From 567531eeef58ba49bb96d16953032f952651db35 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 30 Aug 2026 22:32:25 -0600 Subject: [PATCH 001/136] Add secure single-port HTTP proxy transport --- VotingPlugin/pom.xml | 17 +- .../http/HttpBackendTransportConnector.java | 204 +++++++++++ .../http/HttpClientCredentialStore.java | 162 +++++++++ .../backendproxy/http/HttpConnectionCode.java | 97 ++++++ .../http/HttpEnrollmentAuthority.java | 175 ++++++++++ .../backendproxy/http/HttpPinnedTls.java | 94 +++++ .../http/HttpProxyTransportServer.java | 252 ++++++++++++++ .../backendproxy/http/HttpTlsIdentity.java | 321 ++++++++++++++++++ .../http/HttpTransportProtocol.java | 174 ++++++++++ .../http/HttpTransportSecrets.java | 64 ++++ .../BackendProxyTransportManager.java | 3 + .../transport/HttpBackendProxyTransport.java | 124 +++++++ .../votingplugin/config/BungeeSettings.java | 6 +- .../control/BackendConfigurationService.java | 13 +- .../votingplugin/listeners/VotiferEvent.java | 5 +- .../votingplugin/proxy/BungeeMethod.java | 8 +- .../votingplugin/proxy/VotingPluginProxy.java | 69 +++- .../proxy/VotingPluginProxyCommand.java | 20 ++ .../proxy/VotingPluginProxyConfig.java | 15 + .../proxy/bungee/BungeeConfig.java | 21 +- .../ProxyMethodConfigurationService.java | 15 + .../proxy/velocity/VelocityConfig.java | 15 + .../src/main/resources/BungeeSettings.yml | 15 +- .../src/main/resources/bungeeconfig.yml | 22 +- .../http/HttpTransportRuntimeTest.java | 146 ++++++++ .../http/HttpTransportSecurityTest.java | 100 ++++++ .../BackendConfigurationServiceTest.java | 13 + .../ProxyMethodConfigurationServiceTest.java | 13 + .../votingplugin/tests/BungeeMethodTest.java | 9 + docs/http-transport.md | 50 +++ 30 files changed, 2221 insertions(+), 21 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecrets.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java create mode 100644 docs/http-transport.md diff --git a/VotingPlugin/pom.xml b/VotingPlugin/pom.xml index 465e2f743..646b1e8ac 100644 --- a/VotingPlugin/pom.xml +++ b/VotingPlugin/pom.xml @@ -15,6 +15,7 @@ 21 21 3.4.0 + 1.85 @@ -151,6 +152,10 @@ ${project.groupId}.votingplugin.bstats + + org.bouncycastle + ${project.groupId}.votingplugin.bouncycastle + xyz.upperlevel.spigot @@ -254,6 +259,16 @@ + + org.bouncycastle + bcpkix-jdk18on + ${bouncycastle.version} + + + org.bouncycastle + bcprov-jdk18on + ${bouncycastle.version} + org.spigotmc spigot-api @@ -692,4 +707,4 @@ - \ No newline at end of file + diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java new file mode 100644 index 000000000..67030e90b --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -0,0 +1,204 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.KeyStore; +import java.security.cert.X509Certificate; +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; + +/** Backend-side, persistent HTTP/1.1 long-poll connector. */ +public final class HttpBackendTransportConnector implements AutoCloseable { + public static final Duration CLIENT_TIMEOUT = Duration.ofSeconds(35); + private final HttpClientCredentialStore.HttpClientProfile profile; + private final String serverId; + private final Consumer onEnvelope; + private final HttpClient client; + private final URI transportEndpoint; + private final ThreadPoolExecutor callbackExecutor; + private final AtomicBoolean running = new AtomicBoolean(); + private final Object state = new Object(); + private final LinkedHashMap outgoing = new LinkedHashMap<>(); + private final Set received = new LinkedHashSet<>(), processing = new LinkedHashSet<>(); + private final ArrayDeque acknowledgements = new ArrayDeque<>(); + private final String session = UUID.randomUUID().toString(); + private volatile Thread poller; + private long sequence; + + public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, + HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { + this(profile(code, serverId), credential, onEnvelope); + } + + /** Starts normal transport from the non-secret profile persisted by enrollment. */ + public HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope) throws Exception { + this(enrolled == null ? null : enrolled.profile(), enrolled == null ? null : enrolled.credential(), onEnvelope); + } + + public HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { + if (profile == null || credential == null || onEnvelope == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); + if (!matchesCredential(profile, credential)) throw new IllegalArgumentException("HTTP client certificate does not match transport profile"); + this.profile = profile; this.serverId = profile.serverId(); this.onEnvelope = onEnvelope; + client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); + transportEndpoint = profile.endpoint().resolve("v1/transport"); + callbackExecutor = executor("VotingPlugin-HTTP-callback", 2, 128); + } + + /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ + public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, Path credentials, + Consumer onEnvelope) throws Exception { + this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope); + if (code == null || !profile(code, serverId).equals(this.profile)) throw new IllegalArgumentException("HTTP transport profile does not match connection code"); + } + + /** Starts normal transport using only the persisted certificate and non-secret profile. */ + public HttpBackendTransportConnector(Path credentials, Consumer onEnvelope) throws Exception { + this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope); + } + + /** Performs enrollment network I/O; call this from a connector/setup worker, never a platform main thread. */ + public static HttpClientCredentialStore.ClientCredential enroll(HttpConnectionCode code, String serverId, Path credentials) throws Exception { + if (code == null || credentials == null || serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) throw new IllegalArgumentException("Enrollment configuration is invalid"); + code.requireActive(Clock.systemUTC()); + byte[] payload = ("{\"server\":\"" + serverId + "\",\"token\":\"" + code.enrollmentToken() + "\"}").getBytes(StandardCharsets.UTF_8); + HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(Duration.ofSeconds(5)).sslContext(HttpPinnedTls.clientContext(code)).build(); + HttpRequest request = HttpRequest.newBuilder(code.endpoint().resolve("v1/enroll")).timeout(CLIENT_TIMEOUT) + .header("Content-Type", "application/json").header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(payload)).build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() != 201 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) throw new IllegalArgumentException("Enrollment was rejected"); + HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); + HttpClientCredentialStore.saveEnrolled(credentials, code, issued); return HttpClientCredentialStore.load(credentials); + } + + public void start() { + if (!running.compareAndSet(false, true)) return; + poller = new Thread(this::pollLoop, "VotingPlugin-HTTP-poll"); poller.setDaemon(true); poller.start(); + } + /** + * Inserts an in-memory at-least-once delivery. It survives retry/lost responses while this process remains alive; + * callers needing restart durability must retain the application operation independently. + */ + public boolean send(JsonEnvelope envelope) { + if (envelope == null || !running.get()) return false; + try { HttpTransportProtocol.validateEnvelope(envelope); } + catch (IllegalArgumentException invalid) { return false; } + synchronized (state) { + if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; + String id = UUID.randomUUID().toString(); outgoing.put(id, new HttpTransportProtocol.Delivery(id, envelope)); return true; + } + } + /** A synchronous single poll, useful for lifecycle-controlled integrations and tests. */ + public boolean pollOnce() { + if (!running.get()) return false; + try { + List acks; List messages; long requestSequence; + synchronized (state) { + acks = first(acknowledgements); requestSequence = sequence++; + messages = HttpTransportProtocol.fittingMessages(serverId, session, requestSequence, acks, outgoing.values()); + for (int index = 0; index < acks.size(); index++) acknowledgements.removeFirst(); + } + HttpRequest request = HttpRequest.newBuilder(transportEndpoint).timeout(CLIENT_TIMEOUT).header("Content-Type", "application/json") + .header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(HttpTransportProtocol.request(serverId, session, requestSequence, acks, messages))).build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() != 200 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) return false; + HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(response.body()); + if (!serverId.equals(packet.server()) || !session.equals(packet.session()) || packet.sequence() != requestSequence) return false; + synchronized (state) { for (String ack : packet.acks()) outgoing.remove(ack); } + for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); + return true; + } catch (Exception failure) { return false; } + } + @Override public void close() { + running.getAndSet(false); + Thread current = poller; if (current != null) current.interrupt(); + callbackExecutor.shutdown(); try { if (!callbackExecutor.awaitTermination(5, TimeUnit.SECONDS)) callbackExecutor.shutdownNow(); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); callbackExecutor.shutdownNow(); } + } + + private void pollLoop() { + long retry = 1000L; + while (running.get()) { if (pollOnce()) { retry = 1000L; continue; } try { Thread.sleep(retry); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); break; } retry = Math.min(30_000L, retry * 2); } + } + List accept(List deliveries) { + synchronized (state) { + List accepted = new java.util.ArrayList<>(); + for (HttpTransportProtocol.Delivery delivery : deliveries) { + if (received.contains(delivery.id())) { queueAck(delivery.id()); continue; } + if (!processing.contains(delivery.id())) { if (received.size() >= HttpTransportProtocol.MAX_QUEUE) break; processing.add(delivery.id()); accepted.add(delivery); } + } + return accepted; + } + } + void dispatch(HttpTransportProtocol.Delivery delivery) { + try { callbackExecutor.execute(() -> { boolean success = false; try { onEnvelope.accept(delivery.envelope()); success = true; } catch (RuntimeException ignored) { } + synchronized (state) { processing.remove(delivery.id()); if (success) { received.add(delivery.id()); while (received.size() > HttpTransportProtocol.MAX_QUEUE) received.remove(received.iterator().next()); queueAck(delivery.id()); } } + }); } catch (RejectedExecutionException rejected) { synchronized (state) { processing.remove(delivery.id()); } } + } + private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } + int queuedOutgoing() { synchronized (state) { return outgoing.size(); } } + List drainAcknowledgements() { synchronized (state) { return drain(acknowledgements); } } + private static List first(Collection values) { List output = new java.util.ArrayList<>(); for (T value : values) { output.add(value); if (output.size() == HttpTransportProtocol.MAX_BATCH) break; } return output; } + private static List drain(ArrayDeque values) { List output = new java.util.ArrayList<>(); while (!values.isEmpty() && output.size() < HttpTransportProtocol.MAX_BATCH) output.add(values.remove()); return output; } + private static ThreadPoolExecutor executor(String name, int threads, int queue) { ThreadFactory factory = task -> { Thread thread = new Thread(task, name); thread.setDaemon(true); return thread; }; return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); } + private static HttpClientCredentialStore.HttpClientProfile profile(HttpConnectionCode code, String serverId) { + if (code == null || serverId == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); + if (!code.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalArgumentException("HTTP connection code belongs to a different backend"); + return new HttpClientCredentialStore.HttpClientProfile(serverId, code.endpoint(), code.serverCertificatePin(), code.caCertificatePin()); + } + private static boolean matchesCredential(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential) { + try { + credential.certificate().checkValidity(); credential.certificate().verify(credential.caCertificate().getPublicKey()); + String expected = "urn:votingplugin:http-backend:" + profile.serverId(); + var names = credential.certificate().getSubjectAlternativeNames(); if (names == null) return false; + for (java.util.List name : names) if (name.size() == 2 && Integer.valueOf(6).equals(name.get(0)) && expected.equals(name.get(1))) return true; + return false; + } catch (Exception invalid) { return false; } + } + private static SSLContext clientContext(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { + KeyStore store = KeyStore.getInstance("PKCS12"); store.load(null, new char[0]); + store.setKeyEntry("client", credential.privateKey(), credential.password(), new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); + KeyManagerFactory keys = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keys.init(store, credential.password()); + SSLContext context = SSLContext.getInstance("TLS"); context.init(keys.getKeyManagers(), new javax.net.ssl.TrustManager[] { new PinnedTrustManager(profile) }, null); return context; + } + private static final class PinnedTrustManager implements javax.net.ssl.X509TrustManager { + private final HttpClientCredentialStore.HttpClientProfile profile; PinnedTrustManager(HttpClientCredentialStore.HttpClientProfile profile) { this.profile = profile; } + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } + @Override public void checkServerTrusted(X509Certificate[] chain, String authType) throws java.security.cert.CertificateException { + if (chain == null || chain.length < 2) throw new java.security.cert.CertificateException("Pinned HTTPS server chain is incomplete"); + chain[0].checkValidity(); chain[chain.length - 1].checkValidity(); + try { chain[0].verify(chain[chain.length - 1].getPublicKey()); } + catch (java.security.GeneralSecurityException invalid) { throw new java.security.cert.CertificateException("Pinned HTTPS server signature is invalid", invalid); } + if (chain[chain.length - 1].getBasicConstraints() < 0 || !HttpTransportSecrets.constantTimeEquals(profile.serverCertificatePin().getBytes(StandardCharsets.US_ASCII), HttpTransportSecrets.certificatePin(chain[0]).getBytes(StandardCharsets.US_ASCII))) throw new java.security.cert.CertificateException("Pinned HTTPS server mismatch"); + String caPin = HttpTransportSecrets.certificatePin(chain[chain.length - 1]); + if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), caPin.getBytes(StandardCharsets.US_ASCII))) + throw new java.security.cert.CertificateException("Pinned HTTPS authority mismatch"); + } + @Override public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java new file mode 100644 index 000000000..ae801ae0a --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -0,0 +1,162 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.net.URI; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.Base64; +import java.util.EnumSet; +import java.util.Properties; +import com.bencodez.votingplugin.util.DurableFiles; + +/** Owner-only persistence for the client certificate bundle returned by enrollment. */ +public final class HttpClientCredentialStore { + private static final String BUNDLE_FILE = "http-transport-client.p12"; + private static final String PASSWORD_FILE = "http-transport-client-password"; + private static final String PROFILE_FILE = "http-transport-profile.properties"; + private HttpClientCredentialStore() { } + + public static void save(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws IOException { + if (issued == null) throw new IllegalArgumentException("Issued credential is required"); + Files.createDirectories(directory); + byte[] bundle = issued.pkcs12(); + try { writePrivate(safe(directory.resolve(BUNDLE_FILE)), bundle); } + finally { java.util.Arrays.fill(bundle, (byte) 0); } + char[] password = issued.password(); + try { writePrivate(safe(directory.resolve(PASSWORD_FILE)), asciiBytes(password)); } + finally { java.util.Arrays.fill(password, '\0'); } + } + + /** Persists the certificate plus the non-secret normal-transport profile after enrollment. */ + public static void saveEnrolled(Path directory, HttpConnectionCode code, HttpTlsIdentity.IssuedClientCertificate issued) + throws IOException { + if (code == null || issued == null) throw new IllegalArgumentException("Connection code and credential are required"); + save(directory, issued); + HttpClientProfile profile = new HttpClientProfile(HttpTlsIdentity.canonicalServerId(issued.serverId()), code.endpoint(), + code.serverCertificatePin(), code.caCertificatePin()); + Properties properties = new Properties(); + properties.setProperty("version", "1"); + properties.setProperty("serverId", profile.serverId()); + properties.setProperty("endpoint", profile.endpoint().toASCIIString()); + properties.setProperty("serverPin", profile.serverCertificatePin()); + properties.setProperty("caPin", profile.caCertificatePin()); + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + properties.store(bytes, "VotingPlugin HTTP transport profile"); + writePrivate(safe(directory.resolve(PROFILE_FILE)), bytes.toByteArray()); + } + + public static ClientCredential load(Path directory) throws Exception { + Path bundle = safe(directory.resolve(BUNDLE_FILE)); + Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); + if (!Files.isRegularFile(bundle, LinkOption.NOFOLLOW_LINKS) || !Files.isRegularFile(passwordFile, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP client certificate has not been enrolled"); + byte[] passwordBytes = Files.readAllBytes(passwordFile); + if (passwordBytes.length < 40 || passwordBytes.length > 128) throw new IOException("HTTP client password is invalid"); + char[] password = new String(passwordBytes, StandardCharsets.US_ASCII).toCharArray(); + java.util.Arrays.fill(passwordBytes, (byte) 0); + try { + KeyStore store = KeyStore.getInstance("PKCS12"); + try (var input = Files.newInputStream(bundle, LinkOption.NOFOLLOW_LINKS)) { store.load(input, password); } + PrivateKey privateKey = (PrivateKey) store.getKey("client", password); + java.security.cert.Certificate[] chain = store.getCertificateChain("client"); + if (privateKey == null || chain == null || chain.length != 2 + || !(chain[0] instanceof X509Certificate client) || !(chain[1] instanceof X509Certificate authority)) + throw new IOException("HTTP client certificate bundle is invalid"); + return new ClientCredential(privateKey, client, authority, password); + } finally { java.util.Arrays.fill(password, '\0'); } + } + + public static HttpClientProfile loadProfile(Path directory) throws IOException { + Path profile = safe(directory.resolve(PROFILE_FILE)); + if (!Files.isRegularFile(profile, LinkOption.NOFOLLOW_LINKS) || Files.size(profile) > 8192) + throw new IOException("HTTP transport profile has not been enrolled"); + Properties properties = new Properties(); + try (var input = Files.newInputStream(profile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } + if (properties.size() != 5 || !"1".equals(properties.getProperty("version"))) + throw new IOException("HTTP transport profile is invalid"); + try { + return new HttpClientProfile(properties.getProperty("serverId"), URI.create(properties.getProperty("endpoint")), + properties.getProperty("serverPin"), properties.getProperty("caPin")); + } catch (IllegalArgumentException failure) { throw new IOException("HTTP transport profile is invalid", failure); } + } + + /** Loads and cross-checks the persisted client key material and bound normal-transport profile. */ + public static EnrolledClient loadEnrolled(Path directory) throws Exception { + ClientCredential credential = load(directory); + HttpClientProfile profile = loadProfile(directory); + if (!matchesProfile(credential, profile)) throw new IOException("HTTP client certificate does not match its profile"); + return new EnrolledClient(profile, credential); + } + + public record ClientCredential(PrivateKey privateKey, X509Certificate certificate, X509Certificate caCertificate, char[] password) { + public ClientCredential { password = password.clone(); } + @Override public char[] password() { return password.clone(); } + } + + public record HttpClientProfile(String serverId, URI endpoint, String serverCertificatePin, String caCertificatePin) { + public HttpClientProfile { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + HttpConnectionCode validation = new HttpConnectionCode(serverId, endpoint, serverCertificatePin, caCertificatePin, + java.time.Instant.now().plusSeconds(1), HttpTransportSecrets.randomToken()); + endpoint = validation.endpoint(); + serverCertificatePin = validation.serverCertificatePin(); + caCertificatePin = validation.caCertificatePin(); + } + } + + public record EnrolledClient(HttpClientProfile profile, ClientCredential credential) { } + + private static boolean matchesProfile(ClientCredential credential, HttpClientProfile profile) { + try { + credential.certificate().checkValidity(); + credential.certificate().verify(credential.caCertificate().getPublicKey()); + java.util.List usage = credential.certificate().getExtendedKeyUsage(); + boolean[] keyUsage = credential.certificate().getKeyUsage(); + if (usage == null || !usage.contains(org.bouncycastle.asn1.x509.KeyPurposeId.id_kp_clientAuth.getId()) + || keyUsage == null || !keyUsage[0]) return false; + String expected = "urn:votingplugin:http-backend:" + profile.serverId(); + var names = credential.certificate().getSubjectAlternativeNames(); + if (names == null) return false; + for (java.util.List name : names) if (name.size() == 2 + && Integer.valueOf(6).equals(name.get(0)) && expected.equals(name.get(1))) return true; + return false; + } catch (Exception invalid) { return false; } + } + + private static Path safe(Path file) throws IOException { + if (Files.isSymbolicLink(file)) throw new IOException("Refusing unsafe HTTP credential path"); + return file.toAbsolutePath().normalize(); + } + + private static void writePrivate(Path file, byte[] contents) throws IOException { + Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); + try { + setOwnerOnly(temporary); + Files.write(temporary, contents, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); } + setOwnerOnly(file); + DurableFiles.forceDirectory(file.getParent()); + } finally { Files.deleteIfExists(temporary); } + } + + private static void setOwnerOnly(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { } + } + + private static byte[] asciiBytes(char[] characters) { + byte[] output = new byte[characters.length]; + for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; + return output; + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java new file mode 100644 index 000000000..812751426 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java @@ -0,0 +1,97 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.Instant; +import java.util.Base64; +import java.util.Locale; + +/** + * A copy/paste connection code. It deliberately includes no long-lived credential: + * its only secret is a short-lived, single-use enrollment token. The trailing MAC is a + * corruption check keyed by that included token; it is not a proxy signature and cannot stop + * someone who can replace the whole code from replacing it with another valid code. + */ +public record HttpConnectionCode(String serverId, URI endpoint, String serverCertificatePin, String caCertificatePin, + Instant expiresAt, String enrollmentToken) { + private static final String VERSION = "VPH1"; + private static final int MAX_CODE_LENGTH = 4096; + + public HttpConnectionCode { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + endpoint = validateEndpoint(endpoint); + serverCertificatePin = validatePin(serverCertificatePin, "server certificate pin"); + caCertificatePin = validatePin(caCertificatePin, "CA certificate pin"); + if (expiresAt == null) throw new IllegalArgumentException("Expiry is required"); + enrollmentToken = validateToken(enrollmentToken); + } + + public String encode() { + String endpointPart = Base64.getUrlEncoder().withoutPadding() + .encodeToString(endpoint.toASCIIString().getBytes(StandardCharsets.UTF_8)); + String unsigned = String.join(".", VERSION, serverId, endpointPart, serverCertificatePin, caCertificatePin, + Long.toString(expiresAt.getEpochSecond()), enrollmentToken); + byte[] token = Base64.getUrlDecoder().decode(enrollmentToken); + return unsigned + "." + HttpTransportSecrets.hmacSha256Url(token, unsigned); + } + + public boolean isActive(Clock clock) { + return expiresAt.isAfter(clock.instant()); + } + + public void requireActive(Clock clock) { + if (!isActive(clock)) throw new IllegalArgumentException("Connection code has expired"); + } + + public static HttpConnectionCode parse(String code) { + if (code == null || code.length() > MAX_CODE_LENGTH || code.indexOf('\n') >= 0 || code.indexOf('\r') >= 0) + throw new IllegalArgumentException("Connection code is invalid"); + String[] parts = code.split("\\.", -1); + if (parts.length != 8 || !VERSION.equals(parts[0])) throw new IllegalArgumentException("Connection code is invalid"); + try { + String endpoint = new String(Base64.getUrlDecoder().decode(parts[2]), StandardCharsets.UTF_8); + String unsigned = String.join(".", parts[0], parts[1], parts[2], parts[3], parts[4], parts[5], parts[6]); + byte[] token = Base64.getUrlDecoder().decode(parts[6]); + String expected = HttpTransportSecrets.hmacSha256Url(token, unsigned); + if (!HttpTransportSecrets.constantTimeEquals(expected.getBytes(StandardCharsets.US_ASCII), + parts[7].getBytes(StandardCharsets.US_ASCII))) throw new IllegalArgumentException("Connection code is invalid"); + return new HttpConnectionCode(parts[1], new URI(endpoint), parts[3], parts[4], Instant.ofEpochSecond(Long.parseLong(parts[5])), + parts[6]); + } catch (IllegalArgumentException | URISyntaxException failure) { + throw new IllegalArgumentException("Connection code is invalid", failure); + } + } + + private static URI validateEndpoint(URI value) { + if (value == null || !"https".equalsIgnoreCase(value.getScheme()) || value.getHost() == null + || value.getUserInfo() != null || value.getFragment() != null || value.getRawQuery() != null) + throw new IllegalArgumentException("Endpoint must be an absolute HTTPS URL without credentials or query"); + if (value.getPort() > 65535 || value.getPort() < -1) throw new IllegalArgumentException("Endpoint port is invalid"); + String path = value.getRawPath(); + if (path == null || path.isEmpty()) path = "/"; + if (!path.endsWith("/")) path += "/"; + try { + return new URI("https", null, value.getHost().toLowerCase(Locale.ROOT), value.getPort(), path, null, null); + } catch (URISyntaxException failure) { + throw new IllegalArgumentException("Endpoint is invalid", failure); + } + } + + private static String validatePin(String pin, String name) { + if (pin == null || !pin.matches("[0-9a-fA-F]{64}")) throw new IllegalArgumentException(name + " is invalid"); + return pin.toLowerCase(Locale.ROOT); + } + + private static String validateToken(String token) { + if (token == null || token.length() < 43 || token.length() > 128 || !token.matches("[A-Za-z0-9_-]+")) + throw new IllegalArgumentException("Enrollment token is invalid"); + try { + if (Base64.getUrlDecoder().decode(token).length < 32) throw new IllegalArgumentException("Enrollment token is invalid"); + return token; + } catch (IllegalArgumentException failure) { + throw new IllegalArgumentException("Enrollment token is invalid", failure); + } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java new file mode 100644 index 000000000..5d16a0eeb --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java @@ -0,0 +1,175 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.Properties; +import java.util.Base64; +import java.util.EnumSet; +import com.bencodez.votingplugin.util.DurableFiles; + +/** + * Single-use enrollment tokens and client-certificate binding. Token material is never retained; + * only SHA-256 hashes are kept until expiry. This type is thread-safe. + */ +public final class HttpEnrollmentAuthority { + private static final Duration MAX_ENROLLMENT_LIFETIME = Duration.ofMinutes(15); + private final HttpTlsIdentity identity; + private final Clock clock; + private final Path stateFile; + private final Map enrollments = new HashMap<>(); + private final Map bindings = new HashMap<>(); + private final Set revokedCertificatePins = new HashSet<>(); + private boolean persistenceFailure; + + /** Creates a restart-safe authority. State contains only public certificate pins and revocations. */ + public HttpEnrollmentAuthority(HttpTlsIdentity identity, Path stateDirectory) throws java.io.IOException { + this(identity, Clock.systemUTC(), stateFile(stateDirectory)); + loadState(); + } + + HttpEnrollmentAuthority(HttpTlsIdentity identity, Clock clock) { + this(identity, clock, null); + } + + HttpEnrollmentAuthority(HttpTlsIdentity identity, Clock clock, Path stateFile) { + if (identity == null || clock == null) throw new IllegalArgumentException("Identity and clock are required"); + this.identity = identity; + this.clock = clock; + this.stateFile = stateFile; + } + + public synchronized HttpConnectionCode createConnectionCode(String serverId, URI endpoint, Duration lifetime) { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + if (lifetime == null || lifetime.isNegative() || lifetime.isZero() || lifetime.compareTo(MAX_ENROLLMENT_LIFETIME) > 0) + throw new IllegalArgumentException("Enrollment lifetime must be between one second and fifteen minutes"); + expireEnrollments(); + Instant expiresAt = clock.instant().plus(lifetime); + String token = HttpTransportSecrets.randomToken(); + byte[] tokenHash = HttpTransportSecrets.sha256(token.getBytes(StandardCharsets.US_ASCII)); + String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(tokenHash); + enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId)); + return new HttpConnectionCode(serverId, endpoint, identity.serverCertificatePin(), identity.caCertificatePin(), expiresAt, token); + } + + public synchronized HttpTlsIdentity.IssuedClientCertificate enroll(String serverId, String enrollmentToken) throws Exception { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + if (enrollmentToken == null || enrollmentToken.length() > 128) throw new IllegalArgumentException("Enrollment was rejected"); + expireEnrollments(); + byte[] suppliedHash = HttpTransportSecrets.sha256(enrollmentToken.getBytes(StandardCharsets.US_ASCII)); + String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(suppliedHash); + Enrollment enrollment = enrollments.remove(lookup); // consume before issuing: failed retries require a fresh code. + if (enrollment == null || !HttpTransportSecrets.constantTimeEquals(enrollment.tokenHash(), suppliedHash)) + throw new IllegalArgumentException("Enrollment was rejected"); + if (!serverId.equals(enrollment.serverId())) throw new IllegalArgumentException("Enrollment was rejected"); + ClientBinding existing = bindings.get(serverId); + if (existing != null && !existing.revoked()) throw new IllegalStateException("Server id is already enrolled"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); + bindings.put(serverId, new ClientBinding(HttpTransportSecrets.certificatePin(issued.certificate()), false)); + try { persistState(); } + catch (java.io.IOException failure) { persistenceFailure = true; throw failure; } + return issued; + } + + public synchronized boolean authenticate(String serverId, java.security.cert.X509Certificate certificate) { + if (persistenceFailure || serverId == null || certificate == null) return false; + try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } + catch (IllegalArgumentException invalid) { return false; } + if (!identity.validClientCertificate(serverId, certificate)) return false; + ClientBinding binding = bindings.get(serverId); + String pin = HttpTransportSecrets.certificatePin(certificate); + return binding != null && !binding.revoked() && !revokedCertificatePins.contains(pin) + && HttpTransportSecrets.constantTimeEquals(binding.certificatePin().getBytes(StandardCharsets.US_ASCII), + pin.getBytes(StandardCharsets.US_ASCII)); + } + + public synchronized void revoke(String serverId) { + try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } + catch (IllegalArgumentException invalid) { return; } + ClientBinding binding = bindings.get(serverId); + if (binding != null) { + bindings.put(serverId, new ClientBinding(binding.certificatePin(), true)); + revokedCertificatePins.add(binding.certificatePin()); + try { persistState(); } + catch (java.io.IOException failure) { persistenceFailure = true; throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); } + } + } + + private synchronized void loadState() throws java.io.IOException { + if (stateFile == null || !Files.exists(stateFile, LinkOption.NOFOLLOW_LINKS)) return; + if (!Files.isRegularFile(stateFile, LinkOption.NOFOLLOW_LINKS) || Files.size(stateFile) > 65536) + throw new java.io.IOException("HTTP enrollment state is invalid"); + Properties properties = new Properties(); + try (var input = Files.newInputStream(stateFile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } + for (String key : properties.stringPropertyNames()) { + if (key.startsWith("binding.")) { + String serverId = new String(Base64.getUrlDecoder().decode(key.substring("binding.".length())), StandardCharsets.UTF_8); + serverId = HttpTlsIdentity.canonicalServerId(serverId); + String[] value = properties.getProperty(key, "").split(":", -1); + if (value.length != 2 || !value[0].matches("[0-9a-f]{64}") || !("0".equals(value[1]) || "1".equals(value[1]))) + throw new java.io.IOException("HTTP enrollment state is invalid"); + bindings.put(serverId, new ClientBinding(value[0], "1".equals(value[1]))); + if ("1".equals(value[1])) revokedCertificatePins.add(value[0]); + } else if (!"version".equals(key)) throw new java.io.IOException("HTTP enrollment state is invalid"); + } + if (!"1".equals(properties.getProperty("version"))) throw new java.io.IOException("HTTP enrollment state is invalid"); + } + + private synchronized void persistState() throws java.io.IOException { + if (stateFile == null) return; + Properties properties = new Properties(); + properties.setProperty("version", "1"); + for (Map.Entry entry : bindings.entrySet()) { + String key = Base64.getUrlEncoder().withoutPadding().encodeToString(entry.getKey().getBytes(StandardCharsets.UTF_8)); + properties.setProperty("binding." + key, entry.getValue().certificatePin() + ":" + (entry.getValue().revoked() ? "1" : "0")); + } + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + properties.store(bytes, "VotingPlugin HTTP certificate bindings"); + Path temporary = Files.createTempFile(stateFile.getParent(), stateFile.getFileName().toString(), ".tmp"); + try { + setOwnerOnly(temporary); + Files.write(temporary, bytes.toByteArray(), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, stateFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, stateFile, StandardCopyOption.REPLACE_EXISTING); } + setOwnerOnly(stateFile); + DurableFiles.forceDirectory(stateFile.getParent()); + } finally { Files.deleteIfExists(temporary); } + } + + private static Path stateFile(Path directory) throws java.io.IOException { + if (directory == null) throw new IllegalArgumentException("State directory is required"); + Files.createDirectories(directory); + Path file = directory.toAbsolutePath().normalize().resolve("http-transport-clients.properties"); + if (Files.isSymbolicLink(file)) throw new java.io.IOException("Refusing unsafe HTTP enrollment state path"); + return file; + } + + private static void setOwnerOnly(Path path) throws java.io.IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { } + } + + private void expireEnrollments() { + Instant now = clock.instant(); + enrollments.entrySet().removeIf(entry -> !entry.getValue().expiresAt().isAfter(now)); + } + + private record Enrollment(byte[] tokenHash, Instant expiresAt, String serverId) { + private Enrollment { tokenHash = tokenHash.clone(); } + @Override public byte[] tokenHash() { return tokenHash.clone(); } + } + private record ClientBinding(String certificatePin, boolean revoked) { } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java new file mode 100644 index 000000000..1aaa04cab --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java @@ -0,0 +1,94 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import java.security.cert.X509Certificate; +import java.security.KeyStore; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +/** Builds the backend TLS context. A public CA store is intentionally not consulted. */ +public final class HttpPinnedTls { + private HttpPinnedTls() { } + + public static SSLContext clientContext(HttpConnectionCode code) throws Exception { + if (code == null) throw new IllegalArgumentException("Connection code is required"); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, new TrustManager[] { new PinnedServerTrustManager(code.serverCertificatePin(), code.caCertificatePin()) }, null); + return context; + } + + /** + * Normal transport context: presents the enrolled client certificate and accepts only the + * proxy leaf/CA pins in the connection code. Callers must not override the HttpClient default + * endpoint-identification settings; hostname verification remains enabled. + */ + public static SSLContext mutualTlsContext(HttpConnectionCode code, HttpClientCredentialStore.ClientCredential credential) + throws Exception { + if (code == null || credential == null || credential.privateKey() == null || credential.certificate() == null || credential.caCertificate() == null) + throw new IllegalArgumentException("Enrolled client credential is required"); + char[] password = credential.password(); + try { + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, new char[0]); + store.setKeyEntry("client", credential.privateKey(), password, + new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); + KeyManagerFactory managers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + managers.init(store, password); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(managers.getKeyManagers(), new TrustManager[] { + new PinnedServerTrustManager(code.serverCertificatePin(), code.caCertificatePin()) }, null); + return context; + } finally { java.util.Arrays.fill(password, '\0'); } + } + + public static boolean matchesServerPin(HttpConnectionCode code, X509Certificate certificate) { + if (code == null || certificate == null) return false; + String actual = HttpTransportSecrets.certificatePin(certificate); + return HttpTransportSecrets.constantTimeEquals(code.serverCertificatePin() + .getBytes(java.nio.charset.StandardCharsets.US_ASCII), actual.getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + } + + /** TLS 1.3 is used where the runtime exposes it; hostname verification is deliberately left enabled. */ + public static SSLParameters secureParameters(SSLContext context) { + SSLParameters parameters = context.getDefaultSSLParameters(); + for (String protocol : context.getSupportedSSLParameters().getProtocols()) { + if ("TLSv1.3".equals(protocol)) { + parameters.setProtocols(new String[] { "TLSv1.3" }); + break; + } + } + return parameters; + } + + private static final class PinnedServerTrustManager implements X509TrustManager { + private final String expectedPin; + private final String expectedCaPin; + private PinnedServerTrustManager(String expectedPin, String expectedCaPin) { + this.expectedPin = expectedPin; + this.expectedCaPin = expectedCaPin; + } + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } + @Override public void checkServerTrusted(X509Certificate[] chain, String authType) throws java.security.cert.CertificateException { + if (chain == null || chain.length < 2) throw new java.security.cert.CertificateException("Server certificate chain is incomplete"); + chain[0].checkValidity(); + chain[chain.length - 1].checkValidity(); + try { chain[0].verify(chain[chain.length - 1].getPublicKey()); } + catch (java.security.GeneralSecurityException invalid) { + throw new java.security.cert.CertificateException("Server certificate signature is invalid", invalid); + } + if (chain[chain.length - 1].getBasicConstraints() < 0) + throw new java.security.cert.CertificateException("Server certificate authority is invalid"); + String actual = HttpTransportSecrets.certificatePin(chain[0]); + if (!HttpTransportSecrets.constantTimeEquals(expectedPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII), + actual.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) + throw new java.security.cert.CertificateException("Server certificate pin does not match"); + String issuer = HttpTransportSecrets.certificatePin(chain[chain.length - 1]); + if (!HttpTransportSecrets.constantTimeEquals(expectedCaPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII), + issuer.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) + throw new java.security.cert.CertificateException("Server certificate authority pin does not match"); + } + @Override public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java new file mode 100644 index 000000000..542d1ab6a --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -0,0 +1,252 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsExchange; +import com.sun.net.httpserver.HttpsParameters; +import com.sun.net.httpserver.HttpsServer; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLPeerUnverifiedException; + +/** + * One HTTPS listener for enrollment and the backend-to-proxy long-poll transport. + * Every normal request is certificate-authenticated in the handler, rather than relying on TLS WANT auth. + */ +public final class HttpProxyTransportServer implements AutoCloseable { + static { + // JDK HttpServer reads these once when its internal server configuration is initialized. + // Set conservative process-wide bounds before this transport creates its listener. + setDefault("sun.net.httpserver.maxReqTime", "10"); + setDefault("sun.net.httpserver.maxRspTime", "10"); + setDefault("jdk.httpserver.maxConnections", "144"); + setDefault("sun.net.httpserver.maxReqHeaders", "32"); + setDefault("sun.net.httpserver.maxReqHeaderSize", "16384"); + } + // Keep an idle request open long enough to reuse the TLS connection, but bound backend-origin + // latency when a message is queued immediately after the request body has already been sent. + public static final Duration LONG_POLL = Duration.ofSeconds(2); + private final HttpTlsIdentity identity; + private final HttpEnrollmentAuthority authority; + private final HttpsServer server; + private final ThreadPoolExecutor listenerExecutor; + private final ThreadPoolExecutor handlerExecutor; + private final Semaphore admission = new Semaphore(64); + private final Map backends = new HashMap<>(); + private final Consumer onEnvelope; + private volatile boolean closed; + + public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Consumer onEnvelope) throws Exception { + if (bind == null || identity == null || authority == null || onEnvelope == null) throw new IllegalArgumentException("HTTP transport configuration is required"); + this.identity = identity; this.authority = authority; this.onEnvelope = onEnvelope; + server = HttpsServer.create(bind, 32); + server.setHttpsConfigurator(new HttpsConfigurator(identity.serverContext()) { + @Override public void configure(HttpsParameters parameters) { + SSLParameters ssl = HttpPinnedTls.secureParameters(getSSLContext()); + ssl.setWantClientAuth(true); parameters.setSSLParameters(ssl); + } + }); + // Long polls are blocking by design. Capacity is bounded by admission, while enough workers + // remain available for all admitted polls plus setup requests. + listenerExecutor = executor("VotingPlugin-HTTP-listener", 72, 72); + handlerExecutor = executor("VotingPlugin-HTTP-handler", 4, 128); + server.setExecutor(listenerExecutor); + server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); + server.createContext("/v1/transport", exchange -> transport((HttpsExchange) exchange)); + } + + public void start() { if (closed) throw new IllegalStateException("HTTP transport is closed"); server.start(); } + public int port() { return server.getAddress().getPort(); } + public URI endpoint(String host) { return URI.create("https://" + host + ":" + port() + "/"); } + + /** Queues an in-memory proxy-origin envelope for a specific authenticated backend; this queue is not restart-durable. */ + public boolean send(String serverId, JsonEnvelope envelope) { + if (closed || serverId == null || envelope == null) return false; + try { serverId = HttpTlsIdentity.canonicalServerId(serverId); HttpTransportProtocol.validateEnvelope(envelope); } + catch (IllegalArgumentException invalid) { return false; } + synchronized (backends) { + BackendState backend = backends.computeIfAbsent(serverId, ignored -> new BackendState()); + boolean accepted = backend.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), envelope)); + if (accepted) backend.signal(); + return accepted; + } + } + + @Override public void close() { + if (closed) return; closed = true; server.stop(1); + shutdown(handlerExecutor); shutdown(listenerExecutor); + synchronized (backends) { for (BackendState backend : backends.values()) backend.signal(); backends.clear(); } + } + + private void enroll(HttpsExchange exchange) throws IOException { + if (!"/v1/enroll".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } + if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } + if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } + if (!boundedFixedBody(exchange, 8192) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + try { + HttpTransportProtocol.Enrollment request = HttpTransportProtocol.parseEnrollment(read(exchange.getRequestBody(), 8192)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll(request.server(), request.token()); + reply(exchange, 201, HttpTransportProtocol.enrollmentResponse(issued)); + } catch (Exception rejected) { reply(exchange, 403, new byte[0]); } + finally { admission.release(); } + } + + private void transport(HttpsExchange exchange) throws IOException { + if (!"/v1/transport".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } + if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } + if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } + if (!boundedFixedBody(exchange, HttpTransportProtocol.MAX_BODY_BYTES) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + try { + HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(read(exchange.getRequestBody(), HttpTransportProtocol.MAX_BODY_BYTES)); + X509Certificate certificate = peerCertificate(exchange); + if (certificate == null || !authority.authenticate(packet.server(), certificate)) { reply(exchange, 401, new byte[0]); return; } + BackendState backend; + synchronized (backends) { backend = backends.computeIfAbsent(packet.server(), ignored -> new BackendState()); } + if (!backend.beginPoll(packet.session())) { reply(exchange, 409, new byte[0]); return; } + try { + handlePacket(packet, backend); + Response response = backend.await(packet.server(), packet.session(), packet.sequence()); + reply(exchange, 200, HttpTransportProtocol.response(packet.server(), packet.session(), packet.sequence(), response.acks(), response.messages())); + } finally { backend.endPoll(); } + } catch (IllegalArgumentException rejected) { reply(exchange, 400, new byte[0]); + } catch (Exception failure) { reply(exchange, 503, new byte[0]); + } finally { admission.release(); } + } + + private void handlePacket(HttpTransportProtocol.Packet packet, BackendState backend) { + List accepted; + synchronized (backend) { + if (!backend.allowRequest()) throw new IllegalArgumentException("transport rate limited"); + if (!backend.acceptSession(packet.session(), packet.sequence())) throw new IllegalArgumentException("stale session request"); + backend.acknowledge(packet.acks()); accepted = backend.acceptIncoming(packet.messages()); + } + for (HttpTransportProtocol.Delivery delivery : accepted) dispatch(packet.server(), backend, delivery); + } + private void dispatch(String serverId, BackendState backend, HttpTransportProtocol.Delivery delivery) { + try { handlerExecutor.execute(() -> { + boolean success = false; + try { onEnvelope.accept(new ReceivedEnvelope(serverId, delivery.id(), normalizeBackendIdentity(serverId, delivery.envelope()))); success = true; } + catch (RuntimeException ignored) { } + synchronized (backend) { backend.completeIncoming(delivery.id(), success); } + }); } catch (RejectedExecutionException rejected) { synchronized (backend) { backend.completeIncoming(delivery.id(), false); } } + } + private static JsonEnvelope normalizeBackendIdentity(String serverId, JsonEnvelope envelope) { + // The authenticated TLS identity is authoritative; never forward a forged `server` field. + return envelope.toBuilder().put("server", serverId).build(); + } + private static X509Certificate peerCertificate(HttpsExchange exchange) { + try { Certificate[] peer = exchange.getSSLSession().getPeerCertificates(); + return peer.length > 0 && peer[0] instanceof X509Certificate certificate ? certificate : null; + } catch (SSLPeerUnverifiedException absent) { return null; } + } + private static byte[] read(InputStream input, int maximum) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); byte[] buffer = new byte[4096]; int total = 0, read; + while ((read = input.read(buffer)) >= 0) { total += read; if (total > maximum) throw new IllegalArgumentException("HTTP body is too large"); output.write(buffer, 0, read); } + return output.toByteArray(); + } + private static void reply(HttpsExchange exchange, int status, byte[] body) throws IOException { + Headers headers = exchange.getResponseHeaders(); headers.set("Cache-Control", "no-store"); headers.set("Content-Type", "application/json; charset=utf-8"); + exchange.sendResponseHeaders(status, body.length); try (var output = exchange.getResponseBody()) { output.write(body); } + } + private static boolean json(HttpsExchange exchange) { + String contentType = exchange.getRequestHeaders().getFirst("Content-Type"); + return contentType != null && contentType.toLowerCase(java.util.Locale.ROOT).matches("application/json(?:\\s*;.*)?"); + } + private static boolean boundedFixedBody(HttpsExchange exchange, int maximum) { + if (exchange.getRequestHeaders().getFirst("Transfer-Encoding") != null) return false; + String value = exchange.getRequestHeaders().getFirst("Content-Length"); + try { long length = Long.parseLong(value); return length > 0L && length <= maximum; } + catch (RuntimeException invalid) { return false; } + } + private static ThreadPoolExecutor executor(String name, int threads, int queue) { + ThreadFactory factory = task -> { Thread thread = new Thread(task, name); thread.setDaemon(true); return thread; }; + return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); + } + private static void setDefault(String name, String value) { if (System.getProperty(name) == null) System.setProperty(name, value); } + private static void shutdown(ExecutorService executor) { executor.shutdown(); try { if (!executor.awaitTermination(5, TimeUnit.SECONDS)) executor.shutdownNow(); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); executor.shutdownNow(); } } + + public record ReceivedEnvelope(String serverId, String messageId, JsonEnvelope envelope) { } + static record Response(Collection acks, Collection messages) { } + static final class BackendState { + private String session; private long sequence = -1L; + private final LinkedHashMap outgoing = new LinkedHashMap<>(); + private final Set seen = new LinkedHashSet<>(); private final Set processing = new LinkedHashSet<>(); + private final ArrayDeque acknowledgements = new ArrayDeque<>(); + private final Set delivered = new LinkedHashSet<>(); + private long lastDeliveryNanos; + private double requestTokens = 24.0d; + private long lastTokenNanos = System.nanoTime(); + private boolean activePoll; + private boolean beginPoll(String requestedSession) { synchronized (this) { if (activePoll) return false; activePoll = true; return true; } } + private void endPoll() { synchronized (this) { activePoll = false; notifyAll(); } } + private boolean allowRequest() { + long now = System.nanoTime(); requestTokens = Math.min(24.0d, requestTokens + ((now - lastTokenNanos) / 1_000_000_000.0d) * 2.0d); + lastTokenNanos = now; if (requestTokens < 1.0d) return false; requestTokens -= 1.0d; return true; + } + boolean acceptSession(String requested, long requestedSequence) { + if (!requested.equals(session)) { session = requested; sequence = -1L; delivered.clear(); lastDeliveryNanos = 0L; } + // The connector allocates a fresh monotonic sequence for every attempt. Rejecting equality + // prevents a captured request from being replayed with altered ACKs or a new payload. + if (requestedSequence <= sequence) return false; sequence = requestedSequence; return true; + } + private boolean enqueue(HttpTransportProtocol.Delivery delivery) { if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; outgoing.put(delivery.id(), delivery); return true; } + private void acknowledge(Collection acks) { for (String id : acks) { outgoing.remove(id); delivered.remove(id); } } + List acceptIncoming(List received) { + List accepted = new java.util.ArrayList<>(); + for (HttpTransportProtocol.Delivery delivery : received) { + if (seen.contains(delivery.id())) { queueAck(delivery.id()); continue; } + if (!processing.contains(delivery.id())) { if (seen.size() >= HttpTransportProtocol.MAX_QUEUE) throw new IllegalArgumentException("dedup queue exhausted"); processing.add(delivery.id()); accepted.add(delivery); } + } + return accepted; + } + synchronized void completeIncoming(String id, boolean success) { processing.remove(id); if (success) { seen.add(id); while (seen.size() > HttpTransportProtocol.MAX_QUEUE) seen.remove(seen.iterator().next()); queueAck(id); signal(); } } + private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } + synchronized Response await(String serverId, String requestedSession, long requestedSequence) { + long deadline = System.nanoTime() + LONG_POLL.toNanos(); + while (acknowledgements.isEmpty() && !hasUndelivered() && !redeliveryDue()) { + long remaining = deadline - System.nanoTime(); if (remaining <= 0) break; + try { TimeUnit.NANOSECONDS.timedWait(this, remaining); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); break; } + } + List acks = new java.util.ArrayList<>(); while (!acknowledgements.isEmpty() && acks.size() < HttpTransportProtocol.MAX_BATCH) acks.add(acknowledgements.remove()); + List candidates = new java.util.ArrayList<>(); + if (hasUndelivered() || redeliveryDue()) for (HttpTransportProtocol.Delivery delivery : outgoing.values()) { + if (!delivered.contains(delivery.id()) || redeliveryDue()) candidates.add(delivery); + if (candidates.size() == HttpTransportProtocol.MAX_BATCH) break; + } + List messages = HttpTransportProtocol.fittingMessages(serverId, requestedSession, + requestedSequence, acks, candidates); + for (HttpTransportProtocol.Delivery delivery : messages) delivered.add(delivery.id()); + if (!messages.isEmpty()) lastDeliveryNanos = System.nanoTime(); + return new Response(acks, messages); + } + private boolean hasUndelivered() { for (String id : outgoing.keySet()) if (!delivered.contains(id)) return true; return false; } + private boolean redeliveryDue() { return !outgoing.isEmpty() && lastDeliveryNanos > 0L && System.nanoTime() - lastDeliveryNanos >= LONG_POLL.toNanos(); } + private synchronized void signal() { notifyAll(); } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java new file mode 100644 index 000000000..f3db354b9 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -0,0 +1,321 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import java.io.IOException; +import java.io.OutputStream; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.Security; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.util.Date; +import java.util.EnumSet; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x509.BasicConstraints; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.GeneralName; +import org.bouncycastle.asn1.x509.GeneralNames; +import org.bouncycastle.asn1.x509.KeyUsage; +import org.bouncycastle.asn1.x509.ExtendedKeyUsage; +import org.bouncycastle.asn1.x509.KeyPurposeId; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.X509v3CertificateBuilder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import com.bencodez.votingplugin.util.DurableFiles; + +/** Durable private CA plus server identity used by the proxy HTTP listener. */ +public final class HttpTlsIdentity { + private static final String CA_FILE = "http-transport-ca.p12"; + private static final String SERVER_FILE = "http-transport-server.p12"; + private static final String PASSWORD_FILE = "http-transport-password"; + private static final char[] EMPTY_PASSWORD = new char[0]; + private final PrivateKey caKey; + private final X509Certificate caCertificate; + private final PrivateKey serverKey; + private final X509Certificate serverCertificate; + private final char[] password; + + private HttpTlsIdentity(PrivateKey caKey, X509Certificate caCertificate, PrivateKey serverKey, + X509Certificate serverCertificate, char[] password) { + this.caKey = caKey; + this.caCertificate = caCertificate; + this.serverKey = serverKey; + this.serverCertificate = serverCertificate; + this.password = password.clone(); + } + + public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost) throws Exception { + if (advertisedHost == null || advertisedHost.isBlank() || advertisedHost.length() > 253) + throw new IllegalArgumentException("Advertised HTTPS host is invalid"); + Files.createDirectories(directory); + Path caFile = safe(directory.resolve(CA_FILE)); + Path serverFile = safe(directory.resolve(SERVER_FILE)); + Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); + boolean caExists = Files.exists(caFile, LinkOption.NOFOLLOW_LINKS); + boolean serverExists = Files.exists(serverFile, LinkOption.NOFOLLOW_LINKS); + boolean passwordExists = Files.exists(passwordFile, LinkOption.NOFOLLOW_LINKS); + if (caExists || serverExists || passwordExists) { + if (!(caExists && serverExists && passwordExists)) throw new IOException("HTTP TLS identity files are incomplete"); + char[] password = readPassword(passwordFile); + try { + KeyStore ca = load(caFile, password); + KeyStore server = load(serverFile, password); + PrivateKey caKey = (PrivateKey) ca.getKey("ca", password); + X509Certificate caCertificate = (X509Certificate) ca.getCertificate("ca"); + PrivateKey serverKey = (PrivateKey) server.getKey("server", password); + X509Certificate serverCertificate = (X509Certificate) server.getCertificate("server"); + if (caKey == null || caCertificate == null || serverKey == null || serverCertificate == null) + throw new IOException("HTTP TLS identity files are invalid"); + if (!hasServerName(serverCertificate, advertisedHost)) { + ensureBouncyCastle(); + KeyPair serverPair = keyPair(); + serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, caKey, + CertificateRole.SERVER, advertisedHost); + serverKey = serverPair.getPrivate(); + server = KeyStore.getInstance("PKCS12"); + server.load(null, EMPTY_PASSWORD); + server.setKeyEntry("server", serverKey, password, new Certificate[] { serverCertificate, caCertificate }); + writeStore(serverFile, server, password); + } + return new HttpTlsIdentity(caKey, caCertificate, serverKey, serverCertificate, password); + } finally { Arrays.fill(password, '\0'); } + } + ensureBouncyCastle(); + char[] password = HttpTransportSecrets.randomToken().toCharArray(); + try { + KeyPair caPair = keyPair(); + X509Certificate caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, CertificateRole.CA, null); + KeyPair serverPair = keyPair(); + X509Certificate serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, + caPair.getPrivate(), CertificateRole.SERVER, advertisedHost); + KeyStore ca = KeyStore.getInstance("PKCS12"); + ca.load(null, EMPTY_PASSWORD); + ca.setKeyEntry("ca", caPair.getPrivate(), password, new Certificate[] { caCertificate }); + KeyStore server = KeyStore.getInstance("PKCS12"); + server.load(null, EMPTY_PASSWORD); + server.setKeyEntry("server", serverPair.getPrivate(), password, new Certificate[] { serverCertificate, caCertificate }); + writeStore(caFile, ca, password); + writeStore(serverFile, server, password); + byte[] passwordBytes = asciiBytes(password); + try { writePrivate(passwordFile, passwordBytes); } + finally { Arrays.fill(passwordBytes, (byte) 0); } + return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password); + } finally { Arrays.fill(password, '\0'); } + } + + public String serverCertificatePin() { return HttpTransportSecrets.certificatePin(serverCertificate); } + public String caCertificatePin() { return HttpTransportSecrets.certificatePin(caCertificate); } + public X509Certificate caCertificate() { return caCertificate; } + public X509Certificate serverCertificate() { return serverCertificate; } + + /** + * The listener requests a client certificate but validates its issuance/binding in the HTTP handler. + * This is necessary to share the enrollment and normal endpoints on one JDK HttpsServer listener. + */ + public SSLContext serverContext() throws Exception { + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, EMPTY_PASSWORD); + store.setKeyEntry("server", serverKey, password, new Certificate[] { serverCertificate, caCertificate }); + KeyManagerFactory keyManagers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagers.init(store, password); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(keyManagers.getKeyManagers(), new TrustManager[] { new EnrollmentAwareTrustManager(caCertificate) }, null); + return context; + } + + public IssuedClientCertificate issueClientCertificate(String serverId) throws Exception { + serverId = canonicalServerId(serverId); + ensureBouncyCastle(); + KeyPair pair = keyPair(); + X509Certificate certificate = certificate("CN=" + serverId, pair, caCertificate, caKey, CertificateRole.CLIENT, + "urn:votingplugin:http-backend:" + serverId); + char[] clientPassword = HttpTransportSecrets.randomToken().toCharArray(); + try { + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, EMPTY_PASSWORD); + store.setKeyEntry("client", pair.getPrivate(), clientPassword, new Certificate[] { certificate, caCertificate }); + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + store.store(bytes, clientPassword); + return new IssuedClientCertificate(serverId, certificate, bytes.toByteArray(), clientPassword); + } finally { Arrays.fill(clientPassword, '\0'); } + } + + public static String canonicalServerId(String serverId) { + if (serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) + throw new IllegalArgumentException("Server id is invalid"); + return serverId.toLowerCase(java.util.Locale.ROOT); + } + + public boolean issuedByThisCa(X509Certificate certificate) { + if (certificate == null) return false; + try { + certificate.checkValidity(); + certificate.verify(caCertificate.getPublicKey()); + return true; + } catch (Exception failure) { + return false; + } + } + + public boolean validClientCertificate(String expectedServerId, X509Certificate certificate) { + if (!issuedByThisCa(certificate)) return false; + try { + List usage = certificate.getExtendedKeyUsage(); + boolean[] keyUsage = certificate.getKeyUsage(); + if (usage == null || !usage.contains(KeyPurposeId.id_kp_clientAuth.getId()) || keyUsage == null || !keyUsage[0]) return false; + String expectedUri = "urn:votingplugin:http-backend:" + canonicalServerId(expectedServerId); + Collection> names = certificate.getSubjectAlternativeNames(); + if (names == null) return false; + for (List name : names) { + if (name.size() == 2 && Integer.valueOf(GeneralName.uniformResourceIdentifier).equals(name.get(0)) + && expectedUri.equals(name.get(1))) return true; + } + return false; + } catch (Exception failure) { return false; } + } + + public record IssuedClientCertificate(String serverId, X509Certificate certificate, byte[] pkcs12, char[] password) { + public IssuedClientCertificate { + pkcs12 = pkcs12.clone(); + password = password.clone(); + } + @Override public byte[] pkcs12() { return pkcs12.clone(); } + @Override public char[] password() { return password.clone(); } + } + + private static KeyPair keyPair() throws Exception { + KeyPairGenerator generator = KeyPairGenerator.getInstance("EC"); + generator.initialize(new java.security.spec.ECGenParameterSpec("secp256r1")); + return generator.generateKeyPair(); + } + + private static X509Certificate certificate(String subject, KeyPair subjectKey, X509Certificate issuer, PrivateKey issuerKey, + CertificateRole role, String subjectAlternativeName) throws Exception { + Instant now = Instant.now(); + X500Name issuerName = issuer == null ? new X500Name(subject) : new X500Name(issuer.getSubjectX500Principal().getName()); + X509v3CertificateBuilder builder = new JcaX509v3CertificateBuilder(issuerName, + new BigInteger(160, new java.security.SecureRandom()).setBit(159), Date.from(now.minusSeconds(300)), + Date.from(now.plusSeconds(role == CertificateRole.CA ? 315360000L : 31536000L)), new X500Name(subject), subjectKey.getPublic()); + builder.addExtension(Extension.basicConstraints, true, new BasicConstraints(role == CertificateRole.CA)); + builder.addExtension(Extension.keyUsage, true, new KeyUsage(role == CertificateRole.CA ? KeyUsage.keyCertSign | KeyUsage.cRLSign + : KeyUsage.digitalSignature)); + if (role == CertificateRole.SERVER) builder.addExtension(Extension.extendedKeyUsage, false, + new ExtendedKeyUsage(KeyPurposeId.id_kp_serverAuth)); + if (role == CertificateRole.CLIENT) builder.addExtension(Extension.extendedKeyUsage, false, + new ExtendedKeyUsage(KeyPurposeId.id_kp_clientAuth)); + if (role == CertificateRole.SERVER && subjectAlternativeName != null) { + GeneralName name; + if (subjectAlternativeName.matches("(?:\\d{1,3}\\.){3}\\d{1,3}") || subjectAlternativeName.indexOf(':') >= 0) + name = new GeneralName(GeneralName.iPAddress, subjectAlternativeName); + else name = new GeneralName(GeneralName.dNSName, subjectAlternativeName); + builder.addExtension(Extension.subjectAlternativeName, false, new GeneralNames(name)); + } + if (role == CertificateRole.CLIENT) builder.addExtension(Extension.subjectAlternativeName, false, + new GeneralNames(new GeneralName(GeneralName.uniformResourceIdentifier, subjectAlternativeName))); + ContentSigner signer = new JcaContentSignerBuilder("SHA256withECDSA").setProvider("BC") + .build(issuerKey == null ? subjectKey.getPrivate() : issuerKey); + X509CertificateHolder holder = builder.build(signer); + return new JcaX509CertificateConverter().setProvider("BC").getCertificate(holder); + } + + private static void ensureBouncyCastle() { + if (Security.getProvider("BC") == null) Security.addProvider(new BouncyCastleProvider()); + } + + private static String certificateName(String host) { + return host.replaceAll("[^A-Za-z0-9 ._-]", "_"); + } + + private static boolean hasServerName(X509Certificate certificate, String advertisedHost) { + try { + Collection> names = certificate.getSubjectAlternativeNames(); + if (names == null) return false; + for (List name : names) { + if (name.size() != 2 || !(name.get(1) instanceof String value)) continue; + if ((Integer.valueOf(GeneralName.dNSName).equals(name.get(0)) || Integer.valueOf(GeneralName.iPAddress).equals(name.get(0))) + && advertisedHost.equalsIgnoreCase(value)) return true; + } + return false; + } catch (Exception failure) { return false; } + } + + private static Path safe(Path file) throws IOException { + Path parent = file.toAbsolutePath().normalize().getParent(); + if (parent == null || Files.isSymbolicLink(file)) throw new IOException("Refusing unsafe HTTP TLS identity path"); + return file.toAbsolutePath().normalize(); + } + + private static KeyStore load(Path path, char[] password) throws Exception { + KeyStore store = KeyStore.getInstance("PKCS12"); + try (var input = Files.newInputStream(path, LinkOption.NOFOLLOW_LINKS)) { store.load(input, password); } + return store; + } + + private static char[] readPassword(Path path) throws IOException { + byte[] bytes = Files.readAllBytes(path); + if (bytes.length < 40 || bytes.length > 128) throw new IOException("HTTP TLS password file is invalid"); + try { return new String(bytes, java.nio.charset.StandardCharsets.US_ASCII).toCharArray(); } + finally { Arrays.fill(bytes, (byte) 0); } + } + + private static void writeStore(Path file, KeyStore store, char[] password) throws Exception { + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + store.store(bytes, password); + byte[] contents = bytes.toByteArray(); + try { writePrivate(file, contents); } + finally { Arrays.fill(contents, (byte) 0); } + } + + private static void writePrivate(Path file, byte[] contents) throws IOException { + Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); + try { + setOwnerOnly(temporary); + Files.write(temporary, contents, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); } + setOwnerOnly(file); + DurableFiles.forceDirectory(file.getParent()); + } finally { Files.deleteIfExists(temporary); } + } + + private static byte[] asciiBytes(char[] characters) { + byte[] output = new byte[characters.length]; + for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; + return output; + } + + private static void setOwnerOnly(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { /* Windows ACLs are inherited; never make the file world-readable. */ } + } + + private static final class EnrollmentAwareTrustManager implements X509TrustManager { + private final X509Certificate[] acceptedIssuers; + private EnrollmentAwareTrustManager(X509Certificate caCertificate) { this.acceptedIssuers = new X509Certificate[] { caCertificate }; } + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { } + @Override public void checkServerTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } + @Override public X509Certificate[] getAcceptedIssuers() { return acceptedIssuers.clone(); } + } + private enum CertificateRole { CA, SERVER, CLIENT } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java new file mode 100644 index 000000000..0224dad7d --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java @@ -0,0 +1,174 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collection; +import java.util.List; +import java.util.UUID; + +/** Strict, versioned HTTP transport envelope. Payloads remain canonical JsonEnvelopeCodec values. */ +final class HttpTransportProtocol { + static final int VERSION = 1; + static final int MAX_BODY_BYTES = 256 * 1024; + static final int MAX_BATCH = 64; + static final int MAX_ENVELOPE_BYTES = 48 * 1024; + static final int MAX_QUEUE = 1024; + static final long MAX_CLOCK_SKEW_MILLIS = 90_000L; + + private HttpTransportProtocol() { } + + static void validateEnvelope(JsonEnvelope envelope) { + if (envelope == null || JsonEnvelopeCodec.encode(envelope).getBytes(StandardCharsets.UTF_8).length > MAX_ENVELOPE_BYTES) throw bad(); + } + + static byte[] request(String server, String session, long sequence, Collection acks, + Collection messages) { + JsonObject root = base(server, session, sequence); + root.add("acks", ids(acks)); + root.add("messages", messages(messages)); + byte[] encoded = root.toString().getBytes(StandardCharsets.UTF_8); + if (encoded.length > MAX_BODY_BYTES) throw bad(); + return encoded; + } + + static List fittingMessages(String server, String session, long sequence, Collection acks, + Collection candidates) { + List output = new ArrayList<>(); + for (Delivery candidate : candidates) { + if (output.size() == MAX_BATCH) break; + output.add(candidate); + try { request(server, session, sequence, acks, output); } + catch (IllegalArgumentException tooLarge) { output.remove(output.size() - 1); break; } + } + return output; + } + + static byte[] response(String server, String session, long sequence, Collection acks, + Collection messages) { + return request(server, session, sequence, acks, messages); + } + + static Packet parsePacket(byte[] body) { + if (body == null || body.length == 0 || body.length > MAX_BODY_BYTES) throw bad(); + try { + JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw bad(); + JsonObject root = parsed.getAsJsonObject(); + requireOnly(root, "v", "server", "session", "sequence", "timestamp", "acks", "messages"); + if (integer(root, "v") != VERSION) throw bad(); + String server = HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); + String session = uuid(root, "session"); + long sequence = nonNegative(root, "sequence"); + long timestamp = root.get("timestamp").getAsLong(); + if (Math.abs(Instant.now().toEpochMilli() - timestamp) > MAX_CLOCK_SKEW_MILLIS) throw bad(); + List acks = parseIds(root.get("acks")); + List messages = parseMessages(root.get("messages")); + return new Packet(server, session, sequence, acks, messages); + } catch (RuntimeException invalid) { throw bad(); } + } + + static byte[] enrollmentResponse(HttpTlsIdentity.IssuedClientCertificate certificate) { + JsonObject output = new JsonObject(); + byte[] bundle = certificate.pkcs12(); + try { output.addProperty("bundle", Base64.getUrlEncoder().withoutPadding().encodeToString(bundle)); } + finally { java.util.Arrays.fill(bundle, (byte) 0); } + char[] password = certificate.password(); + try { output.addProperty("password", new String(password)); } + finally { java.util.Arrays.fill(password, '\0'); } + return output.toString().getBytes(StandardCharsets.UTF_8); + } + + static Enrollment parseEnrollment(byte[] body) { + if (body == null || body.length == 0 || body.length > 8192) throw bad(); + try { + JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw bad(); + JsonObject root = parsed.getAsJsonObject(); + requireOnly(root, "server", "token"); + String server = HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); + String token = string(root, "token", 128); + if (!token.matches("[A-Za-z0-9_-]{43,128}")) throw bad(); + return new Enrollment(server, token); + } catch (RuntimeException invalid) { throw bad(); } + } + + static HttpTlsIdentity.IssuedClientCertificate parseEnrollmentResponse(String server, byte[] body) { + if (body == null || body.length == 0 || body.length > MAX_BODY_BYTES) throw bad(); + try { + JsonObject root = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)).getAsJsonObject(); + requireOnly(root, "bundle", "password"); + byte[] bundle = Base64.getUrlDecoder().decode(string(root, "bundle", MAX_BODY_BYTES * 2)); + char[] password = string(root, "password", 128).toCharArray(); + if (bundle.length == 0 || password.length < 40) throw bad(); + try { return new HttpTlsIdentity.IssuedClientCertificate(HttpTlsIdentity.canonicalServerId(server), null, bundle, password); } + finally { java.util.Arrays.fill(bundle, (byte) 0); java.util.Arrays.fill(password, '\0'); } + } catch (RuntimeException invalid) { throw bad(); } + } + + private static JsonObject base(String server, String session, long sequence) { + JsonObject root = new JsonObject(); + root.addProperty("v", VERSION); root.addProperty("server", server); root.addProperty("session", session); + root.addProperty("sequence", sequence); root.addProperty("timestamp", Instant.now().toEpochMilli()); + return root; + } + private static JsonArray ids(Collection values) { + if (values == null || values.size() > MAX_BATCH) throw bad(); + JsonArray output = new JsonArray(); + for (String value : values) { validId(value); output.add(value); } + return output; + } + private static JsonArray messages(Collection values) { + if (values == null || values.size() > MAX_BATCH) throw bad(); + JsonArray output = new JsonArray(); + for (Delivery delivery : values) { + validId(delivery.id()); + String encoded = JsonEnvelopeCodec.encode(delivery.envelope()); + byte[] bytes = encoded.getBytes(StandardCharsets.UTF_8); + if (bytes.length > MAX_ENVELOPE_BYTES) throw bad(); + JsonObject item = new JsonObject(); item.addProperty("id", delivery.id()); + item.addProperty("payload", Base64.getUrlEncoder().withoutPadding().encodeToString(bytes)); output.add(item); + } + return output; + } + private static List parseIds(JsonElement value) { + if (value == null || !value.isJsonArray() || value.getAsJsonArray().size() > MAX_BATCH) throw bad(); + List output = new ArrayList<>(); + for (JsonElement item : value.getAsJsonArray()) { if (!item.isJsonPrimitive()) throw bad(); String id = item.getAsString(); validId(id); output.add(id); } + return output; + } + private static List parseMessages(JsonElement value) { + if (value == null || !value.isJsonArray() || value.getAsJsonArray().size() > MAX_BATCH) throw bad(); + List output = new ArrayList<>(); + for (JsonElement item : value.getAsJsonArray()) { + if (!item.isJsonObject()) throw bad(); JsonObject object = item.getAsJsonObject(); requireOnly(object, "id", "payload"); + String id = string(object, "id", 64); validId(id); + byte[] payload = Base64.getUrlDecoder().decode(string(object, "payload", MAX_ENVELOPE_BYTES * 2)); + if (payload.length == 0 || payload.length > MAX_ENVELOPE_BYTES) throw bad(); + JsonEnvelope envelope = JsonEnvelopeCodec.decode(new String(payload, StandardCharsets.UTF_8)); + output.add(new Delivery(id, envelope)); + } + return output; + } + private static void requireOnly(JsonObject object, String... names) { + for (String name : object.keySet()) { boolean found = false; for (String allowed : names) if (allowed.equals(name)) { found = true; break; } if (!found) throw bad(); } + for (String name : names) if (!object.has(name) || object.get(name).isJsonNull()) throw bad(); + } + private static String string(JsonObject object, String name, int max) { JsonElement v = object.get(name); if (!v.isJsonPrimitive() || !v.getAsJsonPrimitive().isString()) throw bad(); String value = v.getAsString(); if (value.isEmpty() || value.length() > max) throw bad(); return value; } + private static long integer(JsonObject object, String name) { try { return object.get(name).getAsLong(); } catch (RuntimeException failure) { throw bad(); } } + private static long nonNegative(JsonObject object, String name) { long n = integer(object, name); if (n < 0) throw bad(); return n; } + private static String uuid(JsonObject object, String name) { String value = string(object, name, 64); try { return UUID.fromString(value).toString(); } catch (IllegalArgumentException invalid) { throw bad(); } } + private static void validId(String id) { if (id == null || id.length() > 64) throw bad(); try { UUID.fromString(id); } catch (IllegalArgumentException invalid) { throw bad(); } } + private static IllegalArgumentException bad() { return new IllegalArgumentException("Invalid HTTP transport message"); } + + record Delivery(String id, JsonEnvelope envelope) { } + record Packet(String server, String session, long sequence, List acks, List messages) { } + record Enrollment(String server, String token) { } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecrets.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecrets.java new file mode 100644 index 000000000..f1a8ec04b --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecrets.java @@ -0,0 +1,64 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import java.util.Base64; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +/** Small, deliberately dependency-free cryptographic helpers for the HTTP transport. */ +final class HttpTransportSecrets { + private static final SecureRandom RANDOM = new SecureRandom(); + + private HttpTransportSecrets() { } + + static byte[] randomBytes(int length) { + if (length < 16) throw new IllegalArgumentException("Secret length is too small"); + byte[] value = new byte[length]; + RANDOM.nextBytes(value); + return value; + } + + static String randomToken() { + return Base64.getUrlEncoder().withoutPadding().encodeToString(randomBytes(32)); + } + + static byte[] sha256(byte[] value) { + try { + return MessageDigest.getInstance("SHA-256").digest(value); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } + } + + static String sha256Hex(byte[] value) { + StringBuilder output = new StringBuilder(64); + for (byte part : sha256(value)) output.append(String.format("%02x", part & 0xff)); + return output.toString(); + } + + static String certificatePin(X509Certificate certificate) { + try { + return sha256Hex(certificate.getEncoded()); + } catch (Exception failure) { + throw new IllegalArgumentException("Could not encode certificate", failure); + } + } + + static boolean constantTimeEquals(byte[] first, byte[] second) { + return first != null && second != null && MessageDigest.isEqual(first, second); + } + + static String hmacSha256Url(byte[] key, String value) { + try { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return Base64.getUrlEncoder().withoutPadding().encodeToString(mac.doFinal(value.getBytes(StandardCharsets.US_ASCII))); + } catch (Exception failure) { + throw new IllegalStateException("HMAC-SHA-256 is unavailable", failure); + } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 4d7b20872..43b54c128 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -41,6 +41,9 @@ public void start(BungeeMethod method, GlobalMessageHandler messageHandler) { case SOCKETS: transport = new SocketBackendProxyTransport(plugin); break; + case HTTP: + transport = new HttpBackendProxyTransport(plugin); + break; case REDIS: transport = new RedisBackendProxyTransport(plugin, processedVoteCache); break; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java new file mode 100644 index 000000000..9240b93c8 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -0,0 +1,124 @@ +package com.bencodez.votingplugin.backendproxy.transport; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.concurrent.TimeUnit; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; +import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.backendproxy.http.HttpBackendTransportConnector; +import com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore; +import com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode; + +/** Backend adapter for the secure outbound-only HTTP proxy transport. */ +public final class HttpBackendProxyTransport implements BackendProxyTransport { + private static final int MAX_STARTUP_QUEUE = 1024; + private final VotingPluginMain plugin; + private final Object lifecycle = new Object(); + private final ArrayDeque startupQueue = new ArrayDeque<>(); + private volatile HttpBackendTransportConnector connector; + private volatile Thread worker; + private volatile RuntimeException startupFailure; + private volatile boolean closed; + private final java.util.concurrent.atomic.AtomicBoolean queueWarning = new java.util.concurrent.atomic.AtomicBoolean(); + + public HttpBackendProxyTransport(VotingPluginMain plugin) { + this.plugin = plugin; + } + + @Override + public void start(GlobalMessageHandler messageHandler) { + Path directory = plugin.getDataFolder().toPath().resolve("http"); + String serverId = plugin.getBungeeSettings().getServer(); + String connectionCode = plugin.getBungeeSettings().getHttpConnectionCode(); + worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler), + "VotingPlugin-HTTP-Backend-Setup"); + worker.setDaemon(true); + worker.start(); + } + + private void initialize(Path directory, String serverId, String configuredCode, + GlobalMessageHandler messageHandler) { + try { + if (!Files.isRegularFile(directory.resolve("http-transport-profile.properties"))) { + HttpConnectionCode code = HttpConnectionCode.parse(configuredCode); + HttpBackendTransportConnector.enroll(code, serverId, directory); + } + HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); + if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) + throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); + HttpBackendTransportConnector replacement = new HttpBackendTransportConnector(enrolled, messageHandler::onMessage); + synchronized (lifecycle) { + if (closed) { + replacement.close(); + return; + } + connector = replacement; + replacement.start(); + while (!startupQueue.isEmpty()) { + if (!replacement.send(startupQueue.removeFirst())) { + throw new IllegalStateException("HTTP startup queue could not be transferred"); + } + } + } + } catch (Exception failure) { + startupFailure = new IllegalStateException("Secure HTTP backend enrollment or connection failed", failure); + plugin.getLogger().severe("Secure HTTP backend transport is unavailable; check the connection code and proxy endpoint"); + } + } + + @Override + public void send(JsonEnvelope envelope) { + synchronized (lifecycle) { + if (closed) return; + HttpBackendTransportConnector active = connector; + if (active != null) { + if (!active.send(envelope) && queueWarning.compareAndSet(false, true)) + plugin.getLogger().severe("Secure HTTP transport queue is full or rejected an oversized message; delivery was not accepted"); + } else if (startupQueue.size() < MAX_STARTUP_QUEUE) { + startupQueue.addLast(envelope); + } + } + } + + @Override + public void validate() { + RuntimeException failure = startupFailure; + if (failure != null) throw failure; + String serverId = plugin.getBungeeSettings().getServer(); + if (serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) { + throw new IllegalStateException("HTTP requires a valid unique backend Server name"); + } + Path directory = plugin.getDataFolder().toPath().resolve("http"); + if (!Files.isRegularFile(directory.resolve("http-transport-profile.properties")) + && (plugin.getBungeeSettings().getHttpConnectionCode() == null + || plugin.getBungeeSettings().getHttpConnectionCode().isBlank())) { + throw new IllegalStateException("HTTP requires a temporary ConnectionCode for initial enrollment"); + } + } + + @Override + public void close() { + Thread setup; + HttpBackendTransportConnector active; + synchronized (lifecycle) { + closed = true; + startupQueue.clear(); + setup = worker; + worker = null; + active = connector; + connector = null; + } + if (setup != null) { + setup.interrupt(); + try { + setup.join(TimeUnit.SECONDS.toMillis(5)); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + } + if (active != null) active.close(); + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/config/BungeeSettings.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/config/BungeeSettings.java index b9f8b1afc..5c8a05ab7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/config/BungeeSettings.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/config/BungeeSettings.java @@ -88,7 +88,11 @@ public class BungeeSettings extends YMLFile { @ConfigDataInt(path = "BungeeServer.Port") @Getter - private int bungeeServerPort = 1297; + private int bungeeServerPort = 1297; + + @ConfigDataString(path = "HTTP.ConnectionCode") + @Getter + private String httpConnectionCode = ""; @ConfigDataBoolean(path = "PerServerPoints") @Getter diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index fc00e3e9b..0bb8d29f4 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -47,7 +47,7 @@ public final class BackendConfigurationService { "WaitUntilVoteDelay", "PermissionToView", "IgnoreCanVote", "VoteDelayDailyHour", "VoteDelayMin", "GiveOffline"); private static final Pattern COMMENT_SECRET = Pattern.compile( - "(?i)([\"']?\\b(?:[\\w-]*(?:password|secret)[\\w-]*|token|api[ _.-]?key|authorization|[\\w.-]*webhook[ _.-]?url)" + "(?i)([\"']?\\b(?:[\\w-]*(?:password|secret)[\\w-]*|token|connection[ _.-]?code|api[ _.-]?key|authorization|[\\w.-]*webhook[ _.-]?url)" + "\\b[\"']?\\s*[:=]\\s*)(.*)$"); private static final Pattern SECRET_PATH_URL = Pattern.compile("(?i)([\"']?\\burl\\b[\"']?\\s*[:=]\\s*)(.*)$"); private static final Pattern BLOCK_SCALAR_INDICATOR = Pattern.compile("[|>](?:[+-][1-9]?|[1-9][+-]?)?"); @@ -592,6 +592,15 @@ private void validateProxyMethod(BungeeMethod method, YamlConfiguration settings case SOCKETS: configuredHostAndPort(settings, "BungeeServer.Host", "BungeeServer.Port", 1297, "BungeeServer"); break; + case HTTP: + String connectionCode = settings.getString("HTTP.ConnectionCode", ""); + if (connectionCode == null || connectionCode.isBlank()) { + Path identity = dataDirectory.resolve("http").resolve("http-transport-client.p12"); + if (!Files.isRegularFile(identity)) { + throw new IllegalArgumentException("HTTP.ConnectionCode must be set for initial enrollment"); + } + } + break; case MYSQL: try { YamlConfiguration main = parse(readRaw(resolve("Config.yml"), false)); @@ -957,7 +966,7 @@ private static List restoreCommentSecrets(List proposed, List clientHandles; private SocketHandler socketHandler; + private HttpProxyTransportServer httpTransportServer; + private HttpEnrollmentAuthority httpEnrollmentAuthority; @Getter @Setter @@ -566,6 +571,8 @@ protected boolean sendProxyBroadcastEnvelopeNow(String server, JsonEnvelope enve // envelopes. This preserves the socket connection and its delivery // acknowledgement instead of creating a second short-lived socket. return sendSocketEnvelope(server, envelope); + case HTTP: + return sendHttpEnvelope(server, envelope); default: return false; } @@ -1328,6 +1335,8 @@ public void onReceiveEnvelope(JsonEnvelope envelope) { }); rebuildSocketClients(); + } else if (method.equals(BungeeMethod.HTTP)) { + startHttpTransport(); } else if (method.equals(BungeeMethod.REDIS)) { redisHandler = new RedisHandler(getConfig().getRedisHost(), getConfig().getRedisPort(), getConfig().getRedisUsername(), getConfig().getRedisPassword(), getConfig().getRedisDbIndex(), @@ -1391,6 +1400,9 @@ public void sendMessage(String server, int delay, JsonEnvelope envelope) { case SOCKETS: sendSocketEnvelope(server, envelope); break; + case HTTP: + sendHttpEnvelope(server, envelope); + break; default: break; } @@ -2418,6 +2430,7 @@ public void completeRuntimeReplacementShutdown() { if (socketHandler != null) socketHandler.closeConnection(); }); runCleanup("socket clients", this::closeSocketClients); + runCleanup("HTTP transport", this::closeHttpTransport); runCleanup("Redis subscriber", () -> { if (redisHandler != null) redisHandler.close(); }); @@ -2653,6 +2666,60 @@ private synchronized boolean sendSocketEnvelope(String server, JsonEnvelope enve } } + private synchronized boolean sendHttpEnvelope(String server, JsonEnvelope envelope) { + HttpProxyTransportServer transport = httpTransportServer; + return transport != null && transport.send(server, envelope); + } + + private void startHttpTransport() { + try { + URI endpoint = URI.create(getConfig().getHttpPublicEndpoint()); + if (!"https".equalsIgnoreCase(endpoint.getScheme()) || endpoint.getHost() == null + || endpoint.getUserInfo() != null || endpoint.getQuery() != null || endpoint.getFragment() != null + || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) { + throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); + } + File directory = new File(getDataFolderPlugin(), "http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.toPath(), endpoint.getHost()); + httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); + httpTransportServer = new HttpProxyTransportServer( + new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, + httpEnrollmentAuthority, received -> { + GlobalMessageProxyHandler handler = globalMessageProxyHandler; + if (handler == null) throw new IllegalStateException("HTTP message router is not ready"); + handler.onMessage(received.envelope()); + }); + httpTransportServer.start(); + logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" + + httpTransportServer.port() + "; use /votingpluginbungee httpcode for each backend"); + } catch (Exception failure) { + closeHttpTransport(); + throw new IllegalStateException("HTTP transport could not start securely", failure); + } + } + + private synchronized void closeHttpTransport() { + HttpProxyTransportServer transport = httpTransportServer; + httpTransportServer = null; + httpEnrollmentAuthority = null; + if (transport != null) transport.close(); + } + + public String createHttpConnectionCode(String serverId) { + HttpEnrollmentAuthority authority = httpEnrollmentAuthority; + if (method != BungeeMethod.HTTP || authority == null) { + throw new IllegalStateException("The HTTP transport is not running"); + } + return authority.createConnectionCode(serverId, URI.create(getConfig().getHttpPublicEndpoint()), Duration.ofMinutes(15)) + .encode(); + } + + public void revokeHttpBackend(String serverId) { + HttpEnrollmentAuthority authority = httpEnrollmentAuthority; + if (method != BungeeMethod.HTTP || authority == null) throw new IllegalStateException("The HTTP transport is not running"); + authority.revoke(HttpTlsIdentity.canonicalServerId(serverId)); + } + private synchronized void closeSocketClients() { HashMap clients = clientHandles; clientHandles = null; @@ -2673,7 +2740,7 @@ static void stopSocketClients(Map clients) { private void warnUnsupportedDedicatedVotingProxyMode() { if (getConfig().getDedicatedVotingProxy() && (method == null || !method.supportsBackendPresence())) { - logSevere("DedicatedVotingProxy requires MYSQL, REDIS, MQTT, or SOCKETS; PLUGINMESSAGING is disabled for " + logSevere("DedicatedVotingProxy requires MYSQL, REDIS, MQTT, SOCKETS, or HTTP; PLUGINMESSAGING is disabled for " + "dedicated-proxy routing. Falling back to normal proxy routing."); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java index 7420f0a49..069796fb6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java @@ -47,6 +47,24 @@ public String execute(String[] args) { case "status": return handleStatusCommand(); + case "httpcode": + if (args.length != 2) return "&cUsage: httpcode "; + try { + return "&aTemporary HTTP backend connection code (expires in 15 minutes and works once):\n&f" + + plugin.createHttpConnectionCode(args[1]); + } catch (IllegalArgumentException | IllegalStateException failure) { + return "&cThe secure HTTP transport is not running."; + } + + case "httprevoke": + if (args.length != 2) return "&cUsage: httprevoke "; + try { + plugin.revokeHttpBackend(args[1]); + return "&aRevoked HTTP backend identity for " + args[1] + ". Generate a new connection code to re-enroll it."; + } catch (IllegalArgumentException | IllegalStateException failure) { + return "&cCould not revoke that HTTP backend identity."; + } + case "multiproxystatus": plugin.getMultiProxyHandler().sendStatus(); return "&aSent status message across multi-proxy"; @@ -92,6 +110,8 @@ private String getHelpMessage() { helpBuilder.append("/votingplugin vote - Send a vote\n"); helpBuilder.append("/votingplugin forcetimechange - Force a time change\n"); helpBuilder.append("/votingplugin status - Check connection status\n"); + helpBuilder.append("/votingplugin httpcode - Create a node-bound one-time HTTP connection code\n"); + helpBuilder.append("/votingplugin httprevoke - Revoke a backend identity before re-enrollment\n"); helpBuilder.append("/votingplugin multiproxystatus - Send status message across proxies\n"); helpBuilder.append("/votingplugin voteparty - Trigger or modify vote party\n"); return helpBuilder.toString(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java index 3c5eeff94..cd00c91e0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java @@ -241,6 +241,21 @@ default List getProxyBroadcastOfflineForwardServers() { */ public int getBungeePort(); + /** Bind address for the single-port HTTP transport listener. */ + default String getHttpHost() { + return "0.0.0.0"; + } + + /** Public HTTPS origin embedded in newly generated backend connection codes. */ + default String getHttpPublicEndpoint() { + return ""; + } + + /** Listener port for the single-port HTTP transport. */ + default int getHttpPort() { + return 1297; + } + /** * Gets the plugin message channel. * diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index ee1ff540a..4aa394cca 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -209,9 +209,24 @@ public String getBungeeMethod() { } @Override - public int getBungeePort() { - return getData().getInt("BungeeServer.Port", 1297); - } + public int getBungeePort() { + return getData().getInt("BungeeServer.Port", 1297); + } + + @Override + public String getHttpHost() { + return getData().getString("HTTP.Host", "0.0.0.0"); + } + + @Override + public String getHttpPublicEndpoint() { + return getData().getString("HTTP.PublicEndpoint", ""); + } + + @Override + public int getHttpPort() { + return getData().getInt("HTTP.Port", 1297); + } @Override public boolean getDebug() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java index c98f054c8..f2c72515b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java @@ -58,6 +58,21 @@ private void validate(ProxyMethodConfiguration proposal, VotingPluginProxyConfig } } break; + case HTTP: + if (blank(config.getHttpHost()) || config.getHttpPort() < 1 || config.getHttpPort() > 65535) { + throw new IllegalArgumentException("HTTP.Host and HTTP.Port must be set"); + } + String endpoint = config.getHttpPublicEndpoint(); + if (blank(endpoint)) throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); + URI httpEndpoint; + try { httpEndpoint = URI.create(endpoint); } + catch (RuntimeException invalid) { throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); } + if (!"https".equalsIgnoreCase(httpEndpoint.getScheme()) || httpEndpoint.getHost() == null + || httpEndpoint.getUserInfo() != null || httpEndpoint.getQuery() != null || httpEndpoint.getFragment() != null + || (httpEndpoint.getPath() != null && !httpEndpoint.getPath().isEmpty() && !"/".equals(httpEndpoint.getPath()))) { + throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); + } + break; case MYSQL: if (!config.hasDatabaseConfigured()) { throw new IllegalArgumentException("The proxy database Host must be configured for MYSQL"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index 008aa7716..9b6077896 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -360,6 +360,21 @@ public int getBungeePort() { return getInt(getNode("BungeeServer", "Port"), 1297); } + @Override + public String getHttpHost() { + return getString(getNode("HTTP", "Host"), "0.0.0.0"); + } + + @Override + public String getHttpPublicEndpoint() { + return getString(getNode("HTTP", "PublicEndpoint"), ""); + } + + @Override + public int getHttpPort() { + return getInt(getNode("HTTP", "Port"), 1297); + } + @Override public boolean getDebug() { return getBoolean(getNode("Debug"), false); diff --git a/VotingPlugin/src/main/resources/BungeeSettings.yml b/VotingPlugin/src/main/resources/BungeeSettings.yml index 036566edc..a8310126d 100644 --- a/VotingPlugin/src/main/resources/BungeeSettings.yml +++ b/VotingPlugin/src/main/resources/BungeeSettings.yml @@ -10,7 +10,7 @@ # 📑 Config Index (matches this file) # ───────────────────────────────────────────────────────────────────────────── # • Core proxy toggle (UseBungeecord) + method selection (BungeeMethod) -# • Method configs (PLUGINMESSAGING / REDIS / MQTT / MYSQL / SOCKETS) +# • Method configs (PLUGINMESSAGING / HTTP / REDIS / MQTT / MYSQL / SOCKETS) # • Broadcast behavior (BungeeBroadcast / DisableBroadcast / Always) # • Per-server behavior (PerServerRewards / Milestones / Points) # • VoteParty (proxy rewards + global commands) @@ -61,12 +61,19 @@ UseBungeecord: false # Requires restart and set on all servers # https://github.com/BenCodez/VotingPlugin/wiki/Bungeecord-Setups # Available: -# PLUGINMESSAGING (Recommended) -# REDIS (2nd Most Recommended) +# PLUGINMESSAGING (Recommended) +# HTTP (secure direct connection; one proxy port) +# REDIS (2nd Most Recommended) # MQTT # MYSQL # SOCKETS (Not recommended) -BungeeMethod: PLUGINMESSAGING +BungeeMethod: PLUGINMESSAGING + +# Secure direct transport. The proxy generates a temporary ConnectionCode. +# Paste it here once, start the backend, then remove it after enrollment succeeds. +# The backend stores its private client identity in the VotingPlugin data folder. +HTTP: + ConnectionCode: '' # Use Redis for between server communication, set BungeeMethod to REDIS to use this Redis: diff --git a/VotingPlugin/src/main/resources/bungeeconfig.yml b/VotingPlugin/src/main/resources/bungeeconfig.yml index 0c37b46fa..b500d4d7b 100644 --- a/VotingPlugin/src/main/resources/bungeeconfig.yml +++ b/VotingPlugin/src/main/resources/bungeeconfig.yml @@ -249,17 +249,31 @@ WhiteListedServers: [] # Requires restart and set on all servers # https://github.com/BenCodez/VotingPlugin/wiki/Bungeecoord-Setups # Available: -# PLUGINMESSAGING (Recommended) -# REDIS +# PLUGINMESSAGING (Recommended) +# HTTP (secure direct connection; one proxy port) +# REDIS # MQTT # SOCKETS # MYSQL (Not recommended) # PLUGINMESSAGING uses this proxy's player/server state and disables backend # presence messages. Other methods trust each backend's configured Server name. -BungeeMethod: PLUGINMESSAGING +BungeeMethod: PLUGINMESSAGING ###################################################################################### -# Settings for each setup below +# Settings for each setup below + +########################################### +# HTTP Settings +########################################### + +# HTTP uses one encrypted listener on the proxy. Backends connect outbound, so +# they do not need an exposed port. PublicEndpoint must be the HTTPS URL that +# backend servers can reach. VotingPlugin creates the TLS identity, enrollment +# codes, and per-backend client certificates automatically. +HTTP: + Host: '0.0.0.0' + Port: 1297 + PublicEndpoint: '' ########################################### # PLUGINMESSAGING settings diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java new file mode 100644 index 000000000..2273959d1 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -0,0 +1,146 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.net.InetSocketAddress; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpTransportRuntimeTest { + @TempDir Path directory; + + @Test + void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + CountDownLatch proxyReceived = new CountDownLatch(1), backendReceived = new CountDownLatch(1); + AtomicReference received = new AtomicReference<>(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, + message -> { received.set(message); proxyReceived.countDown(); })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, "lobby-1", directory.resolve("client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", credential, + envelope -> backendReceived.countDown())) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("to-proxy").put("server", "forged").build())); + assertTrue(proxyReceived.await(8, TimeUnit.SECONDS)); + assertEquals("lobby-1", received.get().serverId()); + assertEquals("lobby-1", received.get().envelope().getFields().get("server")); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (connector.queuedOutgoing() != 0 && System.nanoTime() < deadline) Thread.sleep(10); + assertEquals(0, connector.queuedOutgoing(), "proxy ACK must remove the exact outbound delivery ID"); + assertTrue(server.send("lobby-1", JsonEnvelope.builder("to-backend").build())); + assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); + } + } + } + + @Test + void normalTransportRejectsAClientWithoutCertificate() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpClient client = HttpClient.newBuilder().sslContext(HttpPinnedTls.clientContext(code)).build(); + byte[] body = HttpTransportProtocol.request("lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(), java.util.List.of()); + HttpResponse response = client.send(HttpRequest.newBuilder(code.endpoint().resolve("v1/transport")) + .timeout(Duration.ofSeconds(5)).header("Content-Type", "application/json").POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(), HttpResponse.BodyHandlers.ofByteArray()); + assertEquals(401, response.statusCode()); + } + } + + @Test + void boundedQueuesFailClosed() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { + for (int i = 0; i < HttpTransportProtocol.MAX_QUEUE; i++) assertTrue(server.send("lobby-1", JsonEnvelope.builder("x").build())); + assertFalse(server.send("lobby-1", JsonEnvelope.builder("x").build())); + assertFalse(server.send("lobby-1", JsonEnvelope.builder("x").put("large", "x".repeat(HttpTransportProtocol.MAX_ENVELOPE_BYTES)).build())); + } + } + + @Test + void aggregatePacketBudgetSplitsLargeValidEnvelopes() { + java.util.List candidates = new java.util.ArrayList<>(); + for (int index = 0; index < 12; index++) candidates.add(new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("large").put("value", "x".repeat(40_000)).build())); + String session = java.util.UUID.randomUUID().toString(); + java.util.List fitted = HttpTransportProtocol.fittingMessages( + "lobby-1", session, 0, java.util.List.of(), candidates); + assertTrue(fitted.size() > 0 && fitted.size() < candidates.size()); + assertTrue(HttpTransportProtocol.request("lobby-1", session, 0, java.util.List.of(), fitted).length + <= HttpTransportProtocol.MAX_BODY_BYTES); + } + + @Test + void persistedProfileStartsAfterTheEnrollmentCodeExpires() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { + server.start(); + HttpConnectionCode active = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(active, "lobby-1", directory.resolve("client")); + HttpConnectionCode expired = new HttpConnectionCode(active.serverId(), active.endpoint(), active.serverCertificatePin(), active.caCertificatePin(), + java.time.Instant.now().minusSeconds(1), active.enrollmentToken()); + try (HttpBackendTransportConnector ignored = new HttpBackendTransportConnector(expired, "lobby-1", directory.resolve("client"), message -> { })) { + assertTrue(true); + } + try (HttpBackendTransportConnector ignored = new HttpBackendTransportConnector(directory.resolve("client"), message -> { })) { + assertTrue(true); + } + } + } + + @Test + void duplicateInboundDeliveryIsReAcknowledgedWithoutSecondDispatch() { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); + String session = java.util.UUID.randomUUID().toString(); + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + assertTrue(state.acceptSession(session, 0)); + assertEquals(1, state.acceptIncoming(java.util.List.of(delivery)).size()); + state.completeIncoming(id, true); + assertEquals(java.util.List.of(id), state.await("lobby-1", session, 0).acks()); + assertTrue(state.acceptSession(session, 1)); + assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty()); + assertEquals(java.util.List.of(id), state.await("lobby-1", session, 1).acks()); + String replacementSession = java.util.UUID.randomUUID().toString(); + assertTrue(state.acceptSession(replacementSession, 0)); + assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty()); + assertEquals(java.util.List.of(id), state.await("lobby-1", replacementSession, 0).acks()); + } + + @Test + void backendReAcknowledgesLostAckDuplicateWithoutSecondCallback() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + CountDownLatch callback = new CountDownLatch(1); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("client")), envelope -> callback.countDown())) { + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + connector.dispatch(delivery); + assertTrue(callback.await(2, TimeUnit.SECONDS)); + assertEquals(java.util.List.of(id), connector.drainAcknowledgements()); + assertTrue(connector.accept(java.util.List.of(delivery)).isEmpty()); + assertEquals(java.util.List.of(id), connector.drainAcknowledgements()); + } + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java new file mode 100644 index 000000000..5b62bdda3 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -0,0 +1,100 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpTransportSecurityTest { + @TempDir Path directory; + + @Test + void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { + HttpConnectionCode original = new HttpConnectionCode("lobby", URI.create("https://Proxy.Example.test:8443/http"), pin('a'), pin('b'), + Instant.parse("2030-01-01T00:00:00Z"), HttpTransportSecrets.randomToken()); + String encoded = original.encode(); + HttpConnectionCode parsed = HttpConnectionCode.parse(encoded); + assertEquals(URI.create("https://proxy.example.test:8443/http/"), parsed.endpoint()); + assertEquals(original.serverCertificatePin(), parsed.serverCertificatePin()); + char last = encoded.charAt(encoded.length() - 1); + assertThrows(IllegalArgumentException.class, () -> HttpConnectionCode.parse(encoded.substring(0, encoded.length() - 1) + + (last == 'A' ? 'B' : 'A'))); + assertThrows(IllegalArgumentException.class, () -> HttpConnectionCode.parse("http://not-a-code")); + } + + @Test + void expiredCodesAreNotActive() { + HttpConnectionCode code = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test/"), pin('a'), pin('b'), + Instant.parse("2029-12-31T23:59:59Z"), HttpTransportSecrets.randomToken()); + assertFalse(code.isActive(Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC))); + assertThrows(IllegalArgumentException.class, () -> code.requireActive(Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC))); + } + + @Test + void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { + HttpTlsIdentity created = HttpTlsIdentity.loadOrCreate(directory, "localhost"); + HttpTlsIdentity loaded = HttpTlsIdentity.loadOrCreate(directory, "localhost"); + assertEquals(created.serverCertificatePin(), loaded.serverCertificatePin()); + assertEquals(created.caCertificatePin(), loaded.caCertificatePin()); + HttpConnectionCode correct = new HttpConnectionCode("lobby", URI.create("https://localhost:8443/"), created.serverCertificatePin(), + created.caCertificatePin(), Instant.now().plusSeconds(60), HttpTransportSecrets.randomToken()); + HttpConnectionCode incorrect = new HttpConnectionCode("lobby", URI.create("https://localhost:8443/"), pin('0'), created.caCertificatePin(), + Instant.now().plusSeconds(60), HttpTransportSecrets.randomToken()); + assertTrue(HttpPinnedTls.matchesServerPin(correct, created.serverCertificate())); + assertFalse(HttpPinnedTls.matchesServerPin(incorrect, created.serverCertificate())); + assertTrue(Files.exists(directory.resolve("http-transport-ca.p12"))); + HttpTlsIdentity rotated = HttpTlsIdentity.loadOrCreate(directory, "127.0.0.1"); + assertEquals(created.caCertificatePin(), rotated.caCertificatePin()); + assertNotEquals(created.serverCertificatePin(), rotated.serverCertificatePin()); + } + + @Test + void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory, "localhost"); + Clock clock = Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock); + HttpConnectionCode wrongTargetCode = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + assertThrows(IllegalArgumentException.class, () -> authority.enroll("attacker", wrongTargetCode.enrollmentToken())); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", code.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", issued.certificate())); + assertTrue(identity.validClientCertificate("LOBBY-1", issued.certificate())); + assertFalse(authority.authenticate("lobby-2", issued.certificate())); + assertThrows(IllegalArgumentException.class, () -> authority.enroll("lobby-2", code.enrollmentToken())); + authority.revoke("lobby-1"); + assertFalse(authority.authenticate("lobby-1", issued.certificate())); + HttpConnectionCode replacementCode = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate replacement = authority.enroll("lobby-1", replacementCode.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", replacement.certificate())); + assertFalse(authority.authenticate("lobby-1", issued.certificate())); + HttpClientCredentialStore.saveEnrolled(directory.resolve("client"), code, issued); + HttpClientCredentialStore.ClientCredential restored = HttpClientCredentialStore.load(directory.resolve("client")); + assertEquals(HttpTransportSecrets.certificatePin(issued.certificate()), HttpTransportSecrets.certificatePin(restored.certificate())); + HttpClientCredentialStore.HttpClientProfile profile = HttpClientCredentialStore.loadProfile(directory.resolve("client")); + assertEquals("lobby-1", profile.serverId()); + assertEquals(code.endpoint(), profile.endpoint()); + assertEquals("lobby-1", HttpClientCredentialStore.loadEnrolled(directory.resolve("client")).profile().serverId()); + assertNotEquals(null, HttpPinnedTls.mutualTlsContext(code, restored)); + Files.writeString(directory.resolve("client").resolve("http-transport-profile.properties"), "version=1\nserverId=lobby-1\n"); + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.loadProfile(directory.resolve("client"))); + HttpEnrollmentAuthority durable = new HttpEnrollmentAuthority(identity, directory.resolve("state")); + HttpConnectionCode durableCode = durable.createConnectionCode("survival", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate durableIssued = durable.enroll("survival", durableCode.enrollmentToken()); + assertTrue(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); + durable.revoke("survival"); + assertFalse(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); + } + + private static String pin(char character) { return String.valueOf(character).repeat(64); } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index bd282b5f8..05e92b462 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -679,6 +679,19 @@ class BackendConfigurationServiceTest { BackendConfigurationService.QuickPreview mqtt = service.previewQuickSetup("proxy-backend", Map.of("server", "lobby", "method", "mqtt")); assertTrue(mqtt.proposal().content().contains("BungeeMethod: MQTT")); + BackendConfigurationService.QuickPreview http = service.previewQuickSetup("proxy-backend", + Map.of("server", "lobby", "method", "http")); + assertTrue(http.proposal().content().contains("BungeeMethod: HTTP")); + } + + @Test void httpConnectionCodeIsRedactedFromManagedConfiguration() throws Exception { + Files.writeString(directory.resolve("BungeeSettings.yml"), + "HTTP:\n ConnectionCode: VPH1-sensitive-enrollment-code\n"); + BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); + + BackendConfigurationService.Document read = service.read("BungeeSettings.yml"); + assertFalse(read.content().contains("VPH1-sensitive-enrollment-code")); + assertTrue(read.content().contains(BackendConfigurationService.REDACTED)); } @Test void proxyMethodSwitchPreflightsRequiredBackendSettings() throws Exception { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java index 4e6a25da7..164c7a7ea 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java @@ -59,6 +59,19 @@ void validatesRequiredSettingsForEveryTransport() { when(config.getSpigotServerConfiguration("lobby")).thenReturn(Map.of("Host", "localhost")); assertDoesNotThrow(() -> service.validate(new ProxyMethodConfiguration(BungeeMethod.SOCKETS))); + assertThrows(IllegalArgumentException.class, + () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + when(config.getHttpHost()).thenReturn("0.0.0.0"); + when(config.getHttpPort()).thenReturn(1297); + when(config.getHttpPublicEndpoint()).thenReturn("http://proxy.example.test:1297"); + assertThrows(IllegalArgumentException.class, + () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test:1297/terminated-path"); + assertThrows(IllegalArgumentException.class, + () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test:1297"); + assertDoesNotThrow(() -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + assertThrows(IllegalArgumentException.class, () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.MYSQL))); when(config.hasDatabaseConfigured()).thenReturn(true); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/BungeeMethodTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/BungeeMethodTest.java index 9d77bd7df..df9c09bc2 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/BungeeMethodTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/BungeeMethodTest.java @@ -1,6 +1,7 @@ package com.bencodez.votingplugin.tests; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; @@ -22,4 +23,12 @@ public void standaloneTransportsSupportBackendPresence() { } } } + + @Test + public void httpMethodNameIsCaseInsensitive() { + assertEquals(BungeeMethod.HTTP, BungeeMethod.getByName("http")); + assertEquals(BungeeMethod.HTTP, BungeeMethod.getByName("HTTP")); + assertFalse(BungeeMethod.HTTP.requiresPlayerOnline()); + assertTrue(BungeeMethod.HTTP.supportsBackendPresence()); + } } diff --git a/docs/http-transport.md b/docs/http-transport.md new file mode 100644 index 000000000..63e1da0a5 --- /dev/null +++ b/docs/http-transport.md @@ -0,0 +1,50 @@ +# Secure HTTP proxy transport + +The `HTTP` bungee method gives every backend an outbound encrypted connection to one HTTPS listener on the proxy. Only the proxy port is opened; backend servers need no inbound transport ports. + +## Quick setup + +1. On the proxy, set the following in `bungeeconfig.yml`: + + ```yaml + BungeeMethod: HTTP + HTTP: + Host: '0.0.0.0' + Port: 1297 + PublicEndpoint: 'https://proxy.example.com:1297/' + ``` + +2. Allow TCP port `1297` to the proxy. `PublicEndpoint` must resolve directly to this VotingPlugin listener. +3. Restart the proxy and run `/votingpluginbungee httpcode `. The name must exactly identify the intended backend; generate a separate code for each backend. +4. On a backend, set a unique `Server`, enable bungee mode, select `HTTP`, and paste its code into `BungeeSettings.yml`: + + ```yaml + UseBungeecord: true + Server: lobby-1 + BungeeMethod: HTTP + HTTP: + ConnectionCode: 'paste-code-here' + ``` + +5. Restart the backend. Once enrollment succeeds, remove `ConnectionCode` from the configuration. The backend's private identity is stored in its VotingPlugin data folder and is reused automatically. + +Connection codes expire after 15 minutes and can be used only once. Treat a fresh code like a temporary password: transfer it privately and do not publish it in logs, tickets, or chat rooms. + +## Security model + +- TLS keys and a private certificate authority are generated automatically on the proxy. No shared transport password or public CA setup is required. +- The backend pins both the exact proxy certificate and its private authority during enrollment. Normal traffic uses a distinct client certificate bound to that backend's canonical `Server` name. +- Every normal request is authenticated again at the application boundary. A payload cannot claim another backend identity, and redirect following is disabled. +- TLS 1.3 is required and weak protocols are disabled. +- Enrollment tokens are 256-bit random capabilities, single-use, short-lived, and retained by the proxy only as hashes. +- Credentials, keys, pins, and revocation state are written atomically with owner-only permissions where the operating system supports them. +- Request bodies, queues, batches, worker pools, concurrent requests, per-backend polling, and request rates are bounded. Exact paths, methods, and JSON content types are enforced. +- Delivery IDs, acknowledgements, and duplicate suppression provide in-process at-least-once delivery. Vote caching remains responsible for application-level restart durability. + +The listener must terminate TLS itself because client-certificate authentication is part of the protocol. Do not put an HTTP TLS-terminating reverse proxy or CDN in front of it. A TCP/L4 proxy that passes TLS through unchanged is suitable. Internet-facing installations should also use the host firewall or provider firewall for volumetric denial-of-service protection; an application cannot fully absorb a link or TCP flood. + +If a backend host or its private credential is compromised, run `/votingpluginbungee httprevoke ` on the proxy before generating a new connection code. Revocation takes effect on the next request and permits a replacement identity to enroll under that server name. Keep the proxy's `http` data directory backed up and private: it contains the transport authority. + +## Performance + +The connector reuses HTTP/1.1 TLS connections, batches messages and acknowledgements, and performs all network and certificate work off the game thread. A two-second bounded long poll avoids busy polling while limiting the worst-case delay for a backend message queued just after an idle request began. All queues are bounded to prevent traffic bursts from causing unbounded memory growth. From 1fd3be90b51150a52c2d390b049bce913250cfbb Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 16:27:15 -0600 Subject: [PATCH 002/136] Validate HTTP client certificates at TLS boundary --- .../backendproxy/http/HttpTlsIdentity.java | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java index f3db354b9..b0c34ae81 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -25,6 +25,7 @@ import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509TrustManager; import org.bouncycastle.asn1.x500.X500Name; import org.bouncycastle.asn1.x509.BasicConstraints; @@ -129,8 +130,9 @@ public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost public X509Certificate serverCertificate() { return serverCertificate; } /** - * The listener requests a client certificate but validates its issuance/binding in the HTTP handler. - * This is necessary to share the enrollment and normal endpoints on one JDK HttpsServer listener. + * The listener requests an optional client certificate so enrollment can share the same port. + * Any certificate that is presented must chain to this transport's private CA; normal requests + * additionally validate the certificate's persisted backend binding in the HTTP handler. */ public SSLContext serverContext() throws Exception { KeyStore store = KeyStore.getInstance("PKCS12"); @@ -311,11 +313,24 @@ private static void setOwnerOnly(Path path) throws IOException { } private static final class EnrollmentAwareTrustManager implements X509TrustManager { - private final X509Certificate[] acceptedIssuers; - private EnrollmentAwareTrustManager(X509Certificate caCertificate) { this.acceptedIssuers = new X509Certificate[] { caCertificate }; } - @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { } + private final X509TrustManager delegate; + private EnrollmentAwareTrustManager(X509Certificate caCertificate) throws Exception { + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + trustStore.load(null, EMPTY_PASSWORD); + trustStore.setCertificateEntry("http-transport-ca", caCertificate); + TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + factory.init(trustStore); + this.delegate = Arrays.stream(factory.getTrustManagers()) + .filter(X509TrustManager.class::isInstance) + .map(X509TrustManager.class::cast) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No X.509 trust manager is available")); + } + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) throws java.security.cert.CertificateException { + delegate.checkClientTrusted(chain, authType); + } @Override public void checkServerTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } - @Override public X509Certificate[] getAcceptedIssuers() { return acceptedIssuers.clone(); } + @Override public X509Certificate[] getAcceptedIssuers() { return delegate.getAcceptedIssuers(); } } private enum CertificateRole { CA, SERVER, CLIENT } } From d27636bdcfe17b61c59b24163b209d62f0759311 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 16:59:14 -0600 Subject: [PATCH 003/136] Harden HTTP transport lifecycle --- .../http/HttpBackendTransportConnector.java | 109 +++++++++++++----- .../http/HttpClientCredentialStore.java | 98 +++++++++++++++- .../http/HttpEnrollmentAuthority.java | 61 +++++++--- .../backendproxy/http/HttpPinnedTls.java | 19 ++- .../http/HttpProxyTransportServer.java | 21 +++- .../backendproxy/http/HttpTlsIdentity.java | 106 +++++++++++++---- .../http/HttpTransportProtocol.java | 17 +++ .../transport/HttpBackendProxyTransport.java | 28 +++-- .../http/HttpTransportRuntimeTest.java | 105 +++++++++++++++++ .../http/HttpTransportSecurityTest.java | 66 ++++++++++- .../HttpBackendProxyTransportTest.java | 49 ++++++++ docs/http-transport.md | 3 +- 12 files changed, 602 insertions(+), 80 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 67030e90b..1c318f6b6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -35,17 +35,21 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private final HttpClientCredentialStore.HttpClientProfile profile; private final String serverId; private final Consumer onEnvelope; - private final HttpClient client; + private volatile HttpClient client; + private volatile HttpClientCredentialStore.ClientCredential credential; + private final Path credentialDirectory; private final URI transportEndpoint; private final ThreadPoolExecutor callbackExecutor; private final AtomicBoolean running = new AtomicBoolean(); private final Object state = new Object(); + private final Object renewal = new Object(); private final LinkedHashMap outgoing = new LinkedHashMap<>(); private final Set received = new LinkedHashSet<>(), processing = new LinkedHashSet<>(); private final ArrayDeque acknowledgements = new ArrayDeque<>(); private final String session = UUID.randomUUID().toString(); private volatile Thread poller; private long sequence; + private volatile long nextRenewalCheckNanos; public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { @@ -54,16 +58,27 @@ public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, /** Starts normal transport from the non-secret profile persisted by enrollment. */ public HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope) throws Exception { - this(enrolled == null ? null : enrolled.profile(), enrolled == null ? null : enrolled.credential(), onEnvelope); + this(enrolled, onEnvelope, null); } public HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { + this(profile, credential, onEnvelope, null); + } + + private HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope, + Path credentialDirectory) throws Exception { + this(enrolled == null ? null : enrolled.profile(), enrolled == null ? null : enrolled.credential(), onEnvelope, credentialDirectory); + } + + private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope, Path credentialDirectory) throws Exception { if (profile == null || credential == null || onEnvelope == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); if (!matchesCredential(profile, credential)) throw new IllegalArgumentException("HTTP client certificate does not match transport profile"); this.profile = profile; this.serverId = profile.serverId(); this.onEnvelope = onEnvelope; - client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) - .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); + this.credential = credential; + this.credentialDirectory = credentialDirectory; + client = client(profile, credential); transportEndpoint = profile.endpoint().resolve("v1/transport"); callbackExecutor = executor("VotingPlugin-HTTP-callback", 2, 128); } @@ -71,18 +86,20 @@ public HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, Path credentials, Consumer onEnvelope) throws Exception { - this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope); + this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope, credentials); if (code == null || !profile(code, serverId).equals(this.profile)) throw new IllegalArgumentException("HTTP transport profile does not match connection code"); } /** Starts normal transport using only the persisted certificate and non-secret profile. */ public HttpBackendTransportConnector(Path credentials, Consumer onEnvelope) throws Exception { - this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope); + this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope, credentials); } /** Performs enrollment network I/O; call this from a connector/setup worker, never a platform main thread. */ public static HttpClientCredentialStore.ClientCredential enroll(HttpConnectionCode code, String serverId, Path credentials) throws Exception { if (code == null || credentials == null || serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) throw new IllegalArgumentException("Enrollment configuration is invalid"); + if (!code.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) + throw new IllegalArgumentException("HTTP connection code belongs to a different backend"); code.requireActive(Clock.systemUTC()); byte[] payload = ("{\"server\":\"" + serverId + "\",\"token\":\"" + code.enrollmentToken() + "\"}").getBytes(StandardCharsets.UTF_8); HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) @@ -113,9 +130,10 @@ public boolean send(JsonEnvelope envelope) { } } /** A synchronous single poll, useful for lifecycle-controlled integrations and tests. */ - public boolean pollOnce() { + public synchronized boolean pollOnce() { if (!running.get()) return false; try { + maybeRenewCredential(); List acks; List messages; long requestSequence; synchronized (state) { acks = first(acknowledgements); requestSequence = sequence++; @@ -149,16 +167,22 @@ List accept(List List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : deliveries) { if (received.contains(delivery.id())) { queueAck(delivery.id()); continue; } - if (!processing.contains(delivery.id())) { if (received.size() >= HttpTransportProtocol.MAX_QUEUE) break; processing.add(delivery.id()); accepted.add(delivery); } + if (!processing.contains(delivery.id())) { + processing.add(delivery.id()); accepted.add(delivery); + } } return accepted; } } void dispatch(HttpTransportProtocol.Delivery delivery) { try { callbackExecutor.execute(() -> { boolean success = false; try { onEnvelope.accept(delivery.envelope()); success = true; } catch (RuntimeException ignored) { } - synchronized (state) { processing.remove(delivery.id()); if (success) { received.add(delivery.id()); while (received.size() > HttpTransportProtocol.MAX_QUEUE) received.remove(received.iterator().next()); queueAck(delivery.id()); } } - }); } catch (RejectedExecutionException rejected) { synchronized (state) { processing.remove(delivery.id()); } } + completeIncoming(delivery.id(), success); + }); } catch (RejectedExecutionException rejected) { completeIncoming(delivery.id(), false); } } + void completeIncoming(String id, boolean success) { synchronized (state) { + processing.remove(id); + if (success) { received.add(id); while (received.size() > HttpTransportProtocol.MAX_QUEUE) received.remove(received.iterator().next()); queueAck(id); } + } } private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } int queuedOutgoing() { synchronized (state) { return outgoing.size(); } } List drainAcknowledgements() { synchronized (state) { return drain(acknowledgements); } } @@ -180,25 +204,52 @@ private static boolean matchesCredential(HttpClientCredentialStore.HttpClientPro return false; } catch (Exception invalid) { return false; } } - private static SSLContext clientContext(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { - KeyStore store = KeyStore.getInstance("PKCS12"); store.load(null, new char[0]); - store.setKeyEntry("client", credential.privateKey(), credential.password(), new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); - KeyManagerFactory keys = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keys.init(store, credential.password()); - SSLContext context = SSLContext.getInstance("TLS"); context.init(keys.getKeyManagers(), new javax.net.ssl.TrustManager[] { new PinnedTrustManager(profile) }, null); return context; - } - private static final class PinnedTrustManager implements javax.net.ssl.X509TrustManager { - private final HttpClientCredentialStore.HttpClientProfile profile; PinnedTrustManager(HttpClientCredentialStore.HttpClientProfile profile) { this.profile = profile; } - @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } - @Override public void checkServerTrusted(X509Certificate[] chain, String authType) throws java.security.cert.CertificateException { - if (chain == null || chain.length < 2) throw new java.security.cert.CertificateException("Pinned HTTPS server chain is incomplete"); - chain[0].checkValidity(); chain[chain.length - 1].checkValidity(); - try { chain[0].verify(chain[chain.length - 1].getPublicKey()); } - catch (java.security.GeneralSecurityException invalid) { throw new java.security.cert.CertificateException("Pinned HTTPS server signature is invalid", invalid); } - if (chain[chain.length - 1].getBasicConstraints() < 0 || !HttpTransportSecrets.constantTimeEquals(profile.serverCertificatePin().getBytes(StandardCharsets.US_ASCII), HttpTransportSecrets.certificatePin(chain[0]).getBytes(StandardCharsets.US_ASCII))) throw new java.security.cert.CertificateException("Pinned HTTPS server mismatch"); - String caPin = HttpTransportSecrets.certificatePin(chain[chain.length - 1]); - if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), caPin.getBytes(StandardCharsets.US_ASCII))) - throw new java.security.cert.CertificateException("Pinned HTTPS authority mismatch"); + private void maybeRenewCredential() { + synchronized (renewal) { + Path directory = credentialDirectory; + if (directory == null || !HttpTlsIdentity.needsRenewal(credential.certificate(), Clock.systemUTC())) return; + long now = System.nanoTime(); + if (nextRenewalCheckNanos != 0L && now - nextRenewalCheckNanos < 0L) return; + nextRenewalCheckNanos = now + Duration.ofHours(6).toNanos(); + try { + byte[] body = HttpTransportProtocol.renewalRequest(serverId); + HttpRequest request = HttpRequest.newBuilder(profile.endpoint().resolve("v1/renew")).timeout(CLIENT_TIMEOUT) + .header("Content-Type", "application/json").header("Cache-Control", "no-store") + .POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() != 201 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) return; + HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(directory, issued); + HttpClientCredentialStore.ClientCredential replacement = staged.credential(); + if (!matchesCredential(profile, replacement)) throw new IllegalArgumentException("Renewed HTTP certificate is invalid"); + HttpClient replacementClient = client(profile, replacement); + HttpClientCredentialStore.activateReplacement(directory, staged); + client = replacementClient; + credential = replacement; + } catch (Exception ignored) { /* The active generation is unchanged; retry on the bounded schedule. */ } } - @Override public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } + } + private static HttpClient client(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential) throws Exception { + return HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); + } + private static SSLContext clientContext(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { + String caPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); + if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), + caPin.getBytes(StandardCharsets.US_ASCII))) throw new IllegalArgumentException("HTTP authority does not match transport profile"); + char[] password = credential.password(); + try { + KeyStore store = KeyStore.getInstance("PKCS12"); store.load(null, new char[0]); + store.setKeyEntry("client", credential.privateKey(), password, + new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); + KeyManagerFactory keys = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keys.init(store, password); + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); trustStore.load(null, new char[0]); + trustStore.setCertificateEntry("http-transport-ca", credential.caCertificate()); + javax.net.ssl.TrustManagerFactory trusts = javax.net.ssl.TrustManagerFactory.getInstance( + javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()); + trusts.init(trustStore); + SSLContext context = SSLContext.getInstance("TLS"); context.init(keys.getKeyManagers(), trusts.getTrustManagers(), null); return context; + } finally { java.util.Arrays.fill(password, '\0'); } } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index ae801ae0a..459a2c360 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -22,6 +22,8 @@ public final class HttpClientCredentialStore { private static final String BUNDLE_FILE = "http-transport-client.p12"; private static final String PASSWORD_FILE = "http-transport-client-password"; private static final String PROFILE_FILE = "http-transport-profile.properties"; + private static final String GENERATIONS_DIRECTORY = "http-transport-client-generations"; + private static final String CURRENT_FILE = "http-transport-client-current"; private HttpClientCredentialStore() { } public static void save(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws IOException { @@ -39,9 +41,16 @@ public static void save(Path directory, HttpTlsIdentity.IssuedClientCertificate public static void saveEnrolled(Path directory, HttpConnectionCode code, HttpTlsIdentity.IssuedClientCertificate issued) throws IOException { if (code == null || issued == null) throw new IllegalArgumentException("Connection code and credential are required"); - save(directory, issued); HttpClientProfile profile = new HttpClientProfile(HttpTlsIdentity.canonicalServerId(issued.serverId()), code.endpoint(), code.serverCertificatePin(), code.caCertificatePin()); + try { + StagedCredential staged = stage(directory, issued, profile); + activateReplacement(directory, staged); + } catch (IOException failure) { throw failure; + } catch (Exception failure) { throw new IOException("Could not persist HTTP client credential", failure); } + } + + private static void writeProfile(Path directory, HttpClientProfile profile) throws IOException { Properties properties = new Properties(); properties.setProperty("version", "1"); properties.setProperty("serverId", profile.serverId()); @@ -54,6 +63,10 @@ public static void saveEnrolled(Path directory, HttpConnectionCode code, HttpTls } public static ClientCredential load(Path directory) throws Exception { + return loadCredential(activeDirectory(directory)); + } + + private static ClientCredential loadCredential(Path directory) throws Exception { Path bundle = safe(directory.resolve(BUNDLE_FILE)); Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); if (!Files.isRegularFile(bundle, LinkOption.NOFOLLOW_LINKS) || !Files.isRegularFile(passwordFile, LinkOption.NOFOLLOW_LINKS)) @@ -74,7 +87,56 @@ public static ClientCredential load(Path directory) throws Exception { } finally { java.util.Arrays.fill(password, '\0'); } } + /** Writes and validates a replacement generation without touching the active credential. */ + static StagedCredential stageReplacement(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws Exception { + return stage(directory, issued, loadProfile(directory)); + } + + private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClientCertificate issued, + HttpClientProfile profile) throws Exception { + if (directory == null || issued == null) throw new IllegalArgumentException("Credential replacement is required"); + Path generations = directory.toAbsolutePath().normalize().resolve(GENERATIONS_DIRECTORY); + Files.createDirectories(generations); + if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential generation directory is unsafe"); + String name = java.util.UUID.randomUUID().toString(); + Path generation = generations.resolve(name); + Files.createDirectory(generation); + setOwnerOnlyDirectory(generation); + try { + save(generation, issued); + writeProfile(generation, profile); + EnrolledClient enrolled = loadEnrolled(generation); + return new StagedCredential(name, enrolled.credential()); + } catch (Exception failure) { + try { Files.deleteIfExists(generation.resolve(BUNDLE_FILE)); Files.deleteIfExists(generation.resolve(PASSWORD_FILE)); + Files.deleteIfExists(generation.resolve(PROFILE_FILE)); Files.deleteIfExists(generation); } + catch (IOException cleanup) { failure.addSuppressed(cleanup); } + throw failure; + } + } + + /** Atomically makes a fully validated generation durable and active. */ + static void activateReplacement(Path directory, StagedCredential staged) throws IOException { + if (directory == null || staged == null || !staged.name().matches("[0-9a-f-]{36}")) + throw new IllegalArgumentException("Staged credential is invalid"); + Path generations = directory.toAbsolutePath().normalize().resolve(GENERATIONS_DIRECTORY); + Path generation = generations.resolve(staged.name()).normalize(); + if (!generation.getParent().equals(generations) || Files.isSymbolicLink(generation) + || !Files.isRegularFile(generation.resolve(BUNDLE_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(generation.resolve(PASSWORD_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(generation.resolve(PROFILE_FILE), LinkOption.NOFOLLOW_LINKS)) + throw new IOException("Staged HTTP credential is incomplete"); + writePrivate(safe(directory.resolve(CURRENT_FILE)), staged.name().getBytes(StandardCharsets.US_ASCII)); + } + + static record StagedCredential(String name, ClientCredential credential) { } + public static HttpClientProfile loadProfile(Path directory) throws IOException { + return loadProfileFile(activeDirectory(directory)); + } + + private static HttpClientProfile loadProfileFile(Path directory) throws IOException { Path profile = safe(directory.resolve(PROFILE_FILE)); if (!Files.isRegularFile(profile, LinkOption.NOFOLLOW_LINKS) || Files.size(profile) > 8192) throw new IOException("HTTP transport profile has not been enrolled"); @@ -88,10 +150,16 @@ public static HttpClientProfile loadProfile(Path directory) throws IOException { } catch (IllegalArgumentException failure) { throw new IOException("HTTP transport profile is invalid", failure); } } + public static boolean hasEnrolledProfile(Path directory) { + try { loadProfile(directory); return true; } + catch (Exception unavailable) { return false; } + } + /** Loads and cross-checks the persisted client key material and bound normal-transport profile. */ public static EnrolledClient loadEnrolled(Path directory) throws Exception { - ClientCredential credential = load(directory); - HttpClientProfile profile = loadProfile(directory); + Path active = activeDirectory(directory); + ClientCredential credential = loadCredential(active); + HttpClientProfile profile = loadProfileFile(active); if (!matchesProfile(credential, profile)) throw new IOException("HTTP client certificate does not match its profile"); return new EnrolledClient(profile, credential); } @@ -116,6 +184,9 @@ public record EnrolledClient(HttpClientProfile profile, ClientCredential credent private static boolean matchesProfile(ClientCredential credential, HttpClientProfile profile) { try { + String authorityPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); + if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), + authorityPin.getBytes(StandardCharsets.US_ASCII))) return false; credential.certificate().checkValidity(); credential.certificate().verify(credential.caCertificate().getPublicKey()); java.util.List usage = credential.certificate().getExtendedKeyUsage(); @@ -136,6 +207,21 @@ private static Path safe(Path file) throws IOException { return file.toAbsolutePath().normalize(); } + private static Path activeDirectory(Path directory) throws IOException { + Path root = directory.toAbsolutePath().normalize(); + Path current = safe(root.resolve(CURRENT_FILE)); + if (!Files.exists(current, LinkOption.NOFOLLOW_LINKS)) return root; + if (!Files.isRegularFile(current, LinkOption.NOFOLLOW_LINKS) || Files.size(current) > 64) + throw new IOException("HTTP client credential pointer is invalid"); + String name = Files.readString(current, StandardCharsets.US_ASCII); + if (!name.matches("[0-9a-f-]{36}")) throw new IOException("HTTP client credential pointer is invalid"); + Path generations = root.resolve(GENERATIONS_DIRECTORY); + Path generation = generations.resolve(name).normalize(); + if (!generation.getParent().equals(generations) || Files.isSymbolicLink(generation) || !Files.isDirectory(generation, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP client credential generation is invalid"); + return generation; + } + private static void writePrivate(Path file, byte[] contents) throws IOException { Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); try { @@ -154,6 +240,12 @@ private static void setOwnerOnly(Path path) throws IOException { catch (UnsupportedOperationException ignored) { } } + private static void setOwnerOnlyDirectory(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } + catch (UnsupportedOperationException ignored) { } + } + private static byte[] asciiBytes(char[] characters) { byte[] output = new byte[characters.length]; for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java index 5d16a0eeb..9ac9b0c51 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java @@ -70,14 +70,15 @@ public synchronized HttpTlsIdentity.IssuedClientCertificate enroll(String server expireEnrollments(); byte[] suppliedHash = HttpTransportSecrets.sha256(enrollmentToken.getBytes(StandardCharsets.US_ASCII)); String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(suppliedHash); - Enrollment enrollment = enrollments.remove(lookup); // consume before issuing: failed retries require a fresh code. + Enrollment enrollment = enrollments.get(lookup); if (enrollment == null || !HttpTransportSecrets.constantTimeEquals(enrollment.tokenHash(), suppliedHash)) throw new IllegalArgumentException("Enrollment was rejected"); if (!serverId.equals(enrollment.serverId())) throw new IllegalArgumentException("Enrollment was rejected"); + enrollments.remove(lookup); // consume only after the token and its intended backend both match. ClientBinding existing = bindings.get(serverId); if (existing != null && !existing.revoked()) throw new IllegalStateException("Server id is already enrolled"); HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); - bindings.put(serverId, new ClientBinding(HttpTransportSecrets.certificatePin(issued.certificate()), false)); + bindings.put(serverId, new ClientBinding(HttpTransportSecrets.certificatePin(issued.certificate()), null, false)); try { persistState(); } catch (java.io.IOException failure) { persistenceFailure = true; throw failure; } return issued; @@ -90,9 +91,28 @@ public synchronized boolean authenticate(String serverId, java.security.cert.X50 if (!identity.validClientCertificate(serverId, certificate)) return false; ClientBinding binding = bindings.get(serverId); String pin = HttpTransportSecrets.certificatePin(certificate); - return binding != null && !binding.revoked() && !revokedCertificatePins.contains(pin) - && HttpTransportSecrets.constantTimeEquals(binding.certificatePin().getBytes(StandardCharsets.US_ASCII), - pin.getBytes(StandardCharsets.US_ASCII)); + if (binding == null || binding.revoked() || revokedCertificatePins.contains(pin)) return false; + if (samePin(binding.certificatePin(), pin)) return true; + if (!samePin(binding.pendingCertificatePin(), pin)) return false; + bindings.put(serverId, new ClientBinding(pin, null, false)); + revokedCertificatePins.add(binding.certificatePin()); + try { persistState(); return true; } + catch (java.io.IOException failure) { persistenceFailure = true; return false; } + } + + /** Issues a replacement while the currently bound certificate is still valid. The old binding remains active + * until the replacement successfully authenticates, making a lost renewal response safe to retry. */ + public synchronized HttpTlsIdentity.IssuedClientCertificate renew(String serverId, + java.security.cert.X509Certificate currentCertificate) throws Exception { + if (!authenticate(serverId, currentCertificate)) throw new IllegalArgumentException("Certificate renewal was rejected"); + serverId = HttpTlsIdentity.canonicalServerId(serverId); + ClientBinding binding = bindings.get(serverId); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); + bindings.put(serverId, new ClientBinding(binding.certificatePin(), + HttpTransportSecrets.certificatePin(issued.certificate()), false)); + try { persistState(); } + catch (java.io.IOException failure) { persistenceFailure = true; throw failure; } + return issued; } public synchronized void revoke(String serverId) { @@ -100,8 +120,9 @@ public synchronized void revoke(String serverId) { catch (IllegalArgumentException invalid) { return; } ClientBinding binding = bindings.get(serverId); if (binding != null) { - bindings.put(serverId, new ClientBinding(binding.certificatePin(), true)); + bindings.put(serverId, new ClientBinding(binding.certificatePin(), binding.pendingCertificatePin(), true)); revokedCertificatePins.add(binding.certificatePin()); + if (binding.pendingCertificatePin() != null) revokedCertificatePins.add(binding.pendingCertificatePin()); try { persistState(); } catch (java.io.IOException failure) { persistenceFailure = true; throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); } } @@ -118,22 +139,31 @@ private synchronized void loadState() throws java.io.IOException { String serverId = new String(Base64.getUrlDecoder().decode(key.substring("binding.".length())), StandardCharsets.UTF_8); serverId = HttpTlsIdentity.canonicalServerId(serverId); String[] value = properties.getProperty(key, "").split(":", -1); - if (value.length != 2 || !value[0].matches("[0-9a-f]{64}") || !("0".equals(value[1]) || "1".equals(value[1]))) + if (!((value.length == 2 && "1".equals(properties.getProperty("version"))) + || (value.length == 3 && "2".equals(properties.getProperty("version")))) + || !value[0].matches("[0-9a-f]{64}")) + throw new java.io.IOException("HTTP enrollment state is invalid"); + String pending = value.length == 3 && !"-".equals(value[1]) ? value[1] : null; + String revoked = value[value.length - 1]; + if ((pending != null && !pending.matches("[0-9a-f]{64}")) || !("0".equals(revoked) || "1".equals(revoked))) throw new java.io.IOException("HTTP enrollment state is invalid"); - bindings.put(serverId, new ClientBinding(value[0], "1".equals(value[1]))); - if ("1".equals(value[1])) revokedCertificatePins.add(value[0]); + bindings.put(serverId, new ClientBinding(value[0], pending, "1".equals(revoked))); + if ("1".equals(revoked)) { revokedCertificatePins.add(value[0]); if (pending != null) revokedCertificatePins.add(pending); } } else if (!"version".equals(key)) throw new java.io.IOException("HTTP enrollment state is invalid"); } - if (!"1".equals(properties.getProperty("version"))) throw new java.io.IOException("HTTP enrollment state is invalid"); + if (!("1".equals(properties.getProperty("version")) || "2".equals(properties.getProperty("version")))) + throw new java.io.IOException("HTTP enrollment state is invalid"); } private synchronized void persistState() throws java.io.IOException { if (stateFile == null) return; Properties properties = new Properties(); - properties.setProperty("version", "1"); + properties.setProperty("version", "2"); for (Map.Entry entry : bindings.entrySet()) { String key = Base64.getUrlEncoder().withoutPadding().encodeToString(entry.getKey().getBytes(StandardCharsets.UTF_8)); - properties.setProperty("binding." + key, entry.getValue().certificatePin() + ":" + (entry.getValue().revoked() ? "1" : "0")); + properties.setProperty("binding." + key, entry.getValue().certificatePin() + ":" + + (entry.getValue().pendingCertificatePin() == null ? "-" : entry.getValue().pendingCertificatePin()) + + ":" + (entry.getValue().revoked() ? "1" : "0")); } java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); properties.store(bytes, "VotingPlugin HTTP certificate bindings"); @@ -171,5 +201,10 @@ private record Enrollment(byte[] tokenHash, Instant expiresAt, String serverId) private Enrollment { tokenHash = tokenHash.clone(); } @Override public byte[] tokenHash() { return tokenHash.clone(); } } - private record ClientBinding(String certificatePin, boolean revoked) { } + private static boolean samePin(String expected, String actual) { + return expected != null && actual != null && HttpTransportSecrets.constantTimeEquals( + expected.getBytes(StandardCharsets.US_ASCII), actual.getBytes(StandardCharsets.US_ASCII)); + } + + private record ClientBinding(String certificatePin, String pendingCertificatePin, boolean revoked) { } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java index 1aaa04cab..9da95637d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java @@ -20,9 +20,9 @@ public static SSLContext clientContext(HttpConnectionCode code) throws Exception } /** - * Normal transport context: presents the enrolled client certificate and accepts only the - * proxy leaf/CA pins in the connection code. Callers must not override the HttpClient default - * endpoint-identification settings; hostname verification remains enabled. + * Normal transport context: presents the enrolled client certificate and trusts only the + * pinned private authority. Callers must not override the HttpClient default endpoint-identification + * settings; hostname verification remains enabled and the proxy leaf may renew under the same CA. */ public static SSLContext mutualTlsContext(HttpConnectionCode code, HttpClientCredentialStore.ClientCredential credential) throws Exception { @@ -30,15 +30,24 @@ public static SSLContext mutualTlsContext(HttpConnectionCode code, HttpClientCre throw new IllegalArgumentException("Enrolled client credential is required"); char[] password = credential.password(); try { + String authorityPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); + if (!HttpTransportSecrets.constantTimeEquals(code.caCertificatePin().getBytes(java.nio.charset.StandardCharsets.US_ASCII), + authorityPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) + throw new IllegalArgumentException("HTTP authority does not match connection code"); KeyStore store = KeyStore.getInstance("PKCS12"); store.load(null, new char[0]); store.setKeyEntry("client", credential.privateKey(), password, new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); KeyManagerFactory managers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); managers.init(store, password); + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + trustStore.load(null, new char[0]); + trustStore.setCertificateEntry("http-transport-ca", credential.caCertificate()); + javax.net.ssl.TrustManagerFactory trusts = javax.net.ssl.TrustManagerFactory.getInstance( + javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()); + trusts.init(trustStore); SSLContext context = SSLContext.getInstance("TLS"); - context.init(managers.getKeyManagers(), new TrustManager[] { - new PinnedServerTrustManager(code.serverCertificatePin(), code.caCertificatePin()) }, null); + context.init(managers.getKeyManagers(), trusts.getTrustManagers(), null); return context; } finally { java.util.Arrays.fill(password, '\0'); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index 542d1ab6a..18bb284d2 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -78,9 +78,26 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity handlerExecutor = executor("VotingPlugin-HTTP-handler", 4, 128); server.setExecutor(listenerExecutor); server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); + server.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); server.createContext("/v1/transport", exchange -> transport((HttpsExchange) exchange)); } + private void renew(HttpsExchange exchange) throws IOException { + if (!"/v1/renew".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } + if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } + if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } + if (!boundedFixedBody(exchange, 1024) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + try { + String serverId = HttpTransportProtocol.parseRenewal(read(exchange.getRequestBody(), 1024)); + X509Certificate certificate = peerCertificate(exchange); + if (certificate == null || !authority.authenticate(serverId, certificate)) { reply(exchange, 401, new byte[0]); return; } + HttpTlsIdentity.IssuedClientCertificate issued = authority.renew(serverId, certificate); + reply(exchange, 201, HttpTransportProtocol.enrollmentResponse(issued)); + } catch (IllegalArgumentException rejected) { reply(exchange, 403, new byte[0]); + } catch (Exception failure) { reply(exchange, 503, new byte[0]); + } finally { admission.release(); } + } + public void start() { if (closed) throw new IllegalStateException("HTTP transport is closed"); server.start(); } public int port() { return server.getAddress().getPort(); } public URI endpoint(String host) { return URI.create("https://" + host + ":" + port() + "/"); } @@ -221,7 +238,9 @@ List acceptIncoming(List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : received) { if (seen.contains(delivery.id())) { queueAck(delivery.id()); continue; } - if (!processing.contains(delivery.id())) { if (seen.size() >= HttpTransportProtocol.MAX_QUEUE) throw new IllegalArgumentException("dedup queue exhausted"); processing.add(delivery.id()); accepted.add(delivery); } + if (!processing.contains(delivery.id())) { + processing.add(delivery.id()); accepted.add(delivery); + } } return accepted; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java index b0c34ae81..afdbb34c3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -13,19 +13,25 @@ import java.security.KeyPairGenerator; import java.security.KeyStore; import java.security.PrivateKey; +import java.security.Principal; import java.security.Security; import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.time.Instant; +import java.time.Clock; +import java.time.Duration; import java.util.Date; import java.util.EnumSet; import java.util.Arrays; import java.util.Collection; import java.util.List; -import javax.net.ssl.KeyManagerFactory; +import java.net.Socket; +import javax.net.ssl.KeyManager; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509ExtendedKeyManager; import javax.net.ssl.X509TrustManager; import org.bouncycastle.asn1.x500.X500Name; import org.bouncycastle.asn1.x509.BasicConstraints; @@ -50,24 +56,34 @@ public final class HttpTlsIdentity { private static final String SERVER_FILE = "http-transport-server.p12"; private static final String PASSWORD_FILE = "http-transport-password"; private static final char[] EMPTY_PASSWORD = new char[0]; + static final Duration RENEW_BEFORE = Duration.ofDays(30); private final PrivateKey caKey; private final X509Certificate caCertificate; - private final PrivateKey serverKey; - private final X509Certificate serverCertificate; + private volatile PrivateKey serverKey; + private volatile X509Certificate serverCertificate; private final char[] password; + private final Path serverFile; + private final String advertisedHost; private HttpTlsIdentity(PrivateKey caKey, X509Certificate caCertificate, PrivateKey serverKey, - X509Certificate serverCertificate, char[] password) { + X509Certificate serverCertificate, char[] password, Path serverFile, String advertisedHost) { this.caKey = caKey; this.caCertificate = caCertificate; this.serverKey = serverKey; this.serverCertificate = serverCertificate; this.password = password.clone(); + this.serverFile = serverFile; + this.advertisedHost = advertisedHost; } public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost) throws Exception { + return loadOrCreate(directory, advertisedHost, Clock.systemUTC()); + } + + static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock clock) throws Exception { if (advertisedHost == null || advertisedHost.isBlank() || advertisedHost.length() > 253) throw new IllegalArgumentException("Advertised HTTPS host is invalid"); + if (clock == null) throw new IllegalArgumentException("Clock is required"); Files.createDirectories(directory); Path caFile = safe(directory.resolve(CA_FILE)); Path serverFile = safe(directory.resolve(SERVER_FILE)); @@ -87,28 +103,29 @@ public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost X509Certificate serverCertificate = (X509Certificate) server.getCertificate("server"); if (caKey == null || caCertificate == null || serverKey == null || serverCertificate == null) throw new IOException("HTTP TLS identity files are invalid"); - if (!hasServerName(serverCertificate, advertisedHost)) { + if (!hasServerName(serverCertificate, advertisedHost) || needsRenewal(serverCertificate, clock)) { ensureBouncyCastle(); KeyPair serverPair = keyPair(); serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, caKey, - CertificateRole.SERVER, advertisedHost); + CertificateRole.SERVER, advertisedHost, clock.instant()); serverKey = serverPair.getPrivate(); server = KeyStore.getInstance("PKCS12"); server.load(null, EMPTY_PASSWORD); server.setKeyEntry("server", serverKey, password, new Certificate[] { serverCertificate, caCertificate }); writeStore(serverFile, server, password); } - return new HttpTlsIdentity(caKey, caCertificate, serverKey, serverCertificate, password); + return new HttpTlsIdentity(caKey, caCertificate, serverKey, serverCertificate, password, serverFile, advertisedHost); } finally { Arrays.fill(password, '\0'); } } ensureBouncyCastle(); char[] password = HttpTransportSecrets.randomToken().toCharArray(); try { KeyPair caPair = keyPair(); - X509Certificate caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, CertificateRole.CA, null); + X509Certificate caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, CertificateRole.CA, null, + clock.instant()); KeyPair serverPair = keyPair(); X509Certificate serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, - caPair.getPrivate(), CertificateRole.SERVER, advertisedHost); + caPair.getPrivate(), CertificateRole.SERVER, advertisedHost, clock.instant()); KeyStore ca = KeyStore.getInstance("PKCS12"); ca.load(null, EMPTY_PASSWORD); ca.setKeyEntry("ca", caPair.getPrivate(), password, new Certificate[] { caCertificate }); @@ -120,11 +137,16 @@ public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost byte[] passwordBytes = asciiBytes(password); try { writePrivate(passwordFile, passwordBytes); } finally { Arrays.fill(passwordBytes, (byte) 0); } - return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password); + return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password, + serverFile, advertisedHost); } finally { Arrays.fill(password, '\0'); } } - public String serverCertificatePin() { return HttpTransportSecrets.certificatePin(serverCertificate); } + public String serverCertificatePin() { + try { renewServerCertificateIfNeeded(); } + catch (Exception failure) { throw new IllegalStateException("Could not renew HTTP server certificate", failure); } + return HttpTransportSecrets.certificatePin(serverCertificate); + } public String caCertificatePin() { return HttpTransportSecrets.certificatePin(caCertificate); } public X509Certificate caCertificate() { return caCertificate; } public X509Certificate serverCertificate() { return serverCertificate; } @@ -135,22 +157,38 @@ public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost * additionally validate the certificate's persisted backend binding in the HTTP handler. */ public SSLContext serverContext() throws Exception { - KeyStore store = KeyStore.getInstance("PKCS12"); - store.load(null, EMPTY_PASSWORD); - store.setKeyEntry("server", serverKey, password, new Certificate[] { serverCertificate, caCertificate }); - KeyManagerFactory keyManagers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagers.init(store, password); + renewServerCertificateIfNeeded(); SSLContext context = SSLContext.getInstance("TLS"); - context.init(keyManagers.getKeyManagers(), new TrustManager[] { new EnrollmentAwareTrustManager(caCertificate) }, null); + context.init(new KeyManager[] { new RotatingServerKeyManager() }, + new TrustManager[] { new EnrollmentAwareTrustManager(caCertificate) }, null); return context; } + private synchronized void renewServerCertificateIfNeeded() throws Exception { + if (!needsRenewal(serverCertificate, Clock.systemUTC())) return; + ensureBouncyCastle(); + KeyPair pair = keyPair(); + X509Certificate replacement = certificate("CN=" + certificateName(advertisedHost), pair, caCertificate, caKey, + CertificateRole.SERVER, advertisedHost, Instant.now()); + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, EMPTY_PASSWORD); + store.setKeyEntry("server", pair.getPrivate(), password, new Certificate[] { replacement, caCertificate }); + writeStore(serverFile, store, password); + serverKey = pair.getPrivate(); + serverCertificate = replacement; + } + public IssuedClientCertificate issueClientCertificate(String serverId) throws Exception { + return issueClientCertificate(serverId, Instant.now()); + } + + IssuedClientCertificate issueClientCertificate(String serverId, Instant issuedAt) throws Exception { serverId = canonicalServerId(serverId); + if (issuedAt == null) throw new IllegalArgumentException("Certificate issuance time is required"); ensureBouncyCastle(); KeyPair pair = keyPair(); X509Certificate certificate = certificate("CN=" + serverId, pair, caCertificate, caKey, CertificateRole.CLIENT, - "urn:votingplugin:http-backend:" + serverId); + "urn:votingplugin:http-backend:" + serverId, issuedAt); char[] clientPassword = HttpTransportSecrets.randomToken().toCharArray(); try { KeyStore store = KeyStore.getInstance("PKCS12"); @@ -212,8 +250,7 @@ private static KeyPair keyPair() throws Exception { } private static X509Certificate certificate(String subject, KeyPair subjectKey, X509Certificate issuer, PrivateKey issuerKey, - CertificateRole role, String subjectAlternativeName) throws Exception { - Instant now = Instant.now(); + CertificateRole role, String subjectAlternativeName, Instant now) throws Exception { X500Name issuerName = issuer == null ? new X500Name(subject) : new X500Name(issuer.getSubjectX500Principal().getName()); X509v3CertificateBuilder builder = new JcaX509v3CertificateBuilder(issuerName, new BigInteger(160, new java.security.SecureRandom()).setBit(159), Date.from(now.minusSeconds(300)), @@ -240,6 +277,10 @@ private static X509Certificate certificate(String subject, KeyPair subjectKey, X return new JcaX509CertificateConverter().setProvider("BC").getCertificate(holder); } + static boolean needsRenewal(X509Certificate certificate, Clock clock) { + return certificate == null || !certificate.getNotAfter().toInstant().isAfter(clock.instant().plus(RENEW_BEFORE)); + } + private static void ensureBouncyCastle() { if (Security.getProvider("BC") == null) Security.addProvider(new BouncyCastleProvider()); } @@ -312,6 +353,31 @@ private static void setOwnerOnly(Path path) throws IOException { catch (UnsupportedOperationException ignored) { /* Windows ACLs are inherited; never make the file world-readable. */ } } + private final class RotatingServerKeyManager extends X509ExtendedKeyManager { + private static final String ALIAS = "server"; + private void refresh() { + try { renewServerCertificateIfNeeded(); } + catch (Exception failure) { throw new IllegalStateException("Could not renew HTTP server certificate", failure); } + } + private String alias(String keyType) { + refresh(); + return keyType != null && ("EC".equalsIgnoreCase(keyType) || keyType.toUpperCase(java.util.Locale.ROOT).startsWith("EC_")) + ? ALIAS : null; + } + @Override public String[] getClientAliases(String keyType, Principal[] issuers) { return null; } + @Override public String chooseClientAlias(String[] keyTypes, Principal[] issuers, Socket socket) { return null; } + @Override public String[] getServerAliases(String keyType, Principal[] issuers) { + return alias(keyType) == null ? null : new String[] { ALIAS }; + } + @Override public String chooseServerAlias(String keyType, Principal[] issuers, Socket socket) { return alias(keyType); } + @Override public String chooseEngineServerAlias(String keyType, Principal[] issuers, SSLEngine engine) { return alias(keyType); } + @Override public X509Certificate[] getCertificateChain(String alias) { + refresh(); + return ALIAS.equals(alias) ? new X509Certificate[] { serverCertificate, caCertificate } : null; + } + @Override public PrivateKey getPrivateKey(String alias) { refresh(); return ALIAS.equals(alias) ? serverKey : null; } + } + private static final class EnrollmentAwareTrustManager implements X509TrustManager { private final X509TrustManager delegate; private EnrollmentAwareTrustManager(X509Certificate caCertificate) throws Exception { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java index 0224dad7d..f275e9b76 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java @@ -100,6 +100,23 @@ static Enrollment parseEnrollment(byte[] body) { } catch (RuntimeException invalid) { throw bad(); } } + static byte[] renewalRequest(String server) { + JsonObject root = new JsonObject(); + root.addProperty("server", HttpTlsIdentity.canonicalServerId(server)); + return root.toString().getBytes(StandardCharsets.UTF_8); + } + + static String parseRenewal(byte[] body) { + if (body == null || body.length == 0 || body.length > 1024) throw bad(); + try { + JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw bad(); + JsonObject root = parsed.getAsJsonObject(); + requireOnly(root, "server"); + return HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); + } catch (RuntimeException invalid) { throw bad(); } + } + static HttpTlsIdentity.IssuedClientCertificate parseEnrollmentResponse(String server, byte[] body) { if (body == null || body.length == 0 || body.length > MAX_BODY_BYTES) throw bad(); try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 9240b93c8..1b8e86569 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -1,7 +1,7 @@ package com.bencodez.votingplugin.backendproxy.transport; -import java.nio.file.Files; import java.nio.file.Path; +import java.time.Clock; import java.util.ArrayDeque; import java.util.concurrent.TimeUnit; @@ -33,6 +33,7 @@ public void start(GlobalMessageHandler messageHandler) { Path directory = plugin.getDataFolder().toPath().resolve("http"); String serverId = plugin.getBungeeSettings().getServer(); String connectionCode = plugin.getBungeeSettings().getHttpConnectionCode(); + validateConfiguration(directory, serverId, connectionCode); worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler), "VotingPlugin-HTTP-Backend-Setup"); worker.setDaemon(true); @@ -42,14 +43,14 @@ public void start(GlobalMessageHandler messageHandler) { private void initialize(Path directory, String serverId, String configuredCode, GlobalMessageHandler messageHandler) { try { - if (!Files.isRegularFile(directory.resolve("http-transport-profile.properties"))) { + if (!HttpClientCredentialStore.hasEnrolledProfile(directory)) { HttpConnectionCode code = HttpConnectionCode.parse(configuredCode); HttpBackendTransportConnector.enroll(code, serverId, directory); } HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); - HttpBackendTransportConnector replacement = new HttpBackendTransportConnector(enrolled, messageHandler::onMessage); + HttpBackendTransportConnector replacement = new HttpBackendTransportConnector(directory, messageHandler::onMessage); synchronized (lifecycle) { if (closed) { replacement.close(); @@ -92,10 +93,23 @@ public void validate() { throw new IllegalStateException("HTTP requires a valid unique backend Server name"); } Path directory = plugin.getDataFolder().toPath().resolve("http"); - if (!Files.isRegularFile(directory.resolve("http-transport-profile.properties")) - && (plugin.getBungeeSettings().getHttpConnectionCode() == null - || plugin.getBungeeSettings().getHttpConnectionCode().isBlank())) { - throw new IllegalStateException("HTTP requires a temporary ConnectionCode for initial enrollment"); + validateConfiguration(directory, serverId, plugin.getBungeeSettings().getHttpConnectionCode()); + } + + private static void validateConfiguration(Path directory, String serverId, String configuredCode) { + try { serverId = com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId); } + catch (IllegalArgumentException invalid) { throw new IllegalStateException("HTTP requires a valid unique backend Server name", invalid); } + if (!HttpClientCredentialStore.hasEnrolledProfile(directory)) { + if (configuredCode == null || configuredCode.isBlank()) + throw new IllegalStateException("HTTP requires a temporary ConnectionCode for initial enrollment"); + try { + HttpConnectionCode code = HttpConnectionCode.parse(configuredCode); + code.requireActive(Clock.systemUTC()); + if (!code.serverId().equals(serverId)) + throw new IllegalArgumentException("Connection code belongs to a different backend"); + } catch (IllegalArgumentException invalid) { + throw new IllegalStateException("HTTP ConnectionCode is invalid, expired, or belongs to a different backend", invalid); + } } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 2273959d1..77d1ac512 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -11,6 +11,7 @@ import java.net.http.HttpResponse; import java.nio.file.Path; import java.time.Duration; +import java.time.Instant; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -105,6 +106,71 @@ void persistedProfileStartsAfterTheEnrollmentCodeExpires() throws Exception { } } + @Test + void persistedBackendConnectsAfterAutomaticServerLeafRotation() throws Exception { + Instant now = Instant.now(); + Path proxyDirectory = directory.resolve("proxy"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost", + java.time.Clock.fixed(now.minus(Duration.ofDays(340)), java.time.ZoneOffset.UTC)); + String originalServerPin = HttpTransportSecrets.certificatePin(original.serverCertificate()); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(original, directory.resolve("authority")); + HttpTlsIdentity rotated = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost"); + CountDownLatch received = new CountDownLatch(1); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), rotated, + authority, ignored -> received.countDown())) { + HttpConnectionCode activeCode = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", activeCode.enrollmentToken()); + HttpConnectionCode oldProfileCode = new HttpConnectionCode(activeCode.serverId(), activeCode.endpoint(), originalServerPin, + activeCode.caCertificatePin(), activeCode.expiresAt(), activeCode.enrollmentToken()); + HttpClientCredentialStore.saveEnrolled(directory.resolve("client"), oldProfileCode, issued); + assertFalse(oldProfileCode.serverCertificatePin().equals(rotated.serverCertificatePin())); + server.start(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(directory.resolve("client"), ignored -> { })) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("after-rotation").build())); + assertTrue(received.await(8, TimeUnit.SECONDS)); + } + } + } + + @Test + void backendRenewsClientCertificateBeforeExpiryWithoutNewConnectionCode() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate expiring = identity.issueClientCertificate("lobby-1", + Instant.now().minus(Duration.ofDays(340))); + String originalPin = HttpTransportSecrets.certificatePin(expiring.certificate()); + Path authorityDirectory = directory.resolve("authority"); + java.nio.file.Files.createDirectories(authorityDirectory); + String key = java.util.Base64.getUrlEncoder().withoutPadding() + .encodeToString("lobby-1".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + java.nio.file.Files.writeString(authorityDirectory.resolve("http-transport-clients.properties"), + "version=2\nbinding." + key + "=" + originalPin + ":-:0\n"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, ignored -> { })) { + HttpConnectionCode profileCode = new HttpConnectionCode("lobby-1", server.endpoint("localhost"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); + Path clientDirectory = directory.resolve("client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, profileCode, expiring); + server.start(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { + connector.start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(8); + String renewedPin = originalPin; + while (renewedPin.equals(originalPin) && System.nanoTime() < deadline) { + Thread.sleep(25); + renewedPin = HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(clientDirectory).certificate()); + } + assertFalse(renewedPin.equals(originalPin)); + HttpClientCredentialStore.ClientCredential renewed = HttpClientCredentialStore.load(clientDirectory); + long promotionDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (authority.authenticate("lobby-1", expiring.certificate()) && System.nanoTime() < promotionDeadline) Thread.sleep(25); + assertTrue(authority.authenticate("lobby-1", renewed.certificate())); + assertFalse(authority.authenticate("lobby-1", expiring.certificate())); + } + } + } + @Test void duplicateInboundDeliveryIsReAcknowledgedWithoutSecondDispatch() { HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); @@ -124,6 +190,24 @@ void duplicateInboundDeliveryIsReAcknowledgedWithoutSecondDispatch() { assertEquals(java.util.List.of(id), state.await("lobby-1", replacementSession, 0).acks()); } + @Test + void proxyDedupWindowEvictsOldestCompletedDeliveryAtCapacity() { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); + String oldest = null, newest = null; + for (int index = 0; index < HttpTransportProtocol.MAX_QUEUE + 1; index++) { + String id = java.util.UUID.randomUUID().toString(); + if (index == 0) oldest = id; + newest = id; + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + assertEquals(1, state.acceptIncoming(java.util.List.of(delivery)).size()); + state.completeIncoming(id, true); + } + HttpTransportProtocol.Delivery evicted = new HttpTransportProtocol.Delivery(oldest, JsonEnvelope.builder("x").build()); + HttpTransportProtocol.Delivery retained = new HttpTransportProtocol.Delivery(newest, JsonEnvelope.builder("x").build()); + assertEquals(1, state.acceptIncoming(java.util.List.of(evicted)).size()); + assertTrue(state.acceptIncoming(java.util.List.of(retained)).isEmpty()); + } + @Test void backendReAcknowledgesLostAckDuplicateWithoutSecondCallback() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); @@ -143,4 +227,25 @@ void backendReAcknowledgesLostAckDuplicateWithoutSecondCallback() throws Excepti assertEquals(java.util.List.of(id), connector.drainAcknowledgements()); } } + + @Test + void backendDedupWindowContinuesAfterCapacity() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("client")), envelope -> { })) { + for (int index = 0; index < HttpTransportProtocol.MAX_QUEUE; index++) { + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + assertEquals(1, connector.accept(java.util.List.of(delivery)).size()); + connector.completeIncoming(id, true); + } + HttpTransportProtocol.Delivery next = new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").build()); + assertEquals(1, connector.accept(java.util.List.of(next)).size()); + } + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 5b62bdda3..996eda9be 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -66,6 +66,9 @@ void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock); HttpConnectionCode wrongTargetCode = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); assertThrows(IllegalArgumentException.class, () -> authority.enroll("attacker", wrongTargetCode.enrollmentToken())); + assertTrue(authority.authenticate("lobby-1", authority.enroll("lobby-1", wrongTargetCode.enrollmentToken()).certificate()), + "a wrong backend must not consume another backend's connection code"); + authority.revoke("lobby-1"); HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", code.enrollmentToken()); assertTrue(authority.authenticate("lobby-1", issued.certificate())); @@ -86,7 +89,10 @@ void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { assertEquals(code.endpoint(), profile.endpoint()); assertEquals("lobby-1", HttpClientCredentialStore.loadEnrolled(directory.resolve("client")).profile().serverId()); assertNotEquals(null, HttpPinnedTls.mutualTlsContext(code, restored)); - Files.writeString(directory.resolve("client").resolve("http-transport-profile.properties"), "version=1\nserverId=lobby-1\n"); + Path clientDirectory = directory.resolve("client"); + String generation = Files.readString(clientDirectory.resolve("http-transport-client-current")); + Files.writeString(clientDirectory.resolve("http-transport-client-generations").resolve(generation) + .resolve("http-transport-profile.properties"), "version=1\nserverId=lobby-1\n"); assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.loadProfile(directory.resolve("client"))); HttpEnrollmentAuthority durable = new HttpEnrollmentAuthority(identity, directory.resolve("state")); HttpConnectionCode durableCode = durable.createConnectionCode("survival", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); @@ -96,5 +102,63 @@ void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { assertFalse(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); } + @Test + void renewalKeepsOldCredentialUntilReplacementAuthenticates() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("state")); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate original = authority.enroll("lobby-1", code.enrollmentToken()); + HttpTlsIdentity.IssuedClientCertificate replacement = authority.renew("lobby-1", original.certificate()); + + assertTrue(authority.authenticate("lobby-1", original.certificate()), "lost renewal responses must leave the old credential usable"); + assertTrue(authority.authenticate("lobby-1", replacement.certificate()), "first replacement request promotes the pending binding"); + assertFalse(authority.authenticate("lobby-1", original.certificate()), "promotion revokes the superseded credential"); + assertTrue(new HttpEnrollmentAuthority(identity, directory.resolve("state")) + .authenticate("lobby-1", replacement.certificate()), "promoted renewal must survive restart"); + } + + @Test + void serverLeafRotatesInsideRenewalWindowAndPreservesAuthority() throws Exception { + Instant now = Instant.now(); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(directory, "localhost", + Clock.fixed(now.minus(Duration.ofDays(340)), ZoneOffset.UTC)); + String originalPin = HttpTransportSecrets.certificatePin(original.serverCertificate()); + HttpTlsIdentity renewed = HttpTlsIdentity.loadOrCreate(directory, "localhost", Clock.fixed(now, ZoneOffset.UTC)); + assertNotEquals(originalPin, renewed.serverCertificatePin()); + assertEquals(original.caCertificatePin(), renewed.caCertificatePin()); + assertFalse(HttpTlsIdentity.needsRenewal(renewed.serverCertificate(), Clock.fixed(now, ZoneOffset.UTC))); + } + + @Test + void activeTlsContextRotatesServerLeafInsideRenewalWindow() throws Exception { + Instant now = Instant.now(); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory, "localhost", + Clock.fixed(now.minus(Duration.ofDays(340)), ZoneOffset.UTC)); + String expiringPin = HttpTransportSecrets.certificatePin(identity.serverCertificate()); + identity.serverContext(); + assertNotEquals(expiringPin, identity.serverCertificatePin()); + assertFalse(HttpTlsIdentity.needsRenewal(identity.serverCertificate(), Clock.systemUTC())); + } + + @Test + void stagedCredentialDoesNotReplaceActiveGenerationUntilAtomicActivation() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + Path client = directory.resolve("client"); + HttpTlsIdentity.IssuedClientCertificate original = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, code, original); + String originalPin = HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate()); + HttpTlsIdentity.IssuedClientCertificate replacement = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(client, replacement); + assertEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); + HttpClientCredentialStore.activateReplacement(client, staged); + assertNotEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); + HttpTlsIdentity.IssuedClientCertificate manuallyReenrolled = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, code, manuallyReenrolled); + assertEquals(HttpTransportSecrets.certificatePin(manuallyReenrolled.certificate()), + HttpTransportSecrets.certificatePin(HttpClientCredentialStore.loadEnrolled(client).credential().certificate())); + } + private static String pin(char character) { return String.valueOf(character).repeat(64); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java new file mode 100644 index 000000000..dd899304c --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -0,0 +1,49 @@ +package com.bencodez.votingplugin.backendproxy.transport; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode; +import com.bencodez.votingplugin.config.BungeeSettings; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; +import java.net.URI; +import java.nio.file.Path; +import java.time.Instant; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpBackendProxyTransportTest { + @TempDir Path directory; + + @Test + void validatesInitialConnectionCodeSynchronously() { + VotingPluginMain plugin = mock(VotingPluginMain.class); + BungeeSettings settings = mock(BungeeSettings.class); + when(plugin.getDataFolder()).thenReturn(directory.toFile()); + when(plugin.getBungeeSettings()).thenReturn(settings); + when(settings.getServer()).thenReturn("lobby-1"); + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(plugin); + + when(settings.getHttpConnectionCode()).thenReturn("malformed"); + assertThrows(IllegalStateException.class, transport::validate); + + when(settings.getHttpConnectionCode()).thenReturn(code("lobby-1", Instant.now().minusSeconds(1)).encode()); + assertThrows(IllegalStateException.class, transport::validate); + + when(settings.getHttpConnectionCode()).thenReturn(code("survival", Instant.now().plusSeconds(60)).encode()); + assertThrows(IllegalStateException.class, transport::validate); + assertThrows(IllegalStateException.class, () -> transport.start(mock(GlobalMessageHandler.class)), + "invalid configuration must fail before the enrollment worker starts"); + + when(settings.getHttpConnectionCode()).thenReturn(code("lobby-1", Instant.now().plusSeconds(60)).encode()); + assertDoesNotThrow(transport::validate); + } + + private static HttpConnectionCode code(String serverId, Instant expiry) { + return new HttpConnectionCode(serverId, URI.create("https://proxy.example.test:1297/"), "a".repeat(64), + "b".repeat(64), expiry, "A".repeat(43)); + } +} diff --git a/docs/http-transport.md b/docs/http-transport.md index 63e1da0a5..7e9bbc4c1 100644 --- a/docs/http-transport.md +++ b/docs/http-transport.md @@ -33,7 +33,8 @@ Connection codes expire after 15 minutes and can be used only once. Treat a fres ## Security model - TLS keys and a private certificate authority are generated automatically on the proxy. No shared transport password or public CA setup is required. -- The backend pins both the exact proxy certificate and its private authority during enrollment. Normal traffic uses a distinct client certificate bound to that backend's canonical `Server` name. +- Enrollment pins both the exact proxy certificate and its private authority. Normal traffic trusts only that pinned private authority and keeps HTTPS hostname verification enabled, allowing the proxy leaf certificate to rotate safely without trusting public certificate authorities. It uses a distinct client certificate bound to that backend's canonical `Server` name. +- Proxy and backend leaf certificates renew automatically during a 30-day pre-expiry window. Backend renewal is authenticated by the still-valid mTLS identity, persisted before use, and switches the proxy binding only after the replacement successfully connects; no new connection code is needed. - Every normal request is authenticated again at the application boundary. A payload cannot claim another backend identity, and redirect following is disabled. - TLS 1.3 is required and weak protocols are disabled. - Enrollment tokens are 256-bit random capabilities, single-use, short-lived, and retained by the proxy only as hashes. From 1d30c9420158266503c7e5c2750bc5d45838a9ca Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 17:11:00 -0600 Subject: [PATCH 004/136] Address HTTP transport review findings --- .../http/HttpClientCredentialStore.java | 38 ++++++++++++++++--- .../http/HttpProxyTransportServer.java | 14 +++---- .../backendproxy/http/HttpTlsIdentity.java | 33 +++++----------- .../transport/HttpBackendProxyTransport.java | 24 +++++++----- .../control/BackendConfigurationService.java | 4 +- .../http/HttpTransportRuntimeTest.java | 8 +++- .../http/HttpTransportSecurityTest.java | 19 ++++++++++ .../HttpBackendProxyTransportTest.java | 22 +++++++++++ .../BackendConfigurationServiceTest.java | 16 ++++++++ docs/http-transport.md | 2 +- 10 files changed, 132 insertions(+), 48 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index 459a2c360..f0d7fe681 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -22,6 +22,7 @@ public final class HttpClientCredentialStore { private static final String BUNDLE_FILE = "http-transport-client.p12"; private static final String PASSWORD_FILE = "http-transport-client-password"; private static final String PROFILE_FILE = "http-transport-profile.properties"; + private static final String CONNECTION_CODE_DIGEST_FILE = "http-transport-connection-code.sha256"; private static final String GENERATIONS_DIRECTORY = "http-transport-client-generations"; private static final String CURRENT_FILE = "http-transport-client-current"; private HttpClientCredentialStore() { } @@ -44,7 +45,7 @@ public static void saveEnrolled(Path directory, HttpConnectionCode code, HttpTls HttpClientProfile profile = new HttpClientProfile(HttpTlsIdentity.canonicalServerId(issued.serverId()), code.endpoint(), code.serverCertificatePin(), code.caCertificatePin()); try { - StagedCredential staged = stage(directory, issued, profile); + StagedCredential staged = stage(directory, issued, profile, connectionCodeDigest(code)); activateReplacement(directory, staged); } catch (IOException failure) { throw failure; } catch (Exception failure) { throw new IOException("Could not persist HTTP client credential", failure); } @@ -89,11 +90,12 @@ private static ClientCredential loadCredential(Path directory) throws Exception /** Writes and validates a replacement generation without touching the active credential. */ static StagedCredential stageReplacement(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws Exception { - return stage(directory, issued, loadProfile(directory)); + Path active = activeDirectory(directory); + return stage(directory, issued, loadProfileFile(active), readConnectionCodeDigest(active)); } private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClientCertificate issued, - HttpClientProfile profile) throws Exception { + HttpClientProfile profile, String connectionCodeDigest) throws Exception { if (directory == null || issued == null) throw new IllegalArgumentException("Credential replacement is required"); Path generations = directory.toAbsolutePath().normalize().resolve(GENERATIONS_DIRECTORY); Files.createDirectories(generations); @@ -106,11 +108,14 @@ private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClie try { save(generation, issued); writeProfile(generation, profile); + if (connectionCodeDigest != null) writePrivate(safe(generation.resolve(CONNECTION_CODE_DIGEST_FILE)), + connectionCodeDigest.getBytes(StandardCharsets.US_ASCII)); EnrolledClient enrolled = loadEnrolled(generation); return new StagedCredential(name, enrolled.credential()); } catch (Exception failure) { try { Files.deleteIfExists(generation.resolve(BUNDLE_FILE)); Files.deleteIfExists(generation.resolve(PASSWORD_FILE)); - Files.deleteIfExists(generation.resolve(PROFILE_FILE)); Files.deleteIfExists(generation); } + Files.deleteIfExists(generation.resolve(PROFILE_FILE)); Files.deleteIfExists(generation.resolve(CONNECTION_CODE_DIGEST_FILE)); + Files.deleteIfExists(generation); } catch (IOException cleanup) { failure.addSuppressed(cleanup); } throw failure; } @@ -151,10 +156,19 @@ private static HttpClientProfile loadProfileFile(Path directory) throws IOExcept } public static boolean hasEnrolledProfile(Path directory) { - try { loadProfile(directory); return true; } + try { loadEnrolled(directory); return true; } catch (Exception unavailable) { return false; } } + /** Returns whether this exact one-time code created the active credential, without persisting the code itself. */ + public static boolean matchesEnrollmentCode(Path directory, HttpConnectionCode code) throws IOException { + if (code == null) throw new IllegalArgumentException("Connection code is required"); + String stored = readConnectionCodeDigest(activeDirectory(directory)); + if (stored == null) return false; + return HttpTransportSecrets.constantTimeEquals(stored.getBytes(StandardCharsets.US_ASCII), + connectionCodeDigest(code).getBytes(StandardCharsets.US_ASCII)); + } + /** Loads and cross-checks the persisted client key material and bound normal-transport profile. */ public static EnrolledClient loadEnrolled(Path directory) throws Exception { Path active = activeDirectory(directory); @@ -222,6 +236,20 @@ private static Path activeDirectory(Path directory) throws IOException { return generation; } + private static String readConnectionCodeDigest(Path directory) throws IOException { + Path digest = safe(directory.resolve(CONNECTION_CODE_DIGEST_FILE)); + if (!Files.exists(digest, LinkOption.NOFOLLOW_LINKS)) return null; + if (!Files.isRegularFile(digest, LinkOption.NOFOLLOW_LINKS) || Files.size(digest) != 64) + throw new IOException("HTTP connection-code marker is invalid"); + String value = Files.readString(digest, StandardCharsets.US_ASCII); + if (!value.matches("[0-9a-f]{64}")) throw new IOException("HTTP connection-code marker is invalid"); + return value; + } + + private static String connectionCodeDigest(HttpConnectionCode code) { + return HttpTransportSecrets.sha256Hex(code.encode().getBytes(StandardCharsets.US_ASCII)); + } + private static void writePrivate(Path file, byte[] contents) throws IOException { Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index 18bb284d2..7d415bdc6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -107,12 +107,9 @@ public boolean send(String serverId, JsonEnvelope envelope) { if (closed || serverId == null || envelope == null) return false; try { serverId = HttpTlsIdentity.canonicalServerId(serverId); HttpTransportProtocol.validateEnvelope(envelope); } catch (IllegalArgumentException invalid) { return false; } - synchronized (backends) { - BackendState backend = backends.computeIfAbsent(serverId, ignored -> new BackendState()); - boolean accepted = backend.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), envelope)); - if (accepted) backend.signal(); - return accepted; - } + BackendState backend; + synchronized (backends) { backend = backends.computeIfAbsent(serverId, ignored -> new BackendState()); } + return backend.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), envelope)); } @Override public void close() { @@ -232,7 +229,10 @@ boolean acceptSession(String requested, long requestedSequence) { // prevents a captured request from being replayed with altered ACKs or a new payload. if (requestedSequence <= sequence) return false; sequence = requestedSequence; return true; } - private boolean enqueue(HttpTransportProtocol.Delivery delivery) { if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; outgoing.put(delivery.id(), delivery); return true; } + synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { + if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; + outgoing.put(delivery.id(), delivery); signal(); return true; + } private void acknowledge(Collection acks) { for (String id : acks) { outgoing.remove(id); delivered.remove(id); } } List acceptIncoming(List received) { List accepted = new java.util.ArrayList<>(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java index afdbb34c3..eb40dda21 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -32,7 +32,6 @@ import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509ExtendedKeyManager; -import javax.net.ssl.X509TrustManager; import org.bouncycastle.asn1.x500.X500Name; import org.bouncycastle.asn1.x509.BasicConstraints; import org.bouncycastle.asn1.x509.Extension; @@ -159,11 +158,19 @@ public String serverCertificatePin() { public SSLContext serverContext() throws Exception { renewServerCertificateIfNeeded(); SSLContext context = SSLContext.getInstance("TLS"); - context.init(new KeyManager[] { new RotatingServerKeyManager() }, - new TrustManager[] { new EnrollmentAwareTrustManager(caCertificate) }, null); + context.init(new KeyManager[] { new RotatingServerKeyManager() }, trustManagers(caCertificate), null); return context; } + static TrustManager[] trustManagers(X509Certificate caCertificate) throws Exception { + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + trustStore.load(null, EMPTY_PASSWORD); + trustStore.setCertificateEntry("http-transport-ca", caCertificate); + TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + factory.init(trustStore); + return factory.getTrustManagers(); + } + private synchronized void renewServerCertificateIfNeeded() throws Exception { if (!needsRenewal(serverCertificate, Clock.systemUTC())) return; ensureBouncyCastle(); @@ -378,25 +385,5 @@ private String alias(String keyType) { @Override public PrivateKey getPrivateKey(String alias) { refresh(); return ALIAS.equals(alias) ? serverKey : null; } } - private static final class EnrollmentAwareTrustManager implements X509TrustManager { - private final X509TrustManager delegate; - private EnrollmentAwareTrustManager(X509Certificate caCertificate) throws Exception { - KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); - trustStore.load(null, EMPTY_PASSWORD); - trustStore.setCertificateEntry("http-transport-ca", caCertificate); - TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - factory.init(trustStore); - this.delegate = Arrays.stream(factory.getTrustManagers()) - .filter(X509TrustManager.class::isInstance) - .map(X509TrustManager.class::cast) - .findFirst() - .orElseThrow(() -> new IllegalStateException("No X.509 trust manager is available")); - } - @Override public void checkClientTrusted(X509Certificate[] chain, String authType) throws java.security.cert.CertificateException { - delegate.checkClientTrusted(chain, authType); - } - @Override public void checkServerTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } - @Override public X509Certificate[] getAcceptedIssuers() { return delegate.getAcceptedIssuers(); } - } private enum CertificateRole { CA, SERVER, CLIENT } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 1b8e86569..1e436fbf1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -43,10 +43,8 @@ public void start(GlobalMessageHandler messageHandler) { private void initialize(Path directory, String serverId, String configuredCode, GlobalMessageHandler messageHandler) { try { - if (!HttpClientCredentialStore.hasEnrolledProfile(directory)) { - HttpConnectionCode code = HttpConnectionCode.parse(configuredCode); - HttpBackendTransportConnector.enroll(code, serverId, directory); - } + HttpConnectionCode code = enrollmentCode(directory, serverId, configuredCode); + if (code != null) HttpBackendTransportConnector.enroll(code, serverId, directory); HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); @@ -97,20 +95,28 @@ public void validate() { } private static void validateConfiguration(Path directory, String serverId, String configuredCode) { + enrollmentCode(directory, serverId, configuredCode); + } + + static HttpConnectionCode enrollmentCode(Path directory, String serverId, String configuredCode) { try { serverId = com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId); } catch (IllegalArgumentException invalid) { throw new IllegalStateException("HTTP requires a valid unique backend Server name", invalid); } - if (!HttpClientCredentialStore.hasEnrolledProfile(directory)) { - if (configuredCode == null || configuredCode.isBlank()) - throw new IllegalStateException("HTTP requires a temporary ConnectionCode for initial enrollment"); + boolean enrolled = HttpClientCredentialStore.hasEnrolledProfile(directory); + if (configuredCode != null && !configuredCode.isBlank()) { try { HttpConnectionCode code = HttpConnectionCode.parse(configuredCode); - code.requireActive(Clock.systemUTC()); if (!code.serverId().equals(serverId)) throw new IllegalArgumentException("Connection code belongs to a different backend"); - } catch (IllegalArgumentException invalid) { + if (enrolled && HttpClientCredentialStore.matchesEnrollmentCode(directory, code)) return null; + code.requireActive(Clock.systemUTC()); + return code; + } catch (Exception invalid) { throw new IllegalStateException("HTTP ConnectionCode is invalid, expired, or belongs to a different backend", invalid); } } + if (!enrolled) + throw new IllegalStateException("HTTP requires a temporary ConnectionCode for initial enrollment"); + return null; } @Override diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 0bb8d29f4..3309b7b93 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -28,6 +28,7 @@ import org.bukkit.configuration.InvalidConfigurationException; import org.bukkit.configuration.file.YamlConfiguration; +import com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore; import com.bencodez.votingplugin.proxy.BungeeMethod; import com.bencodez.votingplugin.util.DurableFiles; @@ -595,8 +596,7 @@ private void validateProxyMethod(BungeeMethod method, YamlConfiguration settings case HTTP: String connectionCode = settings.getString("HTTP.ConnectionCode", ""); if (connectionCode == null || connectionCode.isBlank()) { - Path identity = dataDirectory.resolve("http").resolve("http-transport-client.p12"); - if (!Files.isRegularFile(identity)) { + if (!HttpClientCredentialStore.hasEnrolledProfile(dataDirectory.resolve("http"))) { throw new IllegalArgumentException("HTTP.ConnectionCode must be set for initial enrollment"); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 77d1ac512..9c722a08c 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -222,7 +222,13 @@ void backendReAcknowledgesLostAckDuplicateWithoutSecondCallback() throws Excepti HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); connector.dispatch(delivery); assertTrue(callback.await(2, TimeUnit.SECONDS)); - assertEquals(java.util.List.of(id), connector.drainAcknowledgements()); + java.util.List acknowledgements = java.util.List.of(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (acknowledgements.isEmpty() && System.nanoTime() < deadline) { + acknowledgements = connector.drainAcknowledgements(); + if (acknowledgements.isEmpty()) Thread.sleep(5); + } + assertEquals(java.util.List.of(id), acknowledgements); assertTrue(connector.accept(java.util.List.of(delivery)).isEmpty()); assertEquals(java.util.List.of(id), connector.drainAcknowledgements()); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 996eda9be..cb67b5c1e 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -1,5 +1,6 @@ package com.bencodez.votingplugin.backendproxy.http; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -13,6 +14,8 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; +import java.util.Arrays; +import javax.net.ssl.X509TrustManager; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -140,6 +143,20 @@ void activeTlsContextRotatesServerLeafInsideRenewalWindow() throws Exception { assertFalse(HttpTlsIdentity.needsRenewal(identity.serverCertificate(), Clock.systemUTC())); } + @Test + void serverTlsUsesPrivateCaTrustAndRejectsForeignClients() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity foreign = HttpTlsIdentity.loadOrCreate(directory.resolve("foreign"), "localhost"); + X509TrustManager trust = Arrays.stream(HttpTlsIdentity.trustManagers(identity.caCertificate())) + .filter(X509TrustManager.class::isInstance).map(X509TrustManager.class::cast).findFirst().orElseThrow(); + HttpTlsIdentity.IssuedClientCertificate accepted = identity.issueClientCertificate("lobby-1"); + HttpTlsIdentity.IssuedClientCertificate rejected = foreign.issueClientCertificate("lobby-1"); + assertDoesNotThrow(() -> trust.checkClientTrusted( + new java.security.cert.X509Certificate[] { accepted.certificate(), identity.caCertificate() }, "EC")); + assertThrows(java.security.cert.CertificateException.class, () -> trust.checkClientTrusted( + new java.security.cert.X509Certificate[] { rejected.certificate(), foreign.caCertificate() }, "EC")); + } + @Test void stagedCredentialDoesNotReplaceActiveGenerationUntilAtomicActivation() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); @@ -154,6 +171,8 @@ void stagedCredentialDoesNotReplaceActiveGenerationUntilAtomicActivation() throw assertEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); HttpClientCredentialStore.activateReplacement(client, staged); assertNotEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, code), + "automatic certificate renewal must retain the consumed-code marker"); HttpTlsIdentity.IssuedClientCertificate manuallyReenrolled = identity.issueClientCertificate("lobby-1"); HttpClientCredentialStore.saveEnrolled(client, code, manuallyReenrolled); assertEquals(HttpTransportSecrets.certificatePin(manuallyReenrolled.certificate()), diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index dd899304c..0531cb055 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -1,12 +1,16 @@ package com.bencodez.votingplugin.backendproxy.transport; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore; import com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode; +import com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity; import com.bencodez.votingplugin.config.BungeeSettings; import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; import java.net.URI; @@ -42,6 +46,24 @@ void validatesInitialConnectionCodeSynchronously() { assertDoesNotThrow(transport::validate); } + @Test + void freshConnectionCodeOverridesAnExistingEnrollment() throws Exception { + Path credentials = directory.resolve("http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "proxy.example.test"); + HttpConnectionCode original = new HttpConnectionCode("lobby-1", URI.create("https://proxy.example.test:1297/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().minusSeconds(1), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(credentials, original, identity.issueClientCertificate("lobby-1")); + assertNull(HttpBackendProxyTransport.enrollmentCode(credentials, "lobby-1", "")); + assertNull(HttpBackendProxyTransport.enrollmentCode(credentials, "lobby-1", original.encode()), + "the already-consumed code must not be retried, even after it expires"); + + HttpConnectionCode replacement = new HttpConnectionCode("lobby-1", original.endpoint(), original.serverCertificatePin(), + original.caCertificatePin(), Instant.now().plusSeconds(60), "B".repeat(43)); + assertEquals(replacement.encode(), HttpBackendProxyTransport.enrollmentCode(credentials, "lobby-1", replacement.encode()).encode()); + assertThrows(IllegalStateException.class, + () -> HttpBackendProxyTransport.enrollmentCode(credentials, "lobby-1", "malformed")); + } + private static HttpConnectionCode code(String serverId, Instant expiry) { return new HttpConnectionCode(serverId, URI.create("https://proxy.example.test:1297/"), "a".repeat(64), "b".repeat(64), expiry, "A".repeat(43)); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index 05e92b462..23a0dc8ec 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -694,6 +694,22 @@ class BackendConfigurationServiceTest { assertTrue(read.content().contains(BackendConfigurationService.REDACTED)); } + @Test void httpMethodAcceptsGenerationBasedEnrolledProfile() throws Exception { + Files.writeString(directory.resolve("BungeeSettings.yml"), + "UseBungeecord: true\nServer: lobby-1\nBungeeMethod: PLUGINMESSAGING\nPluginMessageChannel: vp:vp\n"); + com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity identity = + com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode code = + new com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode("lobby-1", + java.net.URI.create("https://localhost:1297/"), identity.serverCertificatePin(), + identity.caCertificatePin(), java.time.Instant.now().plusSeconds(60), "A".repeat(43)); + com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore.saveEnrolled(directory.resolve("http"), code, + identity.issueClientCertificate("lobby-1")); + + BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); + assertDoesNotThrow(() -> service.previewQuickSetup("proxy-method", Map.of("method", "HTTP"))); + } + @Test void proxyMethodSwitchPreflightsRequiredBackendSettings() throws Exception { Path settings = directory.resolve("BungeeSettings.yml"); Files.writeString(settings, "UseBungeecord: true\nServer: lobby\nBungeeMethod: PLUGINMESSAGING\n" diff --git a/docs/http-transport.md b/docs/http-transport.md index 7e9bbc4c1..80bd99391 100644 --- a/docs/http-transport.md +++ b/docs/http-transport.md @@ -26,7 +26,7 @@ The `HTTP` bungee method gives every backend an outbound encrypted connection to ConnectionCode: 'paste-code-here' ``` -5. Restart the backend. Once enrollment succeeds, remove `ConnectionCode` from the configuration. The backend's private identity is stored in its VotingPlugin data folder and is reused automatically. +5. Restart the backend. Once enrollment succeeds, remove `ConnectionCode` from the configuration. The backend's private identity is stored in its VotingPlugin data folder and is reused automatically. A different nonblank code is treated as an explicit re-enrollment request; a digest lets harmless restarts recognize the already-consumed code without storing its secret token. Connection codes expire after 15 minutes and can be used only once. Treat a fresh code like a temporary password: transfer it privately and do not publish it in logs, tickets, or chat rooms. From e9bbe03612d276dd3d5a0cfc6ccdad944b4a8e1c Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 17:22:19 -0600 Subject: [PATCH 005/136] Harden HTTP shutdown and protocol parsing --- .../http/HttpTransportProtocol.java | 15 ++++++-- .../transport/HttpBackendProxyTransport.java | 37 +++++++++++-------- .../http/HttpTransportRuntimeTest.java | 22 +++++++++++ .../HttpBackendProxyTransportTest.java | 25 +++++++++++++ 4 files changed, 80 insertions(+), 19 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java index f275e9b76..a846c410b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java @@ -67,8 +67,9 @@ static Packet parsePacket(byte[] body) { String server = HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); String session = uuid(root, "session"); long sequence = nonNegative(root, "sequence"); - long timestamp = root.get("timestamp").getAsLong(); - if (Math.abs(Instant.now().toEpochMilli() - timestamp) > MAX_CLOCK_SKEW_MILLIS) throw bad(); + long timestamp = integer(root, "timestamp"); + long now = Instant.now().toEpochMilli(); + if (timestamp < now - MAX_CLOCK_SKEW_MILLIS || timestamp > now + MAX_CLOCK_SKEW_MILLIS) throw bad(); List acks = parseIds(root.get("acks")); List messages = parseMessages(root.get("messages")); return new Packet(server, session, sequence, acks, messages); @@ -179,7 +180,15 @@ private static void requireOnly(JsonObject object, String... names) { for (String name : names) if (!object.has(name) || object.get(name).isJsonNull()) throw bad(); } private static String string(JsonObject object, String name, int max) { JsonElement v = object.get(name); if (!v.isJsonPrimitive() || !v.getAsJsonPrimitive().isString()) throw bad(); String value = v.getAsString(); if (value.isEmpty() || value.length() > max) throw bad(); return value; } - private static long integer(JsonObject object, String name) { try { return object.get(name).getAsLong(); } catch (RuntimeException failure) { throw bad(); } } + private static long integer(JsonObject object, String name) { + try { + JsonElement value = object.get(name); + if (!value.isJsonPrimitive() || !value.getAsJsonPrimitive().isNumber()) throw bad(); + String token = value.getAsString(); + if (!token.matches("-?(?:0|[1-9][0-9]*)")) throw bad(); + return Long.parseLong(token); + } catch (RuntimeException failure) { throw bad(); } + } private static long nonNegative(JsonObject object, String name) { long n = integer(object, name); if (n < 0) throw bad(); return n; } private static String uuid(JsonObject object, String name) { String value = string(object, name, 64); try { return UUID.fromString(value).toString(); } catch (IllegalArgumentException invalid) { throw bad(); } } private static void validId(String id) { if (id == null || id.length() > 64) throw bad(); try { UUID.fromString(id); } catch (IllegalArgumentException invalid) { throw bad(); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 1e436fbf1..ac077a074 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -49,19 +49,21 @@ private void initialize(Path directory, String serverId, String configuredCode, if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); HttpBackendTransportConnector replacement = new HttpBackendTransportConnector(directory, messageHandler::onMessage); + boolean discard = false; synchronized (lifecycle) { if (closed) { - replacement.close(); - return; - } - connector = replacement; - replacement.start(); - while (!startupQueue.isEmpty()) { - if (!replacement.send(startupQueue.removeFirst())) { - throw new IllegalStateException("HTTP startup queue could not be transferred"); + discard = true; + } else { + connector = replacement; + replacement.start(); + while (!startupQueue.isEmpty()) { + if (!replacement.send(startupQueue.removeFirst())) { + throw new IllegalStateException("HTTP startup queue could not be transferred"); + } } } } + if (discard) replacement.close(); } catch (Exception failure) { startupFailure = new IllegalStateException("Secure HTTP backend enrollment or connection failed", failure); plugin.getLogger().severe("Secure HTTP backend transport is unavailable; check the connection code and proxy endpoint"); @@ -124,6 +126,7 @@ public void close() { Thread setup; HttpBackendTransportConnector active; synchronized (lifecycle) { + if (closed) return; closed = true; startupQueue.clear(); setup = worker; @@ -131,14 +134,16 @@ public void close() { active = connector; connector = null; } - if (setup != null) { - setup.interrupt(); - try { - setup.join(TimeUnit.SECONDS.toMillis(5)); - } catch (InterruptedException interrupted) { - Thread.currentThread().interrupt(); - } - } + if (setup != null) setup.interrupt(); + if (setup == null && active == null) return; + Thread cleanup = new Thread(() -> drain(setup, active), "VotingPlugin-HTTP-Backend-Cleanup"); + cleanup.setDaemon(true); + cleanup.start(); + } + + private static void drain(Thread setup, HttpBackendTransportConnector active) { + if (setup != null) try { setup.join(TimeUnit.SECONDS.toMillis(5)); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } if (active != null) active.close(); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 9c722a08c..75cf657ad 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -1,7 +1,9 @@ package com.bencodez.votingplugin.backendproxy.http; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; @@ -87,6 +89,26 @@ void aggregatePacketBudgetSplitsLargeValidEnvelopes() { <= HttpTransportProtocol.MAX_BODY_BYTES); } + @Test + void packetNumbersMustUseCanonicalJsonIntegerTokens() { + com.google.gson.JsonObject packet = com.google.gson.JsonParser.parseString(new String(HttpTransportProtocol.request( + "lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(), java.util.List.of()), + java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); + assertDoesNotThrow(() -> HttpTransportProtocol.parsePacket(packet.toString() + .getBytes(java.nio.charset.StandardCharsets.UTF_8))); + String timestamp = packet.get("timestamp").getAsString(); + java.util.Map> invalid = java.util.Map.of( + "v", java.util.List.of("\"1\"", "1.0", "1e0"), + "sequence", java.util.List.of("\"0\"", "0.0", "0e0"), + "timestamp", java.util.List.of("\"" + timestamp + "\"", timestamp + ".0", timestamp + "e0")); + for (var field : invalid.entrySet()) for (String token : field.getValue()) { + com.google.gson.JsonObject rejected = packet.deepCopy(); + rejected.add(field.getKey(), com.google.gson.JsonParser.parseString(token)); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + rejected.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8)), field.getKey() + "=" + token); + } + } + @Test void persistedProfileStartsAfterTheEnrollmentCodeExpires() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 0531cb055..d11aea72f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -16,6 +17,8 @@ import java.net.URI; import java.nio.file.Path; import java.time.Instant; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -64,6 +67,28 @@ void freshConnectionCodeOverridesAnExistingEnrollment() throws Exception { () -> HttpBackendProxyTransport.enrollmentCode(credentials, "lobby-1", "malformed")); } + @Test + void closeNeverWaitsForSetupOnTheCallingThread() throws Exception { + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(mock(VotingPluginMain.class)); + CountDownLatch started = new CountDownLatch(1), release = new CountDownLatch(1); + Thread blocked = new Thread(() -> { + started.countDown(); + while (release.getCount() != 0) try { release.await(); } + catch (InterruptedException ignored) { /* Simulate setup I/O that has not unwound yet. */ } + }); + blocked.start(); + assertTrue(started.await(1, TimeUnit.SECONDS)); + java.lang.reflect.Field worker = HttpBackendProxyTransport.class.getDeclaredField("worker"); + worker.setAccessible(true); + worker.set(transport, blocked); + + long startedAt = System.nanoTime(); + transport.close(); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt); + try { assertTrue(elapsedMillis < 500, "close blocked the calling thread for " + elapsedMillis + " ms"); } + finally { release.countDown(); blocked.join(TimeUnit.SECONDS.toMillis(1)); } + } + private static HttpConnectionCode code(String serverId, Instant expiry) { return new HttpConnectionCode(serverId, URI.create("https://proxy.example.test:1297/"), "a".repeat(64), "b".repeat(64), expiry, "A".repeat(43)); From 38a37ddb1b00f4b1f19918d86a2eb63f0da0c728 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 17:31:49 -0600 Subject: [PATCH 006/136] fix(http): retain votes rejected by transport queue --- .../backendproxy/http/HttpConnectionCode.java | 3 +- .../votingplugin/proxy/VotingPluginProxy.java | 63 ++++++++++++++++--- .../ProxyMethodConfigurationService.java | 1 + .../http/HttpTransportSecurityTest.java | 7 +++ .../ProxyMethodConfigurationServiceTest.java | 8 +++ .../tests/VotingPluginProxyTest.java | 43 +++++++++++++ .../tests/VotingPluginProxyTestImpl.java | 13 ++++ 7 files changed, 128 insertions(+), 10 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java index 812751426..5700147fb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java @@ -68,7 +68,8 @@ private static URI validateEndpoint(URI value) { if (value == null || !"https".equalsIgnoreCase(value.getScheme()) || value.getHost() == null || value.getUserInfo() != null || value.getFragment() != null || value.getRawQuery() != null) throw new IllegalArgumentException("Endpoint must be an absolute HTTPS URL without credentials or query"); - if (value.getPort() > 65535 || value.getPort() < -1) throw new IllegalArgumentException("Endpoint port is invalid"); + if (value.getPort() == 0 || value.getPort() > 65535 || value.getPort() < -1) + throw new IllegalArgumentException("Endpoint port is invalid"); String path = value.getRawPath(); if (path == null || path.isEmpty()) path = "/"; if (!path.endsWith("/")) path += "/"; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 9815b6a14..441aaac27 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -578,6 +578,24 @@ protected boolean sendProxyBroadcastEnvelopeNow(String server, JsonEnvelope enve } } + /** + * Sends a reward-bearing vote envelope and reports whether the selected + * transport accepted it. Legacy transports retain their existing asynchronous + * semantics; HTTP exposes its bounded-queue result so a vote is never discarded + * when the queue is full. + */ + protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope) { + if (method == BungeeMethod.HTTP) { + return sendHttpEnvelope(server, envelope); + } + GlobalMessageProxyHandler handler = globalMessageProxyHandler; + if (handler == null) { + return false; + } + handler.sendMessage(server, delay, envelope); + return true; + } + public synchronized void checkCachedVotes(String server) { int delay = 1; if (isServerValid(server)) { @@ -626,11 +644,14 @@ && getConfig().getProxyBroadcastEnabled()) { broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); } - globalMessageProxyHandler.sendMessage(server, delay, + if (!sendVoteEnvelopeAccepted(server, delay, VotingPluginWire.vote(cache.getPlayerName(), cache.getUuid(), cache.getService(), cache.getTime(), false, cache.isRealVote(), cache.getText(), cache.getVoteId(), getConfig().getBungeeManageTotals(), - broadcastHere, num, numberOfVotes)); + broadcastHere, num, numberOfVotes))) { + debug("Retaining cached vote because the transport rejected delivery for " + server); + continue; + } delay++; num++; removed.add(cache); @@ -688,10 +709,14 @@ && getConfig().getProxyBroadcastEnabled()) { } if (!cache.isRewardDelivered()) { - globalMessageProxyHandler.sendMessage(server, delay, + if (!sendVoteEnvelopeAccepted(server, delay, VotingPluginWire.voteOnline(cache.getPlayerName(), cache.getUuid(), cache.getService(), cache.getTime(), false, cache.isRealVote(), cache.getText(), cache.getVoteId(), - getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes)); + getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes))) { + debug("Retaining online vote because the transport rejected delivery for " + server); + retained.add(cache); + continue; + } // The normal envelope is also a valid broadcast delivery for the // current target. Record it so a previously pending standalone // retry cannot announce the same vote again later. @@ -2675,6 +2700,7 @@ private void startHttpTransport() { try { URI endpoint = URI.create(getConfig().getHttpPublicEndpoint()); if (!"https".equalsIgnoreCase(endpoint.getScheme()) || endpoint.getHost() == null + || endpoint.getPort() == 0 || endpoint.getPort() > 65535 || endpoint.getUserInfo() != null || endpoint.getQuery() != null || endpoint.getFragment() != null || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) { throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); @@ -3461,9 +3487,18 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea broadcastHere = proxyBroadcastDecider.shouldBroadcast(s, targets); } - globalMessageProxyHandler.sendMessage(s, 2, + if (!sendVoteEnvelopeAccepted(s, 2, VotingPluginWire.vote(player, uuid, service, time, true, realVote, text.toString(), - voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1)); + voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1))) { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addServerVote(s, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, + text.toString(), broadcastForwarded, standaloneProxyBroadcast, + proxyBroadcastTargets, broadcastForwardedServers, false)); + debug("Caching vote after the transport rejected delivery for " + s); + } } } } else { @@ -3479,11 +3514,21 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); } - globalMessageProxyHandler.sendMessage(server, 1, + boolean rewardAccepted = sendVoteEnvelopeAccepted(server, 1, VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(), voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1)); + if (!rewardAccepted) { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addOnlineVote(uuid, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), + broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, + broadcastForwardedServers, false)); + debug("Caching online vote after the transport rejected delivery for " + server); + } - if (canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled() + if (rewardAccepted && canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled() && !standaloneProxyBroadcast) { Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); @@ -3505,7 +3550,7 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea } // multiproxy: envelope-only clear vote - if (getConfig().getMultiProxySupport() && getConfig().getMultiProxyOneGlobalReward()) { + if (rewardAccepted && getConfig().getMultiProxySupport() && getConfig().getMultiProxyOneGlobalReward()) { multiProxyHandler.sendClearVote(uuid, player); } } else { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java index f2c72515b..d58041214 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java @@ -68,6 +68,7 @@ private void validate(ProxyMethodConfiguration proposal, VotingPluginProxyConfig try { httpEndpoint = URI.create(endpoint); } catch (RuntimeException invalid) { throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); } if (!"https".equalsIgnoreCase(httpEndpoint.getScheme()) || httpEndpoint.getHost() == null + || httpEndpoint.getPort() == 0 || httpEndpoint.getPort() > 65535 || httpEndpoint.getUserInfo() != null || httpEndpoint.getQuery() != null || httpEndpoint.getFragment() != null || (httpEndpoint.getPath() != null && !httpEndpoint.getPath().isEmpty() && !"/".equals(httpEndpoint.getPath()))) { throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index cb67b5c1e..d85574604 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -20,6 +20,13 @@ import org.junit.jupiter.api.io.TempDir; class HttpTransportSecurityTest { + @Test + void connectionCodeRejectsExplicitZeroPort() { + assertThrows(IllegalArgumentException.class, + () -> new HttpConnectionCode("lobby", URI.create("https://proxy.example.test:0/"), pin('a'), + pin('b'), Instant.now().plusSeconds(60), "token")); + } + @TempDir Path directory; @Test diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java index 164c7a7ea..f49600dce 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java @@ -69,6 +69,14 @@ void validatesRequiredSettingsForEveryTransport() { when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test:1297/terminated-path"); assertThrows(IllegalArgumentException.class, () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test:0"); + assertThrows(IllegalArgumentException.class, + () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test:65536"); + assertThrows(IllegalArgumentException.class, + () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test"); + assertDoesNotThrow(() -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test:1297"); assertDoesNotThrow(() -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index b6c587a5a..4f8b711fb 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -604,6 +604,49 @@ void pendingServerBroadcastRetriesBeforeOfflineRewardDelivery() { verify(voteCache).updateServerVote("Server1", vote); } + @Test + void rejectedHttpQueueDeliveryRetainsCachedServerVote() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote vote = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "player-uuid", + "Service", 100L, true, "totals"); + Mockito.when(voteCache.hasVotes("Server1")).thenReturn(true); + Mockito.when(voteCache.getVotes("Server1")) + .thenReturn(new java.util.ArrayList<>(java.util.List.of(vote))); + Mockito.when(votingPluginProxy.getConfig().getBlockedServers()) + .thenReturn(java.util.Collections.emptyList()); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.checkCachedVotes("Server1"); + + verify(voteCache).removeServerVotes(Mockito.eq("Server1"), + Mockito.argThat(java.util.List::isEmpty)); + } + + @Test + void rejectedHttpQueueDeliveryRetainsCachedOnlineVote() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote vote = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "player-uuid", + "Service", 100L, true, "totals"); + Mockito.when(voteCache.hasOnlineVotes("player-uuid")).thenReturn(true); + Mockito.when(voteCache.getOnlineVotes("player-uuid")) + .thenReturn(new java.util.ArrayList<>(java.util.List.of(vote))); + Mockito.when(votingPluginProxy.getConfig().getBlockedServers()) + .thenReturn(java.util.Collections.emptyList()); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.checkOnlineVotes("Player", "player-uuid", "Server1"); + + assertFalse(vote.isRewardDelivered()); + verify(voteCache).addOnlineVote("player-uuid", vote); + verify(multiProxyHandler, never()).sendClearVote(Mockito.anyString(), Mockito.anyString()); + } + @Test void pendingOnlineBroadcastRetriesWhenTargetGainsAnyCarrier() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index ecdd7e6d1..dd22c32ed 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -23,6 +23,7 @@ public class VotingPluginProxyTestImpl extends VotingPluginProxy { private final List warnings = new ArrayList<>(); private VotingPluginProxyConfig config; private boolean pluginMessageDeliveryResult = true; + private boolean voteEnvelopeDeliveryResult = true; private boolean communicationTestDeliveryResult = true; private JsonEnvelope lastCommunicationTestEnvelope; private boolean playerOnline = true; @@ -212,6 +213,18 @@ public boolean sendProxyBroadcastImmediately(String server, JsonEnvelope envelop return sendProxyBroadcastEnvelopeNow(server, envelope); } + @Override + protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope) { + if (getMethod() == com.bencodez.votingplugin.proxy.BungeeMethod.HTTP) { + return voteEnvelopeDeliveryResult; + } + return super.sendVoteEnvelopeAccepted(server, delay, envelope); + } + + public void setVoteEnvelopeDeliveryResult(boolean voteEnvelopeDeliveryResult) { + this.voteEnvelopeDeliveryResult = voteEnvelopeDeliveryResult; + } + @Override protected boolean sendCommunicationTestEnvelopeNow(String server, JsonEnvelope envelope) { lastCommunicationTestEnvelope = envelope; From 57226856ab49dcf52815d27a78fc6a7a02cb0a7f Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 17:42:49 -0600 Subject: [PATCH 007/136] fix(http): persist outbound deliveries until ack --- .../http/HttpProxyTransportServer.java | 146 +++++++++++++++++- .../http/HttpTransportProtocol.java | 24 +++ .../votingplugin/proxy/VotingPluginProxy.java | 2 +- .../http/HttpTransportRuntimeTest.java | 37 +++++ 4 files changed, 205 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index 7d415bdc6..a70ef95ae 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -1,6 +1,7 @@ package com.bencodez.votingplugin.backendproxy.http; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.votingplugin.util.DurableFiles; import com.sun.net.httpserver.Headers; import com.sun.net.httpserver.HttpsConfigurator; import com.sun.net.httpserver.HttpsExchange; @@ -11,6 +12,12 @@ import java.io.InputStream; import java.net.InetSocketAddress; import java.net.URI; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.time.Duration; @@ -58,13 +65,26 @@ public final class HttpProxyTransportServer implements AutoCloseable { private final ThreadPoolExecutor handlerExecutor; private final Semaphore admission = new Semaphore(64); private final Map backends = new HashMap<>(); + private final DurableOutgoingQueue durableOutgoing; private final Consumer onEnvelope; private volatile boolean closed; public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, Consumer onEnvelope) throws Exception { + this(bind, identity, authority, null, onEnvelope); + } + + public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Path outgoingDirectory, Consumer onEnvelope) throws Exception { if (bind == null || identity == null || authority == null || onEnvelope == null) throw new IllegalArgumentException("HTTP transport configuration is required"); this.identity = identity; this.authority = authority; this.onEnvelope = onEnvelope; + durableOutgoing = outgoingDirectory == null ? null : new DurableOutgoingQueue(outgoingDirectory); + if (durableOutgoing != null) for (Map.Entry> pending + : durableOutgoing.load().entrySet()) { + BackendState state = new BackendState(pending.getKey(), durableOutgoing); + state.restore(pending.getValue()); + backends.put(pending.getKey(), state); + } server = HttpsServer.create(bind, 32); server.setHttpsConfigurator(new HttpsConfigurator(identity.serverContext()) { @Override public void configure(HttpsParameters parameters) { @@ -102,13 +122,15 @@ private void renew(HttpsExchange exchange) throws IOException { public int port() { return server.getAddress().getPort(); } public URI endpoint(String host) { return URI.create("https://" + host + ":" + port() + "/"); } - /** Queues an in-memory proxy-origin envelope for a specific authenticated backend; this queue is not restart-durable. */ + /** Queues a proxy-origin envelope durably before reporting acceptance. */ public boolean send(String serverId, JsonEnvelope envelope) { if (closed || serverId == null || envelope == null) return false; try { serverId = HttpTlsIdentity.canonicalServerId(serverId); HttpTransportProtocol.validateEnvelope(envelope); } catch (IllegalArgumentException invalid) { return false; } BackendState backend; - synchronized (backends) { backend = backends.computeIfAbsent(serverId, ignored -> new BackendState()); } + final String canonicalServerId = serverId; + synchronized (backends) { backend = backends.computeIfAbsent(serverId, + ignored -> new BackendState(canonicalServerId, durableOutgoing)); } return backend.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), envelope)); } @@ -208,6 +230,8 @@ private static ThreadPoolExecutor executor(String name, int threads, int queue) public record ReceivedEnvelope(String serverId, String messageId, JsonEnvelope envelope) { } static record Response(Collection acks, Collection messages) { } static final class BackendState { + private final String serverId; + private final DurableOutgoingQueue durableOutgoing; private String session; private long sequence = -1L; private final LinkedHashMap outgoing = new LinkedHashMap<>(); private final Set seen = new LinkedHashSet<>(); private final Set processing = new LinkedHashSet<>(); @@ -217,6 +241,13 @@ static final class BackendState { private double requestTokens = 24.0d; private long lastTokenNanos = System.nanoTime(); private boolean activePoll; + BackendState() { this(null, null); } + private BackendState(String serverId, DurableOutgoingQueue durableOutgoing) { + this.serverId = serverId; this.durableOutgoing = durableOutgoing; + } + private synchronized void restore(Collection deliveries) { + for (HttpTransportProtocol.Delivery delivery : deliveries) outgoing.put(delivery.id(), delivery); + } private boolean beginPoll(String requestedSession) { synchronized (this) { if (activePoll) return false; activePoll = true; return true; } } private void endPoll() { synchronized (this) { activePoll = false; notifyAll(); } } private boolean allowRequest() { @@ -231,9 +262,18 @@ boolean acceptSession(String requested, long requestedSequence) { } synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; + if (durableOutgoing != null) try { durableOutgoing.persist(serverId, delivery); } + catch (IOException failure) { return false; } outgoing.put(delivery.id(), delivery); signal(); return true; } - private void acknowledge(Collection acks) { for (String id : acks) { outgoing.remove(id); delivered.remove(id); } } + private void acknowledge(Collection acks) { + for (String id : acks) { + if (!outgoing.containsKey(id)) continue; + if (durableOutgoing != null) try { durableOutgoing.remove(serverId, id); } + catch (IOException failure) { continue; } + outgoing.remove(id); delivered.remove(id); + } + } List acceptIncoming(List received) { List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : received) { @@ -268,4 +308,104 @@ synchronized Response await(String serverId, String requestedSession, long reque private boolean redeliveryDue() { return !outgoing.isEmpty() && lastDeliveryNanos > 0L && System.nanoTime() - lastDeliveryNanos >= LONG_POLL.toNanos(); } private synchronized void signal() { notifyAll(); } } + + private static final class DurableOutgoingQueue { + private static final String FILE_PATTERN = "[0-9]{20}-[0-9a-f-]{36}\\.json"; + private final Path root; + private final Map> files = new HashMap<>(); + private long sequence; + + private DurableOutgoingQueue(Path root) throws IOException { + this.root = root.toAbsolutePath().normalize(); + Files.createDirectories(this.root); + if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue directory is invalid"); + ownerOnlyDirectory(this.root); + } + + private synchronized Map> load() throws IOException { + Map> loaded = new LinkedHashMap<>(); + try (DirectoryStream servers = Files.newDirectoryStream(root)) { + for (Path directory : servers) { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue contains an invalid entry"); + String serverId; + try { serverId = HttpTlsIdentity.canonicalServerId(directory.getFileName().toString()); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue server is invalid", invalid); } + if (!serverId.equals(directory.getFileName().toString())) + throw new IOException("HTTP outgoing queue server is not canonical"); + List entries = new java.util.ArrayList<>(); + try (DirectoryStream messages = Files.newDirectoryStream(directory)) { + for (Path message : messages) entries.add(message); + } + entries.sort(java.util.Comparator.comparing(path -> path.getFileName().toString())); + List deliveries = new java.util.ArrayList<>(); + Map serverFiles = files.computeIfAbsent(serverId, ignored -> new HashMap<>()); + for (Path message : entries) { + String name = message.getFileName().toString(); + if (name.startsWith(".pending-") && name.endsWith(".tmp") + && !Files.isSymbolicLink(message) && Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS)) { + DurableFiles.deleteIfExists(message); + continue; + } + if (Files.isSymbolicLink(message) || !Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS) + || !name.matches(FILE_PATTERN) || Files.size(message) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L) + throw new IOException("HTTP outgoing queue message is invalid"); + HttpTransportProtocol.Delivery delivery; + try { delivery = HttpTransportProtocol.parseStoredDelivery(Files.readAllBytes(message)); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue message is invalid", invalid); } + if (!name.endsWith("-" + delivery.id() + ".json") || serverFiles.put(delivery.id(), message) != null) + throw new IOException("HTTP outgoing queue message id is invalid"); + deliveries.add(delivery); + if (deliveries.size() > HttpTransportProtocol.MAX_QUEUE) + throw new IOException("HTTP outgoing queue exceeds its bound"); + sequence = Math.max(sequence, Long.parseLong(name.substring(0, 20))); + } + if (!deliveries.isEmpty()) loaded.put(serverId, deliveries); + } + } + return loaded; + } + + private synchronized void persist(String serverId, HttpTransportProtocol.Delivery delivery) throws IOException { + Path directory = root.resolve(serverId).normalize(); + if (!directory.getParent().equals(root)) throw new IOException("HTTP outgoing queue server is invalid"); + Files.createDirectories(directory); ownerOnlyDirectory(directory); + if (sequence == Long.MAX_VALUE) throw new IOException("HTTP outgoing queue sequence is exhausted"); + String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); + Path target = directory.resolve(name); + Path temporary = Files.createTempFile(directory, ".pending-", ".tmp"); + try { + ownerOnlyFile(temporary); + Files.write(temporary, HttpTransportProtocol.storedDelivery(delivery), StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } + ownerOnlyFile(target); DurableFiles.forceDirectory(directory); + files.computeIfAbsent(serverId, ignored -> new HashMap<>()).put(delivery.id(), target); + } finally { Files.deleteIfExists(temporary); } + } + + private synchronized void remove(String serverId, String id) throws IOException { + Map serverFiles = files.get(serverId); + Path file = serverFiles == null ? null : serverFiles.get(id); + if (file == null) throw new IOException("HTTP outgoing queue acknowledgement is unknown"); + DurableFiles.deleteIfExists(file); + serverFiles.remove(id); + } + + private static void ownerOnlyFile(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, java.util.EnumSet.of( + java.nio.file.attribute.PosixFilePermission.OWNER_READ, + java.nio.file.attribute.PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { } + } + private static void ownerOnlyDirectory(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, java.util.EnumSet.of( + java.nio.file.attribute.PosixFilePermission.OWNER_READ, + java.nio.file.attribute.PosixFilePermission.OWNER_WRITE, + java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE)); } + catch (UnsupportedOperationException ignored) { } + } + } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java index a846c410b..3fa01f878 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java @@ -51,6 +51,30 @@ static List fittingMessages(String server, String session, long sequen return output; } + static byte[] storedDelivery(Delivery delivery) { + validId(delivery.id()); + validateEnvelope(delivery.envelope()); + JsonObject root = new JsonObject(); + root.addProperty("v", VERSION); + root.addProperty("id", delivery.id()); + root.addProperty("payload", Base64.getUrlEncoder().withoutPadding().encodeToString( + JsonEnvelopeCodec.encode(delivery.envelope()).getBytes(StandardCharsets.UTF_8))); + return root.toString().getBytes(StandardCharsets.UTF_8); + } + + static Delivery parseStoredDelivery(byte[] body) { + if (body == null || body.length == 0 || body.length > MAX_ENVELOPE_BYTES * 2) throw bad(); + try { + JsonObject root = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)).getAsJsonObject(); + requireOnly(root, "v", "id", "payload"); + if (integer(root, "v") != VERSION) throw bad(); + String id = string(root, "id", 64); validId(id); + byte[] payload = Base64.getUrlDecoder().decode(string(root, "payload", MAX_ENVELOPE_BYTES * 2)); + if (payload.length == 0 || payload.length > MAX_ENVELOPE_BYTES) throw bad(); + return new Delivery(id, JsonEnvelopeCodec.decode(new String(payload, StandardCharsets.UTF_8))); + } catch (RuntimeException invalid) { throw bad(); } + } + static byte[] response(String server, String session, long sequence, Collection acks, Collection messages) { return request(server, session, sequence, acks, messages); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 441aaac27..b7dc11054 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -2710,7 +2710,7 @@ private void startHttpTransport() { httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); httpTransportServer = new HttpProxyTransportServer( new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, - httpEnrollmentAuthority, received -> { + httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), received -> { GlobalMessageProxyHandler handler = globalMessageProxyHandler; if (handler == null) throw new IllegalStateException("HTTP message router is not ready"); handler.onMessage(received.envelope()); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 75cf657ad..a75bb4c1a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -76,6 +76,43 @@ void boundedQueuesFailClosed() throws Exception { } } + @Test + void proxyOutgoingQueueSurvivesRestartUntilBackendAcknowledges() throws Exception { + Path proxyDirectory = directory.resolve("proxy"); + Path authorityDirectory = directory.resolve("authority"); + Path queueDirectory = directory.resolve("outgoing"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); + HttpProxyTransportServer first = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueDirectory, ignored -> { }); + assertTrue(first.send("lobby-1", JsonEnvelope.builder("durable").build())); + first.close(); + + CountDownLatch received = new CountDownLatch(1); + try (HttpProxyTransportServer restarted = new HttpProxyTransportServer( + new InetSocketAddress("localhost", 0), identity, authority, queueDirectory, ignored -> { })) { + restarted.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", restarted.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, + "lobby-1", directory.resolve("durable-client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", + credential, envelope -> received.countDown())) { + connector.start(); + assertTrue(received.await(8, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (countRegularFiles(queueDirectory) != 0L && System.nanoTime() < deadline) Thread.sleep(20); + assertEquals(0L, countRegularFiles(queueDirectory), "backend ACK must durably remove the delivery"); + } + } + } + + private static long countRegularFiles(Path root) throws Exception { + try (java.util.stream.Stream paths = java.nio.file.Files.walk(root)) { + return paths.filter(path -> java.nio.file.Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS)).count(); + } + } + @Test void aggregatePacketBudgetSplitsLargeValidEnvelopes() { java.util.List candidates = new java.util.ArrayList<>(); From aa0f8d8ba5ef5eddeba87d99c511c9001a837593 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 17:51:08 -0600 Subject: [PATCH 008/136] fix(http): persist poll-created backend queues --- .../http/HttpProxyTransportServer.java | 3 +- .../http/HttpTransportRuntimeTest.java | 34 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index a70ef95ae..c77b8a3f0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -163,7 +163,8 @@ private void transport(HttpsExchange exchange) throws IOException { X509Certificate certificate = peerCertificate(exchange); if (certificate == null || !authority.authenticate(packet.server(), certificate)) { reply(exchange, 401, new byte[0]); return; } BackendState backend; - synchronized (backends) { backend = backends.computeIfAbsent(packet.server(), ignored -> new BackendState()); } + synchronized (backends) { backend = backends.computeIfAbsent(packet.server(), + ignored -> new BackendState(packet.server(), durableOutgoing)); } if (!backend.beginPoll(packet.session())) { reply(exchange, 409, new byte[0]); return; } try { handlePacket(packet, backend); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index a75bb4c1a..ac95b1f8b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -107,6 +107,40 @@ void proxyOutgoingQueueSurvivesRestartUntilBackendAcknowledges() throws Exceptio } } + @Test + void pollCreatedBackendStateUsesDurableOutgoingQueue() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("poll-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("poll-authority")); + Path queueDirectory = directory.resolve("poll-outgoing"); + CountDownLatch proxyReceived = new CountDownLatch(1), backendReceived = new CountDownLatch(1); + CountDownLatch releaseBackendCallback = new CountDownLatch(1); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueDirectory, ignored -> proxyReceived.countDown())) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, + "lobby-1", directory.resolve("poll-client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", + credential, envelope -> { + backendReceived.countDown(); + try { releaseBackendCallback.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + })) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("establish-poll").build())); + assertTrue(proxyReceived.await(8, TimeUnit.SECONDS)); + assertTrue(server.send("lobby-1", JsonEnvelope.builder("durable-after-poll").build())); + assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); + assertEquals(1L, countRegularFiles(queueDirectory), + "a poll-created backend state must persist before reporting acceptance"); + releaseBackendCallback.countDown(); + } + } finally { + releaseBackendCallback.countDown(); + } + } + private static long countRegularFiles(Path root) throws Exception { try (java.util.stream.Stream paths = java.nio.file.Files.walk(root)) { return paths.filter(path -> java.nio.file.Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS)).count(); From 63e8f9ddd9bc538392f6fcf6563cacffb69eb4b4 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 17:59:43 -0600 Subject: [PATCH 009/136] fix(http): validate enrollment codes in Control --- .../transport/HttpBackendProxyTransport.java | 2 +- .../control/BackendConfigurationService.java | 9 +++---- .../BackendConfigurationServiceTest.java | 27 +++++++++++++++++++ 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index ac077a074..81f520ed7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -96,7 +96,7 @@ public void validate() { validateConfiguration(directory, serverId, plugin.getBungeeSettings().getHttpConnectionCode()); } - private static void validateConfiguration(Path directory, String serverId, String configuredCode) { + public static void validateConfiguration(Path directory, String serverId, String configuredCode) { enrollmentCode(directory, serverId, configuredCode); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 3309b7b93..3c84caeb0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -28,7 +28,7 @@ import org.bukkit.configuration.InvalidConfigurationException; import org.bukkit.configuration.file.YamlConfiguration; -import com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore; +import com.bencodez.votingplugin.backendproxy.transport.HttpBackendProxyTransport; import com.bencodez.votingplugin.proxy.BungeeMethod; import com.bencodez.votingplugin.util.DurableFiles; @@ -595,11 +595,8 @@ private void validateProxyMethod(BungeeMethod method, YamlConfiguration settings break; case HTTP: String connectionCode = settings.getString("HTTP.ConnectionCode", ""); - if (connectionCode == null || connectionCode.isBlank()) { - if (!HttpClientCredentialStore.hasEnrolledProfile(dataDirectory.resolve("http"))) { - throw new IllegalArgumentException("HTTP.ConnectionCode must be set for initial enrollment"); - } - } + try { HttpBackendProxyTransport.validateConfiguration(dataDirectory.resolve("http"), server, connectionCode); } + catch (IllegalStateException invalid) { throw new IllegalArgumentException(invalid.getMessage(), invalid); } break; case MYSQL: try { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index 23a0dc8ec..a2d111189 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -710,6 +710,33 @@ class BackendConfigurationServiceTest { assertDoesNotThrow(() -> service.previewQuickSetup("proxy-method", Map.of("method", "HTTP"))); } + @Test void httpMethodPreflightRejectsInvalidExpiredAndWrongServerConnectionCodes() throws Exception { + Path settings = directory.resolve("BungeeSettings.yml"); + Files.writeString(settings, "UseBungeecord: true\nServer: lobby-1\nBungeeMethod: PLUGINMESSAGING\n" + + "PluginMessageChannel: vp:vp\nHTTP:\n ConnectionCode: malformed\n"); + BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); + assertThrows(IllegalArgumentException.class, + () -> service.previewQuickSetup("proxy-method", Map.of("method", "HTTP"))); + + com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity identity = + com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.loadOrCreate(directory.resolve("code-proxy"), "localhost"); + com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode expired = + new com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode("lobby-1", + java.net.URI.create("https://localhost:1297/"), identity.serverCertificatePin(), + identity.caCertificatePin(), java.time.Instant.now().minusSeconds(1), "A".repeat(43)); + Files.writeString(settings, Files.readString(settings).replace("malformed", expired.encode())); + assertThrows(IllegalArgumentException.class, + () -> service.previewQuickSetup("proxy-method", Map.of("method", "HTTP"))); + + com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode wrongServer = + new com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode("survival", + expired.endpoint(), expired.serverCertificatePin(), expired.caCertificatePin(), + java.time.Instant.now().plusSeconds(60), "B".repeat(43)); + Files.writeString(settings, Files.readString(settings).replace(expired.encode(), wrongServer.encode())); + assertThrows(IllegalArgumentException.class, + () -> service.previewQuickSetup("proxy-method", Map.of("method", "HTTP"))); + } + @Test void proxyMethodSwitchPreflightsRequiredBackendSettings() throws Exception { Path settings = directory.resolve("BungeeSettings.yml"); Files.writeString(settings, "UseBungeecord: true\nServer: lobby\nBungeeMethod: PLUGINMESSAGING\n" From 5b666dc501aad8675cf6a061effa455a5f84f433 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 19:18:53 -0600 Subject: [PATCH 010/136] fix(http): serialize callbacks and renew transport CA --- .../http/HttpBackendTransportConnector.java | 13 ++-- .../http/HttpClientCredentialStore.java | 7 +- .../backendproxy/http/HttpTlsIdentity.java | 67 ++++++++++++++----- .../http/HttpTransportRuntimeTest.java | 31 +++++++++ .../http/HttpTransportSecurityTest.java | 34 ++++++++++ 5 files changed, 130 insertions(+), 22 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 1c318f6b6..8733c44a5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -32,7 +32,7 @@ /** Backend-side, persistent HTTP/1.1 long-poll connector. */ public final class HttpBackendTransportConnector implements AutoCloseable { public static final Duration CLIENT_TIMEOUT = Duration.ofSeconds(35); - private final HttpClientCredentialStore.HttpClientProfile profile; + private volatile HttpClientCredentialStore.HttpClientProfile profile; private final String serverId; private final Consumer onEnvelope; private volatile HttpClient client; @@ -80,7 +80,10 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil this.credentialDirectory = credentialDirectory; client = client(profile, credential); transportEndpoint = profile.endpoint().resolve("v1/transport"); - callbackExecutor = executor("VotingPlugin-HTTP-callback", 2, 128); + // GlobalMessageHandler routes mutate backend vote state and must observe the + // wire order. One bounded lane preserves batch ordering without running work on + // the long-poll thread. + callbackExecutor = executor("VotingPlugin-HTTP-callback", 1, 128); } /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ @@ -221,9 +224,11 @@ private void maybeRenewCredential() { HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(directory, issued); HttpClientCredentialStore.ClientCredential replacement = staged.credential(); - if (!matchesCredential(profile, replacement)) throw new IllegalArgumentException("Renewed HTTP certificate is invalid"); - HttpClient replacementClient = client(profile, replacement); + HttpClientCredentialStore.HttpClientProfile replacementProfile = staged.profile(); + if (!matchesCredential(replacementProfile, replacement)) throw new IllegalArgumentException("Renewed HTTP certificate is invalid"); + HttpClient replacementClient = client(replacementProfile, replacement); HttpClientCredentialStore.activateReplacement(directory, staged); + profile = replacementProfile; client = replacementClient; credential = replacement; } catch (Exception ignored) { /* The active generation is unchanged; retry on the bounded schedule. */ } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index f0d7fe681..32b80b8f6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -107,11 +107,14 @@ private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClie setOwnerOnlyDirectory(generation); try { save(generation, issued); + ClientCredential replacement = loadCredential(generation); + profile = new HttpClientProfile(profile.serverId(), profile.endpoint(), profile.serverCertificatePin(), + HttpTransportSecrets.certificatePin(replacement.caCertificate())); writeProfile(generation, profile); if (connectionCodeDigest != null) writePrivate(safe(generation.resolve(CONNECTION_CODE_DIGEST_FILE)), connectionCodeDigest.getBytes(StandardCharsets.US_ASCII)); EnrolledClient enrolled = loadEnrolled(generation); - return new StagedCredential(name, enrolled.credential()); + return new StagedCredential(name, enrolled.credential(), enrolled.profile()); } catch (Exception failure) { try { Files.deleteIfExists(generation.resolve(BUNDLE_FILE)); Files.deleteIfExists(generation.resolve(PASSWORD_FILE)); Files.deleteIfExists(generation.resolve(PROFILE_FILE)); Files.deleteIfExists(generation.resolve(CONNECTION_CODE_DIGEST_FILE)); @@ -135,7 +138,7 @@ static void activateReplacement(Path directory, StagedCredential staged) throws writePrivate(safe(directory.resolve(CURRENT_FILE)), staged.name().getBytes(StandardCharsets.US_ASCII)); } - static record StagedCredential(String name, ClientCredential credential) { } + static record StagedCredential(String name, ClientCredential credential, HttpClientProfile profile) { } public static HttpClientProfile loadProfile(Path directory) throws IOException { return loadProfileFile(activeDirectory(directory)); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java index eb40dda21..41522eacc 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -56,21 +56,24 @@ public final class HttpTlsIdentity { private static final String PASSWORD_FILE = "http-transport-password"; private static final char[] EMPTY_PASSWORD = new char[0]; static final Duration RENEW_BEFORE = Duration.ofDays(30); + static final Duration CA_RENEW_BEFORE = Duration.ofDays(365); private final PrivateKey caKey; - private final X509Certificate caCertificate; + private volatile X509Certificate caCertificate; private volatile PrivateKey serverKey; private volatile X509Certificate serverCertificate; private final char[] password; + private final Path caFile; private final Path serverFile; private final String advertisedHost; private HttpTlsIdentity(PrivateKey caKey, X509Certificate caCertificate, PrivateKey serverKey, - X509Certificate serverCertificate, char[] password, Path serverFile, String advertisedHost) { + X509Certificate serverCertificate, char[] password, Path caFile, Path serverFile, String advertisedHost) { this.caKey = caKey; this.caCertificate = caCertificate; this.serverKey = serverKey; this.serverCertificate = serverCertificate; this.password = password.clone(); + this.caFile = caFile; this.serverFile = serverFile; this.advertisedHost = advertisedHost; } @@ -102,7 +105,18 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock X509Certificate serverCertificate = (X509Certificate) server.getCertificate("server"); if (caKey == null || caCertificate == null || serverKey == null || serverCertificate == null) throw new IOException("HTTP TLS identity files are invalid"); - if (!hasServerName(serverCertificate, advertisedHost) || needsRenewal(serverCertificate, clock)) { + boolean caRenewed = needsCaRenewal(caCertificate, clock); + if (caRenewed) { + ensureBouncyCastle(); + KeyPair caPair = new KeyPair(caCertificate.getPublicKey(), caKey); + caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, + CertificateRole.CA, null, clock.instant()); + ca = KeyStore.getInstance("PKCS12"); + ca.load(null, EMPTY_PASSWORD); + ca.setKeyEntry("ca", caKey, password, new Certificate[] { caCertificate }); + writeStore(caFile, ca, password); + } + if (caRenewed || !hasServerName(serverCertificate, advertisedHost) || needsRenewal(serverCertificate, clock)) { ensureBouncyCastle(); KeyPair serverPair = keyPair(); serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, caKey, @@ -113,7 +127,8 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock server.setKeyEntry("server", serverKey, password, new Certificate[] { serverCertificate, caCertificate }); writeStore(serverFile, server, password); } - return new HttpTlsIdentity(caKey, caCertificate, serverKey, serverCertificate, password, serverFile, advertisedHost); + return new HttpTlsIdentity(caKey, caCertificate, serverKey, serverCertificate, password, caFile, serverFile, + advertisedHost); } finally { Arrays.fill(password, '\0'); } } ensureBouncyCastle(); @@ -137,16 +152,15 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock try { writePrivate(passwordFile, passwordBytes); } finally { Arrays.fill(passwordBytes, (byte) 0); } return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password, - serverFile, advertisedHost); + caFile, serverFile, advertisedHost); } finally { Arrays.fill(password, '\0'); } } public String serverCertificatePin() { - try { renewServerCertificateIfNeeded(); } - catch (Exception failure) { throw new IllegalStateException("Could not renew HTTP server certificate", failure); } + refreshIdentity(); return HttpTransportSecrets.certificatePin(serverCertificate); } - public String caCertificatePin() { return HttpTransportSecrets.certificatePin(caCertificate); } + public String caCertificatePin() { refreshIdentity(); return HttpTransportSecrets.certificatePin(caCertificate); } public X509Certificate caCertificate() { return caCertificate; } public X509Certificate serverCertificate() { return serverCertificate; } @@ -156,7 +170,7 @@ public String serverCertificatePin() { * additionally validate the certificate's persisted backend binding in the HTTP handler. */ public SSLContext serverContext() throws Exception { - renewServerCertificateIfNeeded(); + renewIdentityIfNeeded(); SSLContext context = SSLContext.getInstance("TLS"); context.init(new KeyManager[] { new RotatingServerKeyManager() }, trustManagers(caCertificate), null); return context; @@ -171,16 +185,34 @@ static TrustManager[] trustManagers(X509Certificate caCertificate) throws Except return factory.getTrustManagers(); } - private synchronized void renewServerCertificateIfNeeded() throws Exception { - if (!needsRenewal(serverCertificate, Clock.systemUTC())) return; + private void refreshIdentity() { + try { renewIdentityIfNeeded(); } + catch (Exception failure) { throw new IllegalStateException("Could not renew HTTP TLS identity", failure); } + } + + private synchronized void renewIdentityIfNeeded() throws Exception { + Clock clock = Clock.systemUTC(); + boolean renewCa = needsCaRenewal(caCertificate, clock); + if (!renewCa && !needsRenewal(serverCertificate, clock)) return; ensureBouncyCastle(); + X509Certificate replacementCa = caCertificate; + if (renewCa) { + KeyPair caPair = new KeyPair(caCertificate.getPublicKey(), caKey); + replacementCa = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, + CertificateRole.CA, null, clock.instant()); + KeyStore caStore = KeyStore.getInstance("PKCS12"); + caStore.load(null, EMPTY_PASSWORD); + caStore.setKeyEntry("ca", caKey, password, new Certificate[] { replacementCa }); + writeStore(caFile, caStore, password); + } KeyPair pair = keyPair(); - X509Certificate replacement = certificate("CN=" + certificateName(advertisedHost), pair, caCertificate, caKey, - CertificateRole.SERVER, advertisedHost, Instant.now()); + X509Certificate replacement = certificate("CN=" + certificateName(advertisedHost), pair, replacementCa, caKey, + CertificateRole.SERVER, advertisedHost, clock.instant()); KeyStore store = KeyStore.getInstance("PKCS12"); store.load(null, EMPTY_PASSWORD); - store.setKeyEntry("server", pair.getPrivate(), password, new Certificate[] { replacement, caCertificate }); + store.setKeyEntry("server", pair.getPrivate(), password, new Certificate[] { replacement, replacementCa }); writeStore(serverFile, store, password); + caCertificate = replacementCa; serverKey = pair.getPrivate(); serverCertificate = replacement; } @@ -288,6 +320,10 @@ static boolean needsRenewal(X509Certificate certificate, Clock clock) { return certificate == null || !certificate.getNotAfter().toInstant().isAfter(clock.instant().plus(RENEW_BEFORE)); } + static boolean needsCaRenewal(X509Certificate certificate, Clock clock) { + return certificate == null || !certificate.getNotAfter().toInstant().isAfter(clock.instant().plus(CA_RENEW_BEFORE)); + } + private static void ensureBouncyCastle() { if (Security.getProvider("BC") == null) Security.addProvider(new BouncyCastleProvider()); } @@ -363,8 +399,7 @@ private static void setOwnerOnly(Path path) throws IOException { private final class RotatingServerKeyManager extends X509ExtendedKeyManager { private static final String ALIAS = "server"; private void refresh() { - try { renewServerCertificateIfNeeded(); } - catch (Exception failure) { throw new IllegalStateException("Could not renew HTTP server certificate", failure); } + refreshIdentity(); } private String alias(String keyType) { refresh(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index ac95b1f8b..7363f5334 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -327,6 +327,37 @@ void backendReAcknowledgesLostAckDuplicateWithoutSecondCallback() throws Excepti } } + @Test + void backendCallbacksAreSerializedInDeliveryOrder() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ordered-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("ordered-client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1), secondStarted = new CountDownLatch(1); + java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("ordered-client")), envelope -> { + String marker = String.valueOf(envelope.getFields().get("marker")); + order.add(marker); + if ("first".equals(marker)) { + firstStarted.countDown(); + try { releaseFirst.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } else secondStarted.countDown(); + })) { + connector.dispatch(new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").put("marker", "first").build())); + connector.dispatch(new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").put("marker", "second").build())); + assertTrue(firstStarted.await(2, TimeUnit.SECONDS)); + assertFalse(secondStarted.await(150, TimeUnit.MILLISECONDS)); + releaseFirst.countDown(); + assertTrue(secondStarted.await(2, TimeUnit.SECONDS)); + assertEquals(java.util.List.of("first", "second"), order); + } finally { releaseFirst.countDown(); } + } + @Test void backendDedupWindowContinuesAfterCapacity() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index d85574604..1d5a57b72 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -139,6 +139,40 @@ void serverLeafRotatesInsideRenewalWindowAndPreservesAuthority() throws Exceptio assertFalse(HttpTlsIdentity.needsRenewal(renewed.serverCertificate(), Clock.fixed(now, ZoneOffset.UTC))); } + @Test + void runningPrivateCaRollsOverBeforeExpiryWithoutStrandingExistingClients() throws Exception { + Instant now = Instant.now(); + Clock originalClock = Clock.fixed(now.minus(Duration.ofDays(9 * 365L + 30L)), ZoneOffset.UTC); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(directory, "localhost", originalClock); + X509Certificate originalCa = original.caCertificate(); + HttpTlsIdentity.IssuedClientCertificate existingClient = original.issueClientCertificate("lobby-1", now); + Path client = directory.resolve("client"); + HttpConnectionCode oldCode = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + HttpTransportSecrets.certificatePin(original.serverCertificate()), HttpTransportSecrets.certificatePin(originalCa), + now.plusSeconds(60), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, oldCode, existingClient); + + String renewedPin = original.caCertificatePin(); + assertNotEquals(HttpTransportSecrets.certificatePin(originalCa), renewedPin); + assertEquals(originalCa.getPublicKey(), original.caCertificate().getPublicKey(), + "certificate rollover keeps the private authority key so old and new trust anchors overlap"); + assertFalse(HttpTlsIdentity.needsCaRenewal(original.caCertificate(), Clock.fixed(now, ZoneOffset.UTC))); + assertTrue(original.validClientCertificate("lobby-1", existingClient.certificate())); + + X509TrustManager oldClientTrust = Arrays.stream(HttpTlsIdentity.trustManagers(originalCa)) + .filter(X509TrustManager.class::isInstance).map(X509TrustManager.class::cast).findFirst().orElseThrow(); + assertDoesNotThrow(() -> oldClientTrust.checkServerTrusted( + new X509Certificate[] { original.serverCertificate(), original.caCertificate() }, "ECDHE_ECDSA")); + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(client, + original.issueClientCertificate("lobby-1", now)); + assertEquals(HttpTransportSecrets.certificatePin(originalCa), HttpClientCredentialStore.loadProfile(client).caCertificatePin()); + assertEquals(renewedPin, staged.profile().caCertificatePin()); + HttpClientCredentialStore.activateReplacement(client, staged); + assertEquals(renewedPin, HttpClientCredentialStore.loadProfile(client).caCertificatePin()); + assertEquals(renewedPin, HttpTlsIdentity.loadOrCreate(directory, "localhost").caCertificatePin(), + "live CA rollover must survive restart"); + } + @Test void activeTlsContextRotatesServerLeafInsideRenewalWindow() throws Exception { Instant now = Instant.now(); From dbf416f19bfd017d81cd82be11289706181e602b Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 19:25:00 -0600 Subject: [PATCH 011/136] fix(http): verify backend presence against proxy route --- .../votingplugin/proxy/VotingPluginProxy.java | 31 ++++++-- .../http/HttpTransportSecurityTest.java | 1 + .../tests/VotingPluginProxyTest.java | 71 +++++++++++++++++++ .../tests/VotingPluginProxyTestImpl.java | 5 ++ 4 files changed, 103 insertions(+), 5 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index b7dc11054..428f3e151 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -2710,11 +2710,7 @@ private void startHttpTransport() { httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); httpTransportServer = new HttpProxyTransportServer( new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, - httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), received -> { - GlobalMessageProxyHandler handler = globalMessageProxyHandler; - if (handler == null) throw new IllegalStateException("HTTP message router is not ready"); - handler.onMessage(received.envelope()); - }); + httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope); httpTransportServer.start(); logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" + httpTransportServer.port() + "; use /votingpluginbungee httpcode for each backend"); @@ -2724,6 +2720,31 @@ private void startHttpTransport() { } } + /** Keeps the authenticated mTLS backend identity attached to security-sensitive proxy routing. */ + protected void handleHttpTransportEnvelope(HttpProxyTransportServer.ReceivedEnvelope received) { + if (!isAuthenticatedHttpEnvelopeAllowed(received)) { + debug("Ignored HTTP envelope whose player-presence claim did not match its authenticated backend"); + return; + } + GlobalMessageProxyHandler handler = globalMessageProxyHandler; + if (handler == null) throw new IllegalStateException("HTTP message router is not ready"); + handler.onMessage(received.envelope()); + } + + private boolean isAuthenticatedHttpEnvelopeAllowed(HttpProxyTransportServer.ReceivedEnvelope received) { + if (received == null || received.envelope() == null || received.serverId() == null) return false; + String stampedServer = received.envelope().getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + if (!received.serverId().equalsIgnoreCase(stampedServer)) return false; + if (!VotingPluginWire.SUB_LOGIN.equals(received.envelope().getSubChannel())) return true; + VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(received.envelope()); + boolean modern = event.connectionId != null || event.backendIncarnationId != null + || event.backendStartedAt != 0L || event.presenceTimestamp != 0L; + if (!modern || isDedicatedVotingProxyEnabled()) return true; + // A player-facing proxy has a stronger authority than any backend: its live + // player connection supplies both the current route and (in online mode) UUID. + return isLegacyLoginDestinationAuthoritative(event.player, event.uuid, received.serverId()); + } + private synchronized void closeHttpTransport() { HttpProxyTransportServer transport = httpTransportServer; httpTransportServer = null; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 1d5a57b72..bcb614215 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -10,6 +10,7 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; +import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; import java.time.Instant; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 4f8b711fb..6b3fb402d 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -16,6 +16,7 @@ import org.mockito.MockitoAnnotations; import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalDataHandlerProxy; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; import com.bencodez.votingplugin.proxy.BungeeMethod; import com.bencodez.votingplugin.proxy.OfflineBungeeVote; @@ -350,6 +351,76 @@ void pluginMessagingIgnoresExtendedPresenceLogin() { assertEquals(0, spyProxy.getBackendPlayerPresenceTracker().getOnlinePlayerCount()); } + @Test + void httpModernPresenceRejectsAuthenticatedBackendThatDoesNotMatchProxyRoute() { + String uuid = java.util.UUID.randomUUID().toString(); + Mockito.when(votingPluginProxy.getConfig().getOnlineMode()).thenReturn(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(uuid).when(spyProxy).getUUID("Player"); + var handler = Mockito.mock(com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler.class); + spyProxy.setGlobalMessageProxyHandlerForTest(handler); + JsonEnvelope envelope = VotingPluginWire.login("Player", uuid, "Server2", java.util.UUID.randomUUID(), + java.util.UUID.randomUUID(), 1000L, 1100L); + + spyProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer.ReceivedEnvelope( + "Server2", java.util.UUID.randomUUID().toString(), envelope)); + + verify(handler, never()).onMessage(Mockito.any()); + assertEquals(0, spyProxy.getBackendPlayerPresenceTracker().getOnlinePlayerCount()); + } + + @Test + void httpModernPresenceAcceptsAuthenticatedBackendMatchingProxyRouteAndUuid() { + String uuid = java.util.UUID.randomUUID().toString(); + Mockito.when(votingPluginProxy.getConfig().getOnlineMode()).thenReturn(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(uuid).when(spyProxy).getUUID("Player"); + var handler = Mockito.mock(com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler.class); + spyProxy.setGlobalMessageProxyHandlerForTest(handler); + JsonEnvelope envelope = VotingPluginWire.login("Player", uuid, "Server1", java.util.UUID.randomUUID(), + java.util.UUID.randomUUID(), 1000L, 1100L); + + spyProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer.ReceivedEnvelope( + "Server1", java.util.UUID.randomUUID().toString(), envelope)); + + verify(handler).onMessage(envelope); + } + + @Test + void httpModernPresenceRejectsUuidThatDoesNotMatchProxyPlayer() { + String authoritativeUuid = java.util.UUID.randomUUID().toString(); + String claimedUuid = java.util.UUID.randomUUID().toString(); + Mockito.when(votingPluginProxy.getConfig().getOnlineMode()).thenReturn(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(authoritativeUuid).when(spyProxy).getUUID("Player"); + var handler = Mockito.mock(com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler.class); + spyProxy.setGlobalMessageProxyHandlerForTest(handler); + JsonEnvelope envelope = VotingPluginWire.login("Player", claimedUuid, "Server1", java.util.UUID.randomUUID(), + java.util.UUID.randomUUID(), 1000L, 1100L); + + spyProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer.ReceivedEnvelope( + "Server1", java.util.UUID.randomUUID().toString(), envelope)); + + verify(handler, never()).onMessage(Mockito.any()); + assertEquals(0, spyProxy.getBackendPlayerPresenceTracker().getOnlinePlayerCount()); + } + + @Test + void httpEnvelopeRejectsServerFieldThatDoesNotMatchAuthenticatedBackend() { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + var handler = Mockito.mock(com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler.class); + votingPluginProxy.setGlobalMessageProxyHandlerForTest(handler); + JsonEnvelope envelope = JsonEnvelope.builder("vote").put(VotingPluginWire.K_SERVER, "Server2").build(); + + votingPluginProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer.ReceivedEnvelope( + "Server1", java.util.UUID.randomUUID().toString(), envelope)); + + verify(handler, never()).onMessage(Mockito.any()); + } + @Test void dedicatedVotingProxyRoutesUsingConfirmedBackendPresence() { Mockito.when(votingPluginProxy.getConfig().getDedicatedVotingProxy()).thenReturn(true); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index dd22c32ed..a2d18b1c1 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -17,6 +17,7 @@ import com.bencodez.votingplugin.proxy.OfflineBungeeVote; import com.bencodez.votingplugin.proxy.VotingPluginProxy; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; +import com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; public class VotingPluginProxyTestImpl extends VotingPluginProxy { @@ -243,6 +244,10 @@ public void handleLoginMessageForTest(JsonEnvelope envelope) { handleLoginMessage(envelope); } + public void handleHttpTransportEnvelopeForTest(HttpProxyTransportServer.ReceivedEnvelope received) { + handleHttpTransportEnvelope(received); + } + public void handleStatusOkayForTest(JsonEnvelope envelope) { handleStatusOkay(envelope); } From ac6a947ae5d684d3bf46d49a074fca814e2befba Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 19:27:49 -0600 Subject: [PATCH 012/136] fix(http): preserve proxy callback order --- .../http/HttpProxyTransportServer.java | 4 ++- .../http/HttpTransportRuntimeTest.java | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index c77b8a3f0..bdbbaaccd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -95,7 +95,9 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity // Long polls are blocking by design. Capacity is bounded by admission, while enough workers // remain available for all admitted polls plus setup requests. listenerExecutor = executor("VotingPlugin-HTTP-listener", 72, 72); - handlerExecutor = executor("VotingPlugin-HTTP-handler", 4, 128); + // The proxy router mutates shared presence, vote, and reward state. A separate + // bounded FIFO lane keeps wire order without blocking long-poll workers. + handlerExecutor = executor("VotingPlugin-HTTP-handler", 1, 128); server.setExecutor(listenerExecutor); server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); server.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 7363f5334..399f9be88 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -358,6 +358,40 @@ void backendCallbacksAreSerializedInDeliveryOrder() throws Exception { } finally { releaseFirst.countDown(); } } + @Test + void proxyCallbacksAreSerializedInDeliveryOrder() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ordered-proxy-server"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("ordered-authority")); + CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1), secondStarted = new CountDownLatch(1); + java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, received -> { + String marker = String.valueOf(received.envelope().getFields().get("marker")); + order.add(marker); + if ("first".equals(marker)) { + firstStarted.countDown(); + try { releaseFirst.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } else secondStarted.countDown(); + })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, "lobby-1", + directory.resolve("ordered-proxy-client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", credential, + ignored -> { })) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("x").put("marker", "first").build())); + assertTrue(connector.send(JsonEnvelope.builder("x").put("marker", "second").build())); + assertTrue(firstStarted.await(3, TimeUnit.SECONDS)); + assertFalse(secondStarted.await(150, TimeUnit.MILLISECONDS)); + releaseFirst.countDown(); + assertTrue(secondStarted.await(3, TimeUnit.SECONDS)); + assertEquals(java.util.List.of("first", "second"), order); + } + } finally { releaseFirst.countDown(); } + } + @Test void backendDedupWindowContinuesAfterCapacity() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); From f104e61d83a94febf73a4cb3059804d34ed05b8f Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:04:12 -0600 Subject: [PATCH 013/136] fix(http): preserve callback order and crash dedup --- .../http/HttpBackendTransportConnector.java | 84 +++++++++++-- .../http/HttpInboundDeliveryStore.java | 119 ++++++++++++++++++ .../http/HttpProxyTransportServer.java | 18 +-- .../http/HttpTransportRuntimeTest.java | 112 ++++++++++++++++- .../http/HttpTransportSecurityTest.java | 19 +++ docs/http-transport.md | 6 +- 6 files changed, 335 insertions(+), 23 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 8733c44a5..6747e1990 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -1,6 +1,7 @@ package com.bencodez.votingplugin.backendproxy.http; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -32,12 +33,14 @@ /** Backend-side, persistent HTTP/1.1 long-poll connector. */ public final class HttpBackendTransportConnector implements AutoCloseable { public static final Duration CLIENT_TIMEOUT = Duration.ofSeconds(35); + static final int CALLBACK_QUEUE_CAPACITY = 128; private volatile HttpClientCredentialStore.HttpClientProfile profile; private final String serverId; private final Consumer onEnvelope; private volatile HttpClient client; private volatile HttpClientCredentialStore.ClientCredential credential; private final Path credentialDirectory; + private final HttpInboundDeliveryStore inboundDeliveries; private final URI transportEndpoint; private final ThreadPoolExecutor callbackExecutor; private final AtomicBoolean running = new AtomicBoolean(); @@ -51,17 +54,19 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private long sequence; private volatile long nextRenewalCheckNanos; - public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, + /** In-memory test constructor; production transport must use a directory-backed constructor. */ + HttpBackendTransportConnector(HttpConnectionCode code, String serverId, HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { this(profile(code, serverId), credential, onEnvelope); } - /** Starts normal transport from the non-secret profile persisted by enrollment. */ - public HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope) throws Exception { + /** In-memory test constructor; production transport must use a directory-backed constructor. */ + HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope) throws Exception { this(enrolled, onEnvelope, null); } - public HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, + /** In-memory test constructor; production transport must use a directory-backed constructor. */ + HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { this(profile, credential, onEnvelope, null); } @@ -78,12 +83,17 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil this.profile = profile; this.serverId = profile.serverId(); this.onEnvelope = onEnvelope; this.credential = credential; this.credentialDirectory = credentialDirectory; + inboundDeliveries = credentialDirectory == null ? null : new HttpInboundDeliveryStore(credentialDirectory); + if (inboundDeliveries != null) for (String id : inboundDeliveries.snapshot()) { + received.add(id); + queueAck(id); + } client = client(profile, credential); transportEndpoint = profile.endpoint().resolve("v1/transport"); // GlobalMessageHandler routes mutate backend vote state and must observe the // wire order. One bounded lane preserves batch ordering without running work on - // the long-poll thread. - callbackExecutor = executor("VotingPlugin-HTTP-callback", 1, 128); + // the long-poll thread; bounded admission below backpressures this poller. + callbackExecutor = executor("VotingPlugin-HTTP-callback", 1, CALLBACK_QUEUE_CAPACITY); } /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ @@ -135,9 +145,10 @@ public boolean send(JsonEnvelope envelope) { /** A synchronous single poll, useful for lifecycle-controlled integrations and tests. */ public synchronized boolean pollOnce() { if (!running.get()) return false; + List acks = List.of(); boolean acknowledgementsConfirmed = false; try { maybeRenewCredential(); - List acks; List messages; long requestSequence; + List messages; long requestSequence; synchronized (state) { acks = first(acknowledgements); requestSequence = sequence++; messages = HttpTransportProtocol.fittingMessages(serverId, session, requestSequence, acks, outgoing.values()); @@ -149,10 +160,13 @@ public synchronized boolean pollOnce() { if (response.statusCode() != 200 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) return false; HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(response.body()); if (!serverId.equals(packet.server()) || !session.equals(packet.session()) || packet.sequence() != requestSequence) return false; + confirmAcknowledgements(acks); + acknowledgementsConfirmed = true; synchronized (state) { for (String ack : packet.acks()) outgoing.remove(ack); } for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); return true; - } catch (Exception failure) { return false; } + } catch (Exception failure) { return false; + } finally { if (!acknowledgementsConfirmed) requeueAcknowledgements(acks); } } @Override public void close() { running.getAndSet(false); @@ -169,7 +183,9 @@ List accept(List synchronized (state) { List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : deliveries) { - if (received.contains(delivery.id())) { queueAck(delivery.id()); continue; } + if (received.contains(delivery.id()) || inboundDeliveries != null && inboundDeliveries.contains(delivery.id())) { + received.add(delivery.id()); queueAck(delivery.id()); continue; + } if (!processing.contains(delivery.id())) { processing.add(delivery.id()); accepted.add(delivery); } @@ -178,20 +194,66 @@ List accept(List } } void dispatch(HttpTransportProtocol.Delivery delivery) { - try { callbackExecutor.execute(() -> { boolean success = false; try { onEnvelope.accept(delivery.envelope()); success = true; } catch (RuntimeException ignored) { } + Runnable callback = () -> { + boolean success = false, reserved = false; + try { + if (inboundDeliveries != null) { inboundDeliveries.reserve(delivery.id()); reserved = true; } + onEnvelope.accept(delivery.envelope()); + success = true; + } catch (IOException persistenceFailure) { + // Never run a side-effecting callback without first publishing its replay fence. + } catch (RuntimeException callbackFailure) { + if (reserved) try { inboundDeliveries.remove(delivery.id()); } + catch (IOException removalFailure) { + // A callback can fail after partial effects. If the fence cannot be removed, + // fail closed as processed rather than risk awarding again on a retry. + success = true; + } + } completeIncoming(delivery.id(), success); - }); } catch (RejectedExecutionException rejected) { completeIncoming(delivery.id(), false); } + }; + if (!executeOrdered(callbackExecutor, callback)) completeIncoming(delivery.id(), false); } void completeIncoming(String id, boolean success) { synchronized (state) { processing.remove(id); if (success) { received.add(id); while (received.size() > HttpTransportProtocol.MAX_QUEUE) received.remove(received.iterator().next()); queueAck(id); } } } private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } + private void requeueAcknowledgements(List ids) { synchronized (state) { + for (int index = ids.size() - 1; index >= 0; index--) { + String id = ids.get(index); + if (!acknowledgements.contains(id)) { + while (acknowledgements.size() >= HttpTransportProtocol.MAX_QUEUE) acknowledgements.removeLast(); + acknowledgements.addFirst(id); + } + } + } } + private void confirmAcknowledgements(Collection ids) { + for (String id : ids) { + if (inboundDeliveries != null) try { inboundDeliveries.remove(id); } + catch (IOException cleanupFailure) { continue; } + synchronized (state) { received.remove(id); } + } + } int queuedOutgoing() { synchronized (state) { return outgoing.size(); } } List drainAcknowledgements() { synchronized (state) { return drain(acknowledgements); } } private static List first(Collection values) { List output = new java.util.ArrayList<>(); for (T value : values) { output.add(value); if (output.size() == HttpTransportProtocol.MAX_BATCH) break; } return output; } private static List drain(ArrayDeque values) { List output = new java.util.ArrayList<>(); while (!values.isEmpty() && output.size() < HttpTransportProtocol.MAX_BATCH) output.add(values.remove()); return output; } private static ThreadPoolExecutor executor(String name, int threads, int queue) { ThreadFactory factory = task -> { Thread thread = new Thread(task, name); thread.setDaemon(true); return thread; }; return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); } + static boolean executeOrdered(ThreadPoolExecutor executor, Runnable task) { + try { executor.execute(task); return true; } + catch (RejectedExecutionException fullOrClosed) { + if (executor.isShutdown()) return false; + try { + while (!executor.isShutdown()) { + if (!executor.getQueue().offer(task, 100L, TimeUnit.MILLISECONDS)) continue; + if (executor.isShutdown() && executor.remove(task)) return false; + return true; + } + } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + return false; + } + } private static HttpClientCredentialStore.HttpClientProfile profile(HttpConnectionCode code, String serverId) { if (code == null || serverId == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); if (!code.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalArgumentException("HTTP connection code belongs to a different backend"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java new file mode 100644 index 000000000..9a03c87c6 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java @@ -0,0 +1,119 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import com.bencodez.votingplugin.util.DurableFiles; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.util.EnumSet; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.UUID; + +/** + * Crash-durable fence for proxy deliveries that may already have caused backend side effects. + * A delivery is reserved before its callback runs and removed only after the proxy confirms its ACK. + */ +final class HttpInboundDeliveryStore { + private static final String DIRECTORY = "http-transport-inbound-deliveries"; + private static final String SUFFIX = ".seen"; + private static final int MAX_ENTRIES = HttpTransportProtocol.MAX_QUEUE; + private final Path root; + private final Set entries = new LinkedHashSet<>(); + + HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { + Path credentials = credentialDirectory.toAbsolutePath().normalize(); + if (Files.isSymbolicLink(credentials) || !Files.isDirectory(credentials, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential directory is unsafe"); + ownerOnlyDirectory(credentials); + root = credentials.resolve(DIRECTORY).normalize(); + if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); + Files.createDirectories(root); + if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery directory is unsafe"); + ownerOnlyDirectory(root); + load(); + } + + synchronized boolean contains(String id) { return entries.contains(canonical(id)); } + + synchronized void reserve(String id) throws IOException { + id = canonical(id); + if (entries.contains(id)) return; + if (entries.size() >= MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence is full"); + requireRoot(); + Path target = root.resolve(id + SUFFIX); + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery fence is inconsistent"); + Path temporary = Files.createTempFile(root, ".pending-", ".tmp"); + try { + ownerOnlyFile(temporary); + Files.writeString(temporary, id, StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } + ownerOnlyFile(target); + DurableFiles.forceDirectory(root); + entries.add(id); + } finally { Files.deleteIfExists(temporary); } + } + + synchronized boolean remove(String id) throws IOException { + id = canonical(id); + if (!entries.contains(id)) return true; + requireRoot(); + DurableFiles.deleteIfExists(root.resolve(id + SUFFIX)); + entries.remove(id); + return true; + } + + synchronized Set snapshot() { return Set.copyOf(entries); } + + private void load() throws IOException { + try (DirectoryStream files = Files.newDirectoryStream(root)) { + for (Path file : files) { + String name = file.getFileName().toString(); + if (name.startsWith(".pending-") && name.endsWith(".tmp") && !Files.isSymbolicLink(file) + && Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) { + DurableFiles.deleteIfExists(file); + continue; + } + if (Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) + || !name.endsWith(SUFFIX) || Files.size(file) > 64L) + throw new IOException("HTTP inbound delivery fence contains an invalid entry"); + String id = canonical(name.substring(0, name.length() - SUFFIX.length())); + if (!name.equals(id + SUFFIX) || !Files.readString(file, StandardCharsets.US_ASCII).equals(id) + || !entries.add(id)) + throw new IOException("HTTP inbound delivery fence entry is invalid"); + if (entries.size() > MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence exceeds its bound"); + } + } + } + + private static String canonical(String id) { + if (id == null) throw new IllegalArgumentException("HTTP delivery id is invalid"); + String canonical = UUID.fromString(id).toString(); + if (!canonical.equals(id)) throw new IllegalArgumentException("HTTP delivery id is not canonical"); + return canonical; + } + + private void requireRoot() throws IOException { + if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery directory is unsafe"); + } + + private static void ownerOnlyFile(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { } + } + private static void ownerOnlyDirectory(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } + catch (UnsupportedOperationException ignored) { } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index bdbbaaccd..2d3825a44 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -32,7 +32,6 @@ import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; @@ -97,7 +96,7 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity listenerExecutor = executor("VotingPlugin-HTTP-listener", 72, 72); // The proxy router mutates shared presence, vote, and reward state. A separate // bounded FIFO lane keeps wire order without blocking long-poll workers. - handlerExecutor = executor("VotingPlugin-HTTP-handler", 1, 128); + handlerExecutor = executor("VotingPlugin-HTTP-handler", 1, HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY); server.setExecutor(listenerExecutor); server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); server.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); @@ -178,7 +177,7 @@ private void transport(HttpsExchange exchange) throws IOException { } finally { admission.release(); } } - private void handlePacket(HttpTransportProtocol.Packet packet, BackendState backend) { + private void handlePacket(HttpTransportProtocol.Packet packet, BackendState backend) throws IOException { List accepted; synchronized (backend) { if (!backend.allowRequest()) throw new IllegalArgumentException("transport rate limited"); @@ -188,12 +187,14 @@ private void handlePacket(HttpTransportProtocol.Packet packet, BackendState back for (HttpTransportProtocol.Delivery delivery : accepted) dispatch(packet.server(), backend, delivery); } private void dispatch(String serverId, BackendState backend, HttpTransportProtocol.Delivery delivery) { - try { handlerExecutor.execute(() -> { + Runnable callback = () -> { boolean success = false; try { onEnvelope.accept(new ReceivedEnvelope(serverId, delivery.id(), normalizeBackendIdentity(serverId, delivery.envelope()))); success = true; } catch (RuntimeException ignored) { } synchronized (backend) { backend.completeIncoming(delivery.id(), success); } - }); } catch (RejectedExecutionException rejected) { synchronized (backend) { backend.completeIncoming(delivery.id(), false); } } + }; + if (!HttpBackendTransportConnector.executeOrdered(handlerExecutor, callback)) + synchronized (backend) { backend.completeIncoming(delivery.id(), false); } } private static JsonEnvelope normalizeBackendIdentity(String serverId, JsonEnvelope envelope) { // The authenticated TLS identity is authoritative; never forward a forged `server` field. @@ -269,11 +270,12 @@ synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { catch (IOException failure) { return false; } outgoing.put(delivery.id(), delivery); signal(); return true; } - private void acknowledge(Collection acks) { + private void acknowledge(Collection acks) throws IOException { for (String id : acks) { if (!outgoing.containsKey(id)) continue; - if (durableOutgoing != null) try { durableOutgoing.remove(serverId, id); } - catch (IOException failure) { continue; } + // A 200 response is the backend's proof that its durable replay fence may + // be deleted. Never return success while the proxy delivery still exists. + if (durableOutgoing != null) durableOutgoing.remove(serverId, id); outgoing.remove(id); delivered.remove(id); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 399f9be88..1a1250b61 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -33,8 +33,8 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti message -> { received.set(message); proxyReceived.countDown(); })) { server.start(); HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); - HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, "lobby-1", directory.resolve("client")); - try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", credential, + HttpBackendTransportConnector.enroll(code, "lobby-1", directory.resolve("client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(directory.resolve("client"), envelope -> backendReceived.countDown())) { connector.start(); assertTrue(connector.send(JsonEnvelope.builder("to-proxy").put("server", "forged").build())); @@ -46,6 +46,10 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti assertEquals(0, connector.queuedOutgoing(), "proxy ACK must remove the exact outbound delivery ID"); assertTrue(server.send("lobby-1", JsonEnvelope.builder("to-backend").build())); assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); + Path inboundFence = directory.resolve("client").resolve("http-transport-inbound-deliveries"); + long fenceDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (countRegularFiles(inboundFence) != 0L && System.nanoTime() < fenceDeadline) Thread.sleep(20); + assertEquals(0L, countRegularFiles(inboundFence), "a confirmed ACK must remove the backend replay fence"); } } } @@ -358,6 +362,110 @@ void backendCallbacksAreSerializedInDeliveryOrder() throws Exception { } finally { releaseFirst.countDown(); } } + @Test + void backendCallbackQueueBackpressuresWithoutBreakingFifo() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("backpressure-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("backpressure-client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1); + int deliveries = HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY + 2; + CountDownLatch completed = new CountDownLatch(deliveries), overflowSubmitted = new CountDownLatch(1); + java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("backpressure-client")), envelope -> { + int marker = Integer.parseInt(envelope.getFields().get("marker")); + order.add(marker); + if (marker == 0) { + firstStarted.countDown(); + try { releaseFirst.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } + completed.countDown(); + })) { + connector.dispatch(delivery(0)); + assertTrue(firstStarted.await(2, TimeUnit.SECONDS)); + for (int marker = 1; marker <= HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY; marker++) + connector.dispatch(delivery(marker)); + Thread overflow = new Thread(() -> { + connector.dispatch(delivery(deliveries - 1)); + overflowSubmitted.countDown(); + }, "HTTP-overflow-submitter"); + overflow.start(); + assertFalse(overflowSubmitted.await(150, TimeUnit.MILLISECONDS), "a full ordered lane must backpressure its producer"); + releaseFirst.countDown(); + assertTrue(overflowSubmitted.await(2, TimeUnit.SECONDS)); + assertTrue(completed.await(5, TimeUnit.SECONDS)); + assertEquals(java.util.stream.IntStream.range(0, deliveries).boxed().toList(), order); + } finally { releaseFirst.countDown(); } + } + + @Test + void durableBackendFencePreventsCallbackReplayAfterRestartBeforeAck() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("fence-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "A".repeat(43)); + Path clientDirectory = directory.resolve("fence-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + java.util.concurrent.atomic.AtomicInteger callbacks = new java.util.concurrent.atomic.AtomicInteger(); + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("vote").build()); + try (HttpBackendTransportConnector first = new HttpBackendTransportConnector(clientDirectory, + ignored -> callbacks.incrementAndGet())) { + first.dispatch(delivery); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (callbacks.get() != 1 && System.nanoTime() < deadline) Thread.sleep(5); + assertEquals(1, callbacks.get()); + } + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, + ignored -> callbacks.incrementAndGet())) { + assertEquals(java.util.List.of(id), restarted.drainAcknowledgements(), + "restart must retain and acknowledge the pre-callback delivery fence"); + assertTrue(restarted.accept(java.util.List.of(delivery)).isEmpty()); + assertEquals(1, callbacks.get(), "a durable proxy replay must not award twice"); + } + } + + @Test + void failedBackendCallbackRemovesFenceAndRemainsRetryable() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-fence-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "B".repeat(43)); + Path clientDirectory = directory.resolve("retry-fence-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + java.util.concurrent.atomic.AtomicInteger attempts = new java.util.concurrent.atomic.AtomicInteger(); + CountDownLatch failed = new CountDownLatch(1), succeeded = new CountDownLatch(1); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("vote").build()); + try (HttpBackendTransportConnector first = new HttpBackendTransportConnector(clientDirectory, ignored -> { + attempts.incrementAndGet(); failed.countDown(); throw new IllegalStateException("retry"); + })) { + first.dispatch(delivery); + assertTrue(failed.await(2, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (countRegularFiles(clientDirectory.resolve("http-transport-inbound-deliveries")) != 0L + && System.nanoTime() < deadline) Thread.sleep(5); + assertTrue(first.drainAcknowledgements().isEmpty()); + } + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, ignored -> { + attempts.incrementAndGet(); succeeded.countDown(); + })) { + java.util.List accepted = restarted.accept(java.util.List.of(delivery)); + assertEquals(1, accepted.size()); + restarted.dispatch(accepted.get(0)); + assertTrue(succeeded.await(2, TimeUnit.SECONDS)); + assertEquals(2, attempts.get()); + } + } + + private static HttpTransportProtocol.Delivery delivery(int marker) { + return new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").put("marker", marker).build()); + } + @Test void proxyCallbacksAreSerializedInDeliveryOrder() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ordered-proxy-server"), "localhost"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index bcb614215..7f02d54ad 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -52,6 +52,25 @@ void expiredCodesAreNotActive() { assertThrows(IllegalArgumentException.class, () -> code.requireActive(Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC))); } + @Test + void inboundDeliveryFenceRejectsCorruptionAndPathReplacement() throws Exception { + Path corruptCredentials = directory.resolve("corrupt-client"); + Path corruptFence = corruptCredentials.resolve("http-transport-inbound-deliveries"); + Files.createDirectories(corruptFence); + Files.writeString(corruptFence.resolve("not-a-delivery.seen"), "not-a-delivery"); + assertThrows(java.io.IOException.class, () -> new HttpInboundDeliveryStore(corruptCredentials)); + + Path replacedCredentials = directory.resolve("replaced-client"); + Files.createDirectories(replacedCredentials); + HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(replacedCredentials); + Path fence = replacedCredentials.resolve("http-transport-inbound-deliveries"); + Path outside = directory.resolve("outside-fence"); + Files.createDirectory(outside); + Files.delete(fence); + Files.createSymbolicLink(fence, outside); + assertThrows(java.io.IOException.class, () -> store.reserve(java.util.UUID.randomUUID().toString())); + } + @Test void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { HttpTlsIdentity created = HttpTlsIdentity.loadOrCreate(directory, "localhost"); diff --git a/docs/http-transport.md b/docs/http-transport.md index 80bd99391..78587e159 100644 --- a/docs/http-transport.md +++ b/docs/http-transport.md @@ -40,7 +40,9 @@ Connection codes expire after 15 minutes and can be used only once. Treat a fres - Enrollment tokens are 256-bit random capabilities, single-use, short-lived, and retained by the proxy only as hashes. - Credentials, keys, pins, and revocation state are written atomically with owner-only permissions where the operating system supports them. - Request bodies, queues, batches, worker pools, concurrent requests, per-backend polling, and request rates are bounded. Exact paths, methods, and JSON content types are enforced. -- Delivery IDs, acknowledgements, and duplicate suppression provide in-process at-least-once delivery. Vote caching remains responsible for application-level restart durability. +- Proxy-to-backend messages stay in an owner-only durable proxy queue until the backend acknowledgement is durably applied. Before a backend callback can change vote or reward state, the backend fsyncs that delivery ID in its credential directory; a restart therefore acknowledges a replay without awarding it twice. The fence is deleted only after a matching authenticated HTTP 200 confirms the proxy removed the queued message. +- This crash fence deliberately favors preventing duplicate rewards: a process or host failure in the very small interval after the fence is persisted but before the callback begins can suppress that callback after restart. Fully transactional exactly-once reward execution is not possible because arbitrary reward commands and external plugin effects cannot share a transaction with the transport journal. +- Backend-to-proxy messages retain the existing bounded in-process retry semantics. Vote caching remains responsible for application-level durability in that direction. The listener must terminate TLS itself because client-certificate authentication is part of the protocol. Do not put an HTTP TLS-terminating reverse proxy or CDN in front of it. A TCP/L4 proxy that passes TLS through unchanged is suitable. Internet-facing installations should also use the host firewall or provider firewall for volumetric denial-of-service protection; an application cannot fully absorb a link or TCP flood. @@ -48,4 +50,4 @@ If a backend host or its private credential is compromised, run `/votingpluginbu ## Performance -The connector reuses HTTP/1.1 TLS connections, batches messages and acknowledgements, and performs all network and certificate work off the game thread. A two-second bounded long poll avoids busy polling while limiting the worst-case delay for a backend message queued just after an idle request began. All queues are bounded to prevent traffic bursts from causing unbounded memory growth. +The connector reuses HTTP/1.1 TLS connections, batches messages and acknowledgements, and performs all network, certificate, and delivery-journal work off the game thread. A two-second bounded long poll avoids busy polling while limiting the worst-case delay for a backend message queued just after an idle request began. Callback lanes are single-worker FIFO queues; when their fixed capacity is reached, bounded listener/poller workers apply backpressure instead of dropping an older callback and admitting newer work out of order. All queues and admission pools remain bounded to prevent traffic bursts from causing unbounded memory growth. From 13fd2b7b09194c9cc7240215b1d6407c3b5640c5 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:07:03 -0600 Subject: [PATCH 014/136] fix(http): normalize corrupt fence errors --- .../backendproxy/http/HttpInboundDeliveryStore.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java index 9a03c87c6..9fce3ebd0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java @@ -86,7 +86,11 @@ private void load() throws IOException { if (Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) || !name.endsWith(SUFFIX) || Files.size(file) > 64L) throw new IOException("HTTP inbound delivery fence contains an invalid entry"); - String id = canonical(name.substring(0, name.length() - SUFFIX.length())); + String id; + try { id = canonical(name.substring(0, name.length() - SUFFIX.length())); } + catch (IllegalArgumentException invalid) { + throw new IOException("HTTP inbound delivery fence entry is invalid", invalid); + } if (!name.equals(id + SUFFIX) || !Files.readString(file, StandardCharsets.US_ASCII).equals(id) || !entries.add(id)) throw new IOException("HTTP inbound delivery fence entry is invalid"); From 71f5ca4d6daa2c1fbfc287a08ae2a70016ece6cc Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:17:28 -0600 Subject: [PATCH 015/136] fix(http): revoke pending enrollment codes --- .../http/HttpEnrollmentAuthority.java | 2 ++ .../http/HttpTransportSecurityTest.java | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java index 9ac9b0c51..719028472 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java @@ -118,6 +118,8 @@ public synchronized HttpTlsIdentity.IssuedClientCertificate renew(String serverI public synchronized void revoke(String serverId) { try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } catch (IllegalArgumentException invalid) { return; } + final String revokedServer = serverId; + enrollments.entrySet().removeIf(entry -> revokedServer.equals(entry.getValue().serverId())); ClientBinding binding = bindings.get(serverId); if (binding != null) { bindings.put(serverId, new ClientBinding(binding.certificatePin(), binding.pendingCertificatePin(), true)); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 7f02d54ad..455ecae7d 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -132,6 +132,28 @@ void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { assertFalse(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); } + @Test + void revocationInvalidatesEveryPendingCodeForTheBackend() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("revoke-pending"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, + Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC)); + URI endpoint = URI.create("https://localhost:8443/"); + HttpConnectionCode beforeEnrollment = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + authority.revoke("LOBBY-1"); + assertThrows(IllegalArgumentException.class, + () -> authority.enroll("lobby-1", beforeEnrollment.enrollmentToken())); + + HttpConnectionCode active = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + authority.enroll("lobby-1", active.enrollmentToken()); + HttpConnectionCode firstPending = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + HttpConnectionCode secondPending = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + authority.revoke("lobby-1"); + assertThrows(IllegalArgumentException.class, + () -> authority.enroll("lobby-1", firstPending.enrollmentToken())); + assertThrows(IllegalArgumentException.class, + () -> authority.enroll("lobby-1", secondPending.enrollmentToken())); + } + @Test void renewalKeepsOldCredentialUntilReplacementAuthenticates() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); From 6216a04cc35d39c7e7473d357e72a4c118a5d99b Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:30:04 -0600 Subject: [PATCH 016/136] fix(http): distinguish incomplete inbound deliveries --- .../http/HttpBackendTransportConnector.java | 34 ++++--- .../http/HttpInboundDeliveryStore.java | 92 +++++++++++++------ .../http/HttpTransportRuntimeTest.java | 60 ++++++++++-- docs/http-transport.md | 4 +- 4 files changed, 137 insertions(+), 53 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 6747e1990..69aa4a6f8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -84,9 +84,11 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil this.credential = credential; this.credentialDirectory = credentialDirectory; inboundDeliveries = credentialDirectory == null ? null : new HttpInboundDeliveryStore(credentialDirectory); - if (inboundDeliveries != null) for (String id : inboundDeliveries.snapshot()) { - received.add(id); - queueAck(id); + if (inboundDeliveries != null) for (var entry : inboundDeliveries.snapshot().entrySet()) { + if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { + received.add(entry.getKey()); + queueAck(entry.getKey()); + } } client = client(profile, credential); transportEndpoint = profile.endpoint().resolve("v1/transport"); @@ -183,9 +185,14 @@ List accept(List synchronized (state) { List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : deliveries) { - if (received.contains(delivery.id()) || inboundDeliveries != null && inboundDeliveries.contains(delivery.id())) { + HttpInboundDeliveryStore.State persisted = inboundDeliveries == null ? null : inboundDeliveries.state(delivery.id()); + if (received.contains(delivery.id()) || persisted == HttpInboundDeliveryStore.State.COMPLETED) { received.add(delivery.id()); queueAck(delivery.id()); continue; } + // A callback that was running when the process stopped may already have + // produced external side effects. Keep the proxy copy without replaying or + // acknowledging it; arbitrary plugin callbacks cannot share this journal. + if (persisted == HttpInboundDeliveryStore.State.RUNNING) continue; if (!processing.contains(delivery.id())) { processing.add(delivery.id()); accepted.add(delivery); } @@ -195,20 +202,21 @@ List accept(List } void dispatch(HttpTransportProtocol.Delivery delivery) { Runnable callback = () -> { - boolean success = false, reserved = false; + boolean success = false; try { - if (inboundDeliveries != null) { inboundDeliveries.reserve(delivery.id()); reserved = true; } + if (inboundDeliveries != null) { + if (inboundDeliveries.state(delivery.id()) == null) inboundDeliveries.reserve(delivery.id()); + inboundDeliveries.markRunning(delivery.id()); + } onEnvelope.accept(delivery.envelope()); + if (inboundDeliveries != null) inboundDeliveries.markCompleted(delivery.id()); success = true; } catch (IOException persistenceFailure) { - // Never run a side-effecting callback without first publishing its replay fence. + // Never run before RUNNING is durable and never acknowledge until + // COMPLETED is durable. An uncertain transition stays fail-closed. } catch (RuntimeException callbackFailure) { - if (reserved) try { inboundDeliveries.remove(delivery.id()); } - catch (IOException removalFailure) { - // A callback can fail after partial effects. If the fence cannot be removed, - // fail closed as processed rather than risk awarding again on a retry. - success = true; - } + // The callback may have failed after partial external effects. Leave RUNNING + // unacknowledged so a restart cannot silently lose or duplicate the delivery. } completeIncoming(delivery.id(), success); }; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java index 9fce3ebd0..960d02ba5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java @@ -11,20 +11,16 @@ import java.nio.file.StandardOpenOption; import java.nio.file.attribute.PosixFilePermission; import java.util.EnumSet; -import java.util.LinkedHashSet; -import java.util.Set; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.UUID; -/** - * Crash-durable fence for proxy deliveries that may already have caused backend side effects. - * A delivery is reserved before its callback runs and removed only after the proxy confirms its ACK. - */ +/** Crash-durable state for proxy deliveries around a non-transactional application callback. */ final class HttpInboundDeliveryStore { private static final String DIRECTORY = "http-transport-inbound-deliveries"; - private static final String SUFFIX = ".seen"; private static final int MAX_ENTRIES = HttpTransportProtocol.MAX_QUEUE; private final Path root; - private final Set entries = new LinkedHashSet<>(); + private final Map entries = new LinkedHashMap<>(); HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { Path credentials = credentialDirectory.toAbsolutePath().normalize(); @@ -34,20 +30,21 @@ final class HttpInboundDeliveryStore { root = credentials.resolve(DIRECTORY).normalize(); if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); Files.createDirectories(root); - if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP inbound delivery directory is unsafe"); + requireRoot(); ownerOnlyDirectory(root); load(); } - synchronized boolean contains(String id) { return entries.contains(canonical(id)); } + synchronized State state(String id) { return entries.get(canonical(id)); } synchronized void reserve(String id) throws IOException { id = canonical(id); - if (entries.contains(id)) return; + State existing = entries.get(id); + if (existing == State.RESERVED) return; + if (existing != null) throw new IOException("HTTP inbound delivery fence is already active"); if (entries.size() >= MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence is full"); requireRoot(); - Path target = root.resolve(id + SUFFIX); + Path target = file(id, State.RESERVED); if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP inbound delivery fence is inconsistent"); Path temporary = Files.createTempFile(root, ".pending-", ".tmp"); @@ -55,24 +52,39 @@ synchronized void reserve(String id) throws IOException { ownerOnlyFile(temporary); Files.writeString(temporary, id, StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); DurableFiles.forceFile(temporary); - try { Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); } - catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } + move(temporary, target); ownerOnlyFile(target); DurableFiles.forceDirectory(root); - entries.add(id); + entries.put(id, State.RESERVED); } finally { Files.deleteIfExists(temporary); } } - synchronized boolean remove(String id) throws IOException { + synchronized void markRunning(String id) throws IOException { transition(id, State.RESERVED, State.RUNNING); } + synchronized void markCompleted(String id) throws IOException { transition(id, State.RUNNING, State.COMPLETED); } + + synchronized void remove(String id) throws IOException { id = canonical(id); - if (!entries.contains(id)) return true; + State state = entries.get(id); + if (state == null) return; requireRoot(); - DurableFiles.deleteIfExists(root.resolve(id + SUFFIX)); + DurableFiles.deleteIfExists(file(id, state)); entries.remove(id); - return true; } - synchronized Set snapshot() { return Set.copyOf(entries); } + synchronized Map snapshot() { return Map.copyOf(entries); } + + private void transition(String id, State expected, State replacement) throws IOException { + id = canonical(id); + if (entries.get(id) != expected) throw new IOException("HTTP inbound delivery fence state is invalid"); + requireRoot(); + Path source = file(id, expected), target = file(id, replacement); + if (Files.isSymbolicLink(source) || !Files.isRegularFile(source, LinkOption.NOFOLLOW_LINKS) + || Files.exists(target, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery fence state is unsafe"); + move(source, target); + DurableFiles.forceDirectory(root); + entries.put(id, replacement); + } private void load() throws IOException { try (DirectoryStream files = Files.newDirectoryStream(root)) { @@ -83,34 +95,48 @@ private void load() throws IOException { DurableFiles.deleteIfExists(file); continue; } - if (Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) - || !name.endsWith(SUFFIX) || Files.size(file) > 64L) + State state = State.fromFileName(name); + if (state == null || Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) + || Files.size(file) > 64L) throw new IOException("HTTP inbound delivery fence contains an invalid entry"); String id; - try { id = canonical(name.substring(0, name.length() - SUFFIX.length())); } + try { id = canonical(name.substring(0, name.length() - state.suffix.length())); } catch (IllegalArgumentException invalid) { throw new IOException("HTTP inbound delivery fence entry is invalid", invalid); } - if (!name.equals(id + SUFFIX) || !Files.readString(file, StandardCharsets.US_ASCII).equals(id) - || !entries.add(id)) + if (!name.equals(id + state.suffix) || !Files.readString(file, StandardCharsets.US_ASCII).equals(id)) throw new IOException("HTTP inbound delivery fence entry is invalid"); + State existing = entries.get(id); + if (existing == null) entries.put(id, state); + else { + // A provider without atomic moves may expose both names after an + // interrupted transition. Preserve the furthest fail-closed state: + // RUNNING never replays, and COMPLETED alone may be acknowledged. + State retained = existing.ordinal() >= state.ordinal() ? existing : state; + State obsolete = retained == existing ? state : existing; + DurableFiles.deleteIfExists(file(id, obsolete)); + entries.put(id, retained); + } if (entries.size() > MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence exceeds its bound"); } } } + private Path file(String id, State state) { return root.resolve(id + state.suffix); } + private static void move(Path source, Path target) throws IOException { + try { Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(source, target); } + } private static String canonical(String id) { if (id == null) throw new IllegalArgumentException("HTTP delivery id is invalid"); String canonical = UUID.fromString(id).toString(); if (!canonical.equals(id)) throw new IllegalArgumentException("HTTP delivery id is not canonical"); return canonical; } - private void requireRoot() throws IOException { if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP inbound delivery directory is unsafe"); } - private static void ownerOnlyFile(Path path) throws IOException { try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } catch (UnsupportedOperationException ignored) { } @@ -120,4 +146,14 @@ private static void ownerOnlyDirectory(Path path) throws IOException { PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } catch (UnsupportedOperationException ignored) { } } + + enum State { + RESERVED(".reserved"), RUNNING(".running"), COMPLETED(".completed"); + private final String suffix; + State(String suffix) { this.suffix = suffix; } + private static State fromFileName(String name) { + for (State state : values()) if (name.endsWith(state.suffix)) return state; + return null; + } + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 1a1250b61..b4623ed67 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -429,7 +429,7 @@ void durableBackendFencePreventsCallbackReplayAfterRestartBeforeAck() throws Exc } @Test - void failedBackendCallbackRemovesFenceAndRemainsRetryable() throws Exception { + void failedBackendCallbackRemainsUnacknowledgedAndIsNotReplayed() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-fence-proxy"), "localhost"); HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), @@ -437,7 +437,7 @@ void failedBackendCallbackRemovesFenceAndRemainsRetryable() throws Exception { Path clientDirectory = directory.resolve("retry-fence-client"); HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); java.util.concurrent.atomic.AtomicInteger attempts = new java.util.concurrent.atomic.AtomicInteger(); - CountDownLatch failed = new CountDownLatch(1), succeeded = new CountDownLatch(1); + CountDownLatch failed = new CountDownLatch(1); HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("vote").build()); try (HttpBackendTransportConnector first = new HttpBackendTransportConnector(clientDirectory, ignored -> { @@ -445,22 +445,62 @@ void failedBackendCallbackRemovesFenceAndRemainsRetryable() throws Exception { })) { first.dispatch(delivery); assertTrue(failed.await(2, TimeUnit.SECONDS)); - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); - while (countRegularFiles(clientDirectory.resolve("http-transport-inbound-deliveries")) != 0L - && System.nanoTime() < deadline) Thread.sleep(5); assertTrue(first.drainAcknowledgements().isEmpty()); } - try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, ignored -> { - attempts.incrementAndGet(); succeeded.countDown(); - })) { + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, + ignored -> attempts.incrementAndGet())) { + assertTrue(restarted.drainAcknowledgements().isEmpty()); + java.util.List accepted = restarted.accept(java.util.List.of(delivery)); + assertTrue(accepted.isEmpty(), "an ambiguous callback must not be awarded twice"); + assertEquals(1, attempts.get()); + } + } + + @Test + void reservedButNotStartedDeliveryResumesAfterRestart() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("reserved-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "C".repeat(43)); + Path clientDirectory = directory.resolve("reserved-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + String id = java.util.UUID.randomUUID().toString(); + new HttpInboundDeliveryStore(clientDirectory).reserve(id); + CountDownLatch completed = new CountDownLatch(1); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("vote").build()); + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, ignored -> completed.countDown())) { + assertTrue(restarted.drainAcknowledgements().isEmpty(), "a reservation alone must never be acknowledged"); java.util.List accepted = restarted.accept(java.util.List.of(delivery)); assertEquals(1, accepted.size()); restarted.dispatch(accepted.get(0)); - assertTrue(succeeded.await(2, TimeUnit.SECONDS)); - assertEquals(2, attempts.get()); + assertTrue(completed.await(2, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (new HttpInboundDeliveryStore(clientDirectory).state(id) != HttpInboundDeliveryStore.State.COMPLETED + && System.nanoTime() < deadline) Thread.sleep(5); + assertEquals(java.util.List.of(id), restarted.drainAcknowledgements()); } } + @Test + void interruptedStateRenameRetainsTheFurthestSafeState() throws Exception { + Path clientDirectory = directory.resolve("interrupted-state-client"); + String id = java.util.UUID.randomUUID().toString(); + String completedId = java.util.UUID.randomUUID().toString(); + Path states = clientDirectory.resolve("http-transport-inbound-deliveries"); + Files.createDirectories(states); + Files.writeString(states.resolve(id + ".reserved"), id); + Files.writeString(states.resolve(id + ".running"), id); + Files.writeString(states.resolve(completedId + ".running"), completedId); + Files.writeString(states.resolve(completedId + ".completed"), completedId); + HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(clientDirectory); + assertEquals(HttpInboundDeliveryStore.State.RUNNING, store.state(id)); + assertEquals(HttpInboundDeliveryStore.State.COMPLETED, store.state(completedId)); + assertFalse(Files.exists(states.resolve(id + ".reserved"))); + assertTrue(Files.exists(states.resolve(id + ".running"))); + assertFalse(Files.exists(states.resolve(completedId + ".running"))); + assertTrue(Files.exists(states.resolve(completedId + ".completed"))); + } + private static HttpTransportProtocol.Delivery delivery(int marker) { return new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("x").put("marker", marker).build()); diff --git a/docs/http-transport.md b/docs/http-transport.md index 78587e159..740911e45 100644 --- a/docs/http-transport.md +++ b/docs/http-transport.md @@ -40,8 +40,8 @@ Connection codes expire after 15 minutes and can be used only once. Treat a fres - Enrollment tokens are 256-bit random capabilities, single-use, short-lived, and retained by the proxy only as hashes. - Credentials, keys, pins, and revocation state are written atomically with owner-only permissions where the operating system supports them. - Request bodies, queues, batches, worker pools, concurrent requests, per-backend polling, and request rates are bounded. Exact paths, methods, and JSON content types are enforced. -- Proxy-to-backend messages stay in an owner-only durable proxy queue until the backend acknowledgement is durably applied. Before a backend callback can change vote or reward state, the backend fsyncs that delivery ID in its credential directory; a restart therefore acknowledges a replay without awarding it twice. The fence is deleted only after a matching authenticated HTTP 200 confirms the proxy removed the queued message. -- This crash fence deliberately favors preventing duplicate rewards: a process or host failure in the very small interval after the fence is persisted but before the callback begins can suppress that callback after restart. Fully transactional exactly-once reward execution is not possible because arbitrary reward commands and external plugin effects cannot share a transaction with the transport journal. +- Proxy-to-backend messages stay in an owner-only durable proxy queue until the backend acknowledgement is durably applied. The backend fsyncs three delivery states around each callback: a reserved callback resumes after restart, a completed callback is acknowledged without running twice, and a running callback with an uncertain outcome is neither replayed nor acknowledged. The proxy therefore retains the durable source copy instead of silently losing it. The completed entry is deleted only after a matching authenticated HTTP 200 confirms the proxy removed the queued message. +- A host failure while a callback is running has an inherently ambiguous result because arbitrary reward commands and external plugin effects cannot share a transaction with the transport journal. Such a delivery remains quarantined for operator investigation, avoiding both an automatic duplicate reward and an acknowledgement that could hide a missed reward. - Backend-to-proxy messages retain the existing bounded in-process retry semantics. Vote caching remains responsible for application-level durability in that direction. The listener must terminate TLS itself because client-certificate authentication is part of the protocol. Do not put an HTTP TLS-terminating reverse proxy or CDN in front of it. A TCP/L4 proxy that passes TLS through unchanged is suitable. Internet-facing installations should also use the host firewall or provider firewall for volumetric denial-of-service protection; an application cannot fully absorb a link or TCP flood. From 2a9fcc6d847493e55f3a06ce1c727d5477bc35b5 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:31:45 -0600 Subject: [PATCH 017/136] test(http): import filesystem helper --- .../votingplugin/backendproxy/http/HttpTransportRuntimeTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index b4623ed67..4b97184e7 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -11,6 +11,7 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; From 9eae8b5a35a5486e6eb93f70cbbd9c449a2bfb8b Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:42:05 -0600 Subject: [PATCH 018/136] fix(http): bound responses and persist queue roots --- .../http/HttpBackendTransportConnector.java | 29 +++++++++++++++---- .../http/HttpProxyTransportServer.java | 13 ++++++++- .../http/HttpTransportSecurityTest.java | 9 ++++++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 69aa4a6f8..7359e136d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -2,6 +2,7 @@ import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -121,8 +122,8 @@ public static HttpClientCredentialStore.ClientCredential enroll(HttpConnectionCo .connectTimeout(Duration.ofSeconds(5)).sslContext(HttpPinnedTls.clientContext(code)).build(); HttpRequest request = HttpRequest.newBuilder(code.endpoint().resolve("v1/enroll")).timeout(CLIENT_TIMEOUT) .header("Content-Type", "application/json").header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(payload)).build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); - if (response.statusCode() != 201 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) throw new IllegalArgumentException("Enrollment was rejected"); + LimitedResponse response = sendLimited(client, request); + if (response.statusCode() != 201) throw new IllegalArgumentException("Enrollment was rejected"); HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); HttpClientCredentialStore.saveEnrolled(credentials, code, issued); return HttpClientCredentialStore.load(credentials); } @@ -158,8 +159,8 @@ public synchronized boolean pollOnce() { } HttpRequest request = HttpRequest.newBuilder(transportEndpoint).timeout(CLIENT_TIMEOUT).header("Content-Type", "application/json") .header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(HttpTransportProtocol.request(serverId, session, requestSequence, acks, messages))).build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); - if (response.statusCode() != 200 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) return false; + LimitedResponse response = sendLimited(client, request); + if (response.statusCode() != 200) return false; HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(response.body()); if (!serverId.equals(packet.server()) || !session.equals(packet.session()) || packet.sequence() != requestSequence) return false; confirmAcknowledgements(acks); @@ -289,8 +290,8 @@ private void maybeRenewCredential() { HttpRequest request = HttpRequest.newBuilder(profile.endpoint().resolve("v1/renew")).timeout(CLIENT_TIMEOUT) .header("Content-Type", "application/json").header("Cache-Control", "no-store") .POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); - if (response.statusCode() != 201 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) return; + LimitedResponse response = sendLimited(client, request); + if (response.statusCode() != 201) return; HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(directory, issued); HttpClientCredentialStore.ClientCredential replacement = staged.credential(); @@ -309,6 +310,22 @@ private static HttpClient client(HttpClientCredentialStore.HttpClientProfile pro return HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); } + private static LimitedResponse sendLimited(HttpClient client, HttpRequest request) throws IOException, InterruptedException { + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); + try (InputStream body = response.body()) { + long declaredLength = response.headers().firstValueAsLong("Content-Length").orElse(-1L); + if (declaredLength > HttpTransportProtocol.MAX_BODY_BYTES) + throw new IOException("HTTP transport response exceeds its limit"); + return new LimitedResponse(response.statusCode(), readLimited(body)); + } + } + static byte[] readLimited(InputStream body) throws IOException { + byte[] bytes = body.readNBytes(HttpTransportProtocol.MAX_BODY_BYTES + 1); + if (bytes.length > HttpTransportProtocol.MAX_BODY_BYTES) + throw new IOException("HTTP transport response exceeds its limit"); + return bytes; + } + private record LimitedResponse(int statusCode, byte[] body) { } private static SSLContext clientContext(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { String caPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index 2d3825a44..b9e62536c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -375,7 +375,18 @@ private synchronized Map> load() th private synchronized void persist(String serverId, HttpTransportProtocol.Delivery delivery) throws IOException { Path directory = root.resolve(serverId).normalize(); if (!directory.getParent().equals(root)) throw new IOException("HTTP outgoing queue server is invalid"); - Files.createDirectories(directory); ownerOnlyDirectory(directory); + boolean created = false; + try { Files.createDirectory(directory); created = true; } + catch (java.nio.file.FileAlreadyExistsException existing) { } + try { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue server directory is invalid"); + ownerOnlyDirectory(directory); + } finally { + // The child fsync below cannot make this newly published name durable in + // its parent. Persist the root entry before accepting the first message. + if (created) DurableFiles.forceDirectory(root); + } if (sequence == Long.MAX_VALUE) throw new IOException("HTTP outgoing queue sequence is exhausted"); String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); Path target = directory.resolve(name); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 455ecae7d..094d752c9 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -30,6 +30,15 @@ void connectionCodeRejectsExplicitZeroPort() { @TempDir Path directory; + @Test + void backendResponseReaderRejectsBodiesBeyondTheWireLimit() throws Exception { + byte[] maximum = new byte[HttpTransportProtocol.MAX_BODY_BYTES]; + assertEquals(maximum.length, HttpBackendTransportConnector.readLimited( + new java.io.ByteArrayInputStream(maximum)).length); + assertThrows(java.io.IOException.class, () -> HttpBackendTransportConnector.readLimited( + new java.io.ByteArrayInputStream(new byte[HttpTransportProtocol.MAX_BODY_BYTES + 1]))); + } + @Test void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { HttpConnectionCode original = new HttpConnectionCode("lobby", URI.create("https://Proxy.Example.test:8443/http"), pin('a'), pin('b'), From b13ce28b450e071bcedd72cd6ba6393be994bb69 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:44:33 -0600 Subject: [PATCH 019/136] test(http): await resumed delivery acknowledgement --- .../backendproxy/http/HttpTransportRuntimeTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 4b97184e7..fbd30301f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -476,9 +476,12 @@ void reservedButNotStartedDeliveryResumesAfterRestart() throws Exception { restarted.dispatch(accepted.get(0)); assertTrue(completed.await(2, TimeUnit.SECONDS)); long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); - while (new HttpInboundDeliveryStore(clientDirectory).state(id) != HttpInboundDeliveryStore.State.COMPLETED - && System.nanoTime() < deadline) Thread.sleep(5); - assertEquals(java.util.List.of(id), restarted.drainAcknowledgements()); + java.util.List acknowledgements = java.util.List.of(); + while (acknowledgements.isEmpty() && System.nanoTime() < deadline) { + acknowledgements = restarted.drainAcknowledgements(); + if (acknowledgements.isEmpty()) Thread.sleep(5); + } + assertEquals(java.util.List.of(id), acknowledgements); } } From 0a938cf3ad6002bbdb709ee72a9a4991916d7b17 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:53:47 -0600 Subject: [PATCH 020/136] fix(http): persist transport directory entries --- .../http/HttpClientCredentialStore.java | 3 +++ .../http/HttpInboundDeliveryStore.java | 12 +++++++++--- .../http/HttpProxyTransportServer.java | 14 ++++++++++---- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index 32b80b8f6..b100cc5a7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -114,6 +114,9 @@ private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClie if (connectionCodeDigest != null) writePrivate(safe(generation.resolve(CONNECTION_CODE_DIGEST_FILE)), connectionCodeDigest.getBytes(StandardCharsets.US_ASCII)); EnrolledClient enrolled = loadEnrolled(generation); + // Each file is durable within the generation, but the generation name is + // published by its parent. Persist it before CURRENT can activate it. + DurableFiles.forceDirectory(generations); return new StagedCredential(name, enrolled.credential(), enrolled.profile()); } catch (Exception failure) { try { Files.deleteIfExists(generation.resolve(BUNDLE_FILE)); Files.deleteIfExists(generation.resolve(PASSWORD_FILE)); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java index 960d02ba5..40dcb4101 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java @@ -29,9 +29,15 @@ final class HttpInboundDeliveryStore { ownerOnlyDirectory(credentials); root = credentials.resolve(DIRECTORY).normalize(); if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); - Files.createDirectories(root); - requireRoot(); - ownerOnlyDirectory(root); + boolean created = false; + try { Files.createDirectory(root); created = true; } + catch (java.nio.file.FileAlreadyExistsException existing) { } + try { + requireRoot(); + ownerOnlyDirectory(root); + } finally { + if (created) DurableFiles.forceDirectory(credentials); + } load(); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index b9e62536c..8e1ecc925 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -322,10 +322,16 @@ private static final class DurableOutgoingQueue { private DurableOutgoingQueue(Path root) throws IOException { this.root = root.toAbsolutePath().normalize(); - Files.createDirectories(this.root); - if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP outgoing queue directory is invalid"); - ownerOnlyDirectory(this.root); + boolean created = false; + try { Files.createDirectory(this.root); created = true; } + catch (java.nio.file.FileAlreadyExistsException existing) { } + try { + if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue directory is invalid"); + ownerOnlyDirectory(this.root); + } finally { + if (created) DurableFiles.forceDirectory(this.root.getParent()); + } } private synchronized Map> load() throws IOException { From 5949e113bc7c338c3c6de8c5ce1496503e8f8719 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 21:12:40 -0600 Subject: [PATCH 021/136] fix(http): coordinate transport lifecycle handoffs --- .../votingplugin/VotingPluginMain.java | 7 +- .../backendproxy/BackendProxyHandler.java | 7 +- .../http/HttpBackendTransportConnector.java | 4 + .../http/HttpInboundDeliveryStore.java | 8 + .../BackendProxyTransportManager.java | 7 +- .../transport/HttpBackendProxyTransport.java | 64 +- .../control/BackendControlConnector.java | 3 +- .../votingplugin/proxy/VotingPluginProxy.java | 3682 +---------------- .../http/HttpTransportSecurityTest.java | 13 + .../HttpBackendProxyTransportTest.java | 39 + 10 files changed, 139 insertions(+), 3695 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 06cf263bd..a2d243fa5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1215,6 +1215,11 @@ public String getBackendHostedControlStatus() { /** Recreates proxy transports after Control applies BungeeSettings.yml. */ public synchronized void restartBackendProxyHandler() { + restartBackendProxyHandler(System.nanoTime() + TimeUnit.SECONDS.toNanos(25)); + } + + /** Recreates proxy transports while preserving the caller's end-to-end validation deadline. */ + public synchronized void restartBackendProxyHandler(long validationDeadlineNanos) { BackendProxyHandler previous = backendProxyHandler; if (!bungeeSettings.isUseBungeecoord()) { backendProxyHandler = null; @@ -1229,7 +1234,7 @@ public synchronized void restartBackendProxyHandler() { BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); try { replacement.load(); - replacement.validateTransport(); + replacement.validateTransport(validationDeadlineNanos); if (previous != null) previous.completeRedisHandoff(replacement); } catch (RuntimeException failure) { replacement.close(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 8395c38b4..f485374bb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -110,10 +110,15 @@ public void prepareForReplacement(BungeeMethod replacementMethod) { /** Fails a configuration apply when its selected transport did not initialize. */ public void validateTransport() { + validateTransport(System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(25)); + } + + /** Validates transport startup without extending the caller's existing deadline. */ + public void validateTransport(long deadlineNanos) { if (method == null || globalMessageHandler == null || presenceManager == null) { throw new IllegalStateException("Backend proxy handler initialization failed"); } - transportManager.validate(); + transportManager.validate(deadlineNanos); } /** Completes the no-loss/no-duplicate same-Redis subscriber handoff after validation. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 7359e136d..820d4ab5e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -173,6 +173,10 @@ public synchronized boolean pollOnce() { } @Override public void close() { running.getAndSet(false); + // Revoke this connector's journal writer before a replacement snapshots it. + // In-flight transitions serialize with seal(): either COMPLETED is already + // durable, or the delivery remains durably RUNNING and fail-closed. + if (inboundDeliveries != null) inboundDeliveries.seal(); Thread current = poller; if (current != null) current.interrupt(); callbackExecutor.shutdown(); try { if (!callbackExecutor.awaitTermination(5, TimeUnit.SECONDS)) callbackExecutor.shutdownNow(); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); callbackExecutor.shutdownNow(); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java index 40dcb4101..ba04a31ec 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java @@ -21,6 +21,7 @@ final class HttpInboundDeliveryStore { private static final int MAX_ENTRIES = HttpTransportProtocol.MAX_QUEUE; private final Path root; private final Map entries = new LinkedHashMap<>(); + private boolean sealed; HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { Path credentials = credentialDirectory.toAbsolutePath().normalize(); @@ -44,6 +45,7 @@ final class HttpInboundDeliveryStore { synchronized State state(String id) { return entries.get(canonical(id)); } synchronized void reserve(String id) throws IOException { + requireWritable(); id = canonical(id); State existing = entries.get(id); if (existing == State.RESERVED) return; @@ -67,8 +69,10 @@ synchronized void reserve(String id) throws IOException { synchronized void markRunning(String id) throws IOException { transition(id, State.RESERVED, State.RUNNING); } synchronized void markCompleted(String id) throws IOException { transition(id, State.RUNNING, State.COMPLETED); } + synchronized void seal() { sealed = true; } synchronized void remove(String id) throws IOException { + requireWritable(); id = canonical(id); State state = entries.get(id); if (state == null) return; @@ -80,6 +84,7 @@ synchronized void remove(String id) throws IOException { synchronized Map snapshot() { return Map.copyOf(entries); } private void transition(String id, State expected, State replacement) throws IOException { + requireWritable(); id = canonical(id); if (entries.get(id) != expected) throw new IOException("HTTP inbound delivery fence state is invalid"); requireRoot(); @@ -129,6 +134,9 @@ private void load() throws IOException { } private Path file(String id, State state) { return root.resolve(id + state.suffix); } + private void requireWritable() throws IOException { + if (sealed) throw new IOException("HTTP inbound delivery store ownership has ended"); + } private static void move(Path source, Path target) throws IOException { try { Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(source, target); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 43b54c128..fecb46302 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -70,8 +70,13 @@ public void close() { } public void validate() { + validate(System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(25)); + } + + public void validate(long deadlineNanos) { if (transport == null) throw new IllegalStateException("Backend proxy transport was not initialized"); - transport.validate(); + if (transport instanceof HttpBackendProxyTransport http) http.validate(deadlineNanos); + else transport.validate(); } public void prepareForReplacement() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 81f520ed7..944557439 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -3,6 +3,9 @@ import java.nio.file.Path; import java.time.Clock; import java.util.ArrayDeque; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; @@ -15,13 +18,18 @@ /** Backend adapter for the secure outbound-only HTTP proxy transport. */ public final class HttpBackendProxyTransport implements BackendProxyTransport { private static final int MAX_STARTUP_QUEUE = 1024; + private static final long DEFAULT_STARTUP_VALIDATION_SECONDS = 25L; + private static final ConcurrentHashMap DIRECTORY_OWNERS = new ConcurrentHashMap<>(); private final VotingPluginMain plugin; private final Object lifecycle = new Object(); + private final CountDownLatch startupComplete = new CountDownLatch(1); private final ArrayDeque startupQueue = new ArrayDeque<>(); private volatile HttpBackendTransportConnector connector; private volatile Thread worker; private volatile RuntimeException startupFailure; + private volatile boolean started; private volatile boolean closed; + private Semaphore directoryOwner; private final java.util.concurrent.atomic.AtomicBoolean queueWarning = new java.util.concurrent.atomic.AtomicBoolean(); public HttpBackendProxyTransport(VotingPluginMain plugin) { @@ -34,6 +42,7 @@ public void start(GlobalMessageHandler messageHandler) { String serverId = plugin.getBungeeSettings().getServer(); String connectionCode = plugin.getBungeeSettings().getHttpConnectionCode(); validateConfiguration(directory, serverId, connectionCode); + started = true; worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler), "VotingPlugin-HTTP-Backend-Setup"); worker.setDaemon(true); @@ -42,31 +51,46 @@ public void start(GlobalMessageHandler messageHandler) { private void initialize(Path directory, String serverId, String configuredCode, GlobalMessageHandler messageHandler) { + Path ownerKey = directory.toAbsolutePath().normalize(); + Semaphore owner = DIRECTORY_OWNERS.computeIfAbsent(ownerKey, ignored -> new Semaphore(1)); + boolean acquired = false, installed = false; + HttpBackendTransportConnector replacement = null; try { + owner.acquire(); + acquired = true; + synchronized (lifecycle) { if (closed) return; } HttpConnectionCode code = enrollmentCode(directory, serverId, configuredCode); if (code != null) HttpBackendTransportConnector.enroll(code, serverId, directory); HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); - HttpBackendTransportConnector replacement = new HttpBackendTransportConnector(directory, messageHandler::onMessage); + replacement = new HttpBackendTransportConnector(directory, messageHandler::onMessage); boolean discard = false; synchronized (lifecycle) { if (closed) { discard = true; } else { - connector = replacement; replacement.start(); while (!startupQueue.isEmpty()) { if (!replacement.send(startupQueue.removeFirst())) { throw new IllegalStateException("HTTP startup queue could not be transferred"); } } + connector = replacement; + directoryOwner = owner; + installed = true; } } if (discard) replacement.close(); } catch (Exception failure) { startupFailure = new IllegalStateException("Secure HTTP backend enrollment or connection failed", failure); plugin.getLogger().severe("Secure HTTP backend transport is unavailable; check the connection code and proxy endpoint"); + } finally { + if (!installed) { + if (replacement != null) replacement.close(); + if (acquired) owner.release(); + } + startupComplete.countDown(); } } @@ -86,14 +110,28 @@ public void send(JsonEnvelope envelope) { @Override public void validate() { - RuntimeException failure = startupFailure; - if (failure != null) throw failure; + validate(System.nanoTime() + TimeUnit.SECONDS.toNanos(DEFAULT_STARTUP_VALIDATION_SECONDS)); + } + + void validate(long deadlineNanos) { String serverId = plugin.getBungeeSettings().getServer(); if (serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) { throw new IllegalStateException("HTTP requires a valid unique backend Server name"); } Path directory = plugin.getDataFolder().toPath().resolve("http"); validateConfiguration(directory, serverId, plugin.getBungeeSettings().getHttpConnectionCode()); + if (!started) return; + try { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L || !startupComplete.await(remaining, TimeUnit.NANOSECONDS)) + throw new IllegalStateException("Secure HTTP backend setup did not finish within the validation deadline"); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Secure HTTP backend setup validation was interrupted", interrupted); + } + RuntimeException failure = startupFailure; + if (failure != null) throw failure; + if (closed || connector == null) throw new IllegalStateException("Secure HTTP backend transport did not become ready"); } public static void validateConfiguration(Path directory, String serverId, String configuredCode) { @@ -125,6 +163,7 @@ static HttpConnectionCode enrollmentCode(Path directory, String serverId, String public void close() { Thread setup; HttpBackendTransportConnector active; + Semaphore owner; synchronized (lifecycle) { if (closed) return; closed = true; @@ -133,17 +172,22 @@ public void close() { worker = null; active = connector; connector = null; + owner = directoryOwner; + directoryOwner = null; } + startupComplete.countDown(); if (setup != null) setup.interrupt(); - if (setup == null && active == null) return; - Thread cleanup = new Thread(() -> drain(setup, active), "VotingPlugin-HTTP-Backend-Cleanup"); + if (setup == null && active == null && owner == null) return; + Thread cleanup = new Thread(() -> drain(setup, active, owner), "VotingPlugin-HTTP-Backend-Cleanup"); cleanup.setDaemon(true); cleanup.start(); } - private static void drain(Thread setup, HttpBackendTransportConnector active) { - if (setup != null) try { setup.join(TimeUnit.SECONDS.toMillis(5)); } - catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } - if (active != null) active.close(); + private static void drain(Thread setup, HttpBackendTransportConnector active, Semaphore owner) { + try { + if (setup != null) try { setup.join(TimeUnit.SECONDS.toMillis(5)); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + if (active != null) active.close(); + } finally { if (owner != null) owner.release(); } } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 2b903ccea..00899595c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -112,11 +112,12 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set private void reloadConfiguration(String fileName) throws Exception { Future reload; + long validationDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(29); synchronized (operationLifecycle) { if (closed) throw new IllegalStateException("Bukkit Control connector is stopping"); reload = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { plugin.reloadFromControl(); - if ("BungeeSettings.yml".equals(fileName)) plugin.restartBackendProxyHandler(); + if ("BungeeSettings.yml".equals(fileName)) plugin.restartBackendProxyHandler(validationDeadline); return null; }); activeReload = reload; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 428f3e151..36b185f2d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1,3681 +1 @@ -package com.bencodez.votingplugin.proxy; - -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.sql.SQLException; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; - -import javax.net.ssl.SSLParameters; - -import org.eclipse.paho.client.mqttv3.MqttException; - -import com.bencodez.advancedcore.api.time.TimeType; -import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalDataHandlerProxy; -import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalMySQL; -import com.bencodez.advancedcore.bungeeapi.time.BungeeTimeChecker; -import com.bencodez.simpleapi.encryption.EncryptionHandler; -import com.bencodez.simpleapi.json.JsonParser; -import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; -import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec; -import com.bencodez.simpleapi.servercomm.global.GlobalMessageListener; -import com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler; -import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; -import com.bencodez.simpleapi.servercomm.mqtt.MqttServerComm; -import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; -import com.bencodez.simpleapi.servercomm.redis.RedisHandler; -import com.bencodez.simpleapi.servercomm.redis.RedisListener; -import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; -import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; -import com.bencodez.simpleapi.servercomm.sockets.SocketReceiver; -import com.bencodez.simpleapi.sql.Column; -import com.bencodez.simpleapi.sql.DataType; -import com.bencodez.simpleapi.sql.data.DataValue; -import com.bencodez.simpleapi.sql.data.DataValueBoolean; -import com.bencodez.simpleapi.sql.data.DataValueInt; -import com.bencodez.simpleapi.sql.data.DataValueString; -import com.bencodez.simpleapi.sql.mysql.config.MysqlConfig; -import com.bencodez.votingplugin.backendproxy.http.HttpEnrollmentAuthority; -import com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer; -import com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity; -import com.bencodez.votingplugin.proxy.broadcast.ProxyBroadcastDecider; -import com.bencodez.votingplugin.proxy.cache.IVoteCache; -import com.bencodez.votingplugin.proxy.cache.VoteCacheHandler; -import com.bencodez.votingplugin.proxy.cache.nonvoted.INonVotedPlayersStorage; -import com.bencodez.votingplugin.proxy.cache.nonvoted.NonVotedPlayersCache; -import com.bencodez.votingplugin.proxy.control.ControlConnector; -import com.bencodez.votingplugin.proxy.control.HostedControlManager; -import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyHandler; -import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyMethod; -import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfiguration; -import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfigurationBungee; -import com.bencodez.votingplugin.proxy.presence.BackendPlayerPresenceTracker; -import com.bencodez.votingplugin.proxy.presence.PlayerPresence; -import com.bencodez.votingplugin.timequeue.VoteTimeQueue; -import com.bencodez.votingplugin.topvoter.TopVoter; -import com.bencodez.votingplugin.util.MinecraftUsernameValidator; -import com.bencodez.votingplugin.util.ServiceSiteValidator; -import com.bencodez.votingplugin.votelog.VoteLogMysqlTable; -import com.bencodez.votingplugin.votelog.VoteLogMysqlTable.VoteLogStatus; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; - -import redis.clients.jedis.DefaultJedisClientConfig; -import redis.clients.jedis.HostAndPort; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; - -import lombok.Getter; -import lombok.Setter; - -public abstract class VotingPluginProxy { - private static final long PRESENCE_HANDOFF_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(2); - private static final long PRESENCE_STARTUP_RESYNC_DELAY_SECONDS = 5L; - private static final long PRESENCE_MAINTENANCE_INTERVAL_SECONDS = 30L; - private static final long PRESENCE_BACKEND_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(90); - private static final long CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10); - - @Getter - @Setter - private int votePartyVotes = 0; - - @Getter - @Setter - private int currentVotePartyVotesRequired = 0; - - @Getter - @Setter - private ProxyMysqlUserTable proxyMySQL; - - private EncryptionHandler encryptionHandler; - - private HashMap clientHandles; - - private SocketHandler socketHandler; - private HttpProxyTransportServer httpTransportServer; - private HttpEnrollmentAuthority httpEnrollmentAuthority; - - @Getter - @Setter - private boolean votifierEnabled = true; - - @Getter - private ConcurrentHashMap uuidPlayerNameCache = new ConcurrentHashMap<>(); - - @Getter - @Setter - private GlobalDataHandlerProxy globalDataHandler; - - @Getter - private RedisHandler redisHandler; - private JedisPool redisPublisherPool; - private volatile long redisPublisherRetryAfter; - private boolean timeVoteRetryScheduled; - private boolean timeVoteDeliveryRetryScheduled; - private boolean cachedVoteDeliveryRetryScheduled; - - private boolean enabled; - - @Getter - @Setter - private MultiProxyHandler multiProxyHandler; - - @Getter - private BungeeTimeChecker bungeeTimeChecker; - - @Getter - @Setter - private BungeeMethod method; - - @Getter - private MqttHandler mqttHandler; - - @Getter - private GlobalMessageProxyHandler globalMessageProxyHandler; - - @Getter - @Setter - private MySqlMessenger proxyMysqlMessenger; - - @Getter - private VoteCacheHandler voteCacheHandler; - - @Getter - private NonVotedPlayersCache nonVotedPlayersCache; - - @Getter - private final BackendPlayerPresenceTracker backendPlayerPresenceTracker = new BackendPlayerPresenceTracker(); - private final Map pendingPresenceHandoffs = new HashMap<>(); - private final Set pendingBackendRecoverySnapshots = ConcurrentHashMap.newKeySet(); - private final Map controlEnrollmentNextAllowed = new ConcurrentHashMap<>(); - private final Map pendingCommunicationTests = new ConcurrentHashMap<>(); - private volatile ControlConnector controlConnector; - private volatile HostedControlManager hostedControlManager; - private final Object controlLifecycleLock = new Object(); - private final AtomicLong controlServicesGeneration = new AtomicLong(); - private final ExecutorService controlLifecycleExecutor = Executors.newSingleThreadExecutor(task -> { - Thread thread = new Thread(task, "votingplugin-control-lifecycle"); - thread.setDaemon(true); - return thread; - }); - - public VotingPluginProxy() { - enabled = true; - - bungeeTimeChecker = new BungeeTimeChecker(getConfig().getTimeZone(), getConfig().getTimeHourOffSet(), - getConfig().getTimeWeekOffSet()) { - - @Override - public void debug(String text) { - debug2(text); - } - - @Override - public long getLastUpdated() { - return getVoteCacheLastUpdated(); - } - - @Override - public int getPrevDay() { - return getVoteCachePrevDay(); - } - - @Override - public String getPrevMonth() { - return getVoteCachePrevMonth(); - } - - @Override - public int getPrevWeek() { - return getVoteCachePrevWeek(); - } - - @Override - public void info(String text) { - log(text); - } - - @Override - public boolean isEnabled() { - return enabled; - } - - @Override - public boolean isIgnoreTime() { - return isVoteCacheIgnoreTime(); - } - - @Override - public void setIgnoreTime(boolean ignore) { - setVoteCacheVoteCacheIgnoreTime(ignore); - } - - @Override - public void setLastUpdated() { - setVoteCacheLastUpdated(); - } - - @Override - public void setPrevDay(int day) { - setVoteCachePrevDay(day); - } - - @Override - public void setPrevMonth(String text) { - setVoteCachePrevMonth(text); - } - - @Override - public void setPrevWeek(int week) { - setVoteCachePrevWeek(week); - } - - @Override - public void timeChanged(TimeType type, boolean fake, boolean pre, boolean post) { - if (getConfig().getVoteCacheTime() > 0) { - getVoteCacheHandler().checkVoteCacheTime(getConfig().getVoteCacheTime()); - } - if (!getConfig().getGlobalDataEnabled()) { - warn("Global data not enabled, ignoring time change event"); - return; - } - int delay = 1; - for (String s : getAllAvailableServers()) { - if (getGlobalDataHandler().getGlobalMysql().containsKey(s)) { - String lastOnlineStr = getGlobalDataHandler().getString(s, "LastOnline"); - long lastOnline = 0; - try { - lastOnline = Long.valueOf(lastOnlineStr); - } catch (NumberFormatException e) { - // ignore - } - - if (LocalDateTime.now().atZone(ZoneOffset.UTC).toInstant().toEpochMilli() - lastOnline < 1000 - * 60 * 60 * 12) { - HashMap dataToSet = new HashMap<>(); - dataToSet.put("LastUpdated", new DataValueString( - "" + LocalDateTime.now().atZone(ZoneOffset.UTC).toInstant().toEpochMilli())); - dataToSet.put("FinishedProcessing", new DataValueBoolean(false)); - dataToSet.put(type.toString(), new DataValueBoolean(true)); - getGlobalDataHandler().setData(s, dataToSet); - - globalMessageProxyHandler.sendMessage(s, delay, VotingPluginWire.bungeeTimeChange()); - delay++; - } else { - warn("Server " + s + " hasn't been online recently"); - } - } else { - warn("Server " + s + " global data handler disabled?"); - } - } - globalDataHandler.onTimeChange(type); - } - - @Override - public void warning(String text) { - warn(text); - } - }; - } - - public void onTimeChangedFailed(String srv, TimeType type) { - getGlobalDataHandler().setBoolean(srv, type.toString(), false); - getGlobalDataHandler().setBoolean(srv, "FinishedProcessing", true); - getGlobalDataHandler().setBoolean(srv, "Processing", false); - } - - public void onTimeChangedFinished(TimeType type) { - if (type.equals(TimeType.MONTH)) { - getProxyMySQL().copyColumnData(TopVoter.Monthly.getColumnName(), "LastMonthTotal"); - } - getProxyMySQL().wipeColumnData(TopVoter.of(type).getColumnName(), DataType.INTEGER); - - if (!getConfig().getGlobalDataEnabled()) { - return; - } - for (String s : getAllAvailableServers()) { - getGlobalDataHandler().setBoolean(s, "ForceUpdate", true); - getGlobalMessageProxyHandler().sendMessage(s, 1, VotingPluginWire.bungeeTimeChange()); - } - processQueue(); - } - - /** - * Load MySQL + global data handler. - */ - public void loadMysql(MysqlConfig mysqlConfig, MysqlConfig globalDataMysqlConfig) { - if (mysqlConfig.getHostName().isEmpty() || mysqlConfig.getDatabase().isEmpty()) { - logSevere("MySQL is not configured correctly. " + "Missing host/database. host=" + mysqlConfig.getHostName() - + " db=" + mysqlConfig.getDatabase()); - setProxyMySQL(null); - return; - } - - setProxyMySQL(new ProxyMysqlUserTable("VotingPlugin_Users", mysqlConfig, getConfig().getDebug()) { - - @Override - public void debug(SQLException e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void logSevere(String string) { - VotingPluginProxy.this.logSevere(string); - } - - @Override - public void logInfo(String string) { - VotingPluginProxy.this.logInfo(string); - } - - @Override - public void debug(Throwable t) { - if (getConfig().getDebug()) { - t.printStackTrace(); - } - } - - @Override - public void debug(String str) { - debug2(str); - } - }); - - ArrayList servers = new ArrayList(getAllAvailableServers()); - - if (getConfig().getGlobalDataEnabled()) { - if (getConfig().getGlobalDataUseMainMySQL()) { - setGlobalDataHandler(new GlobalDataHandlerProxy( - new GlobalMySQL("VotingPlugin_GlobalData", getProxyMySQL().getMysql()) { - - @Override - public void debugEx(Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void debugLog(String text) { - debug(text); - } - - @Override - public void info(String text) { - logInfo(text); - } - - @Override - public void logSevere(String text) { - VotingPluginProxy.this.logSevere(text); - } - - @Override - public void warning(String text) { - warn(text); - } - }, servers) { - - @Override - public void onTimeChangedFailed(String srv, TimeType type) { - VotingPluginProxy.this.onTimeChangedFailed(srv, type); - } - - @Override - public void onTimeChangedFinished(TimeType type) { - VotingPluginProxy.this.onTimeChangedFinished(type); - } - }); - } else { - setGlobalDataHandler( - new GlobalDataHandlerProxy(new GlobalMySQL("VotingPlugin_GlobalData", globalDataMysqlConfig) { - - @Override - public void debugEx(Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void debugLog(String text) { - debug(text); - } - - @Override - public void info(String text) { - logInfo(text); - } - - @Override - public void logSevere(String text) { - VotingPluginProxy.this.logSevere(text); - } - - @Override - public void warning(String text) { - warn(text); - } - }, servers) { - - @Override - public void onTimeChangedFailed(String srv, TimeType type) { - VotingPluginProxy.this.onTimeChangedFailed(srv, type); - } - - @Override - public void onTimeChangedFinished(TimeType type) { - VotingPluginProxy.this.onTimeChangedFinished(type); - } - }); - } - - // update global schema columns (unchanged from original) - getGlobalDataHandler().getGlobalMysql().alterColumnType("IgnoreTime", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("MONTH", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("WEEK", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("DAY", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("FinishedProcessing", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("Processing", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("ForceUpdate", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("LastUpdated", "MEDIUMTEXT"); - } - - // column types (unchanged from original) - getProxyMySQL().alterColumnType("TopVoterIgnore", "VARCHAR(5)"); - getProxyMySQL().alterColumnType("CheckWorld", "VARCHAR(5)"); - getProxyMySQL().alterColumnType("Reminded", "VARCHAR(5)"); - getProxyMySQL().alterColumnType("DisableBroadcast", "VARCHAR(5)"); - getProxyMySQL().alterColumnType("LastOnline", "VARCHAR(20)"); - getProxyMySQL().alterColumnType("PlayerName", "VARCHAR(30)"); - getProxyMySQL().alterColumnType("DailyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("WeeklyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("DayVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("BestDayVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("WeekVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("BestWeekVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("VotePartyVotes", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("MonthVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("Points", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("HighestDailyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("AllTimeTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("HighestMonthlyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("MonthTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("HighestWeeklyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("LastMonthTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("LastWeeklyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("LastDailyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("OfflineRewards", "MEDIUMTEXT"); - getProxyMySQL().alterColumnType("DayVoteStreakLastUpdate", "MEDIUMTEXT"); - - if (getConfig().getStoreMonthTotalsWithDate()) { - getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now()), "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now().plusMonths(1)), - "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now().plusMonths(2)), - "INT DEFAULT '0'"); - } - } - - public void addCurrentVotePartyVotes(int amount) { - votePartyVotes += amount; - setVoteCacheVotePartyCurrentVotes(votePartyVotes); - debug("Current vote party total: " + votePartyVotes); - } - - public void addNonVotedPlayer(String uuid, String playerName) { - nonVotedPlayersCache.addPlayer(uuid, playerName); - } - - public void addVoteParty() { - if (getConfig().getVotePartyEnabled()) { - addCurrentVotePartyVotes(1); - checkVoteParty(); - } - } - - public abstract void broadcast(String message); - - private Set sendProxyBroadcast(Set targets, String uuid, String player, String service, long time, - String text, boolean wasOnline) { - Set forwarded = new LinkedHashSet<>(); - for (String targetServer : targets) { - JsonEnvelope envelope = VotingPluginWire.voteBroadcast(uuid, player, service, time, text, wasOnline); - if (sendProxyBroadcastEnvelopeNow(targetServer, envelope)) { - forwarded.add(targetServer); - } - } - return forwarded; - } - - /** - * Sends a standalone proxy broadcast through the selected transport and reports - * whether that transport accepted the message. - * - * @param server target backend server - * @param envelope standalone broadcast envelope - * @return true only when the transport accepted the message - */ - protected boolean sendProxyBroadcastEnvelopeNow(String server, JsonEnvelope envelope) { - switch (method) { - case MQTT: - return sendMqttEnvelopeServer(server, envelope); - case MYSQL: - if (proxyMysqlMessenger == null) { - return false; - } - try { - proxyMysqlMessenger.sendToBackend(server, envelope); - return true; - } catch (SQLException e) { - debug(e.getMessage()); - return false; - } - case PLUGINMESSAGING: - return sendPluginMessageServerNow(server, envelope); - case REDIS: - return sendRedisEnvelopeServer(server, envelope, true); - case SOCKETS: - // Standalone broadcasts use the same initialized client as normal - // envelopes. This preserves the socket connection and its delivery - // acknowledgement instead of creating a second short-lived socket. - return sendSocketEnvelope(server, envelope); - case HTTP: - return sendHttpEnvelope(server, envelope); - default: - return false; - } - } - - /** - * Sends a reward-bearing vote envelope and reports whether the selected - * transport accepted it. Legacy transports retain their existing asynchronous - * semantics; HTTP exposes its bounded-queue result so a vote is never discarded - * when the queue is full. - */ - protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope) { - if (method == BungeeMethod.HTTP) { - return sendHttpEnvelope(server, envelope); - } - GlobalMessageProxyHandler handler = globalMessageProxyHandler; - if (handler == null) { - return false; - } - handler.sendMessage(server, delay, envelope); - return true; - } - - public synchronized void checkCachedVotes(String server) { - int delay = 1; - if (isServerValid(server)) { - if (isSomeoneOnlineServerForVoteRouting(server)) { - if (getVoteCacheHandler().hasVotes(server) && !getConfig().getBlockedServers().contains(server)) { - ArrayList c = getVoteCacheHandler().getVotes(server); - ArrayList removed = new ArrayList<>(); - if (!c.isEmpty()) { - int num = 1; - int numberOfVotes = c.size(); - for (OfflineBungeeVote cache : c) { - if (cache.isDeliveryStateDirty() && !persistServerVoteDelivery(server, cache)) { - continue; - } - if (cache.isProxyBroadcastHandled() && cache.needsBroadcastOn(server)) { - Set forwarded = sendProxyBroadcast(Collections.singleton(server), - cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), - cache.getText(), false); - if (cache.getBroadcastForwardedServers().addAll(forwarded)) { - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - if (!persistServerVoteDelivery(server, cache)) { - continue; - } - } - } - - boolean toSend = true; - if (getConfig().getWaitForUserOnline()) { - if (!isPlayerOnlineForVoteRouting(cache.getPlayerName())) { - toSend = false; - } else if (isPlayerOnlineForVoteRouting(cache.getPlayerName()) - && !getCurrentPlayerServerForVoteRouting(cache.getPlayerName()).equals(server)) { - toSend = false; - } - } - if (toSend) { - boolean broadcastHere = cache.needsBroadcastOn(server); - if (!cache.isProxyBroadcastHandled() && broadcastHere - && getConfig().getProxyBroadcastEnabled()) { - boolean playerOnline = isPlayerOnlineForVoteRouting(cache.getPlayerName()); - String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(cache.getPlayerName()) - : null; - - Set targets = proxyBroadcastDecider.resolveTargets(playerOnline, - playerServer); - broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); - } - - if (!sendVoteEnvelopeAccepted(server, delay, - VotingPluginWire.vote(cache.getPlayerName(), cache.getUuid(), - cache.getService(), cache.getTime(), false, cache.isRealVote(), - cache.getText(), cache.getVoteId(), getConfig().getBungeeManageTotals(), - broadcastHere, num, numberOfVotes))) { - debug("Retaining cached vote because the transport rejected delivery for " + server); - continue; - } - delay++; - num++; - removed.add(cache); - } else { - debug("Not sending vote because user isn't on server " + server + ": " - + cache.toString()); - } - } - getVoteCacheHandler().removeServerVotes(server, removed); - } else { - debug("No cached votes for server: " + server); - } - } else { - debug("No cached votes for server: " + server); - } - } - } else { - debug("Server not valid: " + server); - } - } - - public synchronized void checkOnlineVotes(String player, String uuid, String server) { - int delay = 1; - if (isPlayerOnlineForVoteRouting(player) && getVoteCacheHandler().hasOnlineVotes(uuid)) { - ArrayList c = getVoteCacheHandler().getOnlineVotes(uuid); - if (!c.isEmpty()) { - if (server == null) { - server = getCurrentPlayerServerForVoteRouting(player); - } - if (!getConfig().getBlockedServers().contains(server)) { - int num = 1; - int numberOfVotes = (int) c.stream().filter(vote -> !vote.isRewardDelivered()).count(); - boolean deliveredReward = false; - ArrayList retained = new ArrayList<>(); - for (OfflineBungeeVote cache : c) { - if (cache.isProxyBroadcastHandled()) { - Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); - pendingTargets.removeAll(cache.getBroadcastForwardedServers()); - List blockedServers = getConfig().getBlockedServers(); - if (blockedServers != null) { - pendingTargets.removeAll(blockedServers); - } - cache.getBroadcastForwardedServers().addAll(sendProxyBroadcast(pendingTargets, - cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), - cache.getText(), false)); - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - } - boolean broadcastHere = cache.needsBroadcastOn(server); - if (!cache.isProxyBroadcastHandled() && broadcastHere - && getConfig().getProxyBroadcastEnabled()) { - String playerServer = (server != null) ? server : getCurrentPlayerServerForVoteRouting(player); - - Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); - broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); - } - - if (!cache.isRewardDelivered()) { - if (!sendVoteEnvelopeAccepted(server, delay, - VotingPluginWire.voteOnline(cache.getPlayerName(), cache.getUuid(), cache.getService(), - cache.getTime(), false, cache.isRealVote(), cache.getText(), cache.getVoteId(), - getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes))) { - debug("Retaining online vote because the transport rejected delivery for " + server); - retained.add(cache); - continue; - } - // The normal envelope is also a valid broadcast delivery for the - // current target. Record it so a previously pending standalone - // retry cannot announce the same vote again later. - if (cache.isProxyBroadcastHandled() && broadcastHere) { - cache.getBroadcastForwardedServers().add(server); - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - } - cache.setRewardDelivered(true); - deliveredReward = true; - delay++; - num++; - } - - if (cache.isProxyBroadcastHandled() && !cache.isProxyBroadcastComplete()) { - retained.add(cache); - } - } - getVoteCacheHandler().removeOnlineVotes(uuid); - for (OfflineBungeeVote pending : retained) { - getVoteCacheHandler().addOnlineVote(uuid, pending); - } - - // multiproxy: envelope-only - if (deliveredReward && getConfig().getMultiProxySupport() - && getConfig().getMultiProxyOneGlobalReward()) { - multiProxyHandler.sendClearVote(uuid, player); - } - } - } - } - } - - /** - * Retries voter-keyed standalone broadcasts when any player makes a target - * backend available as a plugin-message carrier. - * - * @param server backend server that gained a carrier - */ - protected synchronized void retryPendingOnlineBroadcasts(String server) { - List blockedServers = getConfig().getBlockedServers(); - if (server == null || (blockedServers != null && blockedServers.contains(server))) { - return; - } - for (String cachedUuid : getVoteCacheHandler().getOnlineVoteUUIDs()) { - for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { - if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { - continue; - } - if (!cache.isProxyBroadcastHandled() || !cache.needsBroadcastOn(server)) { - continue; - } - Set forwarded = sendProxyBroadcast(Collections.singleton(server), cache.getUuid(), - cache.getPlayerName(), cache.getService(), cache.getTime(), cache.getText(), false); - if (cache.getBroadcastForwardedServers().addAll(forwarded)) { - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { - getVoteCacheHandler().removeOnlineVote(cachedUuid, cache); - } else { - persistOnlineVoteDelivery(cachedUuid, cache); - } - } - } - } - } - - protected synchronized void retryPendingTimeBroadcasts(String server) { - List blockedServers = getConfig().getBlockedServers(); - if (server == null || (blockedServers != null && blockedServers.contains(server))) { - return; - } - if (getVoteCacheHandler().getTimeChangeQueue() == null) { - return; - } - for (VoteTimeQueue vote : new ArrayList<>(getVoteCacheHandler().getTimeChangeQueue())) { - if (vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { - continue; - } - if (!vote.isProxyBroadcastHandled() || vote.getUuid().isEmpty() || !vote.getBroadcastTargets().contains(server) - || vote.getBroadcastForwardedServers().contains(server)) { - continue; - } - Set forwarded = sendProxyBroadcast(Collections.singleton(server), vote.getUuid(), vote.getName(), - vote.getService(), vote.getTime(), vote.getTotals(), false); - if (vote.getBroadcastForwardedServers().addAll(forwarded)) { - persistTimeVoteDelivery(vote); - } - } - } - - /** - * Periodically retries every pending voter-keyed standalone broadcast. This is - * required for broker transports whose recovery does not produce a player-login - * carrier event. - */ - public synchronized void retryPendingOnlineBroadcasts() { - for (String cachedUuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { - for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { - if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { - continue; - } - if (!cache.isProxyBroadcastHandled() || cache.isProxyBroadcastComplete()) { - continue; - } - Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); - pendingTargets.removeAll(cache.getBroadcastForwardedServers()); - List blockedServers = getConfig().getBlockedServers(); - if (blockedServers != null) { - pendingTargets.removeAll(blockedServers); - } - Set forwarded = sendProxyBroadcast(pendingTargets, cache.getUuid(), cache.getPlayerName(), - cache.getService(), cache.getTime(), cache.getText(), false); - if (cache.getBroadcastForwardedServers().addAll(forwarded)) { - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { - getVoteCacheHandler().removeOnlineVote(cachedUuid, cache); - } else { - persistOnlineVoteDelivery(cachedUuid, cache); - } - } - } - } - retryPendingTimeBroadcasts(); - } - - public synchronized void retryPendingTimeBroadcasts() { - if (getVoteCacheHandler().getTimeChangeQueue() == null) { - return; - } - for (VoteTimeQueue vote : new ArrayList<>(getVoteCacheHandler().getTimeChangeQueue())) { - if (vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { - continue; - } - if (!vote.isProxyBroadcastHandled() || vote.getUuid().isEmpty()) { - continue; - } - Set pendingTargets = new LinkedHashSet<>(vote.getBroadcastTargets()); - pendingTargets.removeAll(vote.getBroadcastForwardedServers()); - List blockedServers = getConfig().getBlockedServers(); - if (blockedServers != null) { - pendingTargets.removeAll(blockedServers); - } - Set forwarded = sendProxyBroadcast(pendingTargets, vote.getUuid(), vote.getName(), vote.getService(), - vote.getTime(), vote.getTotals(), false); - if (vote.getBroadcastForwardedServers().addAll(forwarded)) { - persistTimeVoteDelivery(vote); - } - } - } - - protected synchronized boolean persistTimeVoteDelivery(VoteTimeQueue vote) { - if (getVoteCacheHandler().updateTimeVote(vote)) { - vote.setDeliveryStateDirty(false); - return true; - } - vote.setDeliveryStateDirty(true); - scheduleTimeVoteDeliveryRetry(); - return false; - } - - private void scheduleTimeVoteDeliveryRetry() { - if (timeVoteDeliveryRetryScheduled || getScheduler() == null) { - return; - } - timeVoteDeliveryRetryScheduled = true; - try { - getScheduler().schedule(() -> { - synchronized (VotingPluginProxy.this) { - timeVoteDeliveryRetryScheduled = false; - } - retryPendingTimeBroadcasts(); - }, 5, TimeUnit.SECONDS); - } catch (RuntimeException e) { - timeVoteDeliveryRetryScheduled = false; - debug("Unable to schedule timed broadcast state retry: " + e.getMessage()); - } - } - - protected synchronized boolean persistServerVoteDelivery(String server, OfflineBungeeVote vote) { - if (getVoteCacheHandler().updateServerVote(server, vote)) { - vote.setDeliveryStateDirty(false); - return true; - } - vote.setDeliveryStateDirty(true); - scheduleCachedVoteDeliveryRetry(); - return false; - } - - protected synchronized boolean persistOnlineVoteDelivery(String uuid, OfflineBungeeVote vote) { - if (getVoteCacheHandler().updateOnlineVote(uuid, vote)) { - vote.setDeliveryStateDirty(false); - return true; - } - vote.setDeliveryStateDirty(true); - scheduleCachedVoteDeliveryRetry(); - return false; - } - - private void scheduleCachedVoteDeliveryRetry() { - if (cachedVoteDeliveryRetryScheduled || getScheduler() == null) { - return; - } - cachedVoteDeliveryRetryScheduled = true; - try { - getScheduler().schedule(() -> { - synchronized (VotingPluginProxy.this) { - cachedVoteDeliveryRetryScheduled = false; - } - retryCachedVoteDeliveryPersistence(); - }, 5, TimeUnit.SECONDS); - } catch (RuntimeException e) { - cachedVoteDeliveryRetryScheduled = false; - debug("Unable to schedule cached broadcast state retry: " + e.getMessage()); - } - } - - private synchronized void retryCachedVoteDeliveryPersistence() { - for (String server : getVoteCacheHandler().getCachedVotesServers()) { - for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getVotes(server))) { - if (vote.isDeliveryStateDirty()) { - persistServerVoteDelivery(server, vote); - } - } - } - for (String uuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { - for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(uuid))) { - if (vote.isDeliveryStateDirty()) { - persistOnlineVoteDelivery(uuid, vote); - } - } - } - } - - public void checkVoteParty() { - if (getConfig().getVotePartyEnabled()) { - if (votePartyVotes >= currentVotePartyVotesRequired) { - debug("Vote party reached"); - addCurrentVotePartyVotes(-currentVotePartyVotesRequired); - - currentVotePartyVotesRequired += getConfig().getVotePartyIncreaseVotesRequired(); - setVoteCacheVotePartyIncreaseVotesRequired( - getVoteCacheVotePartyIncreaseVotesRequired() + getConfig().getVotePartyIncreaseVotesRequired()); - - if (!getConfig().getVotePartyBroadcast().isEmpty()) { - broadcast(getConfig().getVotePartyBroadcast()); - } - - for (String command : getConfig().getVotePartyBungeeCommands()) { - runConsoleCommand(command); - } - - if (getConfig().getVotePartySendToAllServers()) { - for (String server : getAllAvailableServers()) { - sendVoteParty(server); - } - } else { - for (String server : getConfig().getVotePartyServersToSend()) { - sendVoteParty(server); - } - } - } - saveVoteCacheFile(); - } - } - - public abstract void debug(String str); - - private void debug2(String message) { - debug(message); - } - - /** - * HTTP client used for Mojang API requests. - */ - private final HttpClient httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); - - /** - * Fetches a player's UUID from the Mojang API. - * - * @param playerName player name - * @return player UUID, or {@code null} if not found - * @throws IOException if the request fails - * @throws InterruptedException if interrupted while waiting for the response - */ - public UUID fetchUUID(String playerName) throws IOException, InterruptedException { - if (playerName == null || playerName.equalsIgnoreCase("null")) { - return null; - } - - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create("https://api.mojang.com/users/profiles/minecraft/" + playerName)).GET() - .timeout(Duration.ofSeconds(5)).build(); - - HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - - if (response.statusCode() == 400 || response.statusCode() == 404) { - log("There is no player with the name \"" + playerName + "\"!"); - return null; - } - - if (response.statusCode() < 200 || response.statusCode() >= 300) { - throw new IOException("Failed to fetch UUID for " + playerName + ", HTTP " + response.statusCode()); - } - - JsonElement element = JsonParser.parseString(response.body()); - if (element == null || !element.isJsonObject()) { - return null; - } - - JsonObject object = element.getAsJsonObject(); - if (!object.has("id") || object.get("id").isJsonNull()) { - return null; - } - - String uuidAsString = object.get("id").getAsString(); - return parseUUIDFromString(uuidAsString); - } - - public abstract Set getAllAvailableServers(); - - /** Complete platform server set before whitelist/blocked routing filters. */ - public abstract Set getAllConfiguredServers(); - - public abstract VotingPluginProxyConfig getConfig(); - - public abstract String getCurrentPlayerServer(String player); - - /** - * Resolves a player's server for vote routing. A dedicated voting proxy has no - * local players, so it uses the backend presence tracker instead. - */ - protected String getCurrentPlayerServerForVoteRouting(String player) { - if (isDedicatedVotingProxyEnabled()) { - return backendPlayerPresenceTracker.getPlayer(player).map(presence -> presence.getServer()).orElse(null); - } - return getCurrentPlayerServer(player); - } - - /** - * Dedicated routing is intentionally unavailable on plugin messaging: that - * transport is attached to a player-facing proxy and does not carry backend - * presence snapshots. - */ - protected boolean isDedicatedVotingProxyEnabled() { - return getConfig().getDedicatedVotingProxy() && method != null && method.supportsBackendPresence(); - } - - public abstract File getDataFolderPlugin(); - - public String getMonthTotalsWithDatePath() { - LocalDateTime cTime = getBungeeTimeChecker().getTime(); - return getMonthTotalsWithDatePath(cTime); - } - - public String getMonthTotalsWithDatePath(LocalDateTime cTime) { - return "MonthTotal-" + cTime.getMonth().toString() + "-" + cTime.getYear(); - } - - public abstract String getProperName(String uuid, String playerName); - - public abstract String getUUID(String playerName); - - private int getValue(ArrayList cols, String column, int toAdd) { - for (Column d : cols) { - if (d.getName().equalsIgnoreCase(column)) { - DataValue value = d.getValue(); - int num = 0; - if (value == null) { - return toAdd; - } - if (value.isInt()) { - num = value.getInt(); - } else if (value.isString()) { - try { - num = Integer.parseInt(value.getString()); - } catch (Exception e) { - // ignore - } - } - return num + toAdd; - } - } - return toAdd; - } - - private VoteTotalsSnapshot getProjectedRolloverTotals(ArrayList data, String player) { - List timeChanges = getGlobalDataHandler().getTimeChanges(); - boolean resetMonth = timeChanges.contains(TimeType.MONTH); - boolean resetWeek = timeChanges.contains(TimeType.WEEK); - boolean resetDay = timeChanges.contains(TimeType.DAY); - int acceptedQueuedVotes = 0; - int acceptedGlobalQueuedVotes = 0; - for (VoteTimeQueue queued : getVoteCacheHandler().getTimeChangeQueue()) { - if (!queued.isProcessed()) { - acceptedGlobalQueuedVotes++; - } - if (!queued.isProcessed() && queued.getName() != null && queued.getName().equalsIgnoreCase(player)) { - acceptedQueuedVotes++; - } - } - int voteIncrement = acceptedQueuedVotes + 1; - - int allTimeTotal = getValue(data, "AllTimeTotal", voteIncrement); - int monthTotal = resetMonth ? voteIncrement : getValue(data, "MonthTotal", voteIncrement); - int weeklyTotal = resetWeek ? voteIncrement : getValue(data, "WeeklyTotal", voteIncrement); - int dailyTotal = resetDay ? voteIncrement : getValue(data, "DailyTotal", voteIncrement); - int points = getValue(data, "Points", voteIncrement * getConfig().getPointsOnVote()); - - int maxVotes = getConfig().getMaxAmountOfVotesPerDay(); - if (maxVotes > 0) { - int days = getBungeeTimeChecker().getTime().getDayOfMonth(); - if (monthTotal > days * maxVotes) { - monthTotal = days * maxVotes; - } - } - if (getConfig().getLimitVotePoints() > 0 && points > getConfig().getLimitVotePoints()) { - points = getConfig().getLimitVotePoints(); - } - - int dateMonthTotal = -1; - if (getConfig().getStoreMonthTotalsWithDate()) { - if (getConfig().getUseMonthDateTotalsAsPrimaryTotal()) { - dateMonthTotal = resetMonth ? voteIncrement - : getValue(data, getMonthTotalsWithDatePath(), voteIncrement); - } else { - dateMonthTotal = monthTotal; - } - } - - int[] projectedVoteParty = getProjectedVotePartyState(acceptedGlobalQueuedVotes + 1); - return new VoteTotalsSnapshot(allTimeTotal, monthTotal, weeklyTotal, dailyTotal, points, - projectedVoteParty[0], projectedVoteParty[1], dateMonthTotal); - } - - protected boolean canForwardStandaloneBroadcast(boolean managesTotals) { - return managesTotals; - } - - protected int[] getProjectedVotePartyState(int acceptedVotes) { - int current = votePartyVotes; - int required = currentVotePartyVotesRequired; - if (!getConfig().getVotePartyEnabled()) { - return new int[] { current, required }; - } - - int increase = getConfig().getVotePartyIncreaseVotesRequired(); - for (int i = 0; i < acceptedVotes; i++) { - current++; - if (current >= required) { - current -= required; - required += increase; - } - } - return new int[] { current, required }; - } - - public abstract String getPluginVersion(); - - public abstract int getVoteCacheCurrentVotePartyVotes(); - - public abstract long getVoteCacheLastUpdated(); - - public abstract int getVoteCachePrevDay(); - - public abstract String getVoteCachePrevMonth(); - - public abstract int getVoteCachePrevWeek(); - - public abstract int getVoteCacheVotePartyIncreaseVotesRequired(); - - public abstract boolean isPlayerOnline(String playerName); - - /** - * Checks online state for vote routing, using backend presence only when this - * proxy is explicitly configured as the dedicated voting proxy. - */ - protected boolean isPlayerOnlineForVoteRouting(String playerName) { - return isDedicatedVotingProxyEnabled() ? backendPlayerPresenceTracker.getPlayer(playerName).isPresent() - : isPlayerOnline(playerName); - } - - public abstract boolean isServerValid(String server); - - public abstract boolean isSomeoneOnlineServer(String server); - - protected boolean isSomeoneOnlineServerForVoteRouting(String server) { - if (!isDedicatedVotingProxyEnabled()) { - return isSomeoneOnlineServer(server); - } - com.bencodez.votingplugin.proxy.presence.BackendPresenceStatus status = backendPlayerPresenceTracker - .getBackendStatus(server); - return status != null && status.isAvailable() && status.getPlayerCount() > 0; - } - - public abstract boolean isVoteCacheIgnoreTime(); - - public abstract MysqlConfig getVoteCacheMySQLConfig(); - - public abstract MysqlConfig getNonVotedCacheMySQLConfig(); - - public abstract MysqlConfig getVoteLoggingMySQLConfig(); - - /** - * Shutdown MySQL-related resources safely. - */ - public void shutdownMySql() { - if (getProxyMysqlMessenger() != null) { - getProxyMysqlMessenger().shutdown(); - setProxyMysqlMessenger(null); - } - - if (getProxyMySQL() != null) { - getProxyMySQL().shutdown(); - setProxyMySQL(null); - } - } - - public void load(IVoteCache jsonStorage, INonVotedPlayersStorage nonVotedCacheJson) { - method = BungeeMethod.getByName(getConfig().getBungeeMethod()); - if (getMethod() == null) { - method = BungeeMethod.PLUGINMESSAGING; - } - warnUnsupportedDedicatedVotingProxyMode(); - uuidPlayerNameCache = getProxyMySQL().getRowsUUIDNameQuery(); - - bungeeTimeChecker.setTimeChangeFailSafeBypass(getConfig().getTimeChangeFailSafeBypass()); - bungeeTimeChecker.loadTimer(); - - voteCacheHandler = new VoteCacheHandler(getVoteCacheMySQLConfig(), getConfig().getVoteCacheUseMySQL(), - getConfig().getVoteCacheUseMainMySQL(), getProxyMySQL().getMysql(), getConfig().getDebug(), - jsonStorage) { - - @Override - public void logInfo1(String msg) { - logInfo(msg); - } - - @Override - public void logSevere1(String msg) { - logSevere(msg); - } - - @Override - public void debug1(Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void debug1(String msg) { - if (getConfig().getDebug()) { - debug(msg); - } - } - - @Override - public void debug1(Throwable e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - }; - voteCacheHandler.load(); - - nonVotedPlayersCache = new NonVotedPlayersCache(getNonVotedCacheMySQLConfig(), - getConfig().getNonVotedCacheUseMySQL(), getConfig().getNonVotedCacheUseMainMySQL(), - getProxyMySQL().getMysql(), nonVotedCacheJson, getConfig().getDebug()) { - - @Override - public boolean userExists(String uuid) { - return getProxyMySQL().containsKeyQuery(uuid); - } - - @Override - public void logInfo1(String msg) { - logInfo(msg); - } - - @Override - public void logSevere1(String msg) { - logSevere(msg); - } - - @Override - public void debug1(Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void debug1(String msg) { - if (getConfig().getDebug()) { - debug(msg); - } - } - - @Override - public Set getAllUUIDs() { - return getProxyMySQL().getUuids(); - } - }; - - if (method.equals(BungeeMethod.MYSQL)) { - try { - proxyMysqlMessenger = new MySqlMessenger("VotingPlugin", - getProxyMySQL().getMysql().getConnectionManager().getDataSource(), MySqlMessenger.Mode.PROXY, - null, // no serverId in PROXY mode - msg -> { - if (getConfig().getDebug()) { - debug("Got from " + msg.source + ": " + msg.envelope.getSubChannel() + " " - + msg.envelope.getFields()); - } - globalMessageProxyHandler.onMessage(msg.envelope); - }); - } catch (SQLException e) { - e.printStackTrace(); - } - } else if (method.equals(BungeeMethod.PLUGINMESSAGING)) { - if (getConfig().getPluginMessageEncryption()) { - encryptionHandler = new EncryptionHandler("VotingPlugin", - new File(getDataFolderPlugin(), "secretkey.key")); - } - } else if (method.equals(BungeeMethod.SOCKETS)) { - encryptionHandler = new EncryptionHandler("VotingPlugin", new File(getDataFolderPlugin(), "secretkey.key")); - - socketHandler = new SocketHandler(getPluginVersion(), getConfig().getBungeeHost(), - getConfig().getBungeePort(), encryptionHandler, getConfig().getDebug()) { - - @Override - public void log(String str) { - logInfo(str); - } - }; - - socketHandler.add(new SocketReceiver() { - @Override - public void onReceiveEnvelope(JsonEnvelope envelope) { - globalMessageProxyHandler.onMessage(envelope); - } - }); - - rebuildSocketClients(); - } else if (method.equals(BungeeMethod.HTTP)) { - startHttpTransport(); - } else if (method.equals(BungeeMethod.REDIS)) { - redisHandler = new RedisHandler(getConfig().getRedisHost(), getConfig().getRedisPort(), - getConfig().getRedisUsername(), getConfig().getRedisPassword(), getConfig().getRedisDbIndex(), - getConfig().getRedisSsl()) { - - @Override - public void debug(String message) { - debug2(message); - } - }; - redisPublisherPool = new JedisPool(new HostAndPort(getConfig().getRedisHost(), getConfig().getRedisPort()), - buildRedisClientConfig(getConfig())); - - runAsync(() -> { - RedisListener listener = redisHandler.createEnvelopeListener( - getConfig().getRedisPrefix() + "VotingPlugin", - (ch, env) -> globalMessageProxyHandler.onMessage(env)); - redisHandler.loadListener(listener); - }); - - } else if (method.equals(BungeeMethod.MQTT)) { - try { - mqttHandler = new MqttHandler(new MqttServerComm(getConfig().getMqttClientID(), - getConfig().getMqttBrokerURL(), getConfig().getMqttUsername(), getConfig().getMqttPassword()), - 2); - - mqttHandler.subscribeEnvelopes(getConfig().getMqttPrefix() + "votingplugin/servers/proxy", - (topic, env) -> globalMessageProxyHandler.onMessage(env)); - - } catch (MqttException e) { - e.printStackTrace(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - currentVotePartyVotesRequired = getConfig().getVotePartyVotesRequired() - + getVoteCacheVotePartyIncreaseVotesRequired(); - votePartyVotes = getVoteCacheCurrentVotePartyVotes(); - - globalMessageProxyHandler = new GlobalMessageProxyHandler() { - @Override - public void sendMessage(String server, int delay, JsonEnvelope envelope) { - switch (method) { - case MQTT: - sendMqttEnvelopeServer(server, envelope); - break; - case MYSQL: - try { - proxyMysqlMessenger.sendToBackend(server, envelope); - } catch (SQLException e) { - e.printStackTrace(); - } - break; - case PLUGINMESSAGING: - sendPluginMessageServer(server, delay, envelope); - break; - case REDIS: - sendRedisEnvelopeServer(server, envelope); - break; - case SOCKETS: - sendSocketEnvelope(server, envelope); - break; - case HTTP: - sendHttpEnvelope(server, envelope); - break; - default: - break; - } - } - }; - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_LOGIN) { - @Override - public void onReceive(JsonEnvelope message) { - handleLoginMessage(message); - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_LOGOUT) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(message); - if (!isPresenceServerValid(event.server, VotingPluginWire.SUB_LOGOUT) - || !isPresenceGenerationValid(event.backendIncarnationId, event.backendStartedAt, - event.presenceTimestamp, - VotingPluginWire.SUB_LOGOUT)) { - return; - } - if (!backendPlayerPresenceTracker.playerOffline(event.uuid, event.server, event.connectionId, - event.backendIncarnationId, event.backendStartedAt, event.presenceTimestamp, - System.currentTimeMillis())) { - debug("Ignored invalid or stale logout envelope: " + message.getFields()); - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_STARTED) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); - long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); - long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); - if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_STARTED) - && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, - VotingPluginWire.SUB_BACKEND_STARTED)) { - if (backendPlayerPresenceTracker.backendStarted(server, backendIncarnationId, backendStartedAt, - presenceTimestamp, System.currentTimeMillis())) { - discardPendingPresenceHandoffs(server); - pendingBackendRecoverySnapshots.add(presenceServerKey(server)); - requestBackendPresenceSnapshot(server); - } - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_STOPPED) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); - long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); - long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); - if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_STOPPED) - && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, - VotingPluginWire.SUB_BACKEND_STOPPED)) { - if (backendPlayerPresenceTracker.backendStopped(server, backendIncarnationId, backendStartedAt, - presenceTimestamp, System.currentTimeMillis())) { - discardPendingPresenceHandoffs(server); - pendingBackendRecoverySnapshots.remove(presenceServerKey(server)); - } - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_HEARTBEAT) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); - long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); - long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); - if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_HEARTBEAT) - && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, - VotingPluginWire.SUB_BACKEND_HEARTBEAT)) { - backendPlayerPresenceTracker.heartbeat(server, backendIncarnationId, backendStartedAt, - presenceTimestamp, System.currentTimeMillis()); - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_PRESENCE_SNAPSHOT) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - if (!isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_SNAPSHOT)) { - return; - } - VotingPluginWire.PresenceSnapshot snapshot = VotingPluginWire.readPresenceSnapshot(message); - long now = System.currentTimeMillis(); - boolean accepted = snapshot.valid - && isPresenceGenerationValid(snapshot.backendIncarnationId, snapshot.backendStartedAt, - snapshot.presenceTimestamp, - VotingPluginWire.SUB_PRESENCE_SNAPSHOT) - && backendPlayerPresenceTracker.applySnapshotChunk(snapshot.server, - snapshot.requestId, snapshot.chunkIndex, snapshot.chunkCount, snapshot.players, - snapshot.backendIncarnationId, snapshot.backendStartedAt, - snapshot.presenceTimestamp, now); - if (!accepted) { - debug("Ignored invalid or unexpected presence snapshot from " + snapshot.server); - if (backendPlayerPresenceTracker.getPendingSnapshotRequestId(snapshot.server, now) == null) { - discardPendingPresenceHandoffs(snapshot.requestId); - } - } else if (backendPlayerPresenceTracker.getPendingSnapshotRequestId(snapshot.server, now) == null) { - pendingBackendRecoverySnapshots.remove(presenceServerKey(snapshot.server)); - Set handoffPlayers = completePendingPresenceHandoffs(snapshot.requestId, snapshot.server, - snapshot.backendIncarnationId, snapshot.backendStartedAt, now); - processDedicatedSnapshotLogins(snapshot.server, handoffPlayers); - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_STATUS_OKAY) { - @Override - public void onReceive(JsonEnvelope message) { - handleStatusOkay(message); - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener("voteupdate") { - @Override - public void onReceive(JsonEnvelope message) { - int delay = 1; - for (String send : getAllAvailableServers()) { - globalMessageProxyHandler.sendMessage(send, delay, message); - delay++; - } - } - }); - - proxyBroadcastDecider = new ProxyBroadcastDecider(() -> getConfig(), () -> getAllAvailableServers(), - s -> isServerValid(s), - s -> getConfig().getBlockedServers() != null && getConfig().getBlockedServers().contains(s)); - - loadMultiProxySupport(); - loadVoteLoggingMySQL(); - if (method.supportsBackendPresence()) { - scheduleBackendPresenceStartupResync(); - loadTaskTimer(this::maintainBackendPresence, PRESENCE_MAINTENANCE_INTERVAL_SECONDS, - PRESENCE_MAINTENANCE_INTERVAL_SECONDS); - } - startControlServices(); - - debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); - } - - private void startControlServices() { - synchronized (controlLifecycleLock) { - ControlConnector predecessor = controlConnector; - if (predecessor != null && predecessor.deferReplacementUntilSafe(this::restartControlServicesAsync)) { - log("[Control] service restart deferred until the current result is acknowledged"); - return; - } - stopControlServicesLocked(true); - startControlServicesLocked(); - } - } - - /** Keeps potentially long hosted-Control handoffs off proxy command/event threads. */ - private void restartControlServicesAsync() { - final long generation = controlServicesGeneration.incrementAndGet(); - try { - controlLifecycleExecutor.execute(() -> { - try { - synchronized (controlLifecycleLock) { - if (!enabled || generation != controlServicesGeneration.get()) return; - ControlConnector predecessor = controlConnector; - if (predecessor != null - && predecessor.deferReplacementUntilSafe(this::restartControlServicesAsync)) { - log("[Control] service restart deferred until the current result is acknowledged"); - return; - } - stopControlServicesLocked(true); - startControlServicesLocked(); - } - } catch (RuntimeException failure) { - if (generation == controlServicesGeneration.get()) { - logSevere("[Control] asynchronous service restart failed: " + failure.getMessage()); - } - } - }); - } catch (RuntimeException failure) { - logSevere("[Control] services were not restarted because async scheduling failed"); - } - } - - /** Rebuilds a recovery connector from current settings after its durable result is acknowledged. */ - public final void restartControlServicesAfterRecovery() { - restartControlServicesAsync(); - } - - private void stopControlServices(boolean waitForHosted) { - synchronized (controlLifecycleLock) { - stopControlServicesLocked(waitForHosted); - } - } - - private void startControlServicesLocked() { - if (getConfig().getControlHostedEnabled()) { - try { - hostedControlManager = HostedControlManager.create(this); - if (hostedControlManager != null) hostedControlManager.start(); - } catch (IOException | IllegalArgumentException e) { - hostedControlManager = null; - logSevere("[Control Host] configuration or automatic enrollment is invalid; VotingPlugin remains unaffected"); - } - } - try { - controlConnector = ControlConnector.create(this); - if (controlConnector != null) controlConnector.start(); - } catch (IOException | IllegalArgumentException e) { - controlConnector = null; - logSevere("[Control] connector configuration or credential is invalid; voting remains unaffected"); - } - } - - private void stopControlServicesLocked(boolean waitForHosted) { - ControlConnector connector = controlConnector; - if (connector != null) { - try { - connector.close(); - if (controlConnector == connector) controlConnector = null; - } catch (RuntimeException failure) { - if (waitForHosted) throw failure; - if (controlConnector == connector) controlConnector = null; - logSevere("[Control] connector did not stop cleanly; proxy cleanup will continue"); - } - } - HostedControlManager manager = hostedControlManager; - if (manager != null) { - try { - if (waitForHosted) { - manager.closeAndWait(); - } else { - manager.close(); - } - if (hostedControlManager == manager) hostedControlManager = null; - } catch (RuntimeException failure) { - if (waitForHosted) throw failure; - if (hostedControlManager == manager) hostedControlManager = null; - logSevere("[Control Host] manager did not stop cleanly; proxy cleanup will continue"); - } - } - } - - public String getControlConnectorStatus() { - ControlConnector connector = controlConnector; - return connector == null ? "DISABLED" : connector.status().name(); - } - - public String getHostedControlStatus() { - HostedControlManager manager = hostedControlManager; - return manager == null ? "DISABLED" : manager.status().name(); - } - - /** - * Handles both the original login notification and extended presence logins. - * Kept protected so transport-policy behavior can be regression tested without - * initializing a live proxy transport. - * - * @param message login envelope - */ - protected void handleLoginMessage(JsonEnvelope message) { - VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(message); - String player = event.player; - String uuid = event.uuid; - String server = event.server; - - if (player.isEmpty() || uuid.isEmpty()) { - logSevere("Invalid login envelope received: " + message.getFields()); - return; - } - boolean legacy = event.connectionId == null && event.backendIncarnationId == null - && event.backendStartedAt == 0L && event.presenceTimestamp == 0L; - boolean accepted = false; - String deliveryServer = server; - if (legacy) { - if (method == BungeeMethod.PLUGINMESSAGING) { - String proxyServer = getCurrentPlayerServer(player); - accepted = isLegacyLoginDestinationAuthoritative(player, uuid, proxyServer); - if (accepted) { - deliveryServer = proxyServer; - } - } else if (method != null && method.supportsBackendPresence() - && isPresenceServerValid(server, VotingPluginWire.SUB_LOGIN)) { - accepted = isLegacyLoginDestinationAuthoritative(player, uuid, server); - } - } else if (method != null && method.supportsBackendPresence() && event.connectionId != null - && isPresenceServerValid(server, VotingPluginWire.SUB_LOGIN) - && isPresenceGenerationValid(event.backendIncarnationId, event.backendStartedAt, - event.presenceTimestamp, VotingPluginWire.SUB_LOGIN)) { - BackendPlayerPresenceTracker.PlayerOnlineResult result = backendPlayerPresenceTracker.playerOnlineResult( - player, uuid, server, event.connectionId, - event.backendIncarnationId, event.backendStartedAt, event.presenceTimestamp, - System.currentTimeMillis()); - accepted = result.isAccepted(); - if (result.isConflictingPresence()) { - requestBackendPresenceSnapshot(server, - new PendingPresenceHandoff(player, uuid, server, event.connectionId, - event.backendIncarnationId, event.backendStartedAt, - result.getConflictSequence(), System.currentTimeMillis())); - } - } - - debug("Login: " + player + "/" + uuid + " " + server); - if (accepted) { - discardPendingPresenceHandoff(uuid); - login(player, uuid, deliveryServer); - } else { - debug("Ignored invalid or stale login envelope: " + message.getFields()); - } - } - - /** - * Validates a legacy login against an authority independent of the envelope. - * Player-facing proxies use their native live route and UUID. A dedicated - * voting proxy has no native player session, so it requires an exact modern - * presence match for the claimed destination. - */ - private boolean isLegacyLoginDestinationAuthoritative(String player, String uuid, String server) { - if (server == null || server.isBlank()) { - return false; - } - - UUID claimedUuid; - try { - claimedUuid = UUID.fromString(uuid.trim()); - } catch (RuntimeException e) { - return false; - } - - if (isDedicatedVotingProxyEnabled()) { - PlayerPresence presence = backendPlayerPresenceTracker.getPlayer(player).orElse(null); - return presence != null && presence.getServer().equalsIgnoreCase(server) - && (!getConfig().getOnlineMode() || presence.getUuid().equals(claimedUuid)); - } - - if (!isPlayerOnline(player)) { - return false; - } - String proxyServer = getCurrentPlayerServer(player); - if (proxyServer == null || !proxyServer.equalsIgnoreCase(server)) { - return false; - } - if (!getConfig().getOnlineMode()) { - return true; - } - - String authoritativeUuid = getUUID(player); - if (authoritativeUuid == null || authoritativeUuid.isBlank()) { - return false; - } - try { - return claimedUuid.equals(UUID.fromString(authoritativeUuid.trim())); - } catch (IllegalArgumentException e) { - return false; - } - } - - private VoteLogMysqlTable voteLogMysqlTable; - - @Getter - private ProxyBroadcastDecider proxyBroadcastDecider; - - public void loadVoteLoggingMySQL() { - if (getConfig().getVoteLoggingEnabled()) { - if (getConfig().getVoteLoggingUseMainMySQL()) { - voteLogMysqlTable = new VoteLogMysqlTable("votingplugin_votelog", getProxyMySQL().getMysql(), - getVoteLoggingMySQLConfig(), getConfig().getDebug()) { - - @Override - public void logSevere(String string) { - VotingPluginProxy.this.logSevere(string); - } - - @Override - public void logInfo(String string) { - VotingPluginProxy.this.logInfo(string); - } - - @Override - public void debug(Throwable e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public String getServerName() { - return "Proxy"; - } - }; - } else { - voteLogMysqlTable = new VoteLogMysqlTable("votingplugin_votelog", getVoteLoggingMySQLConfig(), - getConfig().getDebug()) { - - @Override - public void logSevere(String string) { - VotingPluginProxy.this.logSevere(string); - } - - @Override - public void logInfo(String string) { - VotingPluginProxy.this.logInfo(string); - } - - @Override - public void debug(Throwable e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public String getServerName() { - return "Proxy"; - } - }; - } - - if (getConfig().getVoteLoggingPurgeDays() > 0) { - loadTaskTimer(() -> voteLogMysqlTable.purgeOlderThanDays(getConfig().getVoteLoggingPurgeDays(), 100), - 60, 60 * 60); - } - - debug("Vote logging MySQL enabled"); - } else { - debug("Vote logging MySQL disabled"); - } - } - - public abstract void loadTaskTimer(Runnable runnable, long delaySeconds, long repeatSeconds); - - public void loadMultiProxySupport() { - if (multiProxyHandler != null) { - multiProxyHandler.close(); - } - multiProxyHandler = new MultiProxyHandler() { - - @Override - public void addNonVotedPlayerCache(String uuid, String player) { - addNonVotedPlayer(uuid, player); - } - - @Override - public void clearVote(String uuid) { - getVoteCacheHandler().clearOnlineVoteRewards(uuid); - } - - @Override - public boolean getDebug() { - return getConfig().getDebug(); - } - - @Override - public EncryptionHandler getEncryptionHandler() { - return encryptionHandler; - } - - @Override - public MultiProxyMethod getMultiProxyMethod() { - return MultiProxyMethod.getByName(getConfig().getMultiProxyMethod()); - } - - @Override - public String getMultiProxyPassword() { - return getConfig().getMultiProxyRedisPassword(); - } - - @Override - public String getMultiProxyRedisHost() { - return getConfig().getMultiProxyRedisHost(); - } - - @Override - public int getMultiProxyRedisPort() { - return getConfig().getMultiProxyRedisPort(); - } - - @Override - public boolean getMultiProxyRedisSsl() { - return getConfig().getMultiProxyRedisSsl(); - } - - @Override - public int getMultiProxyRedisDbIndex() { - return getConfig().getMultiProxyRedisDbIndex(); - } - - @Override - public boolean getMultiProxyRedisUseExistingConnection() { - return getConfig().getMultiProxyRedisUseExistingConnection(); - } - - @Override - public String getMultiProxyServerName() { - return getConfig().getProxyServerName(); - } - - @Override - public Collection getMultiProxyServers() { - return getConfig().getMultiProxyServers(); - } - - @Override - public MultiProxyServerSocketConfiguration getMultiProxyServersConfiguration(String s) { - return new MultiProxyServerSocketConfigurationBungee(s, - getConfig().getMultiProxyServersConfiguration(s)); - } - - @Override - public String getMultiProxySocketHostHost() { - return getConfig().getMultiProxySocketHostHost(); - } - - @Override - public int getMultiProxySocketHostPort() { - return getConfig().getMultiProxySocketHostPort(); - } - - @Override - public boolean getMultiProxySupportEnabled() { - return getConfig().getMultiProxySupport(); - } - - @Override - public String getMultiProxyUsername() { - return getConfig().getMultiProxyRedisUsername(); - } - - @Override - public File getPluginDataFolder() { - return getDataFolderPlugin(); - } - - @Override - public boolean getPrimaryServer() { - return getConfig().getPrimaryServer(); - } - - @Override - public List getProxyServers() { - return getConfig().getProxyServers(); - } - - @Override - public RedisHandler getRedisHandler() { - return redisHandler; - } - - @Override - public String getVersion() { - return getPluginVersion(); - } - - @Override - public void logInfo(String msg) { - log(msg); - } - - @Override - public void runAsnc(Runnable runnable) { - runAsync(runnable); - } - - @Override - public void setEncryptionHandler(EncryptionHandler encryptionHandler1) { - encryptionHandler = encryptionHandler1; - } - - @Override - public void triggerVote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, - VoteTotalsSnapshot text, String uuid) { - vote(player, service, realVote, timeQueue, queueTime, text, uuid); - } - }; - multiProxyHandler.loadMultiProxySupport(); - } - - public abstract void log(String message); - - /** - * Requests a complete player-presence snapshot from one backend server. - * - * @param server configured backend server name - * @return new or already-active request identifier, or null when the server is - * invalid or is inside the snapshot-request cooldown - */ - public UUID requestBackendPresenceSnapshot(String server) { - return requestBackendPresenceSnapshot(server, null); - } - - private UUID requestBackendPresenceSnapshot(String server, PendingPresenceHandoff handoff) { - return requestBackendPresenceSnapshot(server, handoff, System.currentTimeMillis(), false); - } - - private UUID requestBackendPresenceSnapshot(String server, PendingPresenceHandoff handoff, long now, - boolean handoffAlreadyQueued) { - if (method == null || !method.supportsBackendPresence() || globalMessageProxyHandler == null - || !isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_SNAPSHOT_REQUEST)) { - return null; - } - long backendStartedAt = backendPlayerPresenceTracker.getBackendStartedAt(server); - UUID backendIncarnationId = backendPlayerPresenceTracker.getBackendIncarnationId(server); - if (backendStartedAt <= 0L || backendIncarnationId == null) { - return null; - } - if (handoff != null && (!server.equalsIgnoreCase(handoff.server) - || !backendIncarnationId.equals(handoff.backendIncarnationId) - || backendStartedAt != handoff.backendStartedAt)) { - return null; - } - if (handoff != null && (handoffAlreadyQueued ? !isPendingPresenceHandoff(handoff, now) - : !queuePendingPresenceHandoff(handoff, now))) { - return null; - } - UUID requestId = handoff == null - ? backendPlayerPresenceTracker.beginSnapshot(server, UUID.randomUUID(), backendIncarnationId, - backendStartedAt, now) - : backendPlayerPresenceTracker.beginSnapshotForDestinationClaim(server, UUID.randomUUID(), - backendIncarnationId, backendStartedAt, handoff.playerUuid, handoff.conflictSequence, now); - boolean created = requestId != null; - if (!created) { - requestId = handoff == null ? backendPlayerPresenceTracker.getPendingSnapshotRequestId(server, now) - : backendPlayerPresenceTracker.getPendingSnapshotRequestIdForDestinationClaim(server, - handoff.playerUuid, handoff.conflictSequence, now); - } - if (requestId == null) { - if (handoff != null && !backendPlayerPresenceTracker.isCurrentDestinationClaim(handoff.playerUuid, - handoff.server, handoff.conflictSequence)) { - discardPendingPresenceHandoff(handoff); - } - // A handoff stays unassigned while the destination is inside its snapshot - // cooldown. Presence maintenance will attach it to the next allowed snapshot. - return null; - } - if (handoff != null) { - assignPendingPresenceHandoff(handoff, requestId, now); - } - if (created) { - JsonEnvelope request = VotingPluginWire.presenceSnapshotRequest(server, requestId, backendIncarnationId, - backendStartedAt, now); - globalMessageProxyHandler.sendMessage(server, 1, request); - } - return requestId; - } - - private boolean queuePendingPresenceHandoff(PendingPresenceHandoff handoff, long now) { - if (!isPresenceHandoffValid(handoff, now)) { - return false; - } - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - PendingPresenceHandoff current = pendingPresenceHandoffs.get(handoff.playerUuid); - if (current != null && current.conflictSequence > handoff.conflictSequence) { - return false; - } - handoff.requestId = null; - pendingPresenceHandoffs.put(handoff.playerUuid, handoff); - return true; - } - } - - private boolean isPendingPresenceHandoff(PendingPresenceHandoff handoff, long now) { - if (!isPresenceHandoffValid(handoff, now)) { - return false; - } - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - return pendingPresenceHandoffs.get(handoff.playerUuid) == handoff; - } - } - - private void assignPendingPresenceHandoff(PendingPresenceHandoff handoff, UUID requestId, long now) { - if (requestId == null || !isPresenceHandoffValid(handoff, now)) { - return; - } - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - if (pendingPresenceHandoffs.get(handoff.playerUuid) == handoff) { - handoff.requestId = requestId; - } - } - } - - private boolean isPresenceHandoffValid(PendingPresenceHandoff handoff, long now) { - return handoff != null && handoff.playerUuid != null && handoff.connectionId != null - && handoff.conflictSequence > 0L - && now >= handoff.createdAt && now - handoff.createdAt <= PRESENCE_HANDOFF_TIMEOUT_MILLIS; - } - - private Set completePendingPresenceHandoffs(UUID requestId, String server, UUID backendIncarnationId, - long backendStartedAt, long now) { - List completed = new ArrayList<>(); - Set completedPlayers = new LinkedHashSet<>(); - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - pendingPresenceHandoffs.entrySet().removeIf(entry -> { - PendingPresenceHandoff handoff = entry.getValue(); - if (!requestId.equals(handoff.requestId)) { - return false; - } - if (handoff.server.equalsIgnoreCase(server) - && handoff.backendIncarnationId.equals(backendIncarnationId) - && handoff.backendStartedAt == backendStartedAt) { - completed.add(handoff); - } - return true; - }); - } - for (PendingPresenceHandoff handoff : completed) { - PlayerPresence presence = backendPlayerPresenceTracker.getPlayer(handoff.playerUuid).orElse(null); - if (presence != null && presence.getServer().equalsIgnoreCase(handoff.server) - && presence.getConnectionId().equals(handoff.connectionId)) { - login(handoff.playerName, handoff.uuid, handoff.server); - completedPlayers.add(handoff.playerUuid); - } - releaseDestinationClaim(handoff); - } - return completedPlayers; - } - - /** - * Drains voter-keyed cached rewards when a complete recovery snapshot first - * confirms a player on a dedicated voting proxy. Cross-backend handoffs are - * already processed by their token-bound completion path and are excluded to - * avoid a second login callback. - */ - protected void processDedicatedSnapshotLogins(String server, Set handoffPlayers) { - if (!isDedicatedVotingProxyEnabled() || server == null || server.isBlank()) { - return; - } - Set excluded = handoffPlayers == null ? Collections.emptySet() : handoffPlayers; - for (PlayerPresence presence : backendPlayerPresenceTracker.getOnlinePlayers()) { - if (presence.getServer().equalsIgnoreCase(server) && !excluded.contains(presence.getUuid())) { - login(presence.getPlayerName(), presence.getUuid().toString(), presence.getServer()); - } - } - } - - private void discardPendingPresenceHandoff(String uuid) { - try { - UUID playerUuid = UUID.fromString(uuid.trim()); - PendingPresenceHandoff removed; - synchronized (pendingPresenceHandoffs) { - removed = pendingPresenceHandoffs.remove(playerUuid); - } - releaseDestinationClaim(removed); - } catch (Exception ignored) { - // Invalid identities are rejected by the presence tracker. - } - } - - private void discardPendingPresenceHandoff(PendingPresenceHandoff handoff) { - boolean removed = false; - synchronized (pendingPresenceHandoffs) { - if (handoff != null && pendingPresenceHandoffs.get(handoff.playerUuid) == handoff) { - pendingPresenceHandoffs.remove(handoff.playerUuid); - removed = true; - } - } - if (removed) { - releaseDestinationClaim(handoff); - } - } - - private void discardPendingPresenceHandoffs(String server) { - synchronized (pendingPresenceHandoffs) { - pendingPresenceHandoffs.entrySet().removeIf(entry -> { - if (!entry.getValue().server.equalsIgnoreCase(server)) { - return false; - } - releaseDestinationClaim(entry.getValue()); - return true; - }); - } - } - - private void discardPendingPresenceHandoffs(UUID requestId) { - if (requestId == null) { - return; - } - synchronized (pendingPresenceHandoffs) { - pendingPresenceHandoffs.entrySet().removeIf(entry -> { - if (!requestId.equals(entry.getValue().requestId)) { - return false; - } - releaseDestinationClaim(entry.getValue()); - return true; - }); - } - } - - private void prunePendingPresenceHandoffs(long now) { - pendingPresenceHandoffs.entrySet().removeIf(entry -> { - PendingPresenceHandoff handoff = entry.getValue(); - if (now >= handoff.createdAt && now - handoff.createdAt <= PRESENCE_HANDOFF_TIMEOUT_MILLIS) { - return false; - } - releaseDestinationClaim(handoff); - return true; - }); - } - - private void releaseDestinationClaim(PendingPresenceHandoff handoff) { - if (handoff != null) { - backendPlayerPresenceTracker.releaseDestinationClaim(handoff.playerUuid, handoff.server, - handoff.conflictSequence); - } - } - - protected void retryPendingPresenceHandoffs(long now) { - List retry = new ArrayList<>(); - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - for (PendingPresenceHandoff handoff : pendingPresenceHandoffs.values()) { - UUID activeRequestId = backendPlayerPresenceTracker.getPendingSnapshotRequestId(handoff.server, now); - if (handoff.requestId != null && !handoff.requestId.equals(activeRequestId)) { - handoff.requestId = null; - } - if (handoff.requestId == null) { - retry.add(handoff); - } - } - } - for (PendingPresenceHandoff handoff : retry) { - requestBackendPresenceSnapshot(handoff.server, handoff, now, true); - } - } - - protected int getPendingPresenceHandoffCount() { - synchronized (pendingPresenceHandoffs) { - return pendingPresenceHandoffs.size(); - } - } - - protected void scheduleBackendPresenceStartupResync() { - ScheduledExecutorService scheduler = getScheduler(); - if (method == null || !method.supportsBackendPresence() || scheduler == null) { - return; - } - scheduler.schedule(this::requestBackendPresenceStartupResync, - PRESENCE_STARTUP_RESYNC_DELAY_SECONDS, TimeUnit.SECONDS); - } - - protected void requestBackendPresenceStartupResync() { - if (!enabled || method == null || !method.supportsBackendPresence() || globalMessageProxyHandler == null) { - return; - } - long requestedAt = System.currentTimeMillis(); - int delay = 1; - for (String server : getAllAvailableServers()) { - if (!isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_RESYNC_REQUEST)) { - continue; - } - globalMessageProxyHandler.sendMessage(server, delay++, - VotingPluginWire.presenceResyncRequest(server, UUID.randomUUID(), requestedAt)); - } - } - - private void maintainBackendPresence() { - if (!enabled || method == null || !method.supportsBackendPresence()) { - return; - } - expireBackendPresence(PRESENCE_BACKEND_TIMEOUT_MILLIS); - for (String server : getAllAvailableServers()) { - if (pendingBackendRecoverySnapshots.contains(presenceServerKey(server))) { - requestBackendPresenceSnapshot(server); - } - } - retryPendingPresenceHandoffs(System.currentTimeMillis()); - } - - private String presenceServerKey(String server) { - return server == null ? "" : server.trim().toLowerCase(java.util.Locale.ROOT); - } - - private boolean isPresenceServerValid(String server, String subChannel) { - // The presence protocol's trust boundary is the configured backend set. The - // selected transport must only be accessible to backend servers trusted not to - // impersonate one another. - if (server == null || server.isBlank() || !isServerValid(server)) { - debug("Ignored " + subChannel + " presence envelope for an unconfigured server"); - return false; - } - return true; - } - - private boolean isPresenceGenerationValid(UUID backendIncarnationId, long backendStartedAt, - long presenceTimestamp, String subChannel) { - if (backendIncarnationId == null || backendStartedAt <= 0L || presenceTimestamp < backendStartedAt) { - debug("Ignored " + subChannel + " presence envelope with an invalid backend generation"); - return false; - } - return true; - } - - /** - * Removes presence owned by backends that have stopped reporting heartbeats. - * Scheduling and timeout configuration are intentionally left to dedicated - * proxy mode. - * - * @param timeoutMillis maximum backend silence before expiry - * @return expired backend server names - */ - public Set expireBackendPresence(long timeoutMillis) { - if (method == null || !method.supportsBackendPresence()) { - return Collections.emptySet(); - } - long now = System.currentTimeMillis(); - Set expired = backendPlayerPresenceTracker.expireBackends(now, timeoutMillis); - for (String server : expired) { - discardPendingPresenceHandoffs(server); - // Keep recovery pending while this generation is unavailable. If the same - // backend process resumes, its heartbeat can mark it available again and the - // maintenance task will request a fresh snapshot of players who stayed online. - pendingBackendRecoverySnapshots.add(presenceServerKey(server)); - } - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - } - return expired; - } - - public void login(String playerName, String uuid, String serverName) { - if (!getConfig().getOnlineMode()) { - uuid = getUUID(playerName); - } - - try { - if (uuid != null && !uuid.isEmpty() && !uuid.equalsIgnoreCase("null")) { - uuid = UUID.fromString(uuid.trim()).toString(); - } - } catch (Exception ignored) { - // ignore - } - - if (getConfig().getOnlineMode()) { - addNonVotedPlayer(uuid, playerName); - } - if (isPlayerOnlineForVoteRouting(playerName)) { - if (getConfig().getGlobalDataEnabled()) { - if (getGlobalDataHandler().isTimeChangedHappened()) { - getGlobalDataHandler().checkForFinishedTimeChanges(); - } - } - - checkCachedVotes(serverName); - retryPendingOnlineBroadcasts(serverName); - retryPendingTimeBroadcasts(serverName); - checkOnlineVotes(playerName, uuid, serverName); - multiProxyHandler.login(uuid, playerName); - } - } - - private void logInfo(String msg) { - log(msg); - } - - public abstract void logSevere(String message); - - public void onDisable() { - onDisable(false); - } - - /** Full runtime replacement waits for hosted workers; final proxy stop remains non-blocking. */ - public void onDisable(boolean waitForHosted) { - if (waitForHosted) { - prepareForRuntimeReplacement(); - } else { - controlServicesGeneration.incrementAndGet(); - controlLifecycleExecutor.shutdownNow(); - stopControlServices(false); - } - completeRuntimeReplacementShutdown(); - } - - /** Fail-closed gate that must complete before a replacement proxy runtime is created. */ - public void prepareForRuntimeReplacement() { - controlServicesGeneration.incrementAndGet(); - synchronized (controlLifecycleLock) { - ControlConnector connector = controlConnector; - if (connector != null && !connector.reserveRuntimeReplacement()) { - throw new IllegalStateException("Control result must be acknowledged before proxy runtime replacement"); - } - controlLifecycleExecutor.shutdown(); - stopControlServicesLocked(true); - } - } - - /** Best-effort remainder of runtime teardown after the Control overlap gate has succeeded. */ - public void completeRuntimeReplacementShutdown() { - cancelCommunicationTests("Proxy runtime stopped before the backend replied"); - runCleanup("vote cache", () -> getVoteCacheHandler().saveVoteCache()); - runCleanup("proxy MySQL messenger", () -> { - if (getProxyMysqlMessenger() != null) getProxyMysqlMessenger().shutdown(); - }); - runCleanup("proxy MySQL", () -> { - if (getProxyMySQL() != null) getProxyMySQL().shutdown(); - }); - runCleanup("multi-proxy handler", () -> { - if (multiProxyHandler != null) multiProxyHandler.close(); - }); - runCleanup("socket listener", () -> { - if (socketHandler != null) socketHandler.closeConnection(); - }); - runCleanup("socket clients", this::closeSocketClients); - runCleanup("HTTP transport", this::closeHttpTransport); - runCleanup("Redis subscriber", () -> { - if (redisHandler != null) redisHandler.close(); - }); - runCleanup("Redis publisher", () -> { - JedisPool pool = redisPublisherPool; - try { - if (pool != null) pool.close(); - } finally { - if (redisPublisherPool == pool) redisPublisherPool = null; - } - }); - runCleanup("MQTT transport", () -> { - if (mqttHandler != null) mqttHandler.disconnect(); - }); - runCleanup("time checker", () -> bungeeTimeChecker.shutdown()); - runCleanup("global data", () -> { - if (getGlobalDataHandler() != null) getGlobalDataHandler().shutdown(); - }); - enabled = false; - } - - private void runCleanup(String service, CleanupAction cleanup) { - try { - cleanup.run(); - } catch (Exception failure) { - logSevere("Unable to stop " + service + "; remaining proxy cleanup will continue"); - } - } - - @FunctionalInterface - private interface CleanupAction { void run() throws Exception; } - - public void onPluginMessageReceived(DataInputStream in) { - onPluginMessageReceived(in, null); - } - - /** Receives a plugin message bound to the backend server connection that sent it. */ - public void onPluginMessageReceived(DataInputStream in, String sourceServer) { - runAsync(() -> { - try { - final String headerSub; - if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { - headerSub = encryptionHandler.decrypt(in.readUTF()); - } else { - headerSub = in.readUTF(); - } - - int size = in.readInt(); // sanity only - - if (getConfig().getDebug()) { - debug("Received plugin message header=" + headerSub + " size=" + size); - } - - String payload = ""; - if (size > 0) { - if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { - payload = encryptionHandler.decrypt(in.readUTF()); - } else { - payload = in.readUTF(); - } - } - - JsonEnvelope envelope = JsonEnvelopeCodec.decode(payload); - - if (!headerSub.equalsIgnoreCase(envelope.getSubChannel())) { - if (getConfig().getDebug()) { - warn("PluginMessage subChannel mismatch: header=" + headerSub + " env=" - + envelope.getSubChannel()); - } - return; - } - - if (VotingPluginWire.SUB_CONTROL_ENROLLMENT_REQUEST.equals(envelope.getSubChannel())) { - handleControlEnrollmentRequest(sourceServer, envelope); - return; - } - - globalMessageProxyHandler.onMessage(envelope); - } catch (Exception e) { - e.printStackTrace(); - } - }); - } - - private void handleControlEnrollmentRequest(String sourceServer, JsonEnvelope envelope) { - VotingPluginWire.ControlEnrollmentRequest request = VotingPluginWire.readControlEnrollmentRequest(envelope); - if (!request.valid || sourceServer == null || sourceServer.isBlank()) return; - if (!sourceServer.equals(request.nodeId)) { - sendPluginMessageServer(sourceServer, 0, - VotingPluginWire.controlEnrollmentResult(sourceServer, request.requestId, false)); - return; - } - long now = System.nanoTime(); - AtomicBoolean allowed = new AtomicBoolean(); - controlEnrollmentNextAllowed.compute(sourceServer, (ignored, nextAllowed) -> { - if (nextAllowed == null || now >= nextAllowed) { - allowed.set(true); - return now + CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS; - } - return nextAllowed; - }); - if (!allowed.get()) return; - HostedControlManager manager = hostedControlManager; - if (manager == null) { - sendPluginMessageServer(sourceServer, 0, - VotingPluginWire.controlEnrollmentResult(sourceServer, request.requestId, false)); - return; - } - manager.installNodeVerifier(sourceServer, request.verifier, request.endpoint).whenComplete((installed, failure) -> { - boolean success = failure == null && Boolean.TRUE.equals(installed); - sendPluginMessageServer(sourceServer, 0, - VotingPluginWire.controlEnrollmentResult(sourceServer, request.requestId, success)); - if (success) log("[Control] automatically enrolled backend node " + sourceServer); - }); - } - - private UUID parseUUIDFromString(String uuidAsString) { - String[] parts = { "0x" + uuidAsString.substring(0, 8), "0x" + uuidAsString.substring(8, 12), - "0x" + uuidAsString.substring(12, 16), "0x" + uuidAsString.substring(16, 20), - "0x" + uuidAsString.substring(20, 32) }; - - long mostSigBits = Long.decode(parts[0]).longValue(); - mostSigBits <<= 16; - mostSigBits |= Long.decode(parts[1]).longValue(); - mostSigBits <<= 16; - mostSigBits |= Long.decode(parts[2]).longValue(); - - long leastSigBits = Long.decode(parts[3]).longValue(); - leastSigBits <<= 48; - leastSigBits |= Long.decode(parts[4]).longValue(); - - return new UUID(mostSigBits, leastSigBits); - } - - public synchronized void processQueue() { - while (getVoteCacheHandler().getTimeChangeQueue().size() > 0) { - VoteTimeQueue vote = getVoteCacheHandler().getTimeChangeQueue().element(); - if (!vote.isProcessed()) { - VoteTotalsSnapshot queuedTotals = vote.getTotals() == null || vote.getTotals().isEmpty() ? null - : VoteTotalsSnapshot.parseStorage(vote.getTotals()); - QueuedVoteResult result = vote(vote.getName(), vote.getService(), true, false, vote.getTime(), queuedTotals, - vote.getUuid(), vote); - if (result == QueuedVoteResult.RETRY) { - scheduleTimeVoteRetry(); - return; - } - if (result == QueuedVoteResult.TERMINAL) { - warn("Removing terminal rollover vote " + vote.getVoteId() + " for " + vote.getName() + "/" - + ServiceSiteValidator.sanitizeForLog(vote.getService())); - } - } - if (!getVoteCacheHandler().removeTimeVote(vote)) { - scheduleTimeVoteRetry(); - return; - } - } - } - - private void scheduleTimeVoteRetry() { - if (timeVoteRetryScheduled || getScheduler() == null) { - return; - } - timeVoteRetryScheduled = true; - try { - getScheduler().schedule(() -> { - synchronized (VotingPluginProxy.this) { - timeVoteRetryScheduled = false; - } - processQueue(); - }, 5, TimeUnit.SECONDS); - } catch (RuntimeException e) { - timeVoteRetryScheduled = false; - debug("Unable to schedule rollover vote retry: " + e.getMessage()); - } - } - - public void reload() { - reloadRuntime(true); - } - - /** Applies a Control-originated configuration reload without stopping its connector or hosted service. */ - public void reloadFromControl() { - reloadRuntime(false); - } - - private void reloadRuntime(boolean restartControlServices) { - method = BungeeMethod.getByName(getConfig().getBungeeMethod()); - if (getMethod() == null) { - method = BungeeMethod.PLUGINMESSAGING; - } - warnUnsupportedDedicatedVotingProxyMode(); - if (!restartControlServices && method == BungeeMethod.SOCKETS) { - rebuildSocketClients(); - } - - setCurrentVotePartyVotesRequired( - getConfig().getVotePartyVotesRequired() + getVoteCacheVotePartyIncreaseVotesRequired()); - if (restartControlServices) { - loadMultiProxySupport(); - restartControlServicesAsync(); - } - } - - private synchronized void rebuildSocketClients() { - HashMap rebuilt = new HashMap<>(); - try { - List blocked = getConfig().getBlockedServers(); - for (String server : getConfig().getSpigotServers()) { - if (blocked.contains(server)) continue; - Map data = getConfig().getSpigotServerConfiguration(server); - String host = data.containsKey("Host") ? (String) data.get("Host") : ""; - int port = data.containsKey("Port") ? (int) data.get("Port") : 1298; - rebuilt.put(server, new ClientHandler(host, port, encryptionHandler, getConfig().getDebug())); - } - } catch (RuntimeException failure) { - stopSocketClients(rebuilt); - throw failure; - } - HashMap previous = clientHandles; - clientHandles = rebuilt; - stopSocketClients(previous); - } - - private synchronized boolean sendSocketEnvelope(String server, JsonEnvelope envelope) { - ClientHandler socketClient = clientHandles == null ? null : clientHandles.get(server); - if (socketClient == null) return false; - try { - socketClient.sendEnvelope(envelope); - return true; - } catch (RuntimeException e) { - debug(e.getMessage()); - return false; - } - } - - private synchronized boolean sendHttpEnvelope(String server, JsonEnvelope envelope) { - HttpProxyTransportServer transport = httpTransportServer; - return transport != null && transport.send(server, envelope); - } - - private void startHttpTransport() { - try { - URI endpoint = URI.create(getConfig().getHttpPublicEndpoint()); - if (!"https".equalsIgnoreCase(endpoint.getScheme()) || endpoint.getHost() == null - || endpoint.getPort() == 0 || endpoint.getPort() > 65535 - || endpoint.getUserInfo() != null || endpoint.getQuery() != null || endpoint.getFragment() != null - || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) { - throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); - } - File directory = new File(getDataFolderPlugin(), "http"); - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.toPath(), endpoint.getHost()); - httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); - httpTransportServer = new HttpProxyTransportServer( - new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, - httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope); - httpTransportServer.start(); - logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" - + httpTransportServer.port() + "; use /votingpluginbungee httpcode for each backend"); - } catch (Exception failure) { - closeHttpTransport(); - throw new IllegalStateException("HTTP transport could not start securely", failure); - } - } - - /** Keeps the authenticated mTLS backend identity attached to security-sensitive proxy routing. */ - protected void handleHttpTransportEnvelope(HttpProxyTransportServer.ReceivedEnvelope received) { - if (!isAuthenticatedHttpEnvelopeAllowed(received)) { - debug("Ignored HTTP envelope whose player-presence claim did not match its authenticated backend"); - return; - } - GlobalMessageProxyHandler handler = globalMessageProxyHandler; - if (handler == null) throw new IllegalStateException("HTTP message router is not ready"); - handler.onMessage(received.envelope()); - } - - private boolean isAuthenticatedHttpEnvelopeAllowed(HttpProxyTransportServer.ReceivedEnvelope received) { - if (received == null || received.envelope() == null || received.serverId() == null) return false; - String stampedServer = received.envelope().getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - if (!received.serverId().equalsIgnoreCase(stampedServer)) return false; - if (!VotingPluginWire.SUB_LOGIN.equals(received.envelope().getSubChannel())) return true; - VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(received.envelope()); - boolean modern = event.connectionId != null || event.backendIncarnationId != null - || event.backendStartedAt != 0L || event.presenceTimestamp != 0L; - if (!modern || isDedicatedVotingProxyEnabled()) return true; - // A player-facing proxy has a stronger authority than any backend: its live - // player connection supplies both the current route and (in online mode) UUID. - return isLegacyLoginDestinationAuthoritative(event.player, event.uuid, received.serverId()); - } - - private synchronized void closeHttpTransport() { - HttpProxyTransportServer transport = httpTransportServer; - httpTransportServer = null; - httpEnrollmentAuthority = null; - if (transport != null) transport.close(); - } - - public String createHttpConnectionCode(String serverId) { - HttpEnrollmentAuthority authority = httpEnrollmentAuthority; - if (method != BungeeMethod.HTTP || authority == null) { - throw new IllegalStateException("The HTTP transport is not running"); - } - return authority.createConnectionCode(serverId, URI.create(getConfig().getHttpPublicEndpoint()), Duration.ofMinutes(15)) - .encode(); - } - - public void revokeHttpBackend(String serverId) { - HttpEnrollmentAuthority authority = httpEnrollmentAuthority; - if (method != BungeeMethod.HTTP || authority == null) throw new IllegalStateException("The HTTP transport is not running"); - authority.revoke(HttpTlsIdentity.canonicalServerId(serverId)); - } - - private synchronized void closeSocketClients() { - HashMap clients = clientHandles; - clientHandles = null; - stopSocketClients(clients); - } - - static void stopSocketClients(Map clients) { - if (clients == null) return; - for (ClientHandler client : clients.values()) { - if (client == null) continue; - try { - client.stopConnection(); - } catch (RuntimeException ignored) { - // Best effort: one broken client must not prevent the remaining sockets from closing. - } - } - } - - private void warnUnsupportedDedicatedVotingProxyMode() { - if (getConfig().getDedicatedVotingProxy() && (method == null || !method.supportsBackendPresence())) { - logSevere("DedicatedVotingProxy requires MYSQL, REDIS, MQTT, SOCKETS, or HTTP; PLUGINMESSAGING is disabled for " - + "dedicated-proxy routing. Falling back to normal proxy routing."); - } - } - - public abstract void runAsync(Runnable run); - - /** Platform name used only for the transport-neutral Control discovery contract. */ - public abstract String getProxyPlatform(); - - public abstract void runConsoleCommand(String command); - - public abstract void saveVoteCacheFile(); - - public abstract void reloadCore(boolean mysql); - - /** Strict Control reload path; failures propagate so the caller can restore its backup. */ - public abstract void reloadControlConfiguration() throws Exception; - - public abstract boolean sendPluginMessageData(String server, String channel, byte[] data, boolean queue); - - private static final int PLUGIN_MESSAGE_HARD_LIMIT = 32767; - private static final int PLUGIN_MESSAGE_SOFT_LIMIT = 30000; - - public void sendPluginMessageServer(String server, int delay, JsonEnvelope envelope) { - getScheduler().schedule(() -> sendPluginMessageServerNow(server, envelope), delay * 5L, TimeUnit.MILLISECONDS); - } - - /** - * Sends a plugin-message envelope immediately and reports whether the proxy - * accepted it for delivery. - * - * @param server target backend server - * @param envelope envelope to send - * @return true when the proxy accepted the message for delivery - */ - protected boolean sendPluginMessageServerNow(String server, JsonEnvelope envelope) { - final String subChannel = envelope.getSubChannel(); - final String payload = JsonEnvelopeCodec.encode(envelope); - - final byte[] subChannelBytes = subChannel.getBytes(java.nio.charset.StandardCharsets.UTF_8); - final byte[] payloadBytes = payload.getBytes(java.nio.charset.StandardCharsets.UTF_8); - - // Estimate bytes written: - // - writeUTF adds 2-byte length prefix + UTF-8 bytes - // - writeInt is 4 bytes - int estimatedSize = 2 + subChannelBytes.length + // subChannel UTF (len prefix + bytes) - 4 + // payload length int - 2 + payloadBytes.length; // payload UTF (len prefix + bytes) - - if (estimatedSize > PLUGIN_MESSAGE_SOFT_LIMIT) { - debug("[PluginMessage] Payload nearing limit (" + estimatedSize + " bytes) server=" + server - + " subChannel=" + subChannel + " — consider Redis instead"); - } - - if (estimatedSize > PLUGIN_MESSAGE_HARD_LIMIT) { - debug("[PluginMessage] Payload TOO LARGE (" + estimatedSize + " bytes, max=" + PLUGIN_MESSAGE_HARD_LIMIT - + ") server=" + server + " subChannel=" + subChannel + " — NOT sent"); - return false; - } - - try (ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(); - DataOutputStream out = new DataOutputStream(byteOutStream)) { - if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { - out.writeUTF(encryptionHandler.encrypt(subChannel)); - } else { - out.writeUTF(subChannel); - } - - // sanity only: MUST be bytes, not chars - out.writeInt(payloadBytes.length); - - if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { - out.writeUTF(encryptionHandler.encrypt(payload)); - } else { - out.writeUTF(payload); - } - out.flush(); - - boolean sent = sendPluginMessageData(server, getConfig().getPluginMessageChannel().toLowerCase(), - byteOutStream.toByteArray(), false); - if (getConfig().getDebug()) { - debug((sent ? "Sent" : "Could not send") + " plugin envelope (" + estimatedSize + " bytes) " + server - + " " + subChannel + " " + envelope.getFields()); - } - return sent; - } catch (Exception e) { - e.printStackTrace(); - return false; - } - } - - static DefaultJedisClientConfig buildRedisClientConfig(VotingPluginProxyConfig configSource) { - DefaultJedisClientConfig.Builder config = DefaultJedisClientConfig.builder() - .database(configSource.getRedisDbIndex()).ssl(configSource.getRedisSsl()).connectionTimeoutMillis(2000) - .socketTimeoutMillis(2000); - if (configSource.getRedisSsl()) { - SSLParameters sslParameters = new SSLParameters(); - sslParameters.setEndpointIdentificationAlgorithm("HTTPS"); - config.sslParameters(sslParameters); - } - if (configSource.getRedisUsername() != null && !configSource.getRedisUsername().isEmpty()) { - config.user(configSource.getRedisUsername()); - } - if (configSource.getRedisPassword() != null && !configSource.getRedisPassword().isEmpty()) { - config.password(configSource.getRedisPassword()); - } - return config.build(); - } - - public boolean sendRedisEnvelopeServer(String server, JsonEnvelope envelope) { - return sendRedisEnvelopeServer(server, envelope, false); - } - - private boolean sendRedisEnvelopeServer(String server, JsonEnvelope envelope, boolean useRetryCooldown) { - JedisPool publisherPool = redisPublisherPool; - if (publisherPool == null || (useRetryCooldown && System.currentTimeMillis() < redisPublisherRetryAfter)) { - return false; - } - - try (Jedis jedis = publisherPool.getResource()) { - String channel = getConfig().getRedisPrefix() + "VotingPlugin_" + server; - long subscribers = jedis.publish(channel, - JsonEnvelopeCodec.encode(VotingPluginWire.withRedisDeliveryId(envelope))); - redisPublisherRetryAfter = 0L; - return subscribers > 0; - } catch (Exception e) { - if (useRetryCooldown) { - // Standalone broadcasts remain queued, so their retries can be throttled safely. - redisPublisherRetryAfter = System.currentTimeMillis() + 2000L; - } - debug(e.getMessage()); - return false; - } - } - - public boolean sendMqttEnvelopeServer(String server, JsonEnvelope envelope) { - if (mqttHandler == null) { - return false; - } - try { - mqttHandler.publishEnvelope(getConfig().getMqttPrefix() + "votingplugin/servers/" + server, envelope); - return true; - } catch (Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - return false; - } - } - - public boolean sendSocketEnvelopeServer(String server, JsonEnvelope envelope) { - Map configuration = getConfig().getSpigotServerConfiguration(server); - if (configuration == null) { - return false; - } - String host = configuration.get("Host") instanceof String ? (String) configuration.get("Host") : ""; - int port = configuration.get("Port") instanceof Number ? ((Number) configuration.get("Port")).intValue() : 1298; - if (host.isEmpty()) { - return false; - } - - String payload = JsonEnvelopeCodec.encode(envelope); - String encoded = encryptionHandler != null ? encryptionHandler.encrypt(payload) : payload; - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(host, port), 2000); - try (DataOutputStream output = new DataOutputStream(socket.getOutputStream())) { - output.writeUTF(encoded); - output.flush(); - } - return true; - } catch (Exception e) { - debug(e.getMessage()); - return false; - } - } - - public void sendServerNameMessage() { - for (String s : getAllAvailableServers()) { - sendPluginMessageServer(s, 1, VotingPluginWire.serverName(s)); - } - } - - public void sendVoteParty(String server) { - if (isSomeoneOnlineServerForVoteRouting(server)) { - globalMessageProxyHandler.sendMessage(server, 1, VotingPluginWire.votePartyBungee()); - } - } - - public void setCurrentVotePartyVotes(int amount) { - votePartyVotes = amount; - setVoteCacheVotePartyCurrentVotes(amount); - debug("Current vote party total: " + votePartyVotes); - } - - public abstract void setVoteCacheLastUpdated(); - - public abstract void setVoteCachePrevDay(int day); - - public abstract void setVoteCachePrevMonth(String text); - - public abstract void setVoteCachePrevWeek(int week); - - public abstract void setVoteCacheVoteCacheIgnoreTime(boolean ignore); - - public abstract void setVoteCacheVotePartyCurrentVotes(int votes); - - public abstract void setVoteCacheVotePartyIncreaseVotesRequired(int votes); - - public void status() { - for (String s : getAllAvailableServers()) { - if (!isSomeoneOnlineServerForVoteRouting(s)) { - log("No players on server " + s + " to send test status message, please retest with someone online"); - } else { - log("Sending request for status message on " + s); - globalMessageProxyHandler.sendMessage(s, 1, VotingPluginWire.status(s)); - } - } - } - - /** Runs a correlated, non-vote round trip over the active backend transport. */ - public CompletableFuture testBackendCommunication(String requestedServer, - long timeoutMillis) { - String server = requestedServer == null ? "" : requestedServer.trim(); - BungeeMethod activeMethod = method; - if (server.isEmpty() || !getAllAvailableServers().contains(server)) { - return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, - "UNKNOWN_BACKEND", "The backend is not configured on this proxy")); - } - if (activeMethod == null || globalMessageProxyHandler == null) { - return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, - "TRANSPORT_UNAVAILABLE", "The proxy communication transport is not running")); - } - if (activeMethod == BungeeMethod.PLUGINMESSAGING && !isSomeoneOnlineServerForVoteRouting(server)) { - return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, - "PLAYER_REQUIRED", "Plugin messaging requires an online player on the selected backend")); - } - ScheduledExecutorService scheduler = getScheduler(); - if (scheduler == null) { - return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, - "TRANSPORT_UNAVAILABLE", "The proxy scheduler is not running")); - } - long boundedTimeout = Math.max(500L, Math.min(timeoutMillis, 30000L)); - UUID requestId = UUID.randomUUID(); - CompletableFuture result = new CompletableFuture<>(); - PendingCommunicationTest pending = new PendingCommunicationTest(server, activeMethod, System.nanoTime(), result); - pendingCommunicationTests.put(requestId, pending); - result.whenComplete((ignored, failure) -> pendingCommunicationTests.remove(requestId, pending)); - try { - if (!sendCommunicationTestEnvelopeNow(server, VotingPluginWire.status(server, requestId))) { - result.complete(CommunicationTestResult.failure(server, activeMethod, "TRANSPORT_UNAVAILABLE", - "The active transport could not accept the communication test")); - return result; - } - scheduler.schedule(() -> result.complete(CommunicationTestResult.failure(server, activeMethod, - "TIMEOUT", "No correlated reply arrived before the timeout")), boundedTimeout, TimeUnit.MILLISECONDS); - } catch (RuntimeException failure) { - result.complete(CommunicationTestResult.failure(server, activeMethod, "SEND_FAILED", - "The proxy could not send the communication test")); - } - return result; - } - - /** Sends a diagnostic immediately and reports whether the active transport accepted it. */ - protected boolean sendCommunicationTestEnvelopeNow(String server, JsonEnvelope envelope) { - return sendProxyBroadcastEnvelopeNow(server, envelope); - } - - protected void handleStatusOkay(JsonEnvelope message) { - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - String request = message.getFields().getOrDefault(VotingPluginWire.K_REQUEST_ID, ""); - if (request.isEmpty()) { - log("Status okay for " + server); - return; - } - UUID requestId; - try { - requestId = UUID.fromString(request); - } catch (IllegalArgumentException ignored) { - debug("Ignored status reply with an invalid request ID from " + server); - return; - } - PendingCommunicationTest pending = pendingCommunicationTests.get(requestId); - if (pending == null || !pending.server().equals(server)) { - debug("Ignored unexpected status reply from " + server); - return; - } - long roundTripMillis = Math.max(0L, - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - pending.startedAtNanos())); - pending.result().complete(CommunicationTestResult.success(server, pending.method(), roundTripMillis)); - } - - private void cancelCommunicationTests(String message) { - pendingCommunicationTests.forEach((requestId, pending) -> pending.result().complete( - CommunicationTestResult.failure(pending.server(), pending.method(), "TRANSPORT_STOPPED", message))); - pendingCommunicationTests.clear(); - } - - public record CommunicationTestResult(boolean success, String code, String message, String server, - String method, long roundTripMillis) { - private static CommunicationTestResult success(String server, BungeeMethod method, long roundTripMillis) { - return new CommunicationTestResult(true, "OK", "Backend replied over the active transport", server, - method == null ? "" : method.name(), roundTripMillis); - } - - private static CommunicationTestResult failure(String server, BungeeMethod method, String code, String message) { - return new CommunicationTestResult(false, code, message, server, - method == null ? "" : method.name(), -1L); - } - } - - private record PendingCommunicationTest(String server, BungeeMethod method, long startedAtNanos, - CompletableFuture result) { } - - private void sendVoteDelayRejected(String player, String uuid, String service, boolean playerOnline, - String playerServer) { - if (!playerOnline || playerServer == null || !getAllAvailableServers().contains(playerServer)) { - debug("Not sending vote delay rejection for " + player + " because the player is offline"); - return; - } - - globalMessageProxyHandler.sendMessage(playerServer, 1, - VotingPluginWire.voteDelayRejected(player, uuid, service, true)); - } - - public String getWaitUntilDelaySiteFromService(String service) { - for (String site : getConfig().getWaitUntilVoteDelaySites()) { - if (getConfig().getWaitUntilVoteDelayService(site).equalsIgnoreCase(service)) { - return site; - } - } - return ""; - } - - private long getLastVotesTime(String uuid, ArrayList cols, String site, String service, String player, - boolean includeTimeChangeQueue) { - long mostRecentTime = 0; - - if (getVoteCacheHandler().hasOnlineVotes(uuid)) { - ArrayList onlineVotes = getVoteCacheHandler().getOnlineVotes(uuid); - for (OfflineBungeeVote vote : onlineVotes) { - if (vote.getService().equalsIgnoreCase(service)) { - mostRecentTime = Math.max(mostRecentTime, vote.getTime()); - } - } - } - - for (String server : getAllAvailableServers()) { - for (OfflineBungeeVote vote : getVoteCacheHandler().getVotes(server)) { - if (vote.getUuid().equals(uuid) && vote.getService().equalsIgnoreCase(service)) { - mostRecentTime = Math.max(mostRecentTime, vote.getTime()); - } - } - } - - if (includeTimeChangeQueue && player != null) { - for (VoteTimeQueue queuedVote : getVoteCacheHandler().getTimeChangeQueue()) { - if (queuedVote.getName().equalsIgnoreCase(player) - && queuedVote.getService().equalsIgnoreCase(service)) { - mostRecentTime = Math.max(mostRecentTime, queuedVote.getTime()); - } - } - } - - for (Column d : cols) { - if (d.getName().equalsIgnoreCase("LastVotes")) { - DataValue value = d.getValue(); - String[] list = value.getString().split("%line%"); - for (String str : list) { - String[] data = str.split("//"); - if (data[0].equalsIgnoreCase(site)) { - mostRecentTime = Math.max(mostRecentTime, Long.valueOf(data[1])); - } - } - } - } - return mostRecentTime; - } - - public boolean checkVoteDelay(String uuid, String service, ArrayList data) { - return checkVoteDelay(uuid, null, service, data, false); - } - - /** - * Checks the configured vote delay, optionally including accepted votes waiting - * for a GlobalData time change to finish. - * - * @param uuid player UUID - * @param player player name used by the time-change queue - * @param service vote service - * @param data current player data - * @param includeTimeChangeQueue whether queued votes reserve their delay slot - * @return true when the vote may be accepted - */ - public boolean checkVoteDelay(String uuid, String player, String service, ArrayList data, - boolean includeTimeChangeQueue) { - String site = getWaitUntilDelaySiteFromService(service); - if (site.isEmpty()) { - debug("No service site set for " + service + ", skipping vote delay check"); - return true; - } - - int voteDelay = getConfig().getWaitUntilVoteDelayVoteDelay(site); - int voteDelayMin = getConfig().getWaitUntilVoteDelayVoteDelayMin(site); - - long lastVote = getLastVotesTime(uuid, data, site, service, player, includeTimeChangeQueue); - if (lastVote == 0) { - debug("No last vote time found for " + uuid + "/" + service + ", skipping vote delay check"); - return true; - } - - try { - LocalDateTime now = getBungeeTimeChecker().getTime(); - LocalDateTime lastVoteTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastVote), ZoneId.systemDefault()) - .plusHours(getConfig().getTimeHourOffSet()); - - if (!getConfig().getWaitUntilVoteDelayVoteDelayDaily(site)) { - if (voteDelay == 0 && voteDelayMin == 0) { - debug("Vote delay is 0 for " + site + ", skipping vote delay check"); - return true; - } - - LocalDateTime nextvote = lastVoteTime.plusHours((long) voteDelay).plusMinutes((long) voteDelayMin); - return now.isAfter(nextvote); - } - LocalDateTime resetTime = lastVoteTime.withHour(getConfig().getWaitUntilVoteDelayVoteDelayHour(site)) - .withMinute(0).withSecond(0); - LocalDateTime resetTimeTomorrow = resetTime.plusHours(24); - - if (lastVoteTime.isBefore(resetTime)) { - if (now.isAfter(resetTime)) { - debug("Vote delay is met for " + uuid + "/" + service + ", vote can be processed"); - return true; - } - } else { - if (now.isAfter(resetTimeTomorrow)) { - debug("Vote delay is met for " + uuid + "/" + service + ", vote can be processed"); - return true; - } - } - } catch (Exception e) { - e.printStackTrace(); - } - - debug("Vote delay is not met for " + uuid + "/" + service + ", skipping vote"); - return false; - } - - public synchronized void vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, - VoteTotalsSnapshot text, String uuid) { - vote(player, service, realVote, timeQueue, queueTime, text, uuid, null); - } - - private enum QueuedVoteResult { - SUCCESS, RETRY, TERMINAL - } - - private synchronized QueuedVoteResult vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, - VoteTotalsSnapshot text, String uuid, VoteTimeQueue queuedVote) { - try { - if (!ServiceSiteValidator.isValid(service)) { - warn("Rejected vote with invalid service site '" + ServiceSiteValidator.sanitizeForLog(service) + "'"); - return QueuedVoteResult.TERMINAL; - } - if (!MinecraftUsernameValidator.isValid(player, getConfig().getBedrockPlayerPrefix())) { - warn("Rejected vote with invalid Minecraft username '" - + MinecraftUsernameValidator.sanitizeForLog(player) + "' from service '" - + MinecraftUsernameValidator.sanitizeForLog(service) + "'"); - return QueuedVoteResult.TERMINAL; - } - - UUID voteId = queuedVote == null ? null : queuedVote.getVoteId(); - if (voteId == null) { - voteId = UUID.randomUUID(); - } - - // UUID resolution - if (!getConfig().getOnlineMode()) { - uuid = getUUID(player); - } - - if (uuid == null || uuid.isEmpty()) { - uuid = getUUID(player); - - // Bedrock prefix auto-detect - if (uuid.isEmpty() && !getConfig().getBedrockPlayerPrefix().isEmpty() - && !player.startsWith(getConfig().getBedrockPlayerPrefix())) { - String uuid1 = getUUID(getConfig().getBedrockPlayerPrefix() + player); - if (!uuid1.isEmpty()) { - debug("Detected bedrock player without prefix, adjusting..."); - player = getConfig().getBedrockPlayerPrefix() + player; - uuid = uuid1; - } - } - } - - if (uuid.isEmpty()) { - if (player.startsWith(getConfig().getBedrockPlayerPrefix())) { - log("Ignoring vote since unable to get UUID of bedrock player"); - return QueuedVoteResult.TERMINAL; - } - if (!getConfig().getAllowUnJoined()) { - log("Ignoring vote from " + player + " since player hasn't joined before"); - return QueuedVoteResult.TERMINAL; - } - if (!getConfig().getUUIDLookup()) { - log("Failed to get uuid for " + player); - return QueuedVoteResult.TERMINAL; - } - - debug("Fetching UUID online, since allowunjoined is enabled"); - UUID u = null; - try { - if (getConfig().getOnlineMode()) { - u = fetchUUID(player); - } - } catch (Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - if (u == null) { - debug("Failed to get uuid for " + player); - return QueuedVoteResult.TERMINAL; - } - uuid = u.toString(); - } - - // Normalize UUID string if possible - try { - if (uuid != null && !uuid.isEmpty() && !uuid.equalsIgnoreCase("null")) { - uuid = UUID.fromString(uuid.trim()).toString(); - } - } catch (Exception ignored) { - // ignore - } - - player = getProperName(uuid, player); - - // Cache online state/server once (IMPORTANT for broadcast logic correctness) - final boolean playerOnline = isPlayerOnlineForVoteRouting(player); - final String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(player) : null; - long time = queueTime != 0 ? queueTime - : LocalDateTime.now().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); - - Set broadcastTargets = queuedVote == null ? new LinkedHashSet<>() - : new LinkedHashSet<>(queuedVote.getBroadcastTargets()); - Set broadcastForwardedServers = queuedVote == null ? new LinkedHashSet<>() - : new LinkedHashSet<>(queuedVote.getBroadcastForwardedServers()); - boolean proxyBroadcastHandled = queuedVote != null && queuedVote.isProxyBroadcastHandled(); - boolean processesTotals = getConfig().getPrimaryServer() || !getConfig().getMultiProxySupport(); - boolean managesTotals = processesTotals && getConfig().getBungeeManageTotals(); - boolean canValidateStandaloneBroadcast = canForwardStandaloneBroadcast(managesTotals); - ArrayList data = null; - boolean queueForTimeChange = false; - - // A completion callback can wipe totals and replay older queued votes. Run it - // before loading this vote's database snapshot so the calculations below use - // the post-rollover state. - if (getConfig().getGlobalDataEnabled() && getGlobalDataHandler().isTimeChangedHappened()) { - getGlobalDataHandler().checkForFinishedTimeChanges(); - queueForTimeChange = timeQueue && getGlobalDataHandler().isTimeChangedHappened(); - } - - // Validate the vote before any immediate announcement. This keeps duplicate - // votes rejected by the delay check out of the GlobalData rollover queue and - // prevents announcing a vote that will not be processed. - if (managesTotals) { - if (getProxyMySQL() == null) { - logSevere("Mysql is not loaded correctly, stopping vote processing"); - return QueuedVoteResult.RETRY; - } - - if (!getProxyMySQL().containsKeyQuery(uuid)) { - getProxyMySQL().update(uuid, "PlayerName", new DataValueString(player)); - getProxyMySQL().getUuids().add(uuid); - } - - data = getProxyMySQL().getExactQuery(new Column("uuid", new DataValueString(uuid))); - if (!checkVoteDelay(uuid, player, service, data, queuedVote == null)) { - log("Vote delay is not met for " + player + "/" + service + ", skipping vote"); - sendVoteDelayRejected(player, uuid, service, playerOnline, playerServer); - return QueuedVoteResult.TERMINAL; - } - } - - // Forward an accepted offline broadcast before the still-active GlobalData - // change queues the reward/totals work. The queued delivery state prevents - // replaying broadcasts that already reached a backend. - if (queueForTimeChange) { - VoteTotalsSnapshot projectedTotals = managesTotals ? getProjectedRolloverTotals(data, player) : text; - if (canValidateStandaloneBroadcast && proxyBroadcastDecider.usesImmediateForwarding(playerOnline)) { - broadcastTargets.addAll(proxyBroadcastDecider.resolveTargets(false, null)); - proxyBroadcastHandled = true; - } - VoteTimeQueue delayedVote = new VoteTimeQueue(voteId, player, service, time, - proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, - projectedTotals == null ? "" : projectedTotals.toString(), false, uuid); - if (!getVoteCacheHandler().addTimeVoteToCache(delayedVote)) { - logSevere("Unable to persist queued rollover vote for " + player + "/" + service - + "; skipping proxy broadcast"); - return QueuedVoteResult.RETRY; - } - if (proxyBroadcastHandled) { - for (String target : broadcastTargets) { - Set forwarded = sendProxyBroadcast(Collections.singleton(target), uuid, player, - service, time, projectedTotals == null ? "" : projectedTotals.toString(), false); - if (delayedVote.getBroadcastForwardedServers().addAll(forwarded)) { - broadcastForwardedServers.addAll(forwarded); - persistTimeVoteDelivery(delayedVote); - } - } - } - log("Caching vote from " + player + "/" + service - + " because time change is happening right now"); - return QueuedVoteResult.SUCCESS; - } - - addVoteParty(); - - // Totals processing (primary server OR no multiproxy) - if (processesTotals) { - if (managesTotals) { - int allTimeTotal = getValue(data, "AllTimeTotal", 1); - int monthTotal = getValue(data, "MonthTotal", 1); - - int dateMonthTotal = -1; - if (getConfig().getStoreMonthTotalsWithDate()) { - if (getConfig().getUseMonthDateTotalsAsPrimaryTotal()) { - dateMonthTotal = getValue(data, getMonthTotalsWithDatePath(), 1); - } else { - dateMonthTotal = monthTotal; - } - } - - int weeklyTotal = getValue(data, "WeeklyTotal", 1); - int dailyTotal = getValue(data, "DailyTotal", 1); - int points = getValue(data, "Points", getConfig().getPointsOnVote()); - - int maxVotes = getConfig().getMaxAmountOfVotesPerDay(); - if (maxVotes > 0) { - LocalDateTime cTime = getBungeeTimeChecker().getTime(); - int days = cTime.getDayOfMonth(); - if (monthTotal > days * maxVotes) { - monthTotal = days * maxVotes; - } - } - - if (getConfig().getLimitVotePoints() > 0 && points > getConfig().getLimitVotePoints()) { - points = getConfig().getLimitVotePoints(); - } - - text = new VoteTotalsSnapshot(allTimeTotal, monthTotal, weeklyTotal, dailyTotal, points, - votePartyVotes, currentVotePartyVotesRequired, dateMonthTotal); - - ArrayList update = new ArrayList<>(); - update.add(new Column("AllTimeTotal", new DataValueInt(allTimeTotal))); - update.add(new Column("MonthTotal", new DataValueInt(monthTotal))); - if (getConfig().getStoreMonthTotalsWithDate()) { - update.add(new Column(getMonthTotalsWithDatePath(), new DataValueInt(dateMonthTotal))); - } - update.add(new Column("WeeklyTotal", new DataValueInt(weeklyTotal))); - update.add(new Column("DailyTotal", new DataValueInt(dailyTotal))); - update.add(new Column("Points", new DataValueInt(points))); - - debug("Setting totals " + text.toString() + ", voteId=" + voteId + " for " + player + "/" - + service); - getProxyMySQL().update(uuid, update); - } else { - text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); - } - } - if (text == null) { - text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); - } - - VoteLogStatus voteStatus = VoteLogStatus.IMMEDIATE; - boolean standaloneProxyBroadcast = canValidateStandaloneBroadcast && (proxyBroadcastHandled - || proxyBroadcastDecider.usesImmediateForwarding(playerOnline)); - Set proxyBroadcastTargets = Collections.emptySet(); - if (standaloneProxyBroadcast) { - // A handled queued broadcast was necessarily sampled while the player was - // offline. Retry only targets that did not previously accept delivery. - proxyBroadcastTargets = proxyBroadcastHandled ? new LinkedHashSet<>(broadcastTargets) - : proxyBroadcastDecider.resolveTargets(false, null); - Set remainingTargets = new LinkedHashSet<>(proxyBroadcastTargets); - remainingTargets.removeAll(broadcastForwardedServers); - broadcastForwardedServers.addAll(sendProxyBroadcast(remainingTargets, uuid, player, service, time, - text == null ? "" : text.toString(), false)); - } - - // =========================== - // Send vote(s) to backend(s) - // =========================== - if (getConfig().getSendVotesToAllServers()) { - for (String s : getAllAvailableServers()) { - - boolean forceCache = getConfig().getWaitForUserOnline() - && (!playerOnline || playerServer == null || !playerServer.equalsIgnoreCase(s)); - - if (forceCache) { - debug("Forcing vote to cache for server " + s); - } - - if ((!isSomeoneOnlineServerForVoteRouting(s) && method.requiresPlayerOnline()) || forceCache) { - voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addServerVote(s, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, - text.toString(), broadcastForwarded, standaloneProxyBroadcast, - proxyBroadcastTargets, broadcastForwardedServers, false)); - debug("Caching vote for " + player + " on " + service + " for " + s); - } else { - boolean broadcastHere = !broadcastForwardedServers.contains(s); - if (broadcastHere && getConfig().getProxyBroadcastEnabled()) { - Set targets = standaloneProxyBroadcast ? proxyBroadcastTargets - : proxyBroadcastDecider.resolveTargets(playerOnline, playerServer); - broadcastHere = proxyBroadcastDecider.shouldBroadcast(s, targets); - } - - if (!sendVoteEnvelopeAccepted(s, 2, - VotingPluginWire.vote(player, uuid, service, time, true, realVote, text.toString(), - voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1))) { - voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addServerVote(s, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, - text.toString(), broadcastForwarded, standaloneProxyBroadcast, - proxyBroadcastTargets, broadcastForwardedServers, false)); - debug("Caching vote after the transport rejected delivery for " + s); - } - } - } - } else { - // Single-server mode: online goes to player server; otherwise queue as "online - // vote" - if (playerOnline && playerServer != null && getAllAvailableServers().contains(playerServer)) { - String server = playerServer; - - boolean broadcastHere = !broadcastForwardedServers.contains(server); - if (broadcastHere && getConfig().getProxyBroadcastEnabled()) { - Set targets = standaloneProxyBroadcast ? proxyBroadcastTargets - : proxyBroadcastDecider.resolveTargets(true, playerServer); - broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); - } - - boolean rewardAccepted = sendVoteEnvelopeAccepted(server, 1, - VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(), - voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1)); - if (!rewardAccepted) { - voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addOnlineVote(uuid, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), - broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, - broadcastForwardedServers, false)); - debug("Caching online vote after the transport rejected delivery for " + server); - } - - if (rewardAccepted && canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled() - && !standaloneProxyBroadcast) { - Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); - - int bDelay = 2; - for (String targetServer : targets) { - // avoid double-broadcast on the same server that already got the voteOnline - if (targetServer.equalsIgnoreCase(server)) { - continue; - } - if (getConfig().getBlockedServers().contains(targetServer)) { - continue; - } - - globalMessageProxyHandler.sendMessage(targetServer, bDelay, - VotingPluginWire.voteBroadcast(uuid, player, service, time, - text == null ? "" : text.toString(), true)); - bDelay++; - } - } - - // multiproxy: envelope-only clear vote - if (rewardAccepted && getConfig().getMultiProxySupport() && getConfig().getMultiProxyOneGlobalReward()) { - multiProxyHandler.sendClearVote(uuid, player); - } - } else { - voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addOnlineVote(uuid, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), - broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, - broadcastForwardedServers, false)); - debug("Caching online vote for " + player + " on " + service); - } - - int delay = 2; - for (String s : getAllAvailableServers()) { - globalMessageProxyHandler.sendMessage(s, delay + 1, VotingPluginWire.voteUpdate(uuid, - votePartyVotes, currentVotePartyVotesRequired, service, time, text.toString())); - delay += 2; - } - } - - // Vote logging - if (voteLogMysqlTable != null && getConfig().getVoteLoggingEnabled()) { - voteLogMysqlTable.logVote(voteId, voteStatus, service, uuid, player, time, - getVoteCacheHandler().getProxyCachedTotal(uuid)); - } - - // =========================== - // Multiproxy forwarding - // =========================== - if (getConfig().getMultiProxySupport() && getConfig().getPrimaryServer()) { - if (!getConfig().getMultiProxyOneGlobalReward()) { - debug("Sending global proxy vote envelope"); - multiProxyHandler.sendMultiProxyEnvelope(VotingPluginWire.vote(player, uuid, service, time, false, - realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); - } else { - // Only send to other proxies if the player DID NOT already receive reward on a - // backend - boolean shouldSend = true; - if (playerOnline && playerServer != null) { - if (!getConfig().getBlockedServers().contains(playerServer)) { - shouldSend = false; - } - } - - if (shouldSend) { - debug("Sending global proxy voteonline envelope"); - multiProxyHandler - .sendMultiProxyEnvelope(VotingPluginWire.voteOnline(player, uuid, service, time, false, - realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); - } else { - debug("Not sending global proxy message for voteonline, player already got reward"); - } - } - } - if (queuedVote != null) { - queuedVote.setProcessed(true); - if (!getVoteCacheHandler().updateTimeVote(queuedVote)) { - warn("Unable to persist completed rollover vote " + queuedVote.getVoteId() - + "; attempting durable removal immediately"); - } - } - return QueuedVoteResult.SUCCESS; - } catch (Exception e) { - e.printStackTrace(); - return QueuedVoteResult.RETRY; - } - } - - private static final class PendingPresenceHandoff { - private UUID requestId; - private final UUID playerUuid; - private final String playerName; - private final String uuid; - private final String server; - private final UUID connectionId; - private final UUID backendIncarnationId; - private final long backendStartedAt; - private final long conflictSequence; - private final long createdAt; - - private PendingPresenceHandoff(String playerName, String uuid, String server, UUID connectionId, - UUID backendIncarnationId, long backendStartedAt, long conflictSequence, long createdAt) { - this.playerUuid = parsePlayerUuid(uuid); - this.playerName = playerName; - this.uuid = uuid; - this.server = server; - this.connectionId = connectionId; - this.backendIncarnationId = backendIncarnationId; - this.backendStartedAt = backendStartedAt; - this.conflictSequence = conflictSequence; - this.createdAt = createdAt; - } - - private static UUID parsePlayerUuid(String uuid) { - try { - return UUID.fromString(uuid.trim()); - } catch (Exception ignored) { - return null; - } - } - } - - public abstract void warn(String message); - - public abstract ScheduledExecutorService getScheduler(); -} +YªçŠx-®éÜj×¢ëiºÚ+Š§j[h‘éÜ¢éíã}å:-jZ.¶›­–)Þ³W6¶vR6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡“° ¦–×÷'B¦fæ–òä'—FT'&”÷WGWE7G&VÓ°¦–×÷'B¦fæ–òäFF–çWE7G&VÓ°¦–×÷'B¦fæ–òäFF÷WGWE7G&VÓ°¦–×÷'B¦fæ–òäf–ÆS°¦–×÷'B¦fæ–òä”ôW†6WF–öã°¦–×÷'B¦fææWBä–æWE6ö6¶WDFG&W73°¦–×÷'B¦fææWBå6ö6¶WC°¦–×÷'B¦fææWBåU$“°¦–×÷'B¦fææWBæ‡GGä‡GG6Æ–VçC°¦–×÷'B¦fææWBæ‡GGä‡GG&WVW7C°¦–×÷'B¦fææWBæ‡GGä‡GG&W7öç6S°¦–×÷'B¦fç7Âå5ÄW†6WF–öã°¦–×÷'B¦fçF–ÖRäGW&F–öã°¦–×÷'B¦fçF–ÖRä–ç7FçC°¦–×÷'B¦fçF–ÖRäÆö6ÄFFUF–ÖS°¦–×÷'B¦fçF–ÖRå¦öæT–C°¦–×÷'B¦fçF–ÖRå¦öæTöfg6WC°¦–×÷'B¦fçWF–Âä'&”Æ—7C°¦–×÷'B¦fçWF–Âä6öÆÆV7F–öã°¦–×÷'B¦fçWF–Âä6öÆÆV7F–öç3°¦–×÷'B¦fçWF–Âä†6„Ö°¦–×÷'B¦fçWF–ÂäÆ–æ¶VD†6…6WC°¦–×÷'B¦fçWF–ÂäÆ—7C°¦–×÷'B¦fçWF–ÂäÖ°¦–×÷'B¦fçWF–Âå6WC°¦–×÷'B¦fçWF–ÂåUT”C°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBä6ö×ÆWF&ÆTgWGW&S°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBä6öæ7W'&VçD†6„Ö°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBäW†V7WF÷%6W'f–6S°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBäW†V7WF÷'3°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBå66†VGVÆVDW†V7WF÷%6W'f–6S°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBåF–ÖUVæ—C°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBæFöÖ–2äFöÖ–4&ööÆVã°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBæFöÖ–2äFöÖ–4Æöæs° ¦–×÷'B¦f‚ææWBç76Âå54Å&ÖWFW'3° ¦–×÷'B÷&ræV6Æ—6Rç†òæ6Æ–VçBæ×GGc2ä×GDW†6WF–öã° ¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ’çF–ÖRåF–ÖUG—S°¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ'VævVV’ævÆö&ÆFFävÆö&ÄFF†æFÆW%&÷‡“°¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ'VævVV’ævÆö&ÆFFävÆö&Äו5ð¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ'VævVV’çF–ÖRä'VævVUF–ÖT6†V6¶W#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’æVæ7'—F–öâäVæ7'—F–öä†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’æ§6öâä§6öå'6W#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ6öFV2ä§6öäVçfVÆ÷S°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ6öFV2ä§6öäVçfVÆ÷T6öFV3°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒævÆö&ÂävÆö&ÄÖW76vTÆ—7FVæW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒævÆö&ÂävÆö&ÄÖW76vU&÷‡”†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ×GBä×GD†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ×GBä×GE6W'fW$6öÖÓ°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ×—7Âäו7ÄÖW76VævW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç&VF—2å&VF—4†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç&VF—2å&VF—4Æ—7FVæW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç6ö6¶WG2ä6Æ–VçD†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç6ö6¶WG2å6ö6¶WD†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç6ö6¶WG2å6ö6¶WE&V6V—fW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7Âä6öÇVÖã°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂäFFG—S°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVS°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVT&ööÆVã°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVT–çC°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVU7G&–æs°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7Âæ×—7Âæ6öæf–rä×—7Ä6öæf–s°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âæ&6¶VæG&÷‡’æ‡GGä‡GGVç&öÆÆÖVçDWF†÷&—G“°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âæ&6¶VæG&÷‡’æ‡GGä‡GG&÷‡•G&ç7÷'E6W'fW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âæ&6¶VæG&÷‡’æ‡GGä‡GGFÇ4–FVçF—G“°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ'&öF67Bå&÷‡”'&öF67DFV6–FW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Rä•f÷FT66†S°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Råf÷FT66†T†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Rææöçf÷FVBä”æöåf÷FVEÆ–W'57F÷&vS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Rææöçf÷FVBäæöåf÷FVEÆ–W'466†S°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ6öçG&öÂä6öçG&öÄ6öææV7F÷#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ6öçG&öÂä†÷7FVD6öçG&öÄÖævW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡”†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡”ÖWF†öC°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡•6W'fW%6ö6¶WD6öæf–wW&F–öã°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡•6W'fW%6ö6¶WD6öæf–wW&F–öä'VævVS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’ç&W6Væ6Rä&6¶VæEÆ–W%&W6Væ6UG&6¶W#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’ç&W6Væ6RåÆ–W%&W6Væ6S°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçF–ÖWVWVRåf÷FUF–ÖUVWVS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçF÷f÷FW"åF÷f÷FW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçWF–ÂäÖ–æV7&gEW6W&æÖUfÆ–FF÷#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçWF–Âå6W'f–6U6—FUfÆ–FF÷#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçf÷FVÆöråf÷FTÆöt×—7ÅF&ÆS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçf÷FVÆöråf÷FTÆöt×—7ÅF&ÆRåf÷FTÆöu7FGW3°¦–×÷'B6öÒævöövÆRæw6öâä§6öäVÆVÖVçC°¦–×÷'B6öÒævöövÆRæw6öâä§6öäö&¦V7C° ¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2äFVfVÇD¦VF—46Æ–VçD6öæf–s°¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2ä†÷7DæE÷'C°¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2ä¦VF—3°¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2ä¦VF—5ööð ¦–×÷'BÆöÖ&ö²ävWGFW#°¦–×÷'BÆöÖ&ö²å6WGFW#° §V&Æ–2'7G&7B6Æ72f÷F–æuÇVv–å&÷‡’° —&—fFR7FF–2f–æÂÆöær$U4Tä4Uô„äDôdeõD”ÔTõUEôÔ”ÄÄ•2ÒF–ÖUVæ—BäÔ”åUDU2çFôÖ–ÆÆ—2ƒ"“° —&—fFR7FF–2f–æÂÆöær$U4Tä4Uõ5D%EUõ$U5”ä5ôDTÄ•õ4T4ôäE2ÒTð —&—fFR7FF–2f–æÂÆöær$U4Tä4UôÔ”åDTää4Uô”åDU%dÅõ4T4ôäE2Ò3ð —&—fFR7FF–2f–æÂÆöær$U4Tä4Uô$4´TäEõD”ÔTõUEôÔ”ÄÄ•2ÒF–ÖUVæ—Bå4T4ôäE2çFôÖ–ÆÆ—2ƒ““° —&—fFR7FF–2f–æÂÆöær4ôåE$ôÅôTå$ôÄÄÔTåEôÔ”åô”åDU%dÅôääõ2ÒF–ÖUVæ—Bå4T4ôäE2çFôææ÷2ƒ“°  ”vWGFW  ”6WGFW  —&—fFR–çBf÷FU'G•f÷FW2Ò°  ”vWGFW  ”6WGFW  —&—fFR–çB7W'&VçEf÷FU'G•f÷FW5&WV—&VBÒ°  ”vWGFW  ”6WGFW  —&—fFR&÷‡”×—7ÅW6W%F&ÆR&÷‡”ו5ð  —&—fFRVæ7'—F–öä†æFÆW"Væ7'—F–öä†æFÆW#°  —&—fFR†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â6Æ–VçD†æFÆW3°  —&—fFR6ö6¶WD†æFÆW"6ö6¶WD†æFÆW#° —&—fFR‡GG&÷‡•G&ç7÷'E6W'fW"‡GGG&ç7÷'E6W'fW#° —&—fFR‡GGVç&öÆÆÖVçDWF†÷&—G’‡GGVç&öÆÆÖVçDWF†÷&—G“°  ”vWGFW  ”6WGFW  —&—fFR&ööÆVâf÷F–f–W$Væ&ÆVBÒG'VS°  ”vWGFW  —&—fFR6öæ7W'&VçD†6„ÖÅUT”BÂ7G&–æsâWV–EÆ–W$æÖT66†RÒæWr6öæ7W'&VçD†6„ÖÃâ‚“°  ”vWGFW  ”6WGFW  —&—fFRvÆö&ÄFF†æFÆW%&÷‡’vÆö&ÄFF†æFÆW#°  ”vWGFW  —&—fFR&VF—4†æFÆW"&VF—4†æFÆW#° —&—fFR¦VF—5ööÂ&VF—5V&Æ—6†W%ööð —&—fFRföÆF–ÆRÆöær&VF—5V&Æ—6†W%&WG'”gFW#° —&—fFR&ööÆVâF–ÖUf÷FU&WG'•66†VGVÆVC° —&—fFR&ööÆVâF–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVC° —&—fFR&ööÆVâ66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVC°  —&—fFR&ööÆVâVæ&ÆVC°  ”vWGFW  ”6WGFW  —&—fFR×VÇF•&÷‡”†æFÆW"×VÇF•&÷‡”†æFÆW#°  ”vWGFW  —&—fFR'VævVUF–ÖT6†V6¶W"'VævVUF–ÖT6†V6¶W#°  ”vWGFW  ”6WGFW  —&—fFR'VævVTÖWF†öBÖWF†öC°  ”vWGFW  —&—fFR×GD†æFÆW"×GD†æFÆW#°  ”vWGFW  —&—fFRvÆö&ÄÖW76vU&÷‡”†æFÆW"vÆö&ÄÖW76vU&÷‡”†æFÆW#°  ”vWGFW  ”6WGFW  —&—fFRו7ÄÖW76VævW"&÷‡”×—7ÄÖW76VævW#°  ”vWGFW  —&—fFRf÷FT66†T†æFÆW"f÷FT66†T†æFÆW#°  ”vWGFW  —&—fFRæöåf÷FVEÆ–W'466†Ræöåf÷FVEÆ–W'466†S°  ”vWGFW  —&—fFRf–æÂ&6¶VæEÆ–W%&W6Væ6UG&6¶W"&6¶VæEÆ–W%&W6Væ6UG&6¶W"ÒæWr&6¶VæEÆ–W%&W6Væ6UG&6¶W"‚“° —&—fFRf–æÂÖÅUT”BÂVæF–æu&W6Væ6T†æFöfcâVæF–æu&W6Væ6T†æFöfg2ÒæWr†6„ÖÃâ‚“° —&—fFRf–æÂ6WCÅ7G&–æsâVæF–æt&6¶VæE&V6÷fW'•6æ6†÷G2Ò6öæ7W'&VçD†6„ÖææWt¶W•6WB‚“° —&—fFRf–æÂÖÅ7G&–ærÂÆöæsâ6öçG&öÄVç&öÆÆÖVçDæW‡DÆÆ÷vVBÒæWr6öæ7W'&VçD†6„ÖÃâ‚“° —&—fFRf–æÂÖÅUT”BÂVæF–æt6öÖ×Væ–6F–öåFW7CâVæF–æt6öÖ×Væ–6F–öåFW7G2ÒæWr6öæ7W'&VçD†6„ÖÃâ‚“° —&—fFRföÆF–ÆR6öçG&öÄ6öææV7F÷"6öçG&öÄ6öææV7F÷#° —&—fFRföÆF–ÆR†÷7FVD6öçG&öÄÖævW"†÷7FVD6öçG&öÄÖævW#° —&—fFRf–æÂö&¦V7B6öçG&öÄÆ–fV7–6ÆTÆö6²ÒæWrö&¦V7B‚“° —&—fFRf–æÂFöÖ–4Æöær6öçG&öÅ6W'f–6W4vVæW&F–öâÒæWrFöÖ–4Æöær‚“° —&—fFRf–æÂW†V7WF÷%6W'f–6R6öçG&öÄÆ–fV7–6ÆTW†V7WF÷"ÒW†V7WF÷'2ææWu6–ævÆUF‡&VDW†V7WF÷"‡F6²Óâ° •F‡&VBF‡&VBÒæWrF‡&VB‡F6²Â'f÷F–æwÇVv–âÖ6öçG&öÂÖÆ–fV7–6ÆR"“° —F‡&VBç6WDFVÖöâ‡G'VR“° —&WGW&âF‡&VC° —Ò“°  —V&Æ–2f÷F–æuÇVv–å&÷‡’‚’° –Væ&ÆVBÒG'VS°  –'VævVUF–ÖT6†V6¶W"ÒæWr'VævVUF–ÖT6†V6¶W"†vWD6öæf–r‚’ævWEF–ÖU¦öæR‚’ÂvWD6öæf–r‚’ævWEF–ÖT†÷W$öfe6WB‚’À –vWD6öæf–r‚’ævWEF–ÖUvVV´öfe6WB‚’’°  ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…7G&–ærFW‡B’° –FV'Vs"‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2ÆöærvWDÆ7EWFFVB‚’° —&WGW&âvWEf÷FT66†TÆ7EWFFVB‚“° —Р ”÷fW'&–FP —V&Æ–2–çBvWE&WdF’‚’° —&WGW&âvWEf÷FT66†U&WdF’‚“° —Р ”÷fW'&–FP —V&Æ–27G&–ærvWE&WdÖöçF‚‚’° —&WGW&âvWEf÷FT66†U&WdÖöçF‚‚“° —Р ”÷fW'&–FP —V&Æ–2–çBvWE&WevVV²‚’° —&WGW&âvWEf÷FT66†U&WevVV²‚“° —Р ”÷fW'&–FP —V&Æ–2fö–B–æfò…7G&–ærFW‡B’° –Æör‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2&ööÆVâ—4Væ&ÆVB‚’° —&WGW&âVæ&ÆVC° —Р ”÷fW'&–FP —V&Æ–2&ööÆVâ—4–væ÷&UF–ÖR‚’° —&WGW&â—5f÷FT66†T–væ÷&UF–ÖR‚“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WD–væ÷&UF–ÖR†&ööÆVâ–væ÷&R’° —6WEf÷FT66†Uf÷FT66†T–væ÷&UF–ÖR†–væ÷&R“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WDÆ7EWFFVB‚’° —6WEf÷FT66†TÆ7EWFFVB‚“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WE&WdF’†–çBF’’° —6WEf÷FT66†U&WdF’†F’“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WE&WdÖöçF‚…7G&–ærFW‡B’° —6WEf÷FT66†U&WdÖöçF‚‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WE&WevVV²†–çBvVV²’° —6WEf÷FT66†U&WevVV²‡vVV²“° —Р ”÷fW'&–FP —V&Æ–2fö–BF–ÖT6†ævVB…F–ÖUG—RG—RÂ&ööÆVâf¶RÂ&ööÆVâ&RÂ&ööÆVâ÷7B’° ––b†vWD6öæf–r‚’ævWEf÷FT66†UF–ÖR‚’â’° –vWEf÷FT66†T†æFÆW"‚’æ6†V6µf÷FT66†UF–ÖR†vWD6öæf–r‚’ævWEf÷FT66†UF–ÖR‚’“° —Р––b‚vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’’° —v&â‚$vÆö&ÂFFæ÷BVæ&ÆVB–væ÷&–ærF–ÖR6†ævRWfVçB"“° —&WGW&ã° —Р––çBFVÆ’Ò° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° ––b†vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æ6öçF–ç4¶W’‡2’’° •7G&–ærÆ7DöæÆ–æU7G"ÒvWDvÆö&ÄFF†æFÆW"‚’ævWE7G&–ær‡2Â$Æ7DöæÆ–æR"“° –ÆöærÆ7DöæÆ–æRÒ° —G'’° –Æ7DöæÆ–æRÒÆöærçfÇVTöb†Æ7DöæÆ–æU7G"“° —Ò6F6‚„çVÖ&W$f÷&ÖDW†6WF–öâR’° ’òò–væ÷&P —Р ––b„Æö6ÄFFUF–ÖRææ÷r‚’æE¦öæR…¦öæTöfg6WBåUD2’çFô–ç7FçB‚’çFôWö6„Ö–ÆÆ’‚’ÒÆ7DöæÆ–æR ’¢c¢c¢"’° ”†6„ÖÅ7G&–ærÂFFfÇVSâFFFõ6WBÒæWr†6„ÖÃâ‚“° –FFFõ6WBçWB‚$Æ7EWFFVB"ÂæWrFFfÇVU7G&–ær€ ’""²Æö6ÄFFUF–ÖRææ÷r‚’æE¦öæR…¦öæTöfg6WBåUD2’çFô–ç7FçB‚’çFôWö6„Ö–ÆÆ’‚’’“° –FFFõ6WBçWB‚$f–æ—6†VE&ö6W76–ær"ÂæWrFFfÇVT&ööÆVâ†fÇ6R’“° –FFFõ6WBçWB‡G—RçFõ7G&–ær‚’ÂæWrFFfÇVT&ööÆVâ‡G'VR’“° –vWDvÆö&ÄFF†æFÆW"‚’ç6WDFF‡2ÂFFFõ6WB“°  –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡2ÂFVÆ’Âf÷F–æuÇVv–åv—&Ræ'VævVUF–ÖT6†ævR‚’“° –FVÆ’²³° —ÒVÇ6R° —v&â‚%6W'fW""²2²"†6âwB&VVâöæÆ–æR&V6VçFÇ’"“° —Р—ÒVÇ6R° —v&â‚%6W'fW""²2²"vÆö&ÂFF†æFÆW"F—6&ÆVCò"“° —Р—Р–vÆö&ÄFF†æFÆW"æöåF–ÖT6†ævR‡G—R“° —Р ”÷fW'&–FP —V&Æ–2fö–Bv&æ–ær…7G&–ærFW‡B’° —v&â‡FW‡B“° —Р—Ó° —Р —V&Æ–2fö–BöåF–ÖT6†ævVDf–ÆVB…7G&–ær7'bÂF–ÖUG—RG—R’° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡7'bÂG—RçFõ7G&–ær‚’ÂfÇ6R“° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡7'bÂ$f–æ—6†VE&ö6W76–ær"ÂG'VR“° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡7'bÂ%&ö6W76–ær"ÂfÇ6R“° —Р —V&Æ–2fö–BöåF–ÖT6†ævVDf–æ—6†VB…F–ÖUG—RG—R’° ––b‡G—RæWVÇ2…F–ÖUG—RäÔôåD‚’’° –vWE&÷‡”ו5‚’æ6÷”6öÇVÖäFF…F÷f÷FW"äÖöçF†Ç’ævWD6öÇVÖäæÖR‚’Â$Æ7DÖöçF…F÷FÂ"“° —Р–vWE&÷‡”ו5‚’çv—T6öÇVÖäFF…F÷f÷FW"æöb‡G—R’ævWD6öÇVÖäæÖR‚’ÂFFG—Rä”åDTtU"“°  ––b‚vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’’° —&WGW&ã° —Р–f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡2Â$f÷&6UWFFR"ÂG'VR“° –vWDvÆö&ÄÖW76vU&÷‡”†æFÆW"‚’ç6VæDÖW76vR‡2ÂÂf÷F–æuÇVv–åv—&Ræ'VævVUF–ÖT6†ævR‚’“° —Р—&ö6W75VWVR‚“° —Р ’ò¢  ’¢ÆöBו5²vÆö&ÂFF†æFÆW"à ’¢ð —V&Æ–2fö–BÆöD×—7„ח7Ä6öæf–r×—7Ä6öæf–rÂ×—7Ä6öæf–rvÆö&ÄFF×—7Ä6öæf–r’° ––b†×—7Ä6öæf–rævWD†÷7DæÖR‚’æ—4V×G’‚’ÇÂ×—7Ä6öæf–rævWDFF&6R‚’æ—4V×G’‚’’° –Æöu6WfW&R‚$ו5—2æ÷B6öæf–wW&VB6÷'&V7FÇ’â"²$Ö—76–ær†÷7BöFF&6Râ†÷7CÒ"²×—7Ä6öæf–rævWD†÷7DæÖR‚ ’²"F#Ò"²×—7Ä6öæf–rævWDFF&6R‚’“° —6WE&÷‡”ו5†çVÆ“° —&WGW&ã° —Р —6WE&÷‡”ו5†æWr&÷‡”×—7ÅW6W%F&ÆR‚%f÷F–æuÇVv–åõW6W'2"Â×—7Ä6öæf–rÂvWD6öæf–r‚’ævWDFV'Vr‚’’°  ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…5ÄW†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&R…7G&–ær7G&–ær’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöu6WfW&R‡7G&–ær“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöt–æfò…7G&–ær7G&–ær’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöt–æfò‡7G&–ær“° —Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…F‡&÷v&ÆRB’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° —Bç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…7G&–ær7G"’° –FV'Vs"‡7G"“° —Р—Ò“°  ”'&”Æ—7CÅ7G&–æsâ6W'fW'2ÒæWr'&”Æ—7CÅ7G&–æsâ†vWDÆÄf–Æ&ÆU6W'fW'2‚’“°  ––b†vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’’° ––b†vWD6öæf–r‚’ævWDvÆö&ÄFFW6TÖ–äו5‚’’° —6WDvÆö&ÄFF†æFÆW"†æWrvÆö&ÄFF†æFÆW%&÷‡’€ –æWrvÆö&Äו5‚%f÷F–æuÇVv–åôvÆö&ÄFF"ÂvWE&÷‡”ו5‚’ævWD×—7‚’’°  ”÷fW'&–FP —V&Æ–2fö–BFV'VtW‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'VtÆör…7G&–ærFW‡B’° –FV'Vr‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–B–æfò…7G&–ærFW‡B’° –Æöt–æfò‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&R…7G&–ærFW‡B’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöu6WfW&R‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–Bv&æ–ær…7G&–ærFW‡B’° —v&â‡FW‡B“° —Р—ÒÂ6W'fW'2’°  ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–ÆVB…7G&–ær7'bÂF–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–ÆVB‡7'bÂG—R“° —Р ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–æ—6†VB…F–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–æ—6†VB‡G—R“° —Р—Ò“° —ÒVÇ6R° —6WDvÆö&ÄFF†æFÆW"€ –æWrvÆö&ÄFF†æFÆW%&÷‡’†æWrvÆö&Äו5‚%f÷F–æuÇVv–åôvÆö&ÄFF"ÂvÆö&ÄFF×—7Ä6öæf–r’°  ”÷fW'&–FP —V&Æ–2fö–BFV'VtW‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'VtÆör…7G&–ærFW‡B’° –FV'Vr‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–B–æfò…7G&–ærFW‡B’° –Æöt–æfò‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&R…7G&–ærFW‡B’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöu6WfW&R‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–Bv&æ–ær…7G&–ærFW‡B’° —v&â‡FW‡B“° —Р—ÒÂ6W'fW'2’°  ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–ÆVB…7G&–ær7'bÂF–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–ÆVB‡7'bÂG—R“° —Р ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–æ—6†VB…F–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–æ—6†VB‡G—R“° —Р—Ò“° —Р ’òòWFFRvÆö&Â66†VÖ6öÇVÖç2‡Væ6†ævVBg&öÒ÷&–v–æ –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$–væ÷&UF–ÖR"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$ÔôåD‚"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚%tTT²"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$D’"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$f–æ—6†VE&ö6W76–ær"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚%&ö6W76–ær"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$f÷&6UWFFR"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$Æ7EWFFVB"Â$ÔTD•TÕDU…B"“° —Р ’òò6öÇVÖâG—W2‡Væ6†ævVBg&öÒ÷&–v–æ –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%F÷f÷FW$–væ÷&R"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$6†V6µv÷&ÆB"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%&VÖ–æFVB"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F—6&ÆT'&öF67B"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7DöæÆ–æR"Â%d$4„"ƒ#’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%Æ–W$æÖR"Â%d$4„"ƒ3’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F–Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%vVV¶Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F•f÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$&W7DF•f÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%vVVµf÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$&W7EvVVµf÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%f÷FU'G•f÷FW2"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$ÖöçF…f÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%ö–çG2"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$†–v†W7DF–Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$ÆÅF–ÖUF÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$†–v†W7DÖöçF†Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$ÖöçF…F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$†–v†W7EvVV¶Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7DÖöçF…F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7EvVV¶Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7DF–Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$öffÆ–æU&Wv&G2"Â$ÔTD•TÕDU…B"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F•f÷FU7G&V´Æ7EWFFR"Â$ÔTD•TÕDU…B"“°  ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R†vWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖRææ÷r‚’’Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R†vWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖRææ÷r‚’çÇW4ÖöçF‡2ƒ’’À ’$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R†vWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖRææ÷r‚’çÇW4ÖöçF‡2ƒ"’’À ’$”åBDTdTÅBsr"“° —Р—Р —V&Æ–2fö–BFD7W'&VçEf÷FU'G•f÷FW2†–çBÖ÷VçB’° —f÷FU'G•f÷FW2³ÒÖ÷VçC° —6WEf÷FT66†Uf÷FU'G”7W'&VçEf÷FW2‡f÷FU'G•f÷FW2“° –FV'Vr‚$7W'&VçBf÷FR'G’F÷Fâ"²f÷FU'G•f÷FW2“° —Р —V&Æ–2fö–BFDæöåf÷FVEÆ–W"…7G&–ærWV–BÂ7G&–ærÆ–W$æÖR’° –æöåf÷FVEÆ–W'466†RæFEÆ–W"‡WV–BÂÆ–W$æÖR“° —Р —V&Æ–2fö–BFEf÷FU'G’‚’° ––b†vWD6öæf–r‚’ævWEf÷FU'G”Væ&ÆVB‚’’° –FD7W'&VçEf÷FU'G•f÷FW2ƒ“° –6†V6µf÷FU'G’‚“° —Р—Р —V&Æ–2'7G&7Bfö–B'&öF67B…7G&–ærÖW76vR“°  —&—fFR6WCÅ7G&–æsâ6VæE&÷‡”'&öF67B…6WCÅ7G&–æsâF&vWG2Â7G&–ærWV–BÂ7G&–ærÆ–W"Â7G&–ær6W'f–6RÂÆöærF–ÖRÀ •7G&–ærFW‡BÂ&ööÆVâv4öæÆ–æR’° •6WCÅ7G&–æsâf÷'v&FVBÒæWrÆ–æ¶VD†6…6WCÃâ‚“° –f÷"…7G&–ærF&vWE6W'fW"¢F&vWG2’° ”§6öäVçfVÆ÷RVçfVÆ÷RÒf÷F–æuÇVv–åv—&Rçf÷FT'&öF67B‡WV–BÂÆ–W"Â6W'f–6RÂF–ÖRÂFW‡BÂv4öæÆ–æR“° ––b‡6VæE&÷‡”'&öF67DVçfVÆ÷Tæ÷r‡F&vWE6W'fW"ÂVçfVÆ÷R’’° –f÷'v&FVBæFB‡F&vWE6W'fW"“° —Р—Р—&WGW&âf÷'v&FVC° —Р ’ò¢  ’¢6VæG27FæFÆöæR&÷‡’'&öF67BF‡&÷Vv‚F†R6VÆV7FVBG&ç7÷'BæB&W÷'G0 ’¢v†WF†W"F†BG&ç7÷'B66WFVBF†RÖW76vRà ’  ’¢&Ò6W'fW"F&vWB&6¶VæB6W'fW  ’¢&ÒVçfVÆ÷R7FæFÆöæR'&öF67BVçfVÆ÷P ’¢&WGW&âG'VRöæÇ’v†VâF†RG&ç7÷'B66WFVBF†RÖW76vP ’¢ð —&÷FV7FVB&ööÆVâ6VæE&÷‡”'&öF67DVçfVÆ÷Tæ÷r…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° —7v—F6‚†ÖWF†öB’° –66RÕEC  —&WGW&â6VæD×GDVçfVÆ÷U6W'fW"‡6W'fW"ÂVçfVÆ÷R“° –66RÕ•5à ––b‡&÷‡”×—7ÄÖW76VævW"ÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р—G'’° —&÷‡”×—7ÄÖW76VævW"ç6VæEFô&6¶VæB‡6W'fW"ÂVçfVÆ÷R“° —&WGW&âG'VS° —Ò6F6‚…5ÄW†6WF–öâR’° –FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р–66RÅTt”äÔU54t”äs  —&WGW&â6VæEÇVv–äÖW76vU6W'fW$æ÷r‡6W'fW"ÂVçfVÆ÷R“° –66R$TD•3  —&WGW&â6VæE&VF—4VçfVÆ÷U6W'fW"‡6W'fW"ÂVçfVÆ÷RÂG'VR“° –66R4ô4´UE3  ’òò7FæFÆöæR'&öF67G2W6RF†R6ÖR–æ—F–Æ—¦VB6Æ–VçB2æ÷&ÖÀ ’òòVçfVÆ÷W2âF†—2&W6W'fW2F†R6ö6¶WB6öææV7F–öâæB—G2FVÆ—fW' ’òò6¶æ÷vÆVFvVÖVçB–ç7FVBöb7&VF–ær6V6öæB6†÷'BÖÆ—fVB6ö6¶WBà —&WGW&â6VæE6ö6¶WDVçfVÆ÷R‡6W'fW"ÂVçfVÆ÷R“° –66R…EE  —&WGW&â6VæD‡GGVçfVÆ÷R‡6W'fW"ÂVçfVÆ÷R“° –FVfVÇC  —&WGW&âfÇ6S° —Р—Р ’ò¢  ’¢6VæG2&Wv&BÖ&V&–ærf÷FRVçfVÆ÷RæB&W÷'G2v†WF†W"F†R6VÆV7FV@ ’¢G&ç7÷'B66WFVB—BâÆVv7’G&ç7÷'G2&WF–âF†V—"W†—7F–ær7–æ6‡&öæ÷W0 ’¢6VÖçF–73²…EEW‡÷6W2—G2&÷VæFVB×VWVR&W7VÇB6òf÷FR—2æWfW"F—66&FV@ ’¢v†VâF†RVWVR—2gVÆÂà ’¢ð —&÷FV7FVB&ööÆVâ6VæEf÷FTVçfVÆ÷T66WFVB…7G&–ær6W'fW"–çBFVĤ6öäVçfVÆ÷RVçfVÆ÷R’° ––b†ÖWF†öBÓÒ'VævVTÖWF†öBä…EE’° —&WGW&â6VæD‡GGVçfVÆ÷R‡6W'fW"ÂVçfVÆ÷R“° —Р”vÆö&ÄÖW76vU&÷‡”†æFÆW"†æFÆW"ÒvÆö&ÄÖW76vU&÷‡”†æFÆW#° ––b††æFÆW"ÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р–†æFÆW"ç6VæDÖW76vR‡6W'fW"ÂFVÆ’ÂVçfVÆ÷R“° —&WGW&âG'VS° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–B6†V6´66†VEf÷FW2…7G&–ær6W'fW"’° ––çBFVÆ’Ò° ––b†—56W'fW%fÆ–B‡6W'fW"’’° ––b†—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡6W'fW"’’° ––b†vWEf÷FT66†T†æFÆW"‚’æ†5f÷FW2‡6W'fW"’bbvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ2ÒvWEf÷FT66†T†æFÆW"‚’ævWEf÷FW2‡6W'fW"“° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ&VÖ÷fVBÒæWr'&”Æ—7CÃâ‚“° ––b‚2æ—4V×G’‚’’° ––çBçVÒÒ° ––çBçVÖ&W$öef÷FW2Ò2ç6—¦R‚“° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢2’° ––b†66†Ræ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7E6W'fW%f÷FTFVÆ—fW'’‡6W'fW"Â66†R’’° –6öçF–çVS° —Р––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb66†RææVVG4'&öF67Döâ‡6W'fW"’’° •6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡6W'fW"’À –66†RævWEWV–B‚’Â66†RævWEÆ–W$æÖR‚’Â66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’À –66†RævWEFW‡B‚’ÂfÇ6R“° ––b†66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° ––b‚W'6—7E6W'fW%f÷FTFVÆ—fW'’‡6W'fW"Â66†R’’° –6öçF–çVS° —Р—Р—Р –&ööÆVâFõ6VæBÒG'VS° ––b†vWD6öæf–r‚’ævWEv—Df÷%W6W$öæÆ–æR‚’’° ––b‚—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’’’° —Fõ6VæBÒfÇ6S° —ÒVÇ6R–b†—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’ ’bbvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’’æWVÇ2‡6W'fW"’’° —Fõ6VæBÒfÇ6S° —Р—Р––b‡Fõ6VæB’° –&ööÆVâ'&öF67D†W&RÒ66†RææVVG4'&öF67Döâ‡6W'fW"“° ––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb'&öF67D†W&P ’bbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° –&ööÆVâÆ–W$öæÆ–æRÒ—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’“° •7G&–ærÆ–W%6W'fW"ÒÆ–W$öæÆ–æRòvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’ “¢çVÆð  •6WCÅ7G&–æsâF&vWG2Ò&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡Æ–W$öæÆ–æRÀ —Æ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡6W'fW"ÂF&vWG2“° —Р ––b‚6VæEf÷FTVçfVÆ÷T66WFVB‡6W'fW"ÂFVÆ’À •f÷F–æuÇVv–åv—&Rçf÷FR†66†RævWEÆ–W$æÖR‚’Â66†RævWEWV–B‚’À –66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’ÂfÇ6RÂ66†Ræ—5&VÅf÷FR‚’À –66†RævWEFW‡B‚’Â66†RævWEf÷FT–B‚’ÂvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’À –'&öF67D†W&RÂçVÒÂçVÖ&W$öef÷FW2’’’° –FV'Vr‚%&WF–æ–ær66†VBf÷FR&V6W6RF†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²6W'fW"“° –6öçF–çVS° —Р–FVÆ’²³° –çVÒ²³° —&VÖ÷fVBæFB†66†R“° —ÒVÇ6R° –FV'Vr‚$æ÷B6VæF–ærf÷FR&V6W6RW6W"—6âwBöâ6W'fW""²6W'fW"²#¢  ’²66†RçFõ7G&–ær‚’“° —Р—Р–vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fU6W'fW%f÷FW2‡6W'fW"Â&VÖ÷fVB“° —ÒVÇ6R° –FV'Vr‚$æò66†VBf÷FW2f÷"6W'fW#¢"²6W'fW"“° —Р—ÒVÇ6R° –FV'Vr‚$æò66†VBf÷FW2f÷"6W'fW#¢"²6W'fW"“° —Р—Р—ÒVÇ6R° –FV'Vr‚%6W'fW"æ÷BfÆ–C¢"²6W'fW"“° —Р—Р —V&Æ–27–æ6‡&öæ—¦VBfö–B6†V6´öæÆ–æUf÷FW2…7G&–ærÆ–W"Â7G&–ærWV–BÂ7G&–ær6W'fW"’° ––çBFVÆ’Ò° ––b†—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær‡Æ–W"’bbvWEf÷FT66†T†æFÆW"‚’æ†4öæÆ–æUf÷FW2‡WV–B’’° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ2ÒvWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2‡WV–B“° ––b‚2æ—4V×G’‚’’° ––b‡6W'fW"ÓÒçVÆÂ’° —6W'fW"ÒvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær‡Æ–W"“° —Р––b‚vWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° ––çBçVÒÒ° ––çBçVÖ&W$öef÷FW2Ò†–çB’2ç7G&VÒ‚’æf–ÇFW"‡f÷FRÓâf÷FRæ—5&Wv&DFVÆ—fW&VB‚’’æ6÷VçB‚“° –&ööÆVâFVÆ—fW&VE&Wv&BÒfÇ6S° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ&WF–æVBÒæWr'&”Æ—7CÃâ‚“° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢2’° ––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’’° •6WCÅ7G&–æsâVæF–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ†66†RævWD'&öF67EF&vWG2‚’“° —VæF–æuF&vWG2ç&VÖ÷fTƆ66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b†&Æö6¶VE6W'fW'2ÒçVÆÂ’° —VæF–æuF&vWG2ç&VÖ÷fTƆ&Æö6¶VE6W'fW'2“° —Р–66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƇ6VæE&÷‡”'&öF67B‡VæF–æuF&vWG2À –66†RævWEWV–B‚’Â66†RævWEÆ–W$æÖR‚’Â66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’À –66†RævWEFW‡B‚’ÂfÇ6R’“° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° —Р–&ööÆVâ'&öF67D†W&RÒ66†RææVVG4'&öF67Döâ‡6W'fW"“° ––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb'&öF67D†W&P ’bbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° •7G&–ærÆ–W%6W'fW"Ò‡6W'fW"ÒçVÆÂ’ò6W'fW"¢vWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær‡Æ–W"“°  •6WCÅ7G&–æsâF&vWG2Ò&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡G'VRÂÆ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡6W'fW"ÂF&vWG2“° —Р ––b‚66†Ræ—5&Wv&DFVÆ—fW&VB‚’’° ––b‚6VæEf÷FTVçfVÆ÷T66WFVB‡6W'fW"ÂFVÆ’À •f÷F–æuÇVv–åv—&Rçf÷FTöæÆ–æR†66†RævWEÆ–W$æÖR‚’Â66†RævWEWV–B‚’Â66†RævWE6W'f–6R‚’À –66†RævWEF–ÖR‚’ÂfÇ6RÂ66†Ræ—5&VÅf÷FR‚’Â66†RævWEFW‡B‚’Â66†RævWEf÷FT–B‚’À –vWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’Â'&öF67D†W&RÂçVÒÂçVÖ&W$öef÷FW2’’’° –FV'Vr‚%&WF–æ–æröæÆ–æRf÷FR&V6W6RF†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²6W'fW"“° —&WF–æVBæFB†66†R“° –6öçF–çVS° —Р’òòF†Ræ÷&ÖÂVçfVÆ÷R—2Ç6òfÆ–B'&öF67BFVÆ—fW'’f÷"F†P ’òò7W'&VçBF&vWBâ&V6÷&B—B6ò&Wf–÷W6Ç’VæF–ær7FæFÆöæP ’òò&WG'’6ææ÷Bææ÷Væ6RF†R6ÖRf÷FRv–âÆFW"à ––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb'&öF67D†W&R’° –66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFB‡6W'fW"“° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° —Р–66†Rç6WE&Wv&DFVÆ—fW&VB‡G'VR“° –FVÆ—fW&VE&Wv&BÒG'VS° –FVÆ’²³° –çVÒ²³° —Р ––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° —&WF–æVBæFB†66†R“° —Р—Р–vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fTöæÆ–æUf÷FW2‡WV–B“° –f÷"„öffÆ–æT'VævVUf÷FRVæF–ær¢&WF–æVB’° –vWEf÷FT66†T†æFÆW"‚’æFDöæÆ–æUf÷FR‡WV–BÂVæF–ær“° —Р ’òò×VÇF—&÷‡“¢VçfVÆ÷RÖöæÇ ––b†FVÆ—fW&VE&Wv&BbbvWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚ ’bbvWD6öæf–r‚’ævWD×VÇF•&÷‡”öæTvÆö&Å&Wv&B‚’’° –×VÇF•&÷‡”†æFÆW"ç6VæD6ÆV%f÷FR‡WV–BÂÆ–W"“° —Р—Р—Р—Р—Р ’ò¢  ’¢&WG&–W2f÷FW"Ö¶W–VB7FæFÆöæR'&öF67G2v†Vâç’Æ–W"Ö¶W2F&vW@ ’¢&6¶VæBf–Æ&ÆR2ÇVv–âÖÖW76vR6'&–W"à ’  ’¢&Ò6W'fW"&6¶VæB6W'fW"F†Bv–æVB6'&–W  ’¢ð —&÷FV7FVB7–æ6‡&öæ—¦VBfö–B&WG'•VæF–ætöæÆ–æT'&öF67G2…7G&–ær6W'fW"’° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b‡6W'fW"ÓÒçVÆÂdž&Æö6¶VE6W'fW'2ÒçVÆÂbb&Æö6¶VE6W'fW'2æ6öçF–ç2‡6W'fW"’’’° —&WGW&ã° —Р–f÷"…7G&–ær66†VEWV–B¢vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FUUT”G2‚’’° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2†66†VEWV–B’’’° ––b†66†Ræ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R’’° –6öçF–çVS° —Р––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂ66†RææVVG4'&öF67Döâ‡6W'fW"’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡6W'fW"’Â66†RævWEWV–B‚’À –66†RævWEÆ–W$æÖR‚’Â66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’Â66†RævWEFW‡B‚’ÂfÇ6R“° ––b†66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° ––b†66†Ræ—5&Wv&DFVÆ—fW&VB‚’bb66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° –vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fTöæÆ–æUf÷FR†66†VEWV–BÂ66†R“° —ÒVÇ6R° —W'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R“° —Р—Р—Р—Р—Р —&÷FV7FVB7–æ6‡&öæ—¦VBfö–B&WG'•VæF–æuF–ÖT'&öF67G2…7G&–ær6W'fW"’° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b‡6W'fW"ÓÒçVÆÂdž&Æö6¶VE6W'fW'2ÒçVÆÂbb&Æö6¶VE6W'fW'2æ6öçF–ç2‡6W'fW"’’’° —&WGW&ã° —Р––b†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’ÓÒçVÆÂ’° —&WGW&ã° —Р–f÷"…f÷FUF–ÖUVWVRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR’’° –6öçF–çVS° —Р––b‚f÷FRæ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂf÷FRævWEWV–B‚’æ—4V×G’‚’ÇÂf÷FRævWD'&öF67EF&vWG2‚’æ6öçF–ç2‡6W'fW" —ÇÂf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡6W'fW"’Âf÷FRævWEWV–B‚’Âf÷FRævWDæÖR‚’À —f÷FRævWE6W'f–6R‚’Âf÷FRævWEF–ÖR‚’Âf÷FRævWEF÷FÇ2‚’ÂfÇ6R“° ––b‡f÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° —W'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR“° —Р—Р—Р ’ò¢  ’¢W&–öF–6ÆÇ’&WG&–W2WfW'’VæF–ærf÷FW"Ö¶W–VB7FæFÆöæR'&öF67BâF†—2—0 ’¢&WV—&VBf÷"'&ö¶W"G&ç7÷'G2v†÷6R&V6÷fW'’FöW2æ÷B&öGV6RÆ–W"ÖÆöv–à ’¢6'&–W"WfVçBà ’¢ð —V&Æ–27–æ6‡&öæ—¦VBfö–B&WG'•VæF–ætöæÆ–æT'&öF67G2‚’° –f÷"…7G&–ær66†VEWV–B¢æWrÆ–æ¶VD†6…6WCÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FUUT”G2‚’’’° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2†66†VEWV–B’’’° ––b†66†Ræ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R’’° –6öçF–çVS° —Р––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂ66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâVæF–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ†66†RævWD'&öF67EF&vWG2‚’“° —VæF–æuF&vWG2ç&VÖ÷fTƆ66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b†&Æö6¶VE6W'fW'2ÒçVÆÂ’° —VæF–æuF&vWG2ç&VÖ÷fTƆ&Æö6¶VE6W'fW'2“° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B‡VæF–æuF&vWG2Â66†RævWEWV–B‚’Â66†RævWEÆ–W$æÖR‚’À –66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’Â66†RævWEFW‡B‚’ÂfÇ6R“° ––b†66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° ––b†66†Ræ—5&Wv&DFVÆ—fW&VB‚’bb66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° –vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fTöæÆ–æUf÷FR†66†VEWV–BÂ66†R“° —ÒVÇ6R° —W'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R“° —Р—Р—Р—Р—&WG'•VæF–æuF–ÖT'&öF67G2‚“° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–B&WG'•VæF–æuF–ÖT'&öF67G2‚’° ––b†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’ÓÒçVÆÂ’° —&WGW&ã° —Р–f÷"…f÷FUF–ÖUVWVRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR’’° –6öçF–çVS° —Р––b‚f÷FRæ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂf÷FRævWEWV–B‚’æ—4V×G’‚’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâVæF–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ‡f÷FRævWD'&öF67EF&vWG2‚’“° —VæF–æuF&vWG2ç&VÖ÷fTƇf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b†&Æö6¶VE6W'fW'2ÒçVÆÂ’° —VæF–æuF&vWG2ç&VÖ÷fTƆ&Æö6¶VE6W'fW'2“° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B‡VæF–æuF&vWG2Âf÷FRævWEWV–B‚’Âf÷FRævWDæÖR‚’Âf÷FRævWE6W'f–6R‚’À —f÷FRævWEF–ÖR‚’Âf÷FRævWEF÷FÇ2‚’ÂfÇ6R“° ––b‡f÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° —W'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR“° —Р—Р—Р —&÷FV7FVB7–æ6‡&öæ—¦VB&ööÆVâW'6—7EF–ÖUf÷FTFVÆ—fW'’…f÷FUF–ÖUVWVRf÷FR’° ––b†vWEf÷FT66†T†æFÆW"‚’çWFFUF–ÖUf÷FR‡f÷FR’’° —f÷FRç6WDFVÆ—fW'•7FFTF—'G’†fÇ6R“° —&WGW&âG'VS° —Р—f÷FRç6WDFVÆ—fW'•7FFTF—'G’‡G'VR“° —66†VGVÆUF–ÖUf÷FTFVÆ—fW'•&WG'’‚“° —&WGW&âfÇ6S° —Р —&—fFRfö–B66†VGVÆUF–ÖUf÷FTFVÆ—fW'•&WG'’‚’° ––b‡F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÇÂvWE66†VGVÆW"‚’ÓÒçVÆÂ’° —&WGW&ã° —Р—F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒG'VS° —G'’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ° —7–æ6‡&öæ—¦VB…f÷F–æuÇVv–å&÷‡’çF†—2’° —F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° —Р—&WG'•VæF–æuF–ÖT'&öF67G2‚“° —ÒÂRÂF–ÖUVæ—Bå4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° —F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° –FV'Vr‚%Væ&ÆRFò66†VGVÆRF–ÖVB'&öF67B7FFR&WG'“¢"²RævWDÖW76vR‚’“° —Р—Р —&÷FV7FVB7–æ6‡&öæ—¦VB&ööÆVâW'6—7E6W'fW%f÷FTFVÆ—fW'’…7G&–ær6W'fW"ÂöffÆ–æT'VævVUf÷FRf÷FR’° ––b†vWEf÷FT66†T†æFÆW"‚’çWFFU6W'fW%f÷FR‡6W'fW"Âf÷FR’’° —f÷FRç6WDFVÆ—fW'•7FFTF—'G’†fÇ6R“° —&WGW&âG'VS° —Р—f÷FRç6WDFVÆ—fW'•7FFTF—'G’‡G'VR“° —66†VGVÆT66†VEf÷FTFVÆ—fW'•&WG'’‚“° —&WGW&âfÇ6S° —Р —&÷FV7FVB7–æ6‡&öæ—¦VB&ööÆVâW'6—7DöæÆ–æUf÷FTFVÆ—fW'’…7G&–ærWV–BÂöffÆ–æT'VævVUf÷FRf÷FR’° ––b†vWEf÷FT66†T†æFÆW"‚’çWFFTöæÆ–æUf÷FR‡WV–BÂf÷FR’’° —f÷FRç6WDFVÆ—fW'•7FFTF—'G’†fÇ6R“° —&WGW&âG'VS° —Р—f÷FRç6WDFVÆ—fW'•7FFTF—'G’‡G'VR“° —66†VGVÆT66†VEf÷FTFVÆ—fW'•&WG'’‚“° —&WGW&âfÇ6S° —Р —&—fFRfö–B66†VGVÆT66†VEf÷FTFVÆ—fW'•&WG'’‚’° ––b†66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÇÂvWE66†VGVÆW"‚’ÓÒçVÆÂ’° —&WGW&ã° —Р–66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒG'VS° —G'’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ° —7–æ6‡&öæ—¦VB…f÷F–æuÇVv–å&÷‡’çF†—2’° –66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° —Р—&WG'”66†VEf÷FTFVÆ—fW'•W'6—7FVæ6R‚“° —ÒÂRÂF–ÖUVæ—Bå4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° –66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° –FV'Vr‚%Væ&ÆRFò66†VGVÆR66†VB'&öF67B7FFR&WG'“¢"²RævWDÖW76vR‚’“° —Р—Р —&—fFR7–æ6‡&öæ—¦VBfö–B&WG'”66†VEf÷FTFVÆ—fW'•W'6—7FVæ6R‚’° –f÷"…7G&–ær6W'fW"¢vWEf÷FT66†T†æFÆW"‚’ævWD66†VEf÷FW56W'fW'2‚’’° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWEf÷FW2‡6W'fW"’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’’° —W'6—7E6W'fW%f÷FTFVÆ—fW'’‡6W'fW"Âf÷FR“° —Р—Р—Р–f÷"…7G&–ærWV–B¢æWrÆ–æ¶VD†6…6WCÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FUUT”G2‚’’’° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2‡WV–B’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’’° —W'6—7DöæÆ–æUf÷FTFVÆ—fW'’‡WV–BÂf÷FR“° —Р—Р—Р—Р —V&Æ–2fö–B6†V6µf÷FU'G’‚’° ––b†vWD6öæf–r‚’ævWEf÷FU'G”Væ&ÆVB‚’’° ––b‡f÷FU'G•f÷FW2ãÒ7W'&VçEf÷FU'G•f÷FW5&WV—&VB’° –FV'Vr‚%f÷FR'G’&V6†VB"“° –FD7W'&VçEf÷FU'G•f÷FW2‚Ö7W'&VçEf÷FU'G•f÷FW5&WV—&VB“°  –7W'&VçEf÷FU'G•f÷FW5&WV—&VB³ÒvWD6öæf–r‚’ævWEf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚“° —6WEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB€ –vWEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚’²vWD6öæf–r‚’ævWEf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚’“°  ––b‚vWD6öæf–r‚’ævWEf÷FU'G”'&öF67B‚’æ—4V×G’‚’’° –'&öF67B†vWD6öæf–r‚’ævWEf÷FU'G”'&öF67B‚’“° —Р –f÷"…7G&–ær6öÖÖæB¢vWD6öæf–r‚’ævWEf÷FU'G”'VævVT6öÖÖæG2‚’’° —'Vä6öç6öÆT6öÖÖæB†6öÖÖæB“° —Р ––b†vWD6öæf–r‚’ævWEf÷FU'G•6VæEFôÆÅ6W'fW'2‚’’° –f÷"…7G&–ær6W'fW"¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° —6VæEf÷FU'G’‡6W'fW"“° —Р—ÒVÇ6R° –f÷"…7G&–ær6W'fW"¢vWD6öæf–r‚’ævWEf÷FU'G•6W'fW'5Fõ6VæB‚’’° —6VæEf÷FU'G’‡6W'fW"“° —Р—Р—Р—6fUf÷FT66†Tf–ÆR‚“° —Р—Р —V&Æ–2'7G&7Bfö–BFV'Vr…7G&–ær7G"“°  —&—fFRfö–BFV'Vs"…7G&–ærÖW76vR’° –FV'Vr†ÖW76vR“° —Р ’ò¢  ’¢…EE6Æ–VçBW6VBf÷"Öö¦ær’&WVW7G2à ’¢ð —&—fFRf–æ‡GG6Æ–VçB‡GG6Æ–VçBÒ‡GG6Æ–VçBææWt'V–ÆFW"‚’æ6öææV7EF–ÖV÷WB„GW&F–öâæöe6V6öæG2ƒR’’æ'V–ÆB‚“°  ’ò¢  ’¢fWF6†W2Æ–W"w2UT”Bg&öÒF†RÖö¦ær’à ’  ’¢&ÒÆ–W$æÖRÆ–W"æÖP ’¢&WGW&âÆ–W"UT”BÂ÷"´6öFRçVÆÇÒ–bæ÷Bf÷Væ@ ’¢F‡&÷w2”ôW†6WF–öâ–bF†R&WVW7Bf–Ç0 ’¢F‡&÷w2–çFW''WFVDW†6WF–öâ–b–çFW''WFVBv†–ÆRv—F–ærf÷"F†R&W7öç6P ’¢ð —V&Æ–2UT”BfWF6…UT”B…7G&–ærÆ–W$æÖR’F‡&÷w2”ôW†6WF–öâ–çFW''WFVDW†6WF–öâ° ––b‡Æ–W$æÖRÓÒçVÆÂÇÂÆ–W$æÖRæWVÇ4–væ÷&T66R‚&çVÆÂ"’’° —&WGW&âçVÆð —Р ”‡GG&WVW7B&WVW7BÒ‡GG&WVW7BææWt'V–ÆFW"‚ ’çW&’…U$’æ7&VFR‚&‡GG3¢òö’æÖö¦æræ6öÒ÷W6W'2÷&öf–ÆW2öÖ–æV7&gBò"²Æ–W$æÖR’’ätUB‚ ’çF–ÖV÷WB„GW&F–öâæöe6V6öæG2ƒR’’æ'V–ÆB‚“°  ”‡GG&W7öç6SÅ7G&–æsâ&W7öç6RÒ‡GG6Æ–VçBç6VæB‡&WVW7B‡GG&W7öç6Rä&öG”†æFÆW'2æöe7G&–ær‚’“°  ––b‡&W7öç6Rç7FGW46öFR‚’ÓÒCÇÂ&W7öç6Rç7FGW46öFR‚’ÓÒCB’° –Æör‚%F†W&R—2æòÆ–W"v—F‚F†RæÖRÂ""²Æ–W$æÖR²%Â""“° —&WGW&âçVÆð —Р ––b‡&W7öç6Rç7FGW46öFR‚’Â#ÇÂ&W7öç6Rç7FGW46öFR‚’ãÒ3’° —F‡&÷ræWr”ôW†6WF–öâ‚$f–ÆVBFòfWF6‚UT”Bf÷""²Æ–W$æÖR²"Â…EE"²&W7öç6Rç7FGW46öFR‚’“° —Р ”§6öäVÆVÖVçBVÆVÖVçBÒ§6öå'6W"ç'6U7G&–ær‡&W7öç6Ræ&öG’‚’“° ––b†VÆVÖVçBÓÒçVÆÂÇÂVÆVÖVçBæ—4§6öäö&¦V7B‚’’° —&WGW&âçVÆð —Р ”§6öäö&¦V7Bö&¦V7BÒVÆVÖVçBævWD4§6öäö&¦V7B‚“° ––b‚ö&¦V7Bæ†2‚&–B"’ÇÂö&¦V7BævWB‚&–B"’æ—4§6öäçVÆ‚’’° —&WGW&âçVÆð —Р •7G&–ærWV–D57G&–ærÒö&¦V7BævWB‚&–B"’ævWD57G&–ær‚“° —&WGW&â'6UUT”Dg&öÕ7G&–ær‡WV–D57G&–ær“° —Р —V&Æ–2'7G&7B6WCÅ7G&–æsâvWDÆÄf–Æ&ÆU6W'fW'2‚“°  ’ò¢¢6ö×ÆWFRÆFf÷&Ò6W'fW"6WB&Vf÷&Rv†—FVÆ—7Bö&Æö6¶VB&÷WF–ærf–ÇFW'2â¢ð —V&Æ–2'7G&7B6WCÅ7G&–æsâvWDÆÄ6öæf–wW&VE6W'fW'2‚“°  —V&Æ–2'7G&7Bf÷F–æuÇVv–å&÷‡”6öæf–rvWD6öæf–r‚“°  —V&Æ–2'7G&7B7G&–ærvWD7W'&VçEÆ–W%6W'fW"…7G&–ærÆ–W"“°  ’ò¢  ’¢&W6öÇfW2Æ–W"w26W'fW"f÷"f÷FR&÷WF–ærâFVF–6FVBf÷F–ær&÷‡’†2æð ’¢Æö6ÂÆ–W'2Â6ò—BW6W2F†R&6¶VæB&W6Væ6RG&6¶W"–ç7FVBà ’¢ð —&÷FV7FVB7G&–ærvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær…7G&–ærÆ–W"’° ––b†—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’’° —&WGW&â&6¶VæEÆ–W%&W6Væ6UG&6¶W"ævWEÆ–W"‡Æ–W"’æÖ‡&W6Væ6RÓâ&W6Væ6RævWE6W'fW"‚’’æ÷$VÇ6R†çVÆ“° —Р—&WGW&âvWD7W'&VçEÆ–W%6W'fW"‡Æ–W"“° —Р ’ò¢  ’¢FVF–6FVB&÷WF–ær—2–çFVçF–öæÆÇ’Væf–Æ&ÆRöâÇVv–âÖW76v–æs¢F†@ ’¢G&ç7÷'B—2GF6†VBFòÆ–W"Öf6–ær&÷‡’æBFöW2æ÷B6''’&6¶Væ@ ’¢&W6Væ6R6æ6†÷G2à ’¢ð —&÷FV7FVB&ööÆVâ—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’° —&WGW&âvWD6öæf–r‚’ævWDFVF–6FVEf÷F–æu&÷‡’‚’bbÖWF†öBÒçVÆÂbbÖWF†öBç7W÷'G4&6¶VæE&W6Væ6R‚“° —Р —V&Æ–2'7G&7Bf–ÆRvWDFFföÆFW%ÇVv–â‚“°  —V&Æ–27G&–ærvWDÖöçF…F÷FÇ5v—F„FFUF‚‚’° ”Æö6ÄFFUF–ÖR5F–ÖRÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚“° —&WGW&âvWDÖöçF…F÷FÇ5v—F„FFUF‚†5F–ÖR“° —Р —V&Æ–27G&–ærvWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖR5F–ÖR’° —&WGW&â$ÖöçF…F÷FÂÒ"²5F–ÖRævWDÖöçF‚‚’çFõ7G&–ær‚’²"Ò"²5F–ÖRævWE–V"‚“° —Р —V&Æ–2'7G&7B7G&–ærvWE&÷W$æÖR…7G&–ærWV–BÂ7G&–ærÆ–W$æÖR“°  —V&Æ–2'7G&7B7G&–ærvWEUT”B…7G&–ærÆ–W$æÖR“°  —&—fFR–çBvWEfÇVR„'&”Æ—7CÄ6öÇVÖãâ6öÇ2Â7G&–ær6öÇVÖâ–çBFôFB’° –f÷"„6öÇVÖâB¢6öÇ2’° ––b†BævWDæÖR‚’æWVÇ4–væ÷&T66R†6öÇVÖâ’’° ”FFfÇVRfÇVRÒBævWEfÇVR‚“° ––çBçVÒÒ° ––b‡fÇVRÓÒçVÆÂ’° —&WGW&âFôFC° —Р––b‡fÇVRæ—4–çB‚’’° –çVÒÒfÇVRævWD–çB‚“° —ÒVÇ6R–b‡fÇVRæ—57G&–ær‚’’° —G'’° –çVÒÒ–çFVvW"ç'6T–çB‡fÇVRævWE7G&–ær‚’“° —Ò6F6‚„W†6WF–öâR’° ’òò–væ÷&P —Р—Р—&WGW&âçVÒ²FôFC° —Р—Р—&WGW&âFôFC° —Р —&—fFRf÷FUF÷FÇ56æ6†÷BvWE&ö¦V7FVE&öÆÆ÷fW%F÷FÇ2„'&”Æ—7CÄ6öÇVÖãâFFÂ7G&–ærÆ–W"’° ”Æ—7CÅF–ÖUG—SâF–ÖT6†ævW2ÒvWDvÆö&ÄFF†æFÆW"‚’ævWEF–ÖT6†ævW2‚“° –&ööÆVâ&W6WDÖöçF‚ÒF–ÖT6†ævW2æ6öçF–ç2…F–ÖUG—RäÔôåD‚“° –&ööÆVâ&W6WEvVV²ÒF–ÖT6†ævW2æ6öçF–ç2…F–ÖUG—RåtTT²“° –&ööÆVâ&W6WDF’ÒF–ÖT6†ævW2æ6öçF–ç2…F–ÖUG—RäD’“° ––çB66WFVEVWVVEf÷FW2Ò° ––çB66WFVDvÆö&ÅVWVVEf÷FW2Ò° –f÷"…f÷FUF–ÖUVWVRVWVVB¢vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’° ––b‚VWVVBæ—5&ö6W76VB‚’’° –66WFVDvÆö&ÅVWVVEf÷FW2²³° —Р––b‚VWVVBæ—5&ö6W76VB‚’bbVWVVBævWDæÖR‚’ÒçVÆÂbbVWVVBævWDæÖR‚’æWVÇ4–væ÷&T66R‡Æ–W"’’° –66WFVEVWVVEf÷FW2²³° —Р—Р––çBf÷FT–æ7&VÖVçBÒ66WFVEVWVVEf÷FW2²°  ––çBÆÅF–ÖUF÷FÂÒvWEfÇVR†FFÂ$ÆÅF–ÖUF÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBÖöçF…F÷FÂÒ&W6WDÖöçF‚òf÷FT–æ7&VÖVçB¢vWEfÇVR†FFÂ$ÖöçF…F÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBvVV¶Ç•F÷FÂÒ&W6WEvVV²òf÷FT–æ7&VÖVçB¢vWEfÇVR†FFÂ%vVV¶Ç•F÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBF–Ç•F÷FÂÒ&W6WDF’òf÷FT–æ7&VÖVçB¢vWEfÇVR†FFÂ$F–Ç•F÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBö–çG2ÒvWEfÇVR†FFÂ%ö–çG2"Âf÷FT–æ7&VÖVçB¢vWD6öæf–r‚’ævWEö–çG4öåf÷FR‚’“°  ––çBÖ…f÷FW2ÒvWD6öæf–r‚’ævWDÖ„Ö÷VçDöef÷FW5W$F’‚“° ––b†Ö…f÷FW2â’° ––çBF—2ÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚’ævWDF”ödÖöçF‚‚“° ––b†ÖöçF…F÷FÂâF—2¢Ö…f÷FW2’° –ÖöçF…F÷FÂÒF—2¢Ö…f÷FW3° —Р—Р––b†vWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’âbbö–çG2âvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’’° —ö–çG2ÒvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚“° —Р ––çBFFTÖöçF…F÷FÂÒÓ° ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° ––b†vWD6öæf–r‚’ævWEW6TÖöçF„FFUF÷FÇ45&–Ö'•F÷F‚’’° –FFTÖöçF…F÷FÂÒ&W6WDÖöçF‚òf÷FT–æ7&VÖVç@ “¢vWEfÇVR†FFÂvWDÖöçF…F÷FÇ5v—F„FFUF‚‚’Âf÷FT–æ7&VÖVçB“° —ÒVÇ6R° –FFTÖöçF…F÷FÂÒÖöçF…F÷Fð —Р—Р ––çEµÒ&ö¦V7FVEf÷FU'G’ÒvWE&ö¦V7FVEf÷FU'G•7FFR†66WFVDvÆö&ÅVWVVEf÷FW2²“° —&WGW&âæWrf÷FUF÷FÇ56æ6†÷B†ÆÅF–ÖUF÷FÂÂÖöçF…F÷FÂÂvVV¶Ç•F÷FÂÂF–Ç•F÷FÂÂö–çG2À —&ö¦V7FVEf÷FU'G•³ÒÂ&ö¦V7FVEf÷FU'G•³ÒÂFFTÖöçF…F÷F“° —Р —&÷FV7FVB&ööÆVâ6äf÷'v&E7FæFÆöæT'&öF67B†&ööÆVâÖævW5F÷FÇ2’° —&WGW&âÖævW5F÷FÇ3° —Р —&÷FV7FVB–çEµÒvWE&ö¦V7FVEf÷FU'G•7FFR†–çB66WFVEf÷FW2’° ––çB7W'&VçBÒf÷FU'G•f÷FW3° ––çB&WV—&VBÒ7W'&VçEf÷FU'G•f÷FW5&WV—&VC° ––b‚vWD6öæf–r‚’ævWEf÷FU'G”Væ&ÆVB‚’’° —&WGW&âæWr–çEµÒ²7W'&VçBÂ&WV—&VBÓ° —Р ––çB–æ7&V6RÒvWD6öæf–r‚’ævWEf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚“° –f÷"†–çB’Ò²’Â66WFVEf÷FW3²’²²’° –7W'&VçB²³° ––b†7W'&VçBãÒ&WV—&VB’° –7W'&VçBÓÒ&WV—&VC° —&WV—&VB³Ò–æ7&V6S° —Р—Р—&WGW&âæWr–çEµÒ²7W'&VçBÂ&WV—&VBÓ° —Р —V&Æ–2'7G&7B7G&–ærvWEÇVv–åfW'6–öâ‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†T7W'&VçEf÷FU'G•f÷FW2‚“°  —V&Æ–2'7G&7BÆöærvWEf÷FT66†TÆ7EWFFVB‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†U&WdF’‚“°  —V&Æ–2'7G&7B7G&–ærvWEf÷FT66†U&WdÖöçF‚‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†U&WevVV²‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚“°  —V&Æ–2'7G&7B&ööÆVâ—5Æ–W$öæÆ–æR…7G&–ærÆ–W$æÖR“°  ’ò¢  ’¢6†V6·2öæÆ–æR7FFRf÷"f÷FR&÷WF–ærÂW6–ær&6¶VæB&W6Væ6RöæÇ’v†VâF†—0 ’¢&÷‡’—2W‡Æ–6—FÇ’6öæf–wW&VB2F†RFVF–6FVBf÷F–ær&÷‡’à ’¢ð —&÷FV7FVB&ööÆVâ—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær…7G&–ærÆ–W$æÖR’° —&WGW&â—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’ò&6¶VæEÆ–W%&W6Væ6UG&6¶W"ævWEÆ–W"‡Æ–W$æÖR’æ—5&W6VçB‚ “¢—5Æ–W$öæÆ–æR‡Æ–W$æÖR“° —Р —V&Æ–2'7G&7B&ööÆVâ—56W'fW%fÆ–B…7G&–ær6W'fW"“°  —V&Æ–2'7G&7B&ööÆVâ—56öÖVöæTöæÆ–æU6W'fW"…7G&–ær6W'fW"“°  —&÷FV7FVB&ööÆVâ—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær…7G&–ær6W'fW"’° ––b‚—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’’° —&WGW&â—56öÖVöæTöæÆ–æU6W'fW"‡6W'fW"“° —Р–6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’ç&W6Væ6Rä&6¶VæE&W6Væ6U7FGW27FGW2Ò&6¶VæEÆ–W%&W6Væ6UG&6¶W  ’ævWD&6¶VæE7FGW2‡6W'fW"“° —&WGW&â7FGW2ÒçVÆÂbb7FGW2æ—4f–Æ&ÆR‚’bb7FGW2ævWEÆ–W$6÷VçB‚’â° —Р —V&Æ–2'7G&7B&ööÆVâ—5f÷FT66†T–væ÷&UF–ÖR‚“°  —V&Æ–2'7G&7B×—7Ä6öæf–rvWEf÷FT66†Tו5Ä6öæf–r‚“°  —V&Æ–2'7G&7B×—7Ä6öæf–rvWDæöåf÷FVD66†Tו5Ä6öæf–r‚“°  —V&Æ–2'7G&7B×—7Ä6öæf–rvWEf÷FTÆövv–ætו5Ä6öæf–r‚“°  ’ò¢  ’¢6‡WFF÷vâו5Â×&VÆFVB&W6÷W&6W26fVÇ’à ’¢ð —V&Æ–2fö–B6‡WFF÷väו7‚’° ––b†vWE&÷‡”×—7ÄÖW76VævW"‚’ÒçVÆÂ’° –vWE&÷‡”×—7ÄÖW76VævW"‚’ç6‡WFF÷vâ‚“° —6WE&÷‡”×—7ÄÖW76VævW"†çVÆ“° —Р ––b†vWE&÷‡”ו5‚’ÒçVÆÂ’° –vWE&÷‡”ו5‚’ç6‡WFF÷vâ‚“° —6WE&÷‡”ו5†çVÆ“° —Р—Р —V&Æ–2fö–BÆöB„•f÷FT66†R§6öå7F÷&vR”æöåf÷FVEÆ–W'57F÷&vRæöåf÷FVD66†T§6öâ’° –ÖWF†öBÒ'VævVTÖWF†öBævWD'”æÖR†vWD6öæf–r‚’ævWD'VævVTÖWF†öB‚’“° ––b†vWDÖWF†öB‚’ÓÒçVÆÂ’° –ÖWF†öBÒ'VævVTÖWF†öBåÅTt”äÔU54t”äs° —Р—v&åVç7W÷'FVDFVF–6FVEf÷F–æu&÷‡”ÖöFR‚“° —WV–EÆ–W$æÖT66†RÒvWE&÷‡”ו5‚’ævWE&÷w5UT”DæÖUVW'’‚“°  –'VævVUF–ÖT6†V6¶W"ç6WEF–ÖT6†ævTf–Å6fT'—72†vWD6öæf–r‚’ævWEF–ÖT6†ævTf–Å6fT'—72‚’“° –'VævVUF–ÖT6†V6¶W"æÆöEF–ÖW"‚“°  —f÷FT66†T†æFÆW"ÒæWrf÷FT66†T†æFÆW"†vWEf÷FT66†Tו5Ä6öæf–r‚’ÂvWD6öæf–r‚’ævWEf÷FT66†UW6Tו5‚’À –vWD6öæf–r‚’ævWEf÷FT66†UW6TÖ–äו5‚’ÂvWE&÷‡”ו5‚’ævWD×—7‚’ÂvWD6öæf–r‚’ævWDFV'Vr‚’À –§6öå7F÷&vR’°  ”÷fW'&–FP —V&Æ–2fö–BÆöt–æfó…7G&–ær×6r’° –Æöt–æfò†×6r“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&S…7G&–ær×6r’° –Æöu6WfW&R†×6r“° —Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vs„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vs…7G&–ær×6r’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –FV'Vr†×6r“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vs…F‡&÷v&ÆRR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р—Ó° —f÷FT66†T†æFÆW"æÆöB‚“°  –æöåf÷FVEÆ–W'466†RÒæWræöåf÷FVEÆ–W'466†R†vWDæöåf÷FVD66†Tו5Ä6öæf–r‚’À –vWD6öæf–r‚’ævWDæöåf÷FVD66†UW6Tו5‚’ÂvWD6öæf–r‚’ævWDæöåf÷FVD66†UW6TÖ–äו5‚’À –vWE&÷‡”ו5‚’ævWD×—7‚’Âæöåf÷FVD66†T§6öâÂvWD6öæf–r‚’ævWDFV'Vr‚’’°  ”÷fW'&–FP —V&Íx÷Þm¢G§²ÚîÆ­yÔ”B†Ö÷7E6–t&—G2ÂÆV7E6–t&—G2“° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–B&ö6W75VWVR‚’° —v†–ÆR†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’ç6—¦R‚’â’° •f÷FUF–ÖUVWVRf÷FRÒvWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’æVÆVÖVçB‚“° ––b‚f÷FRæ—5&ö6W76VB‚’’° •f÷FUF÷FÇ56æ6†÷BVWVVEF÷FÇ2Òf÷FRævWEF÷FÇ2‚’ÓÒçVÆÂÇÂf÷FRævWEF÷FÇ2‚’æ—4V×G’‚’òçVÆÀ “¢f÷FUF÷FÇ56æ6†÷Bç'6U7F÷&vR‡f÷FRævWEF÷FÇ2‚’“° •VWVVEf÷FU&W7VÇB&W7VÇBÒf÷FR‡f÷FRævWDæÖR‚’Âf÷FRævWE6W'f–6R‚’ÂG'VRÂfÇ6RÂf÷FRævWEF–ÖR‚’ÂVWVVEF÷FÇ2À —f÷FRævWEWV–B‚’Âf÷FR“° ––b‡&W7VÇBÓÒVWVVEf÷FU&W7VÇBå$UE%’’° —66†VGVÆUF–ÖUf÷FU&WG'’‚“° —&WGW&ã° —Р––b‡&W7VÇBÓÒVWVVEf÷FU&W7VÇBåDU$Ô”äÂ’° —v&â‚%&VÖ÷f–ærFW&Ö–æÂ&öÆÆ÷fW"f÷FR"²f÷FRævWEf÷FT–B‚’²"f÷""²f÷FRævWDæÖR‚’²"ò  ’²6W'f–6U6—FUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡f÷FRævWE6W'f–6R‚’’“° —Р—Р––b‚vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fUF–ÖUf÷FR‡f÷FR’’° —66†VGVÆUF–ÖUf÷FU&WG'’‚“° —&WGW&ã° —Р—Р—Р —&—fFRfö–B66†VGVÆUF–ÖUf÷FU&WG'’‚’° ––b‡F–ÖUf÷FU&WG'•66†VGVÆVBÇÂvWE66†VGVÆW"‚’ÓÒçVÆÂ’° —&WGW&ã° —Р—F–ÖUf÷FU&WG'•66†VGVÆVBÒG'VS° —G'’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ° —7–æ6‡&öæ—¦VB…f÷F–æuÇVv–å&÷‡’çF†—2’° —F–ÖUf÷FU&WG'•66†VGVÆVBÒfÇ6S° —Р—&ö6W75VWVR‚“° —ÒÂRÂF–ÖUVæ—Bå4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° —F–ÖUf÷FU&WG'•66†VGVÆVBÒfÇ6S° –FV'Vr‚%Væ&ÆRFò66†VGVÆR&öÆÆ÷fW"f÷FR&WG'“¢"²RævWDÖW76vR‚’“° —Р—Р —V&Æ–2fö–B&VÆöB‚’° —&VÆöE'VçF–ÖR‡G'VR“° —Р ’ò¢¢Æ–W26öçG&öÂÖ÷&–v–æFVB6öæf–wW&F–öâ&VÆöBv—F†÷WB7F÷–ær—G26öææV7F÷"÷"†÷7FVB6W'f–6Râ¢ð —V&Æ–2fö–B&VÆöDg&öÔ6öçG&ö‚’° —&VÆöE'VçF–ÖR†fÇ6R“° —Р —&—fFRfö–B&VÆöE'VçF–ÖR†&ööÆVâ&W7F'D6öçG&öÅ6W'f–6W2’° –ÖWF†öBÒ'VævVTÖWF†öBævWD'”æÖR†vWD6öæf–r‚’ævWD'VævVTÖWF†öB‚’“° ––b†vWDÖWF†öB‚’ÓÒçVÆÂ’° –ÖWF†öBÒ'VævVTÖWF†öBåÅTt”äÔU54t”äs° —Р—v&åVç7W÷'FVDFVF–6FVEf÷F–æu&÷‡”ÖöFR‚“° ––b‚&W7F'D6öçG&öÅ6W'f–6W2bbÖWF†öBÓÒ'VævVTÖWF†öBå4ô4´UE2’° —&V'V–ÆE6ö6¶WD6Æ–VçG2‚“° —Р —6WD7W'&VçEf÷FU'G•f÷FW5&WV—&VB€ –vWD6öæf–r‚’ævWEf÷FU'G•f÷FW5&WV—&VB‚’²vWEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚’“° ––b‡&W7F'D6öçG&öÅ6W'f–6W2’° –ÆöD×VÇF•&÷‡•7W÷'B‚“° —&W7F'D6öçG&öÅ6W'f–6W47–æ2‚“° —Р—Р —&—fFR7–æ6‡&öæ—¦VBfö–B&V'V–ÆE6ö6¶WD6Æ–VçG2‚’° ”†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â&V'V–ÇBÒæWr†6„ÖÃâ‚“° —G'’° ”Æ—7CÅ7G&–æsâ&Æö6¶VBÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° –f÷"…7G&–ær6W'fW"¢vWD6öæf–r‚’ævWE7–v÷E6W'fW'2‚’’° ––b†&Æö6¶VBæ6öçF–ç2‡6W'fW"’’6öçF–çVS° ”ÖÅ7G&–ærÂö&¦V7CâFFÒvWD6öæf–r‚’ævWE7–v÷E6W'fW$6öæf–wW&F–öâ‡6W'fW"“° •7G&–ær†÷7BÒFFæ6öçF–ç4¶W’‚$†÷7B"’ò…7G&–ær’FFævWB‚$†÷7B"’¢"#° ––çB÷'BÒFFæ6öçF–ç4¶W’‚%÷'B"’ò†–çB’FFævWB‚%÷'B"’¢#“ƒ° —&V'V–ÇBçWB‡6W'fW"ÂæWr6Æ–VçD†æFÆW"††÷7BÂ÷'BÂVæ7'—F–öä†æFÆW"ÂvWD6öæf–r‚’ævWDFV'Vr‚’’“° —Р—Ò6F6‚…'VçF–ÖTW†6WF–öâf–ÇW&R’° —7F÷6ö6¶WD6Æ–VçG2‡&V'V–ÇB“° —F‡&÷rf–ÇW&S° —Р”†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â&Wf–÷W2Ò6Æ–VçD†æFÆW3° –6Æ–VçD†æFÆW2Ò&V'V–ÇC° —7F÷6ö6¶WD6Æ–VçG2‡&Wf–÷W2“° —Р —&—fFR7–æ6‡&öæ—¦VB&ööÆVâ6VæE6ö6¶WDVçfVÆ÷R…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ”6Æ–VçD†æFÆW"6ö6¶WD6Æ–VçBÒ6Æ–VçD†æFÆW2ÓÒçVÆÂòçVÆ¢6Æ–VçD†æFÆW2ævWB‡6W'fW"“° ––b‡6ö6¶WD6Æ–VçBÓÒçVÆÂ’&WGW&âfÇ6S° —G'’° —6ö6¶WD6Æ–VçBç6VæDVçfVÆ÷R†VçfVÆ÷R“° —&WGW&âG'VS° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° –FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р—Р —&—fFR7–æ6‡&öæ—¦VB&ööÆVâ6VæD‡GGVçfVÆ÷R…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ”‡GG&÷‡•G&ç7÷'E6W'fW"G&ç7÷'BÒ‡GGG&ç7÷'E6W'fW#° —&WGW&âG&ç7÷'BÒçVÆÂbbG&ç7÷'Bç6VæB‡6W'fW"ÂVçfVÆ÷R“° —Р —&—fFRfö–B7F'D‡GGG&ç7÷'B‚’° —G'’° •U$’VæGö–çBÒU$’æ7&VFR†vWD6öæf–r‚’ævWD‡GGV&Æ–4VæGö–çB‚’“° ––b‚&‡GG2"æWVÇ4–væ÷&T66R†VæGö–çBævWE66†VÖR‚’’ÇÂVæGö–çBævWD†÷7B‚’ÓÒçVÆÀ —ÇÂVæGö–çBævWE÷'B‚’ÓÒÇÂVæGö–çBævWE÷'B‚’âcSS3P —ÇÂVæGö–çBævWEW6W$–æfò‚’ÒçVÆÂÇÂVæGö–çBævWEVW'’‚’ÒçVÆÂÇÂVæGö–çBævWDg&vÖVçB‚’ÒçVÆÀ —džVæGö–çBævWEF‚‚’ÒçVÆÂbbVæGö–çBævWEF‚‚’æ—4V×G’‚’bb"ò"æWVÇ2†VæGö–çBævWEF‚‚’’’’° —F‡&÷ræWr–ÆÆVvÄ&wVÖVçDW†6WF–öâ‚$…EEåV&Æ–4VæGö–çB×W7B&Râ…EE2÷&–v–â"“° —Р”f–ÆRF—&V7F÷'’ÒæWrf–ÆR†vWDFFföÆFW%ÇVv–â‚’Â&‡GG"“° ”‡GGFÇ4–FVçF—G’–FVçF—G’Ò‡GGFÇ4–FVçF—G’æÆöD÷$7&VFR†F—&V7F÷'’çFõF‚‚’ÂVæGö–çBævWD†÷7B‚’“° –‡GGVç&öÆÆÖVçDWF†÷&—G’ÒæWr‡GGVç&öÆÆÖVçDWF†÷&—G’†–FVçF—G’ÂF—&V7F÷'’çFõF‚‚’“° –‡GGG&ç7÷'E6W'fW"ÒæWr‡GG&÷‡•G&ç7÷'E6W'fW"€ –æWr–æWE6ö6¶WDFG&W72†vWD6öæf–r‚’ævWD‡GG†÷7B‚’ÂvWD6öæf–r‚’ævWD‡GG÷'B‚’’–FVçF—G’À –‡GGVç&öÆÆÖVçDWF†÷&—G’ÂF—&V7F÷'’çFõF‚‚’ç&W6öÇfR‚&÷WFvö–ær×c"’ÂF†—3£¦†æFÆT‡GGG&ç7÷'DVçfVÆ÷R“° –‡GGG&ç7÷'E6W'fW"ç7F'B‚“° –Æöt–æfò‚$…EEG&ç7÷'BÆ—7FVæ–ær6V7W&VÇ’öâ"²vWD6öæf–r‚’ævWD‡GG†÷7B‚’²#¢  ’²‡GGG&ç7÷'E6W'fW"ç÷'B‚’²#²W6R÷f÷F–æwÇVv–æ'VævVR‡GG6öFRÇ6W'fW#âf÷"V6‚&6¶VæB"“° —Ò6F6‚„W†6WF–öâf–ÇW&R’° –6Æ÷6T‡GGG&ç7÷'B‚“° —F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚$…EEG&ç7÷'B6÷VÆBæ÷B7F'B6V7W&VÇ’"Âf–ÇW&R“° —Р—Р ’ò¢¢¶VW2F†RWF†VçF–6FVBÕDÅ2&6¶VæB–FVçF—G’GF6†VBFò6V7W&—G’×6Vç6—F—fR&÷‡’&÷WF–ærâ¢ð —&÷FV7FVBfö–B†æFÆT‡GGG&ç7÷'DVçfVÆ÷R„‡GG&÷‡•G&ç7÷'E6W'fW"å&V6V—fVDVçfVÆ÷R&V6V—fVB’° ––b‚—4WF†VçF–6FVD‡GGVçfVÆ÷TÆÆ÷vVB‡&V6V—fVB’’° –FV'Vr‚$–væ÷&VB…EEVçfVÆ÷Rv†÷6RÆ–W"×&W6Væ6R6Æ–ÒF–Bæ÷BÖF6‚—G2WF†VçF–6FVB&6¶VæB"“° —&WGW&ã° —Р”vÆö&ÄÖW76vU&÷‡”†æFÆW"†æFÆW"ÒvÆö&ÄÖW76vU&÷‡”†æFÆW#° ––b††æFÆW"ÓÒçVÆÂ’F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚$…EEÖW76vR&÷WFW"—2æ÷B&VG’"“° –†æFÆW"æöäÖW76vR‡&V6V—fVBæVçfVÆ÷R‚’“° —Р —&—fFR&ööÆVâ—4WF†VçF–6FVD‡GGVçfVÆ÷TÆÆ÷vVB„‡GG&÷‡•G&ç7÷'E6W'fW"å&V6V—fVDVçfVÆ÷R&V6V—fVB’° ––b‡&V6V—fVBÓÒçVÆÂÇÂ&V6V—fVBæVçfVÆ÷R‚’ÓÒçVÆÂÇÂ&V6V—fVBç6W'fW$–B‚’ÓÒçVÆÂ’&WGW&âfÇ6S° •7G&–ær7F×VE6W'fW"Ò&V6V—fVBæVçfVÆ÷R‚’ævWDf–VÆG2‚’ævWD÷$FVfVÇB…f÷F–æuÇVv–åv—&Räµõ4U%dU"Â""“° ––b‚&V6V—fVBç6W'fW$–B‚’æWVÇ4–væ÷&T66R‡7F×VE6W'fW"’’&WGW&âfÇ6S° ––b‚f÷F–æuÇVv–åv—&Rå5T%ôÄôt”âæWVÇ2‡&V6V—fVBæVçfVÆ÷R‚’ævWE7V$6†ææV‚’’’&WGW&âG'VS° •f÷F–æuÇVv–åv—&RåÆ–W%&W6Væ6TWfVçBWfVçBÒf÷F–æuÇVv–åv—&Rç&VEÆ–W%&W6Væ6TWfVçB‡&V6V—fVBæVçfVÆ÷R‚’“° –&ööÆVâÖöFW&âÒWfVçBæ6öææV7F–öä–BÒçVÆÂÇÂWfVçBæ&6¶VæD–æ6&æF–öä–BÒçVÆÀ —ÇÂWfVçBæ&6¶VæE7F'FVDBÒÂÇÂWfVçBç&W6Væ6UF–ÖW7F×Òð ––b‚ÖöFW&âÇ—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’’&WGW&âG'VS° ’òòÆ–W"Öf6–ær&÷‡’†27G&öævW"WF†÷&—G’F†âç’&6¶VæC¢—G2Æ—fP ’òòÆ–W"6öææV7F–öâ7WÆ–W2&÷F‚F†R7W'&VçB&÷WFRæB†–âöæÆ–æRÖöFR’UT”Bà —&WGW&â—4ÆVv7”Æöv–äFW7F–æF–öäWF†÷&—FF—fR†WfVçBçÆ–W"ÂWfVçBçWV–BÂ&V6V—fVBç6W'fW$–B‚’“° —Р —&—fFR7–æ6‡&öæ—¦VBfö–B6Æ÷6T‡GGG&ç7÷'B‚’° ”‡GG&÷‡•G&ç7÷'E6W'fW"G&ç7÷'BÒ‡GGG&ç7÷'E6W'fW#° –‡GGG&ç7÷'E6W'fW"ÒçVÆð –‡GGVç&öÆÆÖVçDWF†÷&—G’ÒçVÆð ––b‡G&ç7÷'BÒçVÆÂ’G&ç7÷'Bæ6Æ÷6R‚“° —Р —V&Æ–27G&–ær7&VFT‡GG6öææV7F–öä6öFR…7G&–ær6W'fW$–B’° ”‡GGVç&öÆÆÖVçDWF†÷&—G’WF†÷&—G’Ò‡GGVç&öÆÆÖVçDWF†÷&—G“° ––b†ÖWF†öBÒ'VævVTÖWF†öBä…EEÇÂWF†÷&—G’ÓÒçVÆÂ’° —F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚%F†R…EEG&ç7÷'B—2æ÷B'Vææ–ær"“° —Р—&WGW&âWF†÷&—G’æ7&VFT6öææV7F–öä6öFR‡6W'fW$–BÂU$’æ7&VFR†vWD6öæf–r‚’ævWD‡GGV&Æ–4VæGö–çB‚’’ÂGW&F–öâæödÖ–çWFW2ƒR’ ’æVæ6öFR‚“° —Р —V&Æ–2fö–B&Wfö¶T‡GG&6¶VæB…7G&–ær6W'fW$–B’° ”‡GGVç&öÆÆÖVçDWF†÷&—G’WF†÷&—G’Ò‡GGVç&öÆÆÖVçDWF†÷&—G“° ––b†ÖWF†öBÒ'VævVTÖWF†öBä…EEÇÂWF†÷&—G’ÓÒçVÆÂ’F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚%F†R…EEG&ç7÷'B—2æ÷B'Vææ–ær"“° –WF†÷&—G’ç&Wfö¶R„‡GGFÇ4–FVçF—G’æ6æöæ–6Å6W'fW$–B‡6W'fW$–B’“° —Р —&—fFR7–æ6‡&öæ—¦VBfö–B6Æ÷6U6ö6¶WD6Æ–VçG2‚’° ”†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â6Æ–VçG2Ò6Æ–VçD†æFÆW3° –6Æ–VçD†æFÆW2ÒçVÆð —7F÷6ö6¶WD6Æ–VçG2†6Æ–VçG2“° —Р —7FF–2fö–B7F÷6ö6¶WD6Æ–VçG2„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â6Æ–VçG2’° ––b†6Æ–VçG2ÓÒçVÆÂ’&WGW&ã° –f÷"„6Æ–VçD†æFÆW"6Æ–VçB¢6Æ–VçG2çfÇVW2‚’’° ––b†6Æ–VçBÓÒçVÆÂ’6öçF–çVS° —G'’° –6Æ–VçBç7F÷6öææV7F–öâ‚“° —Ò6F6‚…'VçF–ÖTW†6WF–öâ–væ÷&VB’° ’òò&W7BVff÷'C¢öæR'&ö¶Vâ6Æ–VçB×W7Bæ÷B&WfVçBF†R&VÖ–æ–ær6ö6¶WG2g&öÒ6Æ÷6–ærࠗР—Р—Р —&—fFRfö–Bv&åVç7W÷'FVDFVF–6FVEf÷F–æu&÷‡”ÖöFR‚’° ––b†vWD6öæf–r‚’ævWDFVF–6FVEf÷F–æu&÷‡’‚’bb†ÖWF†öBÓÒçVÆÂÇÂÖWF†öBç7W÷'G4&6¶VæE&W6Væ6R‚’’’° –Æöu6WfW&R‚$FVF–6FVEf÷F–æu&÷‡’&WV—&W2Õ•5ÂÂ$TD•2ÂÕEBÂ4ô4´UE2Â÷"…EE²ÅTt”äÔU54t”är—2F—6&ÆVBf÷"  ’²&FVF–6FVB×&÷‡’&÷WF–ærâfÆÆ–ær&6²Fòæ÷&ÖÂ&÷‡’&÷WF–ærâ"“° —Р—Р —V&Æ–2'7G&7Bfö–B'Vä7–æ2…'Vææ&ÆR'Vâ“°  ’ò¢¢ÆFf÷&ÒæÖRW6VBöæÇ’f÷"F†RG&ç7÷'BÖæWWG&Â6öçG&öÂF—66÷fW'’6öçG&7Bâ¢ð —V&Æ–2'7G&7B7G&–ærvWE&÷‡•ÆFf÷&Ò‚“°  —V&Æ–2'7G&7Bfö–B'Vä6öç6öÆT6öÖÖæB…7G&–ær6öÖÖæB“°  —V&Æ–2'7G&7Bfö–B6fUf÷FT66†Tf–ÆR‚“°  —V&Æ–2'7G&7Bfö–B&VÆöD6÷&R†&ööÆVâ×—7“°  ’ò¢¢7G&–7B6öçG&öÂ&VÆöBFƒ²f–ÇW&W2&÷vFR6òF†R6ÆÆW"6â&W7F÷&R—G2&6·Wâ¢ð —V&Æ–2'7G&7Bfö–B&VÆöD6öçG&öÄ6öæf–wW&F–öâ‚’F‡&÷w2W†6WF–öã°  —V&Æ–2'7G&7B&ööÆVâ6VæEÇVv–äÖW76vTFF…7G&–ær6W'fW"Â7G&–ær6†ææVÂÂ'—FUµÒFFÂ&ööÆVâVWVR“°  —&—fFR7FF–2f–æ–çBÅTt”åôÔU54tUô„$EôĔԕBÒ3#scs° —&—fFR7FF–2f–æ–çBÅTt”åôÔU54tUõ4ôeEôĔԕBÒ3°  —V&Æ–2fö–B6VæEÇVv–äÖW76vU6W'fW"…7G&–ær6W'fW"–çBFVĤ6öäVçfVÆ÷RVçfVÆ÷R’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ6VæEÇVv–äÖW76vU6W'fW$æ÷r‡6W'fW"ÂVçfVÆ÷R’ÂFVÆ’¢TÂÂF–ÖUVæ—BäÔ”ÄÄ•4T4ôäE2“° —Р ’ò¢  ’¢6VæG2ÇVv–âÖÖW76vRVçfVÆ÷R–ÖÖVF–FVÇ’æB&W÷'G2v†WF†W"F†R&÷‡ ’¢66WFVB—Bf÷"FVÆ—fW'’à ’  ’¢&Ò6W'fW"F&vWB&6¶VæB6W'fW  ’¢&ÒVçfVÆ÷RVçfVÆ÷RFò6Væ@ ’¢&WGW&âG'VRv†VâF†R&÷‡’66WFVBF†RÖW76vRf÷"FVÆ—fW' ’¢ð —&÷FV7FVB&ööÆVâ6VæEÇVv–äÖW76vU6W'fW$æ÷r…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° –f–æÂ7G&–ær7V$6†ææVÂÒVçfVÆ÷RævWE7V$6†ææV‚“° –f–æÂ7G&–ær–ÆöBÒ§6öäVçfVÆ÷T6öFV2æVæ6öFR†VçfVÆ÷R“°  –f–æÂ'—FUµÒ7V$6†ææVÄ'—FW2Ò7V$6†ææVÂævWD'—FW2†¦fææ–òæ6†'6WBå7FæF&D6†'6WG2åUDeó‚“° –f–æÂ'—FUµÒ–ÆöD'—FW2Ò–ÆöBævWD'—FW2†¦fææ–òæ6†'6WBå7FæF&D6†'6WG2åUDeó‚“°  ’òòW7F–ÖFR'—FW2w&—GFVã  ’òòÒw&—FUUDbFG2"Ö'—FRÆVæwF‚&Vf—‚²UDbÓ‚'—FW0 ’òòÒw&—FT–çB—2B'—FW0 ––çBW7F–ÖFVE6—¦RÒ"²7V$6†ææVÄ'—FW2æÆVæwF‚²òò7V$6†ææVÂUDb†ÆVâ&Vf—‚²'—FW2 “B²òò–ÆöBÆVæwF‚–ç@ “"²–ÆöD'—FW2æÆVæwFƒ²òò–ÆöBUDb†ÆVâ&Vf—‚²'—FW2  ––b†W7F–ÖFVE6—¦RâÅTt”åôÔU54tUõ4ôeEôĔԕB’° –FV'Vr‚%µÇVv–äÖW76vUÒ–ÆöBæV&–ærÆ–Ö—B‚"²W7F–ÖFVE6—¦R²"'—FW2’6W'fW#Ò"²6W'fW  ’²"7V$6†ææVÃÒ"²7V$6†ææV²"( B6öç6–FW"&VF—2–ç7FVB"“° —Р ––b†W7F–ÖFVE6—¦RâÅTt”åôÔU54tUô„$EôĔԕB’° –FV'Vr‚%µÇVv–äÖW76vUÒ–ÆöBDôòÄ$tR‚"²W7F–ÖFVE6—¦R²"'—FW2ÂÖƒÒ"²ÅTt”åôÔU54tUô„$EôĔԕ@ ’²"’6W'fW#Ò"²6W'fW"²"7V$6†ææVÃÒ"²7V$6†ææV²"( BäõB6VçB"“° —&WGW&âfÇ6S° —Р —G'’„'—FT'&”÷WGWE7G&VÒ'—FT÷WE7G&VÒÒæWr'—FT'&”÷WGWE7G&VÒ‚“° ”FF÷WGWE7G&VÒ÷WBÒæWrFF÷WGWE7G&VÒ†'—FT÷WE7G&VÒ’’° ––b†vWD6öæf–r‚’ævWEÇVv–äÖW76vTVæ7'—F–öâ‚’bbVæ7'—F–öä†æFÆW"ÒçVÆÂ’° –÷WBçw&—FUUDb†Væ7'—F–öä†æFÆW"æVæ7'—B‡7V$6†ææVÂ’“° —ÒVÇ6R° –÷WBçw&—FUUDb‡7V$6†ææV“° —Р ’òò6æ—G’öæÇ“¢ÕU5B&R'—FW2Âæ÷B6†'0 –÷WBçw&—FT–çB‡–ÆöD'—FW2æÆVæwF‚“°  ––b†vWD6öæf–r‚’ævWEÇVv–äÖW76vTVæ7'—F–öâ‚’bbVæ7'—F–öä†æFÆW"ÒçVÆÂ’° –÷WBçw&—FUUDb†Væ7'—F–öä†æFÆW"æVæ7'—B‡–ÆöB’“° —ÒVÇ6R° –÷WBçw&—FUUDb‡–ÆöB“° —Р–÷WBæfÇW6‚‚“°  –&ööÆVâ6VçBÒ6VæEÇVv–äÖW76vTFF‡6W'fW"ÂvWD6öæf–r‚’ævWEÇVv–äÖW76vT6†ææV‚’çFôÆ÷vW$66R‚’À –'—FT÷WE7G&VÒçFô'—FT'&’‚’ÂfÇ6R“° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –FV'Vr‚‡6VçBò%6VçB"¢$6÷VÆBæ÷B6VæB"’²"ÇVv–âVçfVÆ÷R‚"²W7F–ÖFVE6—¦R²"'—FW2’"²6W'fW  ’²""²7V$6†ææV²""²VçfVÆ÷RævWDf–VÆG2‚’“° —Р—&WGW&â6VçC° —Ò6F6‚„W†6WF–öâR’° –Rç&–çE7F6µG&6R‚“° —&WGW&âfÇ6S° —Р—Р —7FF–2FVfVÇD¦VF—46Æ–VçD6öæf–r'V–ÆE&VF—46Æ–VçD6öæf–r…f÷F–æuÇVv–å&÷‡”6öæf–r6öæf–u6÷W&6R’° ”FVfVÇD¦VF—46Æ–VçD6öæf–rä'V–ÆFW"6öæf–rÒFVfVÇD¦VF—46Æ–VçD6öæf–ræ'V–ÆFW"‚ ’æFF&6R†6öæf–u6÷W&6RævWE&VF—4F$–æFW‚‚’’ç76†6öæf–u6÷W&6RævWE&VF—576‚’’æ6öææV7F–öåF–ÖV÷WDÖ–ÆÆ—2ƒ# ’ç6ö6¶WEF–ÖV÷WDÖ–ÆÆ—2ƒ#“° ––b†6öæf–u6÷W&6RævWE&VF—576‚’’° •54Å&ÖWFW'276Å&ÖWFW'2ÒæWr54Å&ÖWFW'2‚“° —76Å&ÖWFW'2ç6WDVæGö–çD–FVçF–f–6F–öäÆv÷&—F†Ò‚$…EE2"“° –6öæf–rç76Å&ÖWFW'2‡76Å&ÖWFW'2“° —Р––b†6öæf–u6÷W&6RævWE&VF—5W6W&æÖR‚’ÒçVÆÂbb6öæf–u6÷W&6RævWE&VF—5W6W&æÖR‚’æ—4V×G’‚’’° –6öæf–rçW6W"†6öæf–u6÷W&6RævWE&VF—5W6W&æÖR‚’“° —Р––b†6öæf–u6÷W&6RævWE&VF—577v÷&B‚’ÒçVÆÂbb6öæf–u6÷W&6RævWE&VF—577v÷&B‚’æ—4V×G’‚’’° –6öæf–rç77v÷&B†6öæf–u6÷W&6RævWE&VF—577v÷&B‚’“° —Р—&WGW&â6öæf–ræ'V–ÆB‚“° —Р —V&Æ–2&ööÆVâ6VæE&VF—4VçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° —&WGW&â6VæE&VF—4VçfVÆ÷U6W'fW"‡6W'fW"ÂVçfVÆ÷RÂfÇ6R“° —Р —&—fFR&ööÆVâ6VæE&VF—4VçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷RÂ&ööÆVâW6U&WG'”6ööÆF÷vâ’° ”¦VF—5ööÂV&Æ—6†W%ööÂÒ&VF—5V&Æ—6†W%ööð ––b‡V&Æ—6†W%ööÂÓÒçVÆÂLJW6U&WG'”6ööÆF÷vâbb7—7FVÒæ7W'&VçEF–ÖTÖ–ÆÆ—2‚’Â&VF—5V&Æ—6†W%&WG'”gFW"’’° —&WGW&âfÇ6S° —Р —G'’„¦VF—2¦VF—2ÒV&Æ—6†W%ööÂævWE&W6÷W&6R‚’’° •7G&–ær6†ææVÂÒvWD6öæf–r‚’ævWE&VF—5&Vf—‚‚’²%f÷F–æuÇVv–åò"²6W'fW#° –Æöær7V'67&–&W'2Ò¦VF—2çV&Æ—6‚†6†ææVÂÀ ”§6öäVçfVÆ÷T6öFV2æVæ6öFR…f÷F–æuÇVv–åv—&Rçv—F…&VF—4FVÆ—fW'”–B†VçfVÆ÷R’’“° —&VF—5V&Æ—6†W%&WG'”gFW"Òð —&WGW&â7V'67&–&W'2â° —Ò6F6‚„W†6WF–öâR’° ––b‡W6U&WG'”6ööÆF÷vâ’° ’òò7FæFÆöæR'&öF67G2&VÖ–âVWVVBÂ6òF†V—"&WG&–W26â&RF‡&÷GFÆVB6fVÇ’à —&VF—5V&Æ—6†W%&WG'”gFW"Ò7—7FVÒæ7W'&VçEF–ÖTÖ–ÆÆ—2‚’²#ð —Р–FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р—Р —V&Æ–2&ööÆVâ6VæD×GDVçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ––b†×GD†æFÆW"ÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р—G'’° –×GD†æFÆW"çV&Æ—6„VçfVÆ÷R†vWD6öæf–r‚’ævWD×GE&Vf—‚‚’²'f÷F–æwÇVv–â÷6W'fW'2ò"²6W'fW"ÂVçfVÆ÷R“° —&WGW&âG'VS° —Ò6F6‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—&WGW&âfÇ6S° —Р—Р —V&Æ–2&ööÆVâ6VæE6ö6¶WDVçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ”ÖÅ7G&–ærÂö&¦V7Câ6öæf–wW&F–öâÒvWD6öæf–r‚’ævWE7–v÷E6W'fW$6öæf–wW&F–öâ‡6W'fW"“° ––b†6öæf–wW&F–öâÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р•7G&–ær†÷7BÒ6öæf–wW&F–öâævWB‚$†÷7B"’–ç7Fæ6Vöb7G&–ærò…7G&–ær’6öæf–wW&F–öâævWB‚$†÷7B"’¢"#° ––çB÷'BÒ6öæf–wW&F–öâævWB‚%÷'B"’–ç7Fæ6VöbçVÖ&W"ò‚„çVÖ&W"’6öæf–wW&F–öâævWB‚%÷'B"’’æ–çEfÇVR‚’¢#“ƒ° ––b††÷7Bæ—4V×G’‚’’° —&WGW&âfÇ6S° —Р •7G&–ær–ÆöBÒ§6öäVçfVÆ÷T6öFV2æVæ6öFR†VçfVÆ÷R“° •7G&–ærVæ6öFVBÒVæ7'—F–öä†æFÆW"ÒçVÆÂòVæ7'—F–öä†æFÆW"æVæ7'—B‡–ÆöB’¢–ÆöC° —G'’…6ö6¶WB6ö6¶WBÒæWr6ö6¶WB‚’’° —6ö6¶WBæ6öææV7B†æWr–æWE6ö6¶WDFG&W72††÷7BÂ÷'B’Â#“° —G'’„FF÷WGWE7G&VÒ÷WGWBÒæWrFF÷WGWE7G&VÒ‡6ö6¶WBævWD÷WGWE7G&VÒ‚’’’° –÷WGWBçw&—FUUDb†Væ6öFVB“° –÷WGWBæfÇW6‚‚“° —Р—&WGW&âG'VS° —Ò6F6‚„W†6WF–öâR’° –FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р—Р —V&Æ–2fö–B6VæE6W'fW$æÖTÖW76vR‚’° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° —6VæEÇVv–äÖW76vU6W'fW"‡2ÂÂf÷F–æuÇVv–åv—&Rç6W'fW$æÖR‡2’“° —Р—Р —V&Æ–2fö–B6VæEf÷FU'G’…7G&–ær6W'fW"’° ––b†—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡6W'fW"’’° –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡6W'fW"ÂÂf÷F–æuÇVv–åv—&Rçf÷FU'G”'VævVR‚’“° —Р—Р —V&Æ–2fö–B6WD7W'&VçEf÷FU'G•f÷FW2†–çBÖ÷VçB’° —f÷FU'G•f÷FW2ÒÖ÷VçC° —6WEf÷FT66†Uf÷FU'G”7W'&VçEf÷FW2†Ö÷VçB“° –FV'Vr‚$7W'&VçBf÷FR'G’F÷Fâ"²f÷FU'G•f÷FW2“° —Р —V&Æ–2'7G&7Bfö–B6WEf÷FT66†TÆ7EWFFVB‚“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†U&WdF’†–çBF’“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†U&WdÖöçF‚…7G&–ærFW‡B“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†U&WevVV²†–çBvVV²“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†Uf÷FT66†T–væ÷&UF–ÖR†&ööÆVâ–væ÷&R“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†Uf÷FU'G”7W'&VçEf÷FW2†–çBf÷FW2“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB†–çBf÷FW2“°  —V&Æ–2fö–B7FGW2‚’° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° ––b‚—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡2’’° –Æör‚$æòÆ–W'2öâ6W'fW""²2²"Fò6VæBFW7B7FGW2ÖW76vRÂÆV6R&WFW7Bv—F‚6öÖVöæRöæÆ–æR"“° —ÒVÇ6R° –Æör‚%6VæF–ær&WVW7Bf÷"7FGW2ÖW76vRöâ"²2“° –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡2ÂÂf÷F–æuÇVv–åv—&Rç7FGW2‡2’“° —Р—Р—Р ’ò¢¢'Vç26÷'&VÆFVBÂæöâ×f÷FR&÷VæBG&—÷fW"F†R7F—fR&6¶VæBG&ç7÷'Bâ¢ð —V&Æ–26ö×ÆWF&ÆTgWGW&SÄ6öÖ×Væ–6F–öåFW7E&W7VÇCâFW7D&6¶VæD6öÖ×Væ–6F–öâ…7G&–ær&WVW7FVE6W'fW"À –ÆöærF–ÖV÷WDÖ–ÆÆ—2’° •7G&–ær6W'fW"Ò&WVW7FVE6W'fW"ÓÒçVÆÂò""¢&WVW7FVE6W'fW"çG&–Ò‚“° ”'VævVTÖWF†öB7F—fTÖWF†öBÒÖWF†öC° ––b‡6W'fW"æ—4V×G’‚’ÇÂvWDÆÄf–Æ&ÆU6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%Tä´äõtåô$4´TäB"Â%F†R&6¶VæB—2æ÷B6öæf–wW&VBöâF†—2&÷‡’"’“° —Р––b†7F—fTÖWF†öBÓÒçVÆÂÇÂvÆö&ÄÖW76vU&÷‡”†æFÆW"ÓÒçVÆÂ’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%E$å5õ%EõTäd”Ä$ÄR"Â%F†R&÷‡’6öÖ×Væ–6F–öâG&ç7÷'B—2æ÷B'Vææ–ær"’“° —Р––b†7F—fTÖWF†öBÓÒ'VævVTÖWF†öBåÅTt”äÔU54t”ärbb—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡6W'fW"’’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%Ä”U%õ$UT•$TB"Â%ÇVv–âÖW76v–ær&WV—&W2âöæÆ–æRÆ–W"öâF†R6VÆV7FVB&6¶VæB"’“° —Р•66†VGVÆVDW†V7WF÷%6W'f–6R66†VGVÆW"ÒvWE66†VGVÆW"‚“° ––b‡66†VGVÆW"ÓÒçVÆÂ’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%E$å5õ%EõTäd”Ä$ÄR"Â%F†R&÷‡’66†VGVÆW"—2æ÷B'Vææ–ær"’“° —Р–Æöær&÷VæFVEF–ÖV÷WBÒÖF‚æÖ‚ƒSÂÂÖF‚æÖ–â‡F–ÖV÷WDÖ–ÆÆ—2Â3Â’“° •UT”B&WVW7D–BÒUT”Bç&æFöÕUT”B‚“° ”6ö×ÆWF&ÆTgWGW&SÄ6öÖ×Væ–6F–öåFW7E&W7VÇCâ&W7VÇBÒæWr6ö×ÆWF&ÆTgWGW&SÃâ‚“° •VæF–æt6öÖ×Væ–6F–öåFW7BVæF–ærÒæWrVæF–æt6öÖ×Væ–6F–öåFW7B‡6W'fW"Â7F—fTÖWF†öBÂ7—7FVÒæææõF–ÖR‚’Â&W7VÇB“° —VæF–æt6öÖ×Væ–6F–öåFW7G2çWB‡&WVW7D–BÂVæF–ær“° —&W7VÇBçv†Vä6ö×ÆWFR‚†–væ÷&VBÂf–ÇW&R’ÓâVæF–æt6öÖ×Væ–6F–öåFW7G2ç&VÖ÷fR‡&WVW7D–BÂVæF–ær’“° —G'’° ––b‚6VæD6öÖ×Væ–6F–öåFW7DVçfVÆ÷Tæ÷r‡6W'fW"Âf÷F–æuÇVv–åv—&Rç7FGW2‡6W'fW"Â&WVW7D–B’’’° —&W7VÇBæ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÂ%E$å5õ%EõTäd”Ä$ÄR"À ’%F†R7F—fRG&ç7÷'B6÷VÆBæ÷B66WBF†R6öÖ×Væ–6F–öâFW7B"’“° —&WGW&â&W7VÇC° —Р—66†VGVÆW"ç66†VGVÆR‚‚’Óâ&W7VÇBæ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%D”ÔTõUB"Â$æò6÷'&VÆFVB&WÇ’'&—fVB&Vf÷&RF†RF–ÖV÷WB"’’Â&÷VæFVEF–ÖV÷WBÂF–ÖUVæ—BäÔ”ÄÄ•4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâf–ÇW&R’° —&W7VÇBæ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÂ%4TäEôd”ÄTB"À ’%F†R&÷‡’6÷VÆBæ÷B6VæBF†R6öÖ×Væ–6F–öâFW7B"’“° —Р—&WGW&â&W7VÇC° —Р ’ò¢¢6VæG2F–væ÷7F–2–ÖÖVF–FVÇ’æB&W÷'G2v†WF†W"F†R7F—fRG&ç7÷'B66WFVB—Bâ¢ð —&÷FV7FVB&ööÆVâ6VæD6öÖ×Væ–6F–öåFW7DVçfVÆ÷Tæ÷r…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° —&WGW&â6VæE&÷‡”'&öF67DVçfVÆ÷Tæ÷r‡6W'fW"ÂVçfVÆ÷R“° —Р —&÷FV7FVBfö–B†æFÆU7FGW4ö¶’„§6öäVçfVÆ÷RÖW76vR’° •7G&–ær6W'fW"ÒÖW76vRævWDf–VÆG2‚’ævWD÷$FVfVÇB…f÷F–æuÇVv–åv—&Räµõ4U%dU"Â""“° •7G&–ær&WVW7BÒÖW76vRævWDf–VÆG2‚’ævWD÷$FVfVÇB…f÷F–æuÇVv–åv—&Räµõ$UTU5Eô”BÂ""“° ––b‡&WVW7Bæ—4V×G’‚’’° –Æör‚%7FGW2ö¶’f÷""²6W'fW"“° —&WGW&ã° —Р•UT”B&WVW7D–C° —G'’° —&WVW7D–BÒUT”Bæg&öÕ7G&–ær‡&WVW7B“° —Ò6F6‚„–ÆÆVvÄ&wVÖVçDW†6WF–öâ–væ÷&VB’° –FV'Vr‚$–væ÷&VB7FGW2&WÇ’v—F‚â–çfÆ–B&WVW7B”Bg&öÒ"²6W'fW"“° —&WGW&ã° —Р•VæF–æt6öÖ×Væ–6F–öåFW7BVæF–ærÒVæF–æt6öÖ×Væ–6F–öåFW7G2ævWB‡&WVW7D–B“° ––b‡VæF–ærÓÒçVÆÂÇÂVæF–ærç6W'fW"‚’æWVÇ2‡6W'fW"’’° –FV'Vr‚$–væ÷&VBVæW‡V7FVB7FGW2&WÇ’g&öÒ"²6W'fW"“° —&WGW&ã° —Р–Æöær&÷VæEG&—Ö–ÆÆ—2ÒÖF‚æÖ‚ƒÂÀ •F–ÖUVæ—Bäääõ4T4ôäE2çFôÖ–ÆÆ—2…7—7FVÒæææõF–ÖR‚’ÒVæF–ærç7F'FVDDææ÷2‚’’“° —VæF–ærç&W7VÇB‚’æ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBç7V66W72‡6W'fW"ÂVæF–æræÖWF†öB‚’Â&÷VæEG&—Ö–ÆÆ—2’“° —Р —&—fFRfö–B6æ6VÄ6öÖ×Væ–6F–öåFW7G2…7G&–ærÖW76vR’° —VæF–æt6öÖ×Væ–6F–öåFW7G2æf÷$V6‚‚‡&WVW7D–BÂVæF–ær’ÓâVæF–ærç&W7VÇB‚’æ6ö×ÆWFR€ ”6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡VæF–ærç6W'fW"‚’ÂVæF–æræÖWF†öB‚’Â%E$å5õ%Eõ5DõTB"ÂÖW76vR’’“° —VæF–æt6öÖ×Væ–6F–öåFW7G2æ6ÆV"‚“° —Р —V&Æ–2&V6÷&B6öÖ×Væ–6F–öåFW7E&W7VÇB†&ööÆVâ7V66W72Â7G&–ær6öFRÂ7G&–ærÖW76vRÂ7G&–ær6W'fW"À •7G&–ærÖWF†öBÂÆöær&÷VæEG&—Ö–ÆÆ—2’° —&—fFR7FF–26öÖ×Væ–6F–öåFW7E&W7VÇB7V66W72…7G&–ær6W'fW"Â'VævVTÖWF†öBÖWF†öBÂÆöær&÷VæEG&—Ö–ÆÆ—2’° —&WGW&âæWr6öÖ×Væ–6F–öåFW7E&W7VÇB‡G'VRÂ$ô²"Â$&6¶VæB&WÆ–VB÷fW"F†R7F—fRG&ç7÷'B"Â6W'fW"À –ÖWF†öBÓÒçVÆÂò""¢ÖWF†öBææÖR‚’Â&÷VæEG&—Ö–ÆÆ—2“° —Р —&—fFR7FF–26öÖ×Væ–6F–öåFW7E&W7VÇBf–ÇW&R…7G&–ær6W'fW"Â'VævVTÖWF†öBÖWF†öBÂ7G&–ær6öFRÂ7G&–ærÖW76vR’° —&WGW&âæWr6öÖ×Væ–6F–öåFW7E&W7VÇB†fÇ6RÂ6öFRÂÖW76vRÂ6W'fW"À –ÖWF†öBÓÒçVÆÂò""¢ÖWF†öBææÖR‚’ÂÓ“° —Р—Р —&—fFR&V6÷&BVæF–æt6öÖ×Væ–6F–öåFW7B…7G&–ær6W'fW"Â'VævVTÖWF†öBÖWF†öBÂÆöær7F'FVDDææ÷2À ”6ö×ÆWF&ÆTgWGW&SÄ6öÖ×Væ–6F–öåFW7E&W7VÇCâ&W7VÇB’²Р —&—fFRfö–B6VæEf÷FTFVÆ•&V¦V7FVB…7G&–ærÆ–W"Â7G&–ærWV–BÂ7G&–ær6W'f–6RÂ&ööÆVâÆ–W$öæÆ–æRÀ •7G&–ærÆ–W%6W'fW"’° ––b‚Æ–W$öæÆ–æRÇÂÆ–W%6W'fW"ÓÒçVÆÂÇÂvWDÆÄf–Æ&ÆU6W'fW'2‚’æ6öçF–ç2‡Æ–W%6W'fW"’’° –FV'Vr‚$æ÷B6VæF–ærf÷FRFVÆ’&V¦V7F–öâf÷""²Æ–W"²"&V6W6RF†RÆ–W"—2öffÆ–æR"“° —&WGW&ã° —Р –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡Æ–W%6W'fW"ÂÀ •f÷F–æuÇVv–åv—&Rçf÷FTFVÆ•&V¦V7FVB‡Æ–W"ÂWV–BÂ6W'f–6RÂG'VR’“° —Р —V&Æ–27G&–ærvWEv—EVçF–ÄFVÆ•6—FTg&öÕ6W'f–6R…7G&–ær6W'f–6R’° –f÷"…7G&–ær6—FR¢vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•6—FW2‚’’° ––b†vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•6W'f–6R‡6—FR’æWVÇ4–væ÷&T66R‡6W'f–6R’’° —&WGW&â6—FS° —Р—Р—&WGW&â"#° —Р —&—fFRÆöærvWDÆ7Ef÷FW5F–ÖR…7G&–ærWV–BÂ'&”Æ—7CÄ6öÇVÖãâ6öÇ2Â7G&–ær6—FRÂ7G&–ær6W'f–6RÂ7G&–ærÆ–W"À –&ööÆVâ–æ6ÇVFUF–ÖT6†ævUVWVR’° –ÆöærÖ÷7E&V6VçEF–ÖRÒ°  ––b†vWEf÷FT66†T†æFÆW"‚’æ†4öæÆ–æUf÷FW2‡WV–B’’° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâöæÆ–æUf÷FW2ÒvWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2‡WV–B“° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢öæÆ–æUf÷FW2’° ––b‡f÷FRævWE6W'f–6R‚’æWVÇ4–væ÷&T66R‡6W'f–6R’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂf÷FRævWEF–ÖR‚’“° —Р—Р—Р –f÷"…7G&–ær6W'fW"¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢vWEf÷FT66†T†æFÆW"‚’ævWEf÷FW2‡6W'fW"’’° ––b‡f÷FRævWEWV–B‚’æWVÇ2‡WV–B’bbf÷FRævWE6W'f–6R‚’æWVÇ4–væ÷&T66R‡6W'f–6R’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂf÷FRævWEF–ÖR‚’“° —Р—Р—Р ––b†–æ6ÇVFUF–ÖT6†ævUVWVRbbÆ–W"ÒçVÆÂ’° –f÷"…f÷FUF–ÖUVWVRVWVVEf÷FR¢vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’° ––b‡VWVVEf÷FRævWDæÖR‚’æWVÇ4–væ÷&T66R‡Æ–W" ’bbVWVVEf÷FRævWE6W'f–6R‚’æWVÇ4–væ÷&T66R‡6W'f–6R’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂVWVVEf÷FRævWEF–ÖR‚’“° —Р—Р—Р –f÷"„6öÇVÖâB¢6öÇ2’° ––b†BævWDæÖR‚’æWVÇ4–væ÷&T66R‚$Æ7Ef÷FW2"’’° ”FFfÇVRfÇVRÒBævWEfÇVR‚“° •7G&–æuµÒÆ—7BÒfÇVRævWE7G&–ær‚’ç7Æ—B‚"VÆ–æRR"“° –f÷"…7G&–ær7G"¢Æ—7B’° •7G&–æuµÒFFÒ7G"ç7Æ—B‚"òò"“° ––b†FF³ÒæWVÇ4–væ÷&T66R‡6—FR’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂÆöærçfÇVTöb†FF³Ò’“° —Р—Р—Р—Р—&WGW&âÖ÷7E&V6VçEF–ÖS° —Р —V&Æ–2&ööÆVâ6†V6µf÷FTFVÆ’…7G&–ærWV–BÂ7G&–ær6W'f–6RÂ'&”Æ—7CÄ6öÇVÖãâFF’° —&WGW&â6†V6µf÷FTFVÆ’‡WV–BÂçVÆÂÂ6W'f–6RÂFFÂfÇ6R“° —Р ’ò¢  ’¢6†V6·2F†R6öæf–wW&VBf÷FRFVÆ’Â÷F–öæÆÇ’–æ6ÇVF–ær66WFVBf÷FW2v—F–æp ’¢f÷"vÆö&ÄFFF–ÖR6†ævRFòf–æ—6‚à ’  ’¢&ÒWV–BÆ–W"UT”@ ’¢&ÒÆ–W"Æ–W"æÖRW6VB'’F†RF–ÖRÖ6†ævRVWVP ’¢&Ò6W'f–6Rf÷FR6W'f–6P ’¢&ÒFF7W'&VçBÆ–W"FF ’¢&Ò–æ6ÇVFUF–ÖT6†ævUVWVRv†WF†W"VWVVBf÷FW2&W6W'fRF†V—"FVÆ’6Æ÷@ ’¢&WGW&âG'VRv†VâF†Rf÷FRÖ’&R66WFV@ ’¢ð —V&Æ–2&ööÆVâ6†V6µf÷FTFVÆ’…7G&–ærWV–BÂ7G&–ærÆ–W"Â7G&–ær6W'f–6RÂ'&”Æ—7CÄ6öÇVÖãâFFÀ –&ööÆVâ–æ6ÇVFUF–ÖT6†ævUVWVR’° •7G&–ær6—FRÒvWEv—EVçF–ÄFVÆ•6—FTg&öÕ6W'f–6R‡6W'f–6R“° ––b‡6—FRæ—4V×G’‚’’° –FV'Vr‚$æò6W'f–6R6—FR6WBf÷""²6W'f–6R²"Â6¶—–ærf÷FRFVÆ’6†V6²"“° —&WGW&âG'VS° —Р ––çBf÷FTFVÆ’ÒvWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVÆ’‡6—FR“° ––çBf÷FTFVƔ֖âÒvWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVƔ֖â‡6—FR“°  –ÆöærÆ7Ef÷FRÒvWDÆ7Ef÷FW5F–ÖR‡WV–BÂFFÂ6—FRÂ6W'f–6RÂÆ–W"–æ6ÇVFUF–ÖT6†ævUVWVR“° ––b†Æ7Ef÷FRÓÒ’° –FV'Vr‚$æòÆ7Bf÷FRF–ÖRf÷VæBf÷""²WV–B²"ò"²6W'f–6R²"Â6¶—–ærf÷FRFVÆ’6†V6²"“° —&WGW&âG'VS° —Р —G'’° ”Æö6ÄFFUF–ÖRæ÷rÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚“° ”Æö6ÄFFUF–ÖRÆ7Ef÷FUF–ÖRÒÆö6ÄFFUF–ÖRæöd–ç7FçB„–ç7FçBæödWö6„Ö–ÆÆ’†Æ7Ef÷FR’¦öæT–Bç7—7FVÔFVfVÇB‚’ ’çÇW4†÷W'2†vWD6öæf–r‚’ævWEF–ÖT†÷W$öfe6WB‚’“°  ––b‚vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVÆ”F–Ç’‡6—FR’’° ––b‡f÷FTFVÆ’ÓÒbbf÷FTFVƔ֖âÓÒ’° –FV'Vr‚%f÷FRFVÆ’—2f÷""²6—FR²"Â6¶—–ærf÷FRFVÆ’6†V6²"“° —&WGW&âG'VS° —Р ”Æö6ÄFFUF–ÖRæW‡Gf÷FRÒÆ7Ef÷FUF–ÖRçÇW4†÷W'2‚†Æöær’f÷FTFVÆ’’çÇW4Ö–çWFW2‚†Æöær’f÷FTFVƔ֖⓰ —&WGW&âæ÷ræ—4gFW"†æW‡Gf÷FR“° —Р”Æö6ÄFFUF–ÖR&W6WEF–ÖRÒÆ7Ef÷FUF–ÖRçv—F„†÷W"†vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVÆ”†÷W"‡6—FR’ ’çv—F„Ö–çWFRƒ’çv—F…6V6öæBƒ“° ”Æö6ÄFFUF–ÖR&W6WEF–ÖUFöÖ÷'&÷rÒ&W6WEF–ÖRçÇW4†÷W'2ƒ#B“°  ––b†Æ7Ef÷FUF–ÖRæ—4&Vf÷&R‡&W6WEF–ÖR’’° ––b†æ÷ræ—4gFW"‡&W6WEF–ÖR’’° –FV'Vr‚%f÷FRFVÆ’—2ÖWBf÷""²WV–B²"ò"²6W'f–6R²"Âf÷FR6â&R&ö6W76VB"“° —&WGW&âG'VS° —Р—ÒVÇ6R° ––b†æ÷ræ—4gFW"‡&W6WEF–ÖUFöÖ÷'&÷r’’° –FV'Vr‚%f÷FRFVÆ’—2ÖWBf÷""²WV–B²"ò"²6W'f–6R²"Âf÷FR6â&R&ö6W76VB"“° —&WGW&âG'VS° —Р—Р—Ò6F6‚„W†6WF–öâR’° –Rç&–çE7F6µG&6R‚“° —Р –FV'Vr‚%f÷FRFVÆ’—2æ÷BÖWBf÷""²WV–B²"ò"²6W'f–6R²"Â6¶—–ærf÷FR"“° —&WGW&âfÇ6S° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–Bf÷FR…7G&–ærÆ–W"Â7G&–ær6W'f–6RÂ&ööÆVâ&VÅf÷FRÂ&ööÆVâF–ÖUVWVRÂÆöærVWVUF–ÖRÀ •f÷FUF÷FÇ56æ6†÷BFW‡BÂ7G&–ærWV–B’° —f÷FR‡Æ–W"Â6W'f–6RÂ&VÅf÷FRÂF–ÖUVWVRÂVWVUF–ÖRÂFW‡BÂWV–BÂçVÆ“° —Р —&—fFRVçVÒVWVVEf÷FU&W7VÇB° •5T44U52Â$UE%’ÂDU$Ô”äÀ —Р —&—fFR7–æ6‡&öæ—¦VBVWVVEf÷FU&W7VÇBf÷FR…7G&–ærÆ–W"Â7G&–ær6W'f–6RÂ&ööÆVâ&VÅf÷FRÂ&ööÆVâF–ÖUVWVRÂÆöærVWVUF–ÖRÀ •f÷FUF÷FÇ56æ6†÷BFW‡BÂ7G&–ærWV–BÂf÷FUF–ÖUVWVRVWVVEf÷FR’° —G'’° ––b‚6W'f–6U6—FUfÆ–FF÷"æ—5fÆ–B‡6W'f–6R’’° —v&â‚%&V¦V7FVBf÷FRv—F‚–çfÆ–B6W'f–6R6—FRr"²6W'f–6U6—FUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡6W'f–6R’²"r"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р––b‚Ö–æV7&gEW6W&æÖUfÆ–FF÷"æ—5fÆ–B‡Æ–W"ÂvWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’’’° —v&â‚%&V¦V7FVBf÷FRv—F‚–çfÆ–BÖ–æV7&gBW6W&æÖRr  ’²Ö–æV7&gEW6W&æÖUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡Æ–W"’²"rg&öÒ6W'f–6Rr  ’²Ö–æV7&gEW6W&æÖUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡6W'f–6R’²"r"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р •UT”Bf÷FT–BÒVWVVEf÷FRÓÒçVÆÂòçVÆ¢VWVVEf÷FRævWEf÷FT–B‚“° ––b‡f÷FT–BÓÒçVÆÂ’° —f÷FT–BÒUT”Bç&æFöÕUT”B‚“° —Р ’òòUT”B&W6öÇWF–öà ––b‚vWD6öæf–r‚’ævWDöæÆ–æTÖöFR‚’’° —WV–BÒvWEUT”B‡Æ–W"“° —Р ––b‡WV–BÓÒçVÆÂÇÂWV–Bæ—4V×G’‚’’° —WV–BÒvWEUT”B‡Æ–W"“°  ’òò&VG&ö6²&Vf—‚WFòÖFWFV7@ ––b‡WV–Bæ—4V×G’‚’bbvWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’æ—4V×G’‚ ’bbÆ–W"ç7F'G5v—F‚†vWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’’’° •7G&–ærWV–CÒvWEUT”B†vWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’²Æ–W"“° ––b‚WV–Cæ—4V×G’‚’’° –FV'Vr‚$FWFV7FVB&VG&ö6²Æ–W"v—F†÷WB&Vf—‚ÂF§W7F–ærâââ"“° —Æ–W"ÒvWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’²Æ–W#° —WV–BÒWV–C° —Р—Р—Р ––b‡WV–Bæ—4V×G’‚’’° ––b‡Æ–W"ç7F'G5v—F‚†vWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’’’° –Æör‚$–væ÷&–ærf÷FR6–æ6RVæ&ÆRFòvWBUT”Böb&VG&ö6²Æ–W""“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р––b‚vWD6öæf–r‚’ævWDÆÆ÷uVä¦ö–æVB‚’’° –Æör‚$–væ÷&–ærf÷FRg&öÒ"²Æ–W"²"6–æ6RÆ–W"†6âwB¦ö–æVB&Vf÷&R"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р––b‚vWD6öæf–r‚’ævWEUT”DÆöö·W‚’’° –Æör‚$f–ÆVBFòvWBWV–Bf÷""²Æ–W"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р –FV'Vr‚$fWF6†–ærUT”BöæÆ–æRÂ6–æ6RÆÆ÷wVæ¦ö–æVB—2Væ&ÆVB"“° •UT”BRÒçVÆð —G'’° ––b†vWD6öæf–r‚’ævWDöæÆ–æTÖöFR‚’’° —RÒfWF6…UT”B‡Æ–W"“° —Р—Ò6F6‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р––b‡RÓÒçVÆÂ’° –FV'Vr‚$f–ÆVBFòvWBWV–Bf÷""²Æ–W"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р—WV–BÒRçFõ7G&–ær‚“° —Р ’òòæ÷&ÖÆ—¦RUT”B7G&–ær–b÷76–&ÆP —G'’° ––b‡WV–BÒçVÆÂbbWV–Bæ—4V×G’‚’bbWV–BæWVÇ4–væ÷&T66R‚&çVÆÂ"’’° —WV–BÒUT”Bæg&öÕ7G&–ær‡WV–BçG&–Ò‚’’çFõ7G&–ær‚“° —Р—Ò6F6‚„W†6WF–öâ–væ÷&VB’° ’òò–væ÷&P —Р —Æ–W"ÒvWE&÷W$æÖR‡WV–BÂÆ–W"“°  ’òò66†RöæÆ–æR7FFR÷6W'fW"öæ6R„”Õõ%DåBf÷"'&öF67BÆöv–26÷'&V7FæW72 –f–æÂ&ööÆVâÆ–W$öæÆ–æRÒ—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær‡Æ–W"“° –f–æÂ7G&–ærÆ–W%6W'fW"ÒÆ–W$öæÆ–æRòvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær‡Æ–W"’¢çVÆð –ÆöærF–ÖRÒVWVUF–ÖRÒòVWVUF–ÖP “¢Æö6ÄFFUF–ÖRææ÷r‚’æE¦öæR…¦öæT–Bç7—7FVÔFVfVÇB‚’’çFô–ç7FçB‚’çFôWö6„Ö–ÆÆ’‚“°  •6WCÅ7G&–æsâ'&öF67EF&vWG2ÒVWVVEf÷FRÓÒçVÆÂòæWrÆ–æ¶VD†6…6WCÃâ‚ “¢æWrÆ–æ¶VD†6…6WCÃâ‡VWVVEf÷FRævWD'&öF67EF&vWG2‚’“° •6WCÅ7G&–æsâ'&öF67Df÷'v&FVE6W'fW'2ÒVWVVEf÷FRÓÒçVÆÂòæWrÆ–æ¶VD†6…6WCÃâ‚ “¢æWrÆ–æ¶VD†6…6WCÃâ‡VWVVEf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° –&ööÆVâ&÷‡”'&öF67D†æFÆVBÒVWVVEf÷FRÒçVÆÂbbVWVVEf÷FRæ—5&÷‡”'&öF67D†æFÆVB‚“° –&ööÆVâ&ö6W76W5F÷FÇ2ÒvWD6öæf–r‚’ævWE&–Ö'•6W'fW"‚’ÇÂvWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚“° –&ööÆVâÖævW5F÷FÇ2Ò&ö6W76W5F÷FÇ2bbvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚“° –&ööÆVâ6åfÆ–FFU7FæFÆöæT'&öF67BÒ6äf÷'v&E7FæFÆöæT'&öF67B†ÖævW5F÷FÇ2“° ”'&”Æ—7CÄ6öÇVÖãâFFÒçVÆð –&ööÆVâVWVTf÷%F–ÖT6†ævRÒfÇ6S°  ’òò6ö×ÆWF–öâ6ÆÆ&6²6âv—RF÷FÇ2æB&WÆ’öÆFW"VWVVBf÷FW2â'Vâ—@ ’òò&Vf÷&RÆöF–ærF†—2f÷FRw2FF&6R6æ6†÷B6òF†R6Æ7VÆF–öç2&VÆ÷rW6P ’òòF†R÷7B×&öÆÆ÷fW"7FFRà ––b†vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’bbvWDvÆö&ÄFF†æFÆW"‚’æ—5F–ÖT6†ævVD†VæVB‚’’° –vWDvÆö&ÄFF†æFÆW"‚’æ6†V6´f÷$f–æ—6†VEF–ÖT6†ævW2‚“° —VWVTf÷%F–ÖT6†ævRÒF–ÖUVWVRbbvWDvÆö&ÄFF†æFÆW"‚’æ—5F–ÖT6†ævVD†VæVB‚“° —Р ’òòfÆ–FFRF†Rf÷FR&Vf÷&Rç’–ÖÖVF–FRææ÷Væ6VÖVçBâF†—2¶VW2GWÆ–6FP ’òòf÷FW2&V¦V7FVB'’F†RFVÆ’6†V6²÷WBöbF†RvÆö&ÄFF&öÆÆ÷fW"VWVRæ@ ’òò&WfVçG2ææ÷Væ6–ærf÷FRF†Bv–ÆÂæ÷B&R&ö6W76VBà ––b†ÖævW5F÷FÇ2’° ––b†vWE&÷‡”ו5‚’ÓÒçVÆÂ’° –Æöu6WfW&R‚$×—7—2æ÷BÆöFVB6÷'&V7FÇ’Â7F÷–ærf÷FR&ö6W76–ær"“° —&WGW&âVWVVEf÷FU&W7VÇBå$UE%“° —Р ––b‚vWE&÷‡”ו5‚’æ6öçF–ç4¶W•VW'’‡WV–B’’° –vWE&÷‡”ו5‚’çWFFR‡WV–BÂ%Æ–W$æÖR"ÂæWrFFfÇVU7G&–ær‡Æ–W"’“° –vWE&÷‡”ו5‚’ævWEWV–G2‚’æFB‡WV–B“° —Р –FFÒvWE&÷‡”ו5‚’ævWDW†7EVW'’†æWr6öÇVÖâ‚'WV–B"ÂæWrFFfÇVU7G&–ær‡WV–B’’“° ––b‚6†V6µf÷FTFVÆ’‡WV–BÂÆ–W"Â6W'f–6RÂFFÂVWVVEf÷FRÓÒçVÆÂ’’° –Æör‚%f÷FRFVÆ’—2æ÷BÖWBf÷""²Æ–W"²"ò"²6W'f–6R²"Â6¶—–ærf÷FR"“° —6VæEf÷FTFVÆ•&V¦V7FVB‡Æ–W"ÂWV–BÂ6W'f–6RÂÆ–W$öæÆ–æRÂÆ–W%6W'fW"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р—Р ’òòf÷'v&Bâ66WFVBöffÆ–æR'&öF67B&Vf÷&RF†R7F–ÆÂÖ7F—fRvÆö&ÄFF ’òò6†ævRVWVW2F†R&Wv&B÷F÷FÇ2v÷&²âF†RVWVVBFVÆ—fW'’7FFR&WfVçG0 ’òò&WÆ––ær'&öF67G2F†BÇ&VG’&V6†VB&6¶VæBà ––b‡VWVTf÷%F–ÖT6†ævR’° •f÷FUF÷FÇ56æ6†÷B&ö¦V7FVEF÷FÇ2ÒÖævW5F÷FÇ2òvWE&ö¦V7FVE&öÆÆ÷fW%F÷FÇ2†FFÂÆ–W"’¢FW‡C° ––b†6åfÆ–FFU7FæFÆöæT'&öF67Bbb&÷‡”'&öF67DFV6–FW"çW6W4–ÖÖVF–FTf÷'v&F–ær‡Æ–W$öæÆ–æR’’° –'&öF67EF&vWG2æFDƇ&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2†fÇ6RÂçVÆÂ’“° —&÷‡”'&öF67D†æFÆVBÒG'VS° —Р•f÷FUF–ÖUVWVRFVÆ–VEf÷FRÒæWrf÷FUF–ÖUVWVR‡f÷FT–BÂÆ–W"Â6W'f–6RÂF–ÖRÀ —&÷‡”'&öF67D†æFÆVBÂ'&öF67EF&vWG2Â'&öF67Df÷'v&FVE6W'fW'2À —&ö¦V7FVEF÷FÇ2ÓÒçVÆÂò""¢&ö¦V7FVEF÷FÇ2çFõ7G&–ær‚’ÂfÇ6RÂWV–B“° ––b‚vWEf÷FT66†T†æFÆW"‚’æFEF–ÖUf÷FUFô66†R†FVÆ–VEf÷FR’’° –Æöu6WfW&R‚%Væ&ÆRFòW'6—7BVWVVB&öÆÆ÷fW"f÷FRf÷""²Æ–W"²"ò"²6W'f–6P ’²#²6¶—–ær&÷‡’'&öF67B"“° —&WGW&âVWVVEf÷FU&W7VÇBå$UE%“° —Р––b‡&÷‡”'&öF67D†æFÆVB’° –f÷"…7G&–ærF&vWB¢'&öF67EF&vWG2’° •6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡F&vWB’ÂWV–BÂÆ–W"À —6W'f–6RÂF–ÖRÂ&ö¦V7FVEF÷FÇ2ÓÒçVÆÂò""¢&ö¦V7FVEF÷FÇ2çFõ7G&–ær‚’ÂfÇ6R“° ––b†FVÆ–VEf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –'&öF67Df÷'v&FVE6W'fW'2æFDƆf÷'v&FVB“° —W'6—7EF–ÖUf÷FTFVÆ—fW'’†FVÆ–VEf÷FR“° —Р—Р—Р–Æör‚$66†–ærf÷FRg&öÒ"²Æ–W"²"ò"²6W'f–6P ’²"&V6W6RF–ÖR6†ævR—2†Væ–ær&–v‡Bæ÷r"“° —&WGW&âVWVVEf÷FU&W7VÇBå5T44U53° —Р –FEf÷FU'G’‚“°  ’òòF÷FÇ2&ö6W76–ær‡&–Ö'’6W'fW"õ"æò×VÇF—&÷‡’ ––b‡&ö6W76W5F÷FÇ2’° ––b†ÖævW5F÷FÇ2’° ––çBÆÅF–ÖUF÷FÂÒvWEfÇVR†FFÂ$ÆÅF–ÖUF÷FÂ"“° ––çBÖöçF…F÷FÂÒvWEfÇVR†FFÂ$ÖöçF…F÷FÂ"“°  ––çBFFTÖöçF…F÷FÂÒÓ° ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° ––b†vWD6öæf–r‚’ævWEW6TÖöçF„FFUF÷FÇ45&–Ö'•F÷F‚’’° –FFTÖöçF…F÷FÂÒvWEfÇVR†FFÂvWDÖöçF…F÷FÇ5v—F„FFUF‚‚’“° —ÒVÇ6R° –FFTÖöçF…F÷FÂÒÖöçF…F÷Fð —Р—Р ––çBvVV¶Ç•F÷FÂÒvWEfÇVR†FFÂ%vVV¶Ç•F÷FÂ"“° ––çBF–Ç•F÷FÂÒvWEfÇVR†FFÂ$F–Ç•F÷FÂ"“° ––çBö–çG2ÒvWEfÇVR†FFÂ%ö–çG2"ÂvWD6öæf–r‚’ævWEö–çG4öåf÷FR‚’“°  ––çBÖ…f÷FW2ÒvWD6öæf–r‚’ævWDÖ„Ö÷VçDöef÷FW5W$F’‚“° ––b†Ö…f÷FW2â’° ”Æö6ÄFFUF–ÖR5F–ÖRÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚“° ––çBF—2Ò5F–ÖRævWDF”ödÖöçF‚‚“° ––b†ÖöçF…F÷FÂâF—2¢Ö…f÷FW2’° –ÖöçF…F÷FÂÒF—2¢Ö…f÷FW3° —Р—Р ––b†vWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’âbbö–çG2âvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’’° —ö–çG2ÒvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚“° —Р —FW‡BÒæWrf÷FUF÷FÇ56æ6†÷B†ÆÅF–ÖUF÷FÂÂÖöçF…F÷FÂÂvVV¶Ç•F÷FÂÂF–Ç•F÷FÂÂö–çG2À —f÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VBÂFFTÖöçF…F÷F“°  ”'&”Æ—7CÄ6öÇVÖãâWFFRÒæWr'&”Æ—7CÃâ‚“° —WFFRæFB†æWr6öÇVÖâ‚$ÆÅF–ÖUF÷FÂ"ÂæWrFFfÇVT–çB†ÆÅF–ÖUF÷FÂ’’“° —WFFRæFB†æWr6öÇVÖâ‚$ÖöçF…F÷FÂ"ÂæWrFFfÇVT–çB†ÖöçF…F÷FÂ’’“° ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° —WFFRæFB†æWr6öÇVÖâ†vWDÖöçF…F÷FÇ5v—F„FFUF‚‚’ÂæWrFFfÇVT–çB†FFTÖöçF…F÷FÂ’’“° —Р—WFFRæFB†æWr6öÇVÖâ‚%vVV¶Ç•F÷FÂ"ÂæWrFFfÇVT–çB‡vVV¶Ç•F÷FÂ’’“° —WFFRæFB†æWr6öÇVÖâ‚$F–Ç•F÷FÂ"ÂæWrFFfÇVT–çB†F–Ç•F÷FÂ’’“° —WFFRæFB†æWr6öÇVÖâ‚%ö–çG2"ÂæWrFFfÇVT–çB‡ö–çG2’’“°  –FV'Vr‚%6WGF–ærF÷FÇ2"²FW‡BçFõ7G&–ær‚’²"Âf÷FT–CÒ"²f÷FT–B²"f÷""²Æ–W"²"ò  ’²6W'f–6R“° –vWE&÷‡”ו5‚’çWFFR‡WV–BÂWFFR“° —ÒVÇ6R° —FW‡BÒæWrf÷FUF÷FÇ56æ6†÷BƒÂÂÂÂÂf÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VB“° —Р—Р––b‡FW‡BÓÒçVÆÂ’° —FW‡BÒæWrf÷FUF÷FÇ56æ6†÷BƒÂÂÂÂÂf÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VB“° —Р •f÷FTÆöu7FGW2f÷FU7FGW2Òf÷FTÆöu7FGW2ä”ÔÔTD”DS° –&ööÆVâ7FæFÆöæU&÷‡”'&öF67BÒ6åfÆ–FFU7FæFÆöæT'&öF67Bbb‡&÷‡”'&öF67D†æFÆV@ —ÇÂ&÷‡”'&öF67DFV6–FW"çW6W4–ÖÖVF–FTf÷'v&F–ær‡Æ–W$öæÆ–æR’“° •6WCÅ7G&–æsâ&÷‡”'&öF67EF&vWG2Ò6öÆÆV7F–öç2æV×G•6WB‚“° ––b‡7FæFÆöæU&÷‡”'&öF67B’° ’òò†æFÆVBVWVVB'&öF67Bv2æV6W76&–Ç’6×ÆVBv†–ÆRF†RÆ–W"v0 ’òòöffÆ–æRâ&WG'’öæÇ’F&vWG2F†BF–Bæ÷B&Wf–÷W6Ç’66WBFVÆ—fW'’à —&÷‡”'&öF67EF&vWG2Ò&÷‡”'&öF67D†æFÆVBòæWrÆ–æ¶VD†6…6WCÃâ†'&öF67EF&vWG2 “¢&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2†fÇ6RÂçVÆ“° •6WCÅ7G&–æsâ&VÖ–æ–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ‡&÷‡”'&öF67EF&vWG2“° —&VÖ–æ–æuF&vWG2ç&VÖ÷fTƆ'&öF67Df÷'v&FVE6W'fW'2“° –'&öF67Df÷'v&FVE6W'fW'2æFDƇ6VæE&÷‡”'&öF67B‡&VÖ–æ–æuF&vWG2ÂWV–BÂÆ–W"Â6W'f–6RÂF–ÖRÀ —FW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’ÂfÇ6R’“° —Р ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР’òò6VæBf÷FR‡2’Fò&6¶VæB‡2 ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР––b†vWD6öæf–r‚’ævWE6VæEf÷FW5FôÆÅ6W'fW'2‚’’° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’°  –&ööÆVâf÷&6T66†RÒvWD6öæf–r‚’ævWEv—Df÷%W6W$öæÆ–æR‚ ’bb‚Æ–W$öæÆ–æRÇÂÆ–W%6W'fW"ÓÒçVÆÂÇÂÆ–W%6W'fW"æWVÇ4–væ÷&T66R‡2’“°  ––b†f÷&6T66†R’° –FV'Vr‚$f÷&6–ærf÷FRFò66†Rf÷"6W'fW""²2“° —Р ––b‚‚—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡2’bbÖWF†öBç&WV—&W5Æ–W$öæÆ–æR‚’’ÇÂf÷&6T66†R’° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFE6W'fW%f÷FR‡2À –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÀ —FW‡BçFõ7G&–ær‚’Â'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÀ —&÷‡”'&öF67EF&vWG2Â'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–ærf÷FRf÷""²Æ–W"²"öâ"²6W'f–6R²"f÷""²2“° —ÒVÇ6R° –&ööÆVâ'&öF67D†W&RÒ'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç2‡2“° ––b†'&öF67D†W&RbbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° •6WCÅ7G&–æsâF&vWG2Ò7FæFÆöæU&÷‡”'&öF67Bò&÷‡”'&öF67EF&vWG0 “¢&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡Æ–W$öæÆ–æRÂÆ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡2ÂF&vWG2“° —Р ––b‚6VæEf÷FTVçfVÆ÷T66WFVB‡2Â"À •f÷F–æuÇVv–åv—&Rçf÷FR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂG'VRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À —f÷FT–BÂvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’Â'&öF67D†W&RÂÂ’’’° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFE6W'fW%f÷FR‡2À –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÀ —FW‡BçFõ7G&–ær‚’Â'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÀ —&÷‡”'&öF67EF&vWG2Â'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–ærf÷FRgFW"F†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²2“° —Р—Р—Р—ÒVÇ6R° ’òò6–ævÆR×6W'fW"ÖöFS¢öæÆ–æRvöW2FòÆ–W"6W'fW#²÷F†W'v—6RVWVR2&öæÆ–æP ’òòf÷FR  ––b‡Æ–W$öæÆ–æRbbÆ–W%6W'fW"ÒçVÆÂbbvWDÆÄf–Æ&ÆU6W'fW'2‚’æ6öçF–ç2‡Æ–W%6W'fW"’’° •7G&–ær6W'fW"ÒÆ–W%6W'fW#°  –&ööÆVâ'&öF67D†W&RÒ'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç2‡6W'fW"“° ––b†'&öF67D†W&RbbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° •6WCÅ7G&–æsâF&vWG2Ò7FæFÆöæU&÷‡”'&öF67Bò&÷‡”'&öF67EF&vWG0 “¢&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡G'VRÂÆ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡6W'fW"ÂF&vWG2“° —Р –&ööÆVâ&Wv&D66WFVBÒ6VæEf÷FTVçfVÆ÷T66WFVB‡6W'fW"ÂÀ •f÷F–æuÇVv–åv—&Rçf÷FTöæÆ–æR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂG'VRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À —f÷FT–BÂvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’Â'&öF67D†W&RÂÂ’“° ––b‚&Wv&D66WFVB’° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFDöæÆ–æUf÷FR‡WV–BÀ –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À –'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÂ&÷‡”'&öF67EF&vWG2À –'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–æröæÆ–æRf÷FRgFW"F†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²6W'fW"“° —Р ––b‡&Wv&D66WFVBbb6åfÆ–FFU7FæFÆöæT'&öF67BbbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚ ’bb7FæFÆöæU&÷‡”'&öF67B’° •6WCÅ7G&–æsâF&vWG2Ò&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡G'VRÂÆ–W%6W'fW"“°  ––çB$FVÆ’Ò#° –f÷"…7G&–ærF&vWE6W'fW"¢F&vWG2’° ’òòfö–BF÷V&ÆRÖ'&öF67BöâF†R6ÖR6W'fW"F†BÇ&VG’v÷BF†Rf÷FTöæÆ–æP ––b‡F&vWE6W'fW"æWVÇ4–væ÷&T66R‡6W'fW"’’° –6öçF–çVS° —Р––b†vWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡F&vWE6W'fW"’’° –6öçF–çVS° —Р –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡F&vWE6W'fW"Â$FVÆ’À •f÷F–æuÇVv–åv—&Rçf÷FT'&öF67B‡WV–BÂÆ–W"Â6W'f–6RÂF–ÖRÀ —FW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’ÂG'VR’“° –$FVÆ’²³° —Р—Р ’òò×VÇF—&÷‡“¢VçfVÆ÷RÖöæÇ’6ÆV"f÷FP ––b‡&Wv&D66WFVBbbvWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚’bbvWD6öæf–r‚’ævWD×VÇF•&÷‡”öæTvÆö&Å&Wv&B‚’’° –×VÇF•&÷‡”†æFÆW"ç6VæD6ÆV%f÷FR‡WV–BÂÆ–W"“° —Р—ÒVÇ6R° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFDöæÆ–æUf÷FR‡WV–BÀ –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À –'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÂ&÷‡”'&öF67EF&vWG2À –'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–æröæÆ–æRf÷FRf÷""²Æ–W"²"öâ"²6W'f–6R“° —Р ––çBFVÆ’Ò#° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡2ÂFVÆ’²Âf÷F–æuÇVv–åv—&Rçf÷FUWFFR‡WV–BÀ —f÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VBÂ6W'f–6RÂF–ÖRÂFW‡BçFõ7G&–ær‚’’“° –FVÆ’³Ò#° —Р—Р ’òòf÷FRÆövv–æp ––b‡f÷FTÆöt×—7ÅF&ÆRÒçVÆÂbbvWD6öæf–r‚’ævWEf÷FTÆövv–ætVæ&ÆVB‚’’° —f÷FTÆöt×—7ÅF&ÆRæÆöuf÷FR‡f÷FT–BÂf÷FU7FGW2Â6W'f–6RÂWV–BÂÆ–W"ÂF–ÖRÀ –vWEf÷FT66†T†æFÆW"‚’ævWE&÷‡”66†VEF÷F‡WV–B’“° —Р ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР’òò×VÇF—&÷‡’f÷'v&F–æp ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР––b†vWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚’bbvWD6öæf–r‚’ævWE&–Ö'•6W'fW"‚’’° ––b‚vWD6öæf–r‚’ævWD×VÇF•&÷‡”öæTvÆö&Å&Wv&B‚’’° –FV'Vr‚%6VæF–ærvÆö&Â&÷‡’f÷FRVçfVÆ÷R"“° –×VÇF•&÷‡”†æFÆW"ç6VæD×VÇF•&÷‡”VçfVÆ÷R…f÷F–æuÇVv–åv—&Rçf÷FR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂfÇ6RÀ —&VÅf÷FRÂFW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’Âf÷FT–BÂfÇ6RÂfÇ6RÂÂ’“° —ÒVÇ6R° ’òòöæÇ’6VæBFò÷F†W"&÷†–W2–bF†RÆ–W"D”BäõBÇ&VG’&V6V—fR&Wv&Böâ ’òò&6¶Væ@ –&ööÆVâ6†÷VÆE6VæBÒG'VS° ––b‡Æ–W$öæÆ–æRbbÆ–W%6W'fW"ÒçVÆÂ’° ––b‚vWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡Æ–W%6W'fW"’’° —6†÷VÆE6VæBÒfÇ6S° —Р—Р ––b‡6†÷VÆE6VæB’° –FV'Vr‚%6VæF–ærvÆö&Â&÷‡’f÷FVöæÆ–æRVçfVÆ÷R"“° –×VÇF•&÷‡”†æFÆW  ’ç6VæD×VÇF•&÷‡”VçfVÆ÷R…f÷F–æuÇVv–åv—&Rçf÷FTöæÆ–æR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂfÇ6RÀ —&VÅf÷FRÂFW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’Âf÷FT–BÂfÇ6RÂfÇ6RÂÂ’“° —ÒVÇ6R° –FV'Vr‚$æ÷B6VæF–ærvÆö&Â&÷‡’ÖW76vRf÷"f÷FVöæÆ–æRÂÆ–W"Ç&VG’v÷B&Wv&B"“° —Р—Р—Р––b‡VWVVEf÷FRÒçVÆÂ’° —VWVVEf÷FRç6WE&ö6W76VB‡G'VR“° ––b‚vWEf÷FT66†T†æFÆW"‚’çWFFUF–ÖUf÷FR‡VWVVEf÷FR’’° —v&â‚%Væ&ÆRFòW'6—7B6ö×ÆWFVB&öÆÆ÷fW"f÷FR"²VWVVEf÷FRævWEf÷FT–B‚ ’²#²GFV×F–ærGW&&ÆR&VÖ÷f–ÖÖVF–FVÇ’"“° —Р—Р—&WGW&âVWVVEf÷FU&W7VÇBå5T44U53° —Ò6F6‚„W†6WF–öâR’° –Rç&–çE7F6µG&6R‚“° —&WGW&âVWVVEf÷FU&W7VÇBå$UE%“° —Р—Р —&—fFR7FF–2f–æÂ6Æ72VæF–æu&W6Væ6T†æFöfb° —&—fFRUT”B&WVW7D–C° —&—fFRf–æÂUT”BÆ–W%WV–C° —&—fFRf–æÂ7G&–ærÆ–W$æÖS° —&—fFRf–æÂ7G&–ærWV–C° —&—fFRf–æÂ7G&–ær6W'fW#° —&—fFRf–æÂUT”B6öææV7F–öä–C° —&—fFRf–æÂUT”B&6¶VæD–æ6&æF–öä–C° —&—fFRf–æÂÆöær&6¶VæE7F'FVDC° —&—fFRf–æÂÆöær6öæfÆ–7E6WVVæ6S° —&—fFRf–æÂÆöær7&VFVDC°  —&—fFRVæF–æu&W6Væ6T†æFöfb…7G&–ærÆ–W$æÖRÂ7G&–ærWV–BÂ7G&–ær6W'fW"ÂUT”B6öææV7F–öä–BÀ •UT”B&6¶VæD–æ6&æF–öä–BÂÆöær&6¶VæE7F'FVDBÂÆöær6öæfÆ–7E6WVVæ6RÂÆöær7&VFVDB’° —F†—2çÆ–W%WV–BÒ'6UÆ–W%WV–B‡WV–B“° —F†—2çÆ–W$æÖRÒÆ–W$æÖS° —F†—2çWV–BÒWV–C° —F†—2ç6W'fW"Ò6W'fW#° —F†—2æ6öææV7F–öä–BÒ6öææV7F–öä–C° —F†—2æ&6¶VæD–æ6&æF–öä–BÒ&6¶VæD–æ6&æF–öä–C° —F†—2æ&6¶VæE7F'FVDBÒ&6¶VæE7F'FVDC° —F†—2æ6öæfÆ–7E6WVVæ6RÒ6öæfÆ–7E6WVVæ6S° —F†—2æ7&VFVDBÒ7&VFVDC° —Р —&—fFR7FF–2UT”B'6UÆ–W%WV–B…7G&–ærWV–B’° —G'’° —&WGW&âUT”Bæg&öÕ7G&–ær‡WV–BçG&–Ò‚’“° —Ò6F6‚„W†6WF–öâ–væ÷&VB’° —&WGW&âçVÆð —Р—Р—Р —V&Æ–2'7G&7Bfö–Bv&â…7G&–ærÖW76vR“°  —V&Æ–2'7G&7B66†VGVÆVDW†V7WF÷%6W'f–6RvWE66†VGVÆW"‚“°§Ð  \ No newline at end of file diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 094d752c9..4c006c02d 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -80,6 +80,19 @@ void inboundDeliveryFenceRejectsCorruptionAndPathReplacement() throws Exception assertThrows(java.io.IOException.class, () -> store.reserve(java.util.UUID.randomUUID().toString())); } + @Test + void sealedInboundStoreCannotChangeAfterOwnershipHandoff() throws Exception { + Path credentials = directory.resolve("sealed-client"); + Files.createDirectories(credentials); + String id = java.util.UUID.randomUUID().toString(); + HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(credentials); + store.reserve(id); + store.markRunning(id); + store.seal(); + assertThrows(java.io.IOException.class, () -> store.markCompleted(id)); + assertEquals(HttpInboundDeliveryStore.State.RUNNING, new HttpInboundDeliveryStore(credentials).state(id)); + } + @Test void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { HttpTlsIdentity created = HttpTlsIdentity.loadOrCreate(directory, "localhost"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index d11aea72f..dd2fa418a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -89,6 +90,44 @@ void closeNeverWaitsForSetupOnTheCallingThread() throws Exception { finally { release.countDown(); blocked.join(TimeUnit.SECONDS.toMillis(1)); } } + @Test + @SuppressWarnings("unchecked") + void validationWaitsForThePreviousDirectoryOwner() throws Exception { + Path credentials = directory.resolve("http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy-owner"), "proxy.example.test"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://proxy.example.test:1297/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "C".repeat(43)); + HttpClientCredentialStore.saveEnrolled(credentials, code, identity.issueClientCertificate("lobby-1")); + VotingPluginMain plugin = mock(VotingPluginMain.class); + BungeeSettings settings = mock(BungeeSettings.class); + when(plugin.getDataFolder()).thenReturn(directory.toFile()); + when(plugin.getBungeeSettings()).thenReturn(settings); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + when(settings.getServer()).thenReturn("lobby-1"); + when(settings.getHttpConnectionCode()).thenReturn(""); + java.lang.reflect.Field ownersField = HttpBackendProxyTransport.class.getDeclaredField("DIRECTORY_OWNERS"); + ownersField.setAccessible(true); + var owners = (java.util.concurrent.ConcurrentHashMap) ownersField.get(null); + java.util.concurrent.Semaphore predecessor = new java.util.concurrent.Semaphore(0); + owners.put(credentials.toAbsolutePath().normalize(), predecessor); + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(plugin); + transport.start(mock(GlobalMessageHandler.class)); + java.util.concurrent.atomic.AtomicReference failure = new java.util.concurrent.atomic.AtomicReference<>(); + CountDownLatch finished = new CountDownLatch(1); + Thread validation = new Thread(() -> { + try { transport.validate(); } + catch (Throwable thrown) { failure.set(thrown); } + finally { finished.countDown(); } + }); + validation.start(); + assertFalse(finished.await(150, TimeUnit.MILLISECONDS), "validation published before journal handoff"); + predecessor.release(); + assertTrue(finished.await(3, TimeUnit.SECONDS)); + validation.join(TimeUnit.SECONDS.toMillis(1)); + assertNull(failure.get()); + transport.close(); + } + private static HttpConnectionCode code(String serverId, Instant expiry) { return new HttpConnectionCode(serverId, URI.create("https://proxy.example.test:1297/"), "a".repeat(64), "b".repeat(64), expiry, "A".repeat(43)); From dd1ef83f2c3f1b646158679a6e2a0a2c895bb49d Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 21:16:23 -0600 Subject: [PATCH 022/136] fix(ci): restore complete proxy source --- .../votingplugin/proxy/VotingPluginProxy.java | 3684 ++++++++++++++++- 1 file changed, 3683 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 36b185f2d..5608a5ce8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1 +1,3683 @@ -YªçŠx-®éÜj×¢ëiºÚ+Š§j[h‘éÜ¢éíã}å:-jZ.¶›­–)Þ³W6¶vR6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡“° ¦–×÷'B¦fæ–òä'—FT'&”÷WGWE7G&VÓ°¦–×÷'B¦fæ–òäFF–çWE7G&VÓ°¦–×÷'B¦fæ–òäFF÷WGWE7G&VÓ°¦–×÷'B¦fæ–òäf–ÆS°¦–×÷'B¦fæ–òä”ôW†6WF–öã°¦–×÷'B¦fææWBä–æWE6ö6¶WDFG&W73°¦–×÷'B¦fææWBå6ö6¶WC°¦–×÷'B¦fææWBåU$“°¦–×÷'B¦fææWBæ‡GGä‡GG6Æ–VçC°¦–×÷'B¦fææWBæ‡GGä‡GG&WVW7C°¦–×÷'B¦fææWBæ‡GGä‡GG&W7öç6S°¦–×÷'B¦fç7Âå5ÄW†6WF–öã°¦–×÷'B¦fçF–ÖRäGW&F–öã°¦–×÷'B¦fçF–ÖRä–ç7FçC°¦–×÷'B¦fçF–ÖRäÆö6ÄFFUF–ÖS°¦–×÷'B¦fçF–ÖRå¦öæT–C°¦–×÷'B¦fçF–ÖRå¦öæTöfg6WC°¦–×÷'B¦fçWF–Âä'&”Æ—7C°¦–×÷'B¦fçWF–Âä6öÆÆV7F–öã°¦–×÷'B¦fçWF–Âä6öÆÆV7F–öç3°¦–×÷'B¦fçWF–Âä†6„Ö°¦–×÷'B¦fçWF–ÂäÆ–æ¶VD†6…6WC°¦–×÷'B¦fçWF–ÂäÆ—7C°¦–×÷'B¦fçWF–ÂäÖ°¦–×÷'B¦fçWF–Âå6WC°¦–×÷'B¦fçWF–ÂåUT”C°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBä6ö×ÆWF&ÆTgWGW&S°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBä6öæ7W'&VçD†6„Ö°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBäW†V7WF÷%6W'f–6S°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBäW†V7WF÷'3°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBå66†VGVÆVDW†V7WF÷%6W'f–6S°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBåF–ÖUVæ—C°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBæFöÖ–2äFöÖ–4&ööÆVã°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBæFöÖ–2äFöÖ–4Æöæs° ¦–×÷'B¦f‚ææWBç76Âå54Å&ÖWFW'3° ¦–×÷'B÷&ræV6Æ—6Rç†òæ6Æ–VçBæ×GGc2ä×GDW†6WF–öã° ¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ’çF–ÖRåF–ÖUG—S°¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ'VævVV’ævÆö&ÆFFävÆö&ÄFF†æFÆW%&÷‡“°¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ'VævVV’ævÆö&ÆFFävÆö&Äו5ð¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ'VævVV’çF–ÖRä'VævVUF–ÖT6†V6¶W#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’æVæ7'—F–öâäVæ7'—F–öä†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’æ§6öâä§6öå'6W#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ6öFV2ä§6öäVçfVÆ÷S°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ6öFV2ä§6öäVçfVÆ÷T6öFV3°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒævÆö&ÂävÆö&ÄÖW76vTÆ—7FVæW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒævÆö&ÂävÆö&ÄÖW76vU&÷‡”†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ×GBä×GD†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ×GBä×GE6W'fW$6öÖÓ°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ×—7Âäו7ÄÖW76VævW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç&VF—2å&VF—4†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç&VF—2å&VF—4Æ—7FVæW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç6ö6¶WG2ä6Æ–VçD†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç6ö6¶WG2å6ö6¶WD†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç6ö6¶WG2å6ö6¶WE&V6V—fW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7Âä6öÇVÖã°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂäFFG—S°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVS°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVT&ööÆVã°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVT–çC°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVU7G&–æs°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7Âæ×—7Âæ6öæf–rä×—7Ä6öæf–s°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âæ&6¶VæG&÷‡’æ‡GGä‡GGVç&öÆÆÖVçDWF†÷&—G“°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âæ&6¶VæG&÷‡’æ‡GGä‡GG&÷‡•G&ç7÷'E6W'fW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âæ&6¶VæG&÷‡’æ‡GGä‡GGFÇ4–FVçF—G“°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ'&öF67Bå&÷‡”'&öF67DFV6–FW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Rä•f÷FT66†S°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Råf÷FT66†T†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Rææöçf÷FVBä”æöåf÷FVEÆ–W'57F÷&vS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Rææöçf÷FVBäæöåf÷FVEÆ–W'466†S°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ6öçG&öÂä6öçG&öÄ6öææV7F÷#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ6öçG&öÂä†÷7FVD6öçG&öÄÖævW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡”†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡”ÖWF†öC°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡•6W'fW%6ö6¶WD6öæf–wW&F–öã°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡•6W'fW%6ö6¶WD6öæf–wW&F–öä'VævVS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’ç&W6Væ6Rä&6¶VæEÆ–W%&W6Væ6UG&6¶W#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’ç&W6Væ6RåÆ–W%&W6Væ6S°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçF–ÖWVWVRåf÷FUF–ÖUVWVS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçF÷f÷FW"åF÷f÷FW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçWF–ÂäÖ–æV7&gEW6W&æÖUfÆ–FF÷#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçWF–Âå6W'f–6U6—FUfÆ–FF÷#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçf÷FVÆöråf÷FTÆöt×—7ÅF&ÆS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçf÷FVÆöråf÷FTÆöt×—7ÅF&ÆRåf÷FTÆöu7FGW3°¦–×÷'B6öÒævöövÆRæw6öâä§6öäVÆVÖVçC°¦–×÷'B6öÒævöövÆRæw6öâä§6öäö&¦V7C° ¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2äFVfVÇD¦VF—46Æ–VçD6öæf–s°¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2ä†÷7DæE÷'C°¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2ä¦VF—3°¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2ä¦VF—5ööð ¦–×÷'BÆöÖ&ö²ävWGFW#°¦–×÷'BÆöÖ&ö²å6WGFW#° §V&Æ–2'7G&7B6Æ72f÷F–æuÇVv–å&÷‡’° —&—fFR7FF–2f–æÂÆöær$U4Tä4Uô„äDôdeõD”ÔTõUEôÔ”ÄÄ•2ÒF–ÖUVæ—BäÔ”åUDU2çFôÖ–ÆÆ—2ƒ"“° —&—fFR7FF–2f–æÂÆöær$U4Tä4Uõ5D%EUõ$U5”ä5ôDTÄ•õ4T4ôäE2ÒTð —&—fFR7FF–2f–æÂÆöær$U4Tä4UôÔ”åDTää4Uô”åDU%dÅõ4T4ôäE2Ò3ð —&—fFR7FF–2f–æÂÆöær$U4Tä4Uô$4´TäEõD”ÔTõUEôÔ”ÄÄ•2ÒF–ÖUVæ—Bå4T4ôäE2çFôÖ–ÆÆ—2ƒ““° —&—fFR7FF–2f–æÂÆöær4ôåE$ôÅôTå$ôÄÄÔTåEôÔ”åô”åDU%dÅôääõ2ÒF–ÖUVæ—Bå4T4ôäE2çFôææ÷2ƒ“°  ”vWGFW  ”6WGFW  —&—fFR–çBf÷FU'G•f÷FW2Ò°  ”vWGFW  ”6WGFW  —&—fFR–çB7W'&VçEf÷FU'G•f÷FW5&WV—&VBÒ°  ”vWGFW  ”6WGFW  —&—fFR&÷‡”×—7ÅW6W%F&ÆR&÷‡”ו5ð  —&—fFRVæ7'—F–öä†æFÆW"Væ7'—F–öä†æFÆW#°  —&—fFR†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â6Æ–VçD†æFÆW3°  —&—fFR6ö6¶WD†æFÆW"6ö6¶WD†æFÆW#° —&—fFR‡GG&÷‡•G&ç7÷'E6W'fW"‡GGG&ç7÷'E6W'fW#° —&—fFR‡GGVç&öÆÆÖVçDWF†÷&—G’‡GGVç&öÆÆÖVçDWF†÷&—G“°  ”vWGFW  ”6WGFW  —&—fFR&ööÆVâf÷F–f–W$Væ&ÆVBÒG'VS°  ”vWGFW  —&—fFR6öæ7W'&VçD†6„ÖÅUT”BÂ7G&–æsâWV–EÆ–W$æÖT66†RÒæWr6öæ7W'&VçD†6„ÖÃâ‚“°  ”vWGFW  ”6WGFW  —&—fFRvÆö&ÄFF†æFÆW%&÷‡’vÆö&ÄFF†æFÆW#°  ”vWGFW  —&—fFR&VF—4†æFÆW"&VF—4†æFÆW#° —&—fFR¦VF—5ööÂ&VF—5V&Æ—6†W%ööð —&—fFRföÆF–ÆRÆöær&VF—5V&Æ—6†W%&WG'”gFW#° —&—fFR&ööÆVâF–ÖUf÷FU&WG'•66†VGVÆVC° —&—fFR&ööÆVâF–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVC° —&—fFR&ööÆVâ66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVC°  —&—fFR&ööÆVâVæ&ÆVC°  ”vWGFW  ”6WGFW  —&—fFR×VÇF•&÷‡”†æFÆW"×VÇF•&÷‡”†æFÆW#°  ”vWGFW  —&—fFR'VævVUF–ÖT6†V6¶W"'VævVUF–ÖT6†V6¶W#°  ”vWGFW  ”6WGFW  —&—fFR'VævVTÖWF†öBÖWF†öC°  ”vWGFW  —&—fFR×GD†æFÆW"×GD†æFÆW#°  ”vWGFW  —&—fFRvÆö&ÄÖW76vU&÷‡”†æFÆW"vÆö&ÄÖW76vU&÷‡”†æFÆW#°  ”vWGFW  ”6WGFW  —&—fFRו7ÄÖW76VævW"&÷‡”×—7ÄÖW76VævW#°  ”vWGFW  —&—fFRf÷FT66†T†æFÆW"f÷FT66†T†æFÆW#°  ”vWGFW  —&—fFRæöåf÷FVEÆ–W'466†Ræöåf÷FVEÆ–W'466†S°  ”vWGFW  —&—fFRf–æÂ&6¶VæEÆ–W%&W6Væ6UG&6¶W"&6¶VæEÆ–W%&W6Væ6UG&6¶W"ÒæWr&6¶VæEÆ–W%&W6Væ6UG&6¶W"‚“° —&—fFRf–æÂÖÅUT”BÂVæF–æu&W6Væ6T†æFöfcâVæF–æu&W6Væ6T†æFöfg2ÒæWr†6„ÖÃâ‚“° —&—fFRf–æÂ6WCÅ7G&–æsâVæF–æt&6¶VæE&V6÷fW'•6æ6†÷G2Ò6öæ7W'&VçD†6„ÖææWt¶W•6WB‚“° —&—fFRf–æÂÖÅ7G&–ærÂÆöæsâ6öçG&öÄVç&öÆÆÖVçDæW‡DÆÆ÷vVBÒæWr6öæ7W'&VçD†6„ÖÃâ‚“° —&—fFRf–æÂÖÅUT”BÂVæF–æt6öÖ×Væ–6F–öåFW7CâVæF–æt6öÖ×Væ–6F–öåFW7G2ÒæWr6öæ7W'&VçD†6„ÖÃâ‚“° —&—fFRföÆF–ÆR6öçG&öÄ6öææV7F÷"6öçG&öÄ6öææV7F÷#° —&—fFRföÆF–ÆR†÷7FVD6öçG&öÄÖævW"†÷7FVD6öçG&öÄÖævW#° —&—fFRf–æÂö&¦V7B6öçG&öÄÆ–fV7–6ÆTÆö6²ÒæWrö&¦V7B‚“° —&—fFRf–æÂFöÖ–4Æöær6öçG&öÅ6W'f–6W4vVæW&F–öâÒæWrFöÖ–4Æöær‚“° —&—fFRf–æÂW†V7WF÷%6W'f–6R6öçG&öÄÆ–fV7–6ÆTW†V7WF÷"ÒW†V7WF÷'2ææWu6–ævÆUF‡&VDW†V7WF÷"‡F6²Óâ° •F‡&VBF‡&VBÒæWrF‡&VB‡F6²Â'f÷F–æwÇVv–âÖ6öçG&öÂÖÆ–fV7–6ÆR"“° —F‡&VBç6WDFVÖöâ‡G'VR“° —&WGW&âF‡&VC° —Ò“°  —V&Æ–2f÷F–æuÇVv–å&÷‡’‚’° –Væ&ÆVBÒG'VS°  –'VævVUF–ÖT6†V6¶W"ÒæWr'VævVUF–ÖT6†V6¶W"†vWD6öæf–r‚’ævWEF–ÖU¦öæR‚’ÂvWD6öæf–r‚’ævWEF–ÖT†÷W$öfe6WB‚’À –vWD6öæf–r‚’ævWEF–ÖUvVV´öfe6WB‚’’°  ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…7G&–ærFW‡B’° –FV'Vs"‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2ÆöærvWDÆ7EWFFVB‚’° —&WGW&âvWEf÷FT66†TÆ7EWFFVB‚“° —Р ”÷fW'&–FP —V&Æ–2–çBvWE&WdF’‚’° —&WGW&âvWEf÷FT66†U&WdF’‚“° —Р ”÷fW'&–FP —V&Æ–27G&–ærvWE&WdÖöçF‚‚’° —&WGW&âvWEf÷FT66†U&WdÖöçF‚‚“° —Р ”÷fW'&–FP —V&Æ–2–çBvWE&WevVV²‚’° —&WGW&âvWEf÷FT66†U&WevVV²‚“° —Р ”÷fW'&–FP —V&Æ–2fö–B–æfò…7G&–ærFW‡B’° –Æör‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2&ööÆVâ—4Væ&ÆVB‚’° —&WGW&âVæ&ÆVC° —Р ”÷fW'&–FP —V&Æ–2&ööÆVâ—4–væ÷&UF–ÖR‚’° —&WGW&â—5f÷FT66†T–væ÷&UF–ÖR‚“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WD–væ÷&UF–ÖR†&ööÆVâ–væ÷&R’° —6WEf÷FT66†Uf÷FT66†T–væ÷&UF–ÖR†–væ÷&R“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WDÆ7EWFFVB‚’° —6WEf÷FT66†TÆ7EWFFVB‚“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WE&WdF’†–çBF’’° —6WEf÷FT66†U&WdF’†F’“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WE&WdÖöçF‚…7G&–ærFW‡B’° —6WEf÷FT66†U&WdÖöçF‚‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WE&WevVV²†–çBvVV²’° —6WEf÷FT66†U&WevVV²‡vVV²“° —Р ”÷fW'&–FP —V&Æ–2fö–BF–ÖT6†ævVB…F–ÖUG—RG—RÂ&ööÆVâf¶RÂ&ööÆVâ&RÂ&ööÆVâ÷7B’° ––b†vWD6öæf–r‚’ævWEf÷FT66†UF–ÖR‚’â’° –vWEf÷FT66†T†æFÆW"‚’æ6†V6µf÷FT66†UF–ÖR†vWD6öæf–r‚’ævWEf÷FT66†UF–ÖR‚’“° —Р––b‚vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’’° —v&â‚$vÆö&ÂFFæ÷BVæ&ÆVB–væ÷&–ærF–ÖR6†ævRWfVçB"“° —&WGW&ã° —Р––çBFVÆ’Ò° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° ––b†vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æ6öçF–ç4¶W’‡2’’° •7G&–ærÆ7DöæÆ–æU7G"ÒvWDvÆö&ÄFF†æFÆW"‚’ævWE7G&–ær‡2Â$Æ7DöæÆ–æR"“° –ÆöærÆ7DöæÆ–æRÒ° —G'’° –Æ7DöæÆ–æRÒÆöærçfÇVTöb†Æ7DöæÆ–æU7G"“° —Ò6F6‚„çVÖ&W$f÷&ÖDW†6WF–öâR’° ’òò–væ÷&P —Р ––b„Æö6ÄFFUF–ÖRææ÷r‚’æE¦öæR…¦öæTöfg6WBåUD2’çFô–ç7FçB‚’çFôWö6„Ö–ÆÆ’‚’ÒÆ7DöæÆ–æR ’¢c¢c¢"’° ”†6„ÖÅ7G&–ærÂFFfÇVSâFFFõ6WBÒæWr†6„ÖÃâ‚“° –FFFõ6WBçWB‚$Æ7EWFFVB"ÂæWrFFfÇVU7G&–ær€ ’""²Æö6ÄFFUF–ÖRææ÷r‚’æE¦öæR…¦öæTöfg6WBåUD2’çFô–ç7FçB‚’çFôWö6„Ö–ÆÆ’‚’’“° –FFFõ6WBçWB‚$f–æ—6†VE&ö6W76–ær"ÂæWrFFfÇVT&ööÆVâ†fÇ6R’“° –FFFõ6WBçWB‡G—RçFõ7G&–ær‚’ÂæWrFFfÇVT&ööÆVâ‡G'VR’“° –vWDvÆö&ÄFF†æFÆW"‚’ç6WDFF‡2ÂFFFõ6WB“°  –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡2ÂFVÆ’Âf÷F–æuÇVv–åv—&Ræ'VævVUF–ÖT6†ævR‚’“° –FVÆ’²³° —ÒVÇ6R° —v&â‚%6W'fW""²2²"†6âwB&VVâöæÆ–æR&V6VçFÇ’"“° —Р—ÒVÇ6R° —v&â‚%6W'fW""²2²"vÆö&ÂFF†æFÆW"F—6&ÆVCò"“° —Р—Р–vÆö&ÄFF†æFÆW"æöåF–ÖT6†ævR‡G—R“° —Р ”÷fW'&–FP —V&Æ–2fö–Bv&æ–ær…7G&–ærFW‡B’° —v&â‡FW‡B“° —Р—Ó° —Р —V&Æ–2fö–BöåF–ÖT6†ævVDf–ÆVB…7G&–ær7'bÂF–ÖUG—RG—R’° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡7'bÂG—RçFõ7G&–ær‚’ÂfÇ6R“° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡7'bÂ$f–æ—6†VE&ö6W76–ær"ÂG'VR“° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡7'bÂ%&ö6W76–ær"ÂfÇ6R“° —Р —V&Æ–2fö–BöåF–ÖT6†ævVDf–æ—6†VB…F–ÖUG—RG—R’° ––b‡G—RæWVÇ2…F–ÖUG—RäÔôåD‚’’° –vWE&÷‡”ו5‚’æ6÷”6öÇVÖäFF…F÷f÷FW"äÖöçF†Ç’ævWD6öÇVÖäæÖR‚’Â$Æ7DÖöçF…F÷FÂ"“° —Р–vWE&÷‡”ו5‚’çv—T6öÇVÖäFF…F÷f÷FW"æöb‡G—R’ævWD6öÇVÖäæÖR‚’ÂFFG—Rä”åDTtU"“°  ––b‚vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’’° —&WGW&ã° —Р–f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡2Â$f÷&6UWFFR"ÂG'VR“° –vWDvÆö&ÄÖW76vU&÷‡”†æFÆW"‚’ç6VæDÖW76vR‡2ÂÂf÷F–æuÇVv–åv—&Ræ'VævVUF–ÖT6†ævR‚’“° —Р—&ö6W75VWVR‚“° —Р ’ò¢  ’¢ÆöBו5²vÆö&ÂFF†æFÆW"à ’¢ð —V&Æ–2fö–BÆöD×—7„ח7Ä6öæf–r×—7Ä6öæf–rÂ×—7Ä6öæf–rvÆö&ÄFF×—7Ä6öæf–r’° ––b†×—7Ä6öæf–rævWD†÷7DæÖR‚’æ—4V×G’‚’ÇÂ×—7Ä6öæf–rævWDFF&6R‚’æ—4V×G’‚’’° –Æöu6WfW&R‚$ו5—2æ÷B6öæf–wW&VB6÷'&V7FÇ’â"²$Ö—76–ær†÷7BöFF&6Râ†÷7CÒ"²×—7Ä6öæf–rævWD†÷7DæÖR‚ ’²"F#Ò"²×—7Ä6öæf–rævWDFF&6R‚’“° —6WE&÷‡”ו5†çVÆ“° —&WGW&ã° —Р —6WE&÷‡”ו5†æWr&÷‡”×—7ÅW6W%F&ÆR‚%f÷F–æuÇVv–åõW6W'2"Â×—7Ä6öæf–rÂvWD6öæf–r‚’ævWDFV'Vr‚’’°  ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…5ÄW†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&R…7G&–ær7G&–ær’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöu6WfW&R‡7G&–ær“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöt–æfò…7G&–ær7G&–ær’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöt–æfò‡7G&–ær“° —Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…F‡&÷v&ÆRB’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° —Bç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…7G&–ær7G"’° –FV'Vs"‡7G"“° —Р—Ò“°  ”'&”Æ—7CÅ7G&–æsâ6W'fW'2ÒæWr'&”Æ—7CÅ7G&–æsâ†vWDÆÄf–Æ&ÆU6W'fW'2‚’“°  ––b†vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’’° ––b†vWD6öæf–r‚’ævWDvÆö&ÄFFW6TÖ–äו5‚’’° —6WDvÆö&ÄFF†æFÆW"†æWrvÆö&ÄFF†æFÆW%&÷‡’€ –æWrvÆö&Äו5‚%f÷F–æuÇVv–åôvÆö&ÄFF"ÂvWE&÷‡”ו5‚’ævWD×—7‚’’°  ”÷fW'&–FP —V&Æ–2fö–BFV'VtW‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'VtÆör…7G&–ærFW‡B’° –FV'Vr‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–B–æfò…7G&–ærFW‡B’° –Æöt–æfò‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&R…7G&–ærFW‡B’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöu6WfW&R‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–Bv&æ–ær…7G&–ærFW‡B’° —v&â‡FW‡B“° —Р—ÒÂ6W'fW'2’°  ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–ÆVB…7G&–ær7'bÂF–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–ÆVB‡7'bÂG—R“° —Р ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–æ—6†VB…F–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–æ—6†VB‡G—R“° —Р—Ò“° —ÒVÇ6R° —6WDvÆö&ÄFF†æFÆW"€ –æWrvÆö&ÄFF†æFÆW%&÷‡’†æWrvÆö&Äו5‚%f÷F–æuÇVv–åôvÆö&ÄFF"ÂvÆö&ÄFF×—7Ä6öæf–r’°  ”÷fW'&–FP —V&Æ–2fö–BFV'VtW‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'VtÆör…7G&–ærFW‡B’° –FV'Vr‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–B–æfò…7G&–ærFW‡B’° –Æöt–æfò‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&R…7G&–ærFW‡B’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöu6WfW&R‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–Bv&æ–ær…7G&–ærFW‡B’° —v&â‡FW‡B“° —Р—ÒÂ6W'fW'2’°  ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–ÆVB…7G&–ær7'bÂF–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–ÆVB‡7'bÂG—R“° —Р ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–æ—6†VB…F–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–æ—6†VB‡G—R“° —Р—Ò“° —Р ’òòWFFRvÆö&Â66†VÖ6öÇVÖç2‡Væ6†ævVBg&öÒ÷&–v–æ –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$–væ÷&UF–ÖR"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$ÔôåD‚"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚%tTT²"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$D’"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$f–æ—6†VE&ö6W76–ær"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚%&ö6W76–ær"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$f÷&6UWFFR"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$Æ7EWFFVB"Â$ÔTD•TÕDU…B"“° —Р ’òò6öÇVÖâG—W2‡Væ6†ævVBg&öÒ÷&–v–æ –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%F÷f÷FW$–væ÷&R"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$6†V6µv÷&ÆB"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%&VÖ–æFVB"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F—6&ÆT'&öF67B"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7DöæÆ–æR"Â%d$4„"ƒ#’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%Æ–W$æÖR"Â%d$4„"ƒ3’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F–Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%vVV¶Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F•f÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$&W7DF•f÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%vVVµf÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$&W7EvVVµf÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%f÷FU'G•f÷FW2"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$ÖöçF…f÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%ö–çG2"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$†–v†W7DF–Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$ÆÅF–ÖUF÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$†–v†W7DÖöçF†Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$ÖöçF…F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$†–v†W7EvVV¶Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7DÖöçF…F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7EvVV¶Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7DF–Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$öffÆ–æU&Wv&G2"Â$ÔTD•TÕDU…B"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F•f÷FU7G&V´Æ7EWFFR"Â$ÔTD•TÕDU…B"“°  ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R†vWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖRææ÷r‚’’Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R†vWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖRææ÷r‚’çÇW4ÖöçF‡2ƒ’’À ’$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R†vWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖRææ÷r‚’çÇW4ÖöçF‡2ƒ"’’À ’$”åBDTdTÅBsr"“° —Р—Р —V&Æ–2fö–BFD7W'&VçEf÷FU'G•f÷FW2†–çBÖ÷VçB’° —f÷FU'G•f÷FW2³ÒÖ÷VçC° —6WEf÷FT66†Uf÷FU'G”7W'&VçEf÷FW2‡f÷FU'G•f÷FW2“° –FV'Vr‚$7W'&VçBf÷FR'G’F÷Fâ"²f÷FU'G•f÷FW2“° —Р —V&Æ–2fö–BFDæöåf÷FVEÆ–W"…7G&–ærWV–BÂ7G&–ærÆ–W$æÖR’° –æöåf÷FVEÆ–W'466†RæFEÆ–W"‡WV–BÂÆ–W$æÖR“° —Р —V&Æ–2fö–BFEf÷FU'G’‚’° ––b†vWD6öæf–r‚’ævWEf÷FU'G”Væ&ÆVB‚’’° –FD7W'&VçEf÷FU'G•f÷FW2ƒ“° –6†V6µf÷FU'G’‚“° —Р—Р —V&Æ–2'7G&7Bfö–B'&öF67B…7G&–ærÖW76vR“°  —&—fFR6WCÅ7G&–æsâ6VæE&÷‡”'&öF67B…6WCÅ7G&–æsâF&vWG2Â7G&–ærWV–BÂ7G&–ærÆ–W"Â7G&–ær6W'f–6RÂÆöærF–ÖRÀ •7G&–ærFW‡BÂ&ööÆVâv4öæÆ–æR’° •6WCÅ7G&–æsâf÷'v&FVBÒæWrÆ–æ¶VD†6…6WCÃâ‚“° –f÷"…7G&–ærF&vWE6W'fW"¢F&vWG2’° ”§6öäVçfVÆ÷RVçfVÆ÷RÒf÷F–æuÇVv–åv—&Rçf÷FT'&öF67B‡WV–BÂÆ–W"Â6W'f–6RÂF–ÖRÂFW‡BÂv4öæÆ–æR“° ––b‡6VæE&÷‡”'&öF67DVçfVÆ÷Tæ÷r‡F&vWE6W'fW"ÂVçfVÆ÷R’’° –f÷'v&FVBæFB‡F&vWE6W'fW"“° —Р—Р—&WGW&âf÷'v&FVC° —Р ’ò¢  ’¢6VæG27FæFÆöæR&÷‡’'&öF67BF‡&÷Vv‚F†R6VÆV7FVBG&ç7÷'BæB&W÷'G0 ’¢v†WF†W"F†BG&ç7÷'B66WFVBF†RÖW76vRà ’  ’¢&Ò6W'fW"F&vWB&6¶VæB6W'fW  ’¢&ÒVçfVÆ÷R7FæFÆöæR'&öF67BVçfVÆ÷P ’¢&WGW&âG'VRöæÇ’v†VâF†RG&ç7÷'B66WFVBF†RÖW76vP ’¢ð —&÷FV7FVB&ööÆVâ6VæE&÷‡”'&öF67DVçfVÆ÷Tæ÷r…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° —7v—F6‚†ÖWF†öB’° –66RÕEC  —&WGW&â6VæD×GDVçfVÆ÷U6W'fW"‡6W'fW"ÂVçfVÆ÷R“° –66RÕ•5à ––b‡&÷‡”×—7ÄÖW76VævW"ÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р—G'’° —&÷‡”×—7ÄÖW76VævW"ç6VæEFô&6¶VæB‡6W'fW"ÂVçfVÆ÷R“° —&WGW&âG'VS° —Ò6F6‚…5ÄW†6WF–öâR’° –FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р–66RÅTt”äÔU54t”äs  —&WGW&â6VæEÇVv–äÖW76vU6W'fW$æ÷r‡6W'fW"ÂVçfVÆ÷R“° –66R$TD•3  —&WGW&â6VæE&VF—4VçfVÆ÷U6W'fW"‡6W'fW"ÂVçfVÆ÷RÂG'VR“° –66R4ô4´UE3  ’òò7FæFÆöæR'&öF67G2W6RF†R6ÖR–æ—F–Æ—¦VB6Æ–VçB2æ÷&ÖÀ ’òòVçfVÆ÷W2âF†—2&W6W'fW2F†R6ö6¶WB6öææV7F–öâæB—G2FVÆ—fW' ’òò6¶æ÷vÆVFvVÖVçB–ç7FVBöb7&VF–ær6V6öæB6†÷'BÖÆ—fVB6ö6¶WBà —&WGW&â6VæE6ö6¶WDVçfVÆ÷R‡6W'fW"ÂVçfVÆ÷R“° –66R…EE  —&WGW&â6VæD‡GGVçfVÆ÷R‡6W'fW"ÂVçfVÆ÷R“° –FVfVÇC  —&WGW&âfÇ6S° —Р—Р ’ò¢  ’¢6VæG2&Wv&BÖ&V&–ærf÷FRVçfVÆ÷RæB&W÷'G2v†WF†W"F†R6VÆV7FV@ ’¢G&ç7÷'B66WFVB—BâÆVv7’G&ç7÷'G2&WF–âF†V—"W†—7F–ær7–æ6‡&öæ÷W0 ’¢6VÖçF–73²…EEW‡÷6W2—G2&÷VæFVB×VWVR&W7VÇB6òf÷FR—2æWfW"F—66&FV@ ’¢v†VâF†RVWVR—2gVÆÂà ’¢ð —&÷FV7FVB&ööÆVâ6VæEf÷FTVçfVÆ÷T66WFVB…7G&–ær6W'fW"–çBFVĤ6öäVçfVÆ÷RVçfVÆ÷R’° ––b†ÖWF†öBÓÒ'VævVTÖWF†öBä…EE’° —&WGW&â6VæD‡GGVçfVÆ÷R‡6W'fW"ÂVçfVÆ÷R“° —Р”vÆö&ÄÖW76vU&÷‡”†æFÆW"†æFÆW"ÒvÆö&ÄÖW76vU&÷‡”†æFÆW#° ––b††æFÆW"ÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р–†æFÆW"ç6VæDÖW76vR‡6W'fW"ÂFVÆ’ÂVçfVÆ÷R“° —&WGW&âG'VS° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–B6†V6´66†VEf÷FW2…7G&–ær6W'fW"’° ––çBFVÆ’Ò° ––b†—56W'fW%fÆ–B‡6W'fW"’’° ––b†—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡6W'fW"’’° ––b†vWEf÷FT66†T†æFÆW"‚’æ†5f÷FW2‡6W'fW"’bbvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ2ÒvWEf÷FT66†T†æFÆW"‚’ævWEf÷FW2‡6W'fW"“° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ&VÖ÷fVBÒæWr'&”Æ—7CÃâ‚“° ––b‚2æ—4V×G’‚’’° ––çBçVÒÒ° ––çBçVÖ&W$öef÷FW2Ò2ç6—¦R‚“° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢2’° ––b†66†Ræ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7E6W'fW%f÷FTFVÆ—fW'’‡6W'fW"Â66†R’’° –6öçF–çVS° —Р––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb66†RææVVG4'&öF67Döâ‡6W'fW"’’° •6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡6W'fW"’À –66†RævWEWV–B‚’Â66†RævWEÆ–W$æÖR‚’Â66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’À –66†RævWEFW‡B‚’ÂfÇ6R“° ––b†66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° ––b‚W'6—7E6W'fW%f÷FTFVÆ—fW'’‡6W'fW"Â66†R’’° –6öçF–çVS° —Р—Р—Р –&ööÆVâFõ6VæBÒG'VS° ––b†vWD6öæf–r‚’ævWEv—Df÷%W6W$öæÆ–æR‚’’° ––b‚—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’’’° —Fõ6VæBÒfÇ6S° —ÒVÇ6R–b†—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’ ’bbvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’’æWVÇ2‡6W'fW"’’° —Fõ6VæBÒfÇ6S° —Р—Р––b‡Fõ6VæB’° –&ööÆVâ'&öF67D†W&RÒ66†RææVVG4'&öF67Döâ‡6W'fW"“° ––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb'&öF67D†W&P ’bbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° –&ööÆVâÆ–W$öæÆ–æRÒ—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’“° •7G&–ærÆ–W%6W'fW"ÒÆ–W$öæÆ–æRòvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’ “¢çVÆð  •6WCÅ7G&–æsâF&vWG2Ò&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡Æ–W$öæÆ–æRÀ —Æ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡6W'fW"ÂF&vWG2“° —Р ––b‚6VæEf÷FTVçfVÆ÷T66WFVB‡6W'fW"ÂFVÆ’À •f÷F–æuÇVv–åv—&Rçf÷FR†66†RævWEÆ–W$æÖR‚’Â66†RævWEWV–B‚’À –66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’ÂfÇ6RÂ66†Ræ—5&VÅf÷FR‚’À –66†RævWEFW‡B‚’Â66†RævWEf÷FT–B‚’ÂvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’À –'&öF67D†W&RÂçVÒÂçVÖ&W$öef÷FW2’’’° –FV'Vr‚%&WF–æ–ær66†VBf÷FR&V6W6RF†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²6W'fW"“° –6öçF–çVS° —Р–FVÆ’²³° –çVÒ²³° —&VÖ÷fVBæFB†66†R“° —ÒVÇ6R° –FV'Vr‚$æ÷B6VæF–ærf÷FR&V6W6RW6W"—6âwBöâ6W'fW""²6W'fW"²#¢  ’²66†RçFõ7G&–ær‚’“° —Р—Р–vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fU6W'fW%f÷FW2‡6W'fW"Â&VÖ÷fVB“° —ÒVÇ6R° –FV'Vr‚$æò66†VBf÷FW2f÷"6W'fW#¢"²6W'fW"“° —Р—ÒVÇ6R° –FV'Vr‚$æò66†VBf÷FW2f÷"6W'fW#¢"²6W'fW"“° —Р—Р—ÒVÇ6R° –FV'Vr‚%6W'fW"æ÷BfÆ–C¢"²6W'fW"“° —Р—Р —V&Æ–27–æ6‡&öæ—¦VBfö–B6†V6´öæÆ–æUf÷FW2…7G&–ærÆ–W"Â7G&–ærWV–BÂ7G&–ær6W'fW"’° ––çBFVÆ’Ò° ––b†—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær‡Æ–W"’bbvWEf÷FT66†T†æFÆW"‚’æ†4öæÆ–æUf÷FW2‡WV–B’’° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ2ÒvWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2‡WV–B“° ––b‚2æ—4V×G’‚’’° ––b‡6W'fW"ÓÒçVÆÂ’° —6W'fW"ÒvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær‡Æ–W"“° —Р––b‚vWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° ––çBçVÒÒ° ––çBçVÖ&W$öef÷FW2Ò†–çB’2ç7G&VÒ‚’æf–ÇFW"‡f÷FRÓâf÷FRæ—5&Wv&DFVÆ—fW&VB‚’’æ6÷VçB‚“° –&ööÆVâFVÆ—fW&VE&Wv&BÒfÇ6S° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ&WF–æVBÒæWr'&”Æ—7CÃâ‚“° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢2’° ––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’’° •6WCÅ7G&–æsâVæF–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ†66†RævWD'&öF67EF&vWG2‚’“° —VæF–æuF&vWG2ç&VÖ÷fTƆ66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b†&Æö6¶VE6W'fW'2ÒçVÆÂ’° —VæF–æuF&vWG2ç&VÖ÷fTƆ&Æö6¶VE6W'fW'2“° —Р–66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƇ6VæE&÷‡”'&öF67B‡VæF–æuF&vWG2À –66†RævWEWV–B‚’Â66†RævWEÆ–W$æÖR‚’Â66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’À –66†RævWEFW‡B‚’ÂfÇ6R’“° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° —Р–&ööÆVâ'&öF67D†W&RÒ66†RææVVG4'&öF67Döâ‡6W'fW"“° ––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb'&öF67D†W&P ’bbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° •7G&–ærÆ–W%6W'fW"Ò‡6W'fW"ÒçVÆÂ’ò6W'fW"¢vWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær‡Æ–W"“°  •6WCÅ7G&–æsâF&vWG2Ò&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡G'VRÂÆ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡6W'fW"ÂF&vWG2“° —Р ––b‚66†Ræ—5&Wv&DFVÆ—fW&VB‚’’° ––b‚6VæEf÷FTVçfVÆ÷T66WFVB‡6W'fW"ÂFVÆ’À •f÷F–æuÇVv–åv—&Rçf÷FTöæÆ–æR†66†RævWEÆ–W$æÖR‚’Â66†RævWEWV–B‚’Â66†RævWE6W'f–6R‚’À –66†RævWEF–ÖR‚’ÂfÇ6RÂ66†Ræ—5&VÅf÷FR‚’Â66†RævWEFW‡B‚’Â66†RævWEf÷FT–B‚’À –vWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’Â'&öF67D†W&RÂçVÒÂçVÖ&W$öef÷FW2’’’° –FV'Vr‚%&WF–æ–æröæÆ–æRf÷FR&V6W6RF†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²6W'fW"“° —&WF–æVBæFB†66†R“° –6öçF–çVS° —Р’òòF†Ræ÷&ÖÂVçfVÆ÷R—2Ç6òfÆ–B'&öF67BFVÆ—fW'’f÷"F†P ’òò7W'&VçBF&vWBâ&V6÷&B—B6ò&Wf–÷W6Ç’VæF–ær7FæFÆöæP ’òò&WG'’6ææ÷Bææ÷Væ6RF†R6ÖRf÷FRv–âÆFW"à ––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb'&öF67D†W&R’° –66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFB‡6W'fW"“° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° —Р–66†Rç6WE&Wv&DFVÆ—fW&VB‡G'VR“° –FVÆ—fW&VE&Wv&BÒG'VS° –FVÆ’²³° –çVÒ²³° —Р ––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° —&WF–æVBæFB†66†R“° —Р—Р–vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fTöæÆ–æUf÷FW2‡WV–B“° –f÷"„öffÆ–æT'VævVUf÷FRVæF–ær¢&WF–æVB’° –vWEf÷FT66†T†æFÆW"‚’æFDöæÆ–æUf÷FR‡WV–BÂVæF–ær“° —Р ’òò×VÇF—&÷‡“¢VçfVÆ÷RÖöæÇ ––b†FVÆ—fW&VE&Wv&BbbvWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚ ’bbvWD6öæf–r‚’ævWD×VÇF•&÷‡”öæTvÆö&Å&Wv&B‚’’° –×VÇF•&÷‡”†æFÆW"ç6VæD6ÆV%f÷FR‡WV–BÂÆ–W"“° —Р—Р—Р—Р—Р ’ò¢  ’¢&WG&–W2f÷FW"Ö¶W–VB7FæFÆöæR'&öF67G2v†Vâç’Æ–W"Ö¶W2F&vW@ ’¢&6¶VæBf–Æ&ÆR2ÇVv–âÖÖW76vR6'&–W"à ’  ’¢&Ò6W'fW"&6¶VæB6W'fW"F†Bv–æVB6'&–W  ’¢ð —&÷FV7FVB7–æ6‡&öæ—¦VBfö–B&WG'•VæF–ætöæÆ–æT'&öF67G2…7G&–ær6W'fW"’° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b‡6W'fW"ÓÒçVÆÂdž&Æö6¶VE6W'fW'2ÒçVÆÂbb&Æö6¶VE6W'fW'2æ6öçF–ç2‡6W'fW"’’’° —&WGW&ã° —Р–f÷"…7G&–ær66†VEWV–B¢vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FUUT”G2‚’’° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2†66†VEWV–B’’’° ––b†66†Ræ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R’’° –6öçF–çVS° —Р––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂ66†RææVVG4'&öF67Döâ‡6W'fW"’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡6W'fW"’Â66†RævWEWV–B‚’À –66†RævWEÆ–W$æÖR‚’Â66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’Â66†RævWEFW‡B‚’ÂfÇ6R“° ––b†66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° ––b†66†Ræ—5&Wv&DFVÆ—fW&VB‚’bb66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° –vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fTöæÆ–æUf÷FR†66†VEWV–BÂ66†R“° —ÒVÇ6R° —W'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R“° —Р—Р—Р—Р—Р —&÷FV7FVB7–æ6‡&öæ—¦VBfö–B&WG'•VæF–æuF–ÖT'&öF67G2…7G&–ær6W'fW"’° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b‡6W'fW"ÓÒçVÆÂdž&Æö6¶VE6W'fW'2ÒçVÆÂbb&Æö6¶VE6W'fW'2æ6öçF–ç2‡6W'fW"’’’° —&WGW&ã° —Р––b†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’ÓÒçVÆÂ’° —&WGW&ã° —Р–f÷"…f÷FUF–ÖUVWVRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR’’° –6öçF–çVS° —Р––b‚f÷FRæ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂf÷FRævWEWV–B‚’æ—4V×G’‚’ÇÂf÷FRævWD'&öF67EF&vWG2‚’æ6öçF–ç2‡6W'fW" —ÇÂf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡6W'fW"’Âf÷FRævWEWV–B‚’Âf÷FRævWDæÖR‚’À —f÷FRævWE6W'f–6R‚’Âf÷FRævWEF–ÖR‚’Âf÷FRævWEF÷FÇ2‚’ÂfÇ6R“° ––b‡f÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° —W'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR“° —Р—Р—Р ’ò¢  ’¢W&–öF–6ÆÇ’&WG&–W2WfW'’VæF–ærf÷FW"Ö¶W–VB7FæFÆöæR'&öF67BâF†—2—0 ’¢&WV—&VBf÷"'&ö¶W"G&ç7÷'G2v†÷6R&V6÷fW'’FöW2æ÷B&öGV6RÆ–W"ÖÆöv–à ’¢6'&–W"WfVçBà ’¢ð —V&Æ–27–æ6‡&öæ—¦VBfö–B&WG'•VæF–ætöæÆ–æT'&öF67G2‚’° –f÷"…7G&–ær66†VEWV–B¢æWrÆ–æ¶VD†6…6WCÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FUUT”G2‚’’’° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2†66†VEWV–B’’’° ––b†66†Ræ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R’’° –6öçF–çVS° —Р––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂ66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâVæF–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ†66†RævWD'&öF67EF&vWG2‚’“° —VæF–æuF&vWG2ç&VÖ÷fTƆ66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b†&Æö6¶VE6W'fW'2ÒçVÆÂ’° —VæF–æuF&vWG2ç&VÖ÷fTƆ&Æö6¶VE6W'fW'2“° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B‡VæF–æuF&vWG2Â66†RævWEWV–B‚’Â66†RævWEÆ–W$æÖR‚’À –66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’Â66†RævWEFW‡B‚’ÂfÇ6R“° ––b†66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° ––b†66†Ræ—5&Wv&DFVÆ—fW&VB‚’bb66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° –vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fTöæÆ–æUf÷FR†66†VEWV–BÂ66†R“° —ÒVÇ6R° —W'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R“° —Р—Р—Р—Р—&WG'•VæF–æuF–ÖT'&öF67G2‚“° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–B&WG'•VæF–æuF–ÖT'&öF67G2‚’° ––b†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’ÓÒçVÆÂ’° —&WGW&ã° —Р–f÷"…f÷FUF–ÖUVWVRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR’’° –6öçF–çVS° —Р––b‚f÷FRæ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂf÷FRævWEWV–B‚’æ—4V×G’‚’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâVæF–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ‡f÷FRævWD'&öF67EF&vWG2‚’“° —VæF–æuF&vWG2ç&VÖ÷fTƇf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b†&Æö6¶VE6W'fW'2ÒçVÆÂ’° —VæF–æuF&vWG2ç&VÖ÷fTƆ&Æö6¶VE6W'fW'2“° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B‡VæF–æuF&vWG2Âf÷FRævWEWV–B‚’Âf÷FRævWDæÖR‚’Âf÷FRævWE6W'f–6R‚’À —f÷FRævWEF–ÖR‚’Âf÷FRævWEF÷FÇ2‚’ÂfÇ6R“° ––b‡f÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° —W'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR“° —Р—Р—Р —&÷FV7FVB7–æ6‡&öæ—¦VB&ööÆVâW'6—7EF–ÖUf÷FTFVÆ—fW'’…f÷FUF–ÖUVWVRf÷FR’° ––b†vWEf÷FT66†T†æFÆW"‚’çWFFUF–ÖUf÷FR‡f÷FR’’° —f÷FRç6WDFVÆ—fW'•7FFTF—'G’†fÇ6R“° —&WGW&âG'VS° —Р—f÷FRç6WDFVÆ—fW'•7FFTF—'G’‡G'VR“° —66†VGVÆUF–ÖUf÷FTFVÆ—fW'•&WG'’‚“° —&WGW&âfÇ6S° —Р —&—fFRfö–B66†VGVÆUF–ÖUf÷FTFVÆ—fW'•&WG'’‚’° ––b‡F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÇÂvWE66†VGVÆW"‚’ÓÒçVÆÂ’° —&WGW&ã° —Р—F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒG'VS° —G'’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ° —7–æ6‡&öæ—¦VB…f÷F–æuÇVv–å&÷‡’çF†—2’° —F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° —Р—&WG'•VæF–æuF–ÖT'&öF67G2‚“° —ÒÂRÂF–ÖUVæ—Bå4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° —F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° –FV'Vr‚%Væ&ÆRFò66†VGVÆRF–ÖVB'&öF67B7FFR&WG'“¢"²RævWDÖW76vR‚’“° —Р—Р —&÷FV7FVB7–æ6‡&öæ—¦VB&ööÆVâW'6—7E6W'fW%f÷FTFVÆ—fW'’…7G&–ær6W'fW"ÂöffÆ–æT'VævVUf÷FRf÷FR’° ––b†vWEf÷FT66†T†æFÆW"‚’çWFFU6W'fW%f÷FR‡6W'fW"Âf÷FR’’° —f÷FRç6WDFVÆ—fW'•7FFTF—'G’†fÇ6R“° —&WGW&âG'VS° —Р—f÷FRç6WDFVÆ—fW'•7FFTF—'G’‡G'VR“° —66†VGVÆT66†VEf÷FTFVÆ—fW'•&WG'’‚“° —&WGW&âfÇ6S° —Р —&÷FV7FVB7–æ6‡&öæ—¦VB&ööÆVâW'6—7DöæÆ–æUf÷FTFVÆ—fW'’…7G&–ærWV–BÂöffÆ–æT'VævVUf÷FRf÷FR’° ––b†vWEf÷FT66†T†æFÆW"‚’çWFFTöæÆ–æUf÷FR‡WV–BÂf÷FR’’° —f÷FRç6WDFVÆ—fW'•7FFTF—'G’†fÇ6R“° —&WGW&âG'VS° —Р—f÷FRç6WDFVÆ—fW'•7FFTF—'G’‡G'VR“° —66†VGVÆT66†VEf÷FTFVÆ—fW'•&WG'’‚“° —&WGW&âfÇ6S° —Р —&—fFRfö–B66†VGVÆT66†VEf÷FTFVÆ—fW'•&WG'’‚’° ––b†66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÇÂvWE66†VGVÆW"‚’ÓÒçVÆÂ’° —&WGW&ã° —Р–66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒG'VS° —G'’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ° —7–æ6‡&öæ—¦VB…f÷F–æuÇVv–å&÷‡’çF†—2’° –66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° —Р—&WG'”66†VEf÷FTFVÆ—fW'•W'6—7FVæ6R‚“° —ÒÂRÂF–ÖUVæ—Bå4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° –66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° –FV'Vr‚%Væ&ÆRFò66†VGVÆR66†VB'&öF67B7FFR&WG'“¢"²RævWDÖW76vR‚’“° —Р—Р —&—fFR7–æ6‡&öæ—¦VBfö–B&WG'”66†VEf÷FTFVÆ—fW'•W'6—7FVæ6R‚’° –f÷"…7G&–ær6W'fW"¢vWEf÷FT66†T†æFÆW"‚’ævWD66†VEf÷FW56W'fW'2‚’’° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWEf÷FW2‡6W'fW"’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’’° —W'6—7E6W'fW%f÷FTFVÆ—fW'’‡6W'fW"Âf÷FR“° —Р—Р—Р–f÷"…7G&–ærWV–B¢æWrÆ–æ¶VD†6…6WCÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FUUT”G2‚’’’° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2‡WV–B’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’’° —W'6—7DöæÆ–æUf÷FTFVÆ—fW'’‡WV–BÂf÷FR“° —Р—Р—Р—Р —V&Æ–2fö–B6†V6µf÷FU'G’‚’° ––b†vWD6öæf–r‚’ævWEf÷FU'G”Væ&ÆVB‚’’° ––b‡f÷FU'G•f÷FW2ãÒ7W'&VçEf÷FU'G•f÷FW5&WV—&VB’° –FV'Vr‚%f÷FR'G’&V6†VB"“° –FD7W'&VçEf÷FU'G•f÷FW2‚Ö7W'&VçEf÷FU'G•f÷FW5&WV—&VB“°  –7W'&VçEf÷FU'G•f÷FW5&WV—&VB³ÒvWD6öæf–r‚’ævWEf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚“° —6WEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB€ –vWEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚’²vWD6öæf–r‚’ævWEf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚’“°  ––b‚vWD6öæf–r‚’ævWEf÷FU'G”'&öF67B‚’æ—4V×G’‚’’° –'&öF67B†vWD6öæf–r‚’ævWEf÷FU'G”'&öF67B‚’“° —Р –f÷"…7G&–ær6öÖÖæB¢vWD6öæf–r‚’ævWEf÷FU'G”'VævVT6öÖÖæG2‚’’° —'Vä6öç6öÆT6öÖÖæB†6öÖÖæB“° —Р ––b†vWD6öæf–r‚’ævWEf÷FU'G•6VæEFôÆÅ6W'fW'2‚’’° –f÷"…7G&–ær6W'fW"¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° —6VæEf÷FU'G’‡6W'fW"“° —Р—ÒVÇ6R° –f÷"…7G&–ær6W'fW"¢vWD6öæf–r‚’ævWEf÷FU'G•6W'fW'5Fõ6VæB‚’’° —6VæEf÷FU'G’‡6W'fW"“° —Р—Р—Р—6fUf÷FT66†Tf–ÆR‚“° —Р—Р —V&Æ–2'7G&7Bfö–BFV'Vr…7G&–ær7G"“°  —&—fFRfö–BFV'Vs"…7G&–ærÖW76vR’° –FV'Vr†ÖW76vR“° —Р ’ò¢  ’¢…EE6Æ–VçBW6VBf÷"Öö¦ær’&WVW7G2à ’¢ð —&—fFRf–æ‡GG6Æ–VçB‡GG6Æ–VçBÒ‡GG6Æ–VçBææWt'V–ÆFW"‚’æ6öææV7EF–ÖV÷WB„GW&F–öâæöe6V6öæG2ƒR’’æ'V–ÆB‚“°  ’ò¢  ’¢fWF6†W2Æ–W"w2UT”Bg&öÒF†RÖö¦ær’à ’  ’¢&ÒÆ–W$æÖRÆ–W"æÖP ’¢&WGW&âÆ–W"UT”BÂ÷"´6öFRçVÆÇÒ–bæ÷Bf÷Væ@ ’¢F‡&÷w2”ôW†6WF–öâ–bF†R&WVW7Bf–Ç0 ’¢F‡&÷w2–çFW''WFVDW†6WF–öâ–b–çFW''WFVBv†–ÆRv—F–ærf÷"F†R&W7öç6P ’¢ð —V&Æ–2UT”BfWF6…UT”B…7G&–ærÆ–W$æÖR’F‡&÷w2”ôW†6WF–öâ–çFW''WFVDW†6WF–öâ° ––b‡Æ–W$æÖRÓÒçVÆÂÇÂÆ–W$æÖRæWVÇ4–væ÷&T66R‚&çVÆÂ"’’° —&WGW&âçVÆð —Р ”‡GG&WVW7B&WVW7BÒ‡GG&WVW7BææWt'V–ÆFW"‚ ’çW&’…U$’æ7&VFR‚&‡GG3¢òö’æÖö¦æræ6öÒ÷W6W'2÷&öf–ÆW2öÖ–æV7&gBò"²Æ–W$æÖR’’ätUB‚ ’çF–ÖV÷WB„GW&F–öâæöe6V6öæG2ƒR’’æ'V–ÆB‚“°  ”‡GG&W7öç6SÅ7G&–æsâ&W7öç6RÒ‡GG6Æ–VçBç6VæB‡&WVW7B‡GG&W7öç6Rä&öG”†æFÆW'2æöe7G&–ær‚’“°  ––b‡&W7öç6Rç7FGW46öFR‚’ÓÒCÇÂ&W7öç6Rç7FGW46öFR‚’ÓÒCB’° –Æör‚%F†W&R—2æòÆ–W"v—F‚F†RæÖRÂ""²Æ–W$æÖR²%Â""“° —&WGW&âçVÆð —Р ––b‡&W7öç6Rç7FGW46öFR‚’Â#ÇÂ&W7öç6Rç7FGW46öFR‚’ãÒ3’° —F‡&÷ræWr”ôW†6WF–öâ‚$f–ÆVBFòfWF6‚UT”Bf÷""²Æ–W$æÖR²"Â…EE"²&W7öç6Rç7FGW46öFR‚’“° —Р ”§6öäVÆVÖVçBVÆVÖVçBÒ§6öå'6W"ç'6U7G&–ær‡&W7öç6Ræ&öG’‚’“° ––b†VÆVÖVçBÓÒçVÆÂÇÂVÆVÖVçBæ—4§6öäö&¦V7B‚’’° —&WGW&âçVÆð —Р ”§6öäö&¦V7Bö&¦V7BÒVÆVÖVçBævWD4§6öäö&¦V7B‚“° ––b‚ö&¦V7Bæ†2‚&–B"’ÇÂö&¦V7BævWB‚&–B"’æ—4§6öäçVÆ‚’’° —&WGW&âçVÆð —Р •7G&–ærWV–D57G&–ærÒö&¦V7BævWB‚&–B"’ævWD57G&–ær‚“° —&WGW&â'6UUT”Dg&öÕ7G&–ær‡WV–D57G&–ær“° —Р —V&Æ–2'7G&7B6WCÅ7G&–æsâvWDÆÄf–Æ&ÆU6W'fW'2‚“°  ’ò¢¢6ö×ÆWFRÆFf÷&Ò6W'fW"6WB&Vf÷&Rv†—FVÆ—7Bö&Æö6¶VB&÷WF–ærf–ÇFW'2â¢ð —V&Æ–2'7G&7B6WCÅ7G&–æsâvWDÆÄ6öæf–wW&VE6W'fW'2‚“°  —V&Æ–2'7G&7Bf÷F–æuÇVv–å&÷‡”6öæf–rvWD6öæf–r‚“°  —V&Æ–2'7G&7B7G&–ærvWD7W'&VçEÆ–W%6W'fW"…7G&–ærÆ–W"“°  ’ò¢  ’¢&W6öÇfW2Æ–W"w26W'fW"f÷"f÷FR&÷WF–ærâFVF–6FVBf÷F–ær&÷‡’†2æð ’¢Æö6ÂÆ–W'2Â6ò—BW6W2F†R&6¶VæB&W6Væ6RG&6¶W"–ç7FVBà ’¢ð —&÷FV7FVB7G&–ærvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær…7G&–ærÆ–W"’° ––b†—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’’° —&WGW&â&6¶VæEÆ–W%&W6Væ6UG&6¶W"ævWEÆ–W"‡Æ–W"’æÖ‡&W6Væ6RÓâ&W6Væ6RævWE6W'fW"‚’’æ÷$VÇ6R†çVÆ“° —Р—&WGW&âvWD7W'&VçEÆ–W%6W'fW"‡Æ–W"“° —Р ’ò¢  ’¢FVF–6FVB&÷WF–ær—2–çFVçF–öæÆÇ’Væf–Æ&ÆRöâÇVv–âÖW76v–æs¢F†@ ’¢G&ç7÷'B—2GF6†VBFòÆ–W"Öf6–ær&÷‡’æBFöW2æ÷B6''’&6¶Væ@ ’¢&W6Væ6R6æ6†÷G2à ’¢ð —&÷FV7FVB&ööÆVâ—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’° —&WGW&âvWD6öæf–r‚’ævWDFVF–6FVEf÷F–æu&÷‡’‚’bbÖWF†öBÒçVÆÂbbÖWF†öBç7W÷'G4&6¶VæE&W6Væ6R‚“° —Р —V&Æ–2'7G&7Bf–ÆRvWDFFföÆFW%ÇVv–â‚“°  —V&Æ–27G&–ærvWDÖöçF…F÷FÇ5v—F„FFUF‚‚’° ”Æö6ÄFFUF–ÖR5F–ÖRÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚“° —&WGW&âvWDÖöçF…F÷FÇ5v—F„FFUF‚†5F–ÖR“° —Р —V&Æ–27G&–ærvWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖR5F–ÖR’° —&WGW&â$ÖöçF…F÷FÂÒ"²5F–ÖRævWDÖöçF‚‚’çFõ7G&–ær‚’²"Ò"²5F–ÖRævWE–V"‚“° —Р —V&Æ–2'7G&7B7G&–ærvWE&÷W$æÖR…7G&–ærWV–BÂ7G&–ærÆ–W$æÖR“°  —V&Æ–2'7G&7B7G&–ærvWEUT”B…7G&–ærÆ–W$æÖR“°  —&—fFR–çBvWEfÇVR„'&”Æ—7CÄ6öÇVÖãâ6öÇ2Â7G&–ær6öÇVÖâ–çBFôFB’° –f÷"„6öÇVÖâB¢6öÇ2’° ––b†BævWDæÖR‚’æWVÇ4–væ÷&T66R†6öÇVÖâ’’° ”FFfÇVRfÇVRÒBævWEfÇVR‚“° ––çBçVÒÒ° ––b‡fÇVRÓÒçVÆÂ’° —&WGW&âFôFC° —Р––b‡fÇVRæ—4–çB‚’’° –çVÒÒfÇVRævWD–çB‚“° —ÒVÇ6R–b‡fÇVRæ—57G&–ær‚’’° —G'’° –çVÒÒ–çFVvW"ç'6T–çB‡fÇVRævWE7G&–ær‚’“° —Ò6F6‚„W†6WF–öâR’° ’òò–væ÷&P —Р—Р—&WGW&âçVÒ²FôFC° —Р—Р—&WGW&âFôFC° —Р —&—fFRf÷FUF÷FÇ56æ6†÷BvWE&ö¦V7FVE&öÆÆ÷fW%F÷FÇ2„'&”Æ—7CÄ6öÇVÖãâFFÂ7G&–ærÆ–W"’° ”Æ—7CÅF–ÖUG—SâF–ÖT6†ævW2ÒvWDvÆö&ÄFF†æFÆW"‚’ævWEF–ÖT6†ævW2‚“° –&ööÆVâ&W6WDÖöçF‚ÒF–ÖT6†ævW2æ6öçF–ç2…F–ÖUG—RäÔôåD‚“° –&ööÆVâ&W6WEvVV²ÒF–ÖT6†ævW2æ6öçF–ç2…F–ÖUG—RåtTT²“° –&ööÆVâ&W6WDF’ÒF–ÖT6†ævW2æ6öçF–ç2…F–ÖUG—RäD’“° ––çB66WFVEVWVVEf÷FW2Ò° ––çB66WFVDvÆö&ÅVWVVEf÷FW2Ò° –f÷"…f÷FUF–ÖUVWVRVWVVB¢vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’° ––b‚VWVVBæ—5&ö6W76VB‚’’° –66WFVDvÆö&ÅVWVVEf÷FW2²³° —Р––b‚VWVVBæ—5&ö6W76VB‚’bbVWVVBævWDæÖR‚’ÒçVÆÂbbVWVVBævWDæÖR‚’æWVÇ4–væ÷&T66R‡Æ–W"’’° –66WFVEVWVVEf÷FW2²³° —Р—Р––çBf÷FT–æ7&VÖVçBÒ66WFVEVWVVEf÷FW2²°  ––çBÆÅF–ÖUF÷FÂÒvWEfÇVR†FFÂ$ÆÅF–ÖUF÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBÖöçF…F÷FÂÒ&W6WDÖöçF‚òf÷FT–æ7&VÖVçB¢vWEfÇVR†FFÂ$ÖöçF…F÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBvVV¶Ç•F÷FÂÒ&W6WEvVV²òf÷FT–æ7&VÖVçB¢vWEfÇVR†FFÂ%vVV¶Ç•F÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBF–Ç•F÷FÂÒ&W6WDF’òf÷FT–æ7&VÖVçB¢vWEfÇVR†FFÂ$F–Ç•F÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBö–çG2ÒvWEfÇVR†FFÂ%ö–çG2"Âf÷FT–æ7&VÖVçB¢vWD6öæf–r‚’ævWEö–çG4öåf÷FR‚’“°  ––çBÖ…f÷FW2ÒvWD6öæf–r‚’ævWDÖ„Ö÷VçDöef÷FW5W$F’‚“° ––b†Ö…f÷FW2â’° ––çBF—2ÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚’ævWDF”ödÖöçF‚‚“° ––b†ÖöçF…F÷FÂâF—2¢Ö…f÷FW2’° –ÖöçF…F÷FÂÒF—2¢Ö…f÷FW3° —Р—Р––b†vWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’âbbö–çG2âvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’’° —ö–çG2ÒvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚“° —Р ––çBFFTÖöçF…F÷FÂÒÓ° ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° ––b†vWD6öæf–r‚’ævWEW6TÖöçF„FFUF÷FÇ45&–Ö'•F÷F‚’’° –FFTÖöçF…F÷FÂÒ&W6WDÖöçF‚òf÷FT–æ7&VÖVç@ “¢vWEfÇVR†FFÂvWDÖöçF…F÷FÇ5v—F„FFUF‚‚’Âf÷FT–æ7&VÖVçB“° —ÒVÇ6R° –FFTÖöçF…F÷FÂÒÖöçF…F÷Fð —Р—Р ––çEµÒ&ö¦V7FVEf÷FU'G’ÒvWE&ö¦V7FVEf÷FU'G•7FFR†66WFVDvÆö&ÅVWVVEf÷FW2²“° —&WGW&âæWrf÷FUF÷FÇ56æ6†÷B†ÆÅF–ÖUF÷FÂÂÖöçF…F÷FÂÂvVV¶Ç•F÷FÂÂF–Ç•F÷FÂÂö–çG2À —&ö¦V7FVEf÷FU'G•³ÒÂ&ö¦V7FVEf÷FU'G•³ÒÂFFTÖöçF…F÷F“° —Р —&÷FV7FVB&ööÆVâ6äf÷'v&E7FæFÆöæT'&öF67B†&ööÆVâÖævW5F÷FÇ2’° —&WGW&âÖævW5F÷FÇ3° —Р —&÷FV7FVB–çEµÒvWE&ö¦V7FVEf÷FU'G•7FFR†–çB66WFVEf÷FW2’° ––çB7W'&VçBÒf÷FU'G•f÷FW3° ––çB&WV—&VBÒ7W'&VçEf÷FU'G•f÷FW5&WV—&VC° ––b‚vWD6öæf–r‚’ævWEf÷FU'G”Væ&ÆVB‚’’° —&WGW&âæWr–çEµÒ²7W'&VçBÂ&WV—&VBÓ° —Р ––çB–æ7&V6RÒvWD6öæf–r‚’ævWEf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚“° –f÷"†–çB’Ò²’Â66WFVEf÷FW3²’²²’° –7W'&VçB²³° ––b†7W'&VçBãÒ&WV—&VB’° –7W'&VçBÓÒ&WV—&VC° —&WV—&VB³Ò–æ7&V6S° —Р—Р—&WGW&âæWr–çEµÒ²7W'&VçBÂ&WV—&VBÓ° —Р —V&Æ–2'7G&7B7G&–ærvWEÇVv–åfW'6–öâ‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†T7W'&VçEf÷FU'G•f÷FW2‚“°  —V&Æ–2'7G&7BÆöærvWEf÷FT66†TÆ7EWFFVB‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†U&WdF’‚“°  —V&Æ–2'7G&7B7G&–ærvWEf÷FT66†U&WdÖöçF‚‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†U&WevVV²‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚“°  —V&Æ–2'7G&7B&ööÆVâ—5Æ–W$öæÆ–æR…7G&–ærÆ–W$æÖR“°  ’ò¢  ’¢6†V6·2öæÆ–æR7FFRf÷"f÷FR&÷WF–ærÂW6–ær&6¶VæB&W6Væ6RöæÇ’v†VâF†—0 ’¢&÷‡’—2W‡Æ–6—FÇ’6öæf–wW&VB2F†RFVF–6FVBf÷F–ær&÷‡’à ’¢ð —&÷FV7FVB&ööÆVâ—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær…7G&–ærÆ–W$æÖR’° —&WGW&â—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’ò&6¶VæEÆ–W%&W6Væ6UG&6¶W"ævWEÆ–W"‡Æ–W$æÖR’æ—5&W6VçB‚ “¢—5Æ–W$öæÆ–æR‡Æ–W$æÖR“° —Р —V&Æ–2'7G&7B&ööÆVâ—56W'fW%fÆ–B…7G&–ær6W'fW"“°  —V&Æ–2'7G&7B&ööÆVâ—56öÖVöæTöæÆ–æU6W'fW"…7G&–ær6W'fW"“°  —&÷FV7FVB&ööÆVâ—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær…7G&–ær6W'fW"’° ––b‚—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’’° —&WGW&â—56öÖVöæTöæÆ–æU6W'fW"‡6W'fW"“° —Р–6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’ç&W6Væ6Rä&6¶VæE&W6Væ6U7FGW27FGW2Ò&6¶VæEÆ–W%&W6Væ6UG&6¶W  ’ævWD&6¶VæE7FGW2‡6W'fW"“° —&WGW&â7FGW2ÒçVÆÂbb7FGW2æ—4f–Æ&ÆR‚’bb7FGW2ævWEÆ–W$6÷VçB‚’â° —Р —V&Æ–2'7G&7B&ööÆVâ—5f÷FT66†T–væ÷&UF–ÖR‚“°  —V&Æ–2'7G&7B×—7Ä6öæf–rvWEf÷FT66†Tו5Ä6öæf–r‚“°  —V&Æ–2'7G&7B×—7Ä6öæf–rvWDæöåf÷FVD66†Tו5Ä6öæf–r‚“°  —V&Æ–2'7G&7B×—7Ä6öæf–rvWEf÷FTÆövv–ætו5Ä6öæf–r‚“°  ’ò¢  ’¢6‡WFF÷vâו5Â×&VÆFVB&W6÷W&6W26fVÇ’à ’¢ð —V&Æ–2fö–B6‡WFF÷väו7‚’° ––b†vWE&÷‡”×—7ÄÖW76VævW"‚’ÒçVÆÂ’° –vWE&÷‡”×—7ÄÖW76VævW"‚’ç6‡WFF÷vâ‚“° —6WE&÷‡”×—7ÄÖW76VævW"†çVÆ“° —Р ––b†vWE&÷‡”ו5‚’ÒçVÆÂ’° –vWE&÷‡”ו5‚’ç6‡WFF÷vâ‚“° —6WE&÷‡”ו5†çVÆ“° —Р—Р —V&Æ–2fö–BÆöB„•f÷FT66†R§6öå7F÷&vR”æöåf÷FVEÆ–W'57F÷&vRæöåf÷FVD66†T§6öâ’° –ÖWF†öBÒ'VævVTÖWF†öBævWD'”æÖR†vWD6öæf–r‚’ævWD'VævVTÖWF†öB‚’“° ––b†vWDÖWF†öB‚’ÓÒçVÆÂ’° –ÖWF†öBÒ'VævVTÖWF†öBåÅTt”äÔU54t”äs° —Р—v&åVç7W÷'FVDFVF–6FVEf÷F–æu&÷‡”ÖöFR‚“° —WV–EÆ–W$æÖT66†RÒvWE&÷‡”ו5‚’ævWE&÷w5UT”DæÖUVW'’‚“°  –'VævVUF–ÖT6†V6¶W"ç6WEF–ÖT6†ævTf–Å6fT'—72†vWD6öæf–r‚’ævWEF–ÖT6†ævTf–Å6fT'—72‚’“° –'VævVUF–ÖT6†V6¶W"æÆöEF–ÖW"‚“°  —f÷FT66†T†æFÆW"ÒæWrf÷FT66†T†æFÆW"†vWEf÷FT66†Tו5Ä6öæf–r‚’ÂvWD6öæf–r‚’ævWEf÷FT66†UW6Tו5‚’À –vWD6öæf–r‚’ævWEf÷FT66†UW6TÖ–äו5‚’ÂvWE&÷‡”ו5‚’ævWD×—7‚’ÂvWD6öæf–r‚’ævWDFV'Vr‚’À –§6öå7F÷&vR’°  ”÷fW'&–FP —V&Æ–2fö–BÆöt–æfó…7G&–ær×6r’° –Æöt–æfò†×6r“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&S…7G&–ær×6r’° –Æöu6WfW&R†×6r“° —Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vs„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vs…7G&–ær×6r’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –FV'Vr†×6r“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vs…F‡&÷v&ÆRR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р—Ó° —f÷FT66†T†æFÆW"æÆöB‚“°  –æöåf÷FVEÆ–W'466†RÒæWræöåf÷FVEÆ–W'466†R†vWDæöåf÷FVD66†Tו5Ä6öæf–r‚’À –vWD6öæf–r‚’ævWDæöåf÷FVD66†UW6Tו5‚’ÂvWD6öæf–r‚’ævWDæöåf÷FVD66†UW6TÖ–äו5‚’À –vWE&÷‡”ו5‚’ævWD×—7‚’Âæöåf÷FVD66†T§6öâÂvWD6öæf–r‚’ævWDFV'Vr‚’’°  ”÷fW'&–FP —V&Íx÷Þm¢G§²ÚîÆ­yÔ”B†Ö÷7E6–t&—G2ÂÆV7E6–t&—G2“° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–B&ö6W75VWVR‚’° —v†–ÆR†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’ç6—¦R‚’â’° •f÷FUF–ÖUVWVRf÷FRÒvWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’æVÆVÖVçB‚“° ––b‚f÷FRæ—5&ö6W76VB‚’’° •f÷FUF÷FÇ56æ6†÷BVWVVEF÷FÇ2Òf÷FRævWEF÷FÇ2‚’ÓÒçVÆÂÇÂf÷FRævWEF÷FÇ2‚’æ—4V×G’‚’òçVÆÀ “¢f÷FUF÷FÇ56æ6†÷Bç'6U7F÷&vR‡f÷FRævWEF÷FÇ2‚’“° •VWVVEf÷FU&W7VÇB&W7VÇBÒf÷FR‡f÷FRævWDæÖR‚’Âf÷FRævWE6W'f–6R‚’ÂG'VRÂfÇ6RÂf÷FRævWEF–ÖR‚’ÂVWVVEF÷FÇ2À —f÷FRævWEWV–B‚’Âf÷FR“° ––b‡&W7VÇBÓÒVWVVEf÷FU&W7VÇBå$UE%’’° —66†VGVÆUF–ÖUf÷FU&WG'’‚“° —&WGW&ã° —Р––b‡&W7VÇBÓÒVWVVEf÷FU&W7VÇBåDU$Ô”äÂ’° —v&â‚%&VÖ÷f–ærFW&Ö–æÂ&öÆÆ÷fW"f÷FR"²f÷FRævWEf÷FT–B‚’²"f÷""²f÷FRævWDæÖR‚’²"ò  ’²6W'f–6U6—FUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡f÷FRævWE6W'f–6R‚’’“° —Р—Р––b‚vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fUF–ÖUf÷FR‡f÷FR’’° —66†VGVÆUF–ÖUf÷FU&WG'’‚“° —&WGW&ã° —Р—Р—Р —&—fFRfö–B66†VGVÆUF–ÖUf÷FU&WG'’‚’° ––b‡F–ÖUf÷FU&WG'•66†VGVÆVBÇÂvWE66†VGVÆW"‚’ÓÒçVÆÂ’° —&WGW&ã° —Р—F–ÖUf÷FU&WG'•66†VGVÆVBÒG'VS° —G'’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ° —7–æ6‡&öæ—¦VB…f÷F–æuÇVv–å&÷‡’çF†—2’° —F–ÖUf÷FU&WG'•66†VGVÆVBÒfÇ6S° —Р—&ö6W75VWVR‚“° —ÒÂRÂF–ÖUVæ—Bå4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° —F–ÖUf÷FU&WG'•66†VGVÆVBÒfÇ6S° –FV'Vr‚%Væ&ÆRFò66†VGVÆR&öÆÆ÷fW"f÷FR&WG'“¢"²RævWDÖW76vR‚’“° —Р—Р —V&Æ–2fö–B&VÆöB‚’° —&VÆöE'VçF–ÖR‡G'VR“° —Р ’ò¢¢Æ–W26öçG&öÂÖ÷&–v–æFVB6öæf–wW&F–öâ&VÆöBv—F†÷WB7F÷–ær—G26öææV7F÷"÷"†÷7FVB6W'f–6Râ¢ð —V&Æ–2fö–B&VÆöDg&öÔ6öçG&ö‚’° —&VÆöE'VçF–ÖR†fÇ6R“° —Р —&—fFRfö–B&VÆöE'VçF–ÖR†&ööÆVâ&W7F'D6öçG&öÅ6W'f–6W2’° –ÖWF†öBÒ'VævVTÖWF†öBævWD'”æÖR†vWD6öæf–r‚’ævWD'VævVTÖWF†öB‚’“° ––b†vWDÖWF†öB‚’ÓÒçVÆÂ’° –ÖWF†öBÒ'VævVTÖWF†öBåÅTt”äÔU54t”äs° —Р—v&åVç7W÷'FVDFVF–6FVEf÷F–æu&÷‡”ÖöFR‚“° ––b‚&W7F'D6öçG&öÅ6W'f–6W2bbÖWF†öBÓÒ'VævVTÖWF†öBå4ô4´UE2’° —&V'V–ÆE6ö6¶WD6Æ–VçG2‚“° —Р —6WD7W'&VçEf÷FU'G•f÷FW5&WV—&VB€ –vWD6öæf–r‚’ævWEf÷FU'G•f÷FW5&WV—&VB‚’²vWEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚’“° ––b‡&W7F'D6öçG&öÅ6W'f–6W2’° –ÆöD×VÇF•&÷‡•7W÷'B‚“° —&W7F'D6öçG&öÅ6W'f–6W47–æ2‚“° —Р—Р —&—fFR7–æ6‡&öæ—¦VBfö–B&V'V–ÆE6ö6¶WD6Æ–VçG2‚’° ”†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â&V'V–ÇBÒæWr†6„ÖÃâ‚“° —G'’° ”Æ—7CÅ7G&–æsâ&Æö6¶VBÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° –f÷"…7G&–ær6W'fW"¢vWD6öæf–r‚’ævWE7–v÷E6W'fW'2‚’’° ––b†&Æö6¶VBæ6öçF–ç2‡6W'fW"’’6öçF–çVS° ”ÖÅ7G&–ærÂö&¦V7CâFFÒvWD6öæf–r‚’ævWE7–v÷E6W'fW$6öæf–wW&F–öâ‡6W'fW"“° •7G&–ær†÷7BÒFFæ6öçF–ç4¶W’‚$†÷7B"’ò…7G&–ær’FFævWB‚$†÷7B"’¢"#° ––çB÷'BÒFFæ6öçF–ç4¶W’‚%÷'B"’ò†–çB’FFævWB‚%÷'B"’¢#“ƒ° —&V'V–ÇBçWB‡6W'fW"ÂæWr6Æ–VçD†æFÆW"††÷7BÂ÷'BÂVæ7'—F–öä†æFÆW"ÂvWD6öæf–r‚’ævWDFV'Vr‚’’“° —Р—Ò6F6‚…'VçF–ÖTW†6WF–öâf–ÇW&R’° —7F÷6ö6¶WD6Æ–VçG2‡&V'V–ÇB“° —F‡&÷rf–ÇW&S° —Р”†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â&Wf–÷W2Ò6Æ–VçD†æFÆW3° –6Æ–VçD†æFÆW2Ò&V'V–ÇC° —7F÷6ö6¶WD6Æ–VçG2‡&Wf–÷W2“° —Р —&—fFR7–æ6‡&öæ—¦VB&ööÆVâ6VæE6ö6¶WDVçfVÆ÷R…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ”6Æ–VçD†æFÆW"6ö6¶WD6Æ–VçBÒ6Æ–VçD†æFÆW2ÓÒçVÆÂòçVÆ¢6Æ–VçD†æFÆW2ævWB‡6W'fW"“° ––b‡6ö6¶WD6Æ–VçBÓÒçVÆÂ’&WGW&âfÇ6S° —G'’° —6ö6¶WD6Æ–VçBç6VæDVçfVÆ÷R†VçfVÆ÷R“° —&WGW&âG'VS° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° –FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р—Р —&—fFR7–æ6‡&öæ—¦VB&ööÆVâ6VæD‡GGVçfVÆ÷R…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ”‡GG&÷‡•G&ç7÷'E6W'fW"G&ç7÷'BÒ‡GGG&ç7÷'E6W'fW#° —&WGW&âG&ç7÷'BÒçVÆÂbbG&ç7÷'Bç6VæB‡6W'fW"ÂVçfVÆ÷R“° —Р —&—fFRfö–B7F'D‡GGG&ç7÷'B‚’° —G'’° •U$’VæGö–çBÒU$’æ7&VFR†vWD6öæf–r‚’ævWD‡GGV&Æ–4VæGö–çB‚’“° ––b‚&‡GG2"æWVÇ4–væ÷&T66R†VæGö–çBævWE66†VÖR‚’’ÇÂVæGö–çBævWD†÷7B‚’ÓÒçVÆÀ —ÇÂVæGö–çBævWE÷'B‚’ÓÒÇÂVæGö–çBævWE÷'B‚’âcSS3P —ÇÂVæGö–çBævWEW6W$–æfò‚’ÒçVÆÂÇÂVæGö–çBævWEVW'’‚’ÒçVÆÂÇÂVæGö–çBævWDg&vÖVçB‚’ÒçVÆÀ —džVæGö–çBævWEF‚‚’ÒçVÆÂbbVæGö–çBævWEF‚‚’æ—4V×G’‚’bb"ò"æWVÇ2†VæGö–çBævWEF‚‚’’’’° —F‡&÷ræWr–ÆÆVvÄ&wVÖVçDW†6WF–öâ‚$…EEåV&Æ–4VæGö–çB×W7B&Râ…EE2÷&–v–â"“° —Р”f–ÆRF—&V7F÷'’ÒæWrf–ÆR†vWDFFföÆFW%ÇVv–â‚’Â&‡GG"“° ”‡GGFÇ4–FVçF—G’–FVçF—G’Ò‡GGFÇ4–FVçF—G’æÆöD÷$7&VFR†F—&V7F÷'’çFõF‚‚’ÂVæGö–çBævWD†÷7B‚’“° –‡GGVç&öÆÆÖVçDWF†÷&—G’ÒæWr‡GGVç&öÆÆÖVçDWF†÷&—G’†–FVçF—G’ÂF—&V7F÷'’çFõF‚‚’“° –‡GGG&ç7÷'E6W'fW"ÒæWr‡GG&÷‡•G&ç7÷'E6W'fW"€ –æWr–æWE6ö6¶WDFG&W72†vWD6öæf–r‚’ævWD‡GG†÷7B‚’ÂvWD6öæf–r‚’ævWD‡GG÷'B‚’’–FVçF—G’À –‡GGVç&öÆÆÖVçDWF†÷&—G’ÂF—&V7F÷'’çFõF‚‚’ç&W6öÇfR‚&÷WFvö–ær×c"’ÂF†—3£¦†æFÆT‡GGG&ç7÷'DVçfVÆ÷R“° –‡GGG&ç7÷'E6W'fW"ç7F'B‚“° –Æöt–æfò‚$…EEG&ç7÷'BÆ—7FVæ–ær6V7W&VÇ’öâ"²vWD6öæf–r‚’ævWD‡GG†÷7B‚’²#¢  ’²‡GGG&ç7÷'E6W'fW"ç÷'B‚’²#²W6R÷f÷F–æwÇVv–æ'VævVR‡GG6öFRÇ6W'fW#âf÷"V6‚&6¶VæB"“° —Ò6F6‚„W†6WF–öâf–ÇW&R’° –6Æ÷6T‡GGG&ç7÷'B‚“° —F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚$…EEG&ç7÷'B6÷VÆBæ÷B7F'B6V7W&VÇ’"Âf–ÇW&R“° —Р—Р ’ò¢¢¶VW2F†RWF†VçF–6FVBÕDÅ2&6¶VæB–FVçF—G’GF6†VBFò6V7W&—G’×6Vç6—F—fR&÷‡’&÷WF–ærâ¢ð —&÷FV7FVBfö–B†æFÆT‡GGG&ç7÷'DVçfVÆ÷R„‡GG&÷‡•G&ç7÷'E6W'fW"å&V6V—fVDVçfVÆ÷R&V6V—fVB’° ––b‚—4WF†VçF–6FVD‡GGVçfVÆ÷TÆÆ÷vVB‡&V6V—fVB’’° –FV'Vr‚$–væ÷&VB…EEVçfVÆ÷Rv†÷6RÆ–W"×&W6Væ6R6Æ–ÒF–Bæ÷BÖF6‚—G2WF†VçF–6FVB&6¶VæB"“° —&WGW&ã° —Р”vÆö&ÄÖW76vU&÷‡”†æFÆW"†æFÆW"ÒvÆö&ÄÖW76vU&÷‡”†æFÆW#° ––b††æFÆW"ÓÒçVÆÂ’F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚$…EEÖW76vR&÷WFW"—2æ÷B&VG’"“° –†æFÆW"æöäÖW76vR‡&V6V—fVBæVçfVÆ÷R‚’“° —Р —&—fFR&ööÆVâ—4WF†VçF–6FVD‡GGVçfVÆ÷TÆÆ÷vVB„‡GG&÷‡•G&ç7÷'E6W'fW"å&V6V—fVDVçfVÆ÷R&V6V—fVB’° ––b‡&V6V—fVBÓÒçVÆÂÇÂ&V6V—fVBæVçfVÆ÷R‚’ÓÒçVÆÂÇÂ&V6V—fVBç6W'fW$–B‚’ÓÒçVÆÂ’&WGW&âfÇ6S° •7G&–ær7F×VE6W'fW"Ò&V6V—fVBæVçfVÆ÷R‚’ævWDf–VÆG2‚’ævWD÷$FVfVÇB…f÷F–æuÇVv–åv—&Räµõ4U%dU"Â""“° ––b‚&V6V—fVBç6W'fW$–B‚’æWVÇ4–væ÷&T66R‡7F×VE6W'fW"’’&WGW&âfÇ6S° ––b‚f÷F–æuÇVv–åv—&Rå5T%ôÄôt”âæWVÇ2‡&V6V—fVBæVçfVÆ÷R‚’ævWE7V$6†ææV‚’’’&WGW&âG'VS° •f÷F–æuÇVv–åv—&RåÆ–W%&W6Væ6TWfVçBWfVçBÒf÷F–æuÇVv–åv—&Rç&VEÆ–W%&W6Væ6TWfVçB‡&V6V—fVBæVçfVÆ÷R‚’“° –&ööÆVâÖöFW&âÒWfVçBæ6öææV7F–öä–BÒçVÆÂÇÂWfVçBæ&6¶VæD–æ6&æF–öä–BÒçVÆÀ —ÇÂWfVçBæ&6¶VæE7F'FVDBÒÂÇÂWfVçBç&W6Væ6UF–ÖW7F×Òð ––b‚ÖöFW&âÇ—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’’&WGW&âG'VS° ’òòÆ–W"Öf6–ær&÷‡’†27G&öævW"WF†÷&—G’F†âç’&6¶VæC¢—G2Æ—fP ’òòÆ–W"6öææV7F–öâ7WÆ–W2&÷F‚F†R7W'&VçB&÷WFRæB†–âöæÆ–æRÖöFR’UT”Bà —&WGW&â—4ÆVv7”Æöv–äFW7F–æF–öäWF†÷&—FF—fR†WfVçBçÆ–W"ÂWfVçBçWV–BÂ&V6V—fVBç6W'fW$–B‚’“° —Р —&—fFR7–æ6‡&öæ—¦VBfö–B6Æ÷6T‡GGG&ç7÷'B‚’° ”‡GG&÷‡•G&ç7÷'E6W'fW"G&ç7÷'BÒ‡GGG&ç7÷'E6W'fW#° –‡GGG&ç7÷'E6W'fW"ÒçVÆð –‡GGVç&öÆÆÖVçDWF†÷&—G’ÒçVÆð ––b‡G&ç7÷'BÒçVÆÂ’G&ç7÷'Bæ6Æ÷6R‚“° —Р —V&Æ–27G&–ær7&VFT‡GG6öææV7F–öä6öFR…7G&–ær6W'fW$–B’° ”‡GGVç&öÆÆÖVçDWF†÷&—G’WF†÷&—G’Ò‡GGVç&öÆÆÖVçDWF†÷&—G“° ––b†ÖWF†öBÒ'VævVTÖWF†öBä…EEÇÂWF†÷&—G’ÓÒçVÆÂ’° —F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚%F†R…EEG&ç7÷'B—2æ÷B'Vææ–ær"“° —Р—&WGW&âWF†÷&—G’æ7&VFT6öææV7F–öä6öFR‡6W'fW$–BÂU$’æ7&VFR†vWD6öæf–r‚’ævWD‡GGV&Æ–4VæGö–çB‚’’ÂGW&F–öâæödÖ–çWFW2ƒR’ ’æVæ6öFR‚“° —Р —V&Æ–2fö–B&Wfö¶T‡GG&6¶VæB…7G&–ær6W'fW$–B’° ”‡GGVç&öÆÆÖVçDWF†÷&—G’WF†÷&—G’Ò‡GGVç&öÆÆÖVçDWF†÷&—G“° ––b†ÖWF†öBÒ'VævVTÖWF†öBä…EEÇÂWF†÷&—G’ÓÒçVÆÂ’F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚%F†R…EEG&ç7÷'B—2æ÷B'Vææ–ær"“° –WF†÷&—G’ç&Wfö¶R„‡GGFÇ4–FVçF—G’æ6æöæ–6Å6W'fW$–B‡6W'fW$–B’“° —Р —&—fFR7–æ6‡&öæ—¦VBfö–B6Æ÷6U6ö6¶WD6Æ–VçG2‚’° ”†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â6Æ–VçG2Ò6Æ–VçD†æFÆW3° –6Æ–VçD†æFÆW2ÒçVÆð —7F÷6ö6¶WD6Æ–VçG2†6Æ–VçG2“° —Р —7FF–2fö–B7F÷6ö6¶WD6Æ–VçG2„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â6Æ–VçG2’° ––b†6Æ–VçG2ÓÒçVÆÂ’&WGW&ã° –f÷"„6Æ–VçD†æFÆW"6Æ–VçB¢6Æ–VçG2çfÇVW2‚’’° ––b†6Æ–VçBÓÒçVÆÂ’6öçF–çVS° —G'’° –6Æ–VçBç7F÷6öææV7F–öâ‚“° —Ò6F6‚…'VçF–ÖTW†6WF–öâ–væ÷&VB’° ’òò&W7BVff÷'C¢öæR'&ö¶Vâ6Æ–VçB×W7Bæ÷B&WfVçBF†R&VÖ–æ–ær6ö6¶WG2g&öÒ6Æ÷6–ærࠗР—Р—Р —&—fFRfö–Bv&åVç7W÷'FVDFVF–6FVEf÷F–æu&÷‡”ÖöFR‚’° ––b†vWD6öæf–r‚’ævWDFVF–6FVEf÷F–æu&÷‡’‚’bb†ÖWF†öBÓÒçVÆÂÇÂÖWF†öBç7W÷'G4&6¶VæE&W6Væ6R‚’’’° –Æöu6WfW&R‚$FVF–6FVEf÷F–æu&÷‡’&WV—&W2Õ•5ÂÂ$TD•2ÂÕEBÂ4ô4´UE2Â÷"…EE²ÅTt”äÔU54t”är—2F—6&ÆVBf÷"  ’²&FVF–6FVB×&÷‡’&÷WF–ærâfÆÆ–ær&6²Fòæ÷&ÖÂ&÷‡’&÷WF–ærâ"“° —Р—Р —V&Æ–2'7G&7Bfö–B'Vä7–æ2…'Vææ&ÆR'Vâ“°  ’ò¢¢ÆFf÷&ÒæÖRW6VBöæÇ’f÷"F†RG&ç7÷'BÖæWWG&Â6öçG&öÂF—66÷fW'’6öçG&7Bâ¢ð —V&Æ–2'7G&7B7G&–ærvWE&÷‡•ÆFf÷&Ò‚“°  —V&Æ–2'7G&7Bfö–B'Vä6öç6öÆT6öÖÖæB…7G&–ær6öÖÖæB“°  —V&Æ–2'7G&7Bfö–B6fUf÷FT66†Tf–ÆR‚“°  —V&Æ–2'7G&7Bfö–B&VÆöD6÷&R†&ööÆVâ×—7“°  ’ò¢¢7G&–7B6öçG&öÂ&VÆöBFƒ²f–ÇW&W2&÷vFR6òF†R6ÆÆW"6â&W7F÷&R—G2&6·Wâ¢ð —V&Æ–2'7G&7Bfö–B&VÆöD6öçG&öÄ6öæf–wW&F–öâ‚’F‡&÷w2W†6WF–öã°  —V&Æ–2'7G&7B&ööÆVâ6VæEÇVv–äÖW76vTFF…7G&–ær6W'fW"Â7G&–ær6†ææVÂÂ'—FUµÒFFÂ&ööÆVâVWVR“°  —&—fFR7FF–2f–æ–çBÅTt”åôÔU54tUô„$EôĔԕBÒ3#scs° —&—fFR7FF–2f–æ–çBÅTt”åôÔU54tUõ4ôeEôĔԕBÒ3°  —V&Æ–2fö–B6VæEÇVv–äÖW76vU6W'fW"…7G&–ær6W'fW"–çBFVĤ6öäVçfVÆ÷RVçfVÆ÷R’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ6VæEÇVv–äÖW76vU6W'fW$æ÷r‡6W'fW"ÂVçfVÆ÷R’ÂFVÆ’¢TÂÂF–ÖUVæ—BäÔ”ÄÄ•4T4ôäE2“° —Р ’ò¢  ’¢6VæG2ÇVv–âÖÖW76vRVçfVÆ÷R–ÖÖVF–FVÇ’æB&W÷'G2v†WF†W"F†R&÷‡ ’¢66WFVB—Bf÷"FVÆ—fW'’à ’  ’¢&Ò6W'fW"F&vWB&6¶VæB6W'fW  ’¢&ÒVçfVÆ÷RVçfVÆ÷RFò6Væ@ ’¢&WGW&âG'VRv†VâF†R&÷‡’66WFVBF†RÖW76vRf÷"FVÆ—fW' ’¢ð —&÷FV7FVB&ööÆVâ6VæEÇVv–äÖW76vU6W'fW$æ÷r…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° –f–æÂ7G&–ær7V$6†ææVÂÒVçfVÆ÷RævWE7V$6†ææV‚“° –f–æÂ7G&–ær–ÆöBÒ§6öäVçfVÆ÷T6öFV2æVæ6öFR†VçfVÆ÷R“°  –f–æÂ'—FUµÒ7V$6†ææVÄ'—FW2Ò7V$6†ææVÂævWD'—FW2†¦fææ–òæ6†'6WBå7FæF&D6†'6WG2åUDeó‚“° –f–æÂ'—FUµÒ–ÆöD'—FW2Ò–ÆöBævWD'—FW2†¦fææ–òæ6†'6WBå7FæF&D6†'6WG2åUDeó‚“°  ’òòW7F–ÖFR'—FW2w&—GFVã  ’òòÒw&—FUUDbFG2"Ö'—FRÆVæwF‚&Vf—‚²UDbÓ‚'—FW0 ’òòÒw&—FT–çB—2B'—FW0 ––çBW7F–ÖFVE6—¦RÒ"²7V$6†ææVÄ'—FW2æÆVæwF‚²òò7V$6†ææVÂUDb†ÆVâ&Vf—‚²'—FW2 “B²òò–ÆöBÆVæwF‚–ç@ “"²–ÆöD'—FW2æÆVæwFƒ²òò–ÆöBUDb†ÆVâ&Vf—‚²'—FW2  ––b†W7F–ÖFVE6—¦RâÅTt”åôÔU54tUõ4ôeEôĔԕB’° –FV'Vr‚%µÇVv–äÖW76vUÒ–ÆöBæV&–ærÆ–Ö—B‚"²W7F–ÖFVE6—¦R²"'—FW2’6W'fW#Ò"²6W'fW  ’²"7V$6†ææVÃÒ"²7V$6†ææV²"( B6öç6–FW"&VF—2–ç7FVB"“° —Р ––b†W7F–ÖFVE6—¦RâÅTt”åôÔU54tUô„$EôĔԕB’° –FV'Vr‚%µÇVv–äÖW76vUÒ–ÆöBDôòÄ$tR‚"²W7F–ÖFVE6—¦R²"'—FW2ÂÖƒÒ"²ÅTt”åôÔU54tUô„$EôĔԕ@ ’²"’6W'fW#Ò"²6W'fW"²"7V$6†ææVÃÒ"²7V$6†ææV²"( BäõB6VçB"“° —&WGW&âfÇ6S° —Р —G'’„'—FT'&”÷WGWE7G&VÒ'—FT÷WE7G&VÒÒæWr'—FT'&”÷WGWE7G&VÒ‚“° ”FF÷WGWE7G&VÒ÷WBÒæWrFF÷WGWE7G&VÒ†'—FT÷WE7G&VÒ’’° ––b†vWD6öæf–r‚’ævWEÇVv–äÖW76vTVæ7'—F–öâ‚’bbVæ7'—F–öä†æFÆW"ÒçVÆÂ’° –÷WBçw&—FUUDb†Væ7'—F–öä†æFÆW"æVæ7'—B‡7V$6†ææVÂ’“° —ÒVÇ6R° –÷WBçw&—FUUDb‡7V$6†ææV“° —Р ’òò6æ—G’öæÇ“¢ÕU5B&R'—FW2Âæ÷B6†'0 –÷WBçw&—FT–çB‡–ÆöD'—FW2æÆVæwF‚“°  ––b†vWD6öæf–r‚’ævWEÇVv–äÖW76vTVæ7'—F–öâ‚’bbVæ7'—F–öä†æFÆW"ÒçVÆÂ’° –÷WBçw&—FUUDb†Væ7'—F–öä†æFÆW"æVæ7'—B‡–ÆöB’“° —ÒVÇ6R° –÷WBçw&—FUUDb‡–ÆöB“° —Р–÷WBæfÇW6‚‚“°  –&ööÆVâ6VçBÒ6VæEÇVv–äÖW76vTFF‡6W'fW"ÂvWD6öæf–r‚’ævWEÇVv–äÖW76vT6†ææV‚’çFôÆ÷vW$66R‚’À –'—FT÷WE7G&VÒçFô'—FT'&’‚’ÂfÇ6R“° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –FV'Vr‚‡6VçBò%6VçB"¢$6÷VÆBæ÷B6VæB"’²"ÇVv–âVçfVÆ÷R‚"²W7F–ÖFVE6—¦R²"'—FW2’"²6W'fW  ’²""²7V$6†ææV²""²VçfVÆ÷RævWDf–VÆG2‚’“° —Р—&WGW&â6VçC° —Ò6F6‚„W†6WF–öâR’° –Rç&–çE7F6µG&6R‚“° —&WGW&âfÇ6S° —Р—Р —7FF–2FVfVÇD¦VF—46Æ–VçD6öæf–r'V–ÆE&VF—46Æ–VçD6öæf–r…f÷F–æuÇVv–å&÷‡”6öæf–r6öæf–u6÷W&6R’° ”FVfVÇD¦VF—46Æ–VçD6öæf–rä'V–ÆFW"6öæf–rÒFVfVÇD¦VF—46Æ–VçD6öæf–ræ'V–ÆFW"‚ ’æFF&6R†6öæf–u6÷W&6RævWE&VF—4F$–æFW‚‚’’ç76†6öæf–u6÷W&6RævWE&VF—576‚’’æ6öææV7F–öåF–ÖV÷WDÖ–ÆÆ—2ƒ# ’ç6ö6¶WEF–ÖV÷WDÖ–ÆÆ—2ƒ#“° ––b†6öæf–u6÷W&6RævWE&VF—576‚’’° •54Å&ÖWFW'276Å&ÖWFW'2ÒæWr54Å&ÖWFW'2‚“° —76Å&ÖWFW'2ç6WDVæGö–çD–FVçF–f–6F–öäÆv÷&—F†Ò‚$…EE2"“° –6öæf–rç76Å&ÖWFW'2‡76Å&ÖWFW'2“° —Р––b†6öæf–u6÷W&6RævWE&VF—5W6W&æÖR‚’ÒçVÆÂbb6öæf–u6÷W&6RævWE&VF—5W6W&æÖR‚’æ—4V×G’‚’’° –6öæf–rçW6W"†6öæf–u6÷W&6RævWE&VF—5W6W&æÖR‚’“° —Р––b†6öæf–u6÷W&6RævWE&VF—577v÷&B‚’ÒçVÆÂbb6öæf–u6÷W&6RævWE&VF—577v÷&B‚’æ—4V×G’‚’’° –6öæf–rç77v÷&B†6öæf–u6÷W&6RævWE&VF—577v÷&B‚’“° —Р—&WGW&â6öæf–ræ'V–ÆB‚“° —Р —V&Æ–2&ööÆVâ6VæE&VF—4VçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° —&WGW&â6VæE&VF—4VçfVÆ÷U6W'fW"‡6W'fW"ÂVçfVÆ÷RÂfÇ6R“° —Р —&—fFR&ööÆVâ6VæE&VF—4VçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷RÂ&ööÆVâW6U&WG'”6ööÆF÷vâ’° ”¦VF—5ööÂV&Æ—6†W%ööÂÒ&VF—5V&Æ—6†W%ööð ––b‡V&Æ—6†W%ööÂÓÒçVÆÂLJW6U&WG'”6ööÆF÷vâbb7—7FVÒæ7W'&VçEF–ÖTÖ–ÆÆ—2‚’Â&VF—5V&Æ—6†W%&WG'”gFW"’’° —&WGW&âfÇ6S° —Р —G'’„¦VF—2¦VF—2ÒV&Æ—6†W%ööÂævWE&W6÷W&6R‚’’° •7G&–ær6†ææVÂÒvWD6öæf–r‚’ævWE&VF—5&Vf—‚‚’²%f÷F–æuÇVv–åò"²6W'fW#° –Æöær7V'67&–&W'2Ò¦VF—2çV&Æ—6‚†6†ææVÂÀ ”§6öäVçfVÆ÷T6öFV2æVæ6öFR…f÷F–æuÇVv–åv—&Rçv—F…&VF—4FVÆ—fW'”–B†VçfVÆ÷R’’“° —&VF—5V&Æ—6†W%&WG'”gFW"Òð —&WGW&â7V'67&–&W'2â° —Ò6F6‚„W†6WF–öâR’° ––b‡W6U&WG'”6ööÆF÷vâ’° ’òò7FæFÆöæR'&öF67G2&VÖ–âVWVVBÂ6òF†V—"&WG&–W26â&RF‡&÷GFÆVB6fVÇ’à —&VF—5V&Æ—6†W%&WG'”gFW"Ò7—7FVÒæ7W'&VçEF–ÖTÖ–ÆÆ—2‚’²#ð —Р–FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р—Р —V&Æ–2&ööÆVâ6VæD×GDVçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ––b†×GD†æFÆW"ÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р—G'’° –×GD†æFÆW"çV&Æ—6„VçfVÆ÷R†vWD6öæf–r‚’ævWD×GE&Vf—‚‚’²'f÷F–æwÇVv–â÷6W'fW'2ò"²6W'fW"ÂVçfVÆ÷R“° —&WGW&âG'VS° —Ò6F6‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—&WGW&âfÇ6S° —Р—Р —V&Æ–2&ööÆVâ6VæE6ö6¶WDVçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ”ÖÅ7G&–ærÂö&¦V7Câ6öæf–wW&F–öâÒvWD6öæf–r‚’ævWE7–v÷E6W'fW$6öæf–wW&F–öâ‡6W'fW"“° ––b†6öæf–wW&F–öâÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р•7G&–ær†÷7BÒ6öæf–wW&F–öâævWB‚$†÷7B"’–ç7Fæ6Vöb7G&–ærò…7G&–ær’6öæf–wW&F–öâævWB‚$†÷7B"’¢"#° ––çB÷'BÒ6öæf–wW&F–öâævWB‚%÷'B"’–ç7Fæ6VöbçVÖ&W"ò‚„çVÖ&W"’6öæf–wW&F–öâævWB‚%÷'B"’’æ–çEfÇVR‚’¢#“ƒ° ––b††÷7Bæ—4V×G’‚’’° —&WGW&âfÇ6S° —Р •7G&–ær–ÆöBÒ§6öäVçfVÆ÷T6öFV2æVæ6öFR†VçfVÆ÷R“° •7G&–ærVæ6öFVBÒVæ7'—F–öä†æFÆW"ÒçVÆÂòVæ7'—F–öä†æFÆW"æVæ7'—B‡–ÆöB’¢–ÆöC° —G'’…6ö6¶WB6ö6¶WBÒæWr6ö6¶WB‚’’° —6ö6¶WBæ6öææV7B†æWr–æWE6ö6¶WDFG&W72††÷7BÂ÷'B’Â#“° —G'’„FF÷WGWE7G&VÒ÷WGWBÒæWrFF÷WGWE7G&VÒ‡6ö6¶WBævWD÷WGWE7G&VÒ‚’’’° –÷WGWBçw&—FUUDb†Væ6öFVB“° –÷WGWBæfÇW6‚‚“° —Р—&WGW&âG'VS° —Ò6F6‚„W†6WF–öâR’° –FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р—Р —V&Æ–2fö–B6VæE6W'fW$æÖTÖW76vR‚’° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° —6VæEÇVv–äÖW76vU6W'fW"‡2ÂÂf÷F–æuÇVv–åv—&Rç6W'fW$æÖR‡2’“° —Р—Р —V&Æ–2fö–B6VæEf÷FU'G’…7G&–ær6W'fW"’° ––b†—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡6W'fW"’’° –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡6W'fW"ÂÂf÷F–æuÇVv–åv—&Rçf÷FU'G”'VævVR‚’“° —Р—Р —V&Æ–2fö–B6WD7W'&VçEf÷FU'G•f÷FW2†–çBÖ÷VçB’° —f÷FU'G•f÷FW2ÒÖ÷VçC° —6WEf÷FT66†Uf÷FU'G”7W'&VçEf÷FW2†Ö÷VçB“° –FV'Vr‚$7W'&VçBf÷FR'G’F÷Fâ"²f÷FU'G•f÷FW2“° —Р —V&Æ–2'7G&7Bfö–B6WEf÷FT66†TÆ7EWFFVB‚“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†U&WdF’†–çBF’“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†U&WdÖöçF‚…7G&–ærFW‡B“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†U&WevVV²†–çBvVV²“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†Uf÷FT66†T–væ÷&UF–ÖR†&ööÆVâ–væ÷&R“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†Uf÷FU'G”7W'&VçEf÷FW2†–çBf÷FW2“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB†–çBf÷FW2“°  —V&Æ–2fö–B7FGW2‚’° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° ––b‚—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡2’’° –Æör‚$æòÆ–W'2öâ6W'fW""²2²"Fò6VæBFW7B7FGW2ÖW76vRÂÆV6R&WFW7Bv—F‚6öÖVöæRöæÆ–æR"“° —ÒVÇ6R° –Æör‚%6VæF–ær&WVW7Bf÷"7FGW2ÖW76vRöâ"²2“° –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡2ÂÂf÷F–æuÇVv–åv—&Rç7FGW2‡2’“° —Р—Р—Р ’ò¢¢'Vç26÷'&VÆFVBÂæöâ×f÷FR&÷VæBG&—÷fW"F†R7F—fR&6¶VæBG&ç7÷'Bâ¢ð —V&Æ–26ö×ÆWF&ÆTgWGW&SÄ6öÖ×Væ–6F–öåFW7E&W7VÇCâFW7D&6¶VæD6öÖ×Væ–6F–öâ…7G&–ær&WVW7FVE6W'fW"À –ÆöærF–ÖV÷WDÖ–ÆÆ—2’° •7G&–ær6W'fW"Ò&WVW7FVE6W'fW"ÓÒçVÆÂò""¢&WVW7FVE6W'fW"çG&–Ò‚“° ”'VævVTÖWF†öB7F—fTÖWF†öBÒÖWF†öC° ––b‡6W'fW"æ—4V×G’‚’ÇÂvWDÆÄf–Æ&ÆU6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%Tä´äõtåô$4´TäB"Â%F†R&6¶VæB—2æ÷B6öæf–wW&VBöâF†—2&÷‡’"’“° —Р––b†7F—fTÖWF†öBÓÒçVÆÂÇÂvÆö&ÄÖW76vU&÷‡”†æFÆW"ÓÒçVÆÂ’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%E$å5õ%EõTäd”Ä$ÄR"Â%F†R&÷‡’6öÖ×Væ–6F–öâG&ç7÷'B—2æ÷B'Vææ–ær"’“° —Р––b†7F—fTÖWF†öBÓÒ'VævVTÖWF†öBåÅTt”äÔU54t”ärbb—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡6W'fW"’’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%Ä”U%õ$UT•$TB"Â%ÇVv–âÖW76v–ær&WV—&W2âöæÆ–æRÆ–W"öâF†R6VÆV7FVB&6¶VæB"’“° —Р•66†VGVÆVDW†V7WF÷%6W'f–6R66†VGVÆW"ÒvWE66†VGVÆW"‚“° ––b‡66†VGVÆW"ÓÒçVÆÂ’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%E$å5õ%EõTäd”Ä$ÄR"Â%F†R&÷‡’66†VGVÆW"—2æ÷B'Vææ–ær"’“° —Р–Æöær&÷VæFVEF–ÖV÷WBÒÖF‚æÖ‚ƒSÂÂÖF‚æÖ–â‡F–ÖV÷WDÖ–ÆÆ—2Â3Â’“° •UT”B&WVW7D–BÒUT”Bç&æFöÕUT”B‚“° ”6ö×ÆWF&ÆTgWGW&SÄ6öÖ×Væ–6F–öåFW7E&W7VÇCâ&W7VÇBÒæWr6ö×ÆWF&ÆTgWGW&SÃâ‚“° •VæF–æt6öÖ×Væ–6F–öåFW7BVæF–ærÒæWrVæF–æt6öÖ×Væ–6F–öåFW7B‡6W'fW"Â7F—fTÖWF†öBÂ7—7FVÒæææõF–ÖR‚’Â&W7VÇB“° —VæF–æt6öÖ×Væ–6F–öåFW7G2çWB‡&WVW7D–BÂVæF–ær“° —&W7VÇBçv†Vä6ö×ÆWFR‚†–væ÷&VBÂf–ÇW&R’ÓâVæF–æt6öÖ×Væ–6F–öåFW7G2ç&VÖ÷fR‡&WVW7D–BÂVæF–ær’“° —G'’° ––b‚6VæD6öÖ×Væ–6F–öåFW7DVçfVÆ÷Tæ÷r‡6W'fW"Âf÷F–æuÇVv–åv—&Rç7FGW2‡6W'fW"Â&WVW7D–B’’’° —&W7VÇBæ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÂ%E$å5õ%EõTäd”Ä$ÄR"À ’%F†R7F—fRG&ç7÷'B6÷VÆBæ÷B66WBF†R6öÖ×Væ–6F–öâFW7B"’“° —&WGW&â&W7VÇC° —Р—66†VGVÆW"ç66†VGVÆR‚‚’Óâ&W7VÇBæ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%D”ÔTõUB"Â$æò6÷'&VÆFVB&WÇ’'&—fVB&Vf÷&RF†RF–ÖV÷WB"’’Â&÷VæFVEF–ÖV÷WBÂF–ÖUVæ—BäÔ”ÄÄ•4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâf–ÇW&R’° —&W7VÇBæ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÂ%4TäEôd”ÄTB"À ’%F†R&÷‡’6÷VÆBæ÷B6VæBF†R6öÖ×Væ–6F–öâFW7B"’“° —Р—&WGW&â&W7VÇC° —Р ’ò¢¢6VæG2F–væ÷7F–2–ÖÖVF–FVÇ’æB&W÷'G2v†WF†W"F†R7F—fRG&ç7÷'B66WFVB—Bâ¢ð —&÷FV7FVB&ööÆVâ6VæD6öÖ×Væ–6F–öåFW7DVçfVÆ÷Tæ÷r…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° —&WGW&â6VæE&÷‡”'&öF67DVçfVÆ÷Tæ÷r‡6W'fW"ÂVçfVÆ÷R“° —Р —&÷FV7FVBfö–B†æFÆU7FGW4ö¶’„§6öäVçfVÆ÷RÖW76vR’° •7G&–ær6W'fW"ÒÖW76vRævWDf–VÆG2‚’ævWD÷$FVfVÇB…f÷F–æuÇVv–åv—&Räµõ4U%dU"Â""“° •7G&–ær&WVW7BÒÖW76vRævWDf–VÆG2‚’ævWD÷$FVfVÇB…f÷F–æuÇVv–åv—&Räµõ$UTU5Eô”BÂ""“° ––b‡&WVW7Bæ—4V×G’‚’’° –Æör‚%7FGW2ö¶’f÷""²6W'fW"“° —&WGW&ã° —Р•UT”B&WVW7D–C° —G'’° —&WVW7D–BÒUT”Bæg&öÕ7G&–ær‡&WVW7B“° —Ò6F6‚„–ÆÆVvÄ&wVÖVçDW†6WF–öâ–væ÷&VB’° –FV'Vr‚$–væ÷&VB7FGW2&WÇ’v—F‚â–çfÆ–B&WVW7B”Bg&öÒ"²6W'fW"“° —&WGW&ã° —Р•VæF–æt6öÖ×Væ–6F–öåFW7BVæF–ærÒVæF–æt6öÖ×Væ–6F–öåFW7G2ævWB‡&WVW7D–B“° ––b‡VæF–ærÓÒçVÆÂÇÂVæF–ærç6W'fW"‚’æWVÇ2‡6W'fW"’’° –FV'Vr‚$–væ÷&VBVæW‡V7FVB7FGW2&WÇ’g&öÒ"²6W'fW"“° —&WGW&ã° —Р–Æöær&÷VæEG&—Ö–ÆÆ—2ÒÖF‚æÖ‚ƒÂÀ •F–ÖUVæ—Bäääõ4T4ôäE2çFôÖ–ÆÆ—2…7—7FVÒæææõF–ÖR‚’ÒVæF–ærç7F'FVDDææ÷2‚’’“° —VæF–ærç&W7VÇB‚’æ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBç7V66W72‡6W'fW"ÂVæF–æræÖWF†öB‚’Â&÷VæEG&—Ö–ÆÆ—2’“° —Р —&—fFRfö–B6æ6VÄ6öÖ×Væ–6F–öåFW7G2…7G&–ærÖW76vR’° —VæF–æt6öÖ×Væ–6F–öåFW7G2æf÷$V6‚‚‡&WVW7D–BÂVæF–ær’ÓâVæF–ærç&W7VÇB‚’æ6ö×ÆWFR€ ”6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡VæF–ærç6W'fW"‚’ÂVæF–æræÖWF†öB‚’Â%E$å5õ%Eõ5DõTB"ÂÖW76vR’’“° —VæF–æt6öÖ×Væ–6F–öåFW7G2æ6ÆV"‚“° —Р —V&Æ–2&V6÷&B6öÖ×Væ–6F–öåFW7E&W7VÇB†&ööÆVâ7V66W72Â7G&–ær6öFRÂ7G&–ærÖW76vRÂ7G&–ær6W'fW"À •7G&–ærÖWF†öBÂÆöær&÷VæEG&—Ö–ÆÆ—2’° —&—fFR7FF–26öÖ×Væ–6F–öåFW7E&W7VÇB7V66W72…7G&–ær6W'fW"Â'VævVTÖWF†öBÖWF†öBÂÆöær&÷VæEG&—Ö–ÆÆ—2’° —&WGW&âæWr6öÖ×Væ–6F–öåFW7E&W7VÇB‡G'VRÂ$ô²"Â$&6¶VæB&WÆ–VB÷fW"F†R7F—fRG&ç7÷'B"Â6W'fW"À –ÖWF†öBÓÒçVÆÂò""¢ÖWF†öBææÖR‚’Â&÷VæEG&—Ö–ÆÆ—2“° —Р —&—fFR7FF–26öÖ×Væ–6F–öåFW7E&W7VÇBf–ÇW&R…7G&–ær6W'fW"Â'VævVTÖWF†öBÖWF†öBÂ7G&–ær6öFRÂ7G&–ærÖW76vR’° —&WGW&âæWr6öÖ×Væ–6F–öåFW7E&W7VÇB†fÇ6RÂ6öFRÂÖW76vRÂ6W'fW"À –ÖWF†öBÓÒçVÆÂò""¢ÖWF†öBææÖR‚’ÂÓ“° —Р—Р —&—fFR&V6÷&BVæF–æt6öÖ×Væ–6F–öåFW7B…7G&–ær6W'fW"Â'VævVTÖWF†öBÖWF†öBÂÆöær7F'FVDDææ÷2À ”6ö×ÆWF&ÆTgWGW&SÄ6öÖ×Væ–6F–öåFW7E&W7VÇCâ&W7VÇB’²Р —&—fFRfö–B6VæEf÷FTFVÆ•&V¦V7FVB…7G&–ærÆ–W"Â7G&–ærWV–BÂ7G&–ær6W'f–6RÂ&ööÆVâÆ–W$öæÆ–æRÀ •7G&–ærÆ–W%6W'fW"’° ––b‚Æ–W$öæÆ–æRÇÂÆ–W%6W'fW"ÓÒçVÆÂÇÂvWDÆÄf–Æ&ÆU6W'fW'2‚’æ6öçF–ç2‡Æ–W%6W'fW"’’° –FV'Vr‚$æ÷B6VæF–ærf÷FRFVÆ’&V¦V7F–öâf÷""²Æ–W"²"&V6W6RF†RÆ–W"—2öffÆ–æR"“° —&WGW&ã° —Р –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡Æ–W%6W'fW"ÂÀ •f÷F–æuÇVv–åv—&Rçf÷FTFVÆ•&V¦V7FVB‡Æ–W"ÂWV–BÂ6W'f–6RÂG'VR’“° —Р —V&Æ–27G&–ærvWEv—EVçF–ÄFVÆ•6—FTg&öÕ6W'f–6R…7G&–ær6W'f–6R’° –f÷"…7G&–ær6—FR¢vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•6—FW2‚’’° ––b†vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•6W'f–6R‡6—FR’æWVÇ4–væ÷&T66R‡6W'f–6R’’° —&WGW&â6—FS° —Р—Р—&WGW&â"#° —Р —&—fFRÆöærvWDÆ7Ef÷FW5F–ÖR…7G&–ærWV–BÂ'&”Æ—7CÄ6öÇVÖãâ6öÇ2Â7G&–ær6—FRÂ7G&–ær6W'f–6RÂ7G&–ærÆ–W"À –&ööÆVâ–æ6ÇVFUF–ÖT6†ævUVWVR’° –ÆöærÖ÷7E&V6VçEF–ÖRÒ°  ––b†vWEf÷FT66†T†æFÆW"‚’æ†4öæÆ–æUf÷FW2‡WV–B’’° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâöæÆ–æUf÷FW2ÒvWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2‡WV–B“° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢öæÆ–æUf÷FW2’° ––b‡f÷FRævWE6W'f–6R‚’æWVÇ4–væ÷&T66R‡6W'f–6R’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂf÷FRævWEF–ÖR‚’“° —Р—Р—Р –f÷"…7G&–ær6W'fW"¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢vWEf÷FT66†T†æFÆW"‚’ævWEf÷FW2‡6W'fW"’’° ––b‡f÷FRævWEWV–B‚’æWVÇ2‡WV–B’bbf÷FRævWE6W'f–6R‚’æWVÇ4–væ÷&T66R‡6W'f–6R’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂf÷FRævWEF–ÖR‚’“° —Р—Р—Р ––b†–æ6ÇVFUF–ÖT6†ævUVWVRbbÆ–W"ÒçVÆÂ’° –f÷"…f÷FUF–ÖUVWVRVWVVEf÷FR¢vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’° ––b‡VWVVEf÷FRævWDæÖR‚’æWVÇ4–væ÷&T66R‡Æ–W" ’bbVWVVEf÷FRævWE6W'f–6R‚’æWVÇ4–væ÷&T66R‡6W'f–6R’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂVWVVEf÷FRævWEF–ÖR‚’“° —Р—Р—Р –f÷"„6öÇVÖâB¢6öÇ2’° ––b†BævWDæÖR‚’æWVÇ4–væ÷&T66R‚$Æ7Ef÷FW2"’’° ”FFfÇVRfÇVRÒBævWEfÇVR‚“° •7G&–æuµÒÆ—7BÒfÇVRævWE7G&–ær‚’ç7Æ—B‚"VÆ–æRR"“° –f÷"…7G&–ær7G"¢Æ—7B’° •7G&–æuµÒFFÒ7G"ç7Æ—B‚"òò"“° ––b†FF³ÒæWVÇ4–væ÷&T66R‡6—FR’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂÆöærçfÇVTöb†FF³Ò’“° —Р—Р—Р—Р—&WGW&âÖ÷7E&V6VçEF–ÖS° —Р —V&Æ–2&ööÆVâ6†V6µf÷FTFVÆ’…7G&–ærWV–BÂ7G&–ær6W'f–6RÂ'&”Æ—7CÄ6öÇVÖãâFF’° —&WGW&â6†V6µf÷FTFVÆ’‡WV–BÂçVÆÂÂ6W'f–6RÂFFÂfÇ6R“° —Р ’ò¢  ’¢6†V6·2F†R6öæf–wW&VBf÷FRFVÆ’Â÷F–öæÆÇ’–æ6ÇVF–ær66WFVBf÷FW2v—F–æp ’¢f÷"vÆö&ÄFFF–ÖR6†ævRFòf–æ—6‚à ’  ’¢&ÒWV–BÆ–W"UT”@ ’¢&ÒÆ–W"Æ–W"æÖRW6VB'’F†RF–ÖRÖ6†ævRVWVP ’¢&Ò6W'f–6Rf÷FR6W'f–6P ’¢&ÒFF7W'&VçBÆ–W"FF ’¢&Ò–æ6ÇVFUF–ÖT6†ævUVWVRv†WF†W"VWVVBf÷FW2&W6W'fRF†V—"FVÆ’6Æ÷@ ’¢&WGW&âG'VRv†VâF†Rf÷FRÖ’&R66WFV@ ’¢ð —V&Æ–2&ööÆVâ6†V6µf÷FTFVÆ’…7G&–ærWV–BÂ7G&–ærÆ–W"Â7G&–ær6W'f–6RÂ'&”Æ—7CÄ6öÇVÖãâFFÀ –&ööÆVâ–æ6ÇVFUF–ÖT6†ævUVWVR’° •7G&–ær6—FRÒvWEv—EVçF–ÄFVÆ•6—FTg&öÕ6W'f–6R‡6W'f–6R“° ––b‡6—FRæ—4V×G’‚’’° –FV'Vr‚$æò6W'f–6R6—FR6WBf÷""²6W'f–6R²"Â6¶—–ærf÷FRFVÆ’6†V6²"“° —&WGW&âG'VS° —Р ––çBf÷FTFVÆ’ÒvWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVÆ’‡6—FR“° ––çBf÷FTFVƔ֖âÒvWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVƔ֖â‡6—FR“°  –ÆöærÆ7Ef÷FRÒvWDÆ7Ef÷FW5F–ÖR‡WV–BÂFFÂ6—FRÂ6W'f–6RÂÆ–W"–æ6ÇVFUF–ÖT6†ævUVWVR“° ––b†Æ7Ef÷FRÓÒ’° –FV'Vr‚$æòÆ7Bf÷FRF–ÖRf÷VæBf÷""²WV–B²"ò"²6W'f–6R²"Â6¶—–ærf÷FRFVÆ’6†V6²"“° —&WGW&âG'VS° —Р —G'’° ”Æö6ÄFFUF–ÖRæ÷rÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚“° ”Æö6ÄFFUF–ÖRÆ7Ef÷FUF–ÖRÒÆö6ÄFFUF–ÖRæöd–ç7FçB„–ç7FçBæödWö6„Ö–ÆÆ’†Æ7Ef÷FR’¦öæT–Bç7—7FVÔFVfVÇB‚’ ’çÇW4†÷W'2†vWD6öæf–r‚’ævWEF–ÖT†÷W$öfe6WB‚’“°  ––b‚vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVÆ”F–Ç’‡6—FR’’° ––b‡f÷FTFVÆ’ÓÒbbf÷FTFVƔ֖âÓÒ’° –FV'Vr‚%f÷FRFVÆ’—2f÷""²6—FR²"Â6¶—–ærf÷FRFVÆ’6†V6²"“° —&WGW&âG'VS° —Р ”Æö6ÄFFUF–ÖRæW‡Gf÷FRÒÆ7Ef÷FUF–ÖRçÇW4†÷W'2‚†Æöær’f÷FTFVÆ’’çÇW4Ö–çWFW2‚†Æöær’f÷FTFVƔ֖⓰ —&WGW&âæ÷ræ—4gFW"†æW‡Gf÷FR“° —Р”Æö6ÄFFUF–ÖR&W6WEF–ÖRÒÆ7Ef÷FUF–ÖRçv—F„†÷W"†vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVÆ”†÷W"‡6—FR’ ’çv—F„Ö–çWFRƒ’çv—F…6V6öæBƒ“° ”Æö6ÄFFUF–ÖR&W6WEF–ÖUFöÖ÷'&÷rÒ&W6WEF–ÖRçÇW4†÷W'2ƒ#B“°  ––b†Æ7Ef÷FUF–ÖRæ—4&Vf÷&R‡&W6WEF–ÖR’’° ––b†æ÷ræ—4gFW"‡&W6WEF–ÖR’’° –FV'Vr‚%f÷FRFVÆ’—2ÖWBf÷""²WV–B²"ò"²6W'f–6R²"Âf÷FR6â&R&ö6W76VB"“° —&WGW&âG'VS° —Р—ÒVÇ6R° ––b†æ÷ræ—4gFW"‡&W6WEF–ÖUFöÖ÷'&÷r’’° –FV'Vr‚%f÷FRFVÆ’—2ÖWBf÷""²WV–B²"ò"²6W'f–6R²"Âf÷FR6â&R&ö6W76VB"“° —&WGW&âG'VS° —Р—Р—Ò6F6‚„W†6WF–öâR’° –Rç&–çE7F6µG&6R‚“° —Р –FV'Vr‚%f÷FRFVÆ’—2æ÷BÖWBf÷""²WV–B²"ò"²6W'f–6R²"Â6¶—–ærf÷FR"“° —&WGW&âfÇ6S° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–Bf÷FR…7G&–ærÆ–W"Â7G&–ær6W'f–6RÂ&ööÆVâ&VÅf÷FRÂ&ööÆVâF–ÖUVWVRÂÆöærVWVUF–ÖRÀ •f÷FUF÷FÇ56æ6†÷BFW‡BÂ7G&–ærWV–B’° —f÷FR‡Æ–W"Â6W'f–6RÂ&VÅf÷FRÂF–ÖUVWVRÂVWVUF–ÖRÂFW‡BÂWV–BÂçVÆ“° —Р —&—fFRVçVÒVWVVEf÷FU&W7VÇB° •5T44U52Â$UE%’ÂDU$Ô”äÀ —Р —&—fFR7–æ6‡&öæ—¦VBVWVVEf÷FU&W7VÇBf÷FR…7G&–ærÆ–W"Â7G&–ær6W'f–6RÂ&ööÆVâ&VÅf÷FRÂ&ööÆVâF–ÖUVWVRÂÆöærVWVUF–ÖRÀ •f÷FUF÷FÇ56æ6†÷BFW‡BÂ7G&–ærWV–BÂf÷FUF–ÖUVWVRVWVVEf÷FR’° —G'’° ––b‚6W'f–6U6—FUfÆ–FF÷"æ—5fÆ–B‡6W'f–6R’’° —v&â‚%&V¦V7FVBf÷FRv—F‚–çfÆ–B6W'f–6R6—FRr"²6W'f–6U6—FUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡6W'f–6R’²"r"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р––b‚Ö–æV7&gEW6W&æÖUfÆ–FF÷"æ—5fÆ–B‡Æ–W"ÂvWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’’’° —v&â‚%&V¦V7FVBf÷FRv—F‚–çfÆ–BÖ–æV7&gBW6W&æÖRr  ’²Ö–æV7&gEW6W&æÖUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡Æ–W"’²"rg&öÒ6W'f–6Rr  ’²Ö–æV7&gEW6W&æÖUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡6W'f–6R’²"r"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р •UT”Bf÷FT–BÒVWVVEf÷FRÓÒçVÆÂòçVÆ¢VWVVEf÷FRævWEf÷FT–B‚“° ––b‡f÷FT–BÓÒçVÆÂ’° —f÷FT–BÒUT”Bç&æFöÕUT”B‚“° —Р ’òòUT”B&W6öÇWF–öà ––b‚vWD6öæf–r‚’ævWDöæÆ–æTÖöFR‚’’° —WV–BÒvWEUT”B‡Æ–W"“° —Р ––b‡WV–BÓÒçVÆÂÇÂWV–Bæ—4V×G’‚’’° —WV–BÒvWEUT”B‡Æ–W"“°  ’òò&VG&ö6²&Vf—‚WFòÖFWFV7@ ––b‡WV–Bæ—4V×G’‚’bbvWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’æ—4V×G’‚ ’bbÆ–W"ç7F'G5v—F‚†vWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’’’° •7G&–ærWV–CÒvWEUT”B†vWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’²Æ–W"“° ––b‚WV–Cæ—4V×G’‚’’° –FV'Vr‚$FWFV7FVB&VG&ö6²Æ–W"v—F†÷WB&Vf—‚ÂF§W7F–ærâââ"“° —Æ–W"ÒvWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’²Æ–W#° —WV–BÒWV–C° —Р—Р—Р ––b‡WV–Bæ—4V×G’‚’’° ––b‡Æ–W"ç7F'G5v—F‚†vWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’’’° –Æör‚$–væ÷&–ærf÷FR6–æ6RVæ&ÆRFòvWBUT”Böb&VG&ö6²Æ–W""“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р––b‚vWD6öæf–r‚’ævWDÆÆ÷uVä¦ö–æVB‚’’° –Æör‚$–væ÷&–ærf÷FRg&öÒ"²Æ–W"²"6–æ6RÆ–W"†6âwB¦ö–æVB&Vf÷&R"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р––b‚vWD6öæf–r‚’ævWEUT”DÆöö·W‚’’° –Æör‚$f–ÆVBFòvWBWV–Bf÷""²Æ–W"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р –FV'Vr‚$fWF6†–ærUT”BöæÆ–æRÂ6–æ6RÆÆ÷wVæ¦ö–æVB—2Væ&ÆVB"“° •UT”BRÒçVÆð —G'’° ––b†vWD6öæf–r‚’ævWDöæÆ–æTÖöFR‚’’° —RÒfWF6…UT”B‡Æ–W"“° —Р—Ò6F6‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р––b‡RÓÒçVÆÂ’° –FV'Vr‚$f–ÆVBFòvWBWV–Bf÷""²Æ–W"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р—WV–BÒRçFõ7G&–ær‚“° —Р ’òòæ÷&ÖÆ—¦RUT”B7G&–ær–b÷76–&ÆP —G'’° ––b‡WV–BÒçVÆÂbbWV–Bæ—4V×G’‚’bbWV–BæWVÇ4–væ÷&T66R‚&çVÆÂ"’’° —WV–BÒUT”Bæg&öÕ7G&–ær‡WV–BçG&–Ò‚’’çFõ7G&–ær‚“° —Р—Ò6F6‚„W†6WF–öâ–væ÷&VB’° ’òò–væ÷&P —Р —Æ–W"ÒvWE&÷W$æÖR‡WV–BÂÆ–W"“°  ’òò66†RöæÆ–æR7FFR÷6W'fW"öæ6R„”Õõ%DåBf÷"'&öF67BÆöv–26÷'&V7FæW72 –f–æÂ&ööÆVâÆ–W$öæÆ–æRÒ—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær‡Æ–W"“° –f–æÂ7G&–ærÆ–W%6W'fW"ÒÆ–W$öæÆ–æRòvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær‡Æ–W"’¢çVÆð –ÆöærF–ÖRÒVWVUF–ÖRÒòVWVUF–ÖP “¢Æö6ÄFFUF–ÖRææ÷r‚’æE¦öæR…¦öæT–Bç7—7FVÔFVfVÇB‚’’çFô–ç7FçB‚’çFôWö6„Ö–ÆÆ’‚“°  •6WCÅ7G&–æsâ'&öF67EF&vWG2ÒVWVVEf÷FRÓÒçVÆÂòæWrÆ–æ¶VD†6…6WCÃâ‚ “¢æWrÆ–æ¶VD†6…6WCÃâ‡VWVVEf÷FRævWD'&öF67EF&vWG2‚’“° •6WCÅ7G&–æsâ'&öF67Df÷'v&FVE6W'fW'2ÒVWVVEf÷FRÓÒçVÆÂòæWrÆ–æ¶VD†6…6WCÃâ‚ “¢æWrÆ–æ¶VD†6…6WCÃâ‡VWVVEf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° –&ööÆVâ&÷‡”'&öF67D†æFÆVBÒVWVVEf÷FRÒçVÆÂbbVWVVEf÷FRæ—5&÷‡”'&öF67D†æFÆVB‚“° –&ööÆVâ&ö6W76W5F÷FÇ2ÒvWD6öæf–r‚’ævWE&–Ö'•6W'fW"‚’ÇÂvWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚“° –&ööÆVâÖævW5F÷FÇ2Ò&ö6W76W5F÷FÇ2bbvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚“° –&ööÆVâ6åfÆ–FFU7FæFÆöæT'&öF67BÒ6äf÷'v&E7FæFÆöæT'&öF67B†ÖævW5F÷FÇ2“° ”'&”Æ—7CÄ6öÇVÖãâFFÒçVÆð –&ööÆVâVWVTf÷%F–ÖT6†ævRÒfÇ6S°  ’òò6ö×ÆWF–öâ6ÆÆ&6²6âv—RF÷FÇ2æB&WÆ’öÆFW"VWVVBf÷FW2â'Vâ—@ ’òò&Vf÷&RÆöF–ærF†—2f÷FRw2FF&6R6æ6†÷B6òF†R6Æ7VÆF–öç2&VÆ÷rW6P ’òòF†R÷7B×&öÆÆ÷fW"7FFRà ––b†vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’bbvWDvÆö&ÄFF†æFÆW"‚’æ—5F–ÖT6†ævVD†VæVB‚’’° –vWDvÆö&ÄFF†æFÆW"‚’æ6†V6´f÷$f–æ—6†VEF–ÖT6†ævW2‚“° —VWVTf÷%F–ÖT6†ævRÒF–ÖUVWVRbbvWDvÆö&ÄFF†æFÆW"‚’æ—5F–ÖT6†ævVD†VæVB‚“° —Р ’òòfÆ–FFRF†Rf÷FR&Vf÷&Rç’–ÖÖVF–FRææ÷Væ6VÖVçBâF†—2¶VW2GWÆ–6FP ’òòf÷FW2&V¦V7FVB'’F†RFVÆ’6†V6²÷WBöbF†RvÆö&ÄFF&öÆÆ÷fW"VWVRæ@ ’òò&WfVçG2ææ÷Væ6–ærf÷FRF†Bv–ÆÂæ÷B&R&ö6W76VBà ––b†ÖævW5F÷FÇ2’° ––b†vWE&÷‡”ו5‚’ÓÒçVÆÂ’° –Æöu6WfW&R‚$×—7—2æ÷BÆöFVB6÷'&V7FÇ’Â7F÷–ærf÷FR&ö6W76–ær"“° —&WGW&âVWVVEf÷FU&W7VÇBå$UE%“° —Р ––b‚vWE&÷‡”ו5‚’æ6öçF–ç4¶W•VW'’‡WV–B’’° –vWE&÷‡”ו5‚’çWFFR‡WV–BÂ%Æ–W$æÖR"ÂæWrFFfÇVU7G&–ær‡Æ–W"’“° –vWE&÷‡”ו5‚’ævWEWV–G2‚’æFB‡WV–B“° —Р –FFÒvWE&÷‡”ו5‚’ævWDW†7EVW'’†æWr6öÇVÖâ‚'WV–B"ÂæWrFFfÇVU7G&–ær‡WV–B’’“° ––b‚6†V6µf÷FTFVÆ’‡WV–BÂÆ–W"Â6W'f–6RÂFFÂVWVVEf÷FRÓÒçVÆÂ’’° –Æör‚%f÷FRFVÆ’—2æ÷BÖWBf÷""²Æ–W"²"ò"²6W'f–6R²"Â6¶—–ærf÷FR"“° —6VæEf÷FTFVÆ•&V¦V7FVB‡Æ–W"ÂWV–BÂ6W'f–6RÂÆ–W$öæÆ–æRÂÆ–W%6W'fW"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р—Р ’òòf÷'v&Bâ66WFVBöffÆ–æR'&öF67B&Vf÷&RF†R7F–ÆÂÖ7F—fRvÆö&ÄFF ’òò6†ævRVWVW2F†R&Wv&B÷F÷FÇ2v÷&²âF†RVWVVBFVÆ—fW'’7FFR&WfVçG0 ’òò&WÆ––ær'&öF67G2F†BÇ&VG’&V6†VB&6¶VæBà ––b‡VWVTf÷%F–ÖT6†ævR’° •f÷FUF÷FÇ56æ6†÷B&ö¦V7FVEF÷FÇ2ÒÖævW5F÷FÇ2òvWE&ö¦V7FVE&öÆÆ÷fW%F÷FÇ2†FFÂÆ–W"’¢FW‡C° ––b†6åfÆ–FFU7FæFÆöæT'&öF67Bbb&÷‡”'&öF67DFV6–FW"çW6W4–ÖÖVF–FTf÷'v&F–ær‡Æ–W$öæÆ–æR’’° –'&öF67EF&vWG2æFDƇ&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2†fÇ6RÂçVÆÂ’“° —&÷‡”'&öF67D†æFÆVBÒG'VS° —Р•f÷FUF–ÖUVWVRFVÆ–VEf÷FRÒæWrf÷FUF–ÖUVWVR‡f÷FT–BÂÆ–W"Â6W'f–6RÂF–ÖRÀ —&÷‡”'&öF67D†æFÆVBÂ'&öF67EF&vWG2Â'&öF67Df÷'v&FVE6W'fW'2À —&ö¦V7FVEF÷FÇ2ÓÒçVÆÂò""¢&ö¦V7FVEF÷FÇ2çFõ7G&–ær‚’ÂfÇ6RÂWV–B“° ––b‚vWEf÷FT66†T†æFÆW"‚’æFEF–ÖUf÷FUFô66†R†FVÆ–VEf÷FR’’° –Æöu6WfW&R‚%Væ&ÆRFòW'6—7BVWVVB&öÆÆ÷fW"f÷FRf÷""²Æ–W"²"ò"²6W'f–6P ’²#²6¶—–ær&÷‡’'&öF67B"“° —&WGW&âVWVVEf÷FU&W7VÇBå$UE%“° —Р––b‡&÷‡”'&öF67D†æFÆVB’° –f÷"…7G&–ærF&vWB¢'&öF67EF&vWG2’° •6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡F&vWB’ÂWV–BÂÆ–W"À —6W'f–6RÂF–ÖRÂ&ö¦V7FVEF÷FÇ2ÓÒçVÆÂò""¢&ö¦V7FVEF÷FÇ2çFõ7G&–ær‚’ÂfÇ6R“° ––b†FVÆ–VEf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –'&öF67Df÷'v&FVE6W'fW'2æFDƆf÷'v&FVB“° —W'6—7EF–ÖUf÷FTFVÆ—fW'’†FVÆ–VEf÷FR“° —Р—Р—Р–Æör‚$66†–ærf÷FRg&öÒ"²Æ–W"²"ò"²6W'f–6P ’²"&V6W6RF–ÖR6†ævR—2†Væ–ær&–v‡Bæ÷r"“° —&WGW&âVWVVEf÷FU&W7VÇBå5T44U53° —Р –FEf÷FU'G’‚“°  ’òòF÷FÇ2&ö6W76–ær‡&–Ö'’6W'fW"õ"æò×VÇF—&÷‡’ ––b‡&ö6W76W5F÷FÇ2’° ––b†ÖævW5F÷FÇ2’° ––çBÆÅF–ÖUF÷FÂÒvWEfÇVR†FFÂ$ÆÅF–ÖUF÷FÂ"“° ––çBÖöçF…F÷FÂÒvWEfÇVR†FFÂ$ÖöçF…F÷FÂ"“°  ––çBFFTÖöçF…F÷FÂÒÓ° ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° ––b†vWD6öæf–r‚’ævWEW6TÖöçF„FFUF÷FÇ45&–Ö'•F÷F‚’’° –FFTÖöçF…F÷FÂÒvWEfÇVR†FFÂvWDÖöçF…F÷FÇ5v—F„FFUF‚‚’“° —ÒVÇ6R° –FFTÖöçF…F÷FÂÒÖöçF…F÷Fð —Р—Р ––çBvVV¶Ç•F÷FÂÒvWEfÇVR†FFÂ%vVV¶Ç•F÷FÂ"“° ––çBF–Ç•F÷FÂÒvWEfÇVR†FFÂ$F–Ç•F÷FÂ"“° ––çBö–çG2ÒvWEfÇVR†FFÂ%ö–çG2"ÂvWD6öæf–r‚’ævWEö–çG4öåf÷FR‚’“°  ––çBÖ…f÷FW2ÒvWD6öæf–r‚’ævWDÖ„Ö÷VçDöef÷FW5W$F’‚“° ––b†Ö…f÷FW2â’° ”Æö6ÄFFUF–ÖR5F–ÖRÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚“° ––çBF—2Ò5F–ÖRævWDF”ödÖöçF‚‚“° ––b†ÖöçF…F÷FÂâF—2¢Ö…f÷FW2’° –ÖöçF…F÷FÂÒF—2¢Ö…f÷FW3° —Р—Р ––b†vWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’âbbö–çG2âvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’’° —ö–çG2ÒvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚“° —Р —FW‡BÒæWrf÷FUF÷FÇ56æ6†÷B†ÆÅF–ÖUF÷FÂÂÖöçF…F÷FÂÂvVV¶Ç•F÷FÂÂF–Ç•F÷FÂÂö–çG2À —f÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VBÂFFTÖöçF…F÷F“°  ”'&”Æ—7CÄ6öÇVÖãâWFFRÒæWr'&”Æ—7CÃâ‚“° —WFFRæFB†æWr6öÇVÖâ‚$ÆÅF–ÖUF÷FÂ"ÂæWrFFfÇVT–çB†ÆÅF–ÖUF÷FÂ’’“° —WFFRæFB†æWr6öÇVÖâ‚$ÖöçF…F÷FÂ"ÂæWrFFfÇVT–çB†ÖöçF…F÷FÂ’’“° ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° —WFFRæFB†æWr6öÇVÖâ†vWDÖöçF…F÷FÇ5v—F„FFUF‚‚’ÂæWrFFfÇVT–çB†FFTÖöçF…F÷FÂ’’“° —Р—WFFRæFB†æWr6öÇVÖâ‚%vVV¶Ç•F÷FÂ"ÂæWrFFfÇVT–çB‡vVV¶Ç•F÷FÂ’’“° —WFFRæFB†æWr6öÇVÖâ‚$F–Ç•F÷FÂ"ÂæWrFFfÇVT–çB†F–Ç•F÷FÂ’’“° —WFFRæFB†æWr6öÇVÖâ‚%ö–çG2"ÂæWrFFfÇVT–çB‡ö–çG2’’“°  –FV'Vr‚%6WGF–ærF÷FÇ2"²FW‡BçFõ7G&–ær‚’²"Âf÷FT–CÒ"²f÷FT–B²"f÷""²Æ–W"²"ò  ’²6W'f–6R“° –vWE&÷‡”ו5‚’çWFFR‡WV–BÂWFFR“° —ÒVÇ6R° —FW‡BÒæWrf÷FUF÷FÇ56æ6†÷BƒÂÂÂÂÂf÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VB“° —Р—Р––b‡FW‡BÓÒçVÆÂ’° —FW‡BÒæWrf÷FUF÷FÇ56æ6†÷BƒÂÂÂÂÂf÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VB“° —Р •f÷FTÆöu7FGW2f÷FU7FGW2Òf÷FTÆöu7FGW2ä”ÔÔTD”DS° –&ööÆVâ7FæFÆöæU&÷‡”'&öF67BÒ6åfÆ–FFU7FæFÆöæT'&öF67Bbb‡&÷‡”'&öF67D†æFÆV@ —ÇÂ&÷‡”'&öF67DFV6–FW"çW6W4–ÖÖVF–FTf÷'v&F–ær‡Æ–W$öæÆ–æR’“° •6WCÅ7G&–æsâ&÷‡”'&öF67EF&vWG2Ò6öÆÆV7F–öç2æV×G•6WB‚“° ––b‡7FæFÆöæU&÷‡”'&öF67B’° ’òò†æFÆVBVWVVB'&öF67Bv2æV6W76&–Ç’6×ÆVBv†–ÆRF†RÆ–W"v0 ’òòöffÆ–æRâ&WG'’öæÇ’F&vWG2F†BF–Bæ÷B&Wf–÷W6Ç’66WBFVÆ—fW'’à —&÷‡”'&öF67EF&vWG2Ò&÷‡”'&öF67D†æFÆVBòæWrÆ–æ¶VD†6…6WCÃâ†'&öF67EF&vWG2 “¢&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2†fÇ6RÂçVÆ“° •6WCÅ7G&–æsâ&VÖ–æ–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ‡&÷‡”'&öF67EF&vWG2“° —&VÖ–æ–æuF&vWG2ç&VÖ÷fTƆ'&öF67Df÷'v&FVE6W'fW'2“° –'&öF67Df÷'v&FVE6W'fW'2æFDƇ6VæE&÷‡”'&öF67B‡&VÖ–æ–æuF&vWG2ÂWV–BÂÆ–W"Â6W'f–6RÂF–ÖRÀ —FW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’ÂfÇ6R’“° —Р ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР’òò6VæBf÷FR‡2’Fò&6¶VæB‡2 ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР––b†vWD6öæf–r‚’ævWE6VæEf÷FW5FôÆÅ6W'fW'2‚’’° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’°  –&ööÆVâf÷&6T66†RÒvWD6öæf–r‚’ævWEv—Df÷%W6W$öæÆ–æR‚ ’bb‚Æ–W$öæÆ–æRÇÂÆ–W%6W'fW"ÓÒçVÆÂÇÂÆ–W%6W'fW"æWVÇ4–væ÷&T66R‡2’“°  ––b†f÷&6T66†R’° –FV'Vr‚$f÷&6–ærf÷FRFò66†Rf÷"6W'fW""²2“° —Р ––b‚‚—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡2’bbÖWF†öBç&WV—&W5Æ–W$öæÆ–æR‚’’ÇÂf÷&6T66†R’° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFE6W'fW%f÷FR‡2À –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÀ —FW‡BçFõ7G&–ær‚’Â'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÀ —&÷‡”'&öF67EF&vWG2Â'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–ærf÷FRf÷""²Æ–W"²"öâ"²6W'f–6R²"f÷""²2“° —ÒVÇ6R° –&ööÆVâ'&öF67D†W&RÒ'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç2‡2“° ––b†'&öF67D†W&RbbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° •6WCÅ7G&–æsâF&vWG2Ò7FæFÆöæU&÷‡”'&öF67Bò&÷‡”'&öF67EF&vWG0 “¢&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡Æ–W$öæÆ–æRÂÆ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡2ÂF&vWG2“° —Р ––b‚6VæEf÷FTVçfVÆ÷T66WFVB‡2Â"À •f÷F–æuÇVv–åv—&Rçf÷FR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂG'VRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À —f÷FT–BÂvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’Â'&öF67D†W&RÂÂ’’’° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFE6W'fW%f÷FR‡2À –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÀ —FW‡BçFõ7G&–ær‚’Â'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÀ —&÷‡”'&öF67EF&vWG2Â'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–ærf÷FRgFW"F†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²2“° —Р—Р—Р—ÒVÇ6R° ’òò6–ævÆR×6W'fW"ÖöFS¢öæÆ–æRvöW2FòÆ–W"6W'fW#²÷F†W'v—6RVWVR2&öæÆ–æP ’òòf÷FR  ––b‡Æ–W$öæÆ–æRbbÆ–W%6W'fW"ÒçVÆÂbbvWDÆÄf–Æ&ÆU6W'fW'2‚’æ6öçF–ç2‡Æ–W%6W'fW"’’° •7G&–ær6W'fW"ÒÆ–W%6W'fW#°  –&ööÆVâ'&öF67D†W&RÒ'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç2‡6W'fW"“° ––b†'&öF67D†W&RbbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° •6WCÅ7G&–æsâF&vWG2Ò7FæFÆöæU&÷‡”'&öF67Bò&÷‡”'&öF67EF&vWG0 “¢&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡G'VRÂÆ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡6W'fW"ÂF&vWG2“° —Р –&ööÆVâ&Wv&D66WFVBÒ6VæEf÷FTVçfVÆ÷T66WFVB‡6W'fW"ÂÀ •f÷F–æuÇVv–åv—&Rçf÷FTöæÆ–æR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂG'VRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À —f÷FT–BÂvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’Â'&öF67D†W&RÂÂ’“° ––b‚&Wv&D66WFVB’° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFDöæÆ–æUf÷FR‡WV–BÀ –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À –'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÂ&÷‡”'&öF67EF&vWG2À –'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–æröæÆ–æRf÷FRgFW"F†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²6W'fW"“° —Р ––b‡&Wv&D66WFVBbb6åfÆ–FFU7FæFÆöæT'&öF67BbbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚ ’bb7FæFÆöæU&÷‡”'&öF67B’° •6WCÅ7G&–æsâF&vWG2Ò&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡G'VRÂÆ–W%6W'fW"“°  ––çB$FVÆ’Ò#° –f÷"…7G&–ærF&vWE6W'fW"¢F&vWG2’° ’òòfö–BF÷V&ÆRÖ'&öF67BöâF†R6ÖR6W'fW"F†BÇ&VG’v÷BF†Rf÷FTöæÆ–æP ––b‡F&vWE6W'fW"æWVÇ4–væ÷&T66R‡6W'fW"’’° –6öçF–çVS° —Р––b†vWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡F&vWE6W'fW"’’° –6öçF–çVS° —Р –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡F&vWE6W'fW"Â$FVÆ’À •f÷F–æuÇVv–åv—&Rçf÷FT'&öF67B‡WV–BÂÆ–W"Â6W'f–6RÂF–ÖRÀ —FW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’ÂG'VR’“° –$FVÆ’²³° —Р—Р ’òò×VÇF—&÷‡“¢VçfVÆ÷RÖöæÇ’6ÆV"f÷FP ––b‡&Wv&D66WFVBbbvWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚’bbvWD6öæf–r‚’ævWD×VÇF•&÷‡”öæTvÆö&Å&Wv&B‚’’° –×VÇF•&÷‡”†æFÆW"ç6VæD6ÆV%f÷FR‡WV–BÂÆ–W"“° —Р—ÒVÇ6R° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFDöæÆ–æUf÷FR‡WV–BÀ –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À –'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÂ&÷‡”'&öF67EF&vWG2À –'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–æröæÆ–æRf÷FRf÷""²Æ–W"²"öâ"²6W'f–6R“° —Р ––çBFVÆ’Ò#° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡2ÂFVÆ’²Âf÷F–æuÇVv–åv—&Rçf÷FUWFFR‡WV–BÀ —f÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VBÂ6W'f–6RÂF–ÖRÂFW‡BçFõ7G&–ær‚’’“° –FVÆ’³Ò#° —Р—Р ’òòf÷FRÆövv–æp ––b‡f÷FTÆöt×—7ÅF&ÆRÒçVÆÂbbvWD6öæf–r‚’ævWEf÷FTÆövv–ætVæ&ÆVB‚’’° —f÷FTÆöt×—7ÅF&ÆRæÆöuf÷FR‡f÷FT–BÂf÷FU7FGW2Â6W'f–6RÂWV–BÂÆ–W"ÂF–ÖRÀ –vWEf÷FT66†T†æFÆW"‚’ævWE&÷‡”66†VEF÷F‡WV–B’“° —Р ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР’òò×VÇF—&÷‡’f÷'v&F–æp ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР––b†vWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚’bbvWD6öæf–r‚’ævWE&–Ö'•6W'fW"‚’’° ––b‚vWD6öæf–r‚’ævWD×VÇF•&÷‡”öæTvÆö&Å&Wv&B‚’’° –FV'Vr‚%6VæF–ærvÆö&Â&÷‡’f÷FRVçfVÆ÷R"“° –×VÇF•&÷‡”†æFÆW"ç6VæD×VÇF•&÷‡”VçfVÆ÷R…f÷F–æuÇVv–åv—&Rçf÷FR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂfÇ6RÀ —&VÅf÷FRÂFW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’Âf÷FT–BÂfÇ6RÂfÇ6RÂÂ’“° —ÒVÇ6R° ’òòöæÇ’6VæBFò÷F†W"&÷†–W2–bF†RÆ–W"D”BäõBÇ&VG’&V6V—fR&Wv&Böâ ’òò&6¶Væ@ –&ööÆVâ6†÷VÆE6VæBÒG'VS° ––b‡Æ–W$öæÆ–æRbbÆ–W%6W'fW"ÒçVÆÂ’° ––b‚vWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡Æ–W%6W'fW"’’° —6†÷VÆE6VæBÒfÇ6S° —Р—Р ––b‡6†÷VÆE6VæB’° –FV'Vr‚%6VæF–ærvÆö&Â&÷‡’f÷FVöæÆ–æRVçfVÆ÷R"“° –×VÇF•&÷‡”†æFÆW  ’ç6VæD×VÇF•&÷‡”VçfVÆ÷R…f÷F–æuÇVv–åv—&Rçf÷FTöæÆ–æR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂfÇ6RÀ —&VÅf÷FRÂFW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’Âf÷FT–BÂfÇ6RÂfÇ6RÂÂ’“° —ÒVÇ6R° –FV'Vr‚$æ÷B6VæF–ærvÆö&Â&÷‡’ÖW76vRf÷"f÷FVöæÆ–æRÂÆ–W"Ç&VG’v÷B&Wv&B"“° —Р—Р—Р––b‡VWVVEf÷FRÒçVÆÂ’° —VWVVEf÷FRç6WE&ö6W76VB‡G'VR“° ––b‚vWEf÷FT66†T†æFÆW"‚’çWFFUF–ÖUf÷FR‡VWVVEf÷FR’’° —v&â‚%Væ&ÆRFòW'6—7B6ö×ÆWFVB&öÆÆ÷fW"f÷FR"²VWVVEf÷FRævWEf÷FT–B‚ ’²#²GFV×F–ærGW&&ÆR&VÖ÷f–ÖÖVF–FVÇ’"“° —Р—Р—&WGW&âVWVVEf÷FU&W7VÇBå5T44U53° —Ò6F6‚„W†6WF–öâR’° –Rç&–çE7F6µG&6R‚“° —&WGW&âVWVVEf÷FU&W7VÇBå$UE%“° —Р—Р —&—fFR7FF–2f–æÂ6Æ72VæF–æu&W6Væ6T†æFöfb° —&—fFRUT”B&WVW7D–C° —&—fFRf–æÂUT”BÆ–W%WV–C° —&—fFRf–æÂ7G&–ærÆ–W$æÖS° —&—fFRf–æÂ7G&–ærWV–C° —&—fFRf–æÂ7G&–ær6W'fW#° —&—fFRf–æÂUT”B6öææV7F–öä–C° —&—fFRf–æÂUT”B&6¶VæD–æ6&æF–öä–C° —&—fFRf–æÂÆöær&6¶VæE7F'FVDC° —&—fFRf–æÂÆöær6öæfÆ–7E6WVVæ6S° —&—fFRf–æÂÆöær7&VFVDC°  —&—fFRVæF–æu&W6Væ6T†æFöfb…7G&–ærÆ–W$æÖRÂ7G&–ærWV–BÂ7G&–ær6W'fW"ÂUT”B6öææV7F–öä–BÀ •UT”B&6¶VæD–æ6&æF–öä–BÂÆöær&6¶VæE7F'FVDBÂÆöær6öæfÆ–7E6WVVæ6RÂÆöær7&VFVDB’° —F†—2çÆ–W%WV–BÒ'6UÆ–W%WV–B‡WV–B“° —F†—2çÆ–W$æÖRÒÆ–W$æÖS° —F†—2çWV–BÒWV–C° —F†—2ç6W'fW"Ò6W'fW#° —F†—2æ6öææV7F–öä–BÒ6öææV7F–öä–C° —F†—2æ&6¶VæD–æ6&æF–öä–BÒ&6¶VæD–æ6&æF–öä–C° —F†—2æ&6¶VæE7F'FVDBÒ&6¶VæE7F'FVDC° —F†—2æ6öæfÆ–7E6WVVæ6RÒ6öæfÆ–7E6WVVæ6S° —F†—2æ7&VFVDBÒ7&VFVDC° —Р —&—fFR7FF–2UT”B'6UÆ–W%WV–B…7G&–ærWV–B’° —G'’° —&WGW&âUT”Bæg&öÕ7G&–ær‡WV–BçG&–Ò‚’“° —Ò6F6‚„W†6WF–öâ–væ÷&VB’° —&WGW&âçVÆð —Р—Р—Р —V&Æ–2'7G&7Bfö–Bv&â…7G&–ærÖW76vR“°  —V&Æ–2'7G&7B66†VGVÆVDW†V7WF÷%6W'f–6RvWE66†VGVÆW"‚“°§Ð  \ No newline at end of file +package com.bencodez.votingplugin.proxy; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import javax.net.ssl.SSLParameters; + +import org.eclipse.paho.client.mqttv3.MqttException; + +import com.bencodez.advancedcore.api.time.TimeType; +import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalDataHandlerProxy; +import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalMySQL; +import com.bencodez.advancedcore.bungeeapi.time.BungeeTimeChecker; +import com.bencodez.simpleapi.encryption.EncryptionHandler; +import com.bencodez.simpleapi.json.JsonParser; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageListener; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler; +import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; +import com.bencodez.simpleapi.servercomm.mqtt.MqttServerComm; +import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; +import com.bencodez.simpleapi.servercomm.redis.RedisHandler; +import com.bencodez.simpleapi.servercomm.redis.RedisListener; +import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; +import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; +import com.bencodez.simpleapi.servercomm.sockets.SocketReceiver; +import com.bencodez.simpleapi.sql.Column; +import com.bencodez.simpleapi.sql.DataType; +import com.bencodez.simpleapi.sql.data.DataValue; +import com.bencodez.simpleapi.sql.data.DataValueBoolean; +import com.bencodez.simpleapi.sql.data.DataValueInt; +import com.bencodez.simpleapi.sql.data.DataValueString; +import com.bencodez.simpleapi.sql.mysql.config.MysqlConfig; +import com.bencodez.votingplugin.backendproxy.http.HttpEnrollmentAuthority; +import com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer; +import com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity; +import com.bencodez.votingplugin.proxy.broadcast.ProxyBroadcastDecider; +import com.bencodez.votingplugin.proxy.cache.IVoteCache; +import com.bencodez.votingplugin.proxy.cache.VoteCacheHandler; +import com.bencodez.votingplugin.proxy.cache.nonvoted.INonVotedPlayersStorage; +import com.bencodez.votingplugin.proxy.cache.nonvoted.NonVotedPlayersCache; +import com.bencodez.votingplugin.proxy.control.ControlConnector; +import com.bencodez.votingplugin.proxy.control.HostedControlManager; +import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyHandler; +import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyMethod; +import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfiguration; +import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfigurationBungee; +import com.bencodez.votingplugin.proxy.presence.BackendPlayerPresenceTracker; +import com.bencodez.votingplugin.proxy.presence.PlayerPresence; +import com.bencodez.votingplugin.timequeue.VoteTimeQueue; +import com.bencodez.votingplugin.topvoter.TopVoter; +import com.bencodez.votingplugin.util.MinecraftUsernameValidator; +import com.bencodez.votingplugin.util.ServiceSiteValidator; +import com.bencodez.votingplugin.votelog.VoteLogMysqlTable; +import com.bencodez.votingplugin.votelog.VoteLogMysqlTable.VoteLogStatus; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import redis.clients.jedis.DefaultJedisClientConfig; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import lombok.Getter; +import lombok.Setter; + +public abstract class VotingPluginProxy { + private static final long PRESENCE_HANDOFF_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(2); + private static final long PRESENCE_STARTUP_RESYNC_DELAY_SECONDS = 5L; + private static final long PRESENCE_MAINTENANCE_INTERVAL_SECONDS = 30L; + private static final long PRESENCE_BACKEND_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(90); + private static final long CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10); + + @Getter + @Setter + private int votePartyVotes = 0; + + @Getter + @Setter + private int currentVotePartyVotesRequired = 0; + + @Getter + @Setter + private ProxyMysqlUserTable proxyMySQL; + + private EncryptionHandler encryptionHandler; + + private HashMap clientHandles; + + private SocketHandler socketHandler; + private HttpProxyTransportServer httpTransportServer; + private HttpEnrollmentAuthority httpEnrollmentAuthority; + + @Getter + @Setter + private boolean votifierEnabled = true; + + @Getter + private ConcurrentHashMap uuidPlayerNameCache = new ConcurrentHashMap<>(); + + @Getter + @Setter + private GlobalDataHandlerProxy globalDataHandler; + + @Getter + private RedisHandler redisHandler; + private JedisPool redisPublisherPool; + private volatile long redisPublisherRetryAfter; + private boolean timeVoteRetryScheduled; + private boolean timeVoteDeliveryRetryScheduled; + private boolean cachedVoteDeliveryRetryScheduled; + + private boolean enabled; + + @Getter + @Setter + private MultiProxyHandler multiProxyHandler; + + @Getter + private BungeeTimeChecker bungeeTimeChecker; + + @Getter + @Setter + private BungeeMethod method; + + @Getter + private MqttHandler mqttHandler; + + @Getter + private GlobalMessageProxyHandler globalMessageProxyHandler; + + @Getter + @Setter + private MySqlMessenger proxyMysqlMessenger; + + @Getter + private VoteCacheHandler voteCacheHandler; + + @Getter + private NonVotedPlayersCache nonVotedPlayersCache; + + @Getter + private final BackendPlayerPresenceTracker backendPlayerPresenceTracker = new BackendPlayerPresenceTracker(); + private final Map pendingPresenceHandoffs = new HashMap<>(); + private final Set pendingBackendRecoverySnapshots = ConcurrentHashMap.newKeySet(); + private final Map controlEnrollmentNextAllowed = new ConcurrentHashMap<>(); + private final Map pendingCommunicationTests = new ConcurrentHashMap<>(); + private volatile ControlConnector controlConnector; + private volatile HostedControlManager hostedControlManager; + private final Object controlLifecycleLock = new Object(); + private final AtomicLong controlServicesGeneration = new AtomicLong(); + private final ExecutorService controlLifecycleExecutor = Executors.newSingleThreadExecutor(task -> { + Thread thread = new Thread(task, "votingplugin-control-lifecycle"); + thread.setDaemon(true); + return thread; + }); + + public VotingPluginProxy() { + enabled = true; + + bungeeTimeChecker = new BungeeTimeChecker(getConfig().getTimeZone(), getConfig().getTimeHourOffSet(), + getConfig().getTimeWeekOffSet()) { + + @Override + public void debug(String text) { + debug2(text); + } + + @Override + public long getLastUpdated() { + return getVoteCacheLastUpdated(); + } + + @Override + public int getPrevDay() { + return getVoteCachePrevDay(); + } + + @Override + public String getPrevMonth() { + return getVoteCachePrevMonth(); + } + + @Override + public int getPrevWeek() { + return getVoteCachePrevWeek(); + } + + @Override + public void info(String text) { + log(text); + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + public boolean isIgnoreTime() { + return isVoteCacheIgnoreTime(); + } + + @Override + public void setIgnoreTime(boolean ignore) { + setVoteCacheVoteCacheIgnoreTime(ignore); + } + + @Override + public void setLastUpdated() { + setVoteCacheLastUpdated(); + } + + @Override + public void setPrevDay(int day) { + setVoteCachePrevDay(day); + } + + @Override + public void setPrevMonth(String text) { + setVoteCachePrevMonth(text); + } + + @Override + public void setPrevWeek(int week) { + setVoteCachePrevWeek(week); + } + + @Override + public void timeChanged(TimeType type, boolean fake, boolean pre, boolean post) { + if (getConfig().getVoteCacheTime() > 0) { + getVoteCacheHandler().checkVoteCacheTime(getConfig().getVoteCacheTime()); + } + if (!getConfig().getGlobalDataEnabled()) { + warn("Global data not enabled, ignoring time change event"); + return; + } + int delay = 1; + for (String s : getAllAvailableServers()) { + if (getGlobalDataHandler().getGlobalMysql().containsKey(s)) { + String lastOnlineStr = getGlobalDataHandler().getString(s, "LastOnline"); + long lastOnline = 0; + try { + lastOnline = Long.valueOf(lastOnlineStr); + } catch (NumberFormatException e) { + // ignore + } + + if (LocalDateTime.now().atZone(ZoneOffset.UTC).toInstant().toEpochMilli() - lastOnline < 1000 + * 60 * 60 * 12) { + HashMap dataToSet = new HashMap<>(); + dataToSet.put("LastUpdated", new DataValueString( + "" + LocalDateTime.now().atZone(ZoneOffset.UTC).toInstant().toEpochMilli())); + dataToSet.put("FinishedProcessing", new DataValueBoolean(false)); + dataToSet.put(type.toString(), new DataValueBoolean(true)); + getGlobalDataHandler().setData(s, dataToSet); + + globalMessageProxyHandler.sendMessage(s, delay, VotingPluginWire.bungeeTimeChange()); + delay++; + } else { + warn("Server " + s + " hasn't been online recently"); + } + } else { + warn("Server " + s + " global data handler disabled?"); + } + } + globalDataHandler.onTimeChange(type); + } + + @Override + public void warning(String text) { + warn(text); + } + }; + } + + public void onTimeChangedFailed(String srv, TimeType type) { + getGlobalDataHandler().setBoolean(srv, type.toString(), false); + getGlobalDataHandler().setBoolean(srv, "FinishedProcessing", true); + getGlobalDataHandler().setBoolean(srv, "Processing", false); + } + + public void onTimeChangedFinished(TimeType type) { + if (type.equals(TimeType.MONTH)) { + getProxyMySQL().copyColumnData(TopVoter.Monthly.getColumnName(), "LastMonthTotal"); + } + getProxyMySQL().wipeColumnData(TopVoter.of(type).getColumnName(), DataType.INTEGER); + + if (!getConfig().getGlobalDataEnabled()) { + return; + } + for (String s : getAllAvailableServers()) { + getGlobalDataHandler().setBoolean(s, "ForceUpdate", true); + getGlobalMessageProxyHandler().sendMessage(s, 1, VotingPluginWire.bungeeTimeChange()); + } + processQueue(); + } + + /** + * Load MySQL + global data handler. + */ + public void loadMysql(MysqlConfig mysqlConfig, MysqlConfig globalDataMysqlConfig) { + if (mysqlConfig.getHostName().isEmpty() || mysqlConfig.getDatabase().isEmpty()) { + logSevere("MySQL is not configured correctly. " + "Missing host/database. host=" + mysqlConfig.getHostName() + + " db=" + mysqlConfig.getDatabase()); + setProxyMySQL(null); + return; + } + + setProxyMySQL(new ProxyMysqlUserTable("VotingPlugin_Users", mysqlConfig, getConfig().getDebug()) { + + @Override + public void debug(SQLException e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void logSevere(String string) { + VotingPluginProxy.this.logSevere(string); + } + + @Override + public void logInfo(String string) { + VotingPluginProxy.this.logInfo(string); + } + + @Override + public void debug(Throwable t) { + if (getConfig().getDebug()) { + t.printStackTrace(); + } + } + + @Override + public void debug(String str) { + debug2(str); + } + }); + + ArrayList servers = new ArrayList(getAllAvailableServers()); + + if (getConfig().getGlobalDataEnabled()) { + if (getConfig().getGlobalDataUseMainMySQL()) { + setGlobalDataHandler(new GlobalDataHandlerProxy( + new GlobalMySQL("VotingPlugin_GlobalData", getProxyMySQL().getMysql()) { + + @Override + public void debugEx(Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void debugLog(String text) { + debug(text); + } + + @Override + public void info(String text) { + logInfo(text); + } + + @Override + public void logSevere(String text) { + VotingPluginProxy.this.logSevere(text); + } + + @Override + public void warning(String text) { + warn(text); + } + }, servers) { + + @Override + public void onTimeChangedFailed(String srv, TimeType type) { + VotingPluginProxy.this.onTimeChangedFailed(srv, type); + } + + @Override + public void onTimeChangedFinished(TimeType type) { + VotingPluginProxy.this.onTimeChangedFinished(type); + } + }); + } else { + setGlobalDataHandler( + new GlobalDataHandlerProxy(new GlobalMySQL("VotingPlugin_GlobalData", globalDataMysqlConfig) { + + @Override + public void debugEx(Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void debugLog(String text) { + debug(text); + } + + @Override + public void info(String text) { + logInfo(text); + } + + @Override + public void logSevere(String text) { + VotingPluginProxy.this.logSevere(text); + } + + @Override + public void warning(String text) { + warn(text); + } + }, servers) { + + @Override + public void onTimeChangedFailed(String srv, TimeType type) { + VotingPluginProxy.this.onTimeChangedFailed(srv, type); + } + + @Override + public void onTimeChangedFinished(TimeType type) { + VotingPluginProxy.this.onTimeChangedFinished(type); + } + }); + } + + // update global schema columns (unchanged from original) + getGlobalDataHandler().getGlobalMysql().alterColumnType("IgnoreTime", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("MONTH", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("WEEK", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("DAY", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("FinishedProcessing", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("Processing", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("ForceUpdate", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("LastUpdated", "MEDIUMTEXT"); + } + + // column types (unchanged from original) + getProxyMySQL().alterColumnType("TopVoterIgnore", "VARCHAR(5)"); + getProxyMySQL().alterColumnType("CheckWorld", "VARCHAR(5)"); + getProxyMySQL().alterColumnType("Reminded", "VARCHAR(5)"); + getProxyMySQL().alterColumnType("DisableBroadcast", "VARCHAR(5)"); + getProxyMySQL().alterColumnType("LastOnline", "VARCHAR(20)"); + getProxyMySQL().alterColumnType("PlayerName", "VARCHAR(30)"); + getProxyMySQL().alterColumnType("DailyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("WeeklyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("DayVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("BestDayVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("WeekVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("BestWeekVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("VotePartyVotes", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("MonthVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("Points", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("HighestDailyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("AllTimeTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("HighestMonthlyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("MonthTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("HighestWeeklyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("LastMonthTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("LastWeeklyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("LastDailyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("OfflineRewards", "MEDIUMTEXT"); + getProxyMySQL().alterColumnType("DayVoteStreakLastUpdate", "MEDIUMTEXT"); + + if (getConfig().getStoreMonthTotalsWithDate()) { + getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now()), "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now().plusMonths(1)), + "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now().plusMonths(2)), + "INT DEFAULT '0'"); + } + } + + public void addCurrentVotePartyVotes(int amount) { + votePartyVotes += amount; + setVoteCacheVotePartyCurrentVotes(votePartyVotes); + debug("Current vote party total: " + votePartyVotes); + } + + public void addNonVotedPlayer(String uuid, String playerName) { + nonVotedPlayersCache.addPlayer(uuid, playerName); + } + + public void addVoteParty() { + if (getConfig().getVotePartyEnabled()) { + addCurrentVotePartyVotes(1); + checkVoteParty(); + } + } + + public abstract void broadcast(String message); + + private Set sendProxyBroadcast(Set targets, String uuid, String player, String service, long time, + String text, boolean wasOnline) { + Set forwarded = new LinkedHashSet<>(); + for (String targetServer : targets) { + JsonEnvelope envelope = VotingPluginWire.voteBroadcast(uuid, player, service, time, text, wasOnline); + if (sendProxyBroadcastEnvelopeNow(targetServer, envelope)) { + forwarded.add(targetServer); + } + } + return forwarded; + } + + /** + * Sends a standalone proxy broadcast through the selected transport and reports + * whether that transport accepted the message. + * + * @param server target backend server + * @param envelope standalone broadcast envelope + * @return true only when the transport accepted the message + */ + protected boolean sendProxyBroadcastEnvelopeNow(String server, JsonEnvelope envelope) { + switch (method) { + case MQTT: + return sendMqttEnvelopeServer(server, envelope); + case MYSQL: + if (proxyMysqlMessenger == null) { + return false; + } + try { + proxyMysqlMessenger.sendToBackend(server, envelope); + return true; + } catch (SQLException e) { + debug(e.getMessage()); + return false; + } + case PLUGINMESSAGING: + return sendPluginMessageServerNow(server, envelope); + case REDIS: + return sendRedisEnvelopeServer(server, envelope, true); + case SOCKETS: + // Standalone broadcasts use the same initialized client as normal + // envelopes. This preserves the socket connection and its delivery + // acknowledgement instead of creating a second short-lived socket. + return sendSocketEnvelope(server, envelope); + case HTTP: + return sendHttpEnvelope(server, envelope); + default: + return false; + } + } + + /** + * Sends a reward-bearing vote envelope and reports whether the selected + * transport accepted it. Legacy transports retain their existing asynchronous + * semantics; HTTP exposes its bounded-queue result so a vote is never discarded + * when the queue is full. + */ + protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope) { + if (method == BungeeMethod.HTTP) { + return sendHttpEnvelope(server, envelope); + } + GlobalMessageProxyHandler handler = globalMessageProxyHandler; + if (handler == null) { + return false; + } + handler.sendMessage(server, delay, envelope); + return true; + } + + public synchronized void checkCachedVotes(String server) { + int delay = 1; + if (isServerValid(server)) { + if (isSomeoneOnlineServerForVoteRouting(server)) { + if (getVoteCacheHandler().hasVotes(server) && !getConfig().getBlockedServers().contains(server)) { + ArrayList c = getVoteCacheHandler().getVotes(server); + ArrayList removed = new ArrayList<>(); + if (!c.isEmpty()) { + int num = 1; + int numberOfVotes = c.size(); + for (OfflineBungeeVote cache : c) { + if (cache.isDeliveryStateDirty() && !persistServerVoteDelivery(server, cache)) { + continue; + } + if (cache.isProxyBroadcastHandled() && cache.needsBroadcastOn(server)) { + Set forwarded = sendProxyBroadcast(Collections.singleton(server), + cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), + cache.getText(), false); + if (cache.getBroadcastForwardedServers().addAll(forwarded)) { + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + if (!persistServerVoteDelivery(server, cache)) { + continue; + } + } + } + + boolean toSend = true; + if (getConfig().getWaitForUserOnline()) { + if (!isPlayerOnlineForVoteRouting(cache.getPlayerName())) { + toSend = false; + } else if (isPlayerOnlineForVoteRouting(cache.getPlayerName()) + && !getCurrentPlayerServerForVoteRouting(cache.getPlayerName()).equals(server)) { + toSend = false; + } + } + if (toSend) { + boolean broadcastHere = cache.needsBroadcastOn(server); + if (!cache.isProxyBroadcastHandled() && broadcastHere + && getConfig().getProxyBroadcastEnabled()) { + boolean playerOnline = isPlayerOnlineForVoteRouting(cache.getPlayerName()); + String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(cache.getPlayerName()) + : null; + + Set targets = proxyBroadcastDecider.resolveTargets(playerOnline, + playerServer); + broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); + } + + if (!sendVoteEnvelopeAccepted(server, delay, + VotingPluginWire.vote(cache.getPlayerName(), cache.getUuid(), + cache.getService(), cache.getTime(), false, cache.isRealVote(), + cache.getText(), cache.getVoteId(), getConfig().getBungeeManageTotals(), + broadcastHere, num, numberOfVotes))) { + debug("Retaining cached vote because the transport rejected delivery for " + server); + continue; + } + delay++; + num++; + removed.add(cache); + } else { + debug("Not sending vote because user isn't on server " + server + ": " + + cache.toString()); + } + } + getVoteCacheHandler().removeServerVotes(server, removed); + } else { + debug("No cached votes for server: " + server); + } + } else { + debug("No cached votes for server: " + server); + } + } + } else { + debug("Server not valid: " + server); + } + } + + public synchronized void checkOnlineVotes(String player, String uuid, String server) { + int delay = 1; + if (isPlayerOnlineForVoteRouting(player) && getVoteCacheHandler().hasOnlineVotes(uuid)) { + ArrayList c = getVoteCacheHandler().getOnlineVotes(uuid); + if (!c.isEmpty()) { + if (server == null) { + server = getCurrentPlayerServerForVoteRouting(player); + } + if (!getConfig().getBlockedServers().contains(server)) { + int num = 1; + int numberOfVotes = (int) c.stream().filter(vote -> !vote.isRewardDelivered()).count(); + boolean deliveredReward = false; + ArrayList retained = new ArrayList<>(); + for (OfflineBungeeVote cache : c) { + if (cache.isProxyBroadcastHandled()) { + Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); + pendingTargets.removeAll(cache.getBroadcastForwardedServers()); + List blockedServers = getConfig().getBlockedServers(); + if (blockedServers != null) { + pendingTargets.removeAll(blockedServers); + } + cache.getBroadcastForwardedServers().addAll(sendProxyBroadcast(pendingTargets, + cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), + cache.getText(), false)); + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + } + boolean broadcastHere = cache.needsBroadcastOn(server); + if (!cache.isProxyBroadcastHandled() && broadcastHere + && getConfig().getProxyBroadcastEnabled()) { + String playerServer = (server != null) ? server : getCurrentPlayerServerForVoteRouting(player); + + Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); + broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); + } + + if (!cache.isRewardDelivered()) { + if (!sendVoteEnvelopeAccepted(server, delay, + VotingPluginWire.voteOnline(cache.getPlayerName(), cache.getUuid(), cache.getService(), + cache.getTime(), false, cache.isRealVote(), cache.getText(), cache.getVoteId(), + getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes))) { + debug("Retaining online vote because the transport rejected delivery for " + server); + retained.add(cache); + continue; + } + // The normal envelope is also a valid broadcast delivery for the + // current target. Record it so a previously pending standalone + // retry cannot announce the same vote again later. + if (cache.isProxyBroadcastHandled() && broadcastHere) { + cache.getBroadcastForwardedServers().add(server); + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + } + cache.setRewardDelivered(true); + deliveredReward = true; + delay++; + num++; + } + + if (cache.isProxyBroadcastHandled() && !cache.isProxyBroadcastComplete()) { + retained.add(cache); + } + } + getVoteCacheHandler().removeOnlineVotes(uuid); + for (OfflineBungeeVote pending : retained) { + getVoteCacheHandler().addOnlineVote(uuid, pending); + } + + // multiproxy: envelope-only + if (deliveredReward && getConfig().getMultiProxySupport() + && getConfig().getMultiProxyOneGlobalReward()) { + multiProxyHandler.sendClearVote(uuid, player); + } + } + } + } + } + + /** + * Retries voter-keyed standalone broadcasts when any player makes a target + * backend available as a plugin-message carrier. + * + * @param server backend server that gained a carrier + */ + protected synchronized void retryPendingOnlineBroadcasts(String server) { + List blockedServers = getConfig().getBlockedServers(); + if (server == null || (blockedServers != null && blockedServers.contains(server))) { + return; + } + for (String cachedUuid : getVoteCacheHandler().getOnlineVoteUUIDs()) { + for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { + if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { + continue; + } + if (!cache.isProxyBroadcastHandled() || !cache.needsBroadcastOn(server)) { + continue; + } + Set forwarded = sendProxyBroadcast(Collections.singleton(server), cache.getUuid(), + cache.getPlayerName(), cache.getService(), cache.getTime(), cache.getText(), false); + if (cache.getBroadcastForwardedServers().addAll(forwarded)) { + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { + getVoteCacheHandler().removeOnlineVote(cachedUuid, cache); + } else { + persistOnlineVoteDelivery(cachedUuid, cache); + } + } + } + } + } + + protected synchronized void retryPendingTimeBroadcasts(String server) { + List blockedServers = getConfig().getBlockedServers(); + if (server == null || (blockedServers != null && blockedServers.contains(server))) { + return; + } + if (getVoteCacheHandler().getTimeChangeQueue() == null) { + return; + } + for (VoteTimeQueue vote : new ArrayList<>(getVoteCacheHandler().getTimeChangeQueue())) { + if (vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { + continue; + } + if (!vote.isProxyBroadcastHandled() || vote.getUuid().isEmpty() || !vote.getBroadcastTargets().contains(server) + || vote.getBroadcastForwardedServers().contains(server)) { + continue; + } + Set forwarded = sendProxyBroadcast(Collections.singleton(server), vote.getUuid(), vote.getName(), + vote.getService(), vote.getTime(), vote.getTotals(), false); + if (vote.getBroadcastForwardedServers().addAll(forwarded)) { + persistTimeVoteDelivery(vote); + } + } + } + + /** + * Periodically retries every pending voter-keyed standalone broadcast. This is + * required for broker transports whose recovery does not produce a player-login + * carrier event. + */ + public synchronized void retryPendingOnlineBroadcasts() { + for (String cachedUuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { + for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { + if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { + continue; + } + if (!cache.isProxyBroadcastHandled() || cache.isProxyBroadcastComplete()) { + continue; + } + Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); + pendingTargets.removeAll(cache.getBroadcastForwardedServers()); + List blockedServers = getConfig().getBlockedServers(); + if (blockedServers != null) { + pendingTargets.removeAll(blockedServers); + } + Set forwarded = sendProxyBroadcast(pendingTargets, cache.getUuid(), cache.getPlayerName(), + cache.getService(), cache.getTime(), cache.getText(), false); + if (cache.getBroadcastForwardedServers().addAll(forwarded)) { + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { + getVoteCacheHandler().removeOnlineVote(cachedUuid, cache); + } else { + persistOnlineVoteDelivery(cachedUuid, cache); + } + } + } + } + retryPendingTimeBroadcasts(); + } + + public synchronized void retryPendingTimeBroadcasts() { + if (getVoteCacheHandler().getTimeChangeQueue() == null) { + return; + } + for (VoteTimeQueue vote : new ArrayList<>(getVoteCacheHandler().getTimeChangeQueue())) { + if (vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { + continue; + } + if (!vote.isProxyBroadcastHandled() || vote.getUuid().isEmpty()) { + continue; + } + Set pendingTargets = new LinkedHashSet<>(vote.getBroadcastTargets()); + pendingTargets.removeAll(vote.getBroadcastForwardedServers()); + List blockedServers = getConfig().getBlockedServers(); + if (blockedServers != null) { + pendingTargets.removeAll(blockedServers); + } + Set forwarded = sendProxyBroadcast(pendingTargets, vote.getUuid(), vote.getName(), vote.getService(), + vote.getTime(), vote.getTotals(), false); + if (vote.getBroadcastForwardedServers().addAll(forwarded)) { + persistTimeVoteDelivery(vote); + } + } + } + + protected synchronized boolean persistTimeVoteDelivery(VoteTimeQueue vote) { + if (getVoteCacheHandler().updateTimeVote(vote)) { + vote.setDeliveryStateDirty(false); + return true; + } + vote.setDeliveryStateDirty(true); + scheduleTimeVoteDeliveryRetry(); + return false; + } + + private void scheduleTimeVoteDeliveryRetry() { + if (timeVoteDeliveryRetryScheduled || getScheduler() == null) { + return; + } + timeVoteDeliveryRetryScheduled = true; + try { + getScheduler().schedule(() -> { + synchronized (VotingPluginProxy.this) { + timeVoteDeliveryRetryScheduled = false; + } + retryPendingTimeBroadcasts(); + }, 5, TimeUnit.SECONDS); + } catch (RuntimeException e) { + timeVoteDeliveryRetryScheduled = false; + debug("Unable to schedule timed broadcast state retry: " + e.getMessage()); + } + } + + protected synchronized boolean persistServerVoteDelivery(String server, OfflineBungeeVote vote) { + if (getVoteCacheHandler().updateServerVote(server, vote)) { + vote.setDeliveryStateDirty(false); + return true; + } + vote.setDeliveryStateDirty(true); + scheduleCachedVoteDeliveryRetry(); + return false; + } + + protected synchronized boolean persistOnlineVoteDelivery(String uuid, OfflineBungeeVote vote) { + if (getVoteCacheHandler().updateOnlineVote(uuid, vote)) { + vote.setDeliveryStateDirty(false); + return true; + } + vote.setDeliveryStateDirty(true); + scheduleCachedVoteDeliveryRetry(); + return false; + } + + private void scheduleCachedVoteDeliveryRetry() { + if (cachedVoteDeliveryRetryScheduled || getScheduler() == null) { + return; + } + cachedVoteDeliveryRetryScheduled = true; + try { + getScheduler().schedule(() -> { + synchronized (VotingPluginProxy.this) { + cachedVoteDeliveryRetryScheduled = false; + } + retryCachedVoteDeliveryPersistence(); + }, 5, TimeUnit.SECONDS); + } catch (RuntimeException e) { + cachedVoteDeliveryRetryScheduled = false; + debug("Unable to schedule cached broadcast state retry: " + e.getMessage()); + } + } + + private synchronized void retryCachedVoteDeliveryPersistence() { + for (String server : getVoteCacheHandler().getCachedVotesServers()) { + for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getVotes(server))) { + if (vote.isDeliveryStateDirty()) { + persistServerVoteDelivery(server, vote); + } + } + } + for (String uuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { + for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(uuid))) { + if (vote.isDeliveryStateDirty()) { + persistOnlineVoteDelivery(uuid, vote); + } + } + } + } + + public void checkVoteParty() { + if (getConfig().getVotePartyEnabled()) { + if (votePartyVotes >= currentVotePartyVotesRequired) { + debug("Vote party reached"); + addCurrentVotePartyVotes(-currentVotePartyVotesRequired); + + currentVotePartyVotesRequired += getConfig().getVotePartyIncreaseVotesRequired(); + setVoteCacheVotePartyIncreaseVotesRequired( + getVoteCacheVotePartyIncreaseVotesRequired() + getConfig().getVotePartyIncreaseVotesRequired()); + + if (!getConfig().getVotePartyBroadcast().isEmpty()) { + broadcast(getConfig().getVotePartyBroadcast()); + } + + for (String command : getConfig().getVotePartyBungeeCommands()) { + runConsoleCommand(command); + } + + if (getConfig().getVotePartySendToAllServers()) { + for (String server : getAllAvailableServers()) { + sendVoteParty(server); + } + } else { + for (String server : getConfig().getVotePartyServersToSend()) { + sendVoteParty(server); + } + } + } + saveVoteCacheFile(); + } + } + + public abstract void debug(String str); + + private void debug2(String message) { + debug(message); + } + + /** + * HTTP client used for Mojang API requests. + */ + private final HttpClient httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); + + /** + * Fetches a player's UUID from the Mojang API. + * + * @param playerName player name + * @return player UUID, or {@code null} if not found + * @throws IOException if the request fails + * @throws InterruptedException if interrupted while waiting for the response + */ + public UUID fetchUUID(String playerName) throws IOException, InterruptedException { + if (playerName == null || playerName.equalsIgnoreCase("null")) { + return null; + } + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create("https://api.mojang.com/users/profiles/minecraft/" + playerName)).GET() + .timeout(Duration.ofSeconds(5)).build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() == 400 || response.statusCode() == 404) { + log("There is no player with the name \"" + playerName + "\"!"); + return null; + } + + if (response.statusCode() < 200 || response.statusCode() >= 300) { + throw new IOException("Failed to fetch UUID for " + playerName + ", HTTP " + response.statusCode()); + } + + JsonElement element = JsonParser.parseString(response.body()); + if (element == null || !element.isJsonObject()) { + return null; + } + + JsonObject object = element.getAsJsonObject(); + if (!object.has("id") || object.get("id").isJsonNull()) { + return null; + } + + String uuidAsString = object.get("id").getAsString(); + return parseUUIDFromString(uuidAsString); + } + + public abstract Set getAllAvailableServers(); + + /** Complete platform server set before whitelist/blocked routing filters. */ + public abstract Set getAllConfiguredServers(); + + public abstract VotingPluginProxyConfig getConfig(); + + public abstract String getCurrentPlayerServer(String player); + + /** + * Resolves a player's server for vote routing. A dedicated voting proxy has no + * local players, so it uses the backend presence tracker instead. + */ + protected String getCurrentPlayerServerForVoteRouting(String player) { + if (isDedicatedVotingProxyEnabled()) { + return backendPlayerPresenceTracker.getPlayer(player).map(presence -> presence.getServer()).orElse(null); + } + return getCurrentPlayerServer(player); + } + + /** + * Dedicated routing is intentionally unavailable on plugin messaging: that + * transport is attached to a player-facing proxy and does not carry backend + * presence snapshots. + */ + protected boolean isDedicatedVotingProxyEnabled() { + return getConfig().getDedicatedVotingProxy() && method != null && method.supportsBackendPresence(); + } + + public abstract File getDataFolderPlugin(); + + public String getMonthTotalsWithDatePath() { + LocalDateTime cTime = getBungeeTimeChecker().getTime(); + return getMonthTotalsWithDatePath(cTime); + } + + public String getMonthTotalsWithDatePath(LocalDateTime cTime) { + return "MonthTotal-" + cTime.getMonth().toString() + "-" + cTime.getYear(); + } + + public abstract String getProperName(String uuid, String playerName); + + public abstract String getUUID(String playerName); + + private int getValue(ArrayList cols, String column, int toAdd) { + for (Column d : cols) { + if (d.getName().equalsIgnoreCase(column)) { + DataValue value = d.getValue(); + int num = 0; + if (value == null) { + return toAdd; + } + if (value.isInt()) { + num = value.getInt(); + } else if (value.isString()) { + try { + num = Integer.parseInt(value.getString()); + } catch (Exception e) { + // ignore + } + } + return num + toAdd; + } + } + return toAdd; + } + + private VoteTotalsSnapshot getProjectedRolloverTotals(ArrayList data, String player) { + List timeChanges = getGlobalDataHandler().getTimeChanges(); + boolean resetMonth = timeChanges.contains(TimeType.MONTH); + boolean resetWeek = timeChanges.contains(TimeType.WEEK); + boolean resetDay = timeChanges.contains(TimeType.DAY); + int acceptedQueuedVotes = 0; + int acceptedGlobalQueuedVotes = 0; + for (VoteTimeQueue queued : getVoteCacheHandler().getTimeChangeQueue()) { + if (!queued.isProcessed()) { + acceptedGlobalQueuedVotes++; + } + if (!queued.isProcessed() && queued.getName() != null && queued.getName().equalsIgnoreCase(player)) { + acceptedQueuedVotes++; + } + } + int voteIncrement = acceptedQueuedVotes + 1; + + int allTimeTotal = getValue(data, "AllTimeTotal", voteIncrement); + int monthTotal = resetMonth ? voteIncrement : getValue(data, "MonthTotal", voteIncrement); + int weeklyTotal = resetWeek ? voteIncrement : getValue(data, "WeeklyTotal", voteIncrement); + int dailyTotal = resetDay ? voteIncrement : getValue(data, "DailyTotal", voteIncrement); + int points = getValue(data, "Points", voteIncrement * getConfig().getPointsOnVote()); + + int maxVotes = getConfig().getMaxAmountOfVotesPerDay(); + if (maxVotes > 0) { + int days = getBungeeTimeChecker().getTime().getDayOfMonth(); + if (monthTotal > days * maxVotes) { + monthTotal = days * maxVotes; + } + } + if (getConfig().getLimitVotePoints() > 0 && points > getConfig().getLimitVotePoints()) { + points = getConfig().getLimitVotePoints(); + } + + int dateMonthTotal = -1; + if (getConfig().getStoreMonthTotalsWithDate()) { + if (getConfig().getUseMonthDateTotalsAsPrimaryTotal()) { + dateMonthTotal = resetMonth ? voteIncrement + : getValue(data, getMonthTotalsWithDatePath(), voteIncrement); + } else { + dateMonthTotal = monthTotal; + } + } + + int[] projectedVoteParty = getProjectedVotePartyState(acceptedGlobalQueuedVotes + 1); + return new VoteTotalsSnapshot(allTimeTotal, monthTotal, weeklyTotal, dailyTotal, points, + projectedVoteParty[0], projectedVoteParty[1], dateMonthTotal); + } + + protected boolean canForwardStandaloneBroadcast(boolean managesTotals) { + return managesTotals; + } + + protected int[] getProjectedVotePartyState(int acceptedVotes) { + int current = votePartyVotes; + int required = currentVotePartyVotesRequired; + if (!getConfig().getVotePartyEnabled()) { + return new int[] { current, required }; + } + + int increase = getConfig().getVotePartyIncreaseVotesRequired(); + for (int i = 0; i < acceptedVotes; i++) { + current++; + if (current >= required) { + current -= required; + required += increase; + } + } + return new int[] { current, required }; + } + + public abstract String getPluginVersion(); + + public abstract int getVoteCacheCurrentVotePartyVotes(); + + public abstract long getVoteCacheLastUpdated(); + + public abstract int getVoteCachePrevDay(); + + public abstract String getVoteCachePrevMonth(); + + public abstract int getVoteCachePrevWeek(); + + public abstract int getVoteCacheVotePartyIncreaseVotesRequired(); + + public abstract boolean isPlayerOnline(String playerName); + + /** + * Checks online state for vote routing, using backend presence only when this + * proxy is explicitly configured as the dedicated voting proxy. + */ + protected boolean isPlayerOnlineForVoteRouting(String playerName) { + return isDedicatedVotingProxyEnabled() ? backendPlayerPresenceTracker.getPlayer(playerName).isPresent() + : isPlayerOnline(playerName); + } + + public abstract boolean isServerValid(String server); + + public abstract boolean isSomeoneOnlineServer(String server); + + protected boolean isSomeoneOnlineServerForVoteRouting(String server) { + if (!isDedicatedVotingProxyEnabled()) { + return isSomeoneOnlineServer(server); + } + com.bencodez.votingplugin.proxy.presence.BackendPresenceStatus status = backendPlayerPresenceTracker + .getBackendStatus(server); + return status != null && status.isAvailable() && status.getPlayerCount() > 0; + } + + public abstract boolean isVoteCacheIgnoreTime(); + + public abstract MysqlConfig getVoteCacheMySQLConfig(); + + public abstract MysqlConfig getNonVotedCacheMySQLConfig(); + + public abstract MysqlConfig getVoteLoggingMySQLConfig(); + + /** + * Shutdown MySQL-related resources safely. + */ + public void shutdownMySql() { + if (getProxyMysqlMessenger() != null) { + getProxyMysqlMessenger().shutdown(); + setProxyMysqlMessenger(null); + } + + if (getProxyMySQL() != null) { + getProxyMySQL().shutdown(); + setProxyMySQL(null); + } + } + + public void load(IVoteCache jsonStorage, INonVotedPlayersStorage nonVotedCacheJson) { + method = BungeeMethod.getByName(getConfig().getBungeeMethod()); + if (getMethod() == null) { + method = BungeeMethod.PLUGINMESSAGING; + } + warnUnsupportedDedicatedVotingProxyMode(); + uuidPlayerNameCache = getProxyMySQL().getRowsUUIDNameQuery(); + + bungeeTimeChecker.setTimeChangeFailSafeBypass(getConfig().getTimeChangeFailSafeBypass()); + bungeeTimeChecker.loadTimer(); + + voteCacheHandler = new VoteCacheHandler(getVoteCacheMySQLConfig(), getConfig().getVoteCacheUseMySQL(), + getConfig().getVoteCacheUseMainMySQL(), getProxyMySQL().getMysql(), getConfig().getDebug(), + jsonStorage) { + + @Override + public void logInfo1(String msg) { + logInfo(msg); + } + + @Override + public void logSevere1(String msg) { + logSevere(msg); + } + + @Override + public void debug1(Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void debug1(String msg) { + if (getConfig().getDebug()) { + debug(msg); + } + } + + @Override + public void debug1(Throwable e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + }; + voteCacheHandler.load(); + + nonVotedPlayersCache = new NonVotedPlayersCache(getNonVotedCacheMySQLConfig(), + getConfig().getNonVotedCacheUseMySQL(), getConfig().getNonVotedCacheUseMainMySQL(), + getProxyMySQL().getMysql(), nonVotedCacheJson, getConfig().getDebug()) { + + @Override + public boolean userExists(String uuid) { + return getProxyMySQL().containsKeyQuery(uuid); + } + + @Override + public void logInfo1(String msg) { + logInfo(msg); + } + + @Override + public void logSevere1(String msg) { + logSevere(msg); + } + + @Override + public void debug1(Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void debug1(String msg) { + if (getConfig().getDebug()) { + debug(msg); + } + } + + @Override + public Set getAllUUIDs() { + return getProxyMySQL().getUuids(); + } + }; + + if (method.equals(BungeeMethod.MYSQL)) { + try { + proxyMysqlMessenger = new MySqlMessenger("VotingPlugin", + getProxyMySQL().getMysql().getConnectionManager().getDataSource(), MySqlMessenger.Mode.PROXY, + null, // no serverId in PROXY mode + msg -> { + if (getConfig().getDebug()) { + debug("Got from " + msg.source + ": " + msg.envelope.getSubChannel() + " " + + msg.envelope.getFields()); + } + globalMessageProxyHandler.onMessage(msg.envelope); + }); + } catch (SQLException e) { + e.printStackTrace(); + } + } else if (method.equals(BungeeMethod.PLUGINMESSAGING)) { + if (getConfig().getPluginMessageEncryption()) { + encryptionHandler = new EncryptionHandler("VotingPlugin", + new File(getDataFolderPlugin(), "secretkey.key")); + } + } else if (method.equals(BungeeMethod.SOCKETS)) { + encryptionHandler = new EncryptionHandler("VotingPlugin", new File(getDataFolderPlugin(), "secretkey.key")); + + socketHandler = new SocketHandler(getPluginVersion(), getConfig().getBungeeHost(), + getConfig().getBungeePort(), encryptionHandler, getConfig().getDebug()) { + + @Override + public void log(String str) { + logInfo(str); + } + }; + + socketHandler.add(new SocketReceiver() { + @Override + public void onReceiveEnvelope(JsonEnvelope envelope) { + globalMessageProxyHandler.onMessage(envelope); + } + }); + + rebuildSocketClients(); + } else if (method.equals(BungeeMethod.REDIS)) { + redisHandler = new RedisHandler(getConfig().getRedisHost(), getConfig().getRedisPort(), + getConfig().getRedisUsername(), getConfig().getRedisPassword(), getConfig().getRedisDbIndex(), + getConfig().getRedisSsl()) { + + @Override + public void debug(String message) { + debug2(message); + } + }; + redisPublisherPool = new JedisPool(new HostAndPort(getConfig().getRedisHost(), getConfig().getRedisPort()), + buildRedisClientConfig(getConfig())); + + runAsync(() -> { + RedisListener listener = redisHandler.createEnvelopeListener( + getConfig().getRedisPrefix() + "VotingPlugin", + (ch, env) -> globalMessageProxyHandler.onMessage(env)); + redisHandler.loadListener(listener); + }); + + } else if (method.equals(BungeeMethod.MQTT)) { + try { + mqttHandler = new MqttHandler(new MqttServerComm(getConfig().getMqttClientID(), + getConfig().getMqttBrokerURL(), getConfig().getMqttUsername(), getConfig().getMqttPassword()), + 2); + + mqttHandler.subscribeEnvelopes(getConfig().getMqttPrefix() + "votingplugin/servers/proxy", + (topic, env) -> globalMessageProxyHandler.onMessage(env)); + + } catch (MqttException e) { + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + currentVotePartyVotesRequired = getConfig().getVotePartyVotesRequired() + + getVoteCacheVotePartyIncreaseVotesRequired(); + votePartyVotes = getVoteCacheCurrentVotePartyVotes(); + + globalMessageProxyHandler = new GlobalMessageProxyHandler() { + @Override + public void sendMessage(String server, int delay, JsonEnvelope envelope) { + switch (method) { + case MQTT: + sendMqttEnvelopeServer(server, envelope); + break; + case MYSQL: + try { + proxyMysqlMessenger.sendToBackend(server, envelope); + } catch (SQLException e) { + e.printStackTrace(); + } + break; + case PLUGINMESSAGING: + sendPluginMessageServer(server, delay, envelope); + break; + case REDIS: + sendRedisEnvelopeServer(server, envelope); + break; + case SOCKETS: + sendSocketEnvelope(server, envelope); + break; + case HTTP: + sendHttpEnvelope(server, envelope); + break; + default: + break; + } + } + }; + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_LOGIN) { + @Override + public void onReceive(JsonEnvelope message) { + handleLoginMessage(message); + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_LOGOUT) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(message); + if (!isPresenceServerValid(event.server, VotingPluginWire.SUB_LOGOUT) + || !isPresenceGenerationValid(event.backendIncarnationId, event.backendStartedAt, + event.presenceTimestamp, + VotingPluginWire.SUB_LOGOUT)) { + return; + } + if (!backendPlayerPresenceTracker.playerOffline(event.uuid, event.server, event.connectionId, + event.backendIncarnationId, event.backendStartedAt, event.presenceTimestamp, + System.currentTimeMillis())) { + debug("Ignored invalid or stale logout envelope: " + message.getFields()); + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_STARTED) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); + long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); + long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); + if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_STARTED) + && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, + VotingPluginWire.SUB_BACKEND_STARTED)) { + if (backendPlayerPresenceTracker.backendStarted(server, backendIncarnationId, backendStartedAt, + presenceTimestamp, System.currentTimeMillis())) { + discardPendingPresenceHandoffs(server); + pendingBackendRecoverySnapshots.add(presenceServerKey(server)); + requestBackendPresenceSnapshot(server); + } + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_STOPPED) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); + long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); + long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); + if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_STOPPED) + && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, + VotingPluginWire.SUB_BACKEND_STOPPED)) { + if (backendPlayerPresenceTracker.backendStopped(server, backendIncarnationId, backendStartedAt, + presenceTimestamp, System.currentTimeMillis())) { + discardPendingPresenceHandoffs(server); + pendingBackendRecoverySnapshots.remove(presenceServerKey(server)); + } + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_HEARTBEAT) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); + long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); + long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); + if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_HEARTBEAT) + && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, + VotingPluginWire.SUB_BACKEND_HEARTBEAT)) { + backendPlayerPresenceTracker.heartbeat(server, backendIncarnationId, backendStartedAt, + presenceTimestamp, System.currentTimeMillis()); + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_PRESENCE_SNAPSHOT) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + if (!isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_SNAPSHOT)) { + return; + } + VotingPluginWire.PresenceSnapshot snapshot = VotingPluginWire.readPresenceSnapshot(message); + long now = System.currentTimeMillis(); + boolean accepted = snapshot.valid + && isPresenceGenerationValid(snapshot.backendIncarnationId, snapshot.backendStartedAt, + snapshot.presenceTimestamp, + VotingPluginWire.SUB_PRESENCE_SNAPSHOT) + && backendPlayerPresenceTracker.applySnapshotChunk(snapshot.server, + snapshot.requestId, snapshot.chunkIndex, snapshot.chunkCount, snapshot.players, + snapshot.backendIncarnationId, snapshot.backendStartedAt, + snapshot.presenceTimestamp, now); + if (!accepted) { + debug("Ignored invalid or unexpected presence snapshot from " + snapshot.server); + if (backendPlayerPresenceTracker.getPendingSnapshotRequestId(snapshot.server, now) == null) { + discardPendingPresenceHandoffs(snapshot.requestId); + } + } else if (backendPlayerPresenceTracker.getPendingSnapshotRequestId(snapshot.server, now) == null) { + pendingBackendRecoverySnapshots.remove(presenceServerKey(snapshot.server)); + Set handoffPlayers = completePendingPresenceHandoffs(snapshot.requestId, snapshot.server, + snapshot.backendIncarnationId, snapshot.backendStartedAt, now); + processDedicatedSnapshotLogins(snapshot.server, handoffPlayers); + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_STATUS_OKAY) { + @Override + public void onReceive(JsonEnvelope message) { + handleStatusOkay(message); + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener("voteupdate") { + @Override + public void onReceive(JsonEnvelope message) { + int delay = 1; + for (String send : getAllAvailableServers()) { + globalMessageProxyHandler.sendMessage(send, delay, message); + delay++; + } + } + }); + + // Do not accept backend traffic until the router and all of its ordered + // presence/message listeners are installed. + if (method.equals(BungeeMethod.HTTP)) startHttpTransport(); + + proxyBroadcastDecider = new ProxyBroadcastDecider(() -> getConfig(), () -> getAllAvailableServers(), + s -> isServerValid(s), + s -> getConfig().getBlockedServers() != null && getConfig().getBlockedServers().contains(s)); + + loadMultiProxySupport(); + loadVoteLoggingMySQL(); + if (method.supportsBackendPresence()) { + scheduleBackendPresenceStartupResync(); + loadTaskTimer(this::maintainBackendPresence, PRESENCE_MAINTENANCE_INTERVAL_SECONDS, + PRESENCE_MAINTENANCE_INTERVAL_SECONDS); + } + startControlServices(); + + debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); + } + + private void startControlServices() { + synchronized (controlLifecycleLock) { + ControlConnector predecessor = controlConnector; + if (predecessor != null && predecessor.deferReplacementUntilSafe(this::restartControlServicesAsync)) { + log("[Control] service restart deferred until the current result is acknowledged"); + return; + } + stopControlServicesLocked(true); + startControlServicesLocked(); + } + } + + /** Keeps potentially long hosted-Control handoffs off proxy command/event threads. */ + private void restartControlServicesAsync() { + final long generation = controlServicesGeneration.incrementAndGet(); + try { + controlLifecycleExecutor.execute(() -> { + try { + synchronized (controlLifecycleLock) { + if (!enabled || generation != controlServicesGeneration.get()) return; + ControlConnector predecessor = controlConnector; + if (predecessor != null + && predecessor.deferReplacementUntilSafe(this::restartControlServicesAsync)) { + log("[Control] service restart deferred until the current result is acknowledged"); + return; + } + stopControlServicesLocked(true); + startControlServicesLocked(); + } + } catch (RuntimeException failure) { + if (generation == controlServicesGeneration.get()) { + logSevere("[Control] asynchronous service restart failed: " + failure.getMessage()); + } + } + }); + } catch (RuntimeException failure) { + logSevere("[Control] services were not restarted because async scheduling failed"); + } + } + + /** Rebuilds a recovery connector from current settings after its durable result is acknowledged. */ + public final void restartControlServicesAfterRecovery() { + restartControlServicesAsync(); + } + + private void stopControlServices(boolean waitForHosted) { + synchronized (controlLifecycleLock) { + stopControlServicesLocked(waitForHosted); + } + } + + private void startControlServicesLocked() { + if (getConfig().getControlHostedEnabled()) { + try { + hostedControlManager = HostedControlManager.create(this); + if (hostedControlManager != null) hostedControlManager.start(); + } catch (IOException | IllegalArgumentException e) { + hostedControlManager = null; + logSevere("[Control Host] configuration or automatic enrollment is invalid; VotingPlugin remains unaffected"); + } + } + try { + controlConnector = ControlConnector.create(this); + if (controlConnector != null) controlConnector.start(); + } catch (IOException | IllegalArgumentException e) { + controlConnector = null; + logSevere("[Control] connector configuration or credential is invalid; voting remains unaffected"); + } + } + + private void stopControlServicesLocked(boolean waitForHosted) { + ControlConnector connector = controlConnector; + if (connector != null) { + try { + connector.close(); + if (controlConnector == connector) controlConnector = null; + } catch (RuntimeException failure) { + if (waitForHosted) throw failure; + if (controlConnector == connector) controlConnector = null; + logSevere("[Control] connector did not stop cleanly; proxy cleanup will continue"); + } + } + HostedControlManager manager = hostedControlManager; + if (manager != null) { + try { + if (waitForHosted) { + manager.closeAndWait(); + } else { + manager.close(); + } + if (hostedControlManager == manager) hostedControlManager = null; + } catch (RuntimeException failure) { + if (waitForHosted) throw failure; + if (hostedControlManager == manager) hostedControlManager = null; + logSevere("[Control Host] manager did not stop cleanly; proxy cleanup will continue"); + } + } + } + + public String getControlConnectorStatus() { + ControlConnector connector = controlConnector; + return connector == null ? "DISABLED" : connector.status().name(); + } + + public String getHostedControlStatus() { + HostedControlManager manager = hostedControlManager; + return manager == null ? "DISABLED" : manager.status().name(); + } + + /** + * Handles both the original login notification and extended presence logins. + * Kept protected so transport-policy behavior can be regression tested without + * initializing a live proxy transport. + * + * @param message login envelope + */ + protected void handleLoginMessage(JsonEnvelope message) { + VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(message); + String player = event.player; + String uuid = event.uuid; + String server = event.server; + + if (player.isEmpty() || uuid.isEmpty()) { + logSevere("Invalid login envelope received: " + message.getFields()); + return; + } + boolean legacy = event.connectionId == null && event.backendIncarnationId == null + && event.backendStartedAt == 0L && event.presenceTimestamp == 0L; + boolean accepted = false; + String deliveryServer = server; + if (legacy) { + if (method == BungeeMethod.PLUGINMESSAGING) { + String proxyServer = getCurrentPlayerServer(player); + accepted = isLegacyLoginDestinationAuthoritative(player, uuid, proxyServer); + if (accepted) { + deliveryServer = proxyServer; + } + } else if (method != null && method.supportsBackendPresence() + && isPresenceServerValid(server, VotingPluginWire.SUB_LOGIN)) { + accepted = isLegacyLoginDestinationAuthoritative(player, uuid, server); + } + } else if (method != null && method.supportsBackendPresence() && event.connectionId != null + && isPresenceServerValid(server, VotingPluginWire.SUB_LOGIN) + && isPresenceGenerationValid(event.backendIncarnationId, event.backendStartedAt, + event.presenceTimestamp, VotingPluginWire.SUB_LOGIN)) { + BackendPlayerPresenceTracker.PlayerOnlineResult result = backendPlayerPresenceTracker.playerOnlineResult( + player, uuid, server, event.connectionId, + event.backendIncarnationId, event.backendStartedAt, event.presenceTimestamp, + System.currentTimeMillis()); + accepted = result.isAccepted(); + if (result.isConflictingPresence()) { + requestBackendPresenceSnapshot(server, + new PendingPresenceHandoff(player, uuid, server, event.connectionId, + event.backendIncarnationId, event.backendStartedAt, + result.getConflictSequence(), System.currentTimeMillis())); + } + } + + debug("Login: " + player + "/" + uuid + " " + server); + if (accepted) { + discardPendingPresenceHandoff(uuid); + login(player, uuid, deliveryServer); + } else { + debug("Ignored invalid or stale login envelope: " + message.getFields()); + } + } + + /** + * Validates a legacy login against an authority independent of the envelope. + * Player-facing proxies use their native live route and UUID. A dedicated + * voting proxy has no native player session, so it requires an exact modern + * presence match for the claimed destination. + */ + private boolean isLegacyLoginDestinationAuthoritative(String player, String uuid, String server) { + if (server == null || server.isBlank()) { + return false; + } + + UUID claimedUuid; + try { + claimedUuid = UUID.fromString(uuid.trim()); + } catch (RuntimeException e) { + return false; + } + + if (isDedicatedVotingProxyEnabled()) { + PlayerPresence presence = backendPlayerPresenceTracker.getPlayer(player).orElse(null); + return presence != null && presence.getServer().equalsIgnoreCase(server) + && (!getConfig().getOnlineMode() || presence.getUuid().equals(claimedUuid)); + } + + if (!isPlayerOnline(player)) { + return false; + } + String proxyServer = getCurrentPlayerServer(player); + if (proxyServer == null || !proxyServer.equalsIgnoreCase(server)) { + return false; + } + if (!getConfig().getOnlineMode()) { + return true; + } + + String authoritativeUuid = getUUID(player); + if (authoritativeUuid == null || authoritativeUuid.isBlank()) { + return false; + } + try { + return claimedUuid.equals(UUID.fromString(authoritativeUuid.trim())); + } catch (IllegalArgumentException e) { + return false; + } + } + + private VoteLogMysqlTable voteLogMysqlTable; + + @Getter + private ProxyBroadcastDecider proxyBroadcastDecider; + + public void loadVoteLoggingMySQL() { + if (getConfig().getVoteLoggingEnabled()) { + if (getConfig().getVoteLoggingUseMainMySQL()) { + voteLogMysqlTable = new VoteLogMysqlTable("votingplugin_votelog", getProxyMySQL().getMysql(), + getVoteLoggingMySQLConfig(), getConfig().getDebug()) { + + @Override + public void logSevere(String string) { + VotingPluginProxy.this.logSevere(string); + } + + @Override + public void logInfo(String string) { + VotingPluginProxy.this.logInfo(string); + } + + @Override + public void debug(Throwable e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public String getServerName() { + return "Proxy"; + } + }; + } else { + voteLogMysqlTable = new VoteLogMysqlTable("votingplugin_votelog", getVoteLoggingMySQLConfig(), + getConfig().getDebug()) { + + @Override + public void logSevere(String string) { + VotingPluginProxy.this.logSevere(string); + } + + @Override + public void logInfo(String string) { + VotingPluginProxy.this.logInfo(string); + } + + @Override + public void debug(Throwable e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public String getServerName() { + return "Proxy"; + } + }; + } + + if (getConfig().getVoteLoggingPurgeDays() > 0) { + loadTaskTimer(() -> voteLogMysqlTable.purgeOlderThanDays(getConfig().getVoteLoggingPurgeDays(), 100), + 60, 60 * 60); + } + + debug("Vote logging MySQL enabled"); + } else { + debug("Vote logging MySQL disabled"); + } + } + + public abstract void loadTaskTimer(Runnable runnable, long delaySeconds, long repeatSeconds); + + public void loadMultiProxySupport() { + if (multiProxyHandler != null) { + multiProxyHandler.close(); + } + multiProxyHandler = new MultiProxyHandler() { + + @Override + public void addNonVotedPlayerCache(String uuid, String player) { + addNonVotedPlayer(uuid, player); + } + + @Override + public void clearVote(String uuid) { + getVoteCacheHandler().clearOnlineVoteRewards(uuid); + } + + @Override + public boolean getDebug() { + return getConfig().getDebug(); + } + + @Override + public EncryptionHandler getEncryptionHandler() { + return encryptionHandler; + } + + @Override + public MultiProxyMethod getMultiProxyMethod() { + return MultiProxyMethod.getByName(getConfig().getMultiProxyMethod()); + } + + @Override + public String getMultiProxyPassword() { + return getConfig().getMultiProxyRedisPassword(); + } + + @Override + public String getMultiProxyRedisHost() { + return getConfig().getMultiProxyRedisHost(); + } + + @Override + public int getMultiProxyRedisPort() { + return getConfig().getMultiProxyRedisPort(); + } + + @Override + public boolean getMultiProxyRedisSsl() { + return getConfig().getMultiProxyRedisSsl(); + } + + @Override + public int getMultiProxyRedisDbIndex() { + return getConfig().getMultiProxyRedisDbIndex(); + } + + @Override + public boolean getMultiProxyRedisUseExistingConnection() { + return getConfig().getMultiProxyRedisUseExistingConnection(); + } + + @Override + public String getMultiProxyServerName() { + return getConfig().getProxyServerName(); + } + + @Override + public Collection getMultiProxyServers() { + return getConfig().getMultiProxyServers(); + } + + @Override + public MultiProxyServerSocketConfiguration getMultiProxyServersConfiguration(String s) { + return new MultiProxyServerSocketConfigurationBungee(s, + getConfig().getMultiProxyServersConfiguration(s)); + } + + @Override + public String getMultiProxySocketHostHost() { + return getConfig().getMultiProxySocketHostHost(); + } + + @Override + public int getMultiProxySocketHostPort() { + return getConfig().getMultiProxySocketHostPort(); + } + + @Override + public boolean getMultiProxySupportEnabled() { + return getConfig().getMultiProxySupport(); + } + + @Override + public String getMultiProxyUsername() { + return getConfig().getMultiProxyRedisUsername(); + } + + @Override + public File getPluginDataFolder() { + return getDataFolderPlugin(); + } + + @Override + public boolean getPrimaryServer() { + return getConfig().getPrimaryServer(); + } + + @Override + public List getProxyServers() { + return getConfig().getProxyServers(); + } + + @Override + public RedisHandler getRedisHandler() { + return redisHandler; + } + + @Override + public String getVersion() { + return getPluginVersion(); + } + + @Override + public void logInfo(String msg) { + log(msg); + } + + @Override + public void runAsnc(Runnable runnable) { + runAsync(runnable); + } + + @Override + public void setEncryptionHandler(EncryptionHandler encryptionHandler1) { + encryptionHandler = encryptionHandler1; + } + + @Override + public void triggerVote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid) { + vote(player, service, realVote, timeQueue, queueTime, text, uuid); + } + }; + multiProxyHandler.loadMultiProxySupport(); + } + + public abstract void log(String message); + + /** + * Requests a complete player-presence snapshot from one backend server. + * + * @param server configured backend server name + * @return new or already-active request identifier, or null when the server is + * invalid or is inside the snapshot-request cooldown + */ + public UUID requestBackendPresenceSnapshot(String server) { + return requestBackendPresenceSnapshot(server, null); + } + + private UUID requestBackendPresenceSnapshot(String server, PendingPresenceHandoff handoff) { + return requestBackendPresenceSnapshot(server, handoff, System.currentTimeMillis(), false); + } + + private UUID requestBackendPresenceSnapshot(String server, PendingPresenceHandoff handoff, long now, + boolean handoffAlreadyQueued) { + if (method == null || !method.supportsBackendPresence() || globalMessageProxyHandler == null + || !isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_SNAPSHOT_REQUEST)) { + return null; + } + long backendStartedAt = backendPlayerPresenceTracker.getBackendStartedAt(server); + UUID backendIncarnationId = backendPlayerPresenceTracker.getBackendIncarnationId(server); + if (backendStartedAt <= 0L || backendIncarnationId == null) { + return null; + } + if (handoff != null && (!server.equalsIgnoreCase(handoff.server) + || !backendIncarnationId.equals(handoff.backendIncarnationId) + || backendStartedAt != handoff.backendStartedAt)) { + return null; + } + if (handoff != null && (handoffAlreadyQueued ? !isPendingPresenceHandoff(handoff, now) + : !queuePendingPresenceHandoff(handoff, now))) { + return null; + } + UUID requestId = handoff == null + ? backendPlayerPresenceTracker.beginSnapshot(server, UUID.randomUUID(), backendIncarnationId, + backendStartedAt, now) + : backendPlayerPresenceTracker.beginSnapshotForDestinationClaim(server, UUID.randomUUID(), + backendIncarnationId, backendStartedAt, handoff.playerUuid, handoff.conflictSequence, now); + boolean created = requestId != null; + if (!created) { + requestId = handoff == null ? backendPlayerPresenceTracker.getPendingSnapshotRequestId(server, now) + : backendPlayerPresenceTracker.getPendingSnapshotRequestIdForDestinationClaim(server, + handoff.playerUuid, handoff.conflictSequence, now); + } + if (requestId == null) { + if (handoff != null && !backendPlayerPresenceTracker.isCurrentDestinationClaim(handoff.playerUuid, + handoff.server, handoff.conflictSequence)) { + discardPendingPresenceHandoff(handoff); + } + // A handoff stays unassigned while the destination is inside its snapshot + // cooldown. Presence maintenance will attach it to the next allowed snapshot. + return null; + } + if (handoff != null) { + assignPendingPresenceHandoff(handoff, requestId, now); + } + if (created) { + JsonEnvelope request = VotingPluginWire.presenceSnapshotRequest(server, requestId, backendIncarnationId, + backendStartedAt, now); + globalMessageProxyHandler.sendMessage(server, 1, request); + } + return requestId; + } + + private boolean queuePendingPresenceHandoff(PendingPresenceHandoff handoff, long now) { + if (!isPresenceHandoffValid(handoff, now)) { + return false; + } + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + PendingPresenceHandoff current = pendingPresenceHandoffs.get(handoff.playerUuid); + if (current != null && current.conflictSequence > handoff.conflictSequence) { + return false; + } + handoff.requestId = null; + pendingPresenceHandoffs.put(handoff.playerUuid, handoff); + return true; + } + } + + private boolean isPendingPresenceHandoff(PendingPresenceHandoff handoff, long now) { + if (!isPresenceHandoffValid(handoff, now)) { + return false; + } + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + return pendingPresenceHandoffs.get(handoff.playerUuid) == handoff; + } + } + + private void assignPendingPresenceHandoff(PendingPresenceHandoff handoff, UUID requestId, long now) { + if (requestId == null || !isPresenceHandoffValid(handoff, now)) { + return; + } + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + if (pendingPresenceHandoffs.get(handoff.playerUuid) == handoff) { + handoff.requestId = requestId; + } + } + } + + private boolean isPresenceHandoffValid(PendingPresenceHandoff handoff, long now) { + return handoff != null && handoff.playerUuid != null && handoff.connectionId != null + && handoff.conflictSequence > 0L + && now >= handoff.createdAt && now - handoff.createdAt <= PRESENCE_HANDOFF_TIMEOUT_MILLIS; + } + + private Set completePendingPresenceHandoffs(UUID requestId, String server, UUID backendIncarnationId, + long backendStartedAt, long now) { + List completed = new ArrayList<>(); + Set completedPlayers = new LinkedHashSet<>(); + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + pendingPresenceHandoffs.entrySet().removeIf(entry -> { + PendingPresenceHandoff handoff = entry.getValue(); + if (!requestId.equals(handoff.requestId)) { + return false; + } + if (handoff.server.equalsIgnoreCase(server) + && handoff.backendIncarnationId.equals(backendIncarnationId) + && handoff.backendStartedAt == backendStartedAt) { + completed.add(handoff); + } + return true; + }); + } + for (PendingPresenceHandoff handoff : completed) { + PlayerPresence presence = backendPlayerPresenceTracker.getPlayer(handoff.playerUuid).orElse(null); + if (presence != null && presence.getServer().equalsIgnoreCase(handoff.server) + && presence.getConnectionId().equals(handoff.connectionId)) { + login(handoff.playerName, handoff.uuid, handoff.server); + completedPlayers.add(handoff.playerUuid); + } + releaseDestinationClaim(handoff); + } + return completedPlayers; + } + + /** + * Drains voter-keyed cached rewards when a complete recovery snapshot first + * confirms a player on a dedicated voting proxy. Cross-backend handoffs are + * already processed by their token-bound completion path and are excluded to + * avoid a second login callback. + */ + protected void processDedicatedSnapshotLogins(String server, Set handoffPlayers) { + if (!isDedicatedVotingProxyEnabled() || server == null || server.isBlank()) { + return; + } + Set excluded = handoffPlayers == null ? Collections.emptySet() : handoffPlayers; + for (PlayerPresence presence : backendPlayerPresenceTracker.getOnlinePlayers()) { + if (presence.getServer().equalsIgnoreCase(server) && !excluded.contains(presence.getUuid())) { + login(presence.getPlayerName(), presence.getUuid().toString(), presence.getServer()); + } + } + } + + private void discardPendingPresenceHandoff(String uuid) { + try { + UUID playerUuid = UUID.fromString(uuid.trim()); + PendingPresenceHandoff removed; + synchronized (pendingPresenceHandoffs) { + removed = pendingPresenceHandoffs.remove(playerUuid); + } + releaseDestinationClaim(removed); + } catch (Exception ignored) { + // Invalid identities are rejected by the presence tracker. + } + } + + private void discardPendingPresenceHandoff(PendingPresenceHandoff handoff) { + boolean removed = false; + synchronized (pendingPresenceHandoffs) { + if (handoff != null && pendingPresenceHandoffs.get(handoff.playerUuid) == handoff) { + pendingPresenceHandoffs.remove(handoff.playerUuid); + removed = true; + } + } + if (removed) { + releaseDestinationClaim(handoff); + } + } + + private void discardPendingPresenceHandoffs(String server) { + synchronized (pendingPresenceHandoffs) { + pendingPresenceHandoffs.entrySet().removeIf(entry -> { + if (!entry.getValue().server.equalsIgnoreCase(server)) { + return false; + } + releaseDestinationClaim(entry.getValue()); + return true; + }); + } + } + + private void discardPendingPresenceHandoffs(UUID requestId) { + if (requestId == null) { + return; + } + synchronized (pendingPresenceHandoffs) { + pendingPresenceHandoffs.entrySet().removeIf(entry -> { + if (!requestId.equals(entry.getValue().requestId)) { + return false; + } + releaseDestinationClaim(entry.getValue()); + return true; + }); + } + } + + private void prunePendingPresenceHandoffs(long now) { + pendingPresenceHandoffs.entrySet().removeIf(entry -> { + PendingPresenceHandoff handoff = entry.getValue(); + if (now >= handoff.createdAt && now - handoff.createdAt <= PRESENCE_HANDOFF_TIMEOUT_MILLIS) { + return false; + } + releaseDestinationClaim(handoff); + return true; + }); + } + + private void releaseDestinationClaim(PendingPresenceHandoff handoff) { + if (handoff != null) { + backendPlayerPresenceTracker.releaseDestinationClaim(handoff.playerUuid, handoff.server, + handoff.conflictSequence); + } + } + + protected void retryPendingPresenceHandoffs(long now) { + List retry = new ArrayList<>(); + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + for (PendingPresenceHandoff handoff : pendingPresenceHandoffs.values()) { + UUID activeRequestId = backendPlayerPresenceTracker.getPendingSnapshotRequestId(handoff.server, now); + if (handoff.requestId != null && !handoff.requestId.equals(activeRequestId)) { + handoff.requestId = null; + } + if (handoff.requestId == null) { + retry.add(handoff); + } + } + } + for (PendingPresenceHandoff handoff : retry) { + requestBackendPresenceSnapshot(handoff.server, handoff, now, true); + } + } + + protected int getPendingPresenceHandoffCount() { + synchronized (pendingPresenceHandoffs) { + return pendingPresenceHandoffs.size(); + } + } + + protected void scheduleBackendPresenceStartupResync() { + ScheduledExecutorService scheduler = getScheduler(); + if (method == null || !method.supportsBackendPresence() || scheduler == null) { + return; + } + scheduler.schedule(this::requestBackendPresenceStartupResync, + PRESENCE_STARTUP_RESYNC_DELAY_SECONDS, TimeUnit.SECONDS); + } + + protected void requestBackendPresenceStartupResync() { + if (!enabled || method == null || !method.supportsBackendPresence() || globalMessageProxyHandler == null) { + return; + } + long requestedAt = System.currentTimeMillis(); + int delay = 1; + for (String server : getAllAvailableServers()) { + if (!isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_RESYNC_REQUEST)) { + continue; + } + globalMessageProxyHandler.sendMessage(server, delay++, + VotingPluginWire.presenceResyncRequest(server, UUID.randomUUID(), requestedAt)); + } + } + + private void maintainBackendPresence() { + if (!enabled || method == null || !method.supportsBackendPresence()) { + return; + } + expireBackendPresence(PRESENCE_BACKEND_TIMEOUT_MILLIS); + for (String server : getAllAvailableServers()) { + if (pendingBackendRecoverySnapshots.contains(presenceServerKey(server))) { + requestBackendPresenceSnapshot(server); + } + } + retryPendingPresenceHandoffs(System.currentTimeMillis()); + } + + private String presenceServerKey(String server) { + return server == null ? "" : server.trim().toLowerCase(java.util.Locale.ROOT); + } + + private boolean isPresenceServerValid(String server, String subChannel) { + // The presence protocol's trust boundary is the configured backend set. The + // selected transport must only be accessible to backend servers trusted not to + // impersonate one another. + if (server == null || server.isBlank() || !isServerValid(server)) { + debug("Ignored " + subChannel + " presence envelope for an unconfigured server"); + return false; + } + return true; + } + + private boolean isPresenceGenerationValid(UUID backendIncarnationId, long backendStartedAt, + long presenceTimestamp, String subChannel) { + if (backendIncarnationId == null || backendStartedAt <= 0L || presenceTimestamp < backendStartedAt) { + debug("Ignored " + subChannel + " presence envelope with an invalid backend generation"); + return false; + } + return true; + } + + /** + * Removes presence owned by backends that have stopped reporting heartbeats. + * Scheduling and timeout configuration are intentionally left to dedicated + * proxy mode. + * + * @param timeoutMillis maximum backend silence before expiry + * @return expired backend server names + */ + public Set expireBackendPresence(long timeoutMillis) { + if (method == null || !method.supportsBackendPresence()) { + return Collections.emptySet(); + } + long now = System.currentTimeMillis(); + Set expired = backendPlayerPresenceTracker.expireBackends(now, timeoutMillis); + for (String server : expired) { + discardPendingPresenceHandoffs(server); + // Keep recovery pending while this generation is unavailable. If the same + // backend process resumes, its heartbeat can mark it available again and the + // maintenance task will request a fresh snapshot of players who stayed online. + pendingBackendRecoverySnapshots.add(presenceServerKey(server)); + } + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + } + return expired; + } + + public void login(String playerName, String uuid, String serverName) { + if (!getConfig().getOnlineMode()) { + uuid = getUUID(playerName); + } + + try { + if (uuid != null && !uuid.isEmpty() && !uuid.equalsIgnoreCase("null")) { + uuid = UUID.fromString(uuid.trim()).toString(); + } + } catch (Exception ignored) { + // ignore + } + + if (getConfig().getOnlineMode()) { + addNonVotedPlayer(uuid, playerName); + } + if (isPlayerOnlineForVoteRouting(playerName)) { + if (getConfig().getGlobalDataEnabled()) { + if (getGlobalDataHandler().isTimeChangedHappened()) { + getGlobalDataHandler().checkForFinishedTimeChanges(); + } + } + + checkCachedVotes(serverName); + retryPendingOnlineBroadcasts(serverName); + retryPendingTimeBroadcasts(serverName); + checkOnlineVotes(playerName, uuid, serverName); + multiProxyHandler.login(uuid, playerName); + } + } + + private void logInfo(String msg) { + log(msg); + } + + public abstract void logSevere(String message); + + public void onDisable() { + onDisable(false); + } + + /** Full runtime replacement waits for hosted workers; final proxy stop remains non-blocking. */ + public void onDisable(boolean waitForHosted) { + if (waitForHosted) { + prepareForRuntimeReplacement(); + } else { + controlServicesGeneration.incrementAndGet(); + controlLifecycleExecutor.shutdownNow(); + stopControlServices(false); + } + completeRuntimeReplacementShutdown(); + } + + /** Fail-closed gate that must complete before a replacement proxy runtime is created. */ + public void prepareForRuntimeReplacement() { + controlServicesGeneration.incrementAndGet(); + synchronized (controlLifecycleLock) { + ControlConnector connector = controlConnector; + if (connector != null && !connector.reserveRuntimeReplacement()) { + throw new IllegalStateException("Control result must be acknowledged before proxy runtime replacement"); + } + controlLifecycleExecutor.shutdown(); + stopControlServicesLocked(true); + } + } + + /** Best-effort remainder of runtime teardown after the Control overlap gate has succeeded. */ + public void completeRuntimeReplacementShutdown() { + cancelCommunicationTests("Proxy runtime stopped before the backend replied"); + runCleanup("vote cache", () -> getVoteCacheHandler().saveVoteCache()); + runCleanup("proxy MySQL messenger", () -> { + if (getProxyMysqlMessenger() != null) getProxyMysqlMessenger().shutdown(); + }); + runCleanup("proxy MySQL", () -> { + if (getProxyMySQL() != null) getProxyMySQL().shutdown(); + }); + runCleanup("multi-proxy handler", () -> { + if (multiProxyHandler != null) multiProxyHandler.close(); + }); + runCleanup("socket listener", () -> { + if (socketHandler != null) socketHandler.closeConnection(); + }); + runCleanup("socket clients", this::closeSocketClients); + runCleanup("HTTP transport", this::closeHttpTransport); + runCleanup("Redis subscriber", () -> { + if (redisHandler != null) redisHandler.close(); + }); + runCleanup("Redis publisher", () -> { + JedisPool pool = redisPublisherPool; + try { + if (pool != null) pool.close(); + } finally { + if (redisPublisherPool == pool) redisPublisherPool = null; + } + }); + runCleanup("MQTT transport", () -> { + if (mqttHandler != null) mqttHandler.disconnect(); + }); + runCleanup("time checker", () -> bungeeTimeChecker.shutdown()); + runCleanup("global data", () -> { + if (getGlobalDataHandler() != null) getGlobalDataHandler().shutdown(); + }); + enabled = false; + } + + private void runCleanup(String service, CleanupAction cleanup) { + try { + cleanup.run(); + } catch (Exception failure) { + logSevere("Unable to stop " + service + "; remaining proxy cleanup will continue"); + } + } + + @FunctionalInterface + private interface CleanupAction { void run() throws Exception; } + + public void onPluginMessageReceived(DataInputStream in) { + onPluginMessageReceived(in, null); + } + + /** Receives a plugin message bound to the backend server connection that sent it. */ + public void onPluginMessageReceived(DataInputStream in, String sourceServer) { + runAsync(() -> { + try { + final String headerSub; + if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { + headerSub = encryptionHandler.decrypt(in.readUTF()); + } else { + headerSub = in.readUTF(); + } + + int size = in.readInt(); // sanity only + + if (getConfig().getDebug()) { + debug("Received plugin message header=" + headerSub + " size=" + size); + } + + String payload = ""; + if (size > 0) { + if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { + payload = encryptionHandler.decrypt(in.readUTF()); + } else { + payload = in.readUTF(); + } + } + + JsonEnvelope envelope = JsonEnvelopeCodec.decode(payload); + + if (!headerSub.equalsIgnoreCase(envelope.getSubChannel())) { + if (getConfig().getDebug()) { + warn("PluginMessage subChannel mismatch: header=" + headerSub + " env=" + + envelope.getSubChannel()); + } + return; + } + + if (VotingPluginWire.SUB_CONTROL_ENROLLMENT_REQUEST.equals(envelope.getSubChannel())) { + handleControlEnrollmentRequest(sourceServer, envelope); + return; + } + + globalMessageProxyHandler.onMessage(envelope); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + + private void handleControlEnrollmentRequest(String sourceServer, JsonEnvelope envelope) { + VotingPluginWire.ControlEnrollmentRequest request = VotingPluginWire.readControlEnrollmentRequest(envelope); + if (!request.valid || sourceServer == null || sourceServer.isBlank()) return; + if (!sourceServer.equals(request.nodeId)) { + sendPluginMessageServer(sourceServer, 0, + VotingPluginWire.controlEnrollmentResult(sourceServer, request.requestId, false)); + return; + } + long now = System.nanoTime(); + AtomicBoolean allowed = new AtomicBoolean(); + controlEnrollmentNextAllowed.compute(sourceServer, (ignored, nextAllowed) -> { + if (nextAllowed == null || now >= nextAllowed) { + allowed.set(true); + return now + CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS; + } + return nextAllowed; + }); + if (!allowed.get()) return; + HostedControlManager manager = hostedControlManager; + if (manager == null) { + sendPluginMessageServer(sourceServer, 0, + VotingPluginWire.controlEnrollmentResult(sourceServer, request.requestId, false)); + return; + } + manager.installNodeVerifier(sourceServer, request.verifier, request.endpoint).whenComplete((installed, failure) -> { + boolean success = failure == null && Boolean.TRUE.equals(installed); + sendPluginMessageServer(sourceServer, 0, + VotingPluginWire.controlEnrollmentResult(sourceServer, request.requestId, success)); + if (success) log("[Control] automatically enrolled backend node " + sourceServer); + }); + } + + private UUID parseUUIDFromString(String uuidAsString) { + String[] parts = { "0x" + uuidAsString.substring(0, 8), "0x" + uuidAsString.substring(8, 12), + "0x" + uuidAsString.substring(12, 16), "0x" + uuidAsString.substring(16, 20), + "0x" + uuidAsString.substring(20, 32) }; + + long mostSigBits = Long.decode(parts[0]).longValue(); + mostSigBits <<= 16; + mostSigBits |= Long.decode(parts[1]).longValue(); + mostSigBits <<= 16; + mostSigBits |= Long.decode(parts[2]).longValue(); + + long leastSigBits = Long.decode(parts[3]).longValue(); + leastSigBits <<= 48; + leastSigBits |= Long.decode(parts[4]).longValue(); + + return new UUID(mostSigBits, leastSigBits); + } + + public synchronized void processQueue() { + while (getVoteCacheHandler().getTimeChangeQueue().size() > 0) { + VoteTimeQueue vote = getVoteCacheHandler().getTimeChangeQueue().element(); + if (!vote.isProcessed()) { + VoteTotalsSnapshot queuedTotals = vote.getTotals() == null || vote.getTotals().isEmpty() ? null + : VoteTotalsSnapshot.parseStorage(vote.getTotals()); + QueuedVoteResult result = vote(vote.getName(), vote.getService(), true, false, vote.getTime(), queuedTotals, + vote.getUuid(), vote); + if (result == QueuedVoteResult.RETRY) { + scheduleTimeVoteRetry(); + return; + } + if (result == QueuedVoteResult.TERMINAL) { + warn("Removing terminal rollover vote " + vote.getVoteId() + " for " + vote.getName() + "/" + + ServiceSiteValidator.sanitizeForLog(vote.getService())); + } + } + if (!getVoteCacheHandler().removeTimeVote(vote)) { + scheduleTimeVoteRetry(); + return; + } + } + } + + private void scheduleTimeVoteRetry() { + if (timeVoteRetryScheduled || getScheduler() == null) { + return; + } + timeVoteRetryScheduled = true; + try { + getScheduler().schedule(() -> { + synchronized (VotingPluginProxy.this) { + timeVoteRetryScheduled = false; + } + processQueue(); + }, 5, TimeUnit.SECONDS); + } catch (RuntimeException e) { + timeVoteRetryScheduled = false; + debug("Unable to schedule rollover vote retry: " + e.getMessage()); + } + } + + public void reload() { + reloadRuntime(true); + } + + /** Applies a Control-originated configuration reload without stopping its connector or hosted service. */ + public void reloadFromControl() { + reloadRuntime(false); + } + + private void reloadRuntime(boolean restartControlServices) { + method = BungeeMethod.getByName(getConfig().getBungeeMethod()); + if (getMethod() == null) { + method = BungeeMethod.PLUGINMESSAGING; + } + warnUnsupportedDedicatedVotingProxyMode(); + if (!restartControlServices && method == BungeeMethod.SOCKETS) { + rebuildSocketClients(); + } + + setCurrentVotePartyVotesRequired( + getConfig().getVotePartyVotesRequired() + getVoteCacheVotePartyIncreaseVotesRequired()); + if (restartControlServices) { + loadMultiProxySupport(); + restartControlServicesAsync(); + } + } + + private synchronized void rebuildSocketClients() { + HashMap rebuilt = new HashMap<>(); + try { + List blocked = getConfig().getBlockedServers(); + for (String server : getConfig().getSpigotServers()) { + if (blocked.contains(server)) continue; + Map data = getConfig().getSpigotServerConfiguration(server); + String host = data.containsKey("Host") ? (String) data.get("Host") : ""; + int port = data.containsKey("Port") ? (int) data.get("Port") : 1298; + rebuilt.put(server, new ClientHandler(host, port, encryptionHandler, getConfig().getDebug())); + } + } catch (RuntimeException failure) { + stopSocketClients(rebuilt); + throw failure; + } + HashMap previous = clientHandles; + clientHandles = rebuilt; + stopSocketClients(previous); + } + + private synchronized boolean sendSocketEnvelope(String server, JsonEnvelope envelope) { + ClientHandler socketClient = clientHandles == null ? null : clientHandles.get(server); + if (socketClient == null) return false; + try { + socketClient.sendEnvelope(envelope); + return true; + } catch (RuntimeException e) { + debug(e.getMessage()); + return false; + } + } + + private synchronized boolean sendHttpEnvelope(String server, JsonEnvelope envelope) { + HttpProxyTransportServer transport = httpTransportServer; + return transport != null && transport.send(server, envelope); + } + + private void startHttpTransport() { + try { + URI endpoint = URI.create(getConfig().getHttpPublicEndpoint()); + if (!"https".equalsIgnoreCase(endpoint.getScheme()) || endpoint.getHost() == null + || endpoint.getPort() == 0 || endpoint.getPort() > 65535 + || endpoint.getUserInfo() != null || endpoint.getQuery() != null || endpoint.getFragment() != null + || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) { + throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); + } + File directory = new File(getDataFolderPlugin(), "http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.toPath(), endpoint.getHost()); + httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); + httpTransportServer = new HttpProxyTransportServer( + new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, + httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope); + httpTransportServer.start(); + logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" + + httpTransportServer.port() + "; use /votingpluginbungee httpcode for each backend"); + } catch (Exception failure) { + closeHttpTransport(); + throw new IllegalStateException("HTTP transport could not start securely", failure); + } + } + + /** Keeps the authenticated mTLS backend identity attached to security-sensitive proxy routing. */ + protected void handleHttpTransportEnvelope(HttpProxyTransportServer.ReceivedEnvelope received) { + if (!isAuthenticatedHttpEnvelopeAllowed(received)) { + debug("Ignored HTTP envelope whose player-presence claim did not match its authenticated backend"); + return; + } + GlobalMessageProxyHandler handler = globalMessageProxyHandler; + if (handler == null) throw new IllegalStateException("HTTP message router is not ready"); + handler.onMessage(received.envelope()); + } + + private boolean isAuthenticatedHttpEnvelopeAllowed(HttpProxyTransportServer.ReceivedEnvelope received) { + if (received == null || received.envelope() == null || received.serverId() == null) return false; + String stampedServer = received.envelope().getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + if (!received.serverId().equalsIgnoreCase(stampedServer)) return false; + if (!VotingPluginWire.SUB_LOGIN.equals(received.envelope().getSubChannel())) return true; + VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(received.envelope()); + boolean modern = event.connectionId != null || event.backendIncarnationId != null + || event.backendStartedAt != 0L || event.presenceTimestamp != 0L; + if (!modern || isDedicatedVotingProxyEnabled()) return true; + // A player-facing proxy has a stronger authority than any backend: its live + // player connection supplies both the current route and (in online mode) UUID. + return isLegacyLoginDestinationAuthoritative(event.player, event.uuid, received.serverId()); + } + + private synchronized void closeHttpTransport() { + HttpProxyTransportServer transport = httpTransportServer; + httpTransportServer = null; + httpEnrollmentAuthority = null; + if (transport != null) transport.close(); + } + + public String createHttpConnectionCode(String serverId) { + HttpEnrollmentAuthority authority = httpEnrollmentAuthority; + if (method != BungeeMethod.HTTP || authority == null) { + throw new IllegalStateException("The HTTP transport is not running"); + } + return authority.createConnectionCode(serverId, URI.create(getConfig().getHttpPublicEndpoint()), Duration.ofMinutes(15)) + .encode(); + } + + public void revokeHttpBackend(String serverId) { + HttpEnrollmentAuthority authority = httpEnrollmentAuthority; + if (method != BungeeMethod.HTTP || authority == null) throw new IllegalStateException("The HTTP transport is not running"); + authority.revoke(HttpTlsIdentity.canonicalServerId(serverId)); + } + + private synchronized void closeSocketClients() { + HashMap clients = clientHandles; + clientHandles = null; + stopSocketClients(clients); + } + + static void stopSocketClients(Map clients) { + if (clients == null) return; + for (ClientHandler client : clients.values()) { + if (client == null) continue; + try { + client.stopConnection(); + } catch (RuntimeException ignored) { + // Best effort: one broken client must not prevent the remaining sockets from closing. + } + } + } + + private void warnUnsupportedDedicatedVotingProxyMode() { + if (getConfig().getDedicatedVotingProxy() && (method == null || !method.supportsBackendPresence())) { + logSevere("DedicatedVotingProxy requires MYSQL, REDIS, MQTT, SOCKETS, or HTTP; PLUGINMESSAGING is disabled for " + + "dedicated-proxy routing. Falling back to normal proxy routing."); + } + } + + public abstract void runAsync(Runnable run); + + /** Platform name used only for the transport-neutral Control discovery contract. */ + public abstract String getProxyPlatform(); + + public abstract void runConsoleCommand(String command); + + public abstract void saveVoteCacheFile(); + + public abstract void reloadCore(boolean mysql); + + /** Strict Control reload path; failures propagate so the caller can restore its backup. */ + public abstract void reloadControlConfiguration() throws Exception; + + public abstract boolean sendPluginMessageData(String server, String channel, byte[] data, boolean queue); + + private static final int PLUGIN_MESSAGE_HARD_LIMIT = 32767; + private static final int PLUGIN_MESSAGE_SOFT_LIMIT = 30000; + + public void sendPluginMessageServer(String server, int delay, JsonEnvelope envelope) { + getScheduler().schedule(() -> sendPluginMessageServerNow(server, envelope), delay * 5L, TimeUnit.MILLISECONDS); + } + + /** + * Sends a plugin-message envelope immediately and reports whether the proxy + * accepted it for delivery. + * + * @param server target backend server + * @param envelope envelope to send + * @return true when the proxy accepted the message for delivery + */ + protected boolean sendPluginMessageServerNow(String server, JsonEnvelope envelope) { + final String subChannel = envelope.getSubChannel(); + final String payload = JsonEnvelopeCodec.encode(envelope); + + final byte[] subChannelBytes = subChannel.getBytes(java.nio.charset.StandardCharsets.UTF_8); + final byte[] payloadBytes = payload.getBytes(java.nio.charset.StandardCharsets.UTF_8); + + // Estimate bytes written: + // - writeUTF adds 2-byte length prefix + UTF-8 bytes + // - writeInt is 4 bytes + int estimatedSize = 2 + subChannelBytes.length + // subChannel UTF (len prefix + bytes) + 4 + // payload length int + 2 + payloadBytes.length; // payload UTF (len prefix + bytes) + + if (estimatedSize > PLUGIN_MESSAGE_SOFT_LIMIT) { + debug("[PluginMessage] Payload nearing limit (" + estimatedSize + " bytes) server=" + server + + " subChannel=" + subChannel + " — consider Redis instead"); + } + + if (estimatedSize > PLUGIN_MESSAGE_HARD_LIMIT) { + debug("[PluginMessage] Payload TOO LARGE (" + estimatedSize + " bytes, max=" + PLUGIN_MESSAGE_HARD_LIMIT + + ") server=" + server + " subChannel=" + subChannel + " — NOT sent"); + return false; + } + + try (ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(byteOutStream)) { + if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { + out.writeUTF(encryptionHandler.encrypt(subChannel)); + } else { + out.writeUTF(subChannel); + } + + // sanity only: MUST be bytes, not chars + out.writeInt(payloadBytes.length); + + if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { + out.writeUTF(encryptionHandler.encrypt(payload)); + } else { + out.writeUTF(payload); + } + out.flush(); + + boolean sent = sendPluginMessageData(server, getConfig().getPluginMessageChannel().toLowerCase(), + byteOutStream.toByteArray(), false); + if (getConfig().getDebug()) { + debug((sent ? "Sent" : "Could not send") + " plugin envelope (" + estimatedSize + " bytes) " + server + + " " + subChannel + " " + envelope.getFields()); + } + return sent; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + static DefaultJedisClientConfig buildRedisClientConfig(VotingPluginProxyConfig configSource) { + DefaultJedisClientConfig.Builder config = DefaultJedisClientConfig.builder() + .database(configSource.getRedisDbIndex()).ssl(configSource.getRedisSsl()).connectionTimeoutMillis(2000) + .socketTimeoutMillis(2000); + if (configSource.getRedisSsl()) { + SSLParameters sslParameters = new SSLParameters(); + sslParameters.setEndpointIdentificationAlgorithm("HTTPS"); + config.sslParameters(sslParameters); + } + if (configSource.getRedisUsername() != null && !configSource.getRedisUsername().isEmpty()) { + config.user(configSource.getRedisUsername()); + } + if (configSource.getRedisPassword() != null && !configSource.getRedisPassword().isEmpty()) { + config.password(configSource.getRedisPassword()); + } + return config.build(); + } + + public boolean sendRedisEnvelopeServer(String server, JsonEnvelope envelope) { + return sendRedisEnvelopeServer(server, envelope, false); + } + + private boolean sendRedisEnvelopeServer(String server, JsonEnvelope envelope, boolean useRetryCooldown) { + JedisPool publisherPool = redisPublisherPool; + if (publisherPool == null || (useRetryCooldown && System.currentTimeMillis() < redisPublisherRetryAfter)) { + return false; + } + + try (Jedis jedis = publisherPool.getResource()) { + String channel = getConfig().getRedisPrefix() + "VotingPlugin_" + server; + long subscribers = jedis.publish(channel, + JsonEnvelopeCodec.encode(VotingPluginWire.withRedisDeliveryId(envelope))); + redisPublisherRetryAfter = 0L; + return subscribers > 0; + } catch (Exception e) { + if (useRetryCooldown) { + // Standalone broadcasts remain queued, so their retries can be throttled safely. + redisPublisherRetryAfter = System.currentTimeMillis() + 2000L; + } + debug(e.getMessage()); + return false; + } + } + + public boolean sendMqttEnvelopeServer(String server, JsonEnvelope envelope) { + if (mqttHandler == null) { + return false; + } + try { + mqttHandler.publishEnvelope(getConfig().getMqttPrefix() + "votingplugin/servers/" + server, envelope); + return true; + } catch (Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + return false; + } + } + + public boolean sendSocketEnvelopeServer(String server, JsonEnvelope envelope) { + Map configuration = getConfig().getSpigotServerConfiguration(server); + if (configuration == null) { + return false; + } + String host = configuration.get("Host") instanceof String ? (String) configuration.get("Host") : ""; + int port = configuration.get("Port") instanceof Number ? ((Number) configuration.get("Port")).intValue() : 1298; + if (host.isEmpty()) { + return false; + } + + String payload = JsonEnvelopeCodec.encode(envelope); + String encoded = encryptionHandler != null ? encryptionHandler.encrypt(payload) : payload; + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(host, port), 2000); + try (DataOutputStream output = new DataOutputStream(socket.getOutputStream())) { + output.writeUTF(encoded); + output.flush(); + } + return true; + } catch (Exception e) { + debug(e.getMessage()); + return false; + } + } + + public void sendServerNameMessage() { + for (String s : getAllAvailableServers()) { + sendPluginMessageServer(s, 1, VotingPluginWire.serverName(s)); + } + } + + public void sendVoteParty(String server) { + if (isSomeoneOnlineServerForVoteRouting(server)) { + globalMessageProxyHandler.sendMessage(server, 1, VotingPluginWire.votePartyBungee()); + } + } + + public void setCurrentVotePartyVotes(int amount) { + votePartyVotes = amount; + setVoteCacheVotePartyCurrentVotes(amount); + debug("Current vote party total: " + votePartyVotes); + } + + public abstract void setVoteCacheLastUpdated(); + + public abstract void setVoteCachePrevDay(int day); + + public abstract void setVoteCachePrevMonth(String text); + + public abstract void setVoteCachePrevWeek(int week); + + public abstract void setVoteCacheVoteCacheIgnoreTime(boolean ignore); + + public abstract void setVoteCacheVotePartyCurrentVotes(int votes); + + public abstract void setVoteCacheVotePartyIncreaseVotesRequired(int votes); + + public void status() { + for (String s : getAllAvailableServers()) { + if (!isSomeoneOnlineServerForVoteRouting(s)) { + log("No players on server " + s + " to send test status message, please retest with someone online"); + } else { + log("Sending request for status message on " + s); + globalMessageProxyHandler.sendMessage(s, 1, VotingPluginWire.status(s)); + } + } + } + + /** Runs a correlated, non-vote round trip over the active backend transport. */ + public CompletableFuture testBackendCommunication(String requestedServer, + long timeoutMillis) { + String server = requestedServer == null ? "" : requestedServer.trim(); + BungeeMethod activeMethod = method; + if (server.isEmpty() || !getAllAvailableServers().contains(server)) { + return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, + "UNKNOWN_BACKEND", "The backend is not configured on this proxy")); + } + if (activeMethod == null || globalMessageProxyHandler == null) { + return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, + "TRANSPORT_UNAVAILABLE", "The proxy communication transport is not running")); + } + if (activeMethod == BungeeMethod.PLUGINMESSAGING && !isSomeoneOnlineServerForVoteRouting(server)) { + return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, + "PLAYER_REQUIRED", "Plugin messaging requires an online player on the selected backend")); + } + ScheduledExecutorService scheduler = getScheduler(); + if (scheduler == null) { + return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, + "TRANSPORT_UNAVAILABLE", "The proxy scheduler is not running")); + } + long boundedTimeout = Math.max(500L, Math.min(timeoutMillis, 30000L)); + UUID requestId = UUID.randomUUID(); + CompletableFuture result = new CompletableFuture<>(); + PendingCommunicationTest pending = new PendingCommunicationTest(server, activeMethod, System.nanoTime(), result); + pendingCommunicationTests.put(requestId, pending); + result.whenComplete((ignored, failure) -> pendingCommunicationTests.remove(requestId, pending)); + try { + if (!sendCommunicationTestEnvelopeNow(server, VotingPluginWire.status(server, requestId))) { + result.complete(CommunicationTestResult.failure(server, activeMethod, "TRANSPORT_UNAVAILABLE", + "The active transport could not accept the communication test")); + return result; + } + scheduler.schedule(() -> result.complete(CommunicationTestResult.failure(server, activeMethod, + "TIMEOUT", "No correlated reply arrived before the timeout")), boundedTimeout, TimeUnit.MILLISECONDS); + } catch (RuntimeException failure) { + result.complete(CommunicationTestResult.failure(server, activeMethod, "SEND_FAILED", + "The proxy could not send the communication test")); + } + return result; + } + + /** Sends a diagnostic immediately and reports whether the active transport accepted it. */ + protected boolean sendCommunicationTestEnvelopeNow(String server, JsonEnvelope envelope) { + return sendProxyBroadcastEnvelopeNow(server, envelope); + } + + protected void handleStatusOkay(JsonEnvelope message) { + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + String request = message.getFields().getOrDefault(VotingPluginWire.K_REQUEST_ID, ""); + if (request.isEmpty()) { + log("Status okay for " + server); + return; + } + UUID requestId; + try { + requestId = UUID.fromString(request); + } catch (IllegalArgumentException ignored) { + debug("Ignored status reply with an invalid request ID from " + server); + return; + } + PendingCommunicationTest pending = pendingCommunicationTests.get(requestId); + if (pending == null || !pending.server().equals(server)) { + debug("Ignored unexpected status reply from " + server); + return; + } + long roundTripMillis = Math.max(0L, + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - pending.startedAtNanos())); + pending.result().complete(CommunicationTestResult.success(server, pending.method(), roundTripMillis)); + } + + private void cancelCommunicationTests(String message) { + pendingCommunicationTests.forEach((requestId, pending) -> pending.result().complete( + CommunicationTestResult.failure(pending.server(), pending.method(), "TRANSPORT_STOPPED", message))); + pendingCommunicationTests.clear(); + } + + public record CommunicationTestResult(boolean success, String code, String message, String server, + String method, long roundTripMillis) { + private static CommunicationTestResult success(String server, BungeeMethod method, long roundTripMillis) { + return new CommunicationTestResult(true, "OK", "Backend replied over the active transport", server, + method == null ? "" : method.name(), roundTripMillis); + } + + private static CommunicationTestResult failure(String server, BungeeMethod method, String code, String message) { + return new CommunicationTestResult(false, code, message, server, + method == null ? "" : method.name(), -1L); + } + } + + private record PendingCommunicationTest(String server, BungeeMethod method, long startedAtNanos, + CompletableFuture result) { } + + private void sendVoteDelayRejected(String player, String uuid, String service, boolean playerOnline, + String playerServer) { + if (!playerOnline || playerServer == null || !getAllAvailableServers().contains(playerServer)) { + debug("Not sending vote delay rejection for " + player + " because the player is offline"); + return; + } + + globalMessageProxyHandler.sendMessage(playerServer, 1, + VotingPluginWire.voteDelayRejected(player, uuid, service, true)); + } + + public String getWaitUntilDelaySiteFromService(String service) { + for (String site : getConfig().getWaitUntilVoteDelaySites()) { + if (getConfig().getWaitUntilVoteDelayService(site).equalsIgnoreCase(service)) { + return site; + } + } + return ""; + } + + private long getLastVotesTime(String uuid, ArrayList cols, String site, String service, String player, + boolean includeTimeChangeQueue) { + long mostRecentTime = 0; + + if (getVoteCacheHandler().hasOnlineVotes(uuid)) { + ArrayList onlineVotes = getVoteCacheHandler().getOnlineVotes(uuid); + for (OfflineBungeeVote vote : onlineVotes) { + if (vote.getService().equalsIgnoreCase(service)) { + mostRecentTime = Math.max(mostRecentTime, vote.getTime()); + } + } + } + + for (String server : getAllAvailableServers()) { + for (OfflineBungeeVote vote : getVoteCacheHandler().getVotes(server)) { + if (vote.getUuid().equals(uuid) && vote.getService().equalsIgnoreCase(service)) { + mostRecentTime = Math.max(mostRecentTime, vote.getTime()); + } + } + } + + if (includeTimeChangeQueue && player != null) { + for (VoteTimeQueue queuedVote : getVoteCacheHandler().getTimeChangeQueue()) { + if (queuedVote.getName().equalsIgnoreCase(player) + && queuedVote.getService().equalsIgnoreCase(service)) { + mostRecentTime = Math.max(mostRecentTime, queuedVote.getTime()); + } + } + } + + for (Column d : cols) { + if (d.getName().equalsIgnoreCase("LastVotes")) { + DataValue value = d.getValue(); + String[] list = value.getString().split("%line%"); + for (String str : list) { + String[] data = str.split("//"); + if (data[0].equalsIgnoreCase(site)) { + mostRecentTime = Math.max(mostRecentTime, Long.valueOf(data[1])); + } + } + } + } + return mostRecentTime; + } + + public boolean checkVoteDelay(String uuid, String service, ArrayList data) { + return checkVoteDelay(uuid, null, service, data, false); + } + + /** + * Checks the configured vote delay, optionally including accepted votes waiting + * for a GlobalData time change to finish. + * + * @param uuid player UUID + * @param player player name used by the time-change queue + * @param service vote service + * @param data current player data + * @param includeTimeChangeQueue whether queued votes reserve their delay slot + * @return true when the vote may be accepted + */ + public boolean checkVoteDelay(String uuid, String player, String service, ArrayList data, + boolean includeTimeChangeQueue) { + String site = getWaitUntilDelaySiteFromService(service); + if (site.isEmpty()) { + debug("No service site set for " + service + ", skipping vote delay check"); + return true; + } + + int voteDelay = getConfig().getWaitUntilVoteDelayVoteDelay(site); + int voteDelayMin = getConfig().getWaitUntilVoteDelayVoteDelayMin(site); + + long lastVote = getLastVotesTime(uuid, data, site, service, player, includeTimeChangeQueue); + if (lastVote == 0) { + debug("No last vote time found for " + uuid + "/" + service + ", skipping vote delay check"); + return true; + } + + try { + LocalDateTime now = getBungeeTimeChecker().getTime(); + LocalDateTime lastVoteTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastVote), ZoneId.systemDefault()) + .plusHours(getConfig().getTimeHourOffSet()); + + if (!getConfig().getWaitUntilVoteDelayVoteDelayDaily(site)) { + if (voteDelay == 0 && voteDelayMin == 0) { + debug("Vote delay is 0 for " + site + ", skipping vote delay check"); + return true; + } + + LocalDateTime nextvote = lastVoteTime.plusHours((long) voteDelay).plusMinutes((long) voteDelayMin); + return now.isAfter(nextvote); + } + LocalDateTime resetTime = lastVoteTime.withHour(getConfig().getWaitUntilVoteDelayVoteDelayHour(site)) + .withMinute(0).withSecond(0); + LocalDateTime resetTimeTomorrow = resetTime.plusHours(24); + + if (lastVoteTime.isBefore(resetTime)) { + if (now.isAfter(resetTime)) { + debug("Vote delay is met for " + uuid + "/" + service + ", vote can be processed"); + return true; + } + } else { + if (now.isAfter(resetTimeTomorrow)) { + debug("Vote delay is met for " + uuid + "/" + service + ", vote can be processed"); + return true; + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + debug("Vote delay is not met for " + uuid + "/" + service + ", skipping vote"); + return false; + } + + public synchronized void vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid) { + vote(player, service, realVote, timeQueue, queueTime, text, uuid, null); + } + + private enum QueuedVoteResult { + SUCCESS, RETRY, TERMINAL + } + + private synchronized QueuedVoteResult vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid, VoteTimeQueue queuedVote) { + try { + if (!ServiceSiteValidator.isValid(service)) { + warn("Rejected vote with invalid service site '" + ServiceSiteValidator.sanitizeForLog(service) + "'"); + return QueuedVoteResult.TERMINAL; + } + if (!MinecraftUsernameValidator.isValid(player, getConfig().getBedrockPlayerPrefix())) { + warn("Rejected vote with invalid Minecraft username '" + + MinecraftUsernameValidator.sanitizeForLog(player) + "' from service '" + + MinecraftUsernameValidator.sanitizeForLog(service) + "'"); + return QueuedVoteResult.TERMINAL; + } + + UUID voteId = queuedVote == null ? null : queuedVote.getVoteId(); + if (voteId == null) { + voteId = UUID.randomUUID(); + } + + // UUID resolution + if (!getConfig().getOnlineMode()) { + uuid = getUUID(player); + } + + if (uuid == null || uuid.isEmpty()) { + uuid = getUUID(player); + + // Bedrock prefix auto-detect + if (uuid.isEmpty() && !getConfig().getBedrockPlayerPrefix().isEmpty() + && !player.startsWith(getConfig().getBedrockPlayerPrefix())) { + String uuid1 = getUUID(getConfig().getBedrockPlayerPrefix() + player); + if (!uuid1.isEmpty()) { + debug("Detected bedrock player without prefix, adjusting..."); + player = getConfig().getBedrockPlayerPrefix() + player; + uuid = uuid1; + } + } + } + + if (uuid.isEmpty()) { + if (player.startsWith(getConfig().getBedrockPlayerPrefix())) { + log("Ignoring vote since unable to get UUID of bedrock player"); + return QueuedVoteResult.TERMINAL; + } + if (!getConfig().getAllowUnJoined()) { + log("Ignoring vote from " + player + " since player hasn't joined before"); + return QueuedVoteResult.TERMINAL; + } + if (!getConfig().getUUIDLookup()) { + log("Failed to get uuid for " + player); + return QueuedVoteResult.TERMINAL; + } + + debug("Fetching UUID online, since allowunjoined is enabled"); + UUID u = null; + try { + if (getConfig().getOnlineMode()) { + u = fetchUUID(player); + } + } catch (Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + if (u == null) { + debug("Failed to get uuid for " + player); + return QueuedVoteResult.TERMINAL; + } + uuid = u.toString(); + } + + // Normalize UUID string if possible + try { + if (uuid != null && !uuid.isEmpty() && !uuid.equalsIgnoreCase("null")) { + uuid = UUID.fromString(uuid.trim()).toString(); + } + } catch (Exception ignored) { + // ignore + } + + player = getProperName(uuid, player); + + // Cache online state/server once (IMPORTANT for broadcast logic correctness) + final boolean playerOnline = isPlayerOnlineForVoteRouting(player); + final String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(player) : null; + long time = queueTime != 0 ? queueTime + : LocalDateTime.now().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + + Set broadcastTargets = queuedVote == null ? new LinkedHashSet<>() + : new LinkedHashSet<>(queuedVote.getBroadcastTargets()); + Set broadcastForwardedServers = queuedVote == null ? new LinkedHashSet<>() + : new LinkedHashSet<>(queuedVote.getBroadcastForwardedServers()); + boolean proxyBroadcastHandled = queuedVote != null && queuedVote.isProxyBroadcastHandled(); + boolean processesTotals = getConfig().getPrimaryServer() || !getConfig().getMultiProxySupport(); + boolean managesTotals = processesTotals && getConfig().getBungeeManageTotals(); + boolean canValidateStandaloneBroadcast = canForwardStandaloneBroadcast(managesTotals); + ArrayList data = null; + boolean queueForTimeChange = false; + + // A completion callback can wipe totals and replay older queued votes. Run it + // before loading this vote's database snapshot so the calculations below use + // the post-rollover state. + if (getConfig().getGlobalDataEnabled() && getGlobalDataHandler().isTimeChangedHappened()) { + getGlobalDataHandler().checkForFinishedTimeChanges(); + queueForTimeChange = timeQueue && getGlobalDataHandler().isTimeChangedHappened(); + } + + // Validate the vote before any immediate announcement. This keeps duplicate + // votes rejected by the delay check out of the GlobalData rollover queue and + // prevents announcing a vote that will not be processed. + if (managesTotals) { + if (getProxyMySQL() == null) { + logSevere("Mysql is not loaded correctly, stopping vote processing"); + return QueuedVoteResult.RETRY; + } + + if (!getProxyMySQL().containsKeyQuery(uuid)) { + getProxyMySQL().update(uuid, "PlayerName", new DataValueString(player)); + getProxyMySQL().getUuids().add(uuid); + } + + data = getProxyMySQL().getExactQuery(new Column("uuid", new DataValueString(uuid))); + if (!checkVoteDelay(uuid, player, service, data, queuedVote == null)) { + log("Vote delay is not met for " + player + "/" + service + ", skipping vote"); + sendVoteDelayRejected(player, uuid, service, playerOnline, playerServer); + return QueuedVoteResult.TERMINAL; + } + } + + // Forward an accepted offline broadcast before the still-active GlobalData + // change queues the reward/totals work. The queued delivery state prevents + // replaying broadcasts that already reached a backend. + if (queueForTimeChange) { + VoteTotalsSnapshot projectedTotals = managesTotals ? getProjectedRolloverTotals(data, player) : text; + if (canValidateStandaloneBroadcast && proxyBroadcastDecider.usesImmediateForwarding(playerOnline)) { + broadcastTargets.addAll(proxyBroadcastDecider.resolveTargets(false, null)); + proxyBroadcastHandled = true; + } + VoteTimeQueue delayedVote = new VoteTimeQueue(voteId, player, service, time, + proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, + projectedTotals == null ? "" : projectedTotals.toString(), false, uuid); + if (!getVoteCacheHandler().addTimeVoteToCache(delayedVote)) { + logSevere("Unable to persist queued rollover vote for " + player + "/" + service + + "; skipping proxy broadcast"); + return QueuedVoteResult.RETRY; + } + if (proxyBroadcastHandled) { + for (String target : broadcastTargets) { + Set forwarded = sendProxyBroadcast(Collections.singleton(target), uuid, player, + service, time, projectedTotals == null ? "" : projectedTotals.toString(), false); + if (delayedVote.getBroadcastForwardedServers().addAll(forwarded)) { + broadcastForwardedServers.addAll(forwarded); + persistTimeVoteDelivery(delayedVote); + } + } + } + log("Caching vote from " + player + "/" + service + + " because time change is happening right now"); + return QueuedVoteResult.SUCCESS; + } + + addVoteParty(); + + // Totals processing (primary server OR no multiproxy) + if (processesTotals) { + if (managesTotals) { + int allTimeTotal = getValue(data, "AllTimeTotal", 1); + int monthTotal = getValue(data, "MonthTotal", 1); + + int dateMonthTotal = -1; + if (getConfig().getStoreMonthTotalsWithDate()) { + if (getConfig().getUseMonthDateTotalsAsPrimaryTotal()) { + dateMonthTotal = getValue(data, getMonthTotalsWithDatePath(), 1); + } else { + dateMonthTotal = monthTotal; + } + } + + int weeklyTotal = getValue(data, "WeeklyTotal", 1); + int dailyTotal = getValue(data, "DailyTotal", 1); + int points = getValue(data, "Points", getConfig().getPointsOnVote()); + + int maxVotes = getConfig().getMaxAmountOfVotesPerDay(); + if (maxVotes > 0) { + LocalDateTime cTime = getBungeeTimeChecker().getTime(); + int days = cTime.getDayOfMonth(); + if (monthTotal > days * maxVotes) { + monthTotal = days * maxVotes; + } + } + + if (getConfig().getLimitVotePoints() > 0 && points > getConfig().getLimitVotePoints()) { + points = getConfig().getLimitVotePoints(); + } + + text = new VoteTotalsSnapshot(allTimeTotal, monthTotal, weeklyTotal, dailyTotal, points, + votePartyVotes, currentVotePartyVotesRequired, dateMonthTotal); + + ArrayList update = new ArrayList<>(); + update.add(new Column("AllTimeTotal", new DataValueInt(allTimeTotal))); + update.add(new Column("MonthTotal", new DataValueInt(monthTotal))); + if (getConfig().getStoreMonthTotalsWithDate()) { + update.add(new Column(getMonthTotalsWithDatePath(), new DataValueInt(dateMonthTotal))); + } + update.add(new Column("WeeklyTotal", new DataValueInt(weeklyTotal))); + update.add(new Column("DailyTotal", new DataValueInt(dailyTotal))); + update.add(new Column("Points", new DataValueInt(points))); + + debug("Setting totals " + text.toString() + ", voteId=" + voteId + " for " + player + "/" + + service); + getProxyMySQL().update(uuid, update); + } else { + text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); + } + } + if (text == null) { + text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); + } + + VoteLogStatus voteStatus = VoteLogStatus.IMMEDIATE; + boolean standaloneProxyBroadcast = canValidateStandaloneBroadcast && (proxyBroadcastHandled + || proxyBroadcastDecider.usesImmediateForwarding(playerOnline)); + Set proxyBroadcastTargets = Collections.emptySet(); + if (standaloneProxyBroadcast) { + // A handled queued broadcast was necessarily sampled while the player was + // offline. Retry only targets that did not previously accept delivery. + proxyBroadcastTargets = proxyBroadcastHandled ? new LinkedHashSet<>(broadcastTargets) + : proxyBroadcastDecider.resolveTargets(false, null); + Set remainingTargets = new LinkedHashSet<>(proxyBroadcastTargets); + remainingTargets.removeAll(broadcastForwardedServers); + broadcastForwardedServers.addAll(sendProxyBroadcast(remainingTargets, uuid, player, service, time, + text == null ? "" : text.toString(), false)); + } + + // =========================== + // Send vote(s) to backend(s) + // =========================== + if (getConfig().getSendVotesToAllServers()) { + for (String s : getAllAvailableServers()) { + + boolean forceCache = getConfig().getWaitForUserOnline() + && (!playerOnline || playerServer == null || !playerServer.equalsIgnoreCase(s)); + + if (forceCache) { + debug("Forcing vote to cache for server " + s); + } + + if ((!isSomeoneOnlineServerForVoteRouting(s) && method.requiresPlayerOnline()) || forceCache) { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addServerVote(s, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, + text.toString(), broadcastForwarded, standaloneProxyBroadcast, + proxyBroadcastTargets, broadcastForwardedServers, false)); + debug("Caching vote for " + player + " on " + service + " for " + s); + } else { + boolean broadcastHere = !broadcastForwardedServers.contains(s); + if (broadcastHere && getConfig().getProxyBroadcastEnabled()) { + Set targets = standaloneProxyBroadcast ? proxyBroadcastTargets + : proxyBroadcastDecider.resolveTargets(playerOnline, playerServer); + broadcastHere = proxyBroadcastDecider.shouldBroadcast(s, targets); + } + + if (!sendVoteEnvelopeAccepted(s, 2, + VotingPluginWire.vote(player, uuid, service, time, true, realVote, text.toString(), + voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1))) { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addServerVote(s, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, + text.toString(), broadcastForwarded, standaloneProxyBroadcast, + proxyBroadcastTargets, broadcastForwardedServers, false)); + debug("Caching vote after the transport rejected delivery for " + s); + } + } + } + } else { + // Single-server mode: online goes to player server; otherwise queue as "online + // vote" + if (playerOnline && playerServer != null && getAllAvailableServers().contains(playerServer)) { + String server = playerServer; + + boolean broadcastHere = !broadcastForwardedServers.contains(server); + if (broadcastHere && getConfig().getProxyBroadcastEnabled()) { + Set targets = standaloneProxyBroadcast ? proxyBroadcastTargets + : proxyBroadcastDecider.resolveTargets(true, playerServer); + broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); + } + + boolean rewardAccepted = sendVoteEnvelopeAccepted(server, 1, + VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(), + voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1)); + if (!rewardAccepted) { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addOnlineVote(uuid, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), + broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, + broadcastForwardedServers, false)); + debug("Caching online vote after the transport rejected delivery for " + server); + } + + if (rewardAccepted && canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled() + && !standaloneProxyBroadcast) { + Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); + + int bDelay = 2; + for (String targetServer : targets) { + // avoid double-broadcast on the same server that already got the voteOnline + if (targetServer.equalsIgnoreCase(server)) { + continue; + } + if (getConfig().getBlockedServers().contains(targetServer)) { + continue; + } + + globalMessageProxyHandler.sendMessage(targetServer, bDelay, + VotingPluginWire.voteBroadcast(uuid, player, service, time, + text == null ? "" : text.toString(), true)); + bDelay++; + } + } + + // multiproxy: envelope-only clear vote + if (rewardAccepted && getConfig().getMultiProxySupport() && getConfig().getMultiProxyOneGlobalReward()) { + multiProxyHandler.sendClearVote(uuid, player); + } + } else { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addOnlineVote(uuid, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), + broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, + broadcastForwardedServers, false)); + debug("Caching online vote for " + player + " on " + service); + } + + int delay = 2; + for (String s : getAllAvailableServers()) { + globalMessageProxyHandler.sendMessage(s, delay + 1, VotingPluginWire.voteUpdate(uuid, + votePartyVotes, currentVotePartyVotesRequired, service, time, text.toString())); + delay += 2; + } + } + + // Vote logging + if (voteLogMysqlTable != null && getConfig().getVoteLoggingEnabled()) { + voteLogMysqlTable.logVote(voteId, voteStatus, service, uuid, player, time, + getVoteCacheHandler().getProxyCachedTotal(uuid)); + } + + // =========================== + // Multiproxy forwarding + // =========================== + if (getConfig().getMultiProxySupport() && getConfig().getPrimaryServer()) { + if (!getConfig().getMultiProxyOneGlobalReward()) { + debug("Sending global proxy vote envelope"); + multiProxyHandler.sendMultiProxyEnvelope(VotingPluginWire.vote(player, uuid, service, time, false, + realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); + } else { + // Only send to other proxies if the player DID NOT already receive reward on a + // backend + boolean shouldSend = true; + if (playerOnline && playerServer != null) { + if (!getConfig().getBlockedServers().contains(playerServer)) { + shouldSend = false; + } + } + + if (shouldSend) { + debug("Sending global proxy voteonline envelope"); + multiProxyHandler + .sendMultiProxyEnvelope(VotingPluginWire.voteOnline(player, uuid, service, time, false, + realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); + } else { + debug("Not sending global proxy message for voteonline, player already got reward"); + } + } + } + if (queuedVote != null) { + queuedVote.setProcessed(true); + if (!getVoteCacheHandler().updateTimeVote(queuedVote)) { + warn("Unable to persist completed rollover vote " + queuedVote.getVoteId() + + "; attempting durable removal immediately"); + } + } + return QueuedVoteResult.SUCCESS; + } catch (Exception e) { + e.printStackTrace(); + return QueuedVoteResult.RETRY; + } + } + + private static final class PendingPresenceHandoff { + private UUID requestId; + private final UUID playerUuid; + private final String playerName; + private final String uuid; + private final String server; + private final UUID connectionId; + private final UUID backendIncarnationId; + private final long backendStartedAt; + private final long conflictSequence; + private final long createdAt; + + private PendingPresenceHandoff(String playerName, String uuid, String server, UUID connectionId, + UUID backendIncarnationId, long backendStartedAt, long conflictSequence, long createdAt) { + this.playerUuid = parsePlayerUuid(uuid); + this.playerName = playerName; + this.uuid = uuid; + this.server = server; + this.connectionId = connectionId; + this.backendIncarnationId = backendIncarnationId; + this.backendStartedAt = backendStartedAt; + this.conflictSequence = conflictSequence; + this.createdAt = createdAt; + } + + private static UUID parsePlayerUuid(String uuid) { + try { + return UUID.fromString(uuid.trim()); + } catch (Exception ignored) { + return null; + } + } + } + + public abstract void warn(String message); + + public abstract ScheduledExecutorService getScheduler(); +} From 0e9cf5ac05a35eae30c929653f3a1d320aeb04b8 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 21:18:00 -0600 Subject: [PATCH 023/136] fix(test): disambiguate transport validation assertion --- .../backendproxy/transport/HttpBackendProxyTransportTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index dd2fa418a..5c1a68650 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -47,7 +47,7 @@ void validatesInitialConnectionCodeSynchronously() { "invalid configuration must fail before the enrollment worker starts"); when(settings.getHttpConnectionCode()).thenReturn(code("lobby-1", Instant.now().plusSeconds(60)).encode()); - assertDoesNotThrow(transport::validate); + assertDoesNotThrow(() -> transport.validate()); } @Test From f81aa90ba66e32f508e2dd8f5c342c563004c54d Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 21:30:53 -0600 Subject: [PATCH 024/136] fix(http): validate readiness off the server thread --- .../votingplugin/VotingPluginMain.java | 70 +++++++++++++++---- .../http/HttpBackendTransportConnector.java | 9 +++ .../http/HttpClientCredentialStore.java | 7 +- .../backendproxy/http/HttpTlsIdentity.java | 8 ++- .../http/HttpTransportProtocol.java | 11 ++- .../transport/HttpBackendProxyTransport.java | 6 +- .../control/BackendControlConnector.java | 46 ++++++++++-- .../http/HttpTransportRuntimeTest.java | 27 +++++++ .../HttpBackendProxyTransportTest.java | 6 +- 9 files changed, 163 insertions(+), 27 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index a2d243fa5..26d26f3ef 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1214,34 +1214,74 @@ public String getBackendHostedControlStatus() { } /** Recreates proxy transports after Control applies BungeeSettings.yml. */ - public synchronized void restartBackendProxyHandler() { + public void restartBackendProxyHandler() { restartBackendProxyHandler(System.nanoTime() + TimeUnit.SECONDS.toNanos(25)); } /** Recreates proxy transports while preserving the caller's end-to-end validation deadline. */ - public synchronized void restartBackendProxyHandler(long validationDeadlineNanos) { + public void restartBackendProxyHandler(long validationDeadlineNanos) { + BackendProxyRestart restart = prepareBackendProxyHandlerRestart(); + try { + validateBackendProxyHandlerRestart(restart, validationDeadlineNanos); + completeBackendProxyHandlerRestart(restart); + } catch (RuntimeException failure) { + abortBackendProxyHandlerRestart(restart); + throw failure; + } + } + + /** Prepared on the Bukkit thread, validated off-thread, then atomically published on Bukkit. */ + public static final class BackendProxyRestart { + private final BackendProxyHandler previous; + private final BackendProxyHandler replacement; + private final boolean disabled; + private boolean finished; + + private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler replacement, boolean disabled) { + this.previous = previous; + this.replacement = replacement; + this.disabled = disabled; + } + } + + public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { BackendProxyHandler previous = backendProxyHandler; if (!bungeeSettings.isUseBungeecoord()) { - backendProxyHandler = null; - if (previous != null) previous.close(); - BackendControlAutoEnrollment enrollment = backendControlAutoEnrollment; - backendControlAutoEnrollment = null; - if (enrollment != null) enrollment.close(); - return; + return new BackendProxyRestart(previous, null, true); } BungeeMethod replacementMethod = BungeeMethod.getByName(bungeeSettings.getBungeeMethod()); if (previous != null) previous.prepareForReplacement(replacementMethod); BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); try { replacement.load(); - replacement.validateTransport(validationDeadlineNanos); - if (previous != null) previous.completeRedisHandoff(replacement); } catch (RuntimeException failure) { replacement.close(); throw failure; } - backendProxyHandler = replacement; - if (previous != null) previous.close(); + return new BackendProxyRestart(previous, replacement, false); + } + + public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long validationDeadlineNanos) { + if (restart == null) throw new IllegalArgumentException("Backend proxy restart is required"); + if (restart.replacement != null) restart.replacement.validateTransport(validationDeadlineNanos); + } + + public synchronized void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { + if (restart == null || restart.finished) throw new IllegalStateException("Backend proxy restart is no longer active"); + if (backendProxyHandler != restart.previous) throw new IllegalStateException("Backend proxy handler changed during restart"); + if (restart.disabled) { + backendProxyHandler = null; + if (restart.previous != null) restart.previous.close(); + BackendControlAutoEnrollment enrollment = backendControlAutoEnrollment; + backendControlAutoEnrollment = null; + if (enrollment != null) enrollment.close(); + restart.finished = true; + return; + } + if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); + backendProxyHandler = restart.replacement; + if (restart.previous != null) restart.previous.close(); + restart.finished = true; try { refreshBackendControlAutoEnrollment(); } catch (IOException e) { @@ -1249,6 +1289,12 @@ public synchronized void restartBackendProxyHandler(long validationDeadlineNanos } } + public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart restart) { + if (restart == null || restart.finished) return; + if (restart.replacement != null) restart.replacement.close(); + restart.finished = true; + } + /** Keeps one plugin-message listener for the plugin lifetime and atomically swaps its active backend handler. */ public synchronized void activateBackendPluginMessageHandler(GlobalMessageHandler target) { backendPluginMessageTarget.set(target); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 820d4ab5e..f05f5eb61 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -21,6 +21,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadFactory; @@ -45,6 +46,7 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private final URI transportEndpoint; private final ThreadPoolExecutor callbackExecutor; private final AtomicBoolean running = new AtomicBoolean(); + private final CountDownLatch firstResponse = new CountDownLatch(1); private final Object state = new Object(); private final Object renewal = new Object(); private final LinkedHashMap outgoing = new LinkedHashMap<>(); @@ -132,6 +134,11 @@ public void start() { if (!running.compareAndSet(false, true)) return; poller = new Thread(this::pollLoop, "VotingPlugin-HTTP-poll"); poller.setDaemon(true); poller.start(); } + /** Waits for one authenticated, protocol-valid transport response. */ + public boolean awaitFirstResponse(long deadlineNanos) throws InterruptedException { + long remaining = deadlineNanos - System.nanoTime(); + return remaining > 0L && firstResponse.await(remaining, TimeUnit.NANOSECONDS) && running.get(); + } /** * Inserts an in-memory at-least-once delivery. It survives retry/lost responses while this process remains alive; * callers needing restart durability must retain the application operation independently. @@ -167,12 +174,14 @@ public synchronized boolean pollOnce() { acknowledgementsConfirmed = true; synchronized (state) { for (String ack : packet.acks()) outgoing.remove(ack); } for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); + firstResponse.countDown(); return true; } catch (Exception failure) { return false; } finally { if (!acknowledgementsConfirmed) requeueAcknowledgements(acks); } } @Override public void close() { running.getAndSet(false); + firstResponse.countDown(); // Revoke this connector's journal writer before a replacement snapshots it. // In-flight transitions serialize with seal(): either COMPLETED is already // durable, or the delivery remains durably RUNNING and fail-closed. diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index b100cc5a7..518cf4ee7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -97,8 +97,13 @@ static StagedCredential stageReplacement(Path directory, HttpTlsIdentity.IssuedC private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClientCertificate issued, HttpClientProfile profile, String connectionCodeDigest) throws Exception { if (directory == null || issued == null) throw new IllegalArgumentException("Credential replacement is required"); - Path generations = directory.toAbsolutePath().normalize().resolve(GENERATIONS_DIRECTORY); + Path credentialDirectory = directory.toAbsolutePath().normalize(); + boolean created = !Files.exists(credentialDirectory, LinkOption.NOFOLLOW_LINKS); + Path generations = credentialDirectory.resolve(GENERATIONS_DIRECTORY); Files.createDirectories(generations); + // Credential files cannot make the newly created credential-root entry durable. + // Persist its parent before an enrolled transport can activate this root. + if (created) DurableFiles.forceDirectory(credentialDirectory.getParent()); if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP credential generation directory is unsafe"); String name = java.util.UUID.randomUUID().toString(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java index 41522eacc..5a61daefc 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -86,7 +86,13 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock if (advertisedHost == null || advertisedHost.isBlank() || advertisedHost.length() > 253) throw new IllegalArgumentException("Advertised HTTPS host is invalid"); if (clock == null) throw new IllegalArgumentException("Clock is required"); - Files.createDirectories(directory); + Path identityDirectory = directory.toAbsolutePath().normalize(); + boolean created = !Files.exists(identityDirectory, LinkOption.NOFOLLOW_LINKS); + Files.createDirectories(identityDirectory); + // The identity files cannot make the newly created directory entry durable. + // Persist its parent before the TLS identity is returned for listener use. + if (created) DurableFiles.forceDirectory(identityDirectory.getParent()); + directory = identityDirectory; Path caFile = safe(directory.resolve(CA_FILE)); Path serverFile = safe(directory.resolve(SERVER_FILE)); Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java index 3fa01f878..b3c825cdd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java @@ -214,8 +214,15 @@ private static long integer(JsonObject object, String name) { } catch (RuntimeException failure) { throw bad(); } } private static long nonNegative(JsonObject object, String name) { long n = integer(object, name); if (n < 0) throw bad(); return n; } - private static String uuid(JsonObject object, String name) { String value = string(object, name, 64); try { return UUID.fromString(value).toString(); } catch (IllegalArgumentException invalid) { throw bad(); } } - private static void validId(String id) { if (id == null || id.length() > 64) throw bad(); try { UUID.fromString(id); } catch (IllegalArgumentException invalid) { throw bad(); } } + private static String uuid(JsonObject object, String name) { return canonicalUuid(string(object, name, 64)); } + private static void validId(String id) { if (id == null || id.length() > 64) throw bad(); canonicalUuid(id); } + private static String canonicalUuid(String value) { + try { + UUID parsed = UUID.fromString(value); + if (!parsed.toString().equals(value)) throw bad(); + return value; + } catch (IllegalArgumentException invalid) { throw bad(); } + } private static IllegalArgumentException bad() { return new IllegalArgumentException("Invalid HTTP transport message"); } record Delivery(String id, JsonEnvelope envelope) { } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 944557439..2f5566382 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -65,12 +65,16 @@ private void initialize(Path directory, String serverId, String configuredCode, if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); replacement = new HttpBackendTransportConnector(directory, messageHandler::onMessage); + replacement.start(); + if (!replacement.awaitFirstResponse(System.nanoTime() + + TimeUnit.SECONDS.toNanos(DEFAULT_STARTUP_VALIDATION_SECONDS))) { + throw new IllegalStateException("HTTP backend could not authenticate with the proxy"); + } boolean discard = false; synchronized (lifecycle) { if (closed) { discard = true; } else { - replacement.start(); while (!startupQueue.isEmpty()) { if (!replacement.send(startupQueue.removeFirst())) { throw new IllegalStateException("HTTP startup queue could not be transferred"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 00899595c..f2f14567f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -111,26 +111,58 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set } private void reloadConfiguration(String fileName) throws Exception { - Future reload; long validationDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(29); + Future preparation; synchronized (operationLifecycle) { if (closed) throw new IllegalStateException("Bukkit Control connector is stopping"); - reload = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { + preparation = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { plugin.reloadFromControl(); - if ("BungeeSettings.yml".equals(fileName)) plugin.restartBackendProxyHandler(validationDeadline); - return null; + return "BungeeSettings.yml".equals(fileName) ? plugin.prepareBackendProxyHandlerRestart() : null; }); - activeReload = reload; + activeReload = preparation; } + VotingPluginMain.BackendProxyRestart restart = null; try { - reload.get(30, TimeUnit.SECONDS); + restart = preparation.get(remaining(validationDeadline), TimeUnit.NANOSECONDS); + if (restart == null) return; + // Network enrollment/readiness is deliberately awaited on this Control worker, + // never on Bukkit's primary thread. + plugin.validateBackendProxyHandlerRestart(restart, validationDeadline); + VotingPluginMain.BackendProxyRestart prepared = restart; + Future publication = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { + plugin.completeBackendProxyHandlerRestart(prepared); + return null; + }); + synchronized (operationLifecycle) { + if (closed) publication.cancel(true); + activeReload = publication; + } + publication.get(remaining(validationDeadline), TimeUnit.NANOSECONDS); + } catch (Exception failure) { + if (restart != null) { + VotingPluginMain.BackendProxyRestart prepared = restart; + try { + Future abort = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { + plugin.abortBackendProxyHandlerRestart(prepared); + return null; + }); + abort.get(5, TimeUnit.SECONDS); + } catch (Exception cleanupFailure) { failure.addSuppressed(cleanupFailure); } + } + throw failure; } finally { synchronized (operationLifecycle) { - if (activeReload == reload) activeReload = null; + activeReload = null; } } } + private static long remaining(long deadlineNanos) throws java.util.concurrent.TimeoutException { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) throw new java.util.concurrent.TimeoutException("Bukkit configuration reload timed out"); + return remaining; + } + public static BackendControlConnector create(VotingPluginMain plugin) throws IOException { Path root = plugin.getDataFolder().toPath().toAbsolutePath().normalize(); BackendControlResultStore.State recovered = BackendControlResultStore.load(root); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index fbd30301f..3c2658a54 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -38,6 +38,8 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(directory.resolve("client"), envelope -> backendReceived.countDown())) { connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8)), + "an authenticated transport response must make the connector ready"); assertTrue(connector.send(JsonEnvelope.builder("to-proxy").put("server", "forged").build())); assertTrue(proxyReceived.await(8, TimeUnit.SECONDS)); assertEquals("lobby-1", received.get().serverId()); @@ -185,6 +187,31 @@ void packetNumbersMustUseCanonicalJsonIntegerTokens() { } } + @Test + void packetParsingRejectsNoncanonicalUuidForms() { + String deliveryId = java.util.UUID.randomUUID().toString(); + com.google.gson.JsonObject packet = com.google.gson.JsonParser.parseString(new String(HttpTransportProtocol.request( + "lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(deliveryId), + java.util.List.of(new HttpTransportProtocol.Delivery(deliveryId, JsonEnvelope.builder("payload").build()))), + java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); + String abbreviated = "1-1-1-1-1"; + + com.google.gson.JsonObject invalidSession = packet.deepCopy(); + invalidSession.addProperty("session", abbreviated); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidSession.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + com.google.gson.JsonObject invalidAck = packet.deepCopy(); + invalidAck.getAsJsonArray("acks").set(0, new com.google.gson.JsonPrimitive(abbreviated)); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidAck.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + com.google.gson.JsonObject invalidMessage = packet.deepCopy(); + invalidMessage.getAsJsonArray("messages").get(0).getAsJsonObject().addProperty("id", abbreviated); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidMessage.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + @Test void persistedProfileStartsAfterTheEnrollmentCodeExpires() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 5c1a68650..1b8491efd 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -92,7 +92,7 @@ void closeNeverWaitsForSetupOnTheCallingThread() throws Exception { @Test @SuppressWarnings("unchecked") - void validationWaitsForThePreviousDirectoryOwner() throws Exception { + void validationWaitsForThePreviousDirectoryOwnerBeforeReadinessFailure() throws Exception { Path credentials = directory.resolve("http"); HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy-owner"), "proxy.example.test"); HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://proxy.example.test:1297/"), @@ -115,7 +115,7 @@ void validationWaitsForThePreviousDirectoryOwner() throws Exception { java.util.concurrent.atomic.AtomicReference failure = new java.util.concurrent.atomic.AtomicReference<>(); CountDownLatch finished = new CountDownLatch(1); Thread validation = new Thread(() -> { - try { transport.validate(); } + try { transport.validate(System.nanoTime() + TimeUnit.SECONDS.toNanos(1)); } catch (Throwable thrown) { failure.set(thrown); } finally { finished.countDown(); } }); @@ -124,7 +124,7 @@ void validationWaitsForThePreviousDirectoryOwner() throws Exception { predecessor.release(); assertTrue(finished.await(3, TimeUnit.SECONDS)); validation.join(TimeUnit.SECONDS.toMillis(1)); - assertNull(failure.get()); + assertTrue(failure.get() instanceof IllegalStateException); transport.close(); } From b7503b9eb877bc3af8693fca7aff6a4ac1c25840 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 21:41:49 -0600 Subject: [PATCH 025/136] fix(control): cancel stale transport publication --- .../votingplugin/control/BackendControlConnector.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index f2f14567f..1bb3edd2e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -122,6 +122,7 @@ private void reloadConfiguration(String fileName) throws Exception { activeReload = preparation; } VotingPluginMain.BackendProxyRestart restart = null; + Future publication = null; try { restart = preparation.get(remaining(validationDeadline), TimeUnit.NANOSECONDS); if (restart == null) return; @@ -129,7 +130,7 @@ private void reloadConfiguration(String fileName) throws Exception { // never on Bukkit's primary thread. plugin.validateBackendProxyHandlerRestart(restart, validationDeadline); VotingPluginMain.BackendProxyRestart prepared = restart; - Future publication = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { + publication = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { plugin.completeBackendProxyHandlerRestart(prepared); return null; }); @@ -139,6 +140,8 @@ private void reloadConfiguration(String fileName) throws Exception { } publication.get(remaining(validationDeadline), TimeUnit.NANOSECONDS); } catch (Exception failure) { + // A timed-out Bukkit publication must not remain queued ahead of rollback. + if (publication != null) publication.cancel(false); if (restart != null) { VotingPluginMain.BackendProxyRestart prepared = restart; try { From 67e9d1262ad5264c2602800274d5297c1ccc9f4f Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 21:49:09 -0600 Subject: [PATCH 026/136] fix(http): open proxy listener after runtime setup --- .../com/bencodez/votingplugin/proxy/VotingPluginProxy.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 5608a5ce8..1a23fed4a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1576,10 +1576,6 @@ public void onReceive(JsonEnvelope message) { } }); - // Do not accept backend traffic until the router and all of its ordered - // presence/message listeners are installed. - if (method.equals(BungeeMethod.HTTP)) startHttpTransport(); - proxyBroadcastDecider = new ProxyBroadcastDecider(() -> getConfig(), () -> getAllAvailableServers(), s -> isServerValid(s), s -> getConfig().getBlockedServers() != null && getConfig().getBlockedServers().contains(s)); @@ -1592,6 +1588,9 @@ public void onReceive(JsonEnvelope message) { PRESENCE_MAINTENANCE_INTERVAL_SECONDS); } startControlServices(); + // Open the listener last: backend callbacks can immediately reach routing, + // presence, vote-log, multi-proxy, and Control-adjacent runtime helpers. + if (method.equals(BungeeMethod.HTTP)) startHttpTransport(); debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); } From 425664be7cf08f51b693d1f4c68534a04148a6af Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 04:48:52 -0600 Subject: [PATCH 027/136] Restore HTTP transport after failed control reload --- .../votingplugin/VotingPluginMain.java | 15 +++++++++++---- .../backendproxy/BackendProxyHandler.java | 9 ++++++++- .../BackendProxyTransportManager.java | 16 ++++++++++++++++ .../transport/HttpBackendProxyTransport.java | 19 +++++++++++++++++++ .../control/BackendControlConnector.java | 16 ++++++++++++++-- 5 files changed, 68 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 26d26f3ef..867137251 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1235,30 +1235,34 @@ public static final class BackendProxyRestart { private final BackendProxyHandler previous; private final BackendProxyHandler replacement; private final boolean disabled; + private final boolean previousPrepared; private boolean finished; - private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler replacement, boolean disabled) { + private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler replacement, boolean disabled, + boolean previousPrepared) { this.previous = previous; this.replacement = replacement; this.disabled = disabled; + this.previousPrepared = previousPrepared; } } public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { BackendProxyHandler previous = backendProxyHandler; if (!bungeeSettings.isUseBungeecoord()) { - return new BackendProxyRestart(previous, null, true); + return new BackendProxyRestart(previous, null, true, false); } BungeeMethod replacementMethod = BungeeMethod.getByName(bungeeSettings.getBungeeMethod()); - if (previous != null) previous.prepareForReplacement(replacementMethod); + boolean previousPrepared = previous != null && previous.prepareForReplacement(replacementMethod); BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); try { replacement.load(); } catch (RuntimeException failure) { replacement.close(); + if (previousPrepared) previous.restoreAfterFailedReplacement(); throw failure; } - return new BackendProxyRestart(previous, replacement, false); + return new BackendProxyRestart(previous, replacement, false, previousPrepared); } public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long validationDeadlineNanos) { @@ -1292,6 +1296,9 @@ public synchronized void completeBackendProxyHandlerRestart(BackendProxyRestart public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart restart) { if (restart == null || restart.finished) return; if (restart.replacement != null) restart.replacement.close(); + if (restart.previousPrepared && backendProxyHandler == restart.previous) { + restart.previous.restoreAfterFailedReplacement(); + } restart.finished = true; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index f485374bb..10062b023 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -102,10 +102,17 @@ public void close() { } /** Releases a same-method subscriber/listener before its replacement starts. */ - public void prepareForReplacement(BungeeMethod replacementMethod) { + public boolean prepareForReplacement(BungeeMethod replacementMethod) { if (method == replacementMethod && method != BungeeMethod.PLUGINMESSAGING && method != BungeeMethod.REDIS) { transportManager.prepareForReplacement(); + return method == BungeeMethod.HTTP; } + return false; + } + + /** Restores a prepared HTTP transport when its replacement fails validation. */ + public void restoreAfterFailedReplacement() { + transportManager.restorePreparedTransport(); } /** Fails a configuration apply when its selected transport did not initialize. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index fecb46302..6446a23d0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -19,6 +19,7 @@ public class BackendProxyTransportManager { private final VotingPluginMain plugin; private final ProcessedVoteCache processedVoteCache; private BackendProxyTransport transport; + private BackendProxyTransport preparedTransport; public BackendProxyTransportManager(VotingPluginMain plugin) { this(plugin, new ProcessedVoteCache()); @@ -67,6 +68,10 @@ public void close() { transport.close(); transport = null; } + if (preparedTransport != null) { + preparedTransport.close(); + preparedTransport = null; + } } public void validate() { @@ -82,10 +87,21 @@ public void validate(long deadlineNanos) { public void prepareForReplacement() { if (transport != null) { transport.prepareForReplacement(); + preparedTransport = transport; transport = null; } } + public void restorePreparedTransport() { + if (transport != null || preparedTransport == null) return; + if (preparedTransport instanceof HttpBackendProxyTransport http) { + transport = http.recreatePrepared(); + } else { + throw new IllegalStateException("Prepared backend proxy transport cannot be restored"); + } + preparedTransport = null; + } + public void closeRedisForHandoff() { if (!(transport instanceof RedisBackendProxyTransport)) { throw new IllegalStateException("Redis backend proxy transport is unavailable"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 2f5566382..e1dada297 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -29,6 +29,10 @@ public final class HttpBackendProxyTransport implements BackendProxyTransport { private volatile RuntimeException startupFailure; private volatile boolean started; private volatile boolean closed; + private Path configuredDirectory; + private String configuredServerId; + private String configuredConnectionCode; + private GlobalMessageHandler configuredMessageHandler; private Semaphore directoryOwner; private final java.util.concurrent.atomic.AtomicBoolean queueWarning = new java.util.concurrent.atomic.AtomicBoolean(); @@ -41,7 +45,16 @@ public void start(GlobalMessageHandler messageHandler) { Path directory = plugin.getDataFolder().toPath().resolve("http"); String serverId = plugin.getBungeeSettings().getServer(); String connectionCode = plugin.getBungeeSettings().getHttpConnectionCode(); + start(directory, serverId, connectionCode, messageHandler); + } + + private void start(Path directory, String serverId, String connectionCode, + GlobalMessageHandler messageHandler) { validateConfiguration(directory, serverId, connectionCode); + configuredDirectory = directory; + configuredServerId = serverId; + configuredConnectionCode = connectionCode; + configuredMessageHandler = messageHandler; started = true; worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler), "VotingPlugin-HTTP-Backend-Setup"); @@ -49,6 +62,12 @@ public void start(GlobalMessageHandler messageHandler) { worker.start(); } + HttpBackendProxyTransport recreatePrepared() { + HttpBackendProxyTransport restored = new HttpBackendProxyTransport(plugin); + restored.start(configuredDirectory, configuredServerId, configuredConnectionCode, configuredMessageHandler); + return restored; + } + private void initialize(Path directory, String serverId, String configuredCode, GlobalMessageHandler messageHandler) { Path ownerKey = directory.toAbsolutePath().normalize(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 1bb3edd2e..804b0312b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -22,6 +22,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.regex.Pattern; @@ -112,12 +113,20 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set private void reloadConfiguration(String fileName) throws Exception { long validationDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(29); + AtomicBoolean preparationAbandoned = new AtomicBoolean(); + AtomicReference preparedRestart = new AtomicReference<>(); Future preparation; synchronized (operationLifecycle) { if (closed) throw new IllegalStateException("Bukkit Control connector is stopping"); preparation = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { plugin.reloadFromControl(); - return "BungeeSettings.yml".equals(fileName) ? plugin.prepareBackendProxyHandlerRestart() : null; + VotingPluginMain.BackendProxyRestart prepared = "BungeeSettings.yml".equals(fileName) + ? plugin.prepareBackendProxyHandlerRestart() : null; + preparedRestart.set(prepared); + if (prepared != null && preparationAbandoned.get()) { + plugin.abortBackendProxyHandlerRestart(prepared); + } + return prepared; }); activeReload = preparation; } @@ -140,8 +149,11 @@ private void reloadConfiguration(String fileName) throws Exception { } publication.get(remaining(validationDeadline), TimeUnit.NANOSECONDS); } catch (Exception failure) { - // A timed-out Bukkit publication must not remain queued ahead of rollback. + // Timed-out Bukkit work must not remain queued ahead of configuration rollback. + preparationAbandoned.set(true); + preparation.cancel(false); if (publication != null) publication.cancel(false); + if (restart == null) restart = preparedRestart.get(); if (restart != null) { VotingPluginMain.BackendProxyRestart prepared = restart; try { From 57517d7593234c05c7ef7e4718c7a98c7d444f02 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 17:32:42 -0600 Subject: [PATCH 028/136] fix(http): restore credentials after failed reload --- .../votingplugin/VotingPluginMain.java | 69 +++++++++++--- .../backendproxy/BackendProxyHandler.java | 4 + .../http/HttpClientCredentialStore.java | 38 ++++++++ .../BackendProxyTransportManager.java | 4 + .../transport/HttpBackendProxyTransport.java | 62 +++++++++++- .../control/BackendControlConnector.java | 94 +++++++++++++++++-- .../http/HttpTransportSecurityTest.java | 24 +++++ 7 files changed, 270 insertions(+), 25 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 867137251..259f3fa0c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1237,6 +1237,8 @@ public static final class BackendProxyRestart { private final boolean disabled; private final boolean previousPrepared; private boolean finished; + private boolean abandonmentRequested; + private volatile boolean published; private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler replacement, boolean disabled, boolean previousPrepared) { @@ -1247,6 +1249,18 @@ private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler re } } + public static final class BackendProxyRestartPreparationException extends RuntimeException { + private static final long serialVersionUID = 1L; + private final BackendProxyRestart restart; + + private BackendProxyRestartPreparationException(BackendProxyRestart restart, RuntimeException cause) { + super(cause); + this.restart = restart; + } + + public BackendProxyRestart restart() { return restart; } + } + public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { BackendProxyHandler previous = backendProxyHandler; if (!bungeeSettings.isUseBungeecoord()) { @@ -1259,7 +1273,12 @@ public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { replacement.load(); } catch (RuntimeException failure) { replacement.close(); - if (previousPrepared) previous.restoreAfterFailedReplacement(); + if (previousPrepared) { + previous.restoreAfterFailedReplacement(); + BackendProxyRestart failed = new BackendProxyRestart(previous, null, false, true); + failed.finished = true; + throw new BackendProxyRestartPreparationException(failed, failure); + } throw failure; } return new BackendProxyRestart(previous, replacement, false, previousPrepared); @@ -1270,22 +1289,30 @@ public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long if (restart.replacement != null) restart.replacement.validateTransport(validationDeadlineNanos); } - public synchronized void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { - if (restart == null || restart.finished) throw new IllegalStateException("Backend proxy restart is no longer active"); - if (backendProxyHandler != restart.previous) throw new IllegalStateException("Backend proxy handler changed during restart"); - if (restart.disabled) { - backendProxyHandler = null; + public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { + synchronized (this) { + if (restart == null || restart.finished) throw new IllegalStateException("Backend proxy restart is no longer active"); + if (restart.abandonmentRequested) { + abortBackendProxyHandlerRestart(restart); + return; + } + if (backendProxyHandler != restart.previous) throw new IllegalStateException("Backend proxy handler changed during restart"); + if (restart.disabled) { + backendProxyHandler = null; + if (restart.previous != null) restart.previous.close(); + BackendControlAutoEnrollment enrollment = backendControlAutoEnrollment; + backendControlAutoEnrollment = null; + if (enrollment != null) enrollment.close(); + restart.finished = true; + restart.published = true; + return; + } + if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); + backendProxyHandler = restart.replacement; if (restart.previous != null) restart.previous.close(); - BackendControlAutoEnrollment enrollment = backendControlAutoEnrollment; - backendControlAutoEnrollment = null; - if (enrollment != null) enrollment.close(); restart.finished = true; - return; + restart.published = true; } - if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); - backendProxyHandler = restart.replacement; - if (restart.previous != null) restart.previous.close(); - restart.finished = true; try { refreshBackendControlAutoEnrollment(); } catch (IOException e) { @@ -1293,6 +1320,14 @@ public synchronized void completeBackendProxyHandlerRestart(BackendProxyRestart } } + /** Returns false once publication committed and can no longer be rolled back as a failed apply. */ + public synchronized boolean requestBackendProxyHandlerRestartAbandonment(BackendProxyRestart restart) { + if (restart == null) return true; + if (restart.published) return false; + restart.abandonmentRequested = true; + return true; + } + public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart restart) { if (restart == null || restart.finished) return; if (restart.replacement != null) restart.replacement.close(); @@ -1302,6 +1337,12 @@ public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart res restart.finished = true; } + public void awaitBackendProxyHandlerRollback(BackendProxyRestart restart, long deadlineNanos) { + if (restart != null && restart.previousPrepared) { + restart.previous.awaitRestoreAfterFailedReplacement(deadlineNanos); + } + } + /** Keeps one plugin-message listener for the plugin lifetime and atomically swaps its active backend handler. */ public synchronized void activateBackendPluginMessageHandler(GlobalMessageHandler target) { backendPluginMessageTarget.set(target); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 10062b023..ec7998b54 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -115,6 +115,10 @@ public void restoreAfterFailedReplacement() { transportManager.restorePreparedTransport(); } + public void awaitRestoreAfterFailedReplacement(long deadlineNanos) { + transportManager.awaitPreparedTransportRestoration(deadlineNanos); + } + /** Fails a configuration apply when its selected transport did not initialize. */ public void validateTransport() { validateTransport(System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(25)); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index 518cf4ee7..b9b835ef3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -183,12 +183,43 @@ public static boolean matchesEnrollmentCode(Path directory, HttpConnectionCode c /** Loads and cross-checks the persisted client key material and bound normal-transport profile. */ public static EnrolledClient loadEnrolled(Path directory) throws Exception { Path active = activeDirectory(directory); + return loadEnrolledDirectory(active); + } + + private static EnrolledClient loadEnrolledDirectory(Path active) throws Exception { ClientCredential credential = loadCredential(active); HttpClientProfile profile = loadProfileFile(active); if (!matchesProfile(credential, profile)) throw new IOException("HTTP client certificate does not match its profile"); return new EnrolledClient(profile, credential); } + /** Captures the exact credential generation used by a transport before a staged replacement starts. */ + public static ActiveCredentialGeneration snapshotActiveGeneration(Path directory) throws Exception { + Path root = directory.toAbsolutePath().normalize(); + Path active = activeDirectory(root); + loadEnrolledDirectory(active); + return new ActiveCredentialGeneration(active.equals(root) ? "" : active.getFileName().toString()); + } + + /** Atomically restores a previously validated credential generation after replacement rollback. */ + public static void restoreActiveGeneration(Path directory, ActiveCredentialGeneration snapshot) throws Exception { + if (directory == null || snapshot == null) throw new IllegalArgumentException("Credential rollback is required"); + Path root = directory.toAbsolutePath().normalize(); + Path generations = root.resolve(GENERATIONS_DIRECTORY); + Path target = snapshot.name().isEmpty() ? root : generations.resolve(snapshot.name()).normalize(); + if (!snapshot.name().isEmpty() && (!target.getParent().equals(generations) + || Files.isSymbolicLink(target) || !Files.isDirectory(target, LinkOption.NOFOLLOW_LINKS))) + throw new IOException("HTTP client credential generation is invalid"); + loadEnrolledDirectory(target); + Path current = safe(root.resolve(CURRENT_FILE)); + if (snapshot.name().isEmpty()) { + Files.deleteIfExists(current); + DurableFiles.forceDirectory(root); + } else { + writePrivate(current, snapshot.name().getBytes(StandardCharsets.US_ASCII)); + } + } + public record ClientCredential(PrivateKey privateKey, X509Certificate certificate, X509Certificate caCertificate, char[] password) { public ClientCredential { password = password.clone(); } @Override public char[] password() { return password.clone(); } @@ -207,6 +238,13 @@ public record HttpClientProfile(String serverId, URI endpoint, String serverCert public record EnrolledClient(HttpClientProfile profile, ClientCredential credential) { } + public record ActiveCredentialGeneration(String name) { + public ActiveCredentialGeneration { + if (name == null || (!name.isEmpty() && !name.matches("[0-9a-f-]{36}"))) + throw new IllegalArgumentException("HTTP client credential generation is invalid"); + } + } + private static boolean matchesProfile(ClientCredential credential, HttpClientProfile profile) { try { String authorityPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 6446a23d0..309ab7bda 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -102,6 +102,10 @@ public void restorePreparedTransport() { preparedTransport = null; } + public void awaitPreparedTransportRestoration(long deadlineNanos) { + if (transport instanceof HttpBackendProxyTransport http) http.awaitCredentialRestoration(deadlineNanos); + } + public void closeRedisForHandoff() { if (!(transport instanceof RedisBackendProxyTransport)) { throw new IllegalStateException("Redis backend proxy transport is unavailable"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index e1dada297..9651c4203 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -14,6 +14,7 @@ import com.bencodez.votingplugin.backendproxy.http.HttpBackendTransportConnector; import com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore; import com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode; +import com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity; /** Backend adapter for the secure outbound-only HTTP proxy transport. */ public final class HttpBackendProxyTransport implements BackendProxyTransport { @@ -23,16 +24,20 @@ public final class HttpBackendProxyTransport implements BackendProxyTransport { private final VotingPluginMain plugin; private final Object lifecycle = new Object(); private final CountDownLatch startupComplete = new CountDownLatch(1); + private final CountDownLatch credentialRestoreComplete = new CountDownLatch(1); private final ArrayDeque startupQueue = new ArrayDeque<>(); private volatile HttpBackendTransportConnector connector; private volatile Thread worker; private volatile RuntimeException startupFailure; + private volatile RuntimeException credentialRestoreFailure; private volatile boolean started; private volatile boolean closed; private Path configuredDirectory; private String configuredServerId; private String configuredConnectionCode; private GlobalMessageHandler configuredMessageHandler; + private HttpClientCredentialStore.ActiveCredentialGeneration configuredCredentialGeneration; + private HttpClientCredentialStore.ActiveCredentialGeneration credentialGenerationToRestore; private Semaphore directoryOwner; private final java.util.concurrent.atomic.AtomicBoolean queueWarning = new java.util.concurrent.atomic.AtomicBoolean(); @@ -50,11 +55,19 @@ public void start(GlobalMessageHandler messageHandler) { private void start(Path directory, String serverId, String connectionCode, GlobalMessageHandler messageHandler) { - validateConfiguration(directory, serverId, connectionCode); + start(directory, serverId, connectionCode, messageHandler, null); + } + + private void start(Path directory, String serverId, String connectionCode, + GlobalMessageHandler messageHandler, + HttpClientCredentialStore.ActiveCredentialGeneration generationToRestore) { + if (generationToRestore == null) validateConfiguration(directory, serverId, connectionCode); + else HttpTlsIdentity.canonicalServerId(serverId); configuredDirectory = directory; configuredServerId = serverId; configuredConnectionCode = connectionCode; configuredMessageHandler = messageHandler; + credentialGenerationToRestore = generationToRestore; started = true; worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler), "VotingPlugin-HTTP-Backend-Setup"); @@ -64,10 +77,21 @@ private void start(Path directory, String serverId, String connectionCode, HttpBackendProxyTransport recreatePrepared() { HttpBackendProxyTransport restored = new HttpBackendProxyTransport(plugin); - restored.start(configuredDirectory, configuredServerId, configuredConnectionCode, configuredMessageHandler); + restored.start(configuredDirectory, configuredServerId, configuredConnectionCode, configuredMessageHandler, + configuredCredentialGeneration); return restored; } + @Override + public void prepareForReplacement() { + try { + configuredCredentialGeneration = HttpClientCredentialStore.snapshotActiveGeneration(configuredDirectory); + } catch (Exception failure) { + throw new IllegalStateException("Could not preserve the active HTTP client credential", failure); + } + close(); + } + private void initialize(Path directory, String serverId, String configuredCode, GlobalMessageHandler messageHandler) { Path ownerKey = directory.toAbsolutePath().normalize(); @@ -77,7 +101,24 @@ private void initialize(Path directory, String serverId, String configuredCode, try { owner.acquire(); acquired = true; - synchronized (lifecycle) { if (closed) return; } + synchronized (lifecycle) { + if (closed) { + if (credentialGenerationToRestore != null) + credentialRestoreFailure = new IllegalStateException( + "Previous HTTP client credential restoration was cancelled"); + return; + } + } + if (credentialGenerationToRestore != null) { + try { + HttpClientCredentialStore.restoreActiveGeneration(directory, credentialGenerationToRestore); + } catch (Exception failure) { + credentialRestoreFailure = new IllegalStateException( + "Could not restore the previous HTTP client credential", failure); + throw failure; + } + } + credentialRestoreComplete.countDown(); HttpConnectionCode code = enrollmentCode(directory, serverId, configuredCode); if (code != null) HttpBackendTransportConnector.enroll(code, serverId, directory); HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); @@ -109,6 +150,7 @@ private void initialize(Path directory, String serverId, String configuredCode, startupFailure = new IllegalStateException("Secure HTTP backend enrollment or connection failed", failure); plugin.getLogger().severe("Secure HTTP backend transport is unavailable; check the connection code and proxy endpoint"); } finally { + credentialRestoreComplete.countDown(); if (!installed) { if (replacement != null) replacement.close(); if (acquired) owner.release(); @@ -117,6 +159,20 @@ private void initialize(Path directory, String serverId, String configuredCode, } } + void awaitCredentialRestoration(long deadlineNanos) { + if (credentialGenerationToRestore == null) return; + try { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L || !credentialRestoreComplete.await(remaining, TimeUnit.NANOSECONDS)) + throw new IllegalStateException("Previous HTTP client credential restoration timed out"); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Previous HTTP client credential restoration was interrupted", interrupted); + } + RuntimeException failure = credentialRestoreFailure; + if (failure != null) throw failure; + } + @Override public void send(JsonEnvelope envelope) { synchronized (lifecycle) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 804b0312b..e2c4c4009 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -15,6 +15,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; @@ -22,6 +23,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.regex.Pattern; @@ -68,6 +70,7 @@ public final class BackendControlConnector implements AutoCloseable { private final AtomicBoolean running = new AtomicBoolean(); private final AtomicBoolean inspecting = new AtomicBoolean(); private final Object operationLifecycle = new Object(); + private final AtomicReference pendingBackendProxyRollback = new AtomicReference<>(); private final Object journalLifecycle = new Object(); private volatile boolean closed; private volatile boolean registered; @@ -112,21 +115,43 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set } private void reloadConfiguration(String fileName) throws Exception { + finishPendingBackendProxyRollback(); long validationDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(29); AtomicBoolean preparationAbandoned = new AtomicBoolean(); + AtomicInteger preparationState = new AtomicInteger(); + CountDownLatch preparationSettled = new CountDownLatch(1); + CountDownLatch rollbackAbortSettled = new CountDownLatch(1); AtomicReference preparedRestart = new AtomicReference<>(); Future preparation; synchronized (operationLifecycle) { if (closed) throw new IllegalStateException("Bukkit Control connector is stopping"); preparation = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { - plugin.reloadFromControl(); - VotingPluginMain.BackendProxyRestart prepared = "BungeeSettings.yml".equals(fileName) - ? plugin.prepareBackendProxyHandlerRestart() : null; - preparedRestart.set(prepared); - if (prepared != null && preparationAbandoned.get()) { - plugin.abortBackendProxyHandlerRestart(prepared); + try { + if (!preparationState.compareAndSet(0, 1)) return null; + plugin.reloadFromControl(); + VotingPluginMain.BackendProxyRestart prepared; + try { + prepared = "BungeeSettings.yml".equals(fileName) + ? plugin.prepareBackendProxyHandlerRestart() : null; + } catch (VotingPluginMain.BackendProxyRestartPreparationException failure) { + preparedRestart.set(failure.restart()); + throw failure; + } + preparedRestart.set(prepared); + return prepared; + } finally { + try { + VotingPluginMain.BackendProxyRestart prepared = preparedRestart.get(); + if (preparationAbandoned.get()) { + try { + if (prepared != null) plugin.abortBackendProxyHandlerRestart(prepared); + } finally { rollbackAbortSettled.countDown(); } + } + } finally { + preparationState.set(2); + preparationSettled.countDown(); + } } - return prepared; }); activeReload = preparation; } @@ -151,19 +176,49 @@ private void reloadConfiguration(String fileName) throws Exception { } catch (Exception failure) { // Timed-out Bukkit work must not remain queued ahead of configuration rollback. preparationAbandoned.set(true); + boolean abandonedBeforePreparation = preparationState.compareAndSet(0, 3); preparation.cancel(false); + boolean publicationCommitted = publication != null && restart != null + && !plugin.requestBackendProxyHandlerRestartAbandonment(restart); if (publication != null) publication.cancel(false); + // A publication that won the synchronized commit race is the successful + // runtime state; rolling its YAML back would create a split-brain config. + if (publicationCommitted) return; + PendingBackendProxyRollback pendingRollback = null; + if (restart == null && !abandonedBeforePreparation) { + pendingRollback = new PendingBackendProxyRollback(preparationSettled, rollbackAbortSettled, preparedRestart); + pendingBackendProxyRollback.set(pendingRollback); + try { + if (!preparationSettled.await(40, TimeUnit.SECONDS)) + throw new java.util.concurrent.TimeoutException( + "Bukkit configuration preparation did not settle during rollback"); + } catch (Exception preparationFailure) { + failure.addSuppressed(preparationFailure); + throw failure; + } + } if (restart == null) restart = preparedRestart.get(); if (restart != null) { + if (pendingRollback == null) { + pendingRollback = new PendingBackendProxyRollback( + preparationSettled, rollbackAbortSettled, preparedRestart); + pendingBackendProxyRollback.set(pendingRollback); + } VotingPluginMain.BackendProxyRestart prepared = restart; try { Future abort = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { - plugin.abortBackendProxyHandlerRestart(prepared); + try { plugin.abortBackendProxyHandlerRestart(prepared); } + finally { rollbackAbortSettled.countDown(); } return null; }); abort.get(5, TimeUnit.SECONDS); + finishPendingBackendProxyRollback(); } catch (Exception cleanupFailure) { failure.addSuppressed(cleanupFailure); } } + if (pendingRollback != null && restart == null) { + rollbackAbortSettled.countDown(); + pendingBackendProxyRollback.compareAndSet(pendingRollback, null); + } throw failure; } finally { synchronized (operationLifecycle) { @@ -172,6 +227,29 @@ private void reloadConfiguration(String fileName) throws Exception { } } + private void finishPendingBackendProxyRollback() throws Exception { + PendingBackendProxyRollback pending = pendingBackendProxyRollback.get(); + if (pending == null) return; + if (!pending.preparationSettled().await(40, TimeUnit.SECONDS)) + throw new java.util.concurrent.TimeoutException( + "Previous Bukkit configuration preparation is still rolling back"); + VotingPluginMain.BackendProxyRestart restart = pending.preparedRestart().get(); + if (restart != null) { + if (!pending.rollbackAbortSettled().await(40, TimeUnit.SECONDS)) + throw new java.util.concurrent.TimeoutException( + "Previous Bukkit configuration abort is still pending"); + // Keep credential-journal I/O off Bukkit's primary thread and finish it + // before the configuration service starts its automatic backup reload. + plugin.awaitBackendProxyHandlerRollback(restart, + System.nanoTime() + TimeUnit.SECONDS.toNanos(40)); + } + pendingBackendProxyRollback.compareAndSet(pending, null); + } + + private record PendingBackendProxyRollback(CountDownLatch preparationSettled, + CountDownLatch rollbackAbortSettled, + AtomicReference preparedRestart) { } + private static long remaining(long deadlineNanos) throws java.util.concurrent.TimeoutException { long remaining = deadlineNanos - System.nanoTime(); if (remaining <= 0L) throw new java.util.concurrent.TimeoutException("Bukkit configuration reload timed out"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 4c006c02d..5b6a20f17 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -284,5 +284,29 @@ void stagedCredentialDoesNotReplaceActiveGenerationUntilAtomicActivation() throw HttpTransportSecrets.certificatePin(HttpClientCredentialStore.loadEnrolled(client).credential().certificate())); } + @Test + void restoresCredentialGenerationAfterFailedReenrollment() throws Exception { + Path client = directory.resolve("client-rollback"); + HttpTlsIdentity oldIdentity = HttpTlsIdentity.loadOrCreate(directory.resolve("old-proxy"), "old.example.test"); + HttpConnectionCode oldCode = new HttpConnectionCode("lobby-1", URI.create("https://old.example.test:1297/"), + oldIdentity.serverCertificatePin(), oldIdentity.caCertificatePin(), Instant.now().plusSeconds(60), + "R".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, oldCode, oldIdentity.issueClientCertificate("lobby-1")); + HttpClientCredentialStore.ActiveCredentialGeneration previous = + HttpClientCredentialStore.snapshotActiveGeneration(client); + + HttpTlsIdentity replacementIdentity = HttpTlsIdentity.loadOrCreate(directory.resolve("new-proxy"), "new.example.test"); + HttpConnectionCode replacementCode = new HttpConnectionCode("lobby-1", URI.create("https://new.example.test:1297/"), + replacementIdentity.serverCertificatePin(), replacementIdentity.caCertificatePin(), + Instant.now().plusSeconds(60), "S".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, replacementCode, + replacementIdentity.issueClientCertificate("lobby-1")); + assertEquals(replacementCode.endpoint(), HttpClientCredentialStore.loadProfile(client).endpoint()); + + HttpClientCredentialStore.restoreActiveGeneration(client, previous); + assertEquals(oldCode.endpoint(), HttpClientCredentialStore.loadProfile(client).endpoint()); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, oldCode)); + } + private static String pin(char character) { return String.valueOf(character).repeat(64); } } From 9b9232b11033e014a20fcaf4c74c8faf2c65dce4 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 18:43:24 -0600 Subject: [PATCH 029/136] fix(http): preserve independent delivery retries --- .../http/HttpBackendTransportConnector.java | 12 +++ .../http/HttpClientCredentialStore.java | 44 ++++++++++- .../http/HttpProxyTransportServer.java | 57 +++++++++---- .../transport/HttpBackendProxyTransport.java | 9 ++- .../http/HttpTransportRuntimeTest.java | 79 +++++++++++++++++++ .../http/HttpTransportSecurityTest.java | 26 ++++++ 6 files changed, 207 insertions(+), 20 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index f05f5eb61..4370d732d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -189,6 +189,18 @@ public synchronized boolean pollOnce() { Thread current = poller; if (current != null) current.interrupt(); callbackExecutor.shutdown(); try { if (!callbackExecutor.awaitTermination(5, TimeUnit.SECONDS)) callbackExecutor.shutdownNow(); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); callbackExecutor.shutdownNow(); } + // The owning transport may release the credential-directory semaphore as soon + // as close returns. Wait for the interrupted poller so an in-flight renewal + // cannot activate an old credential generation after that ownership handoff. + joinPoller(current); + } + boolean pollerAlive() { Thread current = poller; return current != null && current.isAlive(); } + private static void joinPoller(Thread poller) { + if (poller == null || poller == Thread.currentThread()) return; + boolean interrupted = false; + while (poller.isAlive()) try { poller.join(); } + catch (InterruptedException stopRequested) { interrupted = true; poller.interrupt(); } + if (interrupted) Thread.currentThread().interrupt(); } private void pollLoop() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index b9b835ef3..dc96b9310 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -197,8 +197,33 @@ private static EnrolledClient loadEnrolledDirectory(Path active) throws Exceptio public static ActiveCredentialGeneration snapshotActiveGeneration(Path directory) throws Exception { Path root = directory.toAbsolutePath().normalize(); Path active = activeDirectory(root); - loadEnrolledDirectory(active); - return new ActiveCredentialGeneration(active.equals(root) ? "" : active.getFileName().toString()); + EnrolledClient enrolled = loadEnrolledDirectory(active); + return new ActiveCredentialGeneration(active.equals(root) ? "" : active.getFileName().toString(), + enrolled.profile(), readConnectionCodeDigest(active)); + } + + /** + * Restores a pre-replacement generation unless the replacement already activated a + * newer credential for the same backend endpoint. A successful renewal or same-endpoint + * re-enrollment may revoke the snapshotted certificate at the proxy, so that newer + * generation is the only safe rollback identity. + */ + public static void restoreActiveGenerationAfterReplacement(Path directory, + ActiveCredentialGeneration snapshot) throws Exception { + if (directory == null || snapshot == null) throw new IllegalArgumentException("Credential rollback is required"); + HttpClientProfile previous = snapshot.profile(); + Path active = null; + HttpClientProfile current = null; + if (previous != null) try { + active = activeDirectory(directory); + current = loadEnrolledDirectory(active).profile(); + } catch (Exception unavailable) { active = null; current = null; } + if (active != null && previous.serverId().equals(current.serverId()) + && previous.endpoint().equals(current.endpoint())) { + restoreConnectionCodeDigest(active, snapshot.connectionCodeDigest()); + return; + } + restoreActiveGeneration(directory, snapshot); } /** Atomically restores a previously validated credential generation after replacement rollback. */ @@ -238,10 +263,13 @@ public record HttpClientProfile(String serverId, URI endpoint, String serverCert public record EnrolledClient(HttpClientProfile profile, ClientCredential credential) { } - public record ActiveCredentialGeneration(String name) { + public record ActiveCredentialGeneration(String name, HttpClientProfile profile, String connectionCodeDigest) { + public ActiveCredentialGeneration(String name) { this(name, null, null); } public ActiveCredentialGeneration { if (name == null || (!name.isEmpty() && !name.matches("[0-9a-f-]{36}"))) throw new IllegalArgumentException("HTTP client credential generation is invalid"); + if (connectionCodeDigest != null && !connectionCodeDigest.matches("[0-9a-f]{64}")) + throw new IllegalArgumentException("HTTP connection-code marker is invalid"); } } @@ -299,6 +327,16 @@ private static String connectionCodeDigest(HttpConnectionCode code) { return HttpTransportSecrets.sha256Hex(code.encode().getBytes(StandardCharsets.US_ASCII)); } + private static void restoreConnectionCodeDigest(Path directory, String digest) throws IOException { + Path marker = safe(directory.resolve(CONNECTION_CODE_DIGEST_FILE)); + if (digest == null) { + Files.deleteIfExists(marker); + DurableFiles.forceDirectory(directory); + } else { + writePrivate(marker, digest.getBytes(StandardCharsets.US_ASCII)); + } + } + private static void writePrivate(Path file, byte[] contents) throws IOException { Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index 8e1ecc925..766a741ed 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -37,6 +37,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.function.LongSupplier; import javax.net.ssl.SSLParameters; import javax.net.ssl.SSLPeerUnverifiedException; @@ -236,18 +237,22 @@ static record Response(Collection acks, Collection outgoing = new LinkedHashMap<>(); private final Set seen = new LinkedHashSet<>(); private final Set processing = new LinkedHashSet<>(); private final ArrayDeque acknowledgements = new ArrayDeque<>(); - private final Set delivered = new LinkedHashSet<>(); - private long lastDeliveryNanos; + private final Map deliveredAtNanos = new HashMap<>(); private double requestTokens = 24.0d; private long lastTokenNanos = System.nanoTime(); private boolean activePoll; - BackendState() { this(null, null); } + BackendState() { this(null, null, System::nanoTime); } + BackendState(LongSupplier nanoTime) { this(null, null, nanoTime); } private BackendState(String serverId, DurableOutgoingQueue durableOutgoing) { - this.serverId = serverId; this.durableOutgoing = durableOutgoing; + this(serverId, durableOutgoing, System::nanoTime); + } + private BackendState(String serverId, DurableOutgoingQueue durableOutgoing, LongSupplier nanoTime) { + this.serverId = serverId; this.durableOutgoing = durableOutgoing; this.nanoTime = nanoTime; } private synchronized void restore(Collection deliveries) { for (HttpTransportProtocol.Delivery delivery : deliveries) outgoing.put(delivery.id(), delivery); @@ -259,7 +264,7 @@ private boolean allowRequest() { lastTokenNanos = now; if (requestTokens < 1.0d) return false; requestTokens -= 1.0d; return true; } boolean acceptSession(String requested, long requestedSequence) { - if (!requested.equals(session)) { session = requested; sequence = -1L; delivered.clear(); lastDeliveryNanos = 0L; } + if (!requested.equals(session)) { session = requested; sequence = -1L; deliveredAtNanos.clear(); } // The connector allocates a fresh monotonic sequence for every attempt. Rejecting equality // prevents a captured request from being replayed with altered ACKs or a new payload. if (requestedSequence <= sequence) return false; sequence = requestedSequence; return true; @@ -276,7 +281,7 @@ private void acknowledge(Collection acks) throws IOException { // A 200 response is the backend's proof that its durable replay fence may // be deleted. Never return success while the proxy delivery still exists. if (durableOutgoing != null) durableOutgoing.remove(serverId, id); - outgoing.remove(id); delivered.remove(id); + outgoing.remove(id); deliveredAtNanos.remove(id); } } List acceptIncoming(List received) { @@ -293,24 +298,46 @@ List acceptIncoming(List acks = new java.util.ArrayList<>(); while (!acknowledgements.isEmpty() && acks.size() < HttpTransportProtocol.MAX_BATCH) acks.add(acknowledgements.remove()); List candidates = new java.util.ArrayList<>(); - if (hasUndelivered() || redeliveryDue()) for (HttpTransportProtocol.Delivery delivery : outgoing.values()) { - if (!delivered.contains(delivery.id()) || redeliveryDue()) candidates.add(delivery); + long now = nanoTime.getAsLong(); + if (hasUndelivered() || redeliveryDue(now)) for (HttpTransportProtocol.Delivery delivery : outgoing.values()) { + if (!deliveredAtNanos.containsKey(delivery.id()) || redeliveryDue(delivery.id(), now)) candidates.add(delivery); if (candidates.size() == HttpTransportProtocol.MAX_BATCH) break; } List messages = HttpTransportProtocol.fittingMessages(serverId, requestedSession, requestedSequence, acks, candidates); - for (HttpTransportProtocol.Delivery delivery : messages) delivered.add(delivery.id()); - if (!messages.isEmpty()) lastDeliveryNanos = System.nanoTime(); + long deliveredAt = nanoTime.getAsLong(); + for (HttpTransportProtocol.Delivery delivery : messages) deliveredAtNanos.put(delivery.id(), deliveredAt); return new Response(acks, messages); } - private boolean hasUndelivered() { for (String id : outgoing.keySet()) if (!delivered.contains(id)) return true; return false; } - private boolean redeliveryDue() { return !outgoing.isEmpty() && lastDeliveryNanos > 0L && System.nanoTime() - lastDeliveryNanos >= LONG_POLL.toNanos(); } + private boolean hasUndelivered() { for (String id : outgoing.keySet()) if (!deliveredAtNanos.containsKey(id)) return true; return false; } + private boolean redeliveryDue(long now) { + for (String id : outgoing.keySet()) if (redeliveryDue(id, now)) return true; + return false; + } + private boolean redeliveryDue(String id, long now) { + Long deliveredAt = deliveredAtNanos.get(id); + return deliveredAt != null && now - deliveredAt >= LONG_POLL.toNanos(); + } + private long nanosUntilRedelivery(long now) { + long remaining = Long.MAX_VALUE; + for (String id : outgoing.keySet()) { + Long deliveredAt = deliveredAtNanos.get(id); + if (deliveredAt == null) continue; + long candidate = LONG_POLL.toNanos() - (now - deliveredAt); + if (candidate <= 0L) return 0L; + remaining = Math.min(remaining, candidate); + } + return remaining; + } private synchronized void signal() { notifyAll(); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 9651c4203..8598431c1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -111,7 +111,8 @@ private void initialize(Path directory, String serverId, String configuredCode, } if (credentialGenerationToRestore != null) { try { - HttpClientCredentialStore.restoreActiveGeneration(directory, credentialGenerationToRestore); + HttpClientCredentialStore.restoreActiveGenerationAfterReplacement(directory, + credentialGenerationToRestore); } catch (Exception failure) { credentialRestoreFailure = new IllegalStateException( "Could not restore the previous HTTP client credential", failure); @@ -119,7 +120,11 @@ private void initialize(Path directory, String serverId, String configuredCode, } } credentialRestoreComplete.countDown(); - HttpConnectionCode code = enrollmentCode(directory, serverId, configuredCode); + // A rollback always resumes a validated enrolled generation. Replaying the + // previous temporary code is both unnecessary and unsafe after a same-endpoint + // renewal/re-enrollment retained the newer active generation. + HttpConnectionCode code = enrollmentCode(directory, serverId, + credentialGenerationToRestore == null ? configuredCode : null); if (code != null) HttpBackendTransportConnector.enroll(code, serverId, directory); HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 3c2658a54..0e6a26bd0 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -17,6 +17,7 @@ import java.time.Instant; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -57,6 +58,27 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti } } + @Test + void closeWaitsForTheCredentialOwningPollerToStop() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("close-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("close-authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> { })) { + server.start(); + Path clientDirectory = directory.resolve("close-client"); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(code, "lobby-1", clientDirectory); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { + connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + connector.close(); + assertFalse(connector.pollerAlive(), + "credential-directory ownership must outlive every poller filesystem mutation"); + } + } + } + @Test void normalTransportRejectsAClientWithoutCertificate() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); @@ -315,6 +337,63 @@ void duplicateInboundDeliveryIsReAcknowledgedWithoutSecondDispatch() { assertEquals(java.util.List.of(id), state.await("lobby-1", replacementSession, 0).acks()); } + @Test + void newerDeliveriesDoNotPostponeRetryOfOlderUnacknowledgedDelivery() { + AtomicLong nanoTime = new AtomicLong(1L); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(nanoTime::get); + String session = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery first = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("first").build()); + HttpTransportProtocol.Delivery second = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("second").build()); + + assertTrue(state.acceptSession(session, 0)); + assertTrue(state.enqueue(first)); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 0).messages()); + + nanoTime.addAndGet(TimeUnit.SECONDS.toNanos(1)); + assertTrue(state.acceptSession(session, 1)); + assertTrue(state.enqueue(second)); + assertEquals(java.util.List.of(second), state.await("lobby-1", session, 1).messages()); + + nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(1100)); + assertTrue(state.acceptSession(session, 2)); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 2).messages(), + "sending newer traffic must not reset an older delivery's retry age"); + } + + @Test + void longPollWakesAtTheOldestDeliveryRetryDeadline() throws Exception { + AtomicLong nanoTime = new AtomicLong(1L); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(nanoTime::get); + String session = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery first = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("first").build()); + HttpTransportProtocol.Delivery second = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("second").build()); + assertTrue(state.acceptSession(session, 0)); + assertTrue(state.enqueue(first)); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 0).messages()); + + nanoTime.addAndGet(HttpProxyTransportServer.LONG_POLL.minusMillis(100).toNanos()); + assertTrue(state.acceptSession(session, 1)); + assertTrue(state.enqueue(second)); + assertEquals(java.util.List.of(second), state.await("lobby-1", session, 1).messages()); + assertTrue(state.acceptSession(session, 2)); + Thread clock = new Thread(() -> { + try { Thread.sleep(50L); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(100)); + }, "HTTP-retry-test-clock"); + clock.setDaemon(true); + long started = System.nanoTime(); + clock.start(); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 2).messages()); + clock.join(); + assertTrue(System.nanoTime() - started < TimeUnit.SECONDS.toNanos(1), + "the poll must wake at the oldest delivery deadline, not a fresh long-poll deadline"); + } + @Test void proxyDedupWindowEvictsOldestCompletedDeliveryAtCapacity() { HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 5b6a20f17..662e9e84a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -308,5 +308,31 @@ void restoresCredentialGenerationAfterFailedReenrollment() throws Exception { assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, oldCode)); } + @Test + void rollbackRetainsNewerCredentialForTheSameEndpoint() throws Exception { + Path client = directory.resolve("client-renewal-rollback"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("renewal-proxy"), "renew.example.test"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://renew.example.test:1297/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), + "T".repeat(43)); + HttpTlsIdentity.IssuedClientCertificate original = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, code, original); + HttpClientCredentialStore.ActiveCredentialGeneration previous = + HttpClientCredentialStore.snapshotActiveGeneration(client); + + HttpConnectionCode replacementCode = new HttpConnectionCode("lobby-1", code.endpoint(), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), + "U".repeat(43)); + HttpTlsIdentity.IssuedClientCertificate renewed = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, replacementCode, renewed); + HttpClientCredentialStore.restoreActiveGenerationAfterReplacement(client, previous); + + assertEquals(HttpTransportSecrets.certificatePin(renewed.certificate()), + HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate()), + "rollback must not reactivate a same-endpoint certificate that renewal may have revoked"); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, code), + "the retained credential must recognize the connection code restored in YAML"); + } + private static String pin(char character) { return String.valueOf(character).repeat(64); } } From 59b08052fedcad8437e1ba152f79b6545d747967 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 19:45:39 -0600 Subject: [PATCH 030/136] fix(http): retain rejected vote-party rewards --- .../http/HttpClientCredentialStore.java | 13 +- .../backendproxy/http/HttpConnectionCode.java | 22 +- .../http/HttpProxyTransportServer.java | 69 ++++-- .../http/HttpTransportProtocol.java | 2 +- .../votingplugin/proxy/VotingPluginProxy.java | 203 +++++++++++++++--- .../proxy/bungee/BungeeJsonVoteCache.java | 38 ++++ .../proxy/bungee/VotingPluginBungee.java | 22 ++ .../votingplugin/proxy/cache/IVoteCache.java | 7 + .../proxy/cache/VotePartyCacheDurability.java | 76 +++++++ .../proxy/velocity/VelocityJsonVoteCache.java | 37 ++++ .../proxy/velocity/VotingPluginVelocity.java | 22 ++ .../http/HttpTransportRuntimeTest.java | 29 +++ .../http/HttpTransportSecurityTest.java | 21 +- .../cache/VotePartyCacheDurabilityTest.java | 29 +++ .../tests/VotingPluginProxyTest.java | 49 +++++ .../tests/VotingPluginProxyTestImpl.java | 58 ++++- 16 files changed, 641 insertions(+), 56 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurabilityTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index dc96b9310..9bc247ded 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -176,8 +176,11 @@ public static boolean matchesEnrollmentCode(Path directory, HttpConnectionCode c if (code == null) throw new IllegalArgumentException("Connection code is required"); String stored = readConnectionCodeDigest(activeDirectory(directory)); if (stored == null) return false; - return HttpTransportSecrets.constantTimeEquals(stored.getBytes(StandardCharsets.US_ASCII), - connectionCodeDigest(code).getBytes(StandardCharsets.US_ASCII)); + byte[] storedBytes = stored.getBytes(StandardCharsets.US_ASCII); + return HttpTransportSecrets.constantTimeEquals(storedBytes, + connectionCodeDigest(code.encode()).getBytes(StandardCharsets.US_ASCII)) + || HttpTransportSecrets.constantTimeEquals(storedBytes, + connectionCodeDigest(code.encodeLegacy()).getBytes(StandardCharsets.US_ASCII)); } /** Loads and cross-checks the persisted client key material and bound normal-transport profile. */ @@ -324,7 +327,11 @@ private static String readConnectionCodeDigest(Path directory) throws IOExceptio } private static String connectionCodeDigest(HttpConnectionCode code) { - return HttpTransportSecrets.sha256Hex(code.encode().getBytes(StandardCharsets.US_ASCII)); + return connectionCodeDigest(code.encode()); + } + + private static String connectionCodeDigest(String encoded) { + return HttpTransportSecrets.sha256Hex(encoded.getBytes(StandardCharsets.US_ASCII)); } private static void restoreConnectionCodeDigest(Path directory, String digest) throws IOException { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java index 5700147fb..62bf1ff62 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java @@ -16,7 +16,8 @@ */ public record HttpConnectionCode(String serverId, URI endpoint, String serverCertificatePin, String caCertificatePin, Instant expiresAt, String enrollmentToken) { - private static final String VERSION = "VPH1"; + private static final String LEGACY_VERSION = "VPH1"; + private static final String VERSION = "VPH2"; private static final int MAX_CODE_LENGTH = 4096; public HttpConnectionCode { @@ -29,9 +30,19 @@ public record HttpConnectionCode(String serverId, URI endpoint, String serverCer } public String encode() { + String serverPart = Base64.getUrlEncoder().withoutPadding() + .encodeToString(serverId.getBytes(StandardCharsets.UTF_8)); + return encode(VERSION, serverPart); + } + + String encodeLegacy() { + return encode(LEGACY_VERSION, serverId); + } + + private String encode(String version, String serverPart) { String endpointPart = Base64.getUrlEncoder().withoutPadding() .encodeToString(endpoint.toASCIIString().getBytes(StandardCharsets.UTF_8)); - String unsigned = String.join(".", VERSION, serverId, endpointPart, serverCertificatePin, caCertificatePin, + String unsigned = String.join(".", version, serverPart, endpointPart, serverCertificatePin, caCertificatePin, Long.toString(expiresAt.getEpochSecond()), enrollmentToken); byte[] token = Base64.getUrlDecoder().decode(enrollmentToken); return unsigned + "." + HttpTransportSecrets.hmacSha256Url(token, unsigned); @@ -49,15 +60,18 @@ public static HttpConnectionCode parse(String code) { if (code == null || code.length() > MAX_CODE_LENGTH || code.indexOf('\n') >= 0 || code.indexOf('\r') >= 0) throw new IllegalArgumentException("Connection code is invalid"); String[] parts = code.split("\\.", -1); - if (parts.length != 8 || !VERSION.equals(parts[0])) throw new IllegalArgumentException("Connection code is invalid"); + if (parts.length != 8 || (!VERSION.equals(parts[0]) && !LEGACY_VERSION.equals(parts[0]))) + throw new IllegalArgumentException("Connection code is invalid"); try { + String serverId = LEGACY_VERSION.equals(parts[0]) ? parts[1] + : new String(Base64.getUrlDecoder().decode(parts[1]), StandardCharsets.UTF_8); String endpoint = new String(Base64.getUrlDecoder().decode(parts[2]), StandardCharsets.UTF_8); String unsigned = String.join(".", parts[0], parts[1], parts[2], parts[3], parts[4], parts[5], parts[6]); byte[] token = Base64.getUrlDecoder().decode(parts[6]); String expected = HttpTransportSecrets.hmacSha256Url(token, unsigned); if (!HttpTransportSecrets.constantTimeEquals(expected.getBytes(StandardCharsets.US_ASCII), parts[7].getBytes(StandardCharsets.US_ASCII))) throw new IllegalArgumentException("Connection code is invalid"); - return new HttpConnectionCode(parts[1], new URI(endpoint), parts[3], parts[4], Instant.ofEpochSecond(Long.parseLong(parts[5])), + return new HttpConnectionCode(serverId, new URI(endpoint), parts[3], parts[4], Instant.ofEpochSecond(Long.parseLong(parts[5])), parts[6]); } catch (IllegalArgumentException | URISyntaxException failure) { throw new IllegalArgumentException("Connection code is invalid", failure); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index 766a741ed..4ba34e528 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -22,6 +22,7 @@ import java.security.cert.X509Certificate; import java.time.Duration; import java.util.ArrayDeque; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; @@ -67,21 +68,30 @@ public final class HttpProxyTransportServer implements AutoCloseable { private final Map backends = new HashMap<>(); private final DurableOutgoingQueue durableOutgoing; private final Consumer onEnvelope; + private final DeliveryAcknowledgement onAcknowledged; private volatile boolean closed; public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, Consumer onEnvelope) throws Exception { - this(bind, identity, authority, null, onEnvelope); + this(bind, identity, authority, null, onEnvelope, (serverId, deliveryId) -> { }); } public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, Path outgoingDirectory, Consumer onEnvelope) throws Exception { - if (bind == null || identity == null || authority == null || onEnvelope == null) throw new IllegalArgumentException("HTTP transport configuration is required"); + this(bind, identity, authority, outgoingDirectory, onEnvelope, (serverId, deliveryId) -> { }); + } + + public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Path outgoingDirectory, Consumer onEnvelope, + DeliveryAcknowledgement onAcknowledged) throws Exception { + if (bind == null || identity == null || authority == null || onEnvelope == null || onAcknowledged == null) + throw new IllegalArgumentException("HTTP transport configuration is required"); this.identity = identity; this.authority = authority; this.onEnvelope = onEnvelope; + this.onAcknowledged = onAcknowledged; durableOutgoing = outgoingDirectory == null ? null : new DurableOutgoingQueue(outgoingDirectory); if (durableOutgoing != null) for (Map.Entry> pending : durableOutgoing.load().entrySet()) { - BackendState state = new BackendState(pending.getKey(), durableOutgoing); + BackendState state = new BackendState(pending.getKey(), durableOutgoing, onAcknowledged); state.restore(pending.getValue()); backends.put(pending.getKey(), state); } @@ -126,14 +136,23 @@ private void renew(HttpsExchange exchange) throws IOException { /** Queues a proxy-origin envelope durably before reporting acceptance. */ public boolean send(String serverId, JsonEnvelope envelope) { + return send(serverId, UUID.randomUUID().toString(), envelope); + } + + /** Queues a proxy-origin envelope with a stable, caller-persisted delivery ID. */ + public boolean send(String serverId, String deliveryId, JsonEnvelope envelope) { if (closed || serverId == null || envelope == null) return false; - try { serverId = HttpTlsIdentity.canonicalServerId(serverId); HttpTransportProtocol.validateEnvelope(envelope); } + try { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + HttpTransportProtocol.validId(deliveryId); + HttpTransportProtocol.validateEnvelope(envelope); + } catch (IllegalArgumentException invalid) { return false; } BackendState backend; final String canonicalServerId = serverId; synchronized (backends) { backend = backends.computeIfAbsent(serverId, - ignored -> new BackendState(canonicalServerId, durableOutgoing)); } - return backend.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), envelope)); + ignored -> new BackendState(canonicalServerId, durableOutgoing, onAcknowledged)); } + return backend.enqueue(new HttpTransportProtocol.Delivery(deliveryId, envelope)); } @Override public void close() { @@ -166,7 +185,7 @@ private void transport(HttpsExchange exchange) throws IOException { if (certificate == null || !authority.authenticate(packet.server(), certificate)) { reply(exchange, 401, new byte[0]); return; } BackendState backend; synchronized (backends) { backend = backends.computeIfAbsent(packet.server(), - ignored -> new BackendState(packet.server(), durableOutgoing)); } + ignored -> new BackendState(packet.server(), durableOutgoing, onAcknowledged)); } if (!backend.beginPoll(packet.session())) { reply(exchange, 409, new byte[0]); return; } try { handlePacket(packet, backend); @@ -183,8 +202,9 @@ private void handlePacket(HttpTransportProtocol.Packet packet, BackendState back synchronized (backend) { if (!backend.allowRequest()) throw new IllegalArgumentException("transport rate limited"); if (!backend.acceptSession(packet.session(), packet.sequence())) throw new IllegalArgumentException("stale session request"); - backend.acknowledge(packet.acks()); accepted = backend.acceptIncoming(packet.messages()); } + backend.acknowledge(packet.acks()); + synchronized (backend) { accepted = backend.acceptIncoming(packet.messages()); } for (HttpTransportProtocol.Delivery delivery : accepted) dispatch(packet.server(), backend, delivery); } private void dispatch(String serverId, BackendState backend, HttpTransportProtocol.Delivery delivery) { @@ -233,10 +253,16 @@ private static ThreadPoolExecutor executor(String name, int threads, int queue) private static void shutdown(ExecutorService executor) { executor.shutdown(); try { if (!executor.awaitTermination(5, TimeUnit.SECONDS)) executor.shutdownNow(); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); executor.shutdownNow(); } } public record ReceivedEnvelope(String serverId, String messageId, JsonEnvelope envelope) { } + + @FunctionalInterface + public interface DeliveryAcknowledgement { + void confirm(String serverId, String deliveryId) throws IOException; + } static record Response(Collection acks, Collection messages) { } static final class BackendState { private final String serverId; private final DurableOutgoingQueue durableOutgoing; + private final DeliveryAcknowledgement onAcknowledged; private final LongSupplier nanoTime; private String session; private long sequence = -1L; private final LinkedHashMap outgoing = new LinkedHashMap<>(); @@ -246,13 +272,19 @@ static final class BackendState { private double requestTokens = 24.0d; private long lastTokenNanos = System.nanoTime(); private boolean activePoll; - BackendState() { this(null, null, System::nanoTime); } - BackendState(LongSupplier nanoTime) { this(null, null, nanoTime); } + BackendState() { this(null, null, (serverId, deliveryId) -> { }, System::nanoTime); } + BackendState(LongSupplier nanoTime) { this(null, null, (serverId, deliveryId) -> { }, nanoTime); } private BackendState(String serverId, DurableOutgoingQueue durableOutgoing) { - this(serverId, durableOutgoing, System::nanoTime); + this(serverId, durableOutgoing, (ignoredServer, ignoredDelivery) -> { }, System::nanoTime); } - private BackendState(String serverId, DurableOutgoingQueue durableOutgoing, LongSupplier nanoTime) { - this.serverId = serverId; this.durableOutgoing = durableOutgoing; this.nanoTime = nanoTime; + BackendState(String serverId, DurableOutgoingQueue durableOutgoing, + DeliveryAcknowledgement onAcknowledged) { + this(serverId, durableOutgoing, onAcknowledged, System::nanoTime); + } + private BackendState(String serverId, DurableOutgoingQueue durableOutgoing, + DeliveryAcknowledgement onAcknowledged, LongSupplier nanoTime) { + this.serverId = serverId; this.durableOutgoing = durableOutgoing; + this.onAcknowledged = onAcknowledged; this.nanoTime = nanoTime; } private synchronized void restore(Collection deliveries) { for (HttpTransportProtocol.Delivery delivery : deliveries) outgoing.put(delivery.id(), delivery); @@ -270,18 +302,25 @@ boolean acceptSession(String requested, long requestedSequence) { if (requestedSequence <= sequence) return false; sequence = requestedSequence; return true; } synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { + HttpTransportProtocol.Delivery existing = outgoing.get(delivery.id()); + if (existing != null) return Arrays.equals(HttpTransportProtocol.storedDelivery(existing), + HttpTransportProtocol.storedDelivery(delivery)); if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; if (durableOutgoing != null) try { durableOutgoing.persist(serverId, delivery); } catch (IOException failure) { return false; } outgoing.put(delivery.id(), delivery); signal(); return true; } - private void acknowledge(Collection acks) throws IOException { + void acknowledge(Collection acks) throws IOException { for (String id : acks) { - if (!outgoing.containsKey(id)) continue; + synchronized (this) { if (!outgoing.containsKey(id)) continue; } + onAcknowledged.confirm(serverId, id); + synchronized (this) { + if (!outgoing.containsKey(id)) continue; // A 200 response is the backend's proof that its durable replay fence may // be deleted. Never return success while the proxy delivery still exists. if (durableOutgoing != null) durableOutgoing.remove(serverId, id); outgoing.remove(id); deliveredAtNanos.remove(id); + } } } List acceptIncoming(List received) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java index b3c825cdd..2aafffb33 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java @@ -215,7 +215,7 @@ private static long integer(JsonObject object, String name) { } private static long nonNegative(JsonObject object, String name) { long n = integer(object, name); if (n < 0) throw bad(); return n; } private static String uuid(JsonObject object, String name) { return canonicalUuid(string(object, name, 64)); } - private static void validId(String id) { if (id == null || id.length() > 64) throw bad(); canonicalUuid(id); } + static void validId(String id) { if (id == null || id.length() > 64) throw bad(); canonicalUuid(id); } private static String canonicalUuid(String value) { try { UUID parsed = UUID.fromString(value); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 1a23fed4a..75e81d3c7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -103,6 +103,7 @@ public abstract class VotingPluginProxy { private static final long PRESENCE_MAINTENANCE_INTERVAL_SECONDS = 30L; private static final long PRESENCE_BACKEND_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(90); private static final long CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10); + private static final int MAX_PENDING_VOTE_PARTY_REWARDS = 1024; @Getter @Setter @@ -142,6 +143,7 @@ public abstract class VotingPluginProxy { private boolean timeVoteRetryScheduled; private boolean timeVoteDeliveryRetryScheduled; private boolean cachedVoteDeliveryRetryScheduled; + private boolean votePartyDeliveryRetryScheduled; private boolean enabled; @@ -949,36 +951,77 @@ private synchronized void retryCachedVoteDeliveryPersistence() { } } - public void checkVoteParty() { - if (getConfig().getVotePartyEnabled()) { - if (votePartyVotes >= currentVotePartyVotesRequired) { - debug("Vote party reached"); - addCurrentVotePartyVotes(-currentVotePartyVotesRequired); - - currentVotePartyVotesRequired += getConfig().getVotePartyIncreaseVotesRequired(); - setVoteCacheVotePartyIncreaseVotesRequired( - getVoteCacheVotePartyIncreaseVotesRequired() + getConfig().getVotePartyIncreaseVotesRequired()); - - if (!getConfig().getVotePartyBroadcast().isEmpty()) { - broadcast(getConfig().getVotePartyBroadcast()); - } - - for (String command : getConfig().getVotePartyBungeeCommands()) { - runConsoleCommand(command); - } + public synchronized void checkVoteParty() { + if (!getConfig().getVotePartyEnabled()) return; + if (votePartyVotes < currentVotePartyVotesRequired) { + saveVoteCacheFile(); + return; + } + Collection targets = getConfig().getVotePartySendToAllServers() + ? getAllAvailableServers() : getConfig().getVotePartyServersToSend(); + Map onlineTargets = onlineVotePartyTargets(targets); + if (method == BungeeMethod.HTTP && !canQueueVotePartyRewards(onlineTargets)) { + try { + saveVotePartyStateDurably(); + } catch (IOException failure) { + throw new IllegalStateException("Unable to retain the full HTTP vote-party backlog", failure); + } + return; + } - if (getConfig().getVotePartySendToAllServers()) { - for (String server : getAllAvailableServers()) { - sendVoteParty(server); - } - } else { - for (String server : getConfig().getVotePartyServersToSend()) { - sendVoteParty(server); - } - } + Map stagedRewards = new LinkedHashMap<>(); + if (method == BungeeMethod.HTTP) { + for (String canonicalServer : onlineTargets.keySet()) { + String deliveryId = UUID.randomUUID().toString(); + setVoteCachePendingVotePartyReward(canonicalServer, deliveryId, true); + stagedRewards.put(canonicalServer, deliveryId); + } + } + int previousVotes = votePartyVotes; + int previousRequired = currentVotePartyVotesRequired; + int previousIncrease = getVoteCacheVotePartyIncreaseVotesRequired(); + debug("Vote party reached"); + addCurrentVotePartyVotes(-currentVotePartyVotesRequired); + currentVotePartyVotesRequired += getConfig().getVotePartyIncreaseVotesRequired(); + setVoteCacheVotePartyIncreaseVotesRequired( + previousIncrease + getConfig().getVotePartyIncreaseVotesRequired()); + try { + if (method == BungeeMethod.HTTP) saveVotePartyStateDurably(); + else saveVoteCacheFile(); + } catch (IOException | RuntimeException failure) { + votePartyVotes = previousVotes; + setVoteCacheVotePartyCurrentVotes(previousVotes); + currentVotePartyVotesRequired = previousRequired; + setVoteCacheVotePartyIncreaseVotesRequired(previousIncrease); + for (Map.Entry staged : stagedRewards.entrySet()) + setVoteCachePendingVotePartyReward(staged.getKey(), staged.getValue(), false); + throw failure instanceof RuntimeException runtime ? runtime + : new IllegalStateException("Unable to persist HTTP vote-party rewards", failure); + } + + if (!getConfig().getVotePartyBroadcast().isEmpty()) broadcast(getConfig().getVotePartyBroadcast()); + for (String command : getConfig().getVotePartyBungeeCommands()) runConsoleCommand(command); + if (method == BungeeMethod.HTTP) retryPendingVotePartyRewards(); + else for (String server : targets) sendVoteParty(server); + } + + private Map onlineVotePartyTargets(Collection targets) { + Map online = new LinkedHashMap<>(); + for (String server : targets) if (isSomeoneOnlineServerForVoteRouting(server)) + online.putIfAbsent(server.toLowerCase(Locale.ROOT), server); + return online; + } + + private boolean canQueueVotePartyRewards(Map targets) { + for (String server : targets.keySet()) { + Collection pending = getVoteCachePendingVotePartyRewardIds(server); + if (pending != null && pending.size() >= MAX_PENDING_VOTE_PARTY_REWARDS) { + logSevere("HTTP vote-party reward backlog is full for " + targets.get(server) + + "; retaining the vote-party threshold"); + return false; } - saveVoteCacheFile(); } + return true; } public abstract void debug(String str); @@ -1186,6 +1229,12 @@ protected int[] getProjectedVotePartyState(int acceptedVotes) { public abstract int getVoteCacheVotePartyIncreaseVotesRequired(); + public abstract Collection getVoteCachePendingVotePartyServers(); + + public abstract Collection getVoteCachePendingVotePartyRewardIds(String server); + + public abstract void saveVotePartyStateDurably() throws IOException; + public abstract boolean isPlayerOnline(String playerName); /** @@ -1590,7 +1639,10 @@ public void onReceive(JsonEnvelope message) { startControlServices(); // Open the listener last: backend callbacks can immediately reach routing, // presence, vote-log, multi-proxy, and Control-adjacent runtime helpers. - if (method.equals(BungeeMethod.HTTP)) startHttpTransport(); + if (method.equals(BungeeMethod.HTTP)) { + startHttpTransport(); + scheduleVotePartyDeliveryRetry(); + } debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); } @@ -2697,6 +2749,11 @@ private synchronized boolean sendHttpEnvelope(String server, JsonEnvelope envelo return transport != null && transport.send(server, envelope); } + protected synchronized boolean sendHttpEnvelope(String server, String deliveryId, JsonEnvelope envelope) { + HttpProxyTransportServer transport = httpTransportServer; + return transport != null && transport.send(server, deliveryId, envelope); + } + private void startHttpTransport() { try { URI endpoint = URI.create(getConfig().getHttpPublicEndpoint()); @@ -2711,7 +2768,8 @@ private void startHttpTransport() { httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); httpTransportServer = new HttpProxyTransportServer( new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, - httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope); + httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope, + this::acknowledgeVotePartyDelivery); httpTransportServer.start(); logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" + httpTransportServer.port() + "; use /votingpluginbungee httpcode for each backend"); @@ -2971,9 +3029,90 @@ public void sendServerNameMessage() { } } - public void sendVoteParty(String server) { - if (isSomeoneOnlineServerForVoteRouting(server)) { + public synchronized void sendVoteParty(String server) { + if (!isSomeoneOnlineServerForVoteRouting(server)) return; + if (method != BungeeMethod.HTTP) { globalMessageProxyHandler.sendMessage(server, 1, VotingPluginWire.votePartyBungee()); + return; + } + Collection pending = getVoteCachePendingVotePartyRewardIds(server); + if (pending != null && pending.size() >= MAX_PENDING_VOTE_PARTY_REWARDS) { + logSevere("HTTP vote-party reward backlog is full for " + server); + return; + } + String deliveryId = UUID.randomUUID().toString(); + // Persist intent before the bounded HTTP queue is attempted. A rejection or + // restart therefore leaves a retryable reward instead of silently losing it. + setVoteCachePendingVotePartyReward(server, deliveryId, true); + try { + saveVotePartyStateDurably(); + } catch (IOException failure) { + setVoteCachePendingVotePartyReward(server, deliveryId, false); + throw new IllegalStateException("Unable to persist HTTP vote-party reward", failure); + } + retryPendingVotePartyRewards(); + } + + protected synchronized void retryPendingVotePartyRewards() { + if (!enabled || method != BungeeMethod.HTTP) return; + boolean retryRequired = false; + Collection servers = getVoteCachePendingVotePartyServers(); + if (servers == null) return; + for (String server : new ArrayList<>(servers)) { + Collection pendingIds = getVoteCachePendingVotePartyRewardIds(server); + if (pendingIds == null || pendingIds.isEmpty()) continue; + String routingServer = resolveVotePartyRoutingServer(server); + for (String deliveryId : new ArrayList<>(pendingIds)) { + if (!isSomeoneOnlineServerForVoteRouting(routingServer) + || !sendHttpEnvelope(routingServer, deliveryId, VotingPluginWire.votePartyBungee())) { + retryRequired = true; + break; + } + retryRequired = true; + } + } + if (retryRequired) scheduleVotePartyDeliveryRetry(); + } + + private String resolveVotePartyRoutingServer(String canonicalServer) { + for (String configuredServer : getAllAvailableServers()) { + if (configuredServer.equalsIgnoreCase(canonicalServer)) return configuredServer; + } + return canonicalServer; + } + + protected synchronized void acknowledgeVotePartyDelivery(String server, String deliveryId) throws IOException { + Collection pendingServers = getVoteCachePendingVotePartyServers(); + if (pendingServers == null) return; + for (String pendingServer : new ArrayList<>(pendingServers)) { + if (!pendingServer.equalsIgnoreCase(server)) continue; + Collection pending = getVoteCachePendingVotePartyRewardIds(pendingServer); + if (pending == null || !pending.contains(deliveryId)) return; + setVoteCachePendingVotePartyReward(pendingServer, deliveryId, false); + try { + saveVotePartyStateDurably(); + } catch (IOException | RuntimeException failure) { + setVoteCachePendingVotePartyReward(pendingServer, deliveryId, true); + if (failure instanceof IOException ioFailure) throw ioFailure; + throw (RuntimeException) failure; + } + return; + } + } + + private void scheduleVotePartyDeliveryRetry() { + if (!enabled || votePartyDeliveryRetryScheduled || method != BungeeMethod.HTTP || getScheduler() == null) return; + Collection pendingServers = getVoteCachePendingVotePartyServers(); + if (pendingServers == null || pendingServers.isEmpty()) return; + votePartyDeliveryRetryScheduled = true; + try { + getScheduler().schedule(() -> { + synchronized (VotingPluginProxy.this) { votePartyDeliveryRetryScheduled = false; } + retryPendingVotePartyRewards(); + }, 5, TimeUnit.SECONDS); + } catch (RuntimeException failure) { + votePartyDeliveryRetryScheduled = false; + debug("Unable to schedule HTTP vote-party reward retry: " + failure.getMessage()); } } @@ -2997,6 +3136,8 @@ public void setCurrentVotePartyVotes(int amount) { public abstract void setVoteCacheVotePartyIncreaseVotesRequired(int votes); + public abstract void setVoteCachePendingVotePartyReward(String server, String deliveryId, boolean pending); + public void status() { for (String s : getAllAvailableServers()) { if (!isSomeoneOnlineServerForVoteRouting(s)) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java index 0291a0a48..9fd7db463 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java @@ -1,7 +1,11 @@ package com.bencodez.votingplugin.proxy.bungee; import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; import java.util.Collection; +import java.util.Locale; import com.bencodez.simpleapi.file.BungeeJsonFile; import com.bencodez.votingplugin.proxy.OfflineBungeeVote; @@ -128,6 +132,19 @@ public void removeTimedVotes() { public int getVotePartyCache(String server) { return getInt("VoteParty.Cache." + server, 0); } + + @Override + public Collection getPendingVotePartyRewardServers() { + Collection encoded = getKeys("VoteParty.PendingRewards"); + Collection servers = new ArrayList<>(); + if (encoded != null) for (String key : encoded) servers.add(decodeServerKey(key)); + return servers; + } + + @Override + public Collection getPendingVotePartyRewardIds(String server) { + return getKeys("VoteParty.PendingRewards." + encodeServerKey(server)); + } public int getVotePartyCurrentVotes() { return getInt("VoteParty.CurrentVotes", 0); @@ -140,6 +157,27 @@ public int getVotePartyInreaseVotesRequired() { public void setVotePartyCache(String server, int amount) { setInt("VoteParty.Cache." + server, amount); } + + @Override + public void setPendingVotePartyReward(String server, String deliveryId, boolean pending) { + String serverPath = "VoteParty.PendingRewards." + encodeServerKey(server); + String path = serverPath + "." + deliveryId; + if (pending) setBoolean(path, true); + else { + setString(path, null); + Collection remaining = getKeys(serverPath); + if (remaining == null || remaining.isEmpty()) setString(serverPath, null); + } + } + + private static String encodeServerKey(String server) { + return Base64.getUrlEncoder().withoutPadding() + .encodeToString(server.toLowerCase(Locale.ROOT).getBytes(StandardCharsets.UTF_8)); + } + + private static String decodeServerKey(String server) { + return new String(Base64.getUrlDecoder().decode(server), StandardCharsets.UTF_8); + } public void setVotePartyCurrentVotes(int amount) { setInt("VoteParty.CurrentVotes", amount); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java index bd644c474..629fe1642 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java @@ -7,6 +7,7 @@ import java.io.Reader; import java.net.URL; import java.security.CodeSource; +import java.util.Collection; import java.util.HashSet; import java.util.Locale; import java.util.Map.Entry; @@ -623,6 +624,16 @@ public int getVoteCachePrevWeek() { public int getVoteCacheVotePartyIncreaseVotesRequired() { return voteCacheFile.getVotePartyInreaseVotesRequired(); } + + @Override + public Collection getVoteCachePendingVotePartyServers() { + return voteCacheFile.getPendingVotePartyRewardServers(); + } + + @Override + public Collection getVoteCachePendingVotePartyRewardIds(String server) { + return voteCacheFile.getPendingVotePartyRewardIds(server); + } @Override public boolean isPlayerOnline(String playerName) { @@ -670,6 +681,12 @@ public void runConsoleCommand(String command) { public void saveVoteCacheFile() { voteCacheFile.save(); } + + @Override + public void saveVotePartyStateDurably() throws java.io.IOException { + com.bencodez.votingplugin.proxy.cache.VotePartyCacheDurability.saveAndVerify( + new File(getDataFolder(), "votecache.json").toPath(), voteCacheFile); + } @Override public boolean sendPluginMessageData(String server, String channel, byte[] data, boolean queue) { @@ -718,6 +735,11 @@ public void setVoteCacheVotePartyCurrentVotes(int votes) { public void setVoteCacheVotePartyIncreaseVotesRequired(int votes) { voteCacheFile.setVotePartyInreaseVotesRequired(votes); } + + @Override + public void setVoteCachePendingVotePartyReward(String server, String deliveryId, boolean pending) { + voteCacheFile.setPendingVotePartyReward(server, deliveryId, pending); + } @Override public void warn(String message) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java index 03b6e4977..eb9905725 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java @@ -117,6 +117,11 @@ public interface IVoteCache { */ int getVotePartyCache(String server); + /** Returns backend IDs with persisted, undelivered vote-party rewards. */ + Collection getPendingVotePartyRewardServers(); + + Collection getPendingVotePartyRewardIds(String server); + /** * Gets the current vote party votes. * @@ -139,6 +144,8 @@ public interface IVoteCache { */ void setVotePartyCache(String server, int amount); + void setPendingVotePartyReward(String server, String deliveryId, boolean pending); + /** * Sets the current vote party votes. * diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java new file mode 100644 index 000000000..bfcdf3d76 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java @@ -0,0 +1,76 @@ +package com.bencodez.votingplugin.proxy.cache; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import com.bencodez.votingplugin.util.DurableFiles; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +/** Verifies that the cache library actually persisted the vote-party transaction. */ +public final class VotePartyCacheDurability { + private VotePartyCacheDurability() { } + + public static void saveAndVerify(Path file, IVoteCache cache) throws IOException { + Map> expectedPending = pending(cache); + int expectedVotes = cache.getVotePartyCurrentVotes(); + int expectedIncrease = cache.getVotePartyInreaseVotesRequired(); + cache.save(); + DurableFiles.forceFile(file); + DurableFiles.forceDirectory(file.toAbsolutePath().normalize().getParent()); + try { + JsonObject root = JsonParser.parseString(Files.readString(file, StandardCharsets.UTF_8)).getAsJsonObject(); + JsonObject voteParty = object(root.get("VoteParty")); + if (integer(voteParty, "CurrentVotes") != expectedVotes + || integer(voteParty, "IncreaseVotes") != expectedIncrease + || !pending(voteParty).equals(expectedPending)) + throw new IOException("Vote-party cache state was not persisted"); + } catch (IOException failure) { + throw failure; + } catch (RuntimeException invalid) { + throw new IOException("Vote-party cache state is unreadable", invalid); + } + } + + private static Map> pending(IVoteCache cache) { + Map> pending = new HashMap<>(); + var servers = cache.getPendingVotePartyRewardServers(); + if (servers == null) return pending; + for (String server : servers) { + var ids = cache.getPendingVotePartyRewardIds(server); + if (ids != null && !ids.isEmpty()) pending.put(server, new HashSet<>(ids)); + } + return pending; + } + + private static Map> pending(JsonObject voteParty) { + Map> pending = new HashMap<>(); + JsonObject encodedServers = object(voteParty.get("PendingRewards")); + for (Map.Entry server : encodedServers.entrySet()) { + String serverId = new String(Base64.getUrlDecoder().decode(server.getKey()), StandardCharsets.UTF_8); + JsonObject rewards = object(server.getValue()); + Set ids = new HashSet<>(); + for (Map.Entry reward : rewards.entrySet()) + if (reward.getValue().isJsonPrimitive() && reward.getValue().getAsBoolean()) ids.add(reward.getKey()); + if (!ids.isEmpty()) pending.put(serverId, ids); + } + return pending; + } + + private static JsonObject object(JsonElement element) { + return element == null || element.isJsonNull() ? new JsonObject() : element.getAsJsonObject(); + } + + private static int integer(JsonObject object, String name) { + JsonElement value = object.get(name); + return value == null || value.isJsonNull() ? 0 : value.getAsInt(); + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java index 0569de173..b811ff764 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java @@ -1,7 +1,11 @@ package com.bencodez.votingplugin.proxy.velocity; import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; import java.util.Collection; +import java.util.Locale; import com.bencodez.simpleapi.file.velocity.VelocityJSONFile; import com.bencodez.votingplugin.proxy.OfflineBungeeVote; @@ -134,6 +138,19 @@ public int getVotePartyCache(String server) { return getNode("VoteParty", "Cache", server).getInt(0); } + @Override + public Collection getPendingVotePartyRewardServers() { + Collection encoded = getKeys(getNode("VoteParty", "PendingRewards")); + Collection servers = new ArrayList<>(); + if (encoded != null) for (String key : encoded) servers.add(decodeServerKey(key)); + return servers; + } + + @Override + public Collection getPendingVotePartyRewardIds(String server) { + return getKeys(getNode("VoteParty", "PendingRewards", encodeServerKey(server))); + } + @Override public int getVotePartyCurrentVotes() { return getInt(getNode("VoteParty", "CurrentVotes"), 0); @@ -149,6 +166,26 @@ public void setVotePartyCache(String server, int amount) { setPath(amount, "VoteParty", "Cache", server); } + @Override + public void setPendingVotePartyReward(String server, String deliveryId, boolean pending) { + String serverKey = encodeServerKey(server); + if (pending) setPath(true, "VoteParty", "PendingRewards", serverKey, deliveryId); + else { + remove("VoteParty", "PendingRewards", serverKey, deliveryId); + Collection remaining = getKeys(getNode("VoteParty", "PendingRewards", serverKey)); + if (remaining == null || remaining.isEmpty()) remove("VoteParty", "PendingRewards", serverKey); + } + } + + private static String encodeServerKey(String server) { + return Base64.getUrlEncoder().withoutPadding() + .encodeToString(server.toLowerCase(Locale.ROOT).getBytes(StandardCharsets.UTF_8)); + } + + private static String decodeServerKey(String server) { + return new String(Base64.getUrlDecoder().decode(server), StandardCharsets.UTF_8); + } + @Override public void setVotePartyCurrentVotes(int amount) { setPath(amount, "VoteParty", "CurrentVotes"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java index 176c045c6..9202c866e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java @@ -13,6 +13,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.security.CodeSource; +import java.util.Collection; import java.util.HashSet; import java.util.Locale; import java.util.Map.Entry; @@ -714,6 +715,16 @@ public int getVoteCacheVotePartyIncreaseVotesRequired() { return voteCacheFile.getVotePartyInreaseVotesRequired(); } + @Override + public Collection getVoteCachePendingVotePartyServers() { + return voteCacheFile.getPendingVotePartyRewardServers(); + } + + @Override + public Collection getVoteCachePendingVotePartyRewardIds(String server) { + return voteCacheFile.getPendingVotePartyRewardIds(server); + } + @Override public boolean isVoteCacheIgnoreTime() { return voteCacheFile.getNode("Time", "IgnoreTime").getBoolean(); @@ -759,6 +770,11 @@ public void setVoteCacheVotePartyIncreaseVotesRequired(int votes) { voteCacheFile.setVotePartyInreaseVotesRequired(votes); } + @Override + public void setVoteCachePendingVotePartyReward(String server, String deliveryId, boolean pending) { + voteCacheFile.setPendingVotePartyReward(server, deliveryId, pending); + } + @Override public boolean isPlayerOnline(String playerName) { if (playerName == null) { @@ -805,6 +821,12 @@ public void saveVoteCacheFile() { voteCacheFile.save(); } + @Override + public void saveVotePartyStateDurably() throws java.io.IOException { + com.bencodez.votingplugin.proxy.cache.VotePartyCacheDurability.saveAndVerify( + dataDirectory.resolve("votecache.json"), voteCacheFile); + } + @Override public boolean sendPluginMessageData(String serverName, String channelName, byte[] data, boolean queue) { if (!server.getServer(serverName).isPresent()) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 0e6a26bd0..61b8b785b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -58,6 +58,35 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti } } + @Test + void stableProxyDeliveryIdsAreIdempotentAndAcknowledgedBeforeRemoval() throws Exception { + AtomicLong acknowledged = new AtomicLong(); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", null, + (server, deliveryId) -> acknowledged.incrementAndGet()); + String deliveryId = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("vote-party").build()); + assertTrue(state.enqueue(delivery)); + assertTrue(state.enqueue(delivery)); + assertFalse(state.enqueue(new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("different").build()))); + state.acknowledge(java.util.List.of(deliveryId)); + assertEquals(1L, acknowledged.get()); + assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); + } + + @Test + void failedAcknowledgementCallbackRetainsProxyDelivery() throws Exception { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", null, + (server, deliveryId) -> { throw new java.io.IOException("cache save failed"); }); + String deliveryId = java.util.UUID.randomUUID().toString(); + assertTrue(state.enqueue(new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("vote-party").build()))); + assertThrows(java.io.IOException.class, () -> state.acknowledge(java.util.List.of(deliveryId))); + assertEquals(deliveryId, state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0) + .messages().iterator().next().id()); + } + @Test void closeWaitsForTheCredentialOwningPollerToStop() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("close-proxy"), "localhost"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 662e9e84a..fa00d1342 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -41,10 +41,11 @@ void backendResponseReaderRejectsBodiesBeyondTheWireLimit() throws Exception { @Test void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { - HttpConnectionCode original = new HttpConnectionCode("lobby", URI.create("https://Proxy.Example.test:8443/http"), pin('a'), pin('b'), + HttpConnectionCode original = new HttpConnectionCode("lobby.eu", URI.create("https://Proxy.Example.test:8443/http"), pin('a'), pin('b'), Instant.parse("2030-01-01T00:00:00Z"), HttpTransportSecrets.randomToken()); String encoded = original.encode(); HttpConnectionCode parsed = HttpConnectionCode.parse(encoded); + assertEquals("lobby.eu", parsed.serverId()); assertEquals(URI.create("https://proxy.example.test:8443/http/"), parsed.endpoint()); assertEquals(original.serverCertificatePin(), parsed.serverCertificatePin()); char last = encoded.charAt(encoded.length() - 1); @@ -53,6 +54,24 @@ void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { assertThrows(IllegalArgumentException.class, () -> HttpConnectionCode.parse("http://not-a-code")); } + @Test + void legacyConnectionCodesAndConsumedMarkersRemainCompatible() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("legacy-code-proxy"), "proxy.example.test"); + HttpConnectionCode legacy = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), + HttpTransportSecrets.randomToken()); + assertEquals(legacy, HttpConnectionCode.parse(legacy.encodeLegacy())); + + Path client = directory.resolve("legacy-code-client"); + HttpClientCredentialStore.saveEnrolled(client, legacy, identity.issueClientCertificate("lobby")); + Path active = client.resolve("http-transport-client-generations") + .resolve(Files.readString(client.resolve("http-transport-client-current"))); + Files.writeString(active.resolve("http-transport-connection-code.sha256"), + HttpTransportSecrets.sha256Hex(legacy.encodeLegacy().getBytes(java.nio.charset.StandardCharsets.US_ASCII))); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, + HttpConnectionCode.parse(legacy.encodeLegacy()))); + } + @Test void expiredCodesAreNotActive() { HttpConnectionCode code = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test/"), pin('a'), pin('b'), diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurabilityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurabilityTest.java new file mode 100644 index 000000000..c78157c1a --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurabilityTest.java @@ -0,0 +1,29 @@ +package com.bencodez.votingplugin.proxy.cache; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Set; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class VotePartyCacheDurabilityTest { + @TempDir Path directory; + + @Test + void detectsWhenCacheSaveReturnsWithoutUpdatingTheFile() throws Exception { + IVoteCache cache = mock(IVoteCache.class); + when(cache.getPendingVotePartyRewardServers()).thenReturn(Set.of("lobby")); + when(cache.getPendingVotePartyRewardIds("lobby")).thenReturn(Set.of("delivery-id")); + when(cache.getVotePartyCurrentVotes()).thenReturn(3); + when(cache.getVotePartyInreaseVotesRequired()).thenReturn(2); + Path file = directory.resolve("votecache.json"); + Files.writeString(file, "{\"VoteParty\":{\"CurrentVotes\":3,\"IncreaseVotes\":2}}"); + + assertThrows(java.io.IOException.class, () -> VotePartyCacheDurability.saveAndVerify(file, cache)); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 6b3fb402d..12b4faede 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.never; @@ -80,6 +81,54 @@ void testAddCurrentVotePartyVotes() { assertEquals(5, votingPluginProxy.getVotePartyVotes()); } + @Test + void rejectedHttpVotePartyRewardRemainsPendingUntilAccepted() throws Exception { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + + votingPluginProxy.sendVoteParty("Server1"); + assertEquals(1, votingPluginProxy.getVoteCachePendingVotePartyRewardIds("Server1").size()); + String deliveryId = votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(0); + + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + votingPluginProxy.retryPendingVotePartyRewardsForTest(); + assertEquals(deliveryId, votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(1)); + assertEquals(1, votingPluginProxy.getVoteCachePendingVotePartyRewardIds("Server1").size()); + votingPluginProxy.acknowledgeVotePartyDeliveryForTest("server1", deliveryId); + assertEquals(0, votingPluginProxy.getVoteCachePendingVotePartyRewardIds("Server1").size()); + } + + @Test + void failedVotePartyAcknowledgementSaveRestoresPendingMarker() { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + votingPluginProxy.sendVoteParty("Server1"); + String deliveryId = votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(0); + + votingPluginProxy.failNextVoteCacheSave(); + assertThrows(IllegalStateException.class, + () -> votingPluginProxy.acknowledgeVotePartyDeliveryForTest("server1", deliveryId)); + assertTrue(votingPluginProxy.getVoteCachePendingVotePartyRewardIds("Server1").contains(deliveryId)); + } + + @Test + void httpVotePartyStagesEveryTargetBeforeDelivery() { + Mockito.when(votingPluginProxy.getConfig().getVotePartyEnabled()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getVotePartySendToAllServers()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getVotePartyBroadcast()).thenReturn(""); + Mockito.when(votingPluginProxy.getConfig().getVotePartyBungeeCommands()).thenReturn(java.util.List.of()); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + votingPluginProxy.setVotePartyVotes(1); + votingPluginProxy.setCurrentVotePartyVotesRequired(1); + + votingPluginProxy.checkVoteParty(); + + assertEquals(1, votingPluginProxy.getVoteCachePendingVotePartyRewardIds("server1").size()); + assertEquals(1, votingPluginProxy.getVoteCachePendingVotePartyRewardIds("SERVER2").size()); + assertEquals(0, votingPluginProxy.getVotePartyVotes()); + } + @Test void rolloverProjectionIncludesQueuedVotesAndVotePartyThresholds() { Mockito.when(votingPluginProxy.getConfig().getVotePartyEnabled()).thenReturn(true); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index a2d18b1c1..15ea91aef 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -4,6 +4,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.HashMap; +import java.util.Collection; import java.util.List; import java.util.Set; import java.util.UUID; @@ -29,6 +31,9 @@ public class VotingPluginProxyTestImpl extends VotingPluginProxy { private JsonEnvelope lastCommunicationTestEnvelope; private boolean playerOnline = true; private ScheduledExecutorService scheduler; + private boolean failNextVoteCacheSave; + private final java.util.Map> pendingVotePartyRewards = new HashMap<>(); + private final List attemptedVotePartyDeliveryIds = new ArrayList<>(); public List getWarnings() { return warnings; @@ -125,6 +130,17 @@ public int getVoteCacheVotePartyIncreaseVotesRequired() { return 10; } + @Override + public Collection getVoteCachePendingVotePartyServers() { + return new HashSet<>(pendingVotePartyRewards.keySet()); + } + + @Override + public Collection getVoteCachePendingVotePartyRewardIds(String server) { + return new HashSet<>(pendingVotePartyRewards.getOrDefault(server.toLowerCase(java.util.Locale.ROOT), + java.util.Set.of())); + } + @Override public boolean isPlayerOnline(String playerName) { return playerOnline; @@ -161,7 +177,19 @@ public void runConsoleCommand(String command) { @Override public void saveVoteCacheFile() { - // Mocked for testing + if (failNextVoteCacheSave) { + failNextVoteCacheSave = false; + throw new IllegalStateException("vote cache save failed"); + } + } + + @Override + public void saveVotePartyStateDurably() { + saveVoteCacheFile(); + } + + public void failNextVoteCacheSave() { + failNextVoteCacheSave = true; } @Override @@ -226,6 +254,20 @@ public void setVoteEnvelopeDeliveryResult(boolean voteEnvelopeDeliveryResult) { this.voteEnvelopeDeliveryResult = voteEnvelopeDeliveryResult; } + @Override + protected boolean sendHttpEnvelope(String server, String deliveryId, JsonEnvelope envelope) { + attemptedVotePartyDeliveryIds.add(deliveryId); + return voteEnvelopeDeliveryResult; + } + + public List getAttemptedVotePartyDeliveryIds() { + return attemptedVotePartyDeliveryIds; + } + + public void acknowledgeVotePartyDeliveryForTest(String server, String deliveryId) throws java.io.IOException { + acknowledgeVotePartyDelivery(server, deliveryId); + } + @Override protected boolean sendCommunicationTestEnvelopeNow(String server, JsonEnvelope envelope) { lastCommunicationTestEnvelope = envelope; @@ -286,6 +328,10 @@ public void retryPendingTimeBroadcastsForTest(String server) { retryPendingTimeBroadcasts(server); } + public void retryPendingVotePartyRewardsForTest() { + retryPendingVotePartyRewards(); + } + public int[] getProjectedVotePartyStateForTest(int acceptedVotes) { return getProjectedVotePartyState(acceptedVotes); } @@ -364,6 +410,16 @@ public void setVoteCacheVotePartyIncreaseVotesRequired(int votes) { } + @Override + public void setVoteCachePendingVotePartyReward(String server, String deliveryId, boolean pending) { + server = server.toLowerCase(java.util.Locale.ROOT); + if (pending) pendingVotePartyRewards.computeIfAbsent(server, ignored -> new HashSet<>()).add(deliveryId); + else { + java.util.Set rewards = pendingVotePartyRewards.get(server); + if (rewards != null && rewards.remove(deliveryId) && rewards.isEmpty()) pendingVotePartyRewards.remove(server); + } + } + @Override public ScheduledExecutorService getScheduler() { return scheduler; From c10236a620198c9b4fda5690ca7298eeb82404bd Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 19:47:56 -0600 Subject: [PATCH 031/136] fix(http): import vote-party collection helpers --- .../java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 75e81d3c7..f9b2ffeee 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -21,8 +21,10 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.UUID; From 7fec7976a4f84fc384f028bad711230f39b6d1c5 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 19:51:05 -0600 Subject: [PATCH 032/136] test(http): normalize legacy code expiry precision --- .../backendproxy/http/HttpTransportSecurityTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index fa00d1342..02024377b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -58,7 +58,8 @@ void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { void legacyConnectionCodesAndConsumedMarkersRemainCompatible() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("legacy-code-proxy"), "proxy.example.test"); HttpConnectionCode legacy = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test:8443/"), - identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), + identity.serverCertificatePin(), identity.caCertificatePin(), + Instant.now().plusSeconds(60).truncatedTo(java.time.temporal.ChronoUnit.SECONDS), HttpTransportSecrets.randomToken()); assertEquals(legacy, HttpConnectionCode.parse(legacy.encodeLegacy())); From 01dea25564db4f3c980d0780785289f0a5e67ff8 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 20:01:20 -0600 Subject: [PATCH 033/136] fix(http): bound streamed response reads --- .../http/HttpBackendTransportConnector.java | 76 +++++++++++++++++-- .../http/HttpTransportRuntimeTest.java | 28 +++++++ 2 files changed, 96 insertions(+), 8 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 4370d732d..1f5f83990 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -1,12 +1,14 @@ package com.bencodez.votingplugin.backendproxy.http; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.security.KeyStore; @@ -21,8 +23,11 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Flow; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; @@ -335,13 +340,68 @@ private static HttpClient client(HttpClientCredentialStore.HttpClientProfile pro return HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); } - private static LimitedResponse sendLimited(HttpClient client, HttpRequest request) throws IOException, InterruptedException { - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); - try (InputStream body = response.body()) { - long declaredLength = response.headers().firstValueAsLong("Content-Length").orElse(-1L); - if (declaredLength > HttpTransportProtocol.MAX_BODY_BYTES) - throw new IOException("HTTP transport response exceeds its limit"); - return new LimitedResponse(response.statusCode(), readLimited(body)); + static LimitedResponse sendLimited(HttpClient client, HttpRequest request) throws IOException, InterruptedException { + HttpResponse response = client.send(request, + ignored -> new LimitedBodySubscriber(HttpTransportProtocol.MAX_BODY_BYTES)); + long declaredLength = response.headers().firstValueAsLong("Content-Length").orElse(-1L); + if (declaredLength > HttpTransportProtocol.MAX_BODY_BYTES) + throw new IOException("HTTP transport response exceeds its limit"); + return new LimitedResponse(response.statusCode(), response.body()); + } + + private static final class LimitedBodySubscriber implements HttpResponse.BodySubscriber { + private final int maximum; + private final ByteArrayOutputStream body = new ByteArrayOutputStream(); + private final CompletableFuture result = new CompletableFuture<>(); + private Flow.Subscription subscription; + + private LimitedBodySubscriber(int maximum) { + this.maximum = maximum; + } + + @Override + public CompletionStage getBody() { + return result; + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + if (this.subscription != null) { + subscription.cancel(); + return; + } + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(List buffers) { + try { + for (ByteBuffer buffer : buffers) { + if (buffer.remaining() > maximum - body.size()) { + subscription.cancel(); + result.completeExceptionally(new IOException("HTTP transport response exceeds its limit")); + return; + } + byte[] chunk = new byte[buffer.remaining()]; + buffer.get(chunk); + body.writeBytes(chunk); + } + subscription.request(1); + } catch (RuntimeException failure) { + subscription.cancel(); + result.completeExceptionally(failure); + } + } + + @Override + public void onError(Throwable failure) { + result.completeExceptionally(failure); + } + + @Override + public void onComplete() { + result.complete(body.toByteArray()); } } static byte[] readLimited(InputStream body) throws IOException { @@ -350,7 +410,7 @@ static byte[] readLimited(InputStream body) throws IOException { throw new IOException("HTTP transport response exceeds its limit"); return bytes; } - private record LimitedResponse(int statusCode, byte[] body) { } + record LimitedResponse(int statusCode, byte[] body) { } private static SSLContext clientContext(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { String caPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 61b8b785b..96a826c62 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -4,10 +4,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; import static org.junit.jupiter.api.Assertions.assertTrue; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import java.net.InetSocketAddress; +import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; @@ -58,6 +60,32 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti } } + @Test + void responseBodyConsumptionRemainsBoundedByRequestTimeout() throws Exception { + com.sun.net.httpserver.HttpServer server = com.sun.net.httpserver.HttpServer.create( + new InetSocketAddress("localhost", 0), 1); + CountDownLatch release = new CountDownLatch(1); + server.createContext("/stall", exchange -> { + exchange.sendResponseHeaders(200, 8); + try (var output = exchange.getResponseBody()) { + output.write(1); + output.flush(); + try { release.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } + }); + server.start(); + try { + HttpRequest request = HttpRequest.newBuilder(URI.create("http://localhost:" + server.getAddress().getPort() + + "/stall")).timeout(Duration.ofMillis(250)).GET().build(); + assertTimeoutPreemptively(Duration.ofSeconds(2), () -> assertThrows(java.io.IOException.class, + () -> HttpBackendTransportConnector.sendLimited(HttpClient.newHttpClient(), request))); + } finally { + release.countDown(); + server.stop(0); + } + } + @Test void stableProxyDeliveryIdsAreIdempotentAndAcknowledgedBeforeRemoval() throws Exception { AtomicLong acknowledged = new AtomicLong(); From f8113897b202aeb8dbc18c391fabf05fe785e97d Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 20:04:31 -0600 Subject: [PATCH 034/136] fix(http): cancel stalled body reads at deadline --- .../http/HttpBackendTransportConnector.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 1f5f83990..f10132e50 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -8,6 +8,7 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.net.http.HttpTimeoutException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Path; @@ -27,11 +28,13 @@ import java.util.concurrent.CompletionStage; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Flow; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import javax.net.ssl.KeyManagerFactory; @@ -341,8 +344,23 @@ private static HttpClient client(HttpClientCredentialStore.HttpClientProfile pro .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); } static LimitedResponse sendLimited(HttpClient client, HttpRequest request) throws IOException, InterruptedException { - HttpResponse response = client.send(request, + CompletableFuture> exchange = client.sendAsync(request, ignored -> new LimitedBodySubscriber(HttpTransportProtocol.MAX_BODY_BYTES)); + HttpResponse response; + try { + Duration timeout = request.timeout().orElse(CLIENT_TIMEOUT); + response = exchange.get(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (TimeoutException timeout) { + exchange.cancel(true); + throw new HttpTimeoutException("HTTP transport response timed out"); + } catch (InterruptedException interrupted) { + exchange.cancel(true); + throw interrupted; + } catch (ExecutionException failure) { + Throwable cause = failure.getCause(); + if (cause instanceof IOException ioFailure) throw ioFailure; + throw new IOException("HTTP transport request failed", cause); + } long declaredLength = response.headers().firstValueAsLong("Content-Length").orElse(-1L); if (declaredLength > HttpTransportProtocol.MAX_BODY_BYTES) throw new IOException("HTTP transport response exceeds its limit"); From 7495519d61b42f69feccbbd53cf6ffda762a6abb Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:26:29 -0600 Subject: [PATCH 035/136] fix(http): recover incomplete TLS provisioning --- .../backendproxy/http/HttpTlsIdentity.java | 45 +++++++++++++++++-- .../http/HttpTransportSecurityTest.java | 43 ++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java index 5a61daefc..9aead7e8c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -54,6 +54,9 @@ public final class HttpTlsIdentity { private static final String CA_FILE = "http-transport-ca.p12"; private static final String SERVER_FILE = "http-transport-server.p12"; private static final String PASSWORD_FILE = "http-transport-password"; + private static final String INITIALIZING_FILE = "http-transport-initializing"; + private static final String ENROLLMENT_STATE_FILE = "http-transport-clients.properties"; + private static final String OUTGOING_DIRECTORY = "outgoing-v1"; private static final char[] EMPTY_PASSWORD = new char[0]; static final Duration RENEW_BEFORE = Duration.ofDays(30); static final Duration CA_RENEW_BEFORE = Duration.ofDays(365); @@ -96,11 +99,26 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock Path caFile = safe(directory.resolve(CA_FILE)); Path serverFile = safe(directory.resolve(SERVER_FILE)); Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); + Path initializingFile = safe(directory.resolve(INITIALIZING_FILE)); boolean caExists = Files.exists(caFile, LinkOption.NOFOLLOW_LINKS); boolean serverExists = Files.exists(serverFile, LinkOption.NOFOLLOW_LINKS); boolean passwordExists = Files.exists(passwordFile, LinkOption.NOFOLLOW_LINKS); + boolean initializing = Files.exists(initializingFile, LinkOption.NOFOLLOW_LINKS); + boolean anyIdentityFile = caExists || serverExists || passwordExists; + boolean completeIdentity = caExists && serverExists && passwordExists; + boolean persistentTransportState = hasPersistentTransportState(directory); + if (initializing || (anyIdentityFile && !completeIdentity)) { + if (persistentTransportState) + throw new IOException("HTTP TLS identity files are incomplete"); + if (!initializing) writeInitializationMarker(initializingFile); + discardUncommittedIdentity(caFile, serverFile, passwordFile); + caExists = false; + serverExists = false; + passwordExists = false; + initializing = true; + } if (caExists || serverExists || passwordExists) { - if (!(caExists && serverExists && passwordExists)) throw new IOException("HTTP TLS identity files are incomplete"); + if (!completeIdentity) throw new IOException("HTTP TLS identity files are incomplete"); char[] password = readPassword(passwordFile); try { KeyStore ca = load(caFile, password); @@ -137,6 +155,9 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock advertisedHost); } finally { Arrays.fill(password, '\0'); } } + if (persistentTransportState) + throw new IOException("HTTP TLS identity files are missing"); + if (!initializing) writeInitializationMarker(initializingFile); ensureBouncyCastle(); char[] password = HttpTransportSecrets.randomToken().toCharArray(); try { @@ -154,9 +175,10 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock server.setKeyEntry("server", serverPair.getPrivate(), password, new Certificate[] { serverCertificate, caCertificate }); writeStore(caFile, ca, password); writeStore(serverFile, server, password); - byte[] passwordBytes = asciiBytes(password); - try { writePrivate(passwordFile, passwordBytes); } - finally { Arrays.fill(passwordBytes, (byte) 0); } + byte[] passwordBytes = asciiBytes(password); + try { writePrivate(passwordFile, passwordBytes); } + finally { Arrays.fill(passwordBytes, (byte) 0); } + DurableFiles.deleteIfExists(initializingFile); return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password, caFile, serverFile, advertisedHost); } finally { Arrays.fill(password, '\0'); } @@ -391,6 +413,21 @@ private static void writePrivate(Path file, byte[] contents) throws IOException } finally { Files.deleteIfExists(temporary); } } + private static void writeInitializationMarker(Path file) throws IOException { + writePrivate(file, "initializing\n".getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + } + + private static void discardUncommittedIdentity(Path caFile, Path serverFile, Path passwordFile) throws IOException { + DurableFiles.deleteIfExists(caFile); + DurableFiles.deleteIfExists(serverFile); + DurableFiles.deleteIfExists(passwordFile); + } + + private static boolean hasPersistentTransportState(Path directory) { + return Files.exists(directory.resolve(ENROLLMENT_STATE_FILE), LinkOption.NOFOLLOW_LINKS) + || Files.exists(directory.resolve(OUTGOING_DIRECTORY), LinkOption.NOFOLLOW_LINKS); + } + private static byte[] asciiBytes(char[] characters) { byte[] output = new byte[characters.length]; for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 02024377b..f67173889 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -131,6 +131,49 @@ void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { assertNotEquals(created.serverCertificatePin(), rotated.serverCertificatePin()); } + @Test + void incompleteFirstRunTlsProvisioningRecoversWithoutManualCleanup() throws Exception { + Path source = directory.resolve("complete-identity"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(source, "localhost"); + Path interrupted = directory.resolve("interrupted-identity"); + Files.createDirectories(interrupted); + Files.copy(source.resolve("http-transport-ca.p12"), interrupted.resolve("http-transport-ca.p12")); + Files.copy(source.resolve("http-transport-server.p12"), interrupted.resolve("http-transport-server.p12")); + + HttpTlsIdentity recovered = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); + + assertNotEquals(original.caCertificatePin(), recovered.caCertificatePin()); + assertTrue(Files.exists(interrupted.resolve("http-transport-ca.p12"))); + assertTrue(Files.exists(interrupted.resolve("http-transport-server.p12"))); + assertTrue(Files.exists(interrupted.resolve("http-transport-password"))); + assertFalse(Files.exists(interrupted.resolve("http-transport-initializing"))); + } + + @Test + void completedFirstRunFilesRecoverWhenInitializationMarkerSurvives() throws Exception { + Path interrupted = directory.resolve("marked-identity"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); + String originalCaPin = original.caCertificatePin(); + Files.writeString(interrupted.resolve("http-transport-initializing"), "initializing\n"); + + HttpTlsIdentity recovered = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); + + assertNotEquals(originalCaPin, recovered.caCertificatePin()); + assertFalse(Files.exists(interrupted.resolve("http-transport-initializing"))); + } + + @Test + void incompleteEstablishedTlsIdentityFailsClosed() throws Exception { + Path established = directory.resolve("established-identity"); + HttpTlsIdentity.loadOrCreate(established, "localhost"); + Files.writeString(established.resolve("http-transport-clients.properties"), "version=2\n"); + Files.delete(established.resolve("http-transport-server.p12")); + + assertThrows(java.io.IOException.class, () -> HttpTlsIdentity.loadOrCreate(established, "localhost")); + assertTrue(Files.exists(established.resolve("http-transport-ca.p12"))); + assertTrue(Files.exists(established.resolve("http-transport-password"))); + } + @Test void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory, "localhost"); From e23a9691d0525575057de5c1e953853ac27e130f Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:44:24 -0600 Subject: [PATCH 036/136] fix(http): defer replacement presence publication --- .../votingplugin/VotingPluginMain.java | 3 +- .../backendproxy/BackendProxyHandler.java | 28 +++++++++++-- .../presence/BackendPresenceManager.java | 28 ++++++++----- .../BackendProxyHandlerLifecycleTest.java | 39 +++++++++++++++++++ 4 files changed, 83 insertions(+), 15 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 259f3fa0c..201b84640 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1270,7 +1270,7 @@ public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { boolean previousPrepared = previous != null && previous.prepareForReplacement(replacementMethod); BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); try { - replacement.load(); + replacement.loadForReplacement(); } catch (RuntimeException failure) { replacement.close(); if (previousPrepared) { @@ -1308,6 +1308,7 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { return; } if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); + restart.replacement.activatePresenceReporting(); backendProxyHandler = restart.replacement; if (restart.previous != null) restart.previous.close(); restart.finished = true; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index ec7998b54..eb269f08b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -39,6 +39,7 @@ public class BackendProxyHandler implements Listener { private final BackendGlobalDataSync globalDataSync; private BackendPresenceManager presenceManager; + private boolean presenceReportingActivated; private BackendVotePartySync votePartySync; private BackendProxyMessageRouter messageRouter; @@ -62,6 +63,15 @@ public BackendProxyHandler(VotingPluginMain plugin, ProcessedVoteCache processed * Loads the configured backend/proxy communication components. */ public void load() { + load(true); + } + + /** Loads a replacement without announcing a new presence generation before publication. */ + public void loadForReplacement() { + load(false); + } + + private void load(boolean activatePresenceReporting) { plugin.debug("Loading backend proxy handler"); method = BungeeMethod.getByName(plugin.getBungeeSettings().getBungeeMethod()); plugin.getLogger().info("Using BungeeMethod: " + method.toString()); @@ -84,15 +94,24 @@ public void sendMessage(JsonEnvelope envelope) { if (plugin.getOptions().getServer().equalsIgnoreCase("pleaseset")) { plugin.getLogger().warning("Server name for bungee voting is not set, please set it"); } - presenceManager.start(); + if (activatePresenceReporting) activatePresenceReporting(); + } + + /** Starts presence only after a staged handler reaches the atomic publication boundary. */ + public void activatePresenceReporting() { + if (presenceManager != null && !presenceReportingActivated) { + presenceManager.start(); + presenceReportingActivated = true; + } } /** * Closes backend/proxy components and persists cached proxy state. */ public void close() { - if (presenceManager != null) { + if (presenceManager != null && presenceReportingActivated) { presenceManager.stop(); + presenceReportingActivated = false; } transportManager.close(); if (votePartySync != null) { @@ -152,14 +171,15 @@ public void playerOffline(String playerName) { } public void reloadPresenceReporting() { - if (presenceManager != null) { + if (presenceManager != null && presenceReportingActivated) { presenceManager.reload(); } } public void disablePresenceReporting() { - if (presenceManager != null) { + if (presenceManager != null && presenceReportingActivated) { presenceManager.stop(); + presenceReportingActivated = false; } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java index a82c5551a..9b560a3c3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java @@ -67,20 +67,28 @@ public void start() { lastResyncRequestAtNanos = 0L; lastSnapshotRequestId = null; lastSnapshotRequestAtNanos = 0L; - send(VotingPluginWire.backendStarted(server, incarnationId, startedAt, now)); - send(VotingPluginWire.backendHeartbeat(server, incarnationId, startedAt, nextTimestamp())); - if (heartbeatTask != null) { heartbeatTask.cancel(false); } - heartbeatTask = plugin.getTimer().scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - sendHeartbeat(); - } - }, HEARTBEAT_SECONDS, HEARTBEAT_SECONDS, TimeUnit.SECONDS); + try { + heartbeatTask = plugin.getTimer().scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + sendHeartbeat(); + } + }, HEARTBEAT_SECONDS, HEARTBEAT_SECONDS, TimeUnit.SECONDS); + seedOnlinePlayers(); + } catch (RuntimeException failure) { + if (heartbeatTask != null) heartbeatTask.cancel(false); + heartbeatTask = null; + reporting = false; + server = null; + incarnationId = null; + throw failure; + } + send(VotingPluginWire.backendStarted(server, incarnationId, startedAt, now)); + send(VotingPluginWire.backendHeartbeat(server, incarnationId, startedAt, nextTimestamp())); } - seedOnlinePlayers(); } public void stop() { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index b436fadd5..f26bad9de 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -2,9 +2,12 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; import java.lang.reflect.Field; import java.util.concurrent.ScheduledExecutorService; @@ -12,11 +15,14 @@ import org.junit.jupiter.api.Test; import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; import com.bencodez.simpleapi.servercomm.redis.RedisHandler; import com.bencodez.votingplugin.backendproxy.global.BackendGlobalDataSync; import com.bencodez.votingplugin.backendproxy.cache.ProcessedVoteCache; +import com.bencodez.votingplugin.backendproxy.presence.BackendPresenceManager; +import com.bencodez.votingplugin.config.BungeeSettings; import com.bencodez.votingplugin.backendproxy.transport.MqttBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.MysqlBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.BackendProxyTransport; @@ -26,6 +32,39 @@ import com.bencodez.votingplugin.proxy.BungeeMethod; class BackendProxyHandlerLifecycleTest { + @Test + void failedPresenceActivationDoesNotAnnounceReplacementGeneration() { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BungeeSettings settings = mock(BungeeSettings.class); + ScheduledExecutorService timer = mock(ScheduledExecutorService.class); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + when(plugin.getBungeeSettings()).thenReturn(settings); + when(settings.getServer()).thenReturn("lobby"); + when(plugin.getTimer()).thenReturn(timer); + when(timer.scheduleAtFixedRate(org.mockito.ArgumentMatchers.any(Runnable.class), + org.mockito.ArgumentMatchers.anyLong(), org.mockito.ArgumentMatchers.anyLong(), + org.mockito.ArgumentMatchers.any())).thenThrow(new java.util.concurrent.RejectedExecutionException()); + + BackendPresenceManager presence = new BackendPresenceManager(plugin, BungeeMethod.HTTP, messages); + assertThrows(java.util.concurrent.RejectedExecutionException.class, presence::start); + + verifyNoInteractions(messages); + } + + @Test + void stagedPresenceStartsOnlyAtExplicitPublication() throws Exception { + BackendProxyHandler handler = new BackendProxyHandler(null); + BackendPresenceManager presence = mock(BackendPresenceManager.class); + Field field = BackendProxyHandler.class.getDeclaredField("presenceManager"); + field.setAccessible(true); + field.set(handler, presence); + + verifyNoInteractions(presence); + handler.activatePresenceReporting(); + handler.activatePresenceReporting(); + + verify(presence, times(1)).start(); + } @Test void sharesVoteDeduplicationAcrossHandlerReplacement() { From fa1296ea07c77c1eab239f583e1dfed7b1b7bdf4 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:41:08 -0600 Subject: [PATCH 037/136] fix(http): flush presence on backend shutdown --- .../http/HttpBackendTransportConnector.java | 43 +++++++++++++++++-- .../transport/HttpBackendProxyTransport.java | 6 ++- .../http/HttpTransportRuntimeTest.java | 29 +++++++++++++ 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index f10132e50..284c48a9e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -162,17 +162,20 @@ public boolean send(JsonEnvelope envelope) { } /** A synchronous single poll, useful for lifecycle-controlled integrations and tests. */ public synchronized boolean pollOnce() { - if (!running.get()) return false; + return pollOnce(CLIENT_TIMEOUT, true, true); + } + private boolean pollOnce(Duration timeout, boolean requireRunning, boolean acceptIncoming) { + if (requireRunning && !running.get()) return false; List acks = List.of(); boolean acknowledgementsConfirmed = false; try { - maybeRenewCredential(); + if (requireRunning) maybeRenewCredential(); List messages; long requestSequence; synchronized (state) { acks = first(acknowledgements); requestSequence = sequence++; messages = HttpTransportProtocol.fittingMessages(serverId, session, requestSequence, acks, outgoing.values()); for (int index = 0; index < acks.size(); index++) acknowledgements.removeFirst(); } - HttpRequest request = HttpRequest.newBuilder(transportEndpoint).timeout(CLIENT_TIMEOUT).header("Content-Type", "application/json") + HttpRequest request = HttpRequest.newBuilder(transportEndpoint).timeout(timeout).header("Content-Type", "application/json") .header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(HttpTransportProtocol.request(serverId, session, requestSequence, acks, messages))).build(); LimitedResponse response = sendLimited(client, request); if (response.statusCode() != 200) return false; @@ -181,12 +184,29 @@ public synchronized boolean pollOnce() { confirmAcknowledgements(acks); acknowledgementsConfirmed = true; synchronized (state) { for (String ack : packet.acks()) outgoing.remove(ack); } - for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); + if (acceptIncoming) for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); firstResponse.countDown(); return true; } catch (Exception failure) { return false; } finally { if (!acknowledgementsConfirmed) requeueAcknowledgements(acks); } } + /** Stops normal polling and gives already-queued outbound messages a bounded final delivery attempt. */ + public boolean flushOutgoing(long deadlineNanos) { + running.set(false); + firstResponse.countDown(); + Thread current = poller; + if (current != null) current.interrupt(); + if (!joinPoller(current, deadlineNanos)) return false; + while (queuedOutgoing() != 0) { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) return false; + Duration timeout = Duration.ofNanos(Math.min(CLIENT_TIMEOUT.toNanos(), remaining)); + synchronized (this) { + if (!pollOnce(timeout, false, false)) return false; + } + } + return true; + } @Override public void close() { running.getAndSet(false); firstResponse.countDown(); @@ -203,6 +223,21 @@ public synchronized boolean pollOnce() { joinPoller(current); } boolean pollerAlive() { Thread current = poller; return current != null && current.isAlive(); } + private static boolean joinPoller(Thread poller, long deadlineNanos) { + if (poller == null || poller == Thread.currentThread()) return true; + boolean interrupted = false; + while (poller.isAlive()) { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) { + if (interrupted) Thread.currentThread().interrupt(); + return false; + } + try { TimeUnit.NANOSECONDS.timedJoin(poller, remaining); } + catch (InterruptedException stopRequested) { interrupted = true; poller.interrupt(); } + } + if (interrupted) Thread.currentThread().interrupt(); + return true; + } private static void joinPoller(Thread poller) { if (poller == null || poller == Thread.currentThread()) return; boolean interrupted = false; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 8598431c1..ce8d6172d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -20,6 +20,7 @@ public final class HttpBackendProxyTransport implements BackendProxyTransport { private static final int MAX_STARTUP_QUEUE = 1024; private static final long DEFAULT_STARTUP_VALIDATION_SECONDS = 25L; + private static final long SHUTDOWN_FLUSH_SECONDS = 5L; private static final ConcurrentHashMap DIRECTORY_OWNERS = new ConcurrentHashMap<>(); private final VotingPluginMain plugin; private final Object lifecycle = new Object(); @@ -271,7 +272,10 @@ private static void drain(Thread setup, HttpBackendTransportConnector active, Se try { if (setup != null) try { setup.join(TimeUnit.SECONDS.toMillis(5)); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } - if (active != null) active.close(); + if (active != null) { + active.flushOutgoing(System.nanoTime() + TimeUnit.SECONDS.toNanos(SHUTDOWN_FLUSH_SECONDS)); + active.close(); + } } finally { if (owner != null) owner.release(); } } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 96a826c62..e8be931cc 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -136,6 +136,35 @@ void closeWaitsForTheCredentialOwningPollerToStop() throws Exception { } } + @Test + void finalFlushDeliversMessagesQueuedBehindAnActiveLongPoll() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("flush-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("flush-authority")); + CountDownLatch received = new CountDownLatch(1); + AtomicReference envelope = new AtomicReference<>(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, message -> { + envelope.set(message.envelope()); + received.countDown(); + })) { + server.start(); + Path clientDirectory = directory.resolve("flush-client"); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(code, "lobby-1", clientDirectory); + HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { }); + connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + assertTrue(connector.send(JsonEnvelope.builder("backend-stopped").build())); + try { + assertTrue(connector.flushOutgoing(System.nanoTime() + TimeUnit.SECONDS.toNanos(5))); + assertTrue(received.await(1, TimeUnit.SECONDS)); + assertEquals("backend-stopped", envelope.get().getSubChannel()); + assertEquals(0, connector.queuedOutgoing()); + } finally { connector.close(); } + } + } + @Test void normalTransportRejectsAClientWithoutCertificate() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); From bfea6f1b6357e83f2ef7e75b7415980cc8b9c7a1 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:53:46 -0600 Subject: [PATCH 038/136] fix(control): defer plugin-message relay publication --- .../backendproxy/BackendProxyHandler.java | 3 ++ .../transport/BackendProxyTransport.java | 4 +++ .../BackendProxyTransportManager.java | 4 +++ .../PluginMessagingBackendProxyTransport.java | 15 ++++++++-- .../BackendProxyHandlerLifecycleTest.java | 29 +++++++++++++++++++ 5 files changed, 52 insertions(+), 3 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index eb269f08b..6f2f91cf6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -103,6 +103,9 @@ public void activatePresenceReporting() { presenceManager.start(); presenceReportingActivated = true; } + // Swap shared inbound routing only after presence activation succeeds. An + // aborted publication must leave the previous plugin-message target intact. + transportManager.activateAfterPublication(); } /** diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java index 30a0baa00..008affab1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java @@ -19,5 +19,9 @@ default void prepareForReplacement() { close(); } + /** Activates transport state that must not become visible before handler publication. */ + default void activateAfterPublication() { + } + void close(); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 309ab7bda..7b6ae8d31 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -63,6 +63,10 @@ public void send(JsonEnvelope envelope) { } } + public void activateAfterPublication() { + if (transport != null) transport.activateAfterPublication(); + } + public void close() { if (transport != null) { transport.close(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java index d6a24ff5c..242c7d3cf 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java @@ -11,6 +11,7 @@ public class PluginMessagingBackendProxyTransport implements BackendProxyTranspo private final VotingPluginMain plugin; private GlobalMessageHandler messageHandler; + private boolean active; public PluginMessagingBackendProxyTransport(VotingPluginMain plugin) { this.plugin = plugin; @@ -27,7 +28,14 @@ public void start(GlobalMessageHandler messageHandler) { } plugin.getPluginMessaging().setEncryptionHandler(encryptionHandler); plugin.getPluginMessaging().setDebug(plugin.getBungeeSettings().isBungeeDebug()); - plugin.activateBackendPluginMessageHandler(messageHandler); + } + + @Override + public void activateAfterPublication() { + if (messageHandler != null && !active) { + plugin.activateBackendPluginMessageHandler(messageHandler); + active = true; + } } @Override @@ -37,9 +45,10 @@ public void send(JsonEnvelope envelope) { @Override public void close() { - if (messageHandler != null) { + if (messageHandler != null && active) { plugin.deactivateBackendPluginMessageHandler(messageHandler); - messageHandler = null; } + active = false; + messageHandler = null; } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index f26bad9de..2bd5d3057 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -25,6 +26,7 @@ import com.bencodez.votingplugin.config.BungeeSettings; import com.bencodez.votingplugin.backendproxy.transport.MqttBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.MysqlBackendProxyTransport; +import com.bencodez.votingplugin.backendproxy.transport.PluginMessagingBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.BackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.BackendProxyTransportManager; import com.bencodez.votingplugin.backendproxy.transport.RedisBackendProxyTransport; @@ -32,6 +34,33 @@ import com.bencodez.votingplugin.proxy.BungeeMethod; class BackendProxyHandlerLifecycleTest { + @Test + void stagedPluginMessageTransportDefersRelaySwapUntilPublication() { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BungeeSettings settings = mock(BungeeSettings.class); + com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessage pluginMessages = + mock(com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessage.class); + GlobalMessageHandler replacement = mock(GlobalMessageHandler.class); + when(plugin.getBungeeSettings()).thenReturn(settings); + when(plugin.getPluginMessaging()).thenReturn(pluginMessages); + when(settings.getPluginMessagingChannel()).thenReturn("votingplugin:main"); + + PluginMessagingBackendProxyTransport transport = new PluginMessagingBackendProxyTransport(plugin); + transport.start(replacement); + verify(plugin, never()).activateBackendPluginMessageHandler(replacement); + transport.close(); + verify(plugin, never()).deactivateBackendPluginMessageHandler(replacement); + + transport = new PluginMessagingBackendProxyTransport(plugin); + transport.start(replacement); + transport.activateAfterPublication(); + transport.activateAfterPublication(); + verify(plugin, times(1)).activateBackendPluginMessageHandler(replacement); + + transport.close(); + verify(plugin).deactivateBackendPluginMessageHandler(replacement); + } + @Test void failedPresenceActivationDoesNotAnnounceReplacementGeneration() { com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); From b6d0f774bd13562f7e26d1115531ee300930758e Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:18:53 -0600 Subject: [PATCH 039/136] fix(control): harden transport rollback lifecycle --- .../votingplugin/VotingPluginMain.java | 12 +++-- .../backendproxy/BackendProxyHandler.java | 6 +-- .../http/HttpBackendTransportConnector.java | 9 +++- .../BackendProxyTransportManager.java | 8 +++ .../PluginMessagingBackendProxyTransport.java | 30 +++++++++-- .../BackendProxyHandlerLifecycleTest.java | 51 +++++++++++++++++++ 6 files changed, 103 insertions(+), 13 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 201b84640..f645d7bd1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -165,6 +165,7 @@ public class VotingPluginMain extends AdvancedCorePlugin { private final ProcessedVoteCache backendProcessedVoteCache = new ProcessedVoteCache(); private final AtomicReference backendPluginMessageTarget = new AtomicReference<>(); private PluginMessageHandler backendPluginMessageRelay; + private com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessage backendPluginMessageRelayOwner; private volatile BackendControlAutoEnrollment backendControlAutoEnrollment; private volatile BackendControlConnector backendControlConnector; private final ScheduledExecutorService backendControlConnectorLifecycle = Executors.newSingleThreadScheduledExecutor(runnable -> { @@ -1332,7 +1333,8 @@ public synchronized boolean requestBackendProxyHandlerRestartAbandonment(Backend public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart restart) { if (restart == null || restart.finished) return; if (restart.replacement != null) restart.replacement.close(); - if (restart.previousPrepared && backendProxyHandler == restart.previous) { + if (backendProxyHandler == restart.previous && restart.previous != null + && (restart.previousPrepared || restart.previous.getMethod() == BungeeMethod.PLUGINMESSAGING)) { restart.previous.restoreAfterFailedReplacement(); } restart.finished = true; @@ -1346,7 +1348,6 @@ public void awaitBackendProxyHandlerRollback(BackendProxyRestart restart, long d /** Keeps one plugin-message listener for the plugin lifetime and atomically swaps its active backend handler. */ public synchronized void activateBackendPluginMessageHandler(GlobalMessageHandler target) { - backendPluginMessageTarget.set(target); if (backendPluginMessageRelay == null) { backendPluginMessageRelay = new PluginMessageHandler() { @Override @@ -1355,8 +1356,13 @@ public void onReceive(com.bencodez.simpleapi.servercomm.codec.JsonEnvelope envel if (current != null) current.onMessage(envelope); } }; - getPluginMessaging().add(backendPluginMessageRelay); } + com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessage current = getPluginMessaging(); + if (backendPluginMessageRelayOwner != current) { + current.add(backendPluginMessageRelay); + backendPluginMessageRelayOwner = current; + } + backendPluginMessageTarget.set(target); } public void deactivateBackendPluginMessageHandler(GlobalMessageHandler target) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 6f2f91cf6..0f652cfc3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -99,13 +99,11 @@ public void sendMessage(JsonEnvelope envelope) { /** Starts presence only after a staged handler reaches the atomic publication boundary. */ public void activatePresenceReporting() { + transportManager.activateAfterPublication(); if (presenceManager != null && !presenceReportingActivated) { presenceManager.start(); presenceReportingActivated = true; } - // Swap shared inbound routing only after presence activation succeeds. An - // aborted publication must leave the previous plugin-message target intact. - transportManager.activateAfterPublication(); } /** @@ -134,7 +132,7 @@ public boolean prepareForReplacement(BungeeMethod replacementMethod) { /** Restores a prepared HTTP transport when its replacement fails validation. */ public void restoreAfterFailedReplacement() { - transportManager.restorePreparedTransport(); + transportManager.restoreAfterFailedReplacement(); } public void awaitRestoreAfterFailedReplacement(long deadlineNanos) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 284c48a9e..96a2c5e6c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -202,8 +202,15 @@ public boolean flushOutgoing(long deadlineNanos) { if (remaining <= 0L) return false; Duration timeout = Duration.ofNanos(Math.min(CLIENT_TIMEOUT.toNanos(), remaining)); synchronized (this) { - if (!pollOnce(timeout, false, false)) return false; + if (pollOnce(timeout, false, false)) continue; } + // The proxy may retain its one-active-poll guard briefly after the old + // client request is interrupted. Retry that transient 409 without busy + // spinning, but never extend the caller's shutdown deadline. + remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) return false; + try { TimeUnit.NANOSECONDS.sleep(Math.min(TimeUnit.MILLISECONDS.toNanos(50), remaining)); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); return false; } } return true; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 7b6ae8d31..08e3dccd5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -106,6 +106,14 @@ public void restorePreparedTransport() { preparedTransport = null; } + public void restoreAfterFailedReplacement() { + if (transport instanceof PluginMessagingBackendProxyTransport pluginMessaging) { + pluginMessaging.restoreAfterFailedReplacement(); + } else { + restorePreparedTransport(); + } + } + public void awaitPreparedTransportRestoration(long deadlineNanos) { if (transport instanceof HttpBackendProxyTransport http) http.awaitCredentialRestoration(deadlineNanos); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java index 242c7d3cf..2688eccbb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java @@ -11,6 +11,9 @@ public class PluginMessagingBackendProxyTransport implements BackendProxyTranspo private final VotingPluginMain plugin; private GlobalMessageHandler messageHandler; + private String channel; + private EncryptionHandler encryptionHandler; + private boolean debug; private boolean active; public PluginMessagingBackendProxyTransport(VotingPluginMain plugin) { @@ -20,24 +23,39 @@ public PluginMessagingBackendProxyTransport(VotingPluginMain plugin) { @Override public void start(GlobalMessageHandler messageHandler) { this.messageHandler = messageHandler; - plugin.registerBungeeChannels(plugin.getBungeeSettings().getPluginMessagingChannel()); - EncryptionHandler encryptionHandler = null; + channel = plugin.getBungeeSettings().getPluginMessagingChannel(); + encryptionHandler = null; if (plugin.getBungeeSettings().isPluginMessageEncryption()) { encryptionHandler = new EncryptionHandler(plugin.getName(), new File(plugin.getDataFolder(), "secretkey.key")); } - plugin.getPluginMessaging().setEncryptionHandler(encryptionHandler); - plugin.getPluginMessaging().setDebug(plugin.getBungeeSettings().isBungeeDebug()); + debug = plugin.getBungeeSettings().isBungeeDebug(); } @Override public void activateAfterPublication() { if (messageHandler != null && !active) { - plugin.activateBackendPluginMessageHandler(messageHandler); + publishSharedState(); + active = true; + } + } + + void restoreAfterFailedReplacement() { + if (messageHandler != null) { + publishSharedState(); active = true; } } + private void publishSharedState() { + if (plugin.getPluginMessaging() == null || !channel.equals(plugin.getBungeeChannel())) { + plugin.registerBungeeChannels(channel); + } + plugin.getPluginMessaging().setEncryptionHandler(encryptionHandler); + plugin.getPluginMessaging().setDebug(debug); + plugin.activateBackendPluginMessageHandler(messageHandler); + } + @Override public void send(JsonEnvelope envelope) { plugin.getPluginMessaging().sendEnvelope(envelope); @@ -50,5 +68,7 @@ public void close() { } active = false; messageHandler = null; + channel = null; + encryptionHandler = null; } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 2bd5d3057..20f550879 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -1,6 +1,7 @@ package com.bencodez.votingplugin.backendproxy; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; @@ -11,6 +12,7 @@ import static org.mockito.Mockito.when; import java.lang.reflect.Field; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.ScheduledExecutorService; import org.junit.jupiter.api.Test; @@ -34,6 +36,49 @@ import com.bencodez.votingplugin.proxy.BungeeMethod; class BackendProxyHandlerLifecycleTest { + @Test + void failedPluginMessagePublicationRestoresPreviousSharedState() { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BungeeSettings settings = mock(BungeeSettings.class); + com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessage pluginMessages = + mock(com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessage.class); + GlobalMessageHandler previousHandler = mock(GlobalMessageHandler.class); + GlobalMessageHandler replacementHandler = mock(GlobalMessageHandler.class); + AtomicReference activeChannel = new AtomicReference<>("old:channel"); + when(plugin.getBungeeSettings()).thenReturn(settings); + when(plugin.getPluginMessaging()).thenReturn(pluginMessages); + when(plugin.getBungeeChannel()).thenAnswer(ignored -> activeChannel.get()); + org.mockito.Mockito.doAnswer(invocation -> { + activeChannel.set(invocation.getArgument(0)); + return null; + }).when(plugin).registerBungeeChannels(org.mockito.ArgumentMatchers.anyString()); + + when(settings.getPluginMessagingChannel()).thenReturn("old:channel"); + PluginMessagingBackendProxyTransport previous = new PluginMessagingBackendProxyTransport(plugin); + previous.start(previousHandler); + previous.activateAfterPublication(); + + when(settings.getPluginMessagingChannel()).thenReturn("new:channel"); + PluginMessagingBackendProxyTransport replacement = new PluginMessagingBackendProxyTransport(plugin); + replacement.start(replacementHandler); + replacement.activateAfterPublication(); + replacement.close(); + BackendProxyTransportManager manager = new BackendProxyTransportManager(plugin); + try { + Field transportField = BackendProxyTransportManager.class.getDeclaredField("transport"); + transportField.setAccessible(true); + transportField.set(manager, previous); + manager.restoreAfterFailedReplacement(); + } catch (ReflectiveOperationException failure) { + throw new AssertionError(failure); + } + + assertEquals("old:channel", activeChannel.get()); + verify(plugin, times(2)).activateBackendPluginMessageHandler(previousHandler); + verify(plugin).activateBackendPluginMessageHandler(replacementHandler); + verify(plugin).deactivateBackendPluginMessageHandler(replacementHandler); + } + @Test void stagedPluginMessageTransportDefersRelaySwapUntilPublication() { com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); @@ -48,13 +93,19 @@ void stagedPluginMessageTransportDefersRelaySwapUntilPublication() { PluginMessagingBackendProxyTransport transport = new PluginMessagingBackendProxyTransport(plugin); transport.start(replacement); verify(plugin, never()).activateBackendPluginMessageHandler(replacement); + verify(plugin, never()).registerBungeeChannels(org.mockito.ArgumentMatchers.anyString()); + verifyNoInteractions(pluginMessages); transport.close(); verify(plugin, never()).deactivateBackendPluginMessageHandler(replacement); + verifyNoInteractions(pluginMessages); transport = new PluginMessagingBackendProxyTransport(plugin); transport.start(replacement); transport.activateAfterPublication(); transport.activateAfterPublication(); + verify(plugin, times(1)).registerBungeeChannels("votingplugin:main"); + verify(pluginMessages).setEncryptionHandler(null); + verify(pluginMessages).setDebug(false); verify(plugin, times(1)).activateBackendPluginMessageHandler(replacement); transport.close(); From 48841b80d8e5afc54c1b22ec71f2f32b8cf097a9 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:37:31 -0600 Subject: [PATCH 040/136] Fix durable vote-party effect recovery --- .../http/HttpBackendTransportConnector.java | 19 +- .../http/HttpClientCredentialStore.java | 5 +- .../transport/HttpBackendProxyTransport.java | 11 +- .../votingplugin/proxy/VotingPluginProxy.java | 221 +++++++++++++++++- .../proxy/bungee/BungeeJsonVoteCache.java | 35 ++- .../proxy/bungee/VotingPluginBungee.java | 22 ++ .../votingplugin/proxy/cache/IVoteCache.java | 8 + .../cache/PendingVotePartyProxyEffects.java | 36 +++ .../proxy/cache/VotePartyCacheDurability.java | 22 +- .../proxy/velocity/VelocityJsonVoteCache.java | 35 +++ .../proxy/velocity/VotingPluginVelocity.java | 31 +++ .../http/HttpTransportRuntimeTest.java | 6 + .../HttpBackendProxyTransportTest.java | 20 ++ .../cache/VotePartyCacheDurabilityTest.java | 32 +++ .../tests/VotingPluginProxyTest.java | 139 +++++++++++ .../tests/VotingPluginProxyTestImpl.java | 94 +++++++- docs/http-transport.md | 1 + 17 files changed, 715 insertions(+), 22 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/PendingVotePartyProxyEffects.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 96a2c5e6c..83a5ac91b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -49,6 +49,7 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private final Consumer onEnvelope; private volatile HttpClient client; private volatile HttpClientCredentialStore.ClientCredential credential; + private volatile HttpClientCredentialStore.ActiveCredentialGeneration activeCredentialGeneration; private final Path credentialDirectory; private final HttpInboundDeliveryStore inboundDeliveries; private final URI transportEndpoint; @@ -84,15 +85,24 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope, Path credentialDirectory) throws Exception { - this(enrolled == null ? null : enrolled.profile(), enrolled == null ? null : enrolled.credential(), onEnvelope, credentialDirectory); + this(enrolled == null ? null : enrolled.profile(), enrolled == null ? null : enrolled.credential(), onEnvelope, + credentialDirectory, credentialDirectory == null ? null + : HttpClientCredentialStore.snapshotActiveGeneration(credentialDirectory)); } private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope, Path credentialDirectory) throws Exception { + this(profile, credential, onEnvelope, credentialDirectory, null); + } + + private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope, Path credentialDirectory, + HttpClientCredentialStore.ActiveCredentialGeneration activeCredentialGeneration) throws Exception { if (profile == null || credential == null || onEnvelope == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); if (!matchesCredential(profile, credential)) throw new IllegalArgumentException("HTTP client certificate does not match transport profile"); this.profile = profile; this.serverId = profile.serverId(); this.onEnvelope = onEnvelope; this.credential = credential; + this.activeCredentialGeneration = activeCredentialGeneration; this.credentialDirectory = credentialDirectory; inboundDeliveries = credentialDirectory == null ? null : new HttpInboundDeliveryStore(credentialDirectory); if (inboundDeliveries != null) for (var entry : inboundDeliveries.snapshot().entrySet()) { @@ -142,6 +152,11 @@ public void start() { if (!running.compareAndSet(false, true)) return; poller = new Thread(this::pollLoop, "VotingPlugin-HTTP-poll"); poller.setDaemon(true); poller.start(); } + /** Returns the credential generation already validated off the platform main thread. */ + public HttpClientCredentialStore.ActiveCredentialGeneration activeCredentialGeneration() { + return activeCredentialGeneration; + } + /** Waits for one authenticated, protocol-valid transport response. */ public boolean awaitFirstResponse(long deadlineNanos) throws InterruptedException { long remaining = deadlineNanos - System.nanoTime(); @@ -374,6 +389,8 @@ private void maybeRenewCredential() { if (!matchesCredential(replacementProfile, replacement)) throw new IllegalArgumentException("Renewed HTTP certificate is invalid"); HttpClient replacementClient = client(replacementProfile, replacement); HttpClientCredentialStore.activateReplacement(directory, staged); + activeCredentialGeneration = new HttpClientCredentialStore.ActiveCredentialGeneration(staged.name(), + replacementProfile, staged.connectionCodeDigest()); profile = replacementProfile; client = replacementClient; credential = replacement; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index 9bc247ded..5cf6094ea 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -122,7 +122,7 @@ private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClie // Each file is durable within the generation, but the generation name is // published by its parent. Persist it before CURRENT can activate it. DurableFiles.forceDirectory(generations); - return new StagedCredential(name, enrolled.credential(), enrolled.profile()); + return new StagedCredential(name, enrolled.credential(), enrolled.profile(), connectionCodeDigest); } catch (Exception failure) { try { Files.deleteIfExists(generation.resolve(BUNDLE_FILE)); Files.deleteIfExists(generation.resolve(PASSWORD_FILE)); Files.deleteIfExists(generation.resolve(PROFILE_FILE)); Files.deleteIfExists(generation.resolve(CONNECTION_CODE_DIGEST_FILE)); @@ -146,7 +146,8 @@ static void activateReplacement(Path directory, StagedCredential staged) throws writePrivate(safe(directory.resolve(CURRENT_FILE)), staged.name().getBytes(StandardCharsets.US_ASCII)); } - static record StagedCredential(String name, ClientCredential credential, HttpClientProfile profile) { } + static record StagedCredential(String name, ClientCredential credential, HttpClientProfile profile, + String connectionCodeDigest) { } public static HttpClientProfile loadProfile(Path directory) throws IOException { return loadProfileFile(activeDirectory(directory)); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index ce8d6172d..9e87f225c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -85,11 +85,12 @@ HttpBackendProxyTransport recreatePrepared() { @Override public void prepareForReplacement() { - try { - configuredCredentialGeneration = HttpClientCredentialStore.snapshotActiveGeneration(configuredDirectory); - } catch (Exception failure) { - throw new IllegalStateException("Could not preserve the active HTTP client credential", failure); - } + HttpBackendTransportConnector active = connector; + HttpClientCredentialStore.ActiveCredentialGeneration generation = active == null + ? null : active.activeCredentialGeneration(); + if (generation == null) + throw new IllegalStateException("Could not preserve the active HTTP client credential before it became ready"); + configuredCredentialGeneration = generation; close(); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index f9b2ffeee..8e49773e1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -71,6 +71,7 @@ import com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity; import com.bencodez.votingplugin.proxy.broadcast.ProxyBroadcastDecider; import com.bencodez.votingplugin.proxy.cache.IVoteCache; +import com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects; import com.bencodez.votingplugin.proxy.cache.VoteCacheHandler; import com.bencodez.votingplugin.proxy.cache.nonvoted.INonVotedPlayersStorage; import com.bencodez.votingplugin.proxy.cache.nonvoted.NonVotedPlayersCache; @@ -146,6 +147,8 @@ public abstract class VotingPluginProxy { private boolean timeVoteDeliveryRetryScheduled; private boolean cachedVoteDeliveryRetryScheduled; private boolean votePartyDeliveryRetryScheduled; + private long votePartyProxyCommandAttemptSequence; + private long votePartyProxyCommandInFlight; private boolean enabled; @@ -959,6 +962,21 @@ public synchronized void checkVoteParty() { saveVoteCacheFile(); return; } + if (!retryPendingVotePartyProxyEffects()) { + persistRetainedVotePartyThreshold(); + return; + } + PendingVotePartyProxyEffects stagedProxyEffects = PendingVotePartyProxyEffects.empty(); + if (method == BungeeMethod.HTTP) { + try { + stagedProxyEffects = new PendingVotePartyProxyEffects(getConfig().getVotePartyBroadcast(), + getConfig().getVotePartyBungeeCommands()); + } catch (IllegalArgumentException oversized) { + logSevere("HTTP vote-party proxy effects exceed the durable backlog limit; retaining the vote-party threshold"); + persistRetainedVotePartyThreshold(); + return; + } + } Collection targets = getConfig().getVotePartySendToAllServers() ? getAllAvailableServers() : getConfig().getVotePartyServersToSend(); Map onlineTargets = onlineVotePartyTargets(targets); @@ -982,6 +1000,9 @@ public synchronized void checkVoteParty() { int previousVotes = votePartyVotes; int previousRequired = currentVotePartyVotesRequired; int previousIncrease = getVoteCacheVotePartyIncreaseVotesRequired(); + PendingVotePartyProxyEffects previousProxyEffects = method == BungeeMethod.HTTP + ? getVoteCachePendingVotePartyProxyEffects() : PendingVotePartyProxyEffects.empty(); + if (method == BungeeMethod.HTTP) setVoteCachePendingVotePartyProxyEffects(stagedProxyEffects); debug("Vote party reached"); addCurrentVotePartyVotes(-currentVotePartyVotesRequired); currentVotePartyVotesRequired += getConfig().getVotePartyIncreaseVotesRequired(); @@ -997,14 +1018,26 @@ public synchronized void checkVoteParty() { setVoteCacheVotePartyIncreaseVotesRequired(previousIncrease); for (Map.Entry staged : stagedRewards.entrySet()) setVoteCachePendingVotePartyReward(staged.getKey(), staged.getValue(), false); + if (method == BungeeMethod.HTTP) setVoteCachePendingVotePartyProxyEffects(previousProxyEffects); throw failure instanceof RuntimeException runtime ? runtime : new IllegalStateException("Unable to persist HTTP vote-party rewards", failure); } - if (!getConfig().getVotePartyBroadcast().isEmpty()) broadcast(getConfig().getVotePartyBroadcast()); - for (String command : getConfig().getVotePartyBungeeCommands()) runConsoleCommand(command); - if (method == BungeeMethod.HTTP) retryPendingVotePartyRewards(); - else for (String server : targets) sendVoteParty(server); + if (method == BungeeMethod.HTTP) { + if (retryPendingVotePartyProxyEffects()) retryPendingVotePartyRewards(); + } else { + if (!getConfig().getVotePartyBroadcast().isEmpty()) broadcast(getConfig().getVotePartyBroadcast()); + for (String command : getConfig().getVotePartyBungeeCommands()) runConsoleCommand(command); + for (String server : targets) sendVoteParty(server); + } + } + + private void persistRetainedVotePartyThreshold() { + try { + saveVotePartyStateDurably(); + } catch (IOException failure) { + throw new IllegalStateException("Unable to retain the HTTP vote-party threshold", failure); + } } private Map onlineVotePartyTargets(Collection targets) { @@ -1235,6 +1268,10 @@ protected int[] getProjectedVotePartyState(int acceptedVotes) { public abstract Collection getVoteCachePendingVotePartyRewardIds(String server); + public abstract PendingVotePartyProxyEffects getVoteCachePendingVotePartyProxyEffects(); + + public abstract PendingVotePartyProxyEffects getVoteCacheQuarantinedVotePartyProxyEffects(); + public abstract void saveVotePartyStateDurably() throws IOException; public abstract boolean isPlayerOnline(String playerName); @@ -1643,8 +1680,8 @@ public void onReceive(JsonEnvelope message) { // presence, vote-log, multi-proxy, and Control-adjacent runtime helpers. if (method.equals(BungeeMethod.HTTP)) { startHttpTransport(); - scheduleVotePartyDeliveryRetry(); } + scheduleVotePartyDeliveryRetry(); debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); } @@ -2495,6 +2532,7 @@ public void prepareForRuntimeReplacement() { /** Best-effort remainder of runtime teardown after the Control overlap gate has succeeded. */ public void completeRuntimeReplacementShutdown() { + enabled = false; cancelCommunicationTests("Proxy runtime stopped before the backend replied"); runCleanup("vote cache", () -> getVoteCacheHandler().saveVoteCache()); runCleanup("proxy MySQL messenger", () -> { @@ -2529,7 +2567,6 @@ public void completeRuntimeReplacementShutdown() { runCleanup("global data", () -> { if (getGlobalDataHandler() != null) getGlobalDataHandler().shutdown(); }); - enabled = false; } private void runCleanup(String service, CleanupAction cleanup) { @@ -2860,6 +2897,17 @@ private void warnUnsupportedDedicatedVotingProxyMode() { public abstract void runConsoleCommand(String command); + /** Completion boundary used before durable HTTP vote-party command progress is advanced. */ + protected CompletableFuture runVotePartyConsoleCommand(String command) { + runConsoleCommand(command); + return CompletableFuture.completedFuture(null); + } + + /** Schedules the liveness fence for a platform command whose completion is uncertain. */ + protected void scheduleVotePartyProxyCommandTimeout(Runnable timeout) { + CompletableFuture.delayedExecutor(60, TimeUnit.SECONDS).execute(timeout); + } + public abstract void saveVoteCacheFile(); public abstract void reloadCore(boolean mysql); @@ -3076,6 +3124,148 @@ protected synchronized void retryPendingVotePartyRewards() { if (retryRequired) scheduleVotePartyDeliveryRetry(); } + protected synchronized boolean retryPendingVotePartyProxyEffects() { + if (!enabled) return true; + if (votePartyProxyCommandInFlight != 0L) return false; + PendingVotePartyProxyEffects pending; + try { + pending = getVoteCachePendingVotePartyProxyEffects(); + } catch (RuntimeException invalid) { + logSevere("Pending HTTP vote-party proxy effects are invalid; retaining them without execution"); + return false; + } + while (!pending.isEmpty()) { + PendingVotePartyProxyEffects remaining; + try { + if (!pending.broadcast().isEmpty()) { + broadcast(pending.broadcast()); + remaining = new PendingVotePartyProxyEffects("", pending.commands()); + } else { + CompletableFuture execution = runVotePartyConsoleCommand(pending.commands().get(0)); + remaining = new PendingVotePartyProxyEffects("", pending.commands().subList(1, pending.commands().size())); + if (!execution.isDone()) { + long attempt = ++votePartyProxyCommandAttemptSequence; + if (attempt == 0L) attempt = ++votePartyProxyCommandAttemptSequence; + long commandAttempt = attempt; + votePartyProxyCommandInFlight = commandAttempt; + PendingVotePartyProxyEffects expected = pending; + PendingVotePartyProxyEffects completed = remaining; + execution.whenComplete((ignored, failure) -> + completeVotePartyProxyCommand(commandAttempt, expected, completed, failure)); + long scheduledAttempt = commandAttempt; + try { + scheduleVotePartyProxyCommandTimeout(() -> + quarantineTimedOutVotePartyProxyCommand(scheduledAttempt, expected, completed, execution)); + } catch (RuntimeException unavailable) { + logSevere("Unable to schedule the HTTP vote-party command liveness fence; the command remains pending"); + } + return false; + } + execution.join(); + } + } catch (RuntimeException failure) { + logSevere("A committed HTTP vote-party proxy effect failed and remains pending for retry"); + scheduleVotePartyDeliveryRetry(); + return false; + } + + if (!persistVotePartyProxyEffectProgress(pending, remaining)) return false; + pending = remaining; + } + return true; + } + + private void completeVotePartyProxyCommand(long attempt, PendingVotePartyProxyEffects expected, + PendingVotePartyProxyEffects remaining, Throwable failure) { + synchronized (this) { + if (votePartyProxyCommandInFlight != attempt) return; + votePartyProxyCommandInFlight = 0L; + if (!enabled) return; + if (failure != null) { + logSevere("A committed HTTP vote-party proxy command failed and remains pending for retry"); + scheduleVotePartyDeliveryRetry(); + return; + } + PendingVotePartyProxyEffects current; + try { + current = getVoteCachePendingVotePartyProxyEffects(); + } catch (RuntimeException invalid) { + logSevere("Pending HTTP vote-party proxy effects became invalid while a command was running"); + return; + } + if (!current.equals(expected)) { + logSevere("Pending HTTP vote-party proxy effects changed while a command was running; progress was not advanced"); + return; + } + if (persistVotePartyProxyEffectProgress(expected, remaining) + && retryPendingVotePartyProxyEffects()) { + if (method == BungeeMethod.HTTP) retryPendingVotePartyRewards(); + if (votePartyVotes >= currentVotePartyVotesRequired) checkVoteParty(); + } + } + } + + private void quarantineTimedOutVotePartyProxyCommand(long attempt, PendingVotePartyProxyEffects expected, + PendingVotePartyProxyEffects remaining, CompletableFuture execution) { + synchronized (this) { + if (!enabled || execution.isDone() || votePartyProxyCommandInFlight != attempt) return; + PendingVotePartyProxyEffects current; + PendingVotePartyProxyEffects previousQuarantine; + try { + current = getVoteCachePendingVotePartyProxyEffects(); + previousQuarantine = getVoteCacheQuarantinedVotePartyProxyEffects(); + } catch (RuntimeException invalid) { + logSevere("Pending HTTP vote-party proxy effects became invalid while a command was running"); + return; + } + if (!current.equals(expected)) { + logSevere("Pending HTTP vote-party proxy effects changed while a command was running; the attempt remains fenced"); + return; + } + PendingVotePartyProxyEffects quarantine; + try { + java.util.List commands = new java.util.ArrayList<>(previousQuarantine.commands()); + commands.add(expected.commands().get(0)); + quarantine = new PendingVotePartyProxyEffects(previousQuarantine.broadcast(), commands); + } catch (RuntimeException full) { + logSevere("HTTP vote-party command quarantine is full; the uncertain command remains fenced"); + return; + } + setVoteCacheQuarantinedVotePartyProxyEffects(quarantine); + setVoteCachePendingVotePartyProxyEffects(remaining); + try { + saveVotePartyStateDurably(); + } catch (IOException | RuntimeException failure) { + setVoteCacheQuarantinedVotePartyProxyEffects(previousQuarantine); + setVoteCachePendingVotePartyProxyEffects(expected); + logSevere("Unable to durably quarantine an uncertain HTTP vote-party command; the attempt remains fenced"); + return; + } + votePartyProxyCommandInFlight = 0L; + logSevere("An HTTP vote-party proxy command did not complete within 60 seconds and was durably quarantined without retry"); + if (retryPendingVotePartyProxyEffects()) { + if (method == BungeeMethod.HTTP) retryPendingVotePartyRewards(); + if (votePartyVotes >= currentVotePartyVotesRequired) checkVoteParty(); + } + } + } + + private boolean persistVotePartyProxyEffectProgress(PendingVotePartyProxyEffects previous, + PendingVotePartyProxyEffects remaining) { + setVoteCachePendingVotePartyProxyEffects(remaining); + try { + saveVotePartyStateDurably(); + return true; + } catch (IOException | RuntimeException failure) { + // Execution succeeded, but its progress was not durably confirmed. Restoring + // the marker gives at-least-once recovery rather than silently skipping it. + setVoteCachePendingVotePartyProxyEffects(previous); + logSevere("Unable to persist HTTP vote-party proxy-effect progress; the effect remains pending"); + scheduleVotePartyDeliveryRetry(); + return false; + } + } + private String resolveVotePartyRoutingServer(String canonicalServer) { for (String configuredServer : getAllAvailableServers()) { if (configuredServer.equalsIgnoreCase(canonicalServer)) return configuredServer; @@ -3103,14 +3293,21 @@ protected synchronized void acknowledgeVotePartyDelivery(String server, String d } private void scheduleVotePartyDeliveryRetry() { - if (!enabled || votePartyDeliveryRetryScheduled || method != BungeeMethod.HTTP || getScheduler() == null) return; - Collection pendingServers = getVoteCachePendingVotePartyServers(); - if (pendingServers == null || pendingServers.isEmpty()) return; + if (!enabled || votePartyDeliveryRetryScheduled || getScheduler() == null) return; + boolean pendingProxyEffects; + try { + pendingProxyEffects = !getVoteCachePendingVotePartyProxyEffects().isEmpty(); + } catch (RuntimeException invalid) { + logSevere("Pending HTTP vote-party proxy effects are invalid; automatic execution is disabled"); + return; + } + Collection pendingServers = method == BungeeMethod.HTTP ? getVoteCachePendingVotePartyServers() : null; + if (!pendingProxyEffects && (pendingServers == null || pendingServers.isEmpty())) return; votePartyDeliveryRetryScheduled = true; try { getScheduler().schedule(() -> { synchronized (VotingPluginProxy.this) { votePartyDeliveryRetryScheduled = false; } - retryPendingVotePartyRewards(); + if (retryPendingVotePartyProxyEffects() && method == BungeeMethod.HTTP) retryPendingVotePartyRewards(); }, 5, TimeUnit.SECONDS); } catch (RuntimeException failure) { votePartyDeliveryRetryScheduled = false; @@ -3140,6 +3337,10 @@ public void setCurrentVotePartyVotes(int amount) { public abstract void setVoteCachePendingVotePartyReward(String server, String deliveryId, boolean pending); + public abstract void setVoteCachePendingVotePartyProxyEffects(PendingVotePartyProxyEffects effects); + + public abstract void setVoteCacheQuarantinedVotePartyProxyEffects(PendingVotePartyProxyEffects effects); + public void status() { for (String s : getAllAvailableServers()) { if (!isSomeoneOnlineServerForVoteRouting(s)) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java index 9fd7db463..7bd462002 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java @@ -11,7 +11,8 @@ import com.bencodez.votingplugin.proxy.OfflineBungeeVote; import com.bencodez.votingplugin.proxy.cache.DataNode; import com.bencodez.votingplugin.proxy.cache.GsonDataNode; -import com.bencodez.votingplugin.proxy.cache.IVoteCache; +import com.bencodez.votingplugin.proxy.cache.IVoteCache; +import com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; /** @@ -145,6 +146,18 @@ public Collection getPendingVotePartyRewardServers() { public Collection getPendingVotePartyRewardIds(String server) { return getKeys("VoteParty.PendingRewards." + encodeServerKey(server)); } + + @Override + public PendingVotePartyProxyEffects getPendingVotePartyProxyEffects() { + return new PendingVotePartyProxyEffects(getString("VoteParty.PendingProxyEffects.Broadcast", ""), + getStringList("VoteParty.PendingProxyEffects.Commands", java.util.List.of())); + } + + @Override + public PendingVotePartyProxyEffects getQuarantinedVotePartyProxyEffects() { + return new PendingVotePartyProxyEffects(getString("VoteParty.QuarantinedProxyEffects.Broadcast", ""), + getStringList("VoteParty.QuarantinedProxyEffects.Commands", java.util.List.of())); + } public int getVotePartyCurrentVotes() { return getInt("VoteParty.CurrentVotes", 0); @@ -170,6 +183,26 @@ public void setPendingVotePartyReward(String server, String deliveryId, boolean } } + @Override + public void setPendingVotePartyProxyEffects(PendingVotePartyProxyEffects effects) { + if (effects.isEmpty()) { + remove("VoteParty.PendingProxyEffects"); + return; + } + setString("VoteParty.PendingProxyEffects.Broadcast", effects.broadcast()); + setStringList("VoteParty.PendingProxyEffects.Commands", effects.commands()); + } + + @Override + public void setQuarantinedVotePartyProxyEffects(PendingVotePartyProxyEffects effects) { + if (effects.isEmpty()) { + remove("VoteParty.QuarantinedProxyEffects"); + return; + } + setString("VoteParty.QuarantinedProxyEffects.Broadcast", effects.broadcast()); + setStringList("VoteParty.QuarantinedProxyEffects.Commands", effects.commands()); + } + private static String encodeServerKey(String server) { return Base64.getUrlEncoder().withoutPadding() .encodeToString(server.toLowerCase(Locale.ROOT).getBytes(StandardCharsets.UTF_8)); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java index 629fe1642..41cd4345c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java @@ -634,6 +634,16 @@ public Collection getVoteCachePendingVotePartyServers() { public Collection getVoteCachePendingVotePartyRewardIds(String server) { return voteCacheFile.getPendingVotePartyRewardIds(server); } + + @Override + public com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects getVoteCachePendingVotePartyProxyEffects() { + return voteCacheFile.getPendingVotePartyProxyEffects(); + } + + @Override + public com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects getVoteCacheQuarantinedVotePartyProxyEffects() { + return voteCacheFile.getQuarantinedVotePartyProxyEffects(); + } @Override public boolean isPlayerOnline(String playerName) { @@ -740,6 +750,18 @@ public void setVoteCacheVotePartyIncreaseVotesRequired(int votes) { public void setVoteCachePendingVotePartyReward(String server, String deliveryId, boolean pending) { voteCacheFile.setPendingVotePartyReward(server, deliveryId, pending); } + + @Override + public void setVoteCachePendingVotePartyProxyEffects( + com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects effects) { + voteCacheFile.setPendingVotePartyProxyEffects(effects); + } + + @Override + public void setVoteCacheQuarantinedVotePartyProxyEffects( + com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects effects) { + voteCacheFile.setQuarantinedVotePartyProxyEffects(effects); + } @Override public void warn(String message) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java index eb9905725..12cf1921a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java @@ -122,6 +122,10 @@ public interface IVoteCache { Collection getPendingVotePartyRewardIds(String server); + PendingVotePartyProxyEffects getPendingVotePartyProxyEffects(); + + PendingVotePartyProxyEffects getQuarantinedVotePartyProxyEffects(); + /** * Gets the current vote party votes. * @@ -146,6 +150,10 @@ public interface IVoteCache { void setPendingVotePartyReward(String server, String deliveryId, boolean pending); + void setPendingVotePartyProxyEffects(PendingVotePartyProxyEffects effects); + + void setQuarantinedVotePartyProxyEffects(PendingVotePartyProxyEffects effects); + /** * Sets the current vote party votes. * diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/PendingVotePartyProxyEffects.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/PendingVotePartyProxyEffects.java new file mode 100644 index 000000000..2f351f03e --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/PendingVotePartyProxyEffects.java @@ -0,0 +1,36 @@ +package com.bencodez.votingplugin.proxy.cache; + +import java.util.List; + +/** Durable, ordered proxy-side effects for one committed HTTP vote party. */ +public record PendingVotePartyProxyEffects(String broadcast, List commands) { + public static final int MAX_COMMANDS = 1_024; + public static final int MAX_TOTAL_CHARACTERS = 262_144; + + public PendingVotePartyProxyEffects { + broadcast = broadcast == null ? "" : broadcast; + commands = commands == null ? List.of() : List.copyOf(commands); + if (commands.size() > MAX_COMMANDS) { + throw new IllegalArgumentException("Too many pending vote-party proxy commands"); + } + long characters = broadcast.length(); + if (characters > MAX_TOTAL_CHARACTERS) { + throw new IllegalArgumentException("Pending vote-party proxy effects are too large"); + } + for (String command : commands) { + if (command == null) throw new IllegalArgumentException("Null vote-party proxy command"); + characters += command.length(); + if (characters > MAX_TOTAL_CHARACTERS) { + throw new IllegalArgumentException("Pending vote-party proxy effects are too large"); + } + } + } + + public static PendingVotePartyProxyEffects empty() { + return new PendingVotePartyProxyEffects("", List.of()); + } + + public boolean isEmpty() { + return broadcast.isEmpty() && commands.isEmpty(); + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java index bfcdf3d76..d0c855bba 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java @@ -23,6 +23,8 @@ public static void saveAndVerify(Path file, IVoteCache cache) throws IOException Map> expectedPending = pending(cache); int expectedVotes = cache.getVotePartyCurrentVotes(); int expectedIncrease = cache.getVotePartyInreaseVotesRequired(); + PendingVotePartyProxyEffects expectedProxyEffects = cache.getPendingVotePartyProxyEffects(); + PendingVotePartyProxyEffects expectedQuarantinedEffects = cache.getQuarantinedVotePartyProxyEffects(); cache.save(); DurableFiles.forceFile(file); DurableFiles.forceDirectory(file.toAbsolutePath().normalize().getParent()); @@ -31,7 +33,9 @@ public static void saveAndVerify(Path file, IVoteCache cache) throws IOException JsonObject voteParty = object(root.get("VoteParty")); if (integer(voteParty, "CurrentVotes") != expectedVotes || integer(voteParty, "IncreaseVotes") != expectedIncrease - || !pending(voteParty).equals(expectedPending)) + || !pending(voteParty).equals(expectedPending) + || !proxyEffects(voteParty, "PendingProxyEffects").equals(expectedProxyEffects) + || !proxyEffects(voteParty, "QuarantinedProxyEffects").equals(expectedQuarantinedEffects)) throw new IOException("Vote-party cache state was not persisted"); } catch (IOException failure) { throw failure; @@ -40,6 +44,17 @@ public static void saveAndVerify(Path file, IVoteCache cache) throws IOException } } + private static PendingVotePartyProxyEffects proxyEffects(JsonObject voteParty, String name) { + JsonObject effects = object(voteParty.get(name)); + String broadcast = string(effects, "Broadcast"); + java.util.List commands = new java.util.ArrayList<>(); + JsonElement encodedCommands = effects.get("Commands"); + if (encodedCommands != null && !encodedCommands.isJsonNull()) { + for (JsonElement command : encodedCommands.getAsJsonArray()) commands.add(command.getAsString()); + } + return new PendingVotePartyProxyEffects(broadcast, commands); + } + private static Map> pending(IVoteCache cache) { Map> pending = new HashMap<>(); var servers = cache.getPendingVotePartyRewardServers(); @@ -73,4 +88,9 @@ private static int integer(JsonObject object, String name) { JsonElement value = object.get(name); return value == null || value.isJsonNull() ? 0 : value.getAsInt(); } + + private static String string(JsonObject object, String name) { + JsonElement value = object.get(name); + return value == null || value.isJsonNull() ? "" : value.getAsString(); + } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java index b811ff764..32a0e23cc 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java @@ -12,6 +12,7 @@ import com.bencodez.votingplugin.proxy.cache.ConfigDataNode; import com.bencodez.votingplugin.proxy.cache.DataNode; import com.bencodez.votingplugin.proxy.cache.IVoteCache; +import com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; /** @@ -151,6 +152,20 @@ public Collection getPendingVotePartyRewardIds(String server) { return getKeys(getNode("VoteParty", "PendingRewards", encodeServerKey(server))); } + @Override + public PendingVotePartyProxyEffects getPendingVotePartyProxyEffects() { + return new PendingVotePartyProxyEffects( + getString(getNode("VoteParty", "PendingProxyEffects", "Broadcast"), ""), + getStringList(getNode("VoteParty", "PendingProxyEffects", "Commands"), java.util.List.of())); + } + + @Override + public PendingVotePartyProxyEffects getQuarantinedVotePartyProxyEffects() { + return new PendingVotePartyProxyEffects( + getString(getNode("VoteParty", "QuarantinedProxyEffects", "Broadcast"), ""), + getStringList(getNode("VoteParty", "QuarantinedProxyEffects", "Commands"), java.util.List.of())); + } + @Override public int getVotePartyCurrentVotes() { return getInt(getNode("VoteParty", "CurrentVotes"), 0); @@ -177,6 +192,26 @@ public void setPendingVotePartyReward(String server, String deliveryId, boolean } } + @Override + public void setPendingVotePartyProxyEffects(PendingVotePartyProxyEffects effects) { + if (effects.isEmpty()) { + remove("VoteParty", "PendingProxyEffects"); + return; + } + set(new Object[] { "VoteParty", "PendingProxyEffects", "Broadcast" }, effects.broadcast()); + set(new Object[] { "VoteParty", "PendingProxyEffects", "Commands" }, effects.commands()); + } + + @Override + public void setQuarantinedVotePartyProxyEffects(PendingVotePartyProxyEffects effects) { + if (effects.isEmpty()) { + remove("VoteParty", "QuarantinedProxyEffects"); + return; + } + set(new Object[] { "VoteParty", "QuarantinedProxyEffects", "Broadcast" }, effects.broadcast()); + set(new Object[] { "VoteParty", "QuarantinedProxyEffects", "Commands" }, effects.commands()); + } + private static String encodeServerKey(String server) { return Base64.getUrlEncoder().withoutPadding() .encodeToString(server.toLowerCase(Locale.ROOT).getBytes(StandardCharsets.UTF_8)); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java index 9202c866e..6af9ac551 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java @@ -725,6 +725,16 @@ public Collection getVoteCachePendingVotePartyRewardIds(String server) { return voteCacheFile.getPendingVotePartyRewardIds(server); } + @Override + public com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects getVoteCachePendingVotePartyProxyEffects() { + return voteCacheFile.getPendingVotePartyProxyEffects(); + } + + @Override + public com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects getVoteCacheQuarantinedVotePartyProxyEffects() { + return voteCacheFile.getQuarantinedVotePartyProxyEffects(); + } + @Override public boolean isVoteCacheIgnoreTime() { return voteCacheFile.getNode("Time", "IgnoreTime").getBoolean(); @@ -775,6 +785,18 @@ public void setVoteCachePendingVotePartyReward(String server, String deliveryId, voteCacheFile.setPendingVotePartyReward(server, deliveryId, pending); } + @Override + public void setVoteCachePendingVotePartyProxyEffects( + com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects effects) { + voteCacheFile.setPendingVotePartyProxyEffects(effects); + } + + @Override + public void setVoteCacheQuarantinedVotePartyProxyEffects( + com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects effects) { + voteCacheFile.setQuarantinedVotePartyProxyEffects(effects); + } + @Override public boolean isPlayerOnline(String playerName) { if (playerName == null) { @@ -816,6 +838,15 @@ public void runConsoleCommand(String command) { server.getCommandManager().executeAsync(server.getConsoleCommandSource(), command); } + @Override + protected java.util.concurrent.CompletableFuture runVotePartyConsoleCommand(String command) { + return server.getCommandManager().executeAsync(server.getConsoleCommandSource(), command) + .thenApply(executed -> { + if (!executed) throw new IllegalStateException("Velocity declined the vote-party proxy command"); + return null; + }); + } + @Override public void saveVoteCacheFile() { voteCacheFile.save(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index e8be931cc..e011f0405 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -387,6 +387,8 @@ void backendRenewsClientCertificateBeforeExpiryWithoutNewConnectionCode() throws HttpClientCredentialStore.saveEnrolled(clientDirectory, profileCode, expiring); server.start(); try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { + HttpClientCredentialStore.ActiveCredentialGeneration originalGeneration = + connector.activeCredentialGeneration(); connector.start(); long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(8); String renewedPin = originalPin; @@ -400,6 +402,10 @@ void backendRenewsClientCertificateBeforeExpiryWithoutNewConnectionCode() throws while (authority.authenticate("lobby-1", expiring.certificate()) && System.nanoTime() < promotionDeadline) Thread.sleep(25); assertTrue(authority.authenticate("lobby-1", renewed.certificate())); assertFalse(authority.authenticate("lobby-1", expiring.certificate())); + HttpClientCredentialStore.ActiveCredentialGeneration renewedGeneration = + connector.activeCredentialGeneration(); + assertFalse(renewedGeneration.name().equals(originalGeneration.name())); + assertEquals(originalGeneration.connectionCodeDigest(), renewedGeneration.connectionCodeDigest()); } } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 1b8491efd..5cd5d392c 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -4,12 +4,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.backendproxy.http.HttpBackendTransportConnector; import com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore; import com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode; import com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity; @@ -90,6 +92,24 @@ void closeNeverWaitsForSetupOnTheCallingThread() throws Exception { finally { release.countDown(); blocked.join(TimeUnit.SECONDS.toMillis(1)); } } + @Test + void prepareForReplacementUsesConnectorSnapshotWithoutCredentialIo() throws Exception { + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(mock(VotingPluginMain.class)); + HttpBackendTransportConnector connector = mock(HttpBackendTransportConnector.class); + HttpClientCredentialStore.ActiveCredentialGeneration generation = + new HttpClientCredentialStore.ActiveCredentialGeneration(java.util.UUID.randomUUID().toString()); + when(connector.activeCredentialGeneration()).thenReturn(generation); + java.lang.reflect.Field connectorField = HttpBackendProxyTransport.class.getDeclaredField("connector"); + connectorField.setAccessible(true); + connectorField.set(transport, connector); + + assertDoesNotThrow(transport::prepareForReplacement); + java.lang.reflect.Field generationField = + HttpBackendProxyTransport.class.getDeclaredField("configuredCredentialGeneration"); + generationField.setAccessible(true); + assertSame(generation, generationField.get(transport)); + } + @Test @SuppressWarnings("unchecked") void validationWaitsForThePreviousDirectoryOwnerBeforeReadinessFailure() throws Exception { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurabilityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurabilityTest.java index c78157c1a..7dcec7054 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurabilityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurabilityTest.java @@ -21,6 +21,38 @@ void detectsWhenCacheSaveReturnsWithoutUpdatingTheFile() throws Exception { when(cache.getPendingVotePartyRewardIds("lobby")).thenReturn(Set.of("delivery-id")); when(cache.getVotePartyCurrentVotes()).thenReturn(3); when(cache.getVotePartyInreaseVotesRequired()).thenReturn(2); + when(cache.getPendingVotePartyProxyEffects()).thenReturn(PendingVotePartyProxyEffects.empty()); + when(cache.getQuarantinedVotePartyProxyEffects()).thenReturn(PendingVotePartyProxyEffects.empty()); + Path file = directory.resolve("votecache.json"); + Files.writeString(file, "{\"VoteParty\":{\"CurrentVotes\":3,\"IncreaseVotes\":2}}"); + + assertThrows(java.io.IOException.class, () -> VotePartyCacheDurability.saveAndVerify(file, cache)); + } + + @Test + void detectsWhenPendingProxyEffectsAreMissingFromTheFile() throws Exception { + IVoteCache cache = mock(IVoteCache.class); + when(cache.getPendingVotePartyRewardServers()).thenReturn(Set.of()); + when(cache.getVotePartyCurrentVotes()).thenReturn(3); + when(cache.getVotePartyInreaseVotesRequired()).thenReturn(2); + when(cache.getPendingVotePartyProxyEffects()) + .thenReturn(new PendingVotePartyProxyEffects("Party time", java.util.List.of("reward all"))); + when(cache.getQuarantinedVotePartyProxyEffects()).thenReturn(PendingVotePartyProxyEffects.empty()); + Path file = directory.resolve("votecache.json"); + Files.writeString(file, "{\"VoteParty\":{\"CurrentVotes\":3,\"IncreaseVotes\":2}}"); + + assertThrows(java.io.IOException.class, () -> VotePartyCacheDurability.saveAndVerify(file, cache)); + } + + @Test + void detectsWhenQuarantinedProxyEffectsAreMissingFromTheFile() throws Exception { + IVoteCache cache = mock(IVoteCache.class); + when(cache.getPendingVotePartyRewardServers()).thenReturn(Set.of()); + when(cache.getVotePartyCurrentVotes()).thenReturn(3); + when(cache.getVotePartyInreaseVotesRequired()).thenReturn(2); + when(cache.getPendingVotePartyProxyEffects()).thenReturn(PendingVotePartyProxyEffects.empty()); + when(cache.getQuarantinedVotePartyProxyEffects()) + .thenReturn(new PendingVotePartyProxyEffects("", java.util.List.of("uncertain reward"))); Path file = directory.resolve("votecache.json"); Files.writeString(file, "{\"VoteParty\":{\"CurrentVotes\":3,\"IncreaseVotes\":2}}"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 12b4faede..1cd56cb9b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -129,6 +129,145 @@ void httpVotePartyStagesEveryTargetBeforeDelivery() { assertEquals(0, votingPluginProxy.getVotePartyVotes()); } + @Test + void httpVotePartyRetriesJournaledProxyEffectsAfterExecutionFailure() { + configureHttpVotePartyEffects("Party time", java.util.List.of("first", "second")); + votingPluginProxy.failNextBroadcast(); + + votingPluginProxy.checkVoteParty(); + + assertEquals(0, votingPluginProxy.getVotePartyVotes()); + assertEquals("Party time", votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().broadcast()); + assertEquals(java.util.List.of("first", "second"), + votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().commands()); + assertTrue(votingPluginProxy.getBroadcasts().isEmpty()); + assertTrue(votingPluginProxy.getConsoleCommands().isEmpty()); + + assertTrue(votingPluginProxy.retryPendingVotePartyProxyEffectsForTest()); + assertEquals(java.util.List.of("Party time"), votingPluginProxy.getBroadcasts()); + assertEquals(java.util.List.of("first", "second"), votingPluginProxy.getConsoleCommands()); + assertTrue(votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().isEmpty()); + } + + @Test + void httpVotePartyRestoresExecutedEffectWhenProgressIsNotDurable() { + configureHttpVotePartyEffects("Party time", java.util.List.of("reward all")); + votingPluginProxy.failSaveAfterNextBroadcast(); + + votingPluginProxy.checkVoteParty(); + + assertEquals(java.util.List.of("Party time"), votingPluginProxy.getBroadcasts()); + assertEquals("Party time", votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().broadcast()); + assertEquals(java.util.List.of("reward all"), + votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().commands()); + + assertTrue(votingPluginProxy.retryPendingVotePartyProxyEffectsForTest()); + assertEquals(java.util.List.of("Party time", "Party time"), votingPluginProxy.getBroadcasts()); + assertEquals(java.util.List.of("reward all"), votingPluginProxy.getConsoleCommands()); + assertTrue(votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().isEmpty()); + } + + @Test + void httpVotePartyPersistsOrderedCommandProgress() { + configureHttpVotePartyEffects("", java.util.List.of("first", "second", "third")); + votingPluginProxy.failConsoleCommand("second"); + + votingPluginProxy.checkVoteParty(); + + assertEquals(java.util.List.of("first"), votingPluginProxy.getConsoleCommands()); + assertEquals(java.util.List.of("second", "third"), + votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().commands()); + assertTrue(votingPluginProxy.retryPendingVotePartyProxyEffectsForTest()); + assertEquals(java.util.List.of("first", "second", "third"), votingPluginProxy.getConsoleCommands()); + } + + @Test + void httpVotePartyKeepsCommandPendingUntilAsyncExecutionCompletesAndResumesReachedThreshold() { + configureHttpVotePartyEffects("", java.util.List.of("async command")); + java.util.concurrent.CompletableFuture completion = + votingPluginProxy.delayNextVotePartyCommandCompletion(); + + votingPluginProxy.checkVoteParty(); + + assertEquals(java.util.List.of("async command"), votingPluginProxy.getConsoleCommands()); + assertEquals(java.util.List.of("async command"), + votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().commands()); + assertFalse(votingPluginProxy.retryPendingVotePartyProxyEffectsForTest()); + votingPluginProxy.setVotePartyVotes(1); + + completion.complete(null); + + assertTrue(votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().isEmpty()); + assertEquals(java.util.List.of("async command", "async command"), + votingPluginProxy.getConsoleCommands()); + assertEquals(0, votingPluginProxy.getVotePartyVotes()); + } + + @Test + void httpVotePartyQuarantinesHungCommandWithoutRetryingIt() { + configureHttpVotePartyEffects("", java.util.List.of("hung command", "next command")); + java.util.concurrent.CompletableFuture completion = + votingPluginProxy.delayNextVotePartyCommandCompletion(); + + votingPluginProxy.checkVoteParty(); + votingPluginProxy.runVotePartyProxyCommandTimeoutForTest(); + + assertEquals(java.util.List.of("hung command"), + votingPluginProxy.getVoteCacheQuarantinedVotePartyProxyEffects().commands()); + assertTrue(votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().isEmpty()); + assertEquals(java.util.List.of("hung command", "next command"), votingPluginProxy.getConsoleCommands()); + + completion.complete(null); + assertEquals(java.util.List.of("hung command", "next command"), votingPluginProxy.getConsoleCommands()); + assertEquals(java.util.List.of("hung command"), + votingPluginProxy.getVoteCacheQuarantinedVotePartyProxyEffects().commands()); + } + + @Test + void httpVotePartyRetainsHungCommandFenceWhenQuarantineIsNotDurable() { + configureHttpVotePartyEffects("", java.util.List.of("hung command", "next command")); + java.util.concurrent.CompletableFuture completion = + votingPluginProxy.delayNextVotePartyCommandCompletion(); + votingPluginProxy.checkVoteParty(); + votingPluginProxy.failNextVoteCacheSave(); + + votingPluginProxy.runVotePartyProxyCommandTimeoutForTest(); + + assertEquals(java.util.List.of("hung command", "next command"), + votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().commands()); + assertTrue(votingPluginProxy.getVoteCacheQuarantinedVotePartyProxyEffects().isEmpty()); + assertFalse(votingPluginProxy.retryPendingVotePartyProxyEffectsForTest()); + assertEquals(java.util.List.of("hung command"), votingPluginProxy.getConsoleCommands()); + + completion.complete(null); + assertEquals(java.util.List.of("hung command", "next command"), votingPluginProxy.getConsoleCommands()); + assertTrue(votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().isEmpty()); + } + + @Test + void committedHttpProxyEffectsRecoverAfterTransportModeChanges() { + votingPluginProxy.setMethod(BungeeMethod.SOCKETS); + votingPluginProxy.setVoteCachePendingVotePartyProxyEffects( + new com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects( + "Party time", java.util.List.of("reward all"))); + + assertTrue(votingPluginProxy.retryPendingVotePartyProxyEffectsForTest()); + + assertEquals(java.util.List.of("Party time"), votingPluginProxy.getBroadcasts()); + assertEquals(java.util.List.of("reward all"), votingPluginProxy.getConsoleCommands()); + assertTrue(votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().isEmpty()); + } + + private void configureHttpVotePartyEffects(String broadcast, java.util.List commands) { + Mockito.when(votingPluginProxy.getConfig().getVotePartyEnabled()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getVotePartySendToAllServers()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getVotePartyBroadcast()).thenReturn(broadcast); + Mockito.when(votingPluginProxy.getConfig().getVotePartyBungeeCommands()).thenReturn(commands); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVotePartyVotes(1); + votingPluginProxy.setCurrentVotePartyVotesRequired(1); + } + @Test void rolloverProjectionIncludesQueuedVotesAndVotePartyThresholds() { Mockito.when(votingPluginProxy.getConfig().getVotePartyEnabled()).thenReturn(true); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index 15ea91aef..ca47dd7fd 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -19,6 +19,7 @@ import com.bencodez.votingplugin.proxy.OfflineBungeeVote; import com.bencodez.votingplugin.proxy.VotingPluginProxy; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; +import com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects; import com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; @@ -34,6 +35,15 @@ public class VotingPluginProxyTestImpl extends VotingPluginProxy { private boolean failNextVoteCacheSave; private final java.util.Map> pendingVotePartyRewards = new HashMap<>(); private final List attemptedVotePartyDeliveryIds = new ArrayList<>(); + private PendingVotePartyProxyEffects pendingVotePartyProxyEffects = PendingVotePartyProxyEffects.empty(); + private PendingVotePartyProxyEffects quarantinedVotePartyProxyEffects = PendingVotePartyProxyEffects.empty(); + private final List broadcasts = new ArrayList<>(); + private final List consoleCommands = new ArrayList<>(); + private boolean failNextBroadcast; + private String failConsoleCommand; + private boolean failSaveAfterNextBroadcast; + private java.util.concurrent.CompletableFuture nextVotePartyCommandCompletion; + private Runnable votePartyProxyCommandTimeout; public List getWarnings() { return warnings; @@ -46,7 +56,15 @@ public void addNonVotedPlayer(String uuid, String playerName) { @Override public void broadcast(String message) { - // Implementation for testing purposes + if (failNextBroadcast) { + failNextBroadcast = false; + throw new IllegalStateException("broadcast failed"); + } + broadcasts.add(message); + if (failSaveAfterNextBroadcast) { + failSaveAfterNextBroadcast = false; + failNextVoteCacheSave = true; + } } @Override @@ -141,6 +159,16 @@ public Collection getVoteCachePendingVotePartyRewardIds(String server) { java.util.Set.of())); } + @Override + public PendingVotePartyProxyEffects getVoteCachePendingVotePartyProxyEffects() { + return pendingVotePartyProxyEffects; + } + + @Override + public PendingVotePartyProxyEffects getVoteCacheQuarantinedVotePartyProxyEffects() { + return quarantinedVotePartyProxyEffects; + } + @Override public boolean isPlayerOnline(String playerName) { return playerOnline; @@ -172,7 +200,24 @@ public void runAsync(Runnable run) { @Override public void runConsoleCommand(String command) { - // Mocked for testing + if (command.equals(failConsoleCommand)) { + failConsoleCommand = null; + throw new IllegalStateException("command failed"); + } + consoleCommands.add(command); + } + + @Override + protected java.util.concurrent.CompletableFuture runVotePartyConsoleCommand(String command) { + java.util.concurrent.CompletableFuture completion = nextVotePartyCommandCompletion; + nextVotePartyCommandCompletion = null; + runConsoleCommand(command); + return completion == null ? java.util.concurrent.CompletableFuture.completedFuture(null) : completion; + } + + @Override + protected void scheduleVotePartyProxyCommandTimeout(Runnable timeout) { + votePartyProxyCommandTimeout = timeout; } @Override @@ -192,6 +237,37 @@ public void failNextVoteCacheSave() { failNextVoteCacheSave = true; } + public void failNextBroadcast() { + failNextBroadcast = true; + } + + public void failConsoleCommand(String command) { + failConsoleCommand = command; + } + + public void failSaveAfterNextBroadcast() { + failSaveAfterNextBroadcast = true; + } + + public List getBroadcasts() { + return List.copyOf(broadcasts); + } + + public List getConsoleCommands() { + return List.copyOf(consoleCommands); + } + + public java.util.concurrent.CompletableFuture delayNextVotePartyCommandCompletion() { + nextVotePartyCommandCompletion = new java.util.concurrent.CompletableFuture<>(); + return nextVotePartyCommandCompletion; + } + + public void runVotePartyProxyCommandTimeoutForTest() { + Runnable timeout = votePartyProxyCommandTimeout; + votePartyProxyCommandTimeout = null; + if (timeout != null) timeout.run(); + } + @Override public void reloadCore(boolean mysql) { // Mocked for testing @@ -332,6 +408,10 @@ public void retryPendingVotePartyRewardsForTest() { retryPendingVotePartyRewards(); } + public boolean retryPendingVotePartyProxyEffectsForTest() { + return retryPendingVotePartyProxyEffects(); + } + public int[] getProjectedVotePartyStateForTest(int acceptedVotes) { return getProjectedVotePartyState(acceptedVotes); } @@ -420,6 +500,16 @@ public void setVoteCachePendingVotePartyReward(String server, String deliveryId, } } + @Override + public void setVoteCachePendingVotePartyProxyEffects(PendingVotePartyProxyEffects effects) { + pendingVotePartyProxyEffects = effects; + } + + @Override + public void setVoteCacheQuarantinedVotePartyProxyEffects(PendingVotePartyProxyEffects effects) { + quarantinedVotePartyProxyEffects = effects; + } + @Override public ScheduledExecutorService getScheduler() { return scheduler; diff --git a/docs/http-transport.md b/docs/http-transport.md index 740911e45..b8681176a 100644 --- a/docs/http-transport.md +++ b/docs/http-transport.md @@ -42,6 +42,7 @@ Connection codes expire after 15 minutes and can be used only once. Treat a fres - Request bodies, queues, batches, worker pools, concurrent requests, per-backend polling, and request rates are bounded. Exact paths, methods, and JSON content types are enforced. - Proxy-to-backend messages stay in an owner-only durable proxy queue until the backend acknowledgement is durably applied. The backend fsyncs three delivery states around each callback: a reserved callback resumes after restart, a completed callback is acknowledged without running twice, and a running callback with an uncertain outcome is neither replayed nor acknowledged. The proxy therefore retains the durable source copy instead of silently losing it. The completed entry is deleted only after a matching authenticated HTTP 200 confirms the proxy removed the queued message. - A host failure while a callback is running has an inherently ambiguous result because arbitrary reward commands and external plugin effects cannot share a transaction with the transport journal. Such a delivery remains quarantined for operator investigation, avoiding both an automatic duplicate reward and an acknowledgement that could hide a missed reward. +- HTTP vote-party threshold commits journal the exact proxy broadcast and ordered proxy commands alongside backend reward deliveries. Pending proxy effects resume after restart and advance only after execution completes and progress is fsynced. These local effects are at-least-once: a crash after an effect executes but before its progress record is durable can repeat that effect rather than silently skip it. A Velocity command that does not complete within 60 seconds is moved to the durable `VoteParty.QuarantinedProxyEffects` list without automatic retry so it cannot block later vote-party work or execute concurrently with a retry; operators must inspect and resolve quarantined commands. - Backend-to-proxy messages retain the existing bounded in-process retry semantics. Vote caching remains responsible for application-level durability in that direction. The listener must terminate TLS itself because client-certificate authentication is part of the protocol. Do not put an HTTP TLS-terminating reverse proxy or CDN in front of it. A TCP/L4 proxy that passes TLS through unchanged is suitable. Internet-facing installations should also use the host firewall or provider firewall for volumetric denial-of-service protection; an application cannot fully absorb a link or TCP flood. From a88e7059268c61953b788c69ace029845c9d650f Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:15:21 -0600 Subject: [PATCH 041/136] Use SimpleAPI HTTP transport --- VotingPlugin/pom.xml | 16 +- .../votingplugin/VotingPluginMain.java | 28 +- .../backendproxy/BackendProxyHandler.java | 6 +- .../http/HttpBackendTransportConnector.java | 509 ------------ .../http/HttpClientCredentialStore.java | 377 --------- .../backendproxy/http/HttpConnectionCode.java | 112 --- .../http/HttpEnrollmentAuthority.java | 212 ----- .../http/HttpInboundDeliveryStore.java | 173 ---- .../backendproxy/http/HttpPinnedTls.java | 103 --- .../http/HttpProxyTransportServer.java | 499 ------------ .../backendproxy/http/HttpTlsIdentity.java | 467 ----------- .../http/HttpTransportProtocol.java | 231 ------ .../http/HttpTransportSecrets.java | 64 -- .../transport/HttpBackendProxyTransport.java | 23 +- .../votingplugin/proxy/VotingPluginProxy.java | 8 +- .../proxy/VotingPluginProxyCommand.java | 4 +- .../BackendProxyHandlerLifecycleTest.java | 102 +++ .../http/HttpTransportRuntimeTest.java | 765 ------------------ .../http/HttpTransportSecurityTest.java | 401 --------- .../HttpBackendProxyTransportTest.java | 26 +- .../BackendConfigurationServiceTest.java | 22 +- .../tests/VotingPluginProxyTest.java | 8 +- .../tests/VotingPluginProxyTestImpl.java | 2 +- docs/http-transport.md | 4 +- 24 files changed, 177 insertions(+), 3985 deletions(-) delete mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java delete mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java delete mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java delete mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java delete mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java delete mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java delete mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java delete mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java delete mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java delete mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecrets.java delete mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java delete mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java diff --git a/VotingPlugin/pom.xml b/VotingPlugin/pom.xml index 646b1e8ac..144bc03f9 100644 --- a/VotingPlugin/pom.xml +++ b/VotingPlugin/pom.xml @@ -15,7 +15,6 @@ 21 21 3.4.0 - 1.85 @@ -69,6 +68,11 @@ + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.4 + org.apache.maven.plugins maven-compiler-plugin @@ -259,16 +263,6 @@ - - org.bouncycastle - bcpkix-jdk18on - ${bouncycastle.version} - - - org.bouncycastle - bcprov-jdk18on - ${bouncycastle.version} - org.spigotmc spigot-api diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index f645d7bd1..b5cda2d0b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1231,22 +1231,23 @@ public void restartBackendProxyHandler(long validationDeadlineNanos) { } } - /** Prepared on the Bukkit thread, validated off-thread, then atomically published on Bukkit. */ + /** Constructed on the Bukkit thread, prepared and validated off-thread, then atomically published on Bukkit. */ public static final class BackendProxyRestart { private final BackendProxyHandler previous; private final BackendProxyHandler replacement; private final boolean disabled; - private final boolean previousPrepared; + private final boolean previousRequiresPreparation; + private volatile boolean previousPrepared; private boolean finished; private boolean abandonmentRequested; private volatile boolean published; private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler replacement, boolean disabled, - boolean previousPrepared) { + boolean previousRequiresPreparation) { this.previous = previous; this.replacement = replacement; this.disabled = disabled; - this.previousPrepared = previousPrepared; + this.previousRequiresPreparation = previousRequiresPreparation; } } @@ -1268,25 +1269,25 @@ public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { return new BackendProxyRestart(previous, null, true, false); } BungeeMethod replacementMethod = BungeeMethod.getByName(bungeeSettings.getBungeeMethod()); - boolean previousPrepared = previous != null && previous.prepareForReplacement(replacementMethod); + boolean previousRequiresPreparation = previous != null && previous.getMethod() == replacementMethod + && replacementMethod == BungeeMethod.HTTP; BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); try { replacement.loadForReplacement(); } catch (RuntimeException failure) { replacement.close(); - if (previousPrepared) { - previous.restoreAfterFailedReplacement(); - BackendProxyRestart failed = new BackendProxyRestart(previous, null, false, true); - failed.finished = true; - throw new BackendProxyRestartPreparationException(failed, failure); - } throw failure; } - return new BackendProxyRestart(previous, replacement, false, previousPrepared); + return new BackendProxyRestart(previous, replacement, false, previousRequiresPreparation); } public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long validationDeadlineNanos) { if (restart == null) throw new IllegalArgumentException("Backend proxy restart is required"); + if (restart.previousRequiresPreparation && !restart.previousPrepared) { + if (!restart.previous.prepareForReplacement(restart.replacement.getMethod())) + throw new IllegalStateException("Previous HTTP proxy transport could not be prepared for replacement"); + restart.previousPrepared = true; + } if (restart.replacement != null) restart.replacement.validateTransport(validationDeadlineNanos); } @@ -1359,6 +1360,9 @@ public void onReceive(com.bencodez.simpleapi.servercomm.codec.JsonEnvelope envel } com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessage current = getPluginMessaging(); if (backendPluginMessageRelayOwner != current) { + if (backendPluginMessageRelayOwner != null) { + backendPluginMessageRelayOwner.getPluginMessages().remove(backendPluginMessageRelay); + } current.add(backendPluginMessageRelay); backendPluginMessageRelayOwner = current; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 0f652cfc3..a314bb889 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -121,11 +121,11 @@ public void close() { globalDataSync.close(); } - /** Releases a same-method subscriber/listener before its replacement starts. */ + /** Prepares the restorable HTTP transport before a same-method replacement starts. */ public boolean prepareForReplacement(BungeeMethod replacementMethod) { - if (method == replacementMethod && method != BungeeMethod.PLUGINMESSAGING && method != BungeeMethod.REDIS) { + if (method == replacementMethod && method == BungeeMethod.HTTP) { transportManager.prepareForReplacement(); - return method == BungeeMethod.HTTP; + return true; } return false; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java deleted file mode 100644 index 83a5ac91b..000000000 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ /dev/null @@ -1,509 +0,0 @@ -package com.bencodez.votingplugin.backendproxy.http; - -import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.net.http.HttpTimeoutException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.security.KeyStore; -import java.security.cert.X509Certificate; -import java.time.Clock; -import java.time.Duration; -import java.util.ArrayDeque; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Flow; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; - -/** Backend-side, persistent HTTP/1.1 long-poll connector. */ -public final class HttpBackendTransportConnector implements AutoCloseable { - public static final Duration CLIENT_TIMEOUT = Duration.ofSeconds(35); - static final int CALLBACK_QUEUE_CAPACITY = 128; - private volatile HttpClientCredentialStore.HttpClientProfile profile; - private final String serverId; - private final Consumer onEnvelope; - private volatile HttpClient client; - private volatile HttpClientCredentialStore.ClientCredential credential; - private volatile HttpClientCredentialStore.ActiveCredentialGeneration activeCredentialGeneration; - private final Path credentialDirectory; - private final HttpInboundDeliveryStore inboundDeliveries; - private final URI transportEndpoint; - private final ThreadPoolExecutor callbackExecutor; - private final AtomicBoolean running = new AtomicBoolean(); - private final CountDownLatch firstResponse = new CountDownLatch(1); - private final Object state = new Object(); - private final Object renewal = new Object(); - private final LinkedHashMap outgoing = new LinkedHashMap<>(); - private final Set received = new LinkedHashSet<>(), processing = new LinkedHashSet<>(); - private final ArrayDeque acknowledgements = new ArrayDeque<>(); - private final String session = UUID.randomUUID().toString(); - private volatile Thread poller; - private long sequence; - private volatile long nextRenewalCheckNanos; - - /** In-memory test constructor; production transport must use a directory-backed constructor. */ - HttpBackendTransportConnector(HttpConnectionCode code, String serverId, - HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { - this(profile(code, serverId), credential, onEnvelope); - } - - /** In-memory test constructor; production transport must use a directory-backed constructor. */ - HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope) throws Exception { - this(enrolled, onEnvelope, null); - } - - /** In-memory test constructor; production transport must use a directory-backed constructor. */ - HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, - HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { - this(profile, credential, onEnvelope, null); - } - - private HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope, - Path credentialDirectory) throws Exception { - this(enrolled == null ? null : enrolled.profile(), enrolled == null ? null : enrolled.credential(), onEnvelope, - credentialDirectory, credentialDirectory == null ? null - : HttpClientCredentialStore.snapshotActiveGeneration(credentialDirectory)); - } - - private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, - HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope, Path credentialDirectory) throws Exception { - this(profile, credential, onEnvelope, credentialDirectory, null); - } - - private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, - HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope, Path credentialDirectory, - HttpClientCredentialStore.ActiveCredentialGeneration activeCredentialGeneration) throws Exception { - if (profile == null || credential == null || onEnvelope == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); - if (!matchesCredential(profile, credential)) throw new IllegalArgumentException("HTTP client certificate does not match transport profile"); - this.profile = profile; this.serverId = profile.serverId(); this.onEnvelope = onEnvelope; - this.credential = credential; - this.activeCredentialGeneration = activeCredentialGeneration; - this.credentialDirectory = credentialDirectory; - inboundDeliveries = credentialDirectory == null ? null : new HttpInboundDeliveryStore(credentialDirectory); - if (inboundDeliveries != null) for (var entry : inboundDeliveries.snapshot().entrySet()) { - if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { - received.add(entry.getKey()); - queueAck(entry.getKey()); - } - } - client = client(profile, credential); - transportEndpoint = profile.endpoint().resolve("v1/transport"); - // GlobalMessageHandler routes mutate backend vote state and must observe the - // wire order. One bounded lane preserves batch ordering without running work on - // the long-poll thread; bounded admission below backpressures this poller. - callbackExecutor = executor("VotingPlugin-HTTP-callback", 1, CALLBACK_QUEUE_CAPACITY); - } - - /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ - public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, Path credentials, - Consumer onEnvelope) throws Exception { - this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope, credentials); - if (code == null || !profile(code, serverId).equals(this.profile)) throw new IllegalArgumentException("HTTP transport profile does not match connection code"); - } - - /** Starts normal transport using only the persisted certificate and non-secret profile. */ - public HttpBackendTransportConnector(Path credentials, Consumer onEnvelope) throws Exception { - this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope, credentials); - } - - /** Performs enrollment network I/O; call this from a connector/setup worker, never a platform main thread. */ - public static HttpClientCredentialStore.ClientCredential enroll(HttpConnectionCode code, String serverId, Path credentials) throws Exception { - if (code == null || credentials == null || serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) throw new IllegalArgumentException("Enrollment configuration is invalid"); - if (!code.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) - throw new IllegalArgumentException("HTTP connection code belongs to a different backend"); - code.requireActive(Clock.systemUTC()); - byte[] payload = ("{\"server\":\"" + serverId + "\",\"token\":\"" + code.enrollmentToken() + "\"}").getBytes(StandardCharsets.UTF_8); - HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) - .connectTimeout(Duration.ofSeconds(5)).sslContext(HttpPinnedTls.clientContext(code)).build(); - HttpRequest request = HttpRequest.newBuilder(code.endpoint().resolve("v1/enroll")).timeout(CLIENT_TIMEOUT) - .header("Content-Type", "application/json").header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(payload)).build(); - LimitedResponse response = sendLimited(client, request); - if (response.statusCode() != 201) throw new IllegalArgumentException("Enrollment was rejected"); - HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); - HttpClientCredentialStore.saveEnrolled(credentials, code, issued); return HttpClientCredentialStore.load(credentials); - } - - public void start() { - if (!running.compareAndSet(false, true)) return; - poller = new Thread(this::pollLoop, "VotingPlugin-HTTP-poll"); poller.setDaemon(true); poller.start(); - } - /** Returns the credential generation already validated off the platform main thread. */ - public HttpClientCredentialStore.ActiveCredentialGeneration activeCredentialGeneration() { - return activeCredentialGeneration; - } - - /** Waits for one authenticated, protocol-valid transport response. */ - public boolean awaitFirstResponse(long deadlineNanos) throws InterruptedException { - long remaining = deadlineNanos - System.nanoTime(); - return remaining > 0L && firstResponse.await(remaining, TimeUnit.NANOSECONDS) && running.get(); - } - /** - * Inserts an in-memory at-least-once delivery. It survives retry/lost responses while this process remains alive; - * callers needing restart durability must retain the application operation independently. - */ - public boolean send(JsonEnvelope envelope) { - if (envelope == null || !running.get()) return false; - try { HttpTransportProtocol.validateEnvelope(envelope); } - catch (IllegalArgumentException invalid) { return false; } - synchronized (state) { - if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; - String id = UUID.randomUUID().toString(); outgoing.put(id, new HttpTransportProtocol.Delivery(id, envelope)); return true; - } - } - /** A synchronous single poll, useful for lifecycle-controlled integrations and tests. */ - public synchronized boolean pollOnce() { - return pollOnce(CLIENT_TIMEOUT, true, true); - } - private boolean pollOnce(Duration timeout, boolean requireRunning, boolean acceptIncoming) { - if (requireRunning && !running.get()) return false; - List acks = List.of(); boolean acknowledgementsConfirmed = false; - try { - if (requireRunning) maybeRenewCredential(); - List messages; long requestSequence; - synchronized (state) { - acks = first(acknowledgements); requestSequence = sequence++; - messages = HttpTransportProtocol.fittingMessages(serverId, session, requestSequence, acks, outgoing.values()); - for (int index = 0; index < acks.size(); index++) acknowledgements.removeFirst(); - } - HttpRequest request = HttpRequest.newBuilder(transportEndpoint).timeout(timeout).header("Content-Type", "application/json") - .header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(HttpTransportProtocol.request(serverId, session, requestSequence, acks, messages))).build(); - LimitedResponse response = sendLimited(client, request); - if (response.statusCode() != 200) return false; - HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(response.body()); - if (!serverId.equals(packet.server()) || !session.equals(packet.session()) || packet.sequence() != requestSequence) return false; - confirmAcknowledgements(acks); - acknowledgementsConfirmed = true; - synchronized (state) { for (String ack : packet.acks()) outgoing.remove(ack); } - if (acceptIncoming) for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); - firstResponse.countDown(); - return true; - } catch (Exception failure) { return false; - } finally { if (!acknowledgementsConfirmed) requeueAcknowledgements(acks); } - } - /** Stops normal polling and gives already-queued outbound messages a bounded final delivery attempt. */ - public boolean flushOutgoing(long deadlineNanos) { - running.set(false); - firstResponse.countDown(); - Thread current = poller; - if (current != null) current.interrupt(); - if (!joinPoller(current, deadlineNanos)) return false; - while (queuedOutgoing() != 0) { - long remaining = deadlineNanos - System.nanoTime(); - if (remaining <= 0L) return false; - Duration timeout = Duration.ofNanos(Math.min(CLIENT_TIMEOUT.toNanos(), remaining)); - synchronized (this) { - if (pollOnce(timeout, false, false)) continue; - } - // The proxy may retain its one-active-poll guard briefly after the old - // client request is interrupted. Retry that transient 409 without busy - // spinning, but never extend the caller's shutdown deadline. - remaining = deadlineNanos - System.nanoTime(); - if (remaining <= 0L) return false; - try { TimeUnit.NANOSECONDS.sleep(Math.min(TimeUnit.MILLISECONDS.toNanos(50), remaining)); } - catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); return false; } - } - return true; - } - @Override public void close() { - running.getAndSet(false); - firstResponse.countDown(); - // Revoke this connector's journal writer before a replacement snapshots it. - // In-flight transitions serialize with seal(): either COMPLETED is already - // durable, or the delivery remains durably RUNNING and fail-closed. - if (inboundDeliveries != null) inboundDeliveries.seal(); - Thread current = poller; if (current != null) current.interrupt(); - callbackExecutor.shutdown(); try { if (!callbackExecutor.awaitTermination(5, TimeUnit.SECONDS)) callbackExecutor.shutdownNow(); } - catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); callbackExecutor.shutdownNow(); } - // The owning transport may release the credential-directory semaphore as soon - // as close returns. Wait for the interrupted poller so an in-flight renewal - // cannot activate an old credential generation after that ownership handoff. - joinPoller(current); - } - boolean pollerAlive() { Thread current = poller; return current != null && current.isAlive(); } - private static boolean joinPoller(Thread poller, long deadlineNanos) { - if (poller == null || poller == Thread.currentThread()) return true; - boolean interrupted = false; - while (poller.isAlive()) { - long remaining = deadlineNanos - System.nanoTime(); - if (remaining <= 0L) { - if (interrupted) Thread.currentThread().interrupt(); - return false; - } - try { TimeUnit.NANOSECONDS.timedJoin(poller, remaining); } - catch (InterruptedException stopRequested) { interrupted = true; poller.interrupt(); } - } - if (interrupted) Thread.currentThread().interrupt(); - return true; - } - private static void joinPoller(Thread poller) { - if (poller == null || poller == Thread.currentThread()) return; - boolean interrupted = false; - while (poller.isAlive()) try { poller.join(); } - catch (InterruptedException stopRequested) { interrupted = true; poller.interrupt(); } - if (interrupted) Thread.currentThread().interrupt(); - } - - private void pollLoop() { - long retry = 1000L; - while (running.get()) { if (pollOnce()) { retry = 1000L; continue; } try { Thread.sleep(retry); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); break; } retry = Math.min(30_000L, retry * 2); } - } - List accept(List deliveries) { - synchronized (state) { - List accepted = new java.util.ArrayList<>(); - for (HttpTransportProtocol.Delivery delivery : deliveries) { - HttpInboundDeliveryStore.State persisted = inboundDeliveries == null ? null : inboundDeliveries.state(delivery.id()); - if (received.contains(delivery.id()) || persisted == HttpInboundDeliveryStore.State.COMPLETED) { - received.add(delivery.id()); queueAck(delivery.id()); continue; - } - // A callback that was running when the process stopped may already have - // produced external side effects. Keep the proxy copy without replaying or - // acknowledging it; arbitrary plugin callbacks cannot share this journal. - if (persisted == HttpInboundDeliveryStore.State.RUNNING) continue; - if (!processing.contains(delivery.id())) { - processing.add(delivery.id()); accepted.add(delivery); - } - } - return accepted; - } - } - void dispatch(HttpTransportProtocol.Delivery delivery) { - Runnable callback = () -> { - boolean success = false; - try { - if (inboundDeliveries != null) { - if (inboundDeliveries.state(delivery.id()) == null) inboundDeliveries.reserve(delivery.id()); - inboundDeliveries.markRunning(delivery.id()); - } - onEnvelope.accept(delivery.envelope()); - if (inboundDeliveries != null) inboundDeliveries.markCompleted(delivery.id()); - success = true; - } catch (IOException persistenceFailure) { - // Never run before RUNNING is durable and never acknowledge until - // COMPLETED is durable. An uncertain transition stays fail-closed. - } catch (RuntimeException callbackFailure) { - // The callback may have failed after partial external effects. Leave RUNNING - // unacknowledged so a restart cannot silently lose or duplicate the delivery. - } - completeIncoming(delivery.id(), success); - }; - if (!executeOrdered(callbackExecutor, callback)) completeIncoming(delivery.id(), false); - } - void completeIncoming(String id, boolean success) { synchronized (state) { - processing.remove(id); - if (success) { received.add(id); while (received.size() > HttpTransportProtocol.MAX_QUEUE) received.remove(received.iterator().next()); queueAck(id); } - } } - private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } - private void requeueAcknowledgements(List ids) { synchronized (state) { - for (int index = ids.size() - 1; index >= 0; index--) { - String id = ids.get(index); - if (!acknowledgements.contains(id)) { - while (acknowledgements.size() >= HttpTransportProtocol.MAX_QUEUE) acknowledgements.removeLast(); - acknowledgements.addFirst(id); - } - } - } } - private void confirmAcknowledgements(Collection ids) { - for (String id : ids) { - if (inboundDeliveries != null) try { inboundDeliveries.remove(id); } - catch (IOException cleanupFailure) { continue; } - synchronized (state) { received.remove(id); } - } - } - int queuedOutgoing() { synchronized (state) { return outgoing.size(); } } - List drainAcknowledgements() { synchronized (state) { return drain(acknowledgements); } } - private static List first(Collection values) { List output = new java.util.ArrayList<>(); for (T value : values) { output.add(value); if (output.size() == HttpTransportProtocol.MAX_BATCH) break; } return output; } - private static List drain(ArrayDeque values) { List output = new java.util.ArrayList<>(); while (!values.isEmpty() && output.size() < HttpTransportProtocol.MAX_BATCH) output.add(values.remove()); return output; } - private static ThreadPoolExecutor executor(String name, int threads, int queue) { ThreadFactory factory = task -> { Thread thread = new Thread(task, name); thread.setDaemon(true); return thread; }; return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); } - static boolean executeOrdered(ThreadPoolExecutor executor, Runnable task) { - try { executor.execute(task); return true; } - catch (RejectedExecutionException fullOrClosed) { - if (executor.isShutdown()) return false; - try { - while (!executor.isShutdown()) { - if (!executor.getQueue().offer(task, 100L, TimeUnit.MILLISECONDS)) continue; - if (executor.isShutdown() && executor.remove(task)) return false; - return true; - } - } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } - return false; - } - } - private static HttpClientCredentialStore.HttpClientProfile profile(HttpConnectionCode code, String serverId) { - if (code == null || serverId == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); - if (!code.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalArgumentException("HTTP connection code belongs to a different backend"); - return new HttpClientCredentialStore.HttpClientProfile(serverId, code.endpoint(), code.serverCertificatePin(), code.caCertificatePin()); - } - private static boolean matchesCredential(HttpClientCredentialStore.HttpClientProfile profile, - HttpClientCredentialStore.ClientCredential credential) { - try { - credential.certificate().checkValidity(); credential.certificate().verify(credential.caCertificate().getPublicKey()); - String expected = "urn:votingplugin:http-backend:" + profile.serverId(); - var names = credential.certificate().getSubjectAlternativeNames(); if (names == null) return false; - for (java.util.List name : names) if (name.size() == 2 && Integer.valueOf(6).equals(name.get(0)) && expected.equals(name.get(1))) return true; - return false; - } catch (Exception invalid) { return false; } - } - private void maybeRenewCredential() { - synchronized (renewal) { - Path directory = credentialDirectory; - if (directory == null || !HttpTlsIdentity.needsRenewal(credential.certificate(), Clock.systemUTC())) return; - long now = System.nanoTime(); - if (nextRenewalCheckNanos != 0L && now - nextRenewalCheckNanos < 0L) return; - nextRenewalCheckNanos = now + Duration.ofHours(6).toNanos(); - try { - byte[] body = HttpTransportProtocol.renewalRequest(serverId); - HttpRequest request = HttpRequest.newBuilder(profile.endpoint().resolve("v1/renew")).timeout(CLIENT_TIMEOUT) - .header("Content-Type", "application/json").header("Cache-Control", "no-store") - .POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(); - LimitedResponse response = sendLimited(client, request); - if (response.statusCode() != 201) return; - HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); - HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(directory, issued); - HttpClientCredentialStore.ClientCredential replacement = staged.credential(); - HttpClientCredentialStore.HttpClientProfile replacementProfile = staged.profile(); - if (!matchesCredential(replacementProfile, replacement)) throw new IllegalArgumentException("Renewed HTTP certificate is invalid"); - HttpClient replacementClient = client(replacementProfile, replacement); - HttpClientCredentialStore.activateReplacement(directory, staged); - activeCredentialGeneration = new HttpClientCredentialStore.ActiveCredentialGeneration(staged.name(), - replacementProfile, staged.connectionCodeDigest()); - profile = replacementProfile; - client = replacementClient; - credential = replacement; - } catch (Exception ignored) { /* The active generation is unchanged; retry on the bounded schedule. */ } - } - } - private static HttpClient client(HttpClientCredentialStore.HttpClientProfile profile, - HttpClientCredentialStore.ClientCredential credential) throws Exception { - return HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) - .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); - } - static LimitedResponse sendLimited(HttpClient client, HttpRequest request) throws IOException, InterruptedException { - CompletableFuture> exchange = client.sendAsync(request, - ignored -> new LimitedBodySubscriber(HttpTransportProtocol.MAX_BODY_BYTES)); - HttpResponse response; - try { - Duration timeout = request.timeout().orElse(CLIENT_TIMEOUT); - response = exchange.get(timeout.toMillis(), TimeUnit.MILLISECONDS); - } catch (TimeoutException timeout) { - exchange.cancel(true); - throw new HttpTimeoutException("HTTP transport response timed out"); - } catch (InterruptedException interrupted) { - exchange.cancel(true); - throw interrupted; - } catch (ExecutionException failure) { - Throwable cause = failure.getCause(); - if (cause instanceof IOException ioFailure) throw ioFailure; - throw new IOException("HTTP transport request failed", cause); - } - long declaredLength = response.headers().firstValueAsLong("Content-Length").orElse(-1L); - if (declaredLength > HttpTransportProtocol.MAX_BODY_BYTES) - throw new IOException("HTTP transport response exceeds its limit"); - return new LimitedResponse(response.statusCode(), response.body()); - } - - private static final class LimitedBodySubscriber implements HttpResponse.BodySubscriber { - private final int maximum; - private final ByteArrayOutputStream body = new ByteArrayOutputStream(); - private final CompletableFuture result = new CompletableFuture<>(); - private Flow.Subscription subscription; - - private LimitedBodySubscriber(int maximum) { - this.maximum = maximum; - } - - @Override - public CompletionStage getBody() { - return result; - } - - @Override - public void onSubscribe(Flow.Subscription subscription) { - if (this.subscription != null) { - subscription.cancel(); - return; - } - this.subscription = subscription; - subscription.request(1); - } - - @Override - public void onNext(List buffers) { - try { - for (ByteBuffer buffer : buffers) { - if (buffer.remaining() > maximum - body.size()) { - subscription.cancel(); - result.completeExceptionally(new IOException("HTTP transport response exceeds its limit")); - return; - } - byte[] chunk = new byte[buffer.remaining()]; - buffer.get(chunk); - body.writeBytes(chunk); - } - subscription.request(1); - } catch (RuntimeException failure) { - subscription.cancel(); - result.completeExceptionally(failure); - } - } - - @Override - public void onError(Throwable failure) { - result.completeExceptionally(failure); - } - - @Override - public void onComplete() { - result.complete(body.toByteArray()); - } - } - static byte[] readLimited(InputStream body) throws IOException { - byte[] bytes = body.readNBytes(HttpTransportProtocol.MAX_BODY_BYTES + 1); - if (bytes.length > HttpTransportProtocol.MAX_BODY_BYTES) - throw new IOException("HTTP transport response exceeds its limit"); - return bytes; - } - record LimitedResponse(int statusCode, byte[] body) { } - private static SSLContext clientContext(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { - String caPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); - if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), - caPin.getBytes(StandardCharsets.US_ASCII))) throw new IllegalArgumentException("HTTP authority does not match transport profile"); - char[] password = credential.password(); - try { - KeyStore store = KeyStore.getInstance("PKCS12"); store.load(null, new char[0]); - store.setKeyEntry("client", credential.privateKey(), password, - new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); - KeyManagerFactory keys = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keys.init(store, password); - KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); trustStore.load(null, new char[0]); - trustStore.setCertificateEntry("http-transport-ca", credential.caCertificate()); - javax.net.ssl.TrustManagerFactory trusts = javax.net.ssl.TrustManagerFactory.getInstance( - javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()); - trusts.init(trustStore); - SSLContext context = SSLContext.getInstance("TLS"); context.init(keys.getKeyManagers(), trusts.getTrustManagers(), null); return context; - } finally { java.util.Arrays.fill(password, '\0'); } - } -} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java deleted file mode 100644 index 5cf6094ea..000000000 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ /dev/null @@ -1,377 +0,0 @@ -package com.bencodez.votingplugin.backendproxy.http; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; -import java.net.URI; -import java.security.KeyStore; -import java.security.PrivateKey; -import java.security.cert.X509Certificate; -import java.util.Base64; -import java.util.EnumSet; -import java.util.Properties; -import com.bencodez.votingplugin.util.DurableFiles; - -/** Owner-only persistence for the client certificate bundle returned by enrollment. */ -public final class HttpClientCredentialStore { - private static final String BUNDLE_FILE = "http-transport-client.p12"; - private static final String PASSWORD_FILE = "http-transport-client-password"; - private static final String PROFILE_FILE = "http-transport-profile.properties"; - private static final String CONNECTION_CODE_DIGEST_FILE = "http-transport-connection-code.sha256"; - private static final String GENERATIONS_DIRECTORY = "http-transport-client-generations"; - private static final String CURRENT_FILE = "http-transport-client-current"; - private HttpClientCredentialStore() { } - - public static void save(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws IOException { - if (issued == null) throw new IllegalArgumentException("Issued credential is required"); - Files.createDirectories(directory); - byte[] bundle = issued.pkcs12(); - try { writePrivate(safe(directory.resolve(BUNDLE_FILE)), bundle); } - finally { java.util.Arrays.fill(bundle, (byte) 0); } - char[] password = issued.password(); - try { writePrivate(safe(directory.resolve(PASSWORD_FILE)), asciiBytes(password)); } - finally { java.util.Arrays.fill(password, '\0'); } - } - - /** Persists the certificate plus the non-secret normal-transport profile after enrollment. */ - public static void saveEnrolled(Path directory, HttpConnectionCode code, HttpTlsIdentity.IssuedClientCertificate issued) - throws IOException { - if (code == null || issued == null) throw new IllegalArgumentException("Connection code and credential are required"); - HttpClientProfile profile = new HttpClientProfile(HttpTlsIdentity.canonicalServerId(issued.serverId()), code.endpoint(), - code.serverCertificatePin(), code.caCertificatePin()); - try { - StagedCredential staged = stage(directory, issued, profile, connectionCodeDigest(code)); - activateReplacement(directory, staged); - } catch (IOException failure) { throw failure; - } catch (Exception failure) { throw new IOException("Could not persist HTTP client credential", failure); } - } - - private static void writeProfile(Path directory, HttpClientProfile profile) throws IOException { - Properties properties = new Properties(); - properties.setProperty("version", "1"); - properties.setProperty("serverId", profile.serverId()); - properties.setProperty("endpoint", profile.endpoint().toASCIIString()); - properties.setProperty("serverPin", profile.serverCertificatePin()); - properties.setProperty("caPin", profile.caCertificatePin()); - java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); - properties.store(bytes, "VotingPlugin HTTP transport profile"); - writePrivate(safe(directory.resolve(PROFILE_FILE)), bytes.toByteArray()); - } - - public static ClientCredential load(Path directory) throws Exception { - return loadCredential(activeDirectory(directory)); - } - - private static ClientCredential loadCredential(Path directory) throws Exception { - Path bundle = safe(directory.resolve(BUNDLE_FILE)); - Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); - if (!Files.isRegularFile(bundle, LinkOption.NOFOLLOW_LINKS) || !Files.isRegularFile(passwordFile, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP client certificate has not been enrolled"); - byte[] passwordBytes = Files.readAllBytes(passwordFile); - if (passwordBytes.length < 40 || passwordBytes.length > 128) throw new IOException("HTTP client password is invalid"); - char[] password = new String(passwordBytes, StandardCharsets.US_ASCII).toCharArray(); - java.util.Arrays.fill(passwordBytes, (byte) 0); - try { - KeyStore store = KeyStore.getInstance("PKCS12"); - try (var input = Files.newInputStream(bundle, LinkOption.NOFOLLOW_LINKS)) { store.load(input, password); } - PrivateKey privateKey = (PrivateKey) store.getKey("client", password); - java.security.cert.Certificate[] chain = store.getCertificateChain("client"); - if (privateKey == null || chain == null || chain.length != 2 - || !(chain[0] instanceof X509Certificate client) || !(chain[1] instanceof X509Certificate authority)) - throw new IOException("HTTP client certificate bundle is invalid"); - return new ClientCredential(privateKey, client, authority, password); - } finally { java.util.Arrays.fill(password, '\0'); } - } - - /** Writes and validates a replacement generation without touching the active credential. */ - static StagedCredential stageReplacement(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws Exception { - Path active = activeDirectory(directory); - return stage(directory, issued, loadProfileFile(active), readConnectionCodeDigest(active)); - } - - private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClientCertificate issued, - HttpClientProfile profile, String connectionCodeDigest) throws Exception { - if (directory == null || issued == null) throw new IllegalArgumentException("Credential replacement is required"); - Path credentialDirectory = directory.toAbsolutePath().normalize(); - boolean created = !Files.exists(credentialDirectory, LinkOption.NOFOLLOW_LINKS); - Path generations = credentialDirectory.resolve(GENERATIONS_DIRECTORY); - Files.createDirectories(generations); - // Credential files cannot make the newly created credential-root entry durable. - // Persist its parent before an enrolled transport can activate this root. - if (created) DurableFiles.forceDirectory(credentialDirectory.getParent()); - if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP credential generation directory is unsafe"); - String name = java.util.UUID.randomUUID().toString(); - Path generation = generations.resolve(name); - Files.createDirectory(generation); - setOwnerOnlyDirectory(generation); - try { - save(generation, issued); - ClientCredential replacement = loadCredential(generation); - profile = new HttpClientProfile(profile.serverId(), profile.endpoint(), profile.serverCertificatePin(), - HttpTransportSecrets.certificatePin(replacement.caCertificate())); - writeProfile(generation, profile); - if (connectionCodeDigest != null) writePrivate(safe(generation.resolve(CONNECTION_CODE_DIGEST_FILE)), - connectionCodeDigest.getBytes(StandardCharsets.US_ASCII)); - EnrolledClient enrolled = loadEnrolled(generation); - // Each file is durable within the generation, but the generation name is - // published by its parent. Persist it before CURRENT can activate it. - DurableFiles.forceDirectory(generations); - return new StagedCredential(name, enrolled.credential(), enrolled.profile(), connectionCodeDigest); - } catch (Exception failure) { - try { Files.deleteIfExists(generation.resolve(BUNDLE_FILE)); Files.deleteIfExists(generation.resolve(PASSWORD_FILE)); - Files.deleteIfExists(generation.resolve(PROFILE_FILE)); Files.deleteIfExists(generation.resolve(CONNECTION_CODE_DIGEST_FILE)); - Files.deleteIfExists(generation); } - catch (IOException cleanup) { failure.addSuppressed(cleanup); } - throw failure; - } - } - - /** Atomically makes a fully validated generation durable and active. */ - static void activateReplacement(Path directory, StagedCredential staged) throws IOException { - if (directory == null || staged == null || !staged.name().matches("[0-9a-f-]{36}")) - throw new IllegalArgumentException("Staged credential is invalid"); - Path generations = directory.toAbsolutePath().normalize().resolve(GENERATIONS_DIRECTORY); - Path generation = generations.resolve(staged.name()).normalize(); - if (!generation.getParent().equals(generations) || Files.isSymbolicLink(generation) - || !Files.isRegularFile(generation.resolve(BUNDLE_FILE), LinkOption.NOFOLLOW_LINKS) - || !Files.isRegularFile(generation.resolve(PASSWORD_FILE), LinkOption.NOFOLLOW_LINKS) - || !Files.isRegularFile(generation.resolve(PROFILE_FILE), LinkOption.NOFOLLOW_LINKS)) - throw new IOException("Staged HTTP credential is incomplete"); - writePrivate(safe(directory.resolve(CURRENT_FILE)), staged.name().getBytes(StandardCharsets.US_ASCII)); - } - - static record StagedCredential(String name, ClientCredential credential, HttpClientProfile profile, - String connectionCodeDigest) { } - - public static HttpClientProfile loadProfile(Path directory) throws IOException { - return loadProfileFile(activeDirectory(directory)); - } - - private static HttpClientProfile loadProfileFile(Path directory) throws IOException { - Path profile = safe(directory.resolve(PROFILE_FILE)); - if (!Files.isRegularFile(profile, LinkOption.NOFOLLOW_LINKS) || Files.size(profile) > 8192) - throw new IOException("HTTP transport profile has not been enrolled"); - Properties properties = new Properties(); - try (var input = Files.newInputStream(profile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } - if (properties.size() != 5 || !"1".equals(properties.getProperty("version"))) - throw new IOException("HTTP transport profile is invalid"); - try { - return new HttpClientProfile(properties.getProperty("serverId"), URI.create(properties.getProperty("endpoint")), - properties.getProperty("serverPin"), properties.getProperty("caPin")); - } catch (IllegalArgumentException failure) { throw new IOException("HTTP transport profile is invalid", failure); } - } - - public static boolean hasEnrolledProfile(Path directory) { - try { loadEnrolled(directory); return true; } - catch (Exception unavailable) { return false; } - } - - /** Returns whether this exact one-time code created the active credential, without persisting the code itself. */ - public static boolean matchesEnrollmentCode(Path directory, HttpConnectionCode code) throws IOException { - if (code == null) throw new IllegalArgumentException("Connection code is required"); - String stored = readConnectionCodeDigest(activeDirectory(directory)); - if (stored == null) return false; - byte[] storedBytes = stored.getBytes(StandardCharsets.US_ASCII); - return HttpTransportSecrets.constantTimeEquals(storedBytes, - connectionCodeDigest(code.encode()).getBytes(StandardCharsets.US_ASCII)) - || HttpTransportSecrets.constantTimeEquals(storedBytes, - connectionCodeDigest(code.encodeLegacy()).getBytes(StandardCharsets.US_ASCII)); - } - - /** Loads and cross-checks the persisted client key material and bound normal-transport profile. */ - public static EnrolledClient loadEnrolled(Path directory) throws Exception { - Path active = activeDirectory(directory); - return loadEnrolledDirectory(active); - } - - private static EnrolledClient loadEnrolledDirectory(Path active) throws Exception { - ClientCredential credential = loadCredential(active); - HttpClientProfile profile = loadProfileFile(active); - if (!matchesProfile(credential, profile)) throw new IOException("HTTP client certificate does not match its profile"); - return new EnrolledClient(profile, credential); - } - - /** Captures the exact credential generation used by a transport before a staged replacement starts. */ - public static ActiveCredentialGeneration snapshotActiveGeneration(Path directory) throws Exception { - Path root = directory.toAbsolutePath().normalize(); - Path active = activeDirectory(root); - EnrolledClient enrolled = loadEnrolledDirectory(active); - return new ActiveCredentialGeneration(active.equals(root) ? "" : active.getFileName().toString(), - enrolled.profile(), readConnectionCodeDigest(active)); - } - - /** - * Restores a pre-replacement generation unless the replacement already activated a - * newer credential for the same backend endpoint. A successful renewal or same-endpoint - * re-enrollment may revoke the snapshotted certificate at the proxy, so that newer - * generation is the only safe rollback identity. - */ - public static void restoreActiveGenerationAfterReplacement(Path directory, - ActiveCredentialGeneration snapshot) throws Exception { - if (directory == null || snapshot == null) throw new IllegalArgumentException("Credential rollback is required"); - HttpClientProfile previous = snapshot.profile(); - Path active = null; - HttpClientProfile current = null; - if (previous != null) try { - active = activeDirectory(directory); - current = loadEnrolledDirectory(active).profile(); - } catch (Exception unavailable) { active = null; current = null; } - if (active != null && previous.serverId().equals(current.serverId()) - && previous.endpoint().equals(current.endpoint())) { - restoreConnectionCodeDigest(active, snapshot.connectionCodeDigest()); - return; - } - restoreActiveGeneration(directory, snapshot); - } - - /** Atomically restores a previously validated credential generation after replacement rollback. */ - public static void restoreActiveGeneration(Path directory, ActiveCredentialGeneration snapshot) throws Exception { - if (directory == null || snapshot == null) throw new IllegalArgumentException("Credential rollback is required"); - Path root = directory.toAbsolutePath().normalize(); - Path generations = root.resolve(GENERATIONS_DIRECTORY); - Path target = snapshot.name().isEmpty() ? root : generations.resolve(snapshot.name()).normalize(); - if (!snapshot.name().isEmpty() && (!target.getParent().equals(generations) - || Files.isSymbolicLink(target) || !Files.isDirectory(target, LinkOption.NOFOLLOW_LINKS))) - throw new IOException("HTTP client credential generation is invalid"); - loadEnrolledDirectory(target); - Path current = safe(root.resolve(CURRENT_FILE)); - if (snapshot.name().isEmpty()) { - Files.deleteIfExists(current); - DurableFiles.forceDirectory(root); - } else { - writePrivate(current, snapshot.name().getBytes(StandardCharsets.US_ASCII)); - } - } - - public record ClientCredential(PrivateKey privateKey, X509Certificate certificate, X509Certificate caCertificate, char[] password) { - public ClientCredential { password = password.clone(); } - @Override public char[] password() { return password.clone(); } - } - - public record HttpClientProfile(String serverId, URI endpoint, String serverCertificatePin, String caCertificatePin) { - public HttpClientProfile { - serverId = HttpTlsIdentity.canonicalServerId(serverId); - HttpConnectionCode validation = new HttpConnectionCode(serverId, endpoint, serverCertificatePin, caCertificatePin, - java.time.Instant.now().plusSeconds(1), HttpTransportSecrets.randomToken()); - endpoint = validation.endpoint(); - serverCertificatePin = validation.serverCertificatePin(); - caCertificatePin = validation.caCertificatePin(); - } - } - - public record EnrolledClient(HttpClientProfile profile, ClientCredential credential) { } - - public record ActiveCredentialGeneration(String name, HttpClientProfile profile, String connectionCodeDigest) { - public ActiveCredentialGeneration(String name) { this(name, null, null); } - public ActiveCredentialGeneration { - if (name == null || (!name.isEmpty() && !name.matches("[0-9a-f-]{36}"))) - throw new IllegalArgumentException("HTTP client credential generation is invalid"); - if (connectionCodeDigest != null && !connectionCodeDigest.matches("[0-9a-f]{64}")) - throw new IllegalArgumentException("HTTP connection-code marker is invalid"); - } - } - - private static boolean matchesProfile(ClientCredential credential, HttpClientProfile profile) { - try { - String authorityPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); - if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), - authorityPin.getBytes(StandardCharsets.US_ASCII))) return false; - credential.certificate().checkValidity(); - credential.certificate().verify(credential.caCertificate().getPublicKey()); - java.util.List usage = credential.certificate().getExtendedKeyUsage(); - boolean[] keyUsage = credential.certificate().getKeyUsage(); - if (usage == null || !usage.contains(org.bouncycastle.asn1.x509.KeyPurposeId.id_kp_clientAuth.getId()) - || keyUsage == null || !keyUsage[0]) return false; - String expected = "urn:votingplugin:http-backend:" + profile.serverId(); - var names = credential.certificate().getSubjectAlternativeNames(); - if (names == null) return false; - for (java.util.List name : names) if (name.size() == 2 - && Integer.valueOf(6).equals(name.get(0)) && expected.equals(name.get(1))) return true; - return false; - } catch (Exception invalid) { return false; } - } - - private static Path safe(Path file) throws IOException { - if (Files.isSymbolicLink(file)) throw new IOException("Refusing unsafe HTTP credential path"); - return file.toAbsolutePath().normalize(); - } - - private static Path activeDirectory(Path directory) throws IOException { - Path root = directory.toAbsolutePath().normalize(); - Path current = safe(root.resolve(CURRENT_FILE)); - if (!Files.exists(current, LinkOption.NOFOLLOW_LINKS)) return root; - if (!Files.isRegularFile(current, LinkOption.NOFOLLOW_LINKS) || Files.size(current) > 64) - throw new IOException("HTTP client credential pointer is invalid"); - String name = Files.readString(current, StandardCharsets.US_ASCII); - if (!name.matches("[0-9a-f-]{36}")) throw new IOException("HTTP client credential pointer is invalid"); - Path generations = root.resolve(GENERATIONS_DIRECTORY); - Path generation = generations.resolve(name).normalize(); - if (!generation.getParent().equals(generations) || Files.isSymbolicLink(generation) || !Files.isDirectory(generation, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP client credential generation is invalid"); - return generation; - } - - private static String readConnectionCodeDigest(Path directory) throws IOException { - Path digest = safe(directory.resolve(CONNECTION_CODE_DIGEST_FILE)); - if (!Files.exists(digest, LinkOption.NOFOLLOW_LINKS)) return null; - if (!Files.isRegularFile(digest, LinkOption.NOFOLLOW_LINKS) || Files.size(digest) != 64) - throw new IOException("HTTP connection-code marker is invalid"); - String value = Files.readString(digest, StandardCharsets.US_ASCII); - if (!value.matches("[0-9a-f]{64}")) throw new IOException("HTTP connection-code marker is invalid"); - return value; - } - - private static String connectionCodeDigest(HttpConnectionCode code) { - return connectionCodeDigest(code.encode()); - } - - private static String connectionCodeDigest(String encoded) { - return HttpTransportSecrets.sha256Hex(encoded.getBytes(StandardCharsets.US_ASCII)); - } - - private static void restoreConnectionCodeDigest(Path directory, String digest) throws IOException { - Path marker = safe(directory.resolve(CONNECTION_CODE_DIGEST_FILE)); - if (digest == null) { - Files.deleteIfExists(marker); - DurableFiles.forceDirectory(directory); - } else { - writePrivate(marker, digest.getBytes(StandardCharsets.US_ASCII)); - } - } - - private static void writePrivate(Path file, byte[] contents) throws IOException { - Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); - try { - setOwnerOnly(temporary); - Files.write(temporary, contents, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); - DurableFiles.forceFile(temporary); - try { Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } - catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); } - setOwnerOnly(file); - DurableFiles.forceDirectory(file.getParent()); - } finally { Files.deleteIfExists(temporary); } - } - - private static void setOwnerOnly(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } - catch (UnsupportedOperationException ignored) { } - } - - private static void setOwnerOnlyDirectory(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, - PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } - catch (UnsupportedOperationException ignored) { } - } - - private static byte[] asciiBytes(char[] characters) { - byte[] output = new byte[characters.length]; - for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; - return output; - } -} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java deleted file mode 100644 index 62bf1ff62..000000000 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java +++ /dev/null @@ -1,112 +0,0 @@ -package com.bencodez.votingplugin.backendproxy.http; - -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.time.Instant; -import java.util.Base64; -import java.util.Locale; - -/** - * A copy/paste connection code. It deliberately includes no long-lived credential: - * its only secret is a short-lived, single-use enrollment token. The trailing MAC is a - * corruption check keyed by that included token; it is not a proxy signature and cannot stop - * someone who can replace the whole code from replacing it with another valid code. - */ -public record HttpConnectionCode(String serverId, URI endpoint, String serverCertificatePin, String caCertificatePin, - Instant expiresAt, String enrollmentToken) { - private static final String LEGACY_VERSION = "VPH1"; - private static final String VERSION = "VPH2"; - private static final int MAX_CODE_LENGTH = 4096; - - public HttpConnectionCode { - serverId = HttpTlsIdentity.canonicalServerId(serverId); - endpoint = validateEndpoint(endpoint); - serverCertificatePin = validatePin(serverCertificatePin, "server certificate pin"); - caCertificatePin = validatePin(caCertificatePin, "CA certificate pin"); - if (expiresAt == null) throw new IllegalArgumentException("Expiry is required"); - enrollmentToken = validateToken(enrollmentToken); - } - - public String encode() { - String serverPart = Base64.getUrlEncoder().withoutPadding() - .encodeToString(serverId.getBytes(StandardCharsets.UTF_8)); - return encode(VERSION, serverPart); - } - - String encodeLegacy() { - return encode(LEGACY_VERSION, serverId); - } - - private String encode(String version, String serverPart) { - String endpointPart = Base64.getUrlEncoder().withoutPadding() - .encodeToString(endpoint.toASCIIString().getBytes(StandardCharsets.UTF_8)); - String unsigned = String.join(".", version, serverPart, endpointPart, serverCertificatePin, caCertificatePin, - Long.toString(expiresAt.getEpochSecond()), enrollmentToken); - byte[] token = Base64.getUrlDecoder().decode(enrollmentToken); - return unsigned + "." + HttpTransportSecrets.hmacSha256Url(token, unsigned); - } - - public boolean isActive(Clock clock) { - return expiresAt.isAfter(clock.instant()); - } - - public void requireActive(Clock clock) { - if (!isActive(clock)) throw new IllegalArgumentException("Connection code has expired"); - } - - public static HttpConnectionCode parse(String code) { - if (code == null || code.length() > MAX_CODE_LENGTH || code.indexOf('\n') >= 0 || code.indexOf('\r') >= 0) - throw new IllegalArgumentException("Connection code is invalid"); - String[] parts = code.split("\\.", -1); - if (parts.length != 8 || (!VERSION.equals(parts[0]) && !LEGACY_VERSION.equals(parts[0]))) - throw new IllegalArgumentException("Connection code is invalid"); - try { - String serverId = LEGACY_VERSION.equals(parts[0]) ? parts[1] - : new String(Base64.getUrlDecoder().decode(parts[1]), StandardCharsets.UTF_8); - String endpoint = new String(Base64.getUrlDecoder().decode(parts[2]), StandardCharsets.UTF_8); - String unsigned = String.join(".", parts[0], parts[1], parts[2], parts[3], parts[4], parts[5], parts[6]); - byte[] token = Base64.getUrlDecoder().decode(parts[6]); - String expected = HttpTransportSecrets.hmacSha256Url(token, unsigned); - if (!HttpTransportSecrets.constantTimeEquals(expected.getBytes(StandardCharsets.US_ASCII), - parts[7].getBytes(StandardCharsets.US_ASCII))) throw new IllegalArgumentException("Connection code is invalid"); - return new HttpConnectionCode(serverId, new URI(endpoint), parts[3], parts[4], Instant.ofEpochSecond(Long.parseLong(parts[5])), - parts[6]); - } catch (IllegalArgumentException | URISyntaxException failure) { - throw new IllegalArgumentException("Connection code is invalid", failure); - } - } - - private static URI validateEndpoint(URI value) { - if (value == null || !"https".equalsIgnoreCase(value.getScheme()) || value.getHost() == null - || value.getUserInfo() != null || value.getFragment() != null || value.getRawQuery() != null) - throw new IllegalArgumentException("Endpoint must be an absolute HTTPS URL without credentials or query"); - if (value.getPort() == 0 || value.getPort() > 65535 || value.getPort() < -1) - throw new IllegalArgumentException("Endpoint port is invalid"); - String path = value.getRawPath(); - if (path == null || path.isEmpty()) path = "/"; - if (!path.endsWith("/")) path += "/"; - try { - return new URI("https", null, value.getHost().toLowerCase(Locale.ROOT), value.getPort(), path, null, null); - } catch (URISyntaxException failure) { - throw new IllegalArgumentException("Endpoint is invalid", failure); - } - } - - private static String validatePin(String pin, String name) { - if (pin == null || !pin.matches("[0-9a-fA-F]{64}")) throw new IllegalArgumentException(name + " is invalid"); - return pin.toLowerCase(Locale.ROOT); - } - - private static String validateToken(String token) { - if (token == null || token.length() < 43 || token.length() > 128 || !token.matches("[A-Za-z0-9_-]+")) - throw new IllegalArgumentException("Enrollment token is invalid"); - try { - if (Base64.getUrlDecoder().decode(token).length < 32) throw new IllegalArgumentException("Enrollment token is invalid"); - return token; - } catch (IllegalArgumentException failure) { - throw new IllegalArgumentException("Enrollment token is invalid", failure); - } - } -} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java deleted file mode 100644 index 719028472..000000000 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java +++ /dev/null @@ -1,212 +0,0 @@ -package com.bencodez.votingplugin.backendproxy.http; - -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.Properties; -import java.util.Base64; -import java.util.EnumSet; -import com.bencodez.votingplugin.util.DurableFiles; - -/** - * Single-use enrollment tokens and client-certificate binding. Token material is never retained; - * only SHA-256 hashes are kept until expiry. This type is thread-safe. - */ -public final class HttpEnrollmentAuthority { - private static final Duration MAX_ENROLLMENT_LIFETIME = Duration.ofMinutes(15); - private final HttpTlsIdentity identity; - private final Clock clock; - private final Path stateFile; - private final Map enrollments = new HashMap<>(); - private final Map bindings = new HashMap<>(); - private final Set revokedCertificatePins = new HashSet<>(); - private boolean persistenceFailure; - - /** Creates a restart-safe authority. State contains only public certificate pins and revocations. */ - public HttpEnrollmentAuthority(HttpTlsIdentity identity, Path stateDirectory) throws java.io.IOException { - this(identity, Clock.systemUTC(), stateFile(stateDirectory)); - loadState(); - } - - HttpEnrollmentAuthority(HttpTlsIdentity identity, Clock clock) { - this(identity, clock, null); - } - - HttpEnrollmentAuthority(HttpTlsIdentity identity, Clock clock, Path stateFile) { - if (identity == null || clock == null) throw new IllegalArgumentException("Identity and clock are required"); - this.identity = identity; - this.clock = clock; - this.stateFile = stateFile; - } - - public synchronized HttpConnectionCode createConnectionCode(String serverId, URI endpoint, Duration lifetime) { - serverId = HttpTlsIdentity.canonicalServerId(serverId); - if (lifetime == null || lifetime.isNegative() || lifetime.isZero() || lifetime.compareTo(MAX_ENROLLMENT_LIFETIME) > 0) - throw new IllegalArgumentException("Enrollment lifetime must be between one second and fifteen minutes"); - expireEnrollments(); - Instant expiresAt = clock.instant().plus(lifetime); - String token = HttpTransportSecrets.randomToken(); - byte[] tokenHash = HttpTransportSecrets.sha256(token.getBytes(StandardCharsets.US_ASCII)); - String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(tokenHash); - enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId)); - return new HttpConnectionCode(serverId, endpoint, identity.serverCertificatePin(), identity.caCertificatePin(), expiresAt, token); - } - - public synchronized HttpTlsIdentity.IssuedClientCertificate enroll(String serverId, String enrollmentToken) throws Exception { - serverId = HttpTlsIdentity.canonicalServerId(serverId); - if (enrollmentToken == null || enrollmentToken.length() > 128) throw new IllegalArgumentException("Enrollment was rejected"); - expireEnrollments(); - byte[] suppliedHash = HttpTransportSecrets.sha256(enrollmentToken.getBytes(StandardCharsets.US_ASCII)); - String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(suppliedHash); - Enrollment enrollment = enrollments.get(lookup); - if (enrollment == null || !HttpTransportSecrets.constantTimeEquals(enrollment.tokenHash(), suppliedHash)) - throw new IllegalArgumentException("Enrollment was rejected"); - if (!serverId.equals(enrollment.serverId())) throw new IllegalArgumentException("Enrollment was rejected"); - enrollments.remove(lookup); // consume only after the token and its intended backend both match. - ClientBinding existing = bindings.get(serverId); - if (existing != null && !existing.revoked()) throw new IllegalStateException("Server id is already enrolled"); - HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); - bindings.put(serverId, new ClientBinding(HttpTransportSecrets.certificatePin(issued.certificate()), null, false)); - try { persistState(); } - catch (java.io.IOException failure) { persistenceFailure = true; throw failure; } - return issued; - } - - public synchronized boolean authenticate(String serverId, java.security.cert.X509Certificate certificate) { - if (persistenceFailure || serverId == null || certificate == null) return false; - try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } - catch (IllegalArgumentException invalid) { return false; } - if (!identity.validClientCertificate(serverId, certificate)) return false; - ClientBinding binding = bindings.get(serverId); - String pin = HttpTransportSecrets.certificatePin(certificate); - if (binding == null || binding.revoked() || revokedCertificatePins.contains(pin)) return false; - if (samePin(binding.certificatePin(), pin)) return true; - if (!samePin(binding.pendingCertificatePin(), pin)) return false; - bindings.put(serverId, new ClientBinding(pin, null, false)); - revokedCertificatePins.add(binding.certificatePin()); - try { persistState(); return true; } - catch (java.io.IOException failure) { persistenceFailure = true; return false; } - } - - /** Issues a replacement while the currently bound certificate is still valid. The old binding remains active - * until the replacement successfully authenticates, making a lost renewal response safe to retry. */ - public synchronized HttpTlsIdentity.IssuedClientCertificate renew(String serverId, - java.security.cert.X509Certificate currentCertificate) throws Exception { - if (!authenticate(serverId, currentCertificate)) throw new IllegalArgumentException("Certificate renewal was rejected"); - serverId = HttpTlsIdentity.canonicalServerId(serverId); - ClientBinding binding = bindings.get(serverId); - HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); - bindings.put(serverId, new ClientBinding(binding.certificatePin(), - HttpTransportSecrets.certificatePin(issued.certificate()), false)); - try { persistState(); } - catch (java.io.IOException failure) { persistenceFailure = true; throw failure; } - return issued; - } - - public synchronized void revoke(String serverId) { - try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } - catch (IllegalArgumentException invalid) { return; } - final String revokedServer = serverId; - enrollments.entrySet().removeIf(entry -> revokedServer.equals(entry.getValue().serverId())); - ClientBinding binding = bindings.get(serverId); - if (binding != null) { - bindings.put(serverId, new ClientBinding(binding.certificatePin(), binding.pendingCertificatePin(), true)); - revokedCertificatePins.add(binding.certificatePin()); - if (binding.pendingCertificatePin() != null) revokedCertificatePins.add(binding.pendingCertificatePin()); - try { persistState(); } - catch (java.io.IOException failure) { persistenceFailure = true; throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); } - } - } - - private synchronized void loadState() throws java.io.IOException { - if (stateFile == null || !Files.exists(stateFile, LinkOption.NOFOLLOW_LINKS)) return; - if (!Files.isRegularFile(stateFile, LinkOption.NOFOLLOW_LINKS) || Files.size(stateFile) > 65536) - throw new java.io.IOException("HTTP enrollment state is invalid"); - Properties properties = new Properties(); - try (var input = Files.newInputStream(stateFile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } - for (String key : properties.stringPropertyNames()) { - if (key.startsWith("binding.")) { - String serverId = new String(Base64.getUrlDecoder().decode(key.substring("binding.".length())), StandardCharsets.UTF_8); - serverId = HttpTlsIdentity.canonicalServerId(serverId); - String[] value = properties.getProperty(key, "").split(":", -1); - if (!((value.length == 2 && "1".equals(properties.getProperty("version"))) - || (value.length == 3 && "2".equals(properties.getProperty("version")))) - || !value[0].matches("[0-9a-f]{64}")) - throw new java.io.IOException("HTTP enrollment state is invalid"); - String pending = value.length == 3 && !"-".equals(value[1]) ? value[1] : null; - String revoked = value[value.length - 1]; - if ((pending != null && !pending.matches("[0-9a-f]{64}")) || !("0".equals(revoked) || "1".equals(revoked))) - throw new java.io.IOException("HTTP enrollment state is invalid"); - bindings.put(serverId, new ClientBinding(value[0], pending, "1".equals(revoked))); - if ("1".equals(revoked)) { revokedCertificatePins.add(value[0]); if (pending != null) revokedCertificatePins.add(pending); } - } else if (!"version".equals(key)) throw new java.io.IOException("HTTP enrollment state is invalid"); - } - if (!("1".equals(properties.getProperty("version")) || "2".equals(properties.getProperty("version")))) - throw new java.io.IOException("HTTP enrollment state is invalid"); - } - - private synchronized void persistState() throws java.io.IOException { - if (stateFile == null) return; - Properties properties = new Properties(); - properties.setProperty("version", "2"); - for (Map.Entry entry : bindings.entrySet()) { - String key = Base64.getUrlEncoder().withoutPadding().encodeToString(entry.getKey().getBytes(StandardCharsets.UTF_8)); - properties.setProperty("binding." + key, entry.getValue().certificatePin() + ":" - + (entry.getValue().pendingCertificatePin() == null ? "-" : entry.getValue().pendingCertificatePin()) - + ":" + (entry.getValue().revoked() ? "1" : "0")); - } - java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); - properties.store(bytes, "VotingPlugin HTTP certificate bindings"); - Path temporary = Files.createTempFile(stateFile.getParent(), stateFile.getFileName().toString(), ".tmp"); - try { - setOwnerOnly(temporary); - Files.write(temporary, bytes.toByteArray(), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); - DurableFiles.forceFile(temporary); - try { Files.move(temporary, stateFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } - catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, stateFile, StandardCopyOption.REPLACE_EXISTING); } - setOwnerOnly(stateFile); - DurableFiles.forceDirectory(stateFile.getParent()); - } finally { Files.deleteIfExists(temporary); } - } - - private static Path stateFile(Path directory) throws java.io.IOException { - if (directory == null) throw new IllegalArgumentException("State directory is required"); - Files.createDirectories(directory); - Path file = directory.toAbsolutePath().normalize().resolve("http-transport-clients.properties"); - if (Files.isSymbolicLink(file)) throw new java.io.IOException("Refusing unsafe HTTP enrollment state path"); - return file; - } - - private static void setOwnerOnly(Path path) throws java.io.IOException { - try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } - catch (UnsupportedOperationException ignored) { } - } - - private void expireEnrollments() { - Instant now = clock.instant(); - enrollments.entrySet().removeIf(entry -> !entry.getValue().expiresAt().isAfter(now)); - } - - private record Enrollment(byte[] tokenHash, Instant expiresAt, String serverId) { - private Enrollment { tokenHash = tokenHash.clone(); } - @Override public byte[] tokenHash() { return tokenHash.clone(); } - } - private static boolean samePin(String expected, String actual) { - return expected != null && actual != null && HttpTransportSecrets.constantTimeEquals( - expected.getBytes(StandardCharsets.US_ASCII), actual.getBytes(StandardCharsets.US_ASCII)); - } - - private record ClientBinding(String certificatePin, String pendingCertificatePin, boolean revoked) { } -} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java deleted file mode 100644 index ba04a31ec..000000000 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java +++ /dev/null @@ -1,173 +0,0 @@ -package com.bencodez.votingplugin.backendproxy.http; - -import com.bencodez.votingplugin.util.DurableFiles; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; -import java.util.EnumSet; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; - -/** Crash-durable state for proxy deliveries around a non-transactional application callback. */ -final class HttpInboundDeliveryStore { - private static final String DIRECTORY = "http-transport-inbound-deliveries"; - private static final int MAX_ENTRIES = HttpTransportProtocol.MAX_QUEUE; - private final Path root; - private final Map entries = new LinkedHashMap<>(); - private boolean sealed; - - HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { - Path credentials = credentialDirectory.toAbsolutePath().normalize(); - if (Files.isSymbolicLink(credentials) || !Files.isDirectory(credentials, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP credential directory is unsafe"); - ownerOnlyDirectory(credentials); - root = credentials.resolve(DIRECTORY).normalize(); - if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); - boolean created = false; - try { Files.createDirectory(root); created = true; } - catch (java.nio.file.FileAlreadyExistsException existing) { } - try { - requireRoot(); - ownerOnlyDirectory(root); - } finally { - if (created) DurableFiles.forceDirectory(credentials); - } - load(); - } - - synchronized State state(String id) { return entries.get(canonical(id)); } - - synchronized void reserve(String id) throws IOException { - requireWritable(); - id = canonical(id); - State existing = entries.get(id); - if (existing == State.RESERVED) return; - if (existing != null) throw new IOException("HTTP inbound delivery fence is already active"); - if (entries.size() >= MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence is full"); - requireRoot(); - Path target = file(id, State.RESERVED); - if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP inbound delivery fence is inconsistent"); - Path temporary = Files.createTempFile(root, ".pending-", ".tmp"); - try { - ownerOnlyFile(temporary); - Files.writeString(temporary, id, StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); - DurableFiles.forceFile(temporary); - move(temporary, target); - ownerOnlyFile(target); - DurableFiles.forceDirectory(root); - entries.put(id, State.RESERVED); - } finally { Files.deleteIfExists(temporary); } - } - - synchronized void markRunning(String id) throws IOException { transition(id, State.RESERVED, State.RUNNING); } - synchronized void markCompleted(String id) throws IOException { transition(id, State.RUNNING, State.COMPLETED); } - synchronized void seal() { sealed = true; } - - synchronized void remove(String id) throws IOException { - requireWritable(); - id = canonical(id); - State state = entries.get(id); - if (state == null) return; - requireRoot(); - DurableFiles.deleteIfExists(file(id, state)); - entries.remove(id); - } - - synchronized Map snapshot() { return Map.copyOf(entries); } - - private void transition(String id, State expected, State replacement) throws IOException { - requireWritable(); - id = canonical(id); - if (entries.get(id) != expected) throw new IOException("HTTP inbound delivery fence state is invalid"); - requireRoot(); - Path source = file(id, expected), target = file(id, replacement); - if (Files.isSymbolicLink(source) || !Files.isRegularFile(source, LinkOption.NOFOLLOW_LINKS) - || Files.exists(target, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP inbound delivery fence state is unsafe"); - move(source, target); - DurableFiles.forceDirectory(root); - entries.put(id, replacement); - } - - private void load() throws IOException { - try (DirectoryStream files = Files.newDirectoryStream(root)) { - for (Path file : files) { - String name = file.getFileName().toString(); - if (name.startsWith(".pending-") && name.endsWith(".tmp") && !Files.isSymbolicLink(file) - && Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) { - DurableFiles.deleteIfExists(file); - continue; - } - State state = State.fromFileName(name); - if (state == null || Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) - || Files.size(file) > 64L) - throw new IOException("HTTP inbound delivery fence contains an invalid entry"); - String id; - try { id = canonical(name.substring(0, name.length() - state.suffix.length())); } - catch (IllegalArgumentException invalid) { - throw new IOException("HTTP inbound delivery fence entry is invalid", invalid); - } - if (!name.equals(id + state.suffix) || !Files.readString(file, StandardCharsets.US_ASCII).equals(id)) - throw new IOException("HTTP inbound delivery fence entry is invalid"); - State existing = entries.get(id); - if (existing == null) entries.put(id, state); - else { - // A provider without atomic moves may expose both names after an - // interrupted transition. Preserve the furthest fail-closed state: - // RUNNING never replays, and COMPLETED alone may be acknowledged. - State retained = existing.ordinal() >= state.ordinal() ? existing : state; - State obsolete = retained == existing ? state : existing; - DurableFiles.deleteIfExists(file(id, obsolete)); - entries.put(id, retained); - } - if (entries.size() > MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence exceeds its bound"); - } - } - } - - private Path file(String id, State state) { return root.resolve(id + state.suffix); } - private void requireWritable() throws IOException { - if (sealed) throw new IOException("HTTP inbound delivery store ownership has ended"); - } - private static void move(Path source, Path target) throws IOException { - try { Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); } - catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(source, target); } - } - private static String canonical(String id) { - if (id == null) throw new IllegalArgumentException("HTTP delivery id is invalid"); - String canonical = UUID.fromString(id).toString(); - if (!canonical.equals(id)) throw new IllegalArgumentException("HTTP delivery id is not canonical"); - return canonical; - } - private void requireRoot() throws IOException { - if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP inbound delivery directory is unsafe"); - } - private static void ownerOnlyFile(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } - catch (UnsupportedOperationException ignored) { } - } - private static void ownerOnlyDirectory(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, - PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } - catch (UnsupportedOperationException ignored) { } - } - - enum State { - RESERVED(".reserved"), RUNNING(".running"), COMPLETED(".completed"); - private final String suffix; - State(String suffix) { this.suffix = suffix; } - private static State fromFileName(String name) { - for (State state : values()) if (name.endsWith(state.suffix)) return state; - return null; - } - } -} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java deleted file mode 100644 index 9da95637d..000000000 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java +++ /dev/null @@ -1,103 +0,0 @@ -package com.bencodez.votingplugin.backendproxy.http; - -import java.security.cert.X509Certificate; -import java.security.KeyStore; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLParameters; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; - -/** Builds the backend TLS context. A public CA store is intentionally not consulted. */ -public final class HttpPinnedTls { - private HttpPinnedTls() { } - - public static SSLContext clientContext(HttpConnectionCode code) throws Exception { - if (code == null) throw new IllegalArgumentException("Connection code is required"); - SSLContext context = SSLContext.getInstance("TLS"); - context.init(null, new TrustManager[] { new PinnedServerTrustManager(code.serverCertificatePin(), code.caCertificatePin()) }, null); - return context; - } - - /** - * Normal transport context: presents the enrolled client certificate and trusts only the - * pinned private authority. Callers must not override the HttpClient default endpoint-identification - * settings; hostname verification remains enabled and the proxy leaf may renew under the same CA. - */ - public static SSLContext mutualTlsContext(HttpConnectionCode code, HttpClientCredentialStore.ClientCredential credential) - throws Exception { - if (code == null || credential == null || credential.privateKey() == null || credential.certificate() == null || credential.caCertificate() == null) - throw new IllegalArgumentException("Enrolled client credential is required"); - char[] password = credential.password(); - try { - String authorityPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); - if (!HttpTransportSecrets.constantTimeEquals(code.caCertificatePin().getBytes(java.nio.charset.StandardCharsets.US_ASCII), - authorityPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) - throw new IllegalArgumentException("HTTP authority does not match connection code"); - KeyStore store = KeyStore.getInstance("PKCS12"); - store.load(null, new char[0]); - store.setKeyEntry("client", credential.privateKey(), password, - new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); - KeyManagerFactory managers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - managers.init(store, password); - KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); - trustStore.load(null, new char[0]); - trustStore.setCertificateEntry("http-transport-ca", credential.caCertificate()); - javax.net.ssl.TrustManagerFactory trusts = javax.net.ssl.TrustManagerFactory.getInstance( - javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()); - trusts.init(trustStore); - SSLContext context = SSLContext.getInstance("TLS"); - context.init(managers.getKeyManagers(), trusts.getTrustManagers(), null); - return context; - } finally { java.util.Arrays.fill(password, '\0'); } - } - - public static boolean matchesServerPin(HttpConnectionCode code, X509Certificate certificate) { - if (code == null || certificate == null) return false; - String actual = HttpTransportSecrets.certificatePin(certificate); - return HttpTransportSecrets.constantTimeEquals(code.serverCertificatePin() - .getBytes(java.nio.charset.StandardCharsets.US_ASCII), actual.getBytes(java.nio.charset.StandardCharsets.US_ASCII)); - } - - /** TLS 1.3 is used where the runtime exposes it; hostname verification is deliberately left enabled. */ - public static SSLParameters secureParameters(SSLContext context) { - SSLParameters parameters = context.getDefaultSSLParameters(); - for (String protocol : context.getSupportedSSLParameters().getProtocols()) { - if ("TLSv1.3".equals(protocol)) { - parameters.setProtocols(new String[] { "TLSv1.3" }); - break; - } - } - return parameters; - } - - private static final class PinnedServerTrustManager implements X509TrustManager { - private final String expectedPin; - private final String expectedCaPin; - private PinnedServerTrustManager(String expectedPin, String expectedCaPin) { - this.expectedPin = expectedPin; - this.expectedCaPin = expectedCaPin; - } - @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } - @Override public void checkServerTrusted(X509Certificate[] chain, String authType) throws java.security.cert.CertificateException { - if (chain == null || chain.length < 2) throw new java.security.cert.CertificateException("Server certificate chain is incomplete"); - chain[0].checkValidity(); - chain[chain.length - 1].checkValidity(); - try { chain[0].verify(chain[chain.length - 1].getPublicKey()); } - catch (java.security.GeneralSecurityException invalid) { - throw new java.security.cert.CertificateException("Server certificate signature is invalid", invalid); - } - if (chain[chain.length - 1].getBasicConstraints() < 0) - throw new java.security.cert.CertificateException("Server certificate authority is invalid"); - String actual = HttpTransportSecrets.certificatePin(chain[0]); - if (!HttpTransportSecrets.constantTimeEquals(expectedPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII), - actual.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) - throw new java.security.cert.CertificateException("Server certificate pin does not match"); - String issuer = HttpTransportSecrets.certificatePin(chain[chain.length - 1]); - if (!HttpTransportSecrets.constantTimeEquals(expectedCaPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII), - issuer.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) - throw new java.security.cert.CertificateException("Server certificate authority pin does not match"); - } - @Override public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } - } -} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java deleted file mode 100644 index 4ba34e528..000000000 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ /dev/null @@ -1,499 +0,0 @@ -package com.bencodez.votingplugin.backendproxy.http; - -import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; -import com.bencodez.votingplugin.util.DurableFiles; -import com.sun.net.httpserver.Headers; -import com.sun.net.httpserver.HttpsConfigurator; -import com.sun.net.httpserver.HttpsExchange; -import com.sun.net.httpserver.HttpsParameters; -import com.sun.net.httpserver.HttpsServer; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.InetSocketAddress; -import java.net.URI; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; -import java.security.cert.Certificate; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.util.ArrayDeque; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; -import java.util.function.LongSupplier; -import javax.net.ssl.SSLParameters; -import javax.net.ssl.SSLPeerUnverifiedException; - -/** - * One HTTPS listener for enrollment and the backend-to-proxy long-poll transport. - * Every normal request is certificate-authenticated in the handler, rather than relying on TLS WANT auth. - */ -public final class HttpProxyTransportServer implements AutoCloseable { - static { - // JDK HttpServer reads these once when its internal server configuration is initialized. - // Set conservative process-wide bounds before this transport creates its listener. - setDefault("sun.net.httpserver.maxReqTime", "10"); - setDefault("sun.net.httpserver.maxRspTime", "10"); - setDefault("jdk.httpserver.maxConnections", "144"); - setDefault("sun.net.httpserver.maxReqHeaders", "32"); - setDefault("sun.net.httpserver.maxReqHeaderSize", "16384"); - } - // Keep an idle request open long enough to reuse the TLS connection, but bound backend-origin - // latency when a message is queued immediately after the request body has already been sent. - public static final Duration LONG_POLL = Duration.ofSeconds(2); - private final HttpTlsIdentity identity; - private final HttpEnrollmentAuthority authority; - private final HttpsServer server; - private final ThreadPoolExecutor listenerExecutor; - private final ThreadPoolExecutor handlerExecutor; - private final Semaphore admission = new Semaphore(64); - private final Map backends = new HashMap<>(); - private final DurableOutgoingQueue durableOutgoing; - private final Consumer onEnvelope; - private final DeliveryAcknowledgement onAcknowledged; - private volatile boolean closed; - - public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, - Consumer onEnvelope) throws Exception { - this(bind, identity, authority, null, onEnvelope, (serverId, deliveryId) -> { }); - } - - public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, - Path outgoingDirectory, Consumer onEnvelope) throws Exception { - this(bind, identity, authority, outgoingDirectory, onEnvelope, (serverId, deliveryId) -> { }); - } - - public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, - Path outgoingDirectory, Consumer onEnvelope, - DeliveryAcknowledgement onAcknowledged) throws Exception { - if (bind == null || identity == null || authority == null || onEnvelope == null || onAcknowledged == null) - throw new IllegalArgumentException("HTTP transport configuration is required"); - this.identity = identity; this.authority = authority; this.onEnvelope = onEnvelope; - this.onAcknowledged = onAcknowledged; - durableOutgoing = outgoingDirectory == null ? null : new DurableOutgoingQueue(outgoingDirectory); - if (durableOutgoing != null) for (Map.Entry> pending - : durableOutgoing.load().entrySet()) { - BackendState state = new BackendState(pending.getKey(), durableOutgoing, onAcknowledged); - state.restore(pending.getValue()); - backends.put(pending.getKey(), state); - } - server = HttpsServer.create(bind, 32); - server.setHttpsConfigurator(new HttpsConfigurator(identity.serverContext()) { - @Override public void configure(HttpsParameters parameters) { - SSLParameters ssl = HttpPinnedTls.secureParameters(getSSLContext()); - ssl.setWantClientAuth(true); parameters.setSSLParameters(ssl); - } - }); - // Long polls are blocking by design. Capacity is bounded by admission, while enough workers - // remain available for all admitted polls plus setup requests. - listenerExecutor = executor("VotingPlugin-HTTP-listener", 72, 72); - // The proxy router mutates shared presence, vote, and reward state. A separate - // bounded FIFO lane keeps wire order without blocking long-poll workers. - handlerExecutor = executor("VotingPlugin-HTTP-handler", 1, HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY); - server.setExecutor(listenerExecutor); - server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); - server.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); - server.createContext("/v1/transport", exchange -> transport((HttpsExchange) exchange)); - } - - private void renew(HttpsExchange exchange) throws IOException { - if (!"/v1/renew".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } - if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } - if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } - if (!boundedFixedBody(exchange, 1024) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } - try { - String serverId = HttpTransportProtocol.parseRenewal(read(exchange.getRequestBody(), 1024)); - X509Certificate certificate = peerCertificate(exchange); - if (certificate == null || !authority.authenticate(serverId, certificate)) { reply(exchange, 401, new byte[0]); return; } - HttpTlsIdentity.IssuedClientCertificate issued = authority.renew(serverId, certificate); - reply(exchange, 201, HttpTransportProtocol.enrollmentResponse(issued)); - } catch (IllegalArgumentException rejected) { reply(exchange, 403, new byte[0]); - } catch (Exception failure) { reply(exchange, 503, new byte[0]); - } finally { admission.release(); } - } - - public void start() { if (closed) throw new IllegalStateException("HTTP transport is closed"); server.start(); } - public int port() { return server.getAddress().getPort(); } - public URI endpoint(String host) { return URI.create("https://" + host + ":" + port() + "/"); } - - /** Queues a proxy-origin envelope durably before reporting acceptance. */ - public boolean send(String serverId, JsonEnvelope envelope) { - return send(serverId, UUID.randomUUID().toString(), envelope); - } - - /** Queues a proxy-origin envelope with a stable, caller-persisted delivery ID. */ - public boolean send(String serverId, String deliveryId, JsonEnvelope envelope) { - if (closed || serverId == null || envelope == null) return false; - try { - serverId = HttpTlsIdentity.canonicalServerId(serverId); - HttpTransportProtocol.validId(deliveryId); - HttpTransportProtocol.validateEnvelope(envelope); - } - catch (IllegalArgumentException invalid) { return false; } - BackendState backend; - final String canonicalServerId = serverId; - synchronized (backends) { backend = backends.computeIfAbsent(serverId, - ignored -> new BackendState(canonicalServerId, durableOutgoing, onAcknowledged)); } - return backend.enqueue(new HttpTransportProtocol.Delivery(deliveryId, envelope)); - } - - @Override public void close() { - if (closed) return; closed = true; server.stop(1); - shutdown(handlerExecutor); shutdown(listenerExecutor); - synchronized (backends) { for (BackendState backend : backends.values()) backend.signal(); backends.clear(); } - } - - private void enroll(HttpsExchange exchange) throws IOException { - if (!"/v1/enroll".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } - if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } - if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } - if (!boundedFixedBody(exchange, 8192) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } - try { - HttpTransportProtocol.Enrollment request = HttpTransportProtocol.parseEnrollment(read(exchange.getRequestBody(), 8192)); - HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll(request.server(), request.token()); - reply(exchange, 201, HttpTransportProtocol.enrollmentResponse(issued)); - } catch (Exception rejected) { reply(exchange, 403, new byte[0]); } - finally { admission.release(); } - } - - private void transport(HttpsExchange exchange) throws IOException { - if (!"/v1/transport".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } - if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } - if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } - if (!boundedFixedBody(exchange, HttpTransportProtocol.MAX_BODY_BYTES) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } - try { - HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(read(exchange.getRequestBody(), HttpTransportProtocol.MAX_BODY_BYTES)); - X509Certificate certificate = peerCertificate(exchange); - if (certificate == null || !authority.authenticate(packet.server(), certificate)) { reply(exchange, 401, new byte[0]); return; } - BackendState backend; - synchronized (backends) { backend = backends.computeIfAbsent(packet.server(), - ignored -> new BackendState(packet.server(), durableOutgoing, onAcknowledged)); } - if (!backend.beginPoll(packet.session())) { reply(exchange, 409, new byte[0]); return; } - try { - handlePacket(packet, backend); - Response response = backend.await(packet.server(), packet.session(), packet.sequence()); - reply(exchange, 200, HttpTransportProtocol.response(packet.server(), packet.session(), packet.sequence(), response.acks(), response.messages())); - } finally { backend.endPoll(); } - } catch (IllegalArgumentException rejected) { reply(exchange, 400, new byte[0]); - } catch (Exception failure) { reply(exchange, 503, new byte[0]); - } finally { admission.release(); } - } - - private void handlePacket(HttpTransportProtocol.Packet packet, BackendState backend) throws IOException { - List accepted; - synchronized (backend) { - if (!backend.allowRequest()) throw new IllegalArgumentException("transport rate limited"); - if (!backend.acceptSession(packet.session(), packet.sequence())) throw new IllegalArgumentException("stale session request"); - } - backend.acknowledge(packet.acks()); - synchronized (backend) { accepted = backend.acceptIncoming(packet.messages()); } - for (HttpTransportProtocol.Delivery delivery : accepted) dispatch(packet.server(), backend, delivery); - } - private void dispatch(String serverId, BackendState backend, HttpTransportProtocol.Delivery delivery) { - Runnable callback = () -> { - boolean success = false; - try { onEnvelope.accept(new ReceivedEnvelope(serverId, delivery.id(), normalizeBackendIdentity(serverId, delivery.envelope()))); success = true; } - catch (RuntimeException ignored) { } - synchronized (backend) { backend.completeIncoming(delivery.id(), success); } - }; - if (!HttpBackendTransportConnector.executeOrdered(handlerExecutor, callback)) - synchronized (backend) { backend.completeIncoming(delivery.id(), false); } - } - private static JsonEnvelope normalizeBackendIdentity(String serverId, JsonEnvelope envelope) { - // The authenticated TLS identity is authoritative; never forward a forged `server` field. - return envelope.toBuilder().put("server", serverId).build(); - } - private static X509Certificate peerCertificate(HttpsExchange exchange) { - try { Certificate[] peer = exchange.getSSLSession().getPeerCertificates(); - return peer.length > 0 && peer[0] instanceof X509Certificate certificate ? certificate : null; - } catch (SSLPeerUnverifiedException absent) { return null; } - } - private static byte[] read(InputStream input, int maximum) throws IOException { - ByteArrayOutputStream output = new ByteArrayOutputStream(); byte[] buffer = new byte[4096]; int total = 0, read; - while ((read = input.read(buffer)) >= 0) { total += read; if (total > maximum) throw new IllegalArgumentException("HTTP body is too large"); output.write(buffer, 0, read); } - return output.toByteArray(); - } - private static void reply(HttpsExchange exchange, int status, byte[] body) throws IOException { - Headers headers = exchange.getResponseHeaders(); headers.set("Cache-Control", "no-store"); headers.set("Content-Type", "application/json; charset=utf-8"); - exchange.sendResponseHeaders(status, body.length); try (var output = exchange.getResponseBody()) { output.write(body); } - } - private static boolean json(HttpsExchange exchange) { - String contentType = exchange.getRequestHeaders().getFirst("Content-Type"); - return contentType != null && contentType.toLowerCase(java.util.Locale.ROOT).matches("application/json(?:\\s*;.*)?"); - } - private static boolean boundedFixedBody(HttpsExchange exchange, int maximum) { - if (exchange.getRequestHeaders().getFirst("Transfer-Encoding") != null) return false; - String value = exchange.getRequestHeaders().getFirst("Content-Length"); - try { long length = Long.parseLong(value); return length > 0L && length <= maximum; } - catch (RuntimeException invalid) { return false; } - } - private static ThreadPoolExecutor executor(String name, int threads, int queue) { - ThreadFactory factory = task -> { Thread thread = new Thread(task, name); thread.setDaemon(true); return thread; }; - return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); - } - private static void setDefault(String name, String value) { if (System.getProperty(name) == null) System.setProperty(name, value); } - private static void shutdown(ExecutorService executor) { executor.shutdown(); try { if (!executor.awaitTermination(5, TimeUnit.SECONDS)) executor.shutdownNow(); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); executor.shutdownNow(); } } - - public record ReceivedEnvelope(String serverId, String messageId, JsonEnvelope envelope) { } - - @FunctionalInterface - public interface DeliveryAcknowledgement { - void confirm(String serverId, String deliveryId) throws IOException; - } - static record Response(Collection acks, Collection messages) { } - static final class BackendState { - private final String serverId; - private final DurableOutgoingQueue durableOutgoing; - private final DeliveryAcknowledgement onAcknowledged; - private final LongSupplier nanoTime; - private String session; private long sequence = -1L; - private final LinkedHashMap outgoing = new LinkedHashMap<>(); - private final Set seen = new LinkedHashSet<>(); private final Set processing = new LinkedHashSet<>(); - private final ArrayDeque acknowledgements = new ArrayDeque<>(); - private final Map deliveredAtNanos = new HashMap<>(); - private double requestTokens = 24.0d; - private long lastTokenNanos = System.nanoTime(); - private boolean activePoll; - BackendState() { this(null, null, (serverId, deliveryId) -> { }, System::nanoTime); } - BackendState(LongSupplier nanoTime) { this(null, null, (serverId, deliveryId) -> { }, nanoTime); } - private BackendState(String serverId, DurableOutgoingQueue durableOutgoing) { - this(serverId, durableOutgoing, (ignoredServer, ignoredDelivery) -> { }, System::nanoTime); - } - BackendState(String serverId, DurableOutgoingQueue durableOutgoing, - DeliveryAcknowledgement onAcknowledged) { - this(serverId, durableOutgoing, onAcknowledged, System::nanoTime); - } - private BackendState(String serverId, DurableOutgoingQueue durableOutgoing, - DeliveryAcknowledgement onAcknowledged, LongSupplier nanoTime) { - this.serverId = serverId; this.durableOutgoing = durableOutgoing; - this.onAcknowledged = onAcknowledged; this.nanoTime = nanoTime; - } - private synchronized void restore(Collection deliveries) { - for (HttpTransportProtocol.Delivery delivery : deliveries) outgoing.put(delivery.id(), delivery); - } - private boolean beginPoll(String requestedSession) { synchronized (this) { if (activePoll) return false; activePoll = true; return true; } } - private void endPoll() { synchronized (this) { activePoll = false; notifyAll(); } } - private boolean allowRequest() { - long now = System.nanoTime(); requestTokens = Math.min(24.0d, requestTokens + ((now - lastTokenNanos) / 1_000_000_000.0d) * 2.0d); - lastTokenNanos = now; if (requestTokens < 1.0d) return false; requestTokens -= 1.0d; return true; - } - boolean acceptSession(String requested, long requestedSequence) { - if (!requested.equals(session)) { session = requested; sequence = -1L; deliveredAtNanos.clear(); } - // The connector allocates a fresh monotonic sequence for every attempt. Rejecting equality - // prevents a captured request from being replayed with altered ACKs or a new payload. - if (requestedSequence <= sequence) return false; sequence = requestedSequence; return true; - } - synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { - HttpTransportProtocol.Delivery existing = outgoing.get(delivery.id()); - if (existing != null) return Arrays.equals(HttpTransportProtocol.storedDelivery(existing), - HttpTransportProtocol.storedDelivery(delivery)); - if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; - if (durableOutgoing != null) try { durableOutgoing.persist(serverId, delivery); } - catch (IOException failure) { return false; } - outgoing.put(delivery.id(), delivery); signal(); return true; - } - void acknowledge(Collection acks) throws IOException { - for (String id : acks) { - synchronized (this) { if (!outgoing.containsKey(id)) continue; } - onAcknowledged.confirm(serverId, id); - synchronized (this) { - if (!outgoing.containsKey(id)) continue; - // A 200 response is the backend's proof that its durable replay fence may - // be deleted. Never return success while the proxy delivery still exists. - if (durableOutgoing != null) durableOutgoing.remove(serverId, id); - outgoing.remove(id); deliveredAtNanos.remove(id); - } - } - } - List acceptIncoming(List received) { - List accepted = new java.util.ArrayList<>(); - for (HttpTransportProtocol.Delivery delivery : received) { - if (seen.contains(delivery.id())) { queueAck(delivery.id()); continue; } - if (!processing.contains(delivery.id())) { - processing.add(delivery.id()); accepted.add(delivery); - } - } - return accepted; - } - synchronized void completeIncoming(String id, boolean success) { processing.remove(id); if (success) { seen.add(id); while (seen.size() > HttpTransportProtocol.MAX_QUEUE) seen.remove(seen.iterator().next()); queueAck(id); signal(); } } - private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } - synchronized Response await(String serverId, String requestedSession, long requestedSequence) { - long deadline = System.nanoTime() + LONG_POLL.toNanos(); - while (acknowledgements.isEmpty() && !hasUndelivered()) { - long retryRemaining = nanosUntilRedelivery(nanoTime.getAsLong()); - if (retryRemaining <= 0L) break; - long requestRemaining = deadline - System.nanoTime(); if (requestRemaining <= 0L) break; - long wait = Math.min(requestRemaining, retryRemaining); - try { TimeUnit.NANOSECONDS.timedWait(this, wait); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); break; } - } - List acks = new java.util.ArrayList<>(); while (!acknowledgements.isEmpty() && acks.size() < HttpTransportProtocol.MAX_BATCH) acks.add(acknowledgements.remove()); - List candidates = new java.util.ArrayList<>(); - long now = nanoTime.getAsLong(); - if (hasUndelivered() || redeliveryDue(now)) for (HttpTransportProtocol.Delivery delivery : outgoing.values()) { - if (!deliveredAtNanos.containsKey(delivery.id()) || redeliveryDue(delivery.id(), now)) candidates.add(delivery); - if (candidates.size() == HttpTransportProtocol.MAX_BATCH) break; - } - List messages = HttpTransportProtocol.fittingMessages(serverId, requestedSession, - requestedSequence, acks, candidates); - long deliveredAt = nanoTime.getAsLong(); - for (HttpTransportProtocol.Delivery delivery : messages) deliveredAtNanos.put(delivery.id(), deliveredAt); - return new Response(acks, messages); - } - private boolean hasUndelivered() { for (String id : outgoing.keySet()) if (!deliveredAtNanos.containsKey(id)) return true; return false; } - private boolean redeliveryDue(long now) { - for (String id : outgoing.keySet()) if (redeliveryDue(id, now)) return true; - return false; - } - private boolean redeliveryDue(String id, long now) { - Long deliveredAt = deliveredAtNanos.get(id); - return deliveredAt != null && now - deliveredAt >= LONG_POLL.toNanos(); - } - private long nanosUntilRedelivery(long now) { - long remaining = Long.MAX_VALUE; - for (String id : outgoing.keySet()) { - Long deliveredAt = deliveredAtNanos.get(id); - if (deliveredAt == null) continue; - long candidate = LONG_POLL.toNanos() - (now - deliveredAt); - if (candidate <= 0L) return 0L; - remaining = Math.min(remaining, candidate); - } - return remaining; - } - private synchronized void signal() { notifyAll(); } - } - - private static final class DurableOutgoingQueue { - private static final String FILE_PATTERN = "[0-9]{20}-[0-9a-f-]{36}\\.json"; - private final Path root; - private final Map> files = new HashMap<>(); - private long sequence; - - private DurableOutgoingQueue(Path root) throws IOException { - this.root = root.toAbsolutePath().normalize(); - boolean created = false; - try { Files.createDirectory(this.root); created = true; } - catch (java.nio.file.FileAlreadyExistsException existing) { } - try { - if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP outgoing queue directory is invalid"); - ownerOnlyDirectory(this.root); - } finally { - if (created) DurableFiles.forceDirectory(this.root.getParent()); - } - } - - private synchronized Map> load() throws IOException { - Map> loaded = new LinkedHashMap<>(); - try (DirectoryStream servers = Files.newDirectoryStream(root)) { - for (Path directory : servers) { - if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP outgoing queue contains an invalid entry"); - String serverId; - try { serverId = HttpTlsIdentity.canonicalServerId(directory.getFileName().toString()); } - catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue server is invalid", invalid); } - if (!serverId.equals(directory.getFileName().toString())) - throw new IOException("HTTP outgoing queue server is not canonical"); - List entries = new java.util.ArrayList<>(); - try (DirectoryStream messages = Files.newDirectoryStream(directory)) { - for (Path message : messages) entries.add(message); - } - entries.sort(java.util.Comparator.comparing(path -> path.getFileName().toString())); - List deliveries = new java.util.ArrayList<>(); - Map serverFiles = files.computeIfAbsent(serverId, ignored -> new HashMap<>()); - for (Path message : entries) { - String name = message.getFileName().toString(); - if (name.startsWith(".pending-") && name.endsWith(".tmp") - && !Files.isSymbolicLink(message) && Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS)) { - DurableFiles.deleteIfExists(message); - continue; - } - if (Files.isSymbolicLink(message) || !Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS) - || !name.matches(FILE_PATTERN) || Files.size(message) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L) - throw new IOException("HTTP outgoing queue message is invalid"); - HttpTransportProtocol.Delivery delivery; - try { delivery = HttpTransportProtocol.parseStoredDelivery(Files.readAllBytes(message)); } - catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue message is invalid", invalid); } - if (!name.endsWith("-" + delivery.id() + ".json") || serverFiles.put(delivery.id(), message) != null) - throw new IOException("HTTP outgoing queue message id is invalid"); - deliveries.add(delivery); - if (deliveries.size() > HttpTransportProtocol.MAX_QUEUE) - throw new IOException("HTTP outgoing queue exceeds its bound"); - sequence = Math.max(sequence, Long.parseLong(name.substring(0, 20))); - } - if (!deliveries.isEmpty()) loaded.put(serverId, deliveries); - } - } - return loaded; - } - - private synchronized void persist(String serverId, HttpTransportProtocol.Delivery delivery) throws IOException { - Path directory = root.resolve(serverId).normalize(); - if (!directory.getParent().equals(root)) throw new IOException("HTTP outgoing queue server is invalid"); - boolean created = false; - try { Files.createDirectory(directory); created = true; } - catch (java.nio.file.FileAlreadyExistsException existing) { } - try { - if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP outgoing queue server directory is invalid"); - ownerOnlyDirectory(directory); - } finally { - // The child fsync below cannot make this newly published name durable in - // its parent. Persist the root entry before accepting the first message. - if (created) DurableFiles.forceDirectory(root); - } - if (sequence == Long.MAX_VALUE) throw new IOException("HTTP outgoing queue sequence is exhausted"); - String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); - Path target = directory.resolve(name); - Path temporary = Files.createTempFile(directory, ".pending-", ".tmp"); - try { - ownerOnlyFile(temporary); - Files.write(temporary, HttpTransportProtocol.storedDelivery(delivery), StandardOpenOption.TRUNCATE_EXISTING); - DurableFiles.forceFile(temporary); - try { Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); } - catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } - ownerOnlyFile(target); DurableFiles.forceDirectory(directory); - files.computeIfAbsent(serverId, ignored -> new HashMap<>()).put(delivery.id(), target); - } finally { Files.deleteIfExists(temporary); } - } - - private synchronized void remove(String serverId, String id) throws IOException { - Map serverFiles = files.get(serverId); - Path file = serverFiles == null ? null : serverFiles.get(id); - if (file == null) throw new IOException("HTTP outgoing queue acknowledgement is unknown"); - DurableFiles.deleteIfExists(file); - serverFiles.remove(id); - } - - private static void ownerOnlyFile(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, java.util.EnumSet.of( - java.nio.file.attribute.PosixFilePermission.OWNER_READ, - java.nio.file.attribute.PosixFilePermission.OWNER_WRITE)); } - catch (UnsupportedOperationException ignored) { } - } - private static void ownerOnlyDirectory(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, java.util.EnumSet.of( - java.nio.file.attribute.PosixFilePermission.OWNER_READ, - java.nio.file.attribute.PosixFilePermission.OWNER_WRITE, - java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE)); } - catch (UnsupportedOperationException ignored) { } - } - } -} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java deleted file mode 100644 index 9aead7e8c..000000000 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java +++ /dev/null @@ -1,467 +0,0 @@ -package com.bencodez.votingplugin.backendproxy.http; - -import java.io.IOException; -import java.io.OutputStream; -import java.math.BigInteger; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.security.KeyStore; -import java.security.PrivateKey; -import java.security.Principal; -import java.security.Security; -import java.security.cert.Certificate; -import java.security.cert.X509Certificate; -import java.time.Instant; -import java.time.Clock; -import java.time.Duration; -import java.util.Date; -import java.util.EnumSet; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.net.Socket; -import javax.net.ssl.KeyManager; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLEngine; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; -import javax.net.ssl.X509ExtendedKeyManager; -import org.bouncycastle.asn1.x500.X500Name; -import org.bouncycastle.asn1.x509.BasicConstraints; -import org.bouncycastle.asn1.x509.Extension; -import org.bouncycastle.asn1.x509.GeneralName; -import org.bouncycastle.asn1.x509.GeneralNames; -import org.bouncycastle.asn1.x509.KeyUsage; -import org.bouncycastle.asn1.x509.ExtendedKeyUsage; -import org.bouncycastle.asn1.x509.KeyPurposeId; -import org.bouncycastle.cert.X509CertificateHolder; -import org.bouncycastle.cert.X509v3CertificateBuilder; -import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; -import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.operator.ContentSigner; -import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; -import com.bencodez.votingplugin.util.DurableFiles; - -/** Durable private CA plus server identity used by the proxy HTTP listener. */ -public final class HttpTlsIdentity { - private static final String CA_FILE = "http-transport-ca.p12"; - private static final String SERVER_FILE = "http-transport-server.p12"; - private static final String PASSWORD_FILE = "http-transport-password"; - private static final String INITIALIZING_FILE = "http-transport-initializing"; - private static final String ENROLLMENT_STATE_FILE = "http-transport-clients.properties"; - private static final String OUTGOING_DIRECTORY = "outgoing-v1"; - private static final char[] EMPTY_PASSWORD = new char[0]; - static final Duration RENEW_BEFORE = Duration.ofDays(30); - static final Duration CA_RENEW_BEFORE = Duration.ofDays(365); - private final PrivateKey caKey; - private volatile X509Certificate caCertificate; - private volatile PrivateKey serverKey; - private volatile X509Certificate serverCertificate; - private final char[] password; - private final Path caFile; - private final Path serverFile; - private final String advertisedHost; - - private HttpTlsIdentity(PrivateKey caKey, X509Certificate caCertificate, PrivateKey serverKey, - X509Certificate serverCertificate, char[] password, Path caFile, Path serverFile, String advertisedHost) { - this.caKey = caKey; - this.caCertificate = caCertificate; - this.serverKey = serverKey; - this.serverCertificate = serverCertificate; - this.password = password.clone(); - this.caFile = caFile; - this.serverFile = serverFile; - this.advertisedHost = advertisedHost; - } - - public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost) throws Exception { - return loadOrCreate(directory, advertisedHost, Clock.systemUTC()); - } - - static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock clock) throws Exception { - if (advertisedHost == null || advertisedHost.isBlank() || advertisedHost.length() > 253) - throw new IllegalArgumentException("Advertised HTTPS host is invalid"); - if (clock == null) throw new IllegalArgumentException("Clock is required"); - Path identityDirectory = directory.toAbsolutePath().normalize(); - boolean created = !Files.exists(identityDirectory, LinkOption.NOFOLLOW_LINKS); - Files.createDirectories(identityDirectory); - // The identity files cannot make the newly created directory entry durable. - // Persist its parent before the TLS identity is returned for listener use. - if (created) DurableFiles.forceDirectory(identityDirectory.getParent()); - directory = identityDirectory; - Path caFile = safe(directory.resolve(CA_FILE)); - Path serverFile = safe(directory.resolve(SERVER_FILE)); - Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); - Path initializingFile = safe(directory.resolve(INITIALIZING_FILE)); - boolean caExists = Files.exists(caFile, LinkOption.NOFOLLOW_LINKS); - boolean serverExists = Files.exists(serverFile, LinkOption.NOFOLLOW_LINKS); - boolean passwordExists = Files.exists(passwordFile, LinkOption.NOFOLLOW_LINKS); - boolean initializing = Files.exists(initializingFile, LinkOption.NOFOLLOW_LINKS); - boolean anyIdentityFile = caExists || serverExists || passwordExists; - boolean completeIdentity = caExists && serverExists && passwordExists; - boolean persistentTransportState = hasPersistentTransportState(directory); - if (initializing || (anyIdentityFile && !completeIdentity)) { - if (persistentTransportState) - throw new IOException("HTTP TLS identity files are incomplete"); - if (!initializing) writeInitializationMarker(initializingFile); - discardUncommittedIdentity(caFile, serverFile, passwordFile); - caExists = false; - serverExists = false; - passwordExists = false; - initializing = true; - } - if (caExists || serverExists || passwordExists) { - if (!completeIdentity) throw new IOException("HTTP TLS identity files are incomplete"); - char[] password = readPassword(passwordFile); - try { - KeyStore ca = load(caFile, password); - KeyStore server = load(serverFile, password); - PrivateKey caKey = (PrivateKey) ca.getKey("ca", password); - X509Certificate caCertificate = (X509Certificate) ca.getCertificate("ca"); - PrivateKey serverKey = (PrivateKey) server.getKey("server", password); - X509Certificate serverCertificate = (X509Certificate) server.getCertificate("server"); - if (caKey == null || caCertificate == null || serverKey == null || serverCertificate == null) - throw new IOException("HTTP TLS identity files are invalid"); - boolean caRenewed = needsCaRenewal(caCertificate, clock); - if (caRenewed) { - ensureBouncyCastle(); - KeyPair caPair = new KeyPair(caCertificate.getPublicKey(), caKey); - caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, - CertificateRole.CA, null, clock.instant()); - ca = KeyStore.getInstance("PKCS12"); - ca.load(null, EMPTY_PASSWORD); - ca.setKeyEntry("ca", caKey, password, new Certificate[] { caCertificate }); - writeStore(caFile, ca, password); - } - if (caRenewed || !hasServerName(serverCertificate, advertisedHost) || needsRenewal(serverCertificate, clock)) { - ensureBouncyCastle(); - KeyPair serverPair = keyPair(); - serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, caKey, - CertificateRole.SERVER, advertisedHost, clock.instant()); - serverKey = serverPair.getPrivate(); - server = KeyStore.getInstance("PKCS12"); - server.load(null, EMPTY_PASSWORD); - server.setKeyEntry("server", serverKey, password, new Certificate[] { serverCertificate, caCertificate }); - writeStore(serverFile, server, password); - } - return new HttpTlsIdentity(caKey, caCertificate, serverKey, serverCertificate, password, caFile, serverFile, - advertisedHost); - } finally { Arrays.fill(password, '\0'); } - } - if (persistentTransportState) - throw new IOException("HTTP TLS identity files are missing"); - if (!initializing) writeInitializationMarker(initializingFile); - ensureBouncyCastle(); - char[] password = HttpTransportSecrets.randomToken().toCharArray(); - try { - KeyPair caPair = keyPair(); - X509Certificate caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, CertificateRole.CA, null, - clock.instant()); - KeyPair serverPair = keyPair(); - X509Certificate serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, - caPair.getPrivate(), CertificateRole.SERVER, advertisedHost, clock.instant()); - KeyStore ca = KeyStore.getInstance("PKCS12"); - ca.load(null, EMPTY_PASSWORD); - ca.setKeyEntry("ca", caPair.getPrivate(), password, new Certificate[] { caCertificate }); - KeyStore server = KeyStore.getInstance("PKCS12"); - server.load(null, EMPTY_PASSWORD); - server.setKeyEntry("server", serverPair.getPrivate(), password, new Certificate[] { serverCertificate, caCertificate }); - writeStore(caFile, ca, password); - writeStore(serverFile, server, password); - byte[] passwordBytes = asciiBytes(password); - try { writePrivate(passwordFile, passwordBytes); } - finally { Arrays.fill(passwordBytes, (byte) 0); } - DurableFiles.deleteIfExists(initializingFile); - return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password, - caFile, serverFile, advertisedHost); - } finally { Arrays.fill(password, '\0'); } - } - - public String serverCertificatePin() { - refreshIdentity(); - return HttpTransportSecrets.certificatePin(serverCertificate); - } - public String caCertificatePin() { refreshIdentity(); return HttpTransportSecrets.certificatePin(caCertificate); } - public X509Certificate caCertificate() { return caCertificate; } - public X509Certificate serverCertificate() { return serverCertificate; } - - /** - * The listener requests an optional client certificate so enrollment can share the same port. - * Any certificate that is presented must chain to this transport's private CA; normal requests - * additionally validate the certificate's persisted backend binding in the HTTP handler. - */ - public SSLContext serverContext() throws Exception { - renewIdentityIfNeeded(); - SSLContext context = SSLContext.getInstance("TLS"); - context.init(new KeyManager[] { new RotatingServerKeyManager() }, trustManagers(caCertificate), null); - return context; - } - - static TrustManager[] trustManagers(X509Certificate caCertificate) throws Exception { - KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); - trustStore.load(null, EMPTY_PASSWORD); - trustStore.setCertificateEntry("http-transport-ca", caCertificate); - TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - factory.init(trustStore); - return factory.getTrustManagers(); - } - - private void refreshIdentity() { - try { renewIdentityIfNeeded(); } - catch (Exception failure) { throw new IllegalStateException("Could not renew HTTP TLS identity", failure); } - } - - private synchronized void renewIdentityIfNeeded() throws Exception { - Clock clock = Clock.systemUTC(); - boolean renewCa = needsCaRenewal(caCertificate, clock); - if (!renewCa && !needsRenewal(serverCertificate, clock)) return; - ensureBouncyCastle(); - X509Certificate replacementCa = caCertificate; - if (renewCa) { - KeyPair caPair = new KeyPair(caCertificate.getPublicKey(), caKey); - replacementCa = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, - CertificateRole.CA, null, clock.instant()); - KeyStore caStore = KeyStore.getInstance("PKCS12"); - caStore.load(null, EMPTY_PASSWORD); - caStore.setKeyEntry("ca", caKey, password, new Certificate[] { replacementCa }); - writeStore(caFile, caStore, password); - } - KeyPair pair = keyPair(); - X509Certificate replacement = certificate("CN=" + certificateName(advertisedHost), pair, replacementCa, caKey, - CertificateRole.SERVER, advertisedHost, clock.instant()); - KeyStore store = KeyStore.getInstance("PKCS12"); - store.load(null, EMPTY_PASSWORD); - store.setKeyEntry("server", pair.getPrivate(), password, new Certificate[] { replacement, replacementCa }); - writeStore(serverFile, store, password); - caCertificate = replacementCa; - serverKey = pair.getPrivate(); - serverCertificate = replacement; - } - - public IssuedClientCertificate issueClientCertificate(String serverId) throws Exception { - return issueClientCertificate(serverId, Instant.now()); - } - - IssuedClientCertificate issueClientCertificate(String serverId, Instant issuedAt) throws Exception { - serverId = canonicalServerId(serverId); - if (issuedAt == null) throw new IllegalArgumentException("Certificate issuance time is required"); - ensureBouncyCastle(); - KeyPair pair = keyPair(); - X509Certificate certificate = certificate("CN=" + serverId, pair, caCertificate, caKey, CertificateRole.CLIENT, - "urn:votingplugin:http-backend:" + serverId, issuedAt); - char[] clientPassword = HttpTransportSecrets.randomToken().toCharArray(); - try { - KeyStore store = KeyStore.getInstance("PKCS12"); - store.load(null, EMPTY_PASSWORD); - store.setKeyEntry("client", pair.getPrivate(), clientPassword, new Certificate[] { certificate, caCertificate }); - java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); - store.store(bytes, clientPassword); - return new IssuedClientCertificate(serverId, certificate, bytes.toByteArray(), clientPassword); - } finally { Arrays.fill(clientPassword, '\0'); } - } - - public static String canonicalServerId(String serverId) { - if (serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) - throw new IllegalArgumentException("Server id is invalid"); - return serverId.toLowerCase(java.util.Locale.ROOT); - } - - public boolean issuedByThisCa(X509Certificate certificate) { - if (certificate == null) return false; - try { - certificate.checkValidity(); - certificate.verify(caCertificate.getPublicKey()); - return true; - } catch (Exception failure) { - return false; - } - } - - public boolean validClientCertificate(String expectedServerId, X509Certificate certificate) { - if (!issuedByThisCa(certificate)) return false; - try { - List usage = certificate.getExtendedKeyUsage(); - boolean[] keyUsage = certificate.getKeyUsage(); - if (usage == null || !usage.contains(KeyPurposeId.id_kp_clientAuth.getId()) || keyUsage == null || !keyUsage[0]) return false; - String expectedUri = "urn:votingplugin:http-backend:" + canonicalServerId(expectedServerId); - Collection> names = certificate.getSubjectAlternativeNames(); - if (names == null) return false; - for (List name : names) { - if (name.size() == 2 && Integer.valueOf(GeneralName.uniformResourceIdentifier).equals(name.get(0)) - && expectedUri.equals(name.get(1))) return true; - } - return false; - } catch (Exception failure) { return false; } - } - - public record IssuedClientCertificate(String serverId, X509Certificate certificate, byte[] pkcs12, char[] password) { - public IssuedClientCertificate { - pkcs12 = pkcs12.clone(); - password = password.clone(); - } - @Override public byte[] pkcs12() { return pkcs12.clone(); } - @Override public char[] password() { return password.clone(); } - } - - private static KeyPair keyPair() throws Exception { - KeyPairGenerator generator = KeyPairGenerator.getInstance("EC"); - generator.initialize(new java.security.spec.ECGenParameterSpec("secp256r1")); - return generator.generateKeyPair(); - } - - private static X509Certificate certificate(String subject, KeyPair subjectKey, X509Certificate issuer, PrivateKey issuerKey, - CertificateRole role, String subjectAlternativeName, Instant now) throws Exception { - X500Name issuerName = issuer == null ? new X500Name(subject) : new X500Name(issuer.getSubjectX500Principal().getName()); - X509v3CertificateBuilder builder = new JcaX509v3CertificateBuilder(issuerName, - new BigInteger(160, new java.security.SecureRandom()).setBit(159), Date.from(now.minusSeconds(300)), - Date.from(now.plusSeconds(role == CertificateRole.CA ? 315360000L : 31536000L)), new X500Name(subject), subjectKey.getPublic()); - builder.addExtension(Extension.basicConstraints, true, new BasicConstraints(role == CertificateRole.CA)); - builder.addExtension(Extension.keyUsage, true, new KeyUsage(role == CertificateRole.CA ? KeyUsage.keyCertSign | KeyUsage.cRLSign - : KeyUsage.digitalSignature)); - if (role == CertificateRole.SERVER) builder.addExtension(Extension.extendedKeyUsage, false, - new ExtendedKeyUsage(KeyPurposeId.id_kp_serverAuth)); - if (role == CertificateRole.CLIENT) builder.addExtension(Extension.extendedKeyUsage, false, - new ExtendedKeyUsage(KeyPurposeId.id_kp_clientAuth)); - if (role == CertificateRole.SERVER && subjectAlternativeName != null) { - GeneralName name; - if (subjectAlternativeName.matches("(?:\\d{1,3}\\.){3}\\d{1,3}") || subjectAlternativeName.indexOf(':') >= 0) - name = new GeneralName(GeneralName.iPAddress, subjectAlternativeName); - else name = new GeneralName(GeneralName.dNSName, subjectAlternativeName); - builder.addExtension(Extension.subjectAlternativeName, false, new GeneralNames(name)); - } - if (role == CertificateRole.CLIENT) builder.addExtension(Extension.subjectAlternativeName, false, - new GeneralNames(new GeneralName(GeneralName.uniformResourceIdentifier, subjectAlternativeName))); - ContentSigner signer = new JcaContentSignerBuilder("SHA256withECDSA").setProvider("BC") - .build(issuerKey == null ? subjectKey.getPrivate() : issuerKey); - X509CertificateHolder holder = builder.build(signer); - return new JcaX509CertificateConverter().setProvider("BC").getCertificate(holder); - } - - static boolean needsRenewal(X509Certificate certificate, Clock clock) { - return certificate == null || !certificate.getNotAfter().toInstant().isAfter(clock.instant().plus(RENEW_BEFORE)); - } - - static boolean needsCaRenewal(X509Certificate certificate, Clock clock) { - return certificate == null || !certificate.getNotAfter().toInstant().isAfter(clock.instant().plus(CA_RENEW_BEFORE)); - } - - private static void ensureBouncyCastle() { - if (Security.getProvider("BC") == null) Security.addProvider(new BouncyCastleProvider()); - } - - private static String certificateName(String host) { - return host.replaceAll("[^A-Za-z0-9 ._-]", "_"); - } - - private static boolean hasServerName(X509Certificate certificate, String advertisedHost) { - try { - Collection> names = certificate.getSubjectAlternativeNames(); - if (names == null) return false; - for (List name : names) { - if (name.size() != 2 || !(name.get(1) instanceof String value)) continue; - if ((Integer.valueOf(GeneralName.dNSName).equals(name.get(0)) || Integer.valueOf(GeneralName.iPAddress).equals(name.get(0))) - && advertisedHost.equalsIgnoreCase(value)) return true; - } - return false; - } catch (Exception failure) { return false; } - } - - private static Path safe(Path file) throws IOException { - Path parent = file.toAbsolutePath().normalize().getParent(); - if (parent == null || Files.isSymbolicLink(file)) throw new IOException("Refusing unsafe HTTP TLS identity path"); - return file.toAbsolutePath().normalize(); - } - - private static KeyStore load(Path path, char[] password) throws Exception { - KeyStore store = KeyStore.getInstance("PKCS12"); - try (var input = Files.newInputStream(path, LinkOption.NOFOLLOW_LINKS)) { store.load(input, password); } - return store; - } - - private static char[] readPassword(Path path) throws IOException { - byte[] bytes = Files.readAllBytes(path); - if (bytes.length < 40 || bytes.length > 128) throw new IOException("HTTP TLS password file is invalid"); - try { return new String(bytes, java.nio.charset.StandardCharsets.US_ASCII).toCharArray(); } - finally { Arrays.fill(bytes, (byte) 0); } - } - - private static void writeStore(Path file, KeyStore store, char[] password) throws Exception { - java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); - store.store(bytes, password); - byte[] contents = bytes.toByteArray(); - try { writePrivate(file, contents); } - finally { Arrays.fill(contents, (byte) 0); } - } - - private static void writePrivate(Path file, byte[] contents) throws IOException { - Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); - try { - setOwnerOnly(temporary); - Files.write(temporary, contents, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); - DurableFiles.forceFile(temporary); - try { Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } - catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); } - setOwnerOnly(file); - DurableFiles.forceDirectory(file.getParent()); - } finally { Files.deleteIfExists(temporary); } - } - - private static void writeInitializationMarker(Path file) throws IOException { - writePrivate(file, "initializing\n".getBytes(java.nio.charset.StandardCharsets.US_ASCII)); - } - - private static void discardUncommittedIdentity(Path caFile, Path serverFile, Path passwordFile) throws IOException { - DurableFiles.deleteIfExists(caFile); - DurableFiles.deleteIfExists(serverFile); - DurableFiles.deleteIfExists(passwordFile); - } - - private static boolean hasPersistentTransportState(Path directory) { - return Files.exists(directory.resolve(ENROLLMENT_STATE_FILE), LinkOption.NOFOLLOW_LINKS) - || Files.exists(directory.resolve(OUTGOING_DIRECTORY), LinkOption.NOFOLLOW_LINKS); - } - - private static byte[] asciiBytes(char[] characters) { - byte[] output = new byte[characters.length]; - for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; - return output; - } - - private static void setOwnerOnly(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } - catch (UnsupportedOperationException ignored) { /* Windows ACLs are inherited; never make the file world-readable. */ } - } - - private final class RotatingServerKeyManager extends X509ExtendedKeyManager { - private static final String ALIAS = "server"; - private void refresh() { - refreshIdentity(); - } - private String alias(String keyType) { - refresh(); - return keyType != null && ("EC".equalsIgnoreCase(keyType) || keyType.toUpperCase(java.util.Locale.ROOT).startsWith("EC_")) - ? ALIAS : null; - } - @Override public String[] getClientAliases(String keyType, Principal[] issuers) { return null; } - @Override public String chooseClientAlias(String[] keyTypes, Principal[] issuers, Socket socket) { return null; } - @Override public String[] getServerAliases(String keyType, Principal[] issuers) { - return alias(keyType) == null ? null : new String[] { ALIAS }; - } - @Override public String chooseServerAlias(String keyType, Principal[] issuers, Socket socket) { return alias(keyType); } - @Override public String chooseEngineServerAlias(String keyType, Principal[] issuers, SSLEngine engine) { return alias(keyType); } - @Override public X509Certificate[] getCertificateChain(String alias) { - refresh(); - return ALIAS.equals(alias) ? new X509Certificate[] { serverCertificate, caCertificate } : null; - } - @Override public PrivateKey getPrivateKey(String alias) { refresh(); return ALIAS.equals(alias) ? serverKey : null; } - } - - private enum CertificateRole { CA, SERVER, CLIENT } -} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java deleted file mode 100644 index 2aafffb33..000000000 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java +++ /dev/null @@ -1,231 +0,0 @@ -package com.bencodez.votingplugin.backendproxy.http; - -import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; -import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Base64; -import java.util.Collection; -import java.util.List; -import java.util.UUID; - -/** Strict, versioned HTTP transport envelope. Payloads remain canonical JsonEnvelopeCodec values. */ -final class HttpTransportProtocol { - static final int VERSION = 1; - static final int MAX_BODY_BYTES = 256 * 1024; - static final int MAX_BATCH = 64; - static final int MAX_ENVELOPE_BYTES = 48 * 1024; - static final int MAX_QUEUE = 1024; - static final long MAX_CLOCK_SKEW_MILLIS = 90_000L; - - private HttpTransportProtocol() { } - - static void validateEnvelope(JsonEnvelope envelope) { - if (envelope == null || JsonEnvelopeCodec.encode(envelope).getBytes(StandardCharsets.UTF_8).length > MAX_ENVELOPE_BYTES) throw bad(); - } - - static byte[] request(String server, String session, long sequence, Collection acks, - Collection messages) { - JsonObject root = base(server, session, sequence); - root.add("acks", ids(acks)); - root.add("messages", messages(messages)); - byte[] encoded = root.toString().getBytes(StandardCharsets.UTF_8); - if (encoded.length > MAX_BODY_BYTES) throw bad(); - return encoded; - } - - static List fittingMessages(String server, String session, long sequence, Collection acks, - Collection candidates) { - List output = new ArrayList<>(); - for (Delivery candidate : candidates) { - if (output.size() == MAX_BATCH) break; - output.add(candidate); - try { request(server, session, sequence, acks, output); } - catch (IllegalArgumentException tooLarge) { output.remove(output.size() - 1); break; } - } - return output; - } - - static byte[] storedDelivery(Delivery delivery) { - validId(delivery.id()); - validateEnvelope(delivery.envelope()); - JsonObject root = new JsonObject(); - root.addProperty("v", VERSION); - root.addProperty("id", delivery.id()); - root.addProperty("payload", Base64.getUrlEncoder().withoutPadding().encodeToString( - JsonEnvelopeCodec.encode(delivery.envelope()).getBytes(StandardCharsets.UTF_8))); - return root.toString().getBytes(StandardCharsets.UTF_8); - } - - static Delivery parseStoredDelivery(byte[] body) { - if (body == null || body.length == 0 || body.length > MAX_ENVELOPE_BYTES * 2) throw bad(); - try { - JsonObject root = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)).getAsJsonObject(); - requireOnly(root, "v", "id", "payload"); - if (integer(root, "v") != VERSION) throw bad(); - String id = string(root, "id", 64); validId(id); - byte[] payload = Base64.getUrlDecoder().decode(string(root, "payload", MAX_ENVELOPE_BYTES * 2)); - if (payload.length == 0 || payload.length > MAX_ENVELOPE_BYTES) throw bad(); - return new Delivery(id, JsonEnvelopeCodec.decode(new String(payload, StandardCharsets.UTF_8))); - } catch (RuntimeException invalid) { throw bad(); } - } - - static byte[] response(String server, String session, long sequence, Collection acks, - Collection messages) { - return request(server, session, sequence, acks, messages); - } - - static Packet parsePacket(byte[] body) { - if (body == null || body.length == 0 || body.length > MAX_BODY_BYTES) throw bad(); - try { - JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); - if (!parsed.isJsonObject()) throw bad(); - JsonObject root = parsed.getAsJsonObject(); - requireOnly(root, "v", "server", "session", "sequence", "timestamp", "acks", "messages"); - if (integer(root, "v") != VERSION) throw bad(); - String server = HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); - String session = uuid(root, "session"); - long sequence = nonNegative(root, "sequence"); - long timestamp = integer(root, "timestamp"); - long now = Instant.now().toEpochMilli(); - if (timestamp < now - MAX_CLOCK_SKEW_MILLIS || timestamp > now + MAX_CLOCK_SKEW_MILLIS) throw bad(); - List acks = parseIds(root.get("acks")); - List messages = parseMessages(root.get("messages")); - return new Packet(server, session, sequence, acks, messages); - } catch (RuntimeException invalid) { throw bad(); } - } - - static byte[] enrollmentResponse(HttpTlsIdentity.IssuedClientCertificate certificate) { - JsonObject output = new JsonObject(); - byte[] bundle = certificate.pkcs12(); - try { output.addProperty("bundle", Base64.getUrlEncoder().withoutPadding().encodeToString(bundle)); } - finally { java.util.Arrays.fill(bundle, (byte) 0); } - char[] password = certificate.password(); - try { output.addProperty("password", new String(password)); } - finally { java.util.Arrays.fill(password, '\0'); } - return output.toString().getBytes(StandardCharsets.UTF_8); - } - - static Enrollment parseEnrollment(byte[] body) { - if (body == null || body.length == 0 || body.length > 8192) throw bad(); - try { - JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); - if (!parsed.isJsonObject()) throw bad(); - JsonObject root = parsed.getAsJsonObject(); - requireOnly(root, "server", "token"); - String server = HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); - String token = string(root, "token", 128); - if (!token.matches("[A-Za-z0-9_-]{43,128}")) throw bad(); - return new Enrollment(server, token); - } catch (RuntimeException invalid) { throw bad(); } - } - - static byte[] renewalRequest(String server) { - JsonObject root = new JsonObject(); - root.addProperty("server", HttpTlsIdentity.canonicalServerId(server)); - return root.toString().getBytes(StandardCharsets.UTF_8); - } - - static String parseRenewal(byte[] body) { - if (body == null || body.length == 0 || body.length > 1024) throw bad(); - try { - JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); - if (!parsed.isJsonObject()) throw bad(); - JsonObject root = parsed.getAsJsonObject(); - requireOnly(root, "server"); - return HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); - } catch (RuntimeException invalid) { throw bad(); } - } - - static HttpTlsIdentity.IssuedClientCertificate parseEnrollmentResponse(String server, byte[] body) { - if (body == null || body.length == 0 || body.length > MAX_BODY_BYTES) throw bad(); - try { - JsonObject root = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)).getAsJsonObject(); - requireOnly(root, "bundle", "password"); - byte[] bundle = Base64.getUrlDecoder().decode(string(root, "bundle", MAX_BODY_BYTES * 2)); - char[] password = string(root, "password", 128).toCharArray(); - if (bundle.length == 0 || password.length < 40) throw bad(); - try { return new HttpTlsIdentity.IssuedClientCertificate(HttpTlsIdentity.canonicalServerId(server), null, bundle, password); } - finally { java.util.Arrays.fill(bundle, (byte) 0); java.util.Arrays.fill(password, '\0'); } - } catch (RuntimeException invalid) { throw bad(); } - } - - private static JsonObject base(String server, String session, long sequence) { - JsonObject root = new JsonObject(); - root.addProperty("v", VERSION); root.addProperty("server", server); root.addProperty("session", session); - root.addProperty("sequence", sequence); root.addProperty("timestamp", Instant.now().toEpochMilli()); - return root; - } - private static JsonArray ids(Collection values) { - if (values == null || values.size() > MAX_BATCH) throw bad(); - JsonArray output = new JsonArray(); - for (String value : values) { validId(value); output.add(value); } - return output; - } - private static JsonArray messages(Collection values) { - if (values == null || values.size() > MAX_BATCH) throw bad(); - JsonArray output = new JsonArray(); - for (Delivery delivery : values) { - validId(delivery.id()); - String encoded = JsonEnvelopeCodec.encode(delivery.envelope()); - byte[] bytes = encoded.getBytes(StandardCharsets.UTF_8); - if (bytes.length > MAX_ENVELOPE_BYTES) throw bad(); - JsonObject item = new JsonObject(); item.addProperty("id", delivery.id()); - item.addProperty("payload", Base64.getUrlEncoder().withoutPadding().encodeToString(bytes)); output.add(item); - } - return output; - } - private static List parseIds(JsonElement value) { - if (value == null || !value.isJsonArray() || value.getAsJsonArray().size() > MAX_BATCH) throw bad(); - List output = new ArrayList<>(); - for (JsonElement item : value.getAsJsonArray()) { if (!item.isJsonPrimitive()) throw bad(); String id = item.getAsString(); validId(id); output.add(id); } - return output; - } - private static List parseMessages(JsonElement value) { - if (value == null || !value.isJsonArray() || value.getAsJsonArray().size() > MAX_BATCH) throw bad(); - List output = new ArrayList<>(); - for (JsonElement item : value.getAsJsonArray()) { - if (!item.isJsonObject()) throw bad(); JsonObject object = item.getAsJsonObject(); requireOnly(object, "id", "payload"); - String id = string(object, "id", 64); validId(id); - byte[] payload = Base64.getUrlDecoder().decode(string(object, "payload", MAX_ENVELOPE_BYTES * 2)); - if (payload.length == 0 || payload.length > MAX_ENVELOPE_BYTES) throw bad(); - JsonEnvelope envelope = JsonEnvelopeCodec.decode(new String(payload, StandardCharsets.UTF_8)); - output.add(new Delivery(id, envelope)); - } - return output; - } - private static void requireOnly(JsonObject object, String... names) { - for (String name : object.keySet()) { boolean found = false; for (String allowed : names) if (allowed.equals(name)) { found = true; break; } if (!found) throw bad(); } - for (String name : names) if (!object.has(name) || object.get(name).isJsonNull()) throw bad(); - } - private static String string(JsonObject object, String name, int max) { JsonElement v = object.get(name); if (!v.isJsonPrimitive() || !v.getAsJsonPrimitive().isString()) throw bad(); String value = v.getAsString(); if (value.isEmpty() || value.length() > max) throw bad(); return value; } - private static long integer(JsonObject object, String name) { - try { - JsonElement value = object.get(name); - if (!value.isJsonPrimitive() || !value.getAsJsonPrimitive().isNumber()) throw bad(); - String token = value.getAsString(); - if (!token.matches("-?(?:0|[1-9][0-9]*)")) throw bad(); - return Long.parseLong(token); - } catch (RuntimeException failure) { throw bad(); } - } - private static long nonNegative(JsonObject object, String name) { long n = integer(object, name); if (n < 0) throw bad(); return n; } - private static String uuid(JsonObject object, String name) { return canonicalUuid(string(object, name, 64)); } - static void validId(String id) { if (id == null || id.length() > 64) throw bad(); canonicalUuid(id); } - private static String canonicalUuid(String value) { - try { - UUID parsed = UUID.fromString(value); - if (!parsed.toString().equals(value)) throw bad(); - return value; - } catch (IllegalArgumentException invalid) { throw bad(); } - } - private static IllegalArgumentException bad() { return new IllegalArgumentException("Invalid HTTP transport message"); } - - record Delivery(String id, JsonEnvelope envelope) { } - record Packet(String server, String session, long sequence, List acks, List messages) { } - record Enrollment(String server, String token) { } -} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecrets.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecrets.java deleted file mode 100644 index f1a8ec04b..000000000 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecrets.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.bencodez.votingplugin.backendproxy.http; - -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; -import java.security.cert.X509Certificate; -import java.util.Base64; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; - -/** Small, deliberately dependency-free cryptographic helpers for the HTTP transport. */ -final class HttpTransportSecrets { - private static final SecureRandom RANDOM = new SecureRandom(); - - private HttpTransportSecrets() { } - - static byte[] randomBytes(int length) { - if (length < 16) throw new IllegalArgumentException("Secret length is too small"); - byte[] value = new byte[length]; - RANDOM.nextBytes(value); - return value; - } - - static String randomToken() { - return Base64.getUrlEncoder().withoutPadding().encodeToString(randomBytes(32)); - } - - static byte[] sha256(byte[] value) { - try { - return MessageDigest.getInstance("SHA-256").digest(value); - } catch (NoSuchAlgorithmException impossible) { - throw new IllegalStateException("SHA-256 is unavailable", impossible); - } - } - - static String sha256Hex(byte[] value) { - StringBuilder output = new StringBuilder(64); - for (byte part : sha256(value)) output.append(String.format("%02x", part & 0xff)); - return output.toString(); - } - - static String certificatePin(X509Certificate certificate) { - try { - return sha256Hex(certificate.getEncoded()); - } catch (Exception failure) { - throw new IllegalArgumentException("Could not encode certificate", failure); - } - } - - static boolean constantTimeEquals(byte[] first, byte[] second) { - return first != null && second != null && MessageDigest.isEqual(first, second); - } - - static String hmacSha256Url(byte[] key, String value) { - try { - Mac mac = Mac.getInstance("HmacSHA256"); - mac.init(new SecretKeySpec(key, "HmacSHA256")); - return Base64.getUrlEncoder().withoutPadding().encodeToString(mac.doFinal(value.getBytes(StandardCharsets.US_ASCII))); - } catch (Exception failure) { - throw new IllegalStateException("HMAC-SHA-256 is unavailable", failure); - } - } -} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 9e87f225c..d6e3eb35b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -10,11 +10,11 @@ import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; +import com.bencodez.simpleapi.servercomm.http.HttpBackendTransportConnector; +import com.bencodez.simpleapi.servercomm.http.HttpClientCredentialStore; +import com.bencodez.simpleapi.servercomm.http.HttpConnectionCode; +import com.bencodez.simpleapi.servercomm.http.HttpTlsIdentity; import com.bencodez.votingplugin.VotingPluginMain; -import com.bencodez.votingplugin.backendproxy.http.HttpBackendTransportConnector; -import com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore; -import com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode; -import com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity; /** Backend adapter for the secure outbound-only HTTP proxy transport. */ public final class HttpBackendProxyTransport implements BackendProxyTransport { @@ -85,12 +85,13 @@ HttpBackendProxyTransport recreatePrepared() { @Override public void prepareForReplacement() { - HttpBackendTransportConnector active = connector; - HttpClientCredentialStore.ActiveCredentialGeneration generation = active == null - ? null : active.activeCredentialGeneration(); - if (generation == null) + if (connector == null || configuredDirectory == null) throw new IllegalStateException("Could not preserve the active HTTP client credential before it became ready"); - configuredCredentialGeneration = generation; + try { + configuredCredentialGeneration = HttpClientCredentialStore.snapshotActiveGeneration(configuredDirectory); + } catch (Exception failure) { + throw new IllegalStateException("Could not preserve the active HTTP client credential", failure); + } close(); } @@ -129,7 +130,7 @@ private void initialize(Path directory, String serverId, String configuredCode, credentialGenerationToRestore == null ? configuredCode : null); if (code != null) HttpBackendTransportConnector.enroll(code, serverId, directory); HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); - if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) + if (!enrolled.profile().serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); replacement = new HttpBackendTransportConnector(directory, messageHandler::onMessage); replacement.start(); @@ -225,7 +226,7 @@ public static void validateConfiguration(Path directory, String serverId, String } static HttpConnectionCode enrollmentCode(Path directory, String serverId, String configuredCode) { - try { serverId = com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId); } + try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } catch (IllegalArgumentException invalid) { throw new IllegalStateException("HTTP requires a valid unique backend Server name", invalid); } boolean enrolled = HttpClientCredentialStore.hasEnrolledProfile(directory); if (configuredCode != null && !configuredCode.isBlank()) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 8e49773e1..5cad21d9f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -51,6 +51,9 @@ import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec; import com.bencodez.simpleapi.servercomm.global.GlobalMessageListener; import com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler; +import com.bencodez.simpleapi.servercomm.http.HttpEnrollmentAuthority; +import com.bencodez.simpleapi.servercomm.http.HttpProxyTransportServer; +import com.bencodez.simpleapi.servercomm.http.HttpTlsIdentity; import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; import com.bencodez.simpleapi.servercomm.mqtt.MqttServerComm; import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; @@ -66,9 +69,6 @@ import com.bencodez.simpleapi.sql.data.DataValueInt; import com.bencodez.simpleapi.sql.data.DataValueString; import com.bencodez.simpleapi.sql.mysql.config.MysqlConfig; -import com.bencodez.votingplugin.backendproxy.http.HttpEnrollmentAuthority; -import com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer; -import com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity; import com.bencodez.votingplugin.proxy.broadcast.ProxyBroadcastDecider; import com.bencodez.votingplugin.proxy.cache.IVoteCache; import com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects; @@ -2811,7 +2811,7 @@ private void startHttpTransport() { this::acknowledgeVotePartyDelivery); httpTransportServer.start(); logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" - + httpTransportServer.port() + "; use /votingpluginbungee httpcode for each backend"); + + httpTransportServer.port() + "; use /votingpluginproxy httpcode for each backend"); } catch (Exception failure) { closeHttpTransport(); throw new IllegalStateException("HTTP transport could not start securely", failure); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java index 069796fb6..871a86fd7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java @@ -110,8 +110,8 @@ private String getHelpMessage() { helpBuilder.append("/votingplugin vote - Send a vote\n"); helpBuilder.append("/votingplugin forcetimechange - Force a time change\n"); helpBuilder.append("/votingplugin status - Check connection status\n"); - helpBuilder.append("/votingplugin httpcode - Create a node-bound one-time HTTP connection code\n"); - helpBuilder.append("/votingplugin httprevoke - Revoke a backend identity before re-enrollment\n"); + helpBuilder.append("/votingpluginproxy httpcode - Create a node-bound one-time HTTP connection code\n"); + helpBuilder.append("/votingpluginproxy httprevoke - Revoke a backend identity before re-enrollment\n"); helpBuilder.append("/votingplugin multiproxystatus - Send status message across proxies\n"); helpBuilder.append("/votingplugin voteparty - Trigger or modify vote party\n"); return helpBuilder.toString(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 20f550879..f41eea3d7 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -2,8 +2,11 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -18,6 +21,7 @@ import org.junit.jupiter.api.Test; import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; +import com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessageHandler; import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; @@ -112,6 +116,43 @@ void stagedPluginMessageTransportDefersRelaySwapUntilPublication() { verify(plugin).deactivateBackendPluginMessageHandler(replacement); } + @Test + void movingPluginMessageRelayDetachesItFromThePreviousPluginMessage() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class, + CALLS_REAL_METHODS); + com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessage first = + mock(com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessage.class); + com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessage second = + mock(com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessage.class); + java.util.ArrayList firstHandlers = new java.util.ArrayList<>(); + java.util.ArrayList secondHandlers = new java.util.ArrayList<>(); + GlobalMessageHandler firstTarget = mock(GlobalMessageHandler.class); + GlobalMessageHandler secondTarget = mock(GlobalMessageHandler.class); + AtomicReference current = + new AtomicReference<>(first); + when(plugin.getPluginMessaging()).thenAnswer(ignored -> current.get()); + when(first.getPluginMessages()).thenReturn(firstHandlers); + when(second.getPluginMessages()).thenReturn(secondHandlers); + org.mockito.Mockito.doAnswer(invocation -> firstHandlers.add(invocation.getArgument(0))).when(first) + .add(org.mockito.ArgumentMatchers.any(PluginMessageHandler.class)); + org.mockito.Mockito.doAnswer(invocation -> secondHandlers.add(invocation.getArgument(0))).when(second) + .add(org.mockito.ArgumentMatchers.any(PluginMessageHandler.class)); + Field targetField = com.bencodez.votingplugin.VotingPluginMain.class + .getDeclaredField("backendPluginMessageTarget"); + targetField.setAccessible(true); + targetField.set(plugin, new AtomicReference()); + + plugin.activateBackendPluginMessageHandler(firstTarget); + current.set(second); + plugin.activateBackendPluginMessageHandler(secondTarget); + + assertTrue(firstHandlers.isEmpty()); + assertEquals(1, secondHandlers.size()); + secondHandlers.get(0).onReceive(null); + verify(firstTarget, never()).onMessage(null); + verify(secondTarget).onMessage(null); + } + @Test void failedPresenceActivationDoesNotAnnounceReplacementGeneration() { com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); @@ -226,6 +267,48 @@ void releasesMqttSubscriberBeforeSameMethodReplacement() throws Exception { assertNull(handler.getMqttHandler()); } + @Test + void failedSameSocketReplacementLeavesExistingTransportAvailable() throws Exception { + BackendProxyHandler handler = handlerWithTransport(BungeeMethod.SOCKETS); + SocketHandler socket = mock(SocketHandler.class); + SocketBackendProxyTransport transport = new SocketBackendProxyTransport(null); + setField(transport, "socketHandler", socket); + setTransport(handler, transport); + + assertFalse(handler.prepareForReplacement(BungeeMethod.SOCKETS)); + + assertSame(socket, handler.getSocketHandler()); + verifyNoInteractions(socket); + } + + @Test + void failedSameMqttReplacementLeavesExistingTransportAvailable() throws Exception { + BackendProxyHandler handler = handlerWithTransport(BungeeMethod.MQTT); + MqttHandler mqtt = mock(MqttHandler.class); + MqttBackendProxyTransport transport = new MqttBackendProxyTransport(null); + setField(transport, "mqttHandler", mqtt); + setTransport(handler, transport); + + assertFalse(handler.prepareForReplacement(BungeeMethod.MQTT)); + + assertSame(mqtt, handler.getMqttHandler()); + verifyNoInteractions(mqtt); + } + + @Test + void failedSameMysqlReplacementLeavesExistingTransportAvailable() throws Exception { + BackendProxyHandler handler = handlerWithTransport(BungeeMethod.MYSQL); + MySqlMessenger messenger = mock(MySqlMessenger.class); + MysqlBackendProxyTransport transport = new MysqlBackendProxyTransport(null); + setField(transport, "messenger", messenger); + setTransport(handler, transport); + + assertFalse(handler.prepareForReplacement(BungeeMethod.MYSQL)); + + assertSame(messenger, handler.getBackendMysqlMessenger()); + verifyNoInteractions(messenger); + } + @Test void releasesMysqlSubscriberBeforeSameMethodReplacement() throws Exception { MysqlBackendProxyTransport handler = new MysqlBackendProxyTransport(null); @@ -253,4 +336,23 @@ void stopsGlobalDataTimerWhenHandlerIsReplaced() throws Exception { verify(timer).shutdownNow(); assertNull(handler.getTimer()); } + + private BackendProxyHandler handlerWithTransport(BungeeMethod method) throws Exception { + BackendProxyHandler handler = new BackendProxyHandler(null); + setField(handler, "method", method); + return handler; + } + + private void setTransport(BackendProxyHandler handler, BackendProxyTransport transport) throws Exception { + Field managerField = BackendProxyHandler.class.getDeclaredField("transportManager"); + managerField.setAccessible(true); + BackendProxyTransportManager manager = (BackendProxyTransportManager) managerField.get(handler); + setField(manager, "transport", transport); + } + + private void setField(Object target, String name, Object value) throws Exception { + Field field = target.getClass().getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java deleted file mode 100644 index e011f0405..000000000 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ /dev/null @@ -1,765 +0,0 @@ -package com.bencodez.votingplugin.backendproxy.http; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; -import java.net.InetSocketAddress; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -class HttpTransportRuntimeTest { - @TempDir Path directory; - - @Test - void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); - CountDownLatch proxyReceived = new CountDownLatch(1), backendReceived = new CountDownLatch(1); - AtomicReference received = new AtomicReference<>(); - try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, - message -> { received.set(message); proxyReceived.countDown(); })) { - server.start(); - HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); - HttpBackendTransportConnector.enroll(code, "lobby-1", directory.resolve("client")); - try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(directory.resolve("client"), - envelope -> backendReceived.countDown())) { - connector.start(); - assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8)), - "an authenticated transport response must make the connector ready"); - assertTrue(connector.send(JsonEnvelope.builder("to-proxy").put("server", "forged").build())); - assertTrue(proxyReceived.await(8, TimeUnit.SECONDS)); - assertEquals("lobby-1", received.get().serverId()); - assertEquals("lobby-1", received.get().envelope().getFields().get("server")); - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); - while (connector.queuedOutgoing() != 0 && System.nanoTime() < deadline) Thread.sleep(10); - assertEquals(0, connector.queuedOutgoing(), "proxy ACK must remove the exact outbound delivery ID"); - assertTrue(server.send("lobby-1", JsonEnvelope.builder("to-backend").build())); - assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); - Path inboundFence = directory.resolve("client").resolve("http-transport-inbound-deliveries"); - long fenceDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); - while (countRegularFiles(inboundFence) != 0L && System.nanoTime() < fenceDeadline) Thread.sleep(20); - assertEquals(0L, countRegularFiles(inboundFence), "a confirmed ACK must remove the backend replay fence"); - } - } - } - - @Test - void responseBodyConsumptionRemainsBoundedByRequestTimeout() throws Exception { - com.sun.net.httpserver.HttpServer server = com.sun.net.httpserver.HttpServer.create( - new InetSocketAddress("localhost", 0), 1); - CountDownLatch release = new CountDownLatch(1); - server.createContext("/stall", exchange -> { - exchange.sendResponseHeaders(200, 8); - try (var output = exchange.getResponseBody()) { - output.write(1); - output.flush(); - try { release.await(5, TimeUnit.SECONDS); } - catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } - } - }); - server.start(); - try { - HttpRequest request = HttpRequest.newBuilder(URI.create("http://localhost:" + server.getAddress().getPort() - + "/stall")).timeout(Duration.ofMillis(250)).GET().build(); - assertTimeoutPreemptively(Duration.ofSeconds(2), () -> assertThrows(java.io.IOException.class, - () -> HttpBackendTransportConnector.sendLimited(HttpClient.newHttpClient(), request))); - } finally { - release.countDown(); - server.stop(0); - } - } - - @Test - void stableProxyDeliveryIdsAreIdempotentAndAcknowledgedBeforeRemoval() throws Exception { - AtomicLong acknowledged = new AtomicLong(); - HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", null, - (server, deliveryId) -> acknowledged.incrementAndGet()); - String deliveryId = java.util.UUID.randomUUID().toString(); - HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, - JsonEnvelope.builder("vote-party").build()); - assertTrue(state.enqueue(delivery)); - assertTrue(state.enqueue(delivery)); - assertFalse(state.enqueue(new HttpTransportProtocol.Delivery(deliveryId, - JsonEnvelope.builder("different").build()))); - state.acknowledge(java.util.List.of(deliveryId)); - assertEquals(1L, acknowledged.get()); - assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); - } - - @Test - void failedAcknowledgementCallbackRetainsProxyDelivery() throws Exception { - HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", null, - (server, deliveryId) -> { throw new java.io.IOException("cache save failed"); }); - String deliveryId = java.util.UUID.randomUUID().toString(); - assertTrue(state.enqueue(new HttpTransportProtocol.Delivery(deliveryId, - JsonEnvelope.builder("vote-party").build()))); - assertThrows(java.io.IOException.class, () -> state.acknowledge(java.util.List.of(deliveryId))); - assertEquals(deliveryId, state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0) - .messages().iterator().next().id()); - } - - @Test - void closeWaitsForTheCredentialOwningPollerToStop() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("close-proxy"), "localhost"); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("close-authority")); - try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), - identity, authority, ignored -> { })) { - server.start(); - Path clientDirectory = directory.resolve("close-client"); - HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), - Duration.ofMinutes(5)); - HttpBackendTransportConnector.enroll(code, "lobby-1", clientDirectory); - try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { - connector.start(); - assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); - connector.close(); - assertFalse(connector.pollerAlive(), - "credential-directory ownership must outlive every poller filesystem mutation"); - } - } - } - - @Test - void finalFlushDeliversMessagesQueuedBehindAnActiveLongPoll() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("flush-proxy"), "localhost"); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("flush-authority")); - CountDownLatch received = new CountDownLatch(1); - AtomicReference envelope = new AtomicReference<>(); - try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), - identity, authority, message -> { - envelope.set(message.envelope()); - received.countDown(); - })) { - server.start(); - Path clientDirectory = directory.resolve("flush-client"); - HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), - Duration.ofMinutes(5)); - HttpBackendTransportConnector.enroll(code, "lobby-1", clientDirectory); - HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { }); - connector.start(); - assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); - assertTrue(connector.send(JsonEnvelope.builder("backend-stopped").build())); - try { - assertTrue(connector.flushOutgoing(System.nanoTime() + TimeUnit.SECONDS.toNanos(5))); - assertTrue(received.await(1, TimeUnit.SECONDS)); - assertEquals("backend-stopped", envelope.get().getSubChannel()); - assertEquals(0, connector.queuedOutgoing()); - } finally { connector.close(); } - } - } - - @Test - void normalTransportRejectsAClientWithoutCertificate() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); - try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { - server.start(); - HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); - HttpClient client = HttpClient.newBuilder().sslContext(HttpPinnedTls.clientContext(code)).build(); - byte[] body = HttpTransportProtocol.request("lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(), java.util.List.of()); - HttpResponse response = client.send(HttpRequest.newBuilder(code.endpoint().resolve("v1/transport")) - .timeout(Duration.ofSeconds(5)).header("Content-Type", "application/json").POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(), HttpResponse.BodyHandlers.ofByteArray()); - assertEquals(401, response.statusCode()); - } - } - - @Test - void boundedQueuesFailClosed() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); - try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { - for (int i = 0; i < HttpTransportProtocol.MAX_QUEUE; i++) assertTrue(server.send("lobby-1", JsonEnvelope.builder("x").build())); - assertFalse(server.send("lobby-1", JsonEnvelope.builder("x").build())); - assertFalse(server.send("lobby-1", JsonEnvelope.builder("x").put("large", "x".repeat(HttpTransportProtocol.MAX_ENVELOPE_BYTES)).build())); - } - } - - @Test - void proxyOutgoingQueueSurvivesRestartUntilBackendAcknowledges() throws Exception { - Path proxyDirectory = directory.resolve("proxy"); - Path authorityDirectory = directory.resolve("authority"); - Path queueDirectory = directory.resolve("outgoing"); - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost"); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); - HttpProxyTransportServer first = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), - identity, authority, queueDirectory, ignored -> { }); - assertTrue(first.send("lobby-1", JsonEnvelope.builder("durable").build())); - first.close(); - - CountDownLatch received = new CountDownLatch(1); - try (HttpProxyTransportServer restarted = new HttpProxyTransportServer( - new InetSocketAddress("localhost", 0), identity, authority, queueDirectory, ignored -> { })) { - restarted.start(); - HttpConnectionCode code = authority.createConnectionCode("lobby-1", restarted.endpoint("localhost"), - Duration.ofMinutes(5)); - HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, - "lobby-1", directory.resolve("durable-client")); - try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", - credential, envelope -> received.countDown())) { - connector.start(); - assertTrue(received.await(8, TimeUnit.SECONDS)); - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); - while (countRegularFiles(queueDirectory) != 0L && System.nanoTime() < deadline) Thread.sleep(20); - assertEquals(0L, countRegularFiles(queueDirectory), "backend ACK must durably remove the delivery"); - } - } - } - - @Test - void pollCreatedBackendStateUsesDurableOutgoingQueue() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("poll-proxy"), "localhost"); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("poll-authority")); - Path queueDirectory = directory.resolve("poll-outgoing"); - CountDownLatch proxyReceived = new CountDownLatch(1), backendReceived = new CountDownLatch(1); - CountDownLatch releaseBackendCallback = new CountDownLatch(1); - try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), - identity, authority, queueDirectory, ignored -> proxyReceived.countDown())) { - server.start(); - HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), - Duration.ofMinutes(5)); - HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, - "lobby-1", directory.resolve("poll-client")); - try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", - credential, envelope -> { - backendReceived.countDown(); - try { releaseBackendCallback.await(5, TimeUnit.SECONDS); } - catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } - })) { - connector.start(); - assertTrue(connector.send(JsonEnvelope.builder("establish-poll").build())); - assertTrue(proxyReceived.await(8, TimeUnit.SECONDS)); - assertTrue(server.send("lobby-1", JsonEnvelope.builder("durable-after-poll").build())); - assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); - assertEquals(1L, countRegularFiles(queueDirectory), - "a poll-created backend state must persist before reporting acceptance"); - releaseBackendCallback.countDown(); - } - } finally { - releaseBackendCallback.countDown(); - } - } - - private static long countRegularFiles(Path root) throws Exception { - try (java.util.stream.Stream paths = java.nio.file.Files.walk(root)) { - return paths.filter(path -> java.nio.file.Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS)).count(); - } - } - - @Test - void aggregatePacketBudgetSplitsLargeValidEnvelopes() { - java.util.List candidates = new java.util.ArrayList<>(); - for (int index = 0; index < 12; index++) candidates.add(new HttpTransportProtocol.Delivery( - java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("large").put("value", "x".repeat(40_000)).build())); - String session = java.util.UUID.randomUUID().toString(); - java.util.List fitted = HttpTransportProtocol.fittingMessages( - "lobby-1", session, 0, java.util.List.of(), candidates); - assertTrue(fitted.size() > 0 && fitted.size() < candidates.size()); - assertTrue(HttpTransportProtocol.request("lobby-1", session, 0, java.util.List.of(), fitted).length - <= HttpTransportProtocol.MAX_BODY_BYTES); - } - - @Test - void packetNumbersMustUseCanonicalJsonIntegerTokens() { - com.google.gson.JsonObject packet = com.google.gson.JsonParser.parseString(new String(HttpTransportProtocol.request( - "lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(), java.util.List.of()), - java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); - assertDoesNotThrow(() -> HttpTransportProtocol.parsePacket(packet.toString() - .getBytes(java.nio.charset.StandardCharsets.UTF_8))); - String timestamp = packet.get("timestamp").getAsString(); - java.util.Map> invalid = java.util.Map.of( - "v", java.util.List.of("\"1\"", "1.0", "1e0"), - "sequence", java.util.List.of("\"0\"", "0.0", "0e0"), - "timestamp", java.util.List.of("\"" + timestamp + "\"", timestamp + ".0", timestamp + "e0")); - for (var field : invalid.entrySet()) for (String token : field.getValue()) { - com.google.gson.JsonObject rejected = packet.deepCopy(); - rejected.add(field.getKey(), com.google.gson.JsonParser.parseString(token)); - assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( - rejected.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8)), field.getKey() + "=" + token); - } - } - - @Test - void packetParsingRejectsNoncanonicalUuidForms() { - String deliveryId = java.util.UUID.randomUUID().toString(); - com.google.gson.JsonObject packet = com.google.gson.JsonParser.parseString(new String(HttpTransportProtocol.request( - "lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(deliveryId), - java.util.List.of(new HttpTransportProtocol.Delivery(deliveryId, JsonEnvelope.builder("payload").build()))), - java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); - String abbreviated = "1-1-1-1-1"; - - com.google.gson.JsonObject invalidSession = packet.deepCopy(); - invalidSession.addProperty("session", abbreviated); - assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( - invalidSession.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); - - com.google.gson.JsonObject invalidAck = packet.deepCopy(); - invalidAck.getAsJsonArray("acks").set(0, new com.google.gson.JsonPrimitive(abbreviated)); - assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( - invalidAck.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); - - com.google.gson.JsonObject invalidMessage = packet.deepCopy(); - invalidMessage.getAsJsonArray("messages").get(0).getAsJsonObject().addProperty("id", abbreviated); - assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( - invalidMessage.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); - } - - @Test - void persistedProfileStartsAfterTheEnrollmentCodeExpires() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); - try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { - server.start(); - HttpConnectionCode active = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); - HttpBackendTransportConnector.enroll(active, "lobby-1", directory.resolve("client")); - HttpConnectionCode expired = new HttpConnectionCode(active.serverId(), active.endpoint(), active.serverCertificatePin(), active.caCertificatePin(), - java.time.Instant.now().minusSeconds(1), active.enrollmentToken()); - try (HttpBackendTransportConnector ignored = new HttpBackendTransportConnector(expired, "lobby-1", directory.resolve("client"), message -> { })) { - assertTrue(true); - } - try (HttpBackendTransportConnector ignored = new HttpBackendTransportConnector(directory.resolve("client"), message -> { })) { - assertTrue(true); - } - } - } - - @Test - void persistedBackendConnectsAfterAutomaticServerLeafRotation() throws Exception { - Instant now = Instant.now(); - Path proxyDirectory = directory.resolve("proxy"); - HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost", - java.time.Clock.fixed(now.minus(Duration.ofDays(340)), java.time.ZoneOffset.UTC)); - String originalServerPin = HttpTransportSecrets.certificatePin(original.serverCertificate()); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(original, directory.resolve("authority")); - HttpTlsIdentity rotated = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost"); - CountDownLatch received = new CountDownLatch(1); - try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), rotated, - authority, ignored -> received.countDown())) { - HttpConnectionCode activeCode = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); - HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", activeCode.enrollmentToken()); - HttpConnectionCode oldProfileCode = new HttpConnectionCode(activeCode.serverId(), activeCode.endpoint(), originalServerPin, - activeCode.caCertificatePin(), activeCode.expiresAt(), activeCode.enrollmentToken()); - HttpClientCredentialStore.saveEnrolled(directory.resolve("client"), oldProfileCode, issued); - assertFalse(oldProfileCode.serverCertificatePin().equals(rotated.serverCertificatePin())); - server.start(); - try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(directory.resolve("client"), ignored -> { })) { - connector.start(); - assertTrue(connector.send(JsonEnvelope.builder("after-rotation").build())); - assertTrue(received.await(8, TimeUnit.SECONDS)); - } - } - } - - @Test - void backendRenewsClientCertificateBeforeExpiryWithoutNewConnectionCode() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); - HttpTlsIdentity.IssuedClientCertificate expiring = identity.issueClientCertificate("lobby-1", - Instant.now().minus(Duration.ofDays(340))); - String originalPin = HttpTransportSecrets.certificatePin(expiring.certificate()); - Path authorityDirectory = directory.resolve("authority"); - java.nio.file.Files.createDirectories(authorityDirectory); - String key = java.util.Base64.getUrlEncoder().withoutPadding() - .encodeToString("lobby-1".getBytes(java.nio.charset.StandardCharsets.UTF_8)); - java.nio.file.Files.writeString(authorityDirectory.resolve("http-transport-clients.properties"), - "version=2\nbinding." + key + "=" + originalPin + ":-:0\n"); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); - try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, - authority, ignored -> { })) { - HttpConnectionCode profileCode = new HttpConnectionCode("lobby-1", server.endpoint("localhost"), - identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); - Path clientDirectory = directory.resolve("client"); - HttpClientCredentialStore.saveEnrolled(clientDirectory, profileCode, expiring); - server.start(); - try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { - HttpClientCredentialStore.ActiveCredentialGeneration originalGeneration = - connector.activeCredentialGeneration(); - connector.start(); - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(8); - String renewedPin = originalPin; - while (renewedPin.equals(originalPin) && System.nanoTime() < deadline) { - Thread.sleep(25); - renewedPin = HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(clientDirectory).certificate()); - } - assertFalse(renewedPin.equals(originalPin)); - HttpClientCredentialStore.ClientCredential renewed = HttpClientCredentialStore.load(clientDirectory); - long promotionDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); - while (authority.authenticate("lobby-1", expiring.certificate()) && System.nanoTime() < promotionDeadline) Thread.sleep(25); - assertTrue(authority.authenticate("lobby-1", renewed.certificate())); - assertFalse(authority.authenticate("lobby-1", expiring.certificate())); - HttpClientCredentialStore.ActiveCredentialGeneration renewedGeneration = - connector.activeCredentialGeneration(); - assertFalse(renewedGeneration.name().equals(originalGeneration.name())); - assertEquals(originalGeneration.connectionCodeDigest(), renewedGeneration.connectionCodeDigest()); - } - } - } - - @Test - void duplicateInboundDeliveryIsReAcknowledgedWithoutSecondDispatch() { - HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); - String session = java.util.UUID.randomUUID().toString(); - String id = java.util.UUID.randomUUID().toString(); - HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); - assertTrue(state.acceptSession(session, 0)); - assertEquals(1, state.acceptIncoming(java.util.List.of(delivery)).size()); - state.completeIncoming(id, true); - assertEquals(java.util.List.of(id), state.await("lobby-1", session, 0).acks()); - assertTrue(state.acceptSession(session, 1)); - assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty()); - assertEquals(java.util.List.of(id), state.await("lobby-1", session, 1).acks()); - String replacementSession = java.util.UUID.randomUUID().toString(); - assertTrue(state.acceptSession(replacementSession, 0)); - assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty()); - assertEquals(java.util.List.of(id), state.await("lobby-1", replacementSession, 0).acks()); - } - - @Test - void newerDeliveriesDoNotPostponeRetryOfOlderUnacknowledgedDelivery() { - AtomicLong nanoTime = new AtomicLong(1L); - HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(nanoTime::get); - String session = java.util.UUID.randomUUID().toString(); - HttpTransportProtocol.Delivery first = new HttpTransportProtocol.Delivery( - java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("first").build()); - HttpTransportProtocol.Delivery second = new HttpTransportProtocol.Delivery( - java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("second").build()); - - assertTrue(state.acceptSession(session, 0)); - assertTrue(state.enqueue(first)); - assertEquals(java.util.List.of(first), state.await("lobby-1", session, 0).messages()); - - nanoTime.addAndGet(TimeUnit.SECONDS.toNanos(1)); - assertTrue(state.acceptSession(session, 1)); - assertTrue(state.enqueue(second)); - assertEquals(java.util.List.of(second), state.await("lobby-1", session, 1).messages()); - - nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(1100)); - assertTrue(state.acceptSession(session, 2)); - assertEquals(java.util.List.of(first), state.await("lobby-1", session, 2).messages(), - "sending newer traffic must not reset an older delivery's retry age"); - } - - @Test - void longPollWakesAtTheOldestDeliveryRetryDeadline() throws Exception { - AtomicLong nanoTime = new AtomicLong(1L); - HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(nanoTime::get); - String session = java.util.UUID.randomUUID().toString(); - HttpTransportProtocol.Delivery first = new HttpTransportProtocol.Delivery( - java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("first").build()); - HttpTransportProtocol.Delivery second = new HttpTransportProtocol.Delivery( - java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("second").build()); - assertTrue(state.acceptSession(session, 0)); - assertTrue(state.enqueue(first)); - assertEquals(java.util.List.of(first), state.await("lobby-1", session, 0).messages()); - - nanoTime.addAndGet(HttpProxyTransportServer.LONG_POLL.minusMillis(100).toNanos()); - assertTrue(state.acceptSession(session, 1)); - assertTrue(state.enqueue(second)); - assertEquals(java.util.List.of(second), state.await("lobby-1", session, 1).messages()); - assertTrue(state.acceptSession(session, 2)); - Thread clock = new Thread(() -> { - try { Thread.sleep(50L); } - catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } - nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(100)); - }, "HTTP-retry-test-clock"); - clock.setDaemon(true); - long started = System.nanoTime(); - clock.start(); - assertEquals(java.util.List.of(first), state.await("lobby-1", session, 2).messages()); - clock.join(); - assertTrue(System.nanoTime() - started < TimeUnit.SECONDS.toNanos(1), - "the poll must wake at the oldest delivery deadline, not a fresh long-poll deadline"); - } - - @Test - void proxyDedupWindowEvictsOldestCompletedDeliveryAtCapacity() { - HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); - String oldest = null, newest = null; - for (int index = 0; index < HttpTransportProtocol.MAX_QUEUE + 1; index++) { - String id = java.util.UUID.randomUUID().toString(); - if (index == 0) oldest = id; - newest = id; - HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); - assertEquals(1, state.acceptIncoming(java.util.List.of(delivery)).size()); - state.completeIncoming(id, true); - } - HttpTransportProtocol.Delivery evicted = new HttpTransportProtocol.Delivery(oldest, JsonEnvelope.builder("x").build()); - HttpTransportProtocol.Delivery retained = new HttpTransportProtocol.Delivery(newest, JsonEnvelope.builder("x").build()); - assertEquals(1, state.acceptIncoming(java.util.List.of(evicted)).size()); - assertTrue(state.acceptIncoming(java.util.List.of(retained)).isEmpty()); - } - - @Test - void backendReAcknowledgesLostAckDuplicateWithoutSecondCallback() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); - HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); - HttpClientCredentialStore.save(directory.resolve("client"), issued); - HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", - java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); - CountDownLatch callback = new CountDownLatch(1); - try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, - HttpClientCredentialStore.load(directory.resolve("client")), envelope -> callback.countDown())) { - String id = java.util.UUID.randomUUID().toString(); - HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); - connector.dispatch(delivery); - assertTrue(callback.await(2, TimeUnit.SECONDS)); - java.util.List acknowledgements = java.util.List.of(); - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); - while (acknowledgements.isEmpty() && System.nanoTime() < deadline) { - acknowledgements = connector.drainAcknowledgements(); - if (acknowledgements.isEmpty()) Thread.sleep(5); - } - assertEquals(java.util.List.of(id), acknowledgements); - assertTrue(connector.accept(java.util.List.of(delivery)).isEmpty()); - assertEquals(java.util.List.of(id), connector.drainAcknowledgements()); - } - } - - @Test - void backendCallbacksAreSerializedInDeliveryOrder() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ordered-proxy"), "localhost"); - HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); - HttpClientCredentialStore.save(directory.resolve("ordered-client"), issued); - HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", - java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); - CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1), secondStarted = new CountDownLatch(1); - java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); - try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, - HttpClientCredentialStore.load(directory.resolve("ordered-client")), envelope -> { - String marker = String.valueOf(envelope.getFields().get("marker")); - order.add(marker); - if ("first".equals(marker)) { - firstStarted.countDown(); - try { releaseFirst.await(5, TimeUnit.SECONDS); } - catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } - } else secondStarted.countDown(); - })) { - connector.dispatch(new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), - JsonEnvelope.builder("x").put("marker", "first").build())); - connector.dispatch(new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), - JsonEnvelope.builder("x").put("marker", "second").build())); - assertTrue(firstStarted.await(2, TimeUnit.SECONDS)); - assertFalse(secondStarted.await(150, TimeUnit.MILLISECONDS)); - releaseFirst.countDown(); - assertTrue(secondStarted.await(2, TimeUnit.SECONDS)); - assertEquals(java.util.List.of("first", "second"), order); - } finally { releaseFirst.countDown(); } - } - - @Test - void backendCallbackQueueBackpressuresWithoutBreakingFifo() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("backpressure-proxy"), "localhost"); - HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); - HttpClientCredentialStore.save(directory.resolve("backpressure-client"), issued); - HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", - java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); - CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1); - int deliveries = HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY + 2; - CountDownLatch completed = new CountDownLatch(deliveries), overflowSubmitted = new CountDownLatch(1); - java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); - try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, - HttpClientCredentialStore.load(directory.resolve("backpressure-client")), envelope -> { - int marker = Integer.parseInt(envelope.getFields().get("marker")); - order.add(marker); - if (marker == 0) { - firstStarted.countDown(); - try { releaseFirst.await(5, TimeUnit.SECONDS); } - catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } - } - completed.countDown(); - })) { - connector.dispatch(delivery(0)); - assertTrue(firstStarted.await(2, TimeUnit.SECONDS)); - for (int marker = 1; marker <= HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY; marker++) - connector.dispatch(delivery(marker)); - Thread overflow = new Thread(() -> { - connector.dispatch(delivery(deliveries - 1)); - overflowSubmitted.countDown(); - }, "HTTP-overflow-submitter"); - overflow.start(); - assertFalse(overflowSubmitted.await(150, TimeUnit.MILLISECONDS), "a full ordered lane must backpressure its producer"); - releaseFirst.countDown(); - assertTrue(overflowSubmitted.await(2, TimeUnit.SECONDS)); - assertTrue(completed.await(5, TimeUnit.SECONDS)); - assertEquals(java.util.stream.IntStream.range(0, deliveries).boxed().toList(), order); - } finally { releaseFirst.countDown(); } - } - - @Test - void durableBackendFencePreventsCallbackReplayAfterRestartBeforeAck() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("fence-proxy"), "localhost"); - HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); - HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), - identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "A".repeat(43)); - Path clientDirectory = directory.resolve("fence-client"); - HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); - java.util.concurrent.atomic.AtomicInteger callbacks = new java.util.concurrent.atomic.AtomicInteger(); - String id = java.util.UUID.randomUUID().toString(); - HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("vote").build()); - try (HttpBackendTransportConnector first = new HttpBackendTransportConnector(clientDirectory, - ignored -> callbacks.incrementAndGet())) { - first.dispatch(delivery); - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); - while (callbacks.get() != 1 && System.nanoTime() < deadline) Thread.sleep(5); - assertEquals(1, callbacks.get()); - } - try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, - ignored -> callbacks.incrementAndGet())) { - assertEquals(java.util.List.of(id), restarted.drainAcknowledgements(), - "restart must retain and acknowledge the pre-callback delivery fence"); - assertTrue(restarted.accept(java.util.List.of(delivery)).isEmpty()); - assertEquals(1, callbacks.get(), "a durable proxy replay must not award twice"); - } - } - - @Test - void failedBackendCallbackRemainsUnacknowledgedAndIsNotReplayed() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-fence-proxy"), "localhost"); - HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); - HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), - identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "B".repeat(43)); - Path clientDirectory = directory.resolve("retry-fence-client"); - HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); - java.util.concurrent.atomic.AtomicInteger attempts = new java.util.concurrent.atomic.AtomicInteger(); - CountDownLatch failed = new CountDownLatch(1); - HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), - JsonEnvelope.builder("vote").build()); - try (HttpBackendTransportConnector first = new HttpBackendTransportConnector(clientDirectory, ignored -> { - attempts.incrementAndGet(); failed.countDown(); throw new IllegalStateException("retry"); - })) { - first.dispatch(delivery); - assertTrue(failed.await(2, TimeUnit.SECONDS)); - assertTrue(first.drainAcknowledgements().isEmpty()); - } - try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, - ignored -> attempts.incrementAndGet())) { - assertTrue(restarted.drainAcknowledgements().isEmpty()); - java.util.List accepted = restarted.accept(java.util.List.of(delivery)); - assertTrue(accepted.isEmpty(), "an ambiguous callback must not be awarded twice"); - assertEquals(1, attempts.get()); - } - } - - @Test - void reservedButNotStartedDeliveryResumesAfterRestart() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("reserved-proxy"), "localhost"); - HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); - HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), - identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "C".repeat(43)); - Path clientDirectory = directory.resolve("reserved-client"); - HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); - String id = java.util.UUID.randomUUID().toString(); - new HttpInboundDeliveryStore(clientDirectory).reserve(id); - CountDownLatch completed = new CountDownLatch(1); - HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("vote").build()); - try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, ignored -> completed.countDown())) { - assertTrue(restarted.drainAcknowledgements().isEmpty(), "a reservation alone must never be acknowledged"); - java.util.List accepted = restarted.accept(java.util.List.of(delivery)); - assertEquals(1, accepted.size()); - restarted.dispatch(accepted.get(0)); - assertTrue(completed.await(2, TimeUnit.SECONDS)); - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); - java.util.List acknowledgements = java.util.List.of(); - while (acknowledgements.isEmpty() && System.nanoTime() < deadline) { - acknowledgements = restarted.drainAcknowledgements(); - if (acknowledgements.isEmpty()) Thread.sleep(5); - } - assertEquals(java.util.List.of(id), acknowledgements); - } - } - - @Test - void interruptedStateRenameRetainsTheFurthestSafeState() throws Exception { - Path clientDirectory = directory.resolve("interrupted-state-client"); - String id = java.util.UUID.randomUUID().toString(); - String completedId = java.util.UUID.randomUUID().toString(); - Path states = clientDirectory.resolve("http-transport-inbound-deliveries"); - Files.createDirectories(states); - Files.writeString(states.resolve(id + ".reserved"), id); - Files.writeString(states.resolve(id + ".running"), id); - Files.writeString(states.resolve(completedId + ".running"), completedId); - Files.writeString(states.resolve(completedId + ".completed"), completedId); - HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(clientDirectory); - assertEquals(HttpInboundDeliveryStore.State.RUNNING, store.state(id)); - assertEquals(HttpInboundDeliveryStore.State.COMPLETED, store.state(completedId)); - assertFalse(Files.exists(states.resolve(id + ".reserved"))); - assertTrue(Files.exists(states.resolve(id + ".running"))); - assertFalse(Files.exists(states.resolve(completedId + ".running"))); - assertTrue(Files.exists(states.resolve(completedId + ".completed"))); - } - - private static HttpTransportProtocol.Delivery delivery(int marker) { - return new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), - JsonEnvelope.builder("x").put("marker", marker).build()); - } - - @Test - void proxyCallbacksAreSerializedInDeliveryOrder() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ordered-proxy-server"), "localhost"); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("ordered-authority")); - CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1), secondStarted = new CountDownLatch(1); - java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); - try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, - authority, received -> { - String marker = String.valueOf(received.envelope().getFields().get("marker")); - order.add(marker); - if ("first".equals(marker)) { - firstStarted.countDown(); - try { releaseFirst.await(5, TimeUnit.SECONDS); } - catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } - } else secondStarted.countDown(); - })) { - server.start(); - HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); - HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, "lobby-1", - directory.resolve("ordered-proxy-client")); - try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", credential, - ignored -> { })) { - connector.start(); - assertTrue(connector.send(JsonEnvelope.builder("x").put("marker", "first").build())); - assertTrue(connector.send(JsonEnvelope.builder("x").put("marker", "second").build())); - assertTrue(firstStarted.await(3, TimeUnit.SECONDS)); - assertFalse(secondStarted.await(150, TimeUnit.MILLISECONDS)); - releaseFirst.countDown(); - assertTrue(secondStarted.await(3, TimeUnit.SECONDS)); - assertEquals(java.util.List.of("first", "second"), order); - } - } finally { releaseFirst.countDown(); } - } - - @Test - void backendDedupWindowContinuesAfterCapacity() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); - HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); - HttpClientCredentialStore.save(directory.resolve("client"), issued); - HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", - java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); - try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, - HttpClientCredentialStore.load(directory.resolve("client")), envelope -> { })) { - for (int index = 0; index < HttpTransportProtocol.MAX_QUEUE; index++) { - String id = java.util.UUID.randomUUID().toString(); - HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); - assertEquals(1, connector.accept(java.util.List.of(delivery)).size()); - connector.completeIncoming(id, true); - } - HttpTransportProtocol.Delivery next = new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), - JsonEnvelope.builder("x").build()); - assertEquals(1, connector.accept(java.util.List.of(next)).size()); - } - } -} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java deleted file mode 100644 index f67173889..000000000 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ /dev/null @@ -1,401 +0,0 @@ -package com.bencodez.votingplugin.backendproxy.http; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.cert.X509Certificate; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.time.ZoneOffset; -import java.util.Arrays; -import javax.net.ssl.X509TrustManager; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -class HttpTransportSecurityTest { - @Test - void connectionCodeRejectsExplicitZeroPort() { - assertThrows(IllegalArgumentException.class, - () -> new HttpConnectionCode("lobby", URI.create("https://proxy.example.test:0/"), pin('a'), - pin('b'), Instant.now().plusSeconds(60), "token")); - } - - @TempDir Path directory; - - @Test - void backendResponseReaderRejectsBodiesBeyondTheWireLimit() throws Exception { - byte[] maximum = new byte[HttpTransportProtocol.MAX_BODY_BYTES]; - assertEquals(maximum.length, HttpBackendTransportConnector.readLimited( - new java.io.ByteArrayInputStream(maximum)).length); - assertThrows(java.io.IOException.class, () -> HttpBackendTransportConnector.readLimited( - new java.io.ByteArrayInputStream(new byte[HttpTransportProtocol.MAX_BODY_BYTES + 1]))); - } - - @Test - void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { - HttpConnectionCode original = new HttpConnectionCode("lobby.eu", URI.create("https://Proxy.Example.test:8443/http"), pin('a'), pin('b'), - Instant.parse("2030-01-01T00:00:00Z"), HttpTransportSecrets.randomToken()); - String encoded = original.encode(); - HttpConnectionCode parsed = HttpConnectionCode.parse(encoded); - assertEquals("lobby.eu", parsed.serverId()); - assertEquals(URI.create("https://proxy.example.test:8443/http/"), parsed.endpoint()); - assertEquals(original.serverCertificatePin(), parsed.serverCertificatePin()); - char last = encoded.charAt(encoded.length() - 1); - assertThrows(IllegalArgumentException.class, () -> HttpConnectionCode.parse(encoded.substring(0, encoded.length() - 1) - + (last == 'A' ? 'B' : 'A'))); - assertThrows(IllegalArgumentException.class, () -> HttpConnectionCode.parse("http://not-a-code")); - } - - @Test - void legacyConnectionCodesAndConsumedMarkersRemainCompatible() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("legacy-code-proxy"), "proxy.example.test"); - HttpConnectionCode legacy = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test:8443/"), - identity.serverCertificatePin(), identity.caCertificatePin(), - Instant.now().plusSeconds(60).truncatedTo(java.time.temporal.ChronoUnit.SECONDS), - HttpTransportSecrets.randomToken()); - assertEquals(legacy, HttpConnectionCode.parse(legacy.encodeLegacy())); - - Path client = directory.resolve("legacy-code-client"); - HttpClientCredentialStore.saveEnrolled(client, legacy, identity.issueClientCertificate("lobby")); - Path active = client.resolve("http-transport-client-generations") - .resolve(Files.readString(client.resolve("http-transport-client-current"))); - Files.writeString(active.resolve("http-transport-connection-code.sha256"), - HttpTransportSecrets.sha256Hex(legacy.encodeLegacy().getBytes(java.nio.charset.StandardCharsets.US_ASCII))); - assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, - HttpConnectionCode.parse(legacy.encodeLegacy()))); - } - - @Test - void expiredCodesAreNotActive() { - HttpConnectionCode code = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test/"), pin('a'), pin('b'), - Instant.parse("2029-12-31T23:59:59Z"), HttpTransportSecrets.randomToken()); - assertFalse(code.isActive(Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC))); - assertThrows(IllegalArgumentException.class, () -> code.requireActive(Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC))); - } - - @Test - void inboundDeliveryFenceRejectsCorruptionAndPathReplacement() throws Exception { - Path corruptCredentials = directory.resolve("corrupt-client"); - Path corruptFence = corruptCredentials.resolve("http-transport-inbound-deliveries"); - Files.createDirectories(corruptFence); - Files.writeString(corruptFence.resolve("not-a-delivery.seen"), "not-a-delivery"); - assertThrows(java.io.IOException.class, () -> new HttpInboundDeliveryStore(corruptCredentials)); - - Path replacedCredentials = directory.resolve("replaced-client"); - Files.createDirectories(replacedCredentials); - HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(replacedCredentials); - Path fence = replacedCredentials.resolve("http-transport-inbound-deliveries"); - Path outside = directory.resolve("outside-fence"); - Files.createDirectory(outside); - Files.delete(fence); - Files.createSymbolicLink(fence, outside); - assertThrows(java.io.IOException.class, () -> store.reserve(java.util.UUID.randomUUID().toString())); - } - - @Test - void sealedInboundStoreCannotChangeAfterOwnershipHandoff() throws Exception { - Path credentials = directory.resolve("sealed-client"); - Files.createDirectories(credentials); - String id = java.util.UUID.randomUUID().toString(); - HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(credentials); - store.reserve(id); - store.markRunning(id); - store.seal(); - assertThrows(java.io.IOException.class, () -> store.markCompleted(id)); - assertEquals(HttpInboundDeliveryStore.State.RUNNING, new HttpInboundDeliveryStore(credentials).state(id)); - } - - @Test - void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { - HttpTlsIdentity created = HttpTlsIdentity.loadOrCreate(directory, "localhost"); - HttpTlsIdentity loaded = HttpTlsIdentity.loadOrCreate(directory, "localhost"); - assertEquals(created.serverCertificatePin(), loaded.serverCertificatePin()); - assertEquals(created.caCertificatePin(), loaded.caCertificatePin()); - HttpConnectionCode correct = new HttpConnectionCode("lobby", URI.create("https://localhost:8443/"), created.serverCertificatePin(), - created.caCertificatePin(), Instant.now().plusSeconds(60), HttpTransportSecrets.randomToken()); - HttpConnectionCode incorrect = new HttpConnectionCode("lobby", URI.create("https://localhost:8443/"), pin('0'), created.caCertificatePin(), - Instant.now().plusSeconds(60), HttpTransportSecrets.randomToken()); - assertTrue(HttpPinnedTls.matchesServerPin(correct, created.serverCertificate())); - assertFalse(HttpPinnedTls.matchesServerPin(incorrect, created.serverCertificate())); - assertTrue(Files.exists(directory.resolve("http-transport-ca.p12"))); - HttpTlsIdentity rotated = HttpTlsIdentity.loadOrCreate(directory, "127.0.0.1"); - assertEquals(created.caCertificatePin(), rotated.caCertificatePin()); - assertNotEquals(created.serverCertificatePin(), rotated.serverCertificatePin()); - } - - @Test - void incompleteFirstRunTlsProvisioningRecoversWithoutManualCleanup() throws Exception { - Path source = directory.resolve("complete-identity"); - HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(source, "localhost"); - Path interrupted = directory.resolve("interrupted-identity"); - Files.createDirectories(interrupted); - Files.copy(source.resolve("http-transport-ca.p12"), interrupted.resolve("http-transport-ca.p12")); - Files.copy(source.resolve("http-transport-server.p12"), interrupted.resolve("http-transport-server.p12")); - - HttpTlsIdentity recovered = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); - - assertNotEquals(original.caCertificatePin(), recovered.caCertificatePin()); - assertTrue(Files.exists(interrupted.resolve("http-transport-ca.p12"))); - assertTrue(Files.exists(interrupted.resolve("http-transport-server.p12"))); - assertTrue(Files.exists(interrupted.resolve("http-transport-password"))); - assertFalse(Files.exists(interrupted.resolve("http-transport-initializing"))); - } - - @Test - void completedFirstRunFilesRecoverWhenInitializationMarkerSurvives() throws Exception { - Path interrupted = directory.resolve("marked-identity"); - HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); - String originalCaPin = original.caCertificatePin(); - Files.writeString(interrupted.resolve("http-transport-initializing"), "initializing\n"); - - HttpTlsIdentity recovered = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); - - assertNotEquals(originalCaPin, recovered.caCertificatePin()); - assertFalse(Files.exists(interrupted.resolve("http-transport-initializing"))); - } - - @Test - void incompleteEstablishedTlsIdentityFailsClosed() throws Exception { - Path established = directory.resolve("established-identity"); - HttpTlsIdentity.loadOrCreate(established, "localhost"); - Files.writeString(established.resolve("http-transport-clients.properties"), "version=2\n"); - Files.delete(established.resolve("http-transport-server.p12")); - - assertThrows(java.io.IOException.class, () -> HttpTlsIdentity.loadOrCreate(established, "localhost")); - assertTrue(Files.exists(established.resolve("http-transport-ca.p12"))); - assertTrue(Files.exists(established.resolve("http-transport-password"))); - } - - @Test - void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory, "localhost"); - Clock clock = Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock); - HttpConnectionCode wrongTargetCode = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); - assertThrows(IllegalArgumentException.class, () -> authority.enroll("attacker", wrongTargetCode.enrollmentToken())); - assertTrue(authority.authenticate("lobby-1", authority.enroll("lobby-1", wrongTargetCode.enrollmentToken()).certificate()), - "a wrong backend must not consume another backend's connection code"); - authority.revoke("lobby-1"); - HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); - HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", code.enrollmentToken()); - assertTrue(authority.authenticate("lobby-1", issued.certificate())); - assertTrue(identity.validClientCertificate("LOBBY-1", issued.certificate())); - assertFalse(authority.authenticate("lobby-2", issued.certificate())); - assertThrows(IllegalArgumentException.class, () -> authority.enroll("lobby-2", code.enrollmentToken())); - authority.revoke("lobby-1"); - assertFalse(authority.authenticate("lobby-1", issued.certificate())); - HttpConnectionCode replacementCode = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); - HttpTlsIdentity.IssuedClientCertificate replacement = authority.enroll("lobby-1", replacementCode.enrollmentToken()); - assertTrue(authority.authenticate("lobby-1", replacement.certificate())); - assertFalse(authority.authenticate("lobby-1", issued.certificate())); - HttpClientCredentialStore.saveEnrolled(directory.resolve("client"), code, issued); - HttpClientCredentialStore.ClientCredential restored = HttpClientCredentialStore.load(directory.resolve("client")); - assertEquals(HttpTransportSecrets.certificatePin(issued.certificate()), HttpTransportSecrets.certificatePin(restored.certificate())); - HttpClientCredentialStore.HttpClientProfile profile = HttpClientCredentialStore.loadProfile(directory.resolve("client")); - assertEquals("lobby-1", profile.serverId()); - assertEquals(code.endpoint(), profile.endpoint()); - assertEquals("lobby-1", HttpClientCredentialStore.loadEnrolled(directory.resolve("client")).profile().serverId()); - assertNotEquals(null, HttpPinnedTls.mutualTlsContext(code, restored)); - Path clientDirectory = directory.resolve("client"); - String generation = Files.readString(clientDirectory.resolve("http-transport-client-current")); - Files.writeString(clientDirectory.resolve("http-transport-client-generations").resolve(generation) - .resolve("http-transport-profile.properties"), "version=1\nserverId=lobby-1\n"); - assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.loadProfile(directory.resolve("client"))); - HttpEnrollmentAuthority durable = new HttpEnrollmentAuthority(identity, directory.resolve("state")); - HttpConnectionCode durableCode = durable.createConnectionCode("survival", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); - HttpTlsIdentity.IssuedClientCertificate durableIssued = durable.enroll("survival", durableCode.enrollmentToken()); - assertTrue(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); - durable.revoke("survival"); - assertFalse(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); - } - - @Test - void revocationInvalidatesEveryPendingCodeForTheBackend() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("revoke-pending"), "localhost"); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, - Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC)); - URI endpoint = URI.create("https://localhost:8443/"); - HttpConnectionCode beforeEnrollment = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); - authority.revoke("LOBBY-1"); - assertThrows(IllegalArgumentException.class, - () -> authority.enroll("lobby-1", beforeEnrollment.enrollmentToken())); - - HttpConnectionCode active = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); - authority.enroll("lobby-1", active.enrollmentToken()); - HttpConnectionCode firstPending = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); - HttpConnectionCode secondPending = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); - authority.revoke("lobby-1"); - assertThrows(IllegalArgumentException.class, - () -> authority.enroll("lobby-1", firstPending.enrollmentToken())); - assertThrows(IllegalArgumentException.class, - () -> authority.enroll("lobby-1", secondPending.enrollmentToken())); - } - - @Test - void renewalKeepsOldCredentialUntilReplacementAuthenticates() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("state")); - HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); - HttpTlsIdentity.IssuedClientCertificate original = authority.enroll("lobby-1", code.enrollmentToken()); - HttpTlsIdentity.IssuedClientCertificate replacement = authority.renew("lobby-1", original.certificate()); - - assertTrue(authority.authenticate("lobby-1", original.certificate()), "lost renewal responses must leave the old credential usable"); - assertTrue(authority.authenticate("lobby-1", replacement.certificate()), "first replacement request promotes the pending binding"); - assertFalse(authority.authenticate("lobby-1", original.certificate()), "promotion revokes the superseded credential"); - assertTrue(new HttpEnrollmentAuthority(identity, directory.resolve("state")) - .authenticate("lobby-1", replacement.certificate()), "promoted renewal must survive restart"); - } - - @Test - void serverLeafRotatesInsideRenewalWindowAndPreservesAuthority() throws Exception { - Instant now = Instant.now(); - HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(directory, "localhost", - Clock.fixed(now.minus(Duration.ofDays(340)), ZoneOffset.UTC)); - String originalPin = HttpTransportSecrets.certificatePin(original.serverCertificate()); - HttpTlsIdentity renewed = HttpTlsIdentity.loadOrCreate(directory, "localhost", Clock.fixed(now, ZoneOffset.UTC)); - assertNotEquals(originalPin, renewed.serverCertificatePin()); - assertEquals(original.caCertificatePin(), renewed.caCertificatePin()); - assertFalse(HttpTlsIdentity.needsRenewal(renewed.serverCertificate(), Clock.fixed(now, ZoneOffset.UTC))); - } - - @Test - void runningPrivateCaRollsOverBeforeExpiryWithoutStrandingExistingClients() throws Exception { - Instant now = Instant.now(); - Clock originalClock = Clock.fixed(now.minus(Duration.ofDays(9 * 365L + 30L)), ZoneOffset.UTC); - HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(directory, "localhost", originalClock); - X509Certificate originalCa = original.caCertificate(); - HttpTlsIdentity.IssuedClientCertificate existingClient = original.issueClientCertificate("lobby-1", now); - Path client = directory.resolve("client"); - HttpConnectionCode oldCode = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), - HttpTransportSecrets.certificatePin(original.serverCertificate()), HttpTransportSecrets.certificatePin(originalCa), - now.plusSeconds(60), "A".repeat(43)); - HttpClientCredentialStore.saveEnrolled(client, oldCode, existingClient); - - String renewedPin = original.caCertificatePin(); - assertNotEquals(HttpTransportSecrets.certificatePin(originalCa), renewedPin); - assertEquals(originalCa.getPublicKey(), original.caCertificate().getPublicKey(), - "certificate rollover keeps the private authority key so old and new trust anchors overlap"); - assertFalse(HttpTlsIdentity.needsCaRenewal(original.caCertificate(), Clock.fixed(now, ZoneOffset.UTC))); - assertTrue(original.validClientCertificate("lobby-1", existingClient.certificate())); - - X509TrustManager oldClientTrust = Arrays.stream(HttpTlsIdentity.trustManagers(originalCa)) - .filter(X509TrustManager.class::isInstance).map(X509TrustManager.class::cast).findFirst().orElseThrow(); - assertDoesNotThrow(() -> oldClientTrust.checkServerTrusted( - new X509Certificate[] { original.serverCertificate(), original.caCertificate() }, "ECDHE_ECDSA")); - HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(client, - original.issueClientCertificate("lobby-1", now)); - assertEquals(HttpTransportSecrets.certificatePin(originalCa), HttpClientCredentialStore.loadProfile(client).caCertificatePin()); - assertEquals(renewedPin, staged.profile().caCertificatePin()); - HttpClientCredentialStore.activateReplacement(client, staged); - assertEquals(renewedPin, HttpClientCredentialStore.loadProfile(client).caCertificatePin()); - assertEquals(renewedPin, HttpTlsIdentity.loadOrCreate(directory, "localhost").caCertificatePin(), - "live CA rollover must survive restart"); - } - - @Test - void activeTlsContextRotatesServerLeafInsideRenewalWindow() throws Exception { - Instant now = Instant.now(); - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory, "localhost", - Clock.fixed(now.minus(Duration.ofDays(340)), ZoneOffset.UTC)); - String expiringPin = HttpTransportSecrets.certificatePin(identity.serverCertificate()); - identity.serverContext(); - assertNotEquals(expiringPin, identity.serverCertificatePin()); - assertFalse(HttpTlsIdentity.needsRenewal(identity.serverCertificate(), Clock.systemUTC())); - } - - @Test - void serverTlsUsesPrivateCaTrustAndRejectsForeignClients() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); - HttpTlsIdentity foreign = HttpTlsIdentity.loadOrCreate(directory.resolve("foreign"), "localhost"); - X509TrustManager trust = Arrays.stream(HttpTlsIdentity.trustManagers(identity.caCertificate())) - .filter(X509TrustManager.class::isInstance).map(X509TrustManager.class::cast).findFirst().orElseThrow(); - HttpTlsIdentity.IssuedClientCertificate accepted = identity.issueClientCertificate("lobby-1"); - HttpTlsIdentity.IssuedClientCertificate rejected = foreign.issueClientCertificate("lobby-1"); - assertDoesNotThrow(() -> trust.checkClientTrusted( - new java.security.cert.X509Certificate[] { accepted.certificate(), identity.caCertificate() }, "EC")); - assertThrows(java.security.cert.CertificateException.class, () -> trust.checkClientTrusted( - new java.security.cert.X509Certificate[] { rejected.certificate(), foreign.caCertificate() }, "EC")); - } - - @Test - void stagedCredentialDoesNotReplaceActiveGenerationUntilAtomicActivation() throws Exception { - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); - Path client = directory.resolve("client"); - HttpTlsIdentity.IssuedClientCertificate original = identity.issueClientCertificate("lobby-1"); - HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), - identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); - HttpClientCredentialStore.saveEnrolled(client, code, original); - String originalPin = HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate()); - HttpTlsIdentity.IssuedClientCertificate replacement = identity.issueClientCertificate("lobby-1"); - HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(client, replacement); - assertEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); - HttpClientCredentialStore.activateReplacement(client, staged); - assertNotEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); - assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, code), - "automatic certificate renewal must retain the consumed-code marker"); - HttpTlsIdentity.IssuedClientCertificate manuallyReenrolled = identity.issueClientCertificate("lobby-1"); - HttpClientCredentialStore.saveEnrolled(client, code, manuallyReenrolled); - assertEquals(HttpTransportSecrets.certificatePin(manuallyReenrolled.certificate()), - HttpTransportSecrets.certificatePin(HttpClientCredentialStore.loadEnrolled(client).credential().certificate())); - } - - @Test - void restoresCredentialGenerationAfterFailedReenrollment() throws Exception { - Path client = directory.resolve("client-rollback"); - HttpTlsIdentity oldIdentity = HttpTlsIdentity.loadOrCreate(directory.resolve("old-proxy"), "old.example.test"); - HttpConnectionCode oldCode = new HttpConnectionCode("lobby-1", URI.create("https://old.example.test:1297/"), - oldIdentity.serverCertificatePin(), oldIdentity.caCertificatePin(), Instant.now().plusSeconds(60), - "R".repeat(43)); - HttpClientCredentialStore.saveEnrolled(client, oldCode, oldIdentity.issueClientCertificate("lobby-1")); - HttpClientCredentialStore.ActiveCredentialGeneration previous = - HttpClientCredentialStore.snapshotActiveGeneration(client); - - HttpTlsIdentity replacementIdentity = HttpTlsIdentity.loadOrCreate(directory.resolve("new-proxy"), "new.example.test"); - HttpConnectionCode replacementCode = new HttpConnectionCode("lobby-1", URI.create("https://new.example.test:1297/"), - replacementIdentity.serverCertificatePin(), replacementIdentity.caCertificatePin(), - Instant.now().plusSeconds(60), "S".repeat(43)); - HttpClientCredentialStore.saveEnrolled(client, replacementCode, - replacementIdentity.issueClientCertificate("lobby-1")); - assertEquals(replacementCode.endpoint(), HttpClientCredentialStore.loadProfile(client).endpoint()); - - HttpClientCredentialStore.restoreActiveGeneration(client, previous); - assertEquals(oldCode.endpoint(), HttpClientCredentialStore.loadProfile(client).endpoint()); - assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, oldCode)); - } - - @Test - void rollbackRetainsNewerCredentialForTheSameEndpoint() throws Exception { - Path client = directory.resolve("client-renewal-rollback"); - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("renewal-proxy"), "renew.example.test"); - HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://renew.example.test:1297/"), - identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), - "T".repeat(43)); - HttpTlsIdentity.IssuedClientCertificate original = identity.issueClientCertificate("lobby-1"); - HttpClientCredentialStore.saveEnrolled(client, code, original); - HttpClientCredentialStore.ActiveCredentialGeneration previous = - HttpClientCredentialStore.snapshotActiveGeneration(client); - - HttpConnectionCode replacementCode = new HttpConnectionCode("lobby-1", code.endpoint(), - identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), - "U".repeat(43)); - HttpTlsIdentity.IssuedClientCertificate renewed = identity.issueClientCertificate("lobby-1"); - HttpClientCredentialStore.saveEnrolled(client, replacementCode, renewed); - HttpClientCredentialStore.restoreActiveGenerationAfterReplacement(client, previous); - - assertEquals(HttpTransportSecrets.certificatePin(renewed.certificate()), - HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate()), - "rollback must not reactivate a same-endpoint certificate that renewal may have revoked"); - assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, code), - "the retained credential must recognize the connection code restored in YAML"); - } - - private static String pin(char character) { return String.valueOf(character).repeat(64); } -} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 5cd5d392c..f74f9ef5e 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -4,17 +4,15 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.bencodez.simpleapi.servercomm.http.HttpClientCredentialStore; +import com.bencodez.simpleapi.servercomm.http.HttpConnectionCode; +import com.bencodez.simpleapi.servercomm.http.HttpTlsIdentity; import com.bencodez.votingplugin.VotingPluginMain; -import com.bencodez.votingplugin.backendproxy.http.HttpBackendTransportConnector; -import com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore; -import com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode; -import com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity; import com.bencodez.votingplugin.config.BungeeSettings; import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; import java.net.URI; @@ -93,21 +91,27 @@ void closeNeverWaitsForSetupOnTheCallingThread() throws Exception { } @Test - void prepareForReplacementUsesConnectorSnapshotWithoutCredentialIo() throws Exception { + void prepareForReplacementSnapshotsThePersistedCredentialGeneration() throws Exception { HttpBackendProxyTransport transport = new HttpBackendProxyTransport(mock(VotingPluginMain.class)); - HttpBackendTransportConnector connector = mock(HttpBackendTransportConnector.class); + Path credentials = directory.resolve("http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("replacement-proxy"), "proxy.example.test"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://proxy.example.test:1297/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "D".repeat(43)); + HttpClientCredentialStore.saveEnrolled(credentials, code, identity.issueClientCertificate("lobby-1")); HttpClientCredentialStore.ActiveCredentialGeneration generation = - new HttpClientCredentialStore.ActiveCredentialGeneration(java.util.UUID.randomUUID().toString()); - when(connector.activeCredentialGeneration()).thenReturn(generation); + HttpClientCredentialStore.snapshotActiveGeneration(credentials); + java.lang.reflect.Field directoryField = HttpBackendProxyTransport.class.getDeclaredField("configuredDirectory"); + directoryField.setAccessible(true); + directoryField.set(transport, credentials); java.lang.reflect.Field connectorField = HttpBackendProxyTransport.class.getDeclaredField("connector"); connectorField.setAccessible(true); - connectorField.set(transport, connector); + connectorField.set(transport, mock(com.bencodez.simpleapi.servercomm.http.HttpBackendTransportConnector.class)); assertDoesNotThrow(transport::prepareForReplacement); java.lang.reflect.Field generationField = HttpBackendProxyTransport.class.getDeclaredField("configuredCredentialGeneration"); generationField.setAccessible(true); - assertSame(generation, generationField.get(transport)); + assertEquals(generation, generationField.get(transport)); } @Test diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index a2d111189..f19004774 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -697,13 +697,13 @@ class BackendConfigurationServiceTest { @Test void httpMethodAcceptsGenerationBasedEnrolledProfile() throws Exception { Files.writeString(directory.resolve("BungeeSettings.yml"), "UseBungeecord: true\nServer: lobby-1\nBungeeMethod: PLUGINMESSAGING\nPluginMessageChannel: vp:vp\n"); - com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity identity = - com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); - com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode code = - new com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode("lobby-1", + com.bencodez.simpleapi.servercomm.http.HttpTlsIdentity identity = + com.bencodez.simpleapi.servercomm.http.HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + com.bencodez.simpleapi.servercomm.http.HttpConnectionCode code = + new com.bencodez.simpleapi.servercomm.http.HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:1297/"), identity.serverCertificatePin(), identity.caCertificatePin(), java.time.Instant.now().plusSeconds(60), "A".repeat(43)); - com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore.saveEnrolled(directory.resolve("http"), code, + com.bencodez.simpleapi.servercomm.http.HttpClientCredentialStore.saveEnrolled(directory.resolve("http"), code, identity.issueClientCertificate("lobby-1")); BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); @@ -718,18 +718,18 @@ class BackendConfigurationServiceTest { assertThrows(IllegalArgumentException.class, () -> service.previewQuickSetup("proxy-method", Map.of("method", "HTTP"))); - com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity identity = - com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.loadOrCreate(directory.resolve("code-proxy"), "localhost"); - com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode expired = - new com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode("lobby-1", + com.bencodez.simpleapi.servercomm.http.HttpTlsIdentity identity = + com.bencodez.simpleapi.servercomm.http.HttpTlsIdentity.loadOrCreate(directory.resolve("code-proxy"), "localhost"); + com.bencodez.simpleapi.servercomm.http.HttpConnectionCode expired = + new com.bencodez.simpleapi.servercomm.http.HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:1297/"), identity.serverCertificatePin(), identity.caCertificatePin(), java.time.Instant.now().minusSeconds(1), "A".repeat(43)); Files.writeString(settings, Files.readString(settings).replace("malformed", expired.encode())); assertThrows(IllegalArgumentException.class, () -> service.previewQuickSetup("proxy-method", Map.of("method", "HTTP"))); - com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode wrongServer = - new com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode("survival", + com.bencodez.simpleapi.servercomm.http.HttpConnectionCode wrongServer = + new com.bencodez.simpleapi.servercomm.http.HttpConnectionCode("survival", expired.endpoint(), expired.serverCertificatePin(), expired.caCertificatePin(), java.time.Instant.now().plusSeconds(60), "B".repeat(43)); Files.writeString(settings, Files.readString(settings).replace(expired.encode(), wrongServer.encode())); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 1cd56cb9b..5a1716760 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -551,7 +551,7 @@ void httpModernPresenceRejectsAuthenticatedBackendThatDoesNotMatchProxyRoute() { JsonEnvelope envelope = VotingPluginWire.login("Player", uuid, "Server2", java.util.UUID.randomUUID(), java.util.UUID.randomUUID(), 1000L, 1100L); - spyProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer.ReceivedEnvelope( + spyProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.simpleapi.servercomm.http.HttpProxyTransportServer.ReceivedEnvelope( "Server2", java.util.UUID.randomUUID().toString(), envelope)); verify(handler, never()).onMessage(Mockito.any()); @@ -570,7 +570,7 @@ void httpModernPresenceAcceptsAuthenticatedBackendMatchingProxyRouteAndUuid() { JsonEnvelope envelope = VotingPluginWire.login("Player", uuid, "Server1", java.util.UUID.randomUUID(), java.util.UUID.randomUUID(), 1000L, 1100L); - spyProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer.ReceivedEnvelope( + spyProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.simpleapi.servercomm.http.HttpProxyTransportServer.ReceivedEnvelope( "Server1", java.util.UUID.randomUUID().toString(), envelope)); verify(handler).onMessage(envelope); @@ -589,7 +589,7 @@ void httpModernPresenceRejectsUuidThatDoesNotMatchProxyPlayer() { JsonEnvelope envelope = VotingPluginWire.login("Player", claimedUuid, "Server1", java.util.UUID.randomUUID(), java.util.UUID.randomUUID(), 1000L, 1100L); - spyProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer.ReceivedEnvelope( + spyProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.simpleapi.servercomm.http.HttpProxyTransportServer.ReceivedEnvelope( "Server1", java.util.UUID.randomUUID().toString(), envelope)); verify(handler, never()).onMessage(Mockito.any()); @@ -603,7 +603,7 @@ void httpEnvelopeRejectsServerFieldThatDoesNotMatchAuthenticatedBackend() { votingPluginProxy.setGlobalMessageProxyHandlerForTest(handler); JsonEnvelope envelope = JsonEnvelope.builder("vote").put(VotingPluginWire.K_SERVER, "Server2").build(); - votingPluginProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer.ReceivedEnvelope( + votingPluginProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.simpleapi.servercomm.http.HttpProxyTransportServer.ReceivedEnvelope( "Server1", java.util.UUID.randomUUID().toString(), envelope)); verify(handler, never()).onMessage(Mockito.any()); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index ca47dd7fd..70a3a780a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -20,7 +20,7 @@ import com.bencodez.votingplugin.proxy.VotingPluginProxy; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; import com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects; -import com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer; +import com.bencodez.simpleapi.servercomm.http.HttpProxyTransportServer; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; public class VotingPluginProxyTestImpl extends VotingPluginProxy { diff --git a/docs/http-transport.md b/docs/http-transport.md index b8681176a..544bc1507 100644 --- a/docs/http-transport.md +++ b/docs/http-transport.md @@ -15,7 +15,7 @@ The `HTTP` bungee method gives every backend an outbound encrypted connection to ``` 2. Allow TCP port `1297` to the proxy. `PublicEndpoint` must resolve directly to this VotingPlugin listener. -3. Restart the proxy and run `/votingpluginbungee httpcode `. The name must exactly identify the intended backend; generate a separate code for each backend. +3. Restart the proxy and run `/votingpluginproxy httpcode `. The name must exactly identify the intended backend; generate a separate code for each backend. 4. On a backend, set a unique `Server`, enable bungee mode, select `HTTP`, and paste its code into `BungeeSettings.yml`: ```yaml @@ -47,7 +47,7 @@ Connection codes expire after 15 minutes and can be used only once. Treat a fres The listener must terminate TLS itself because client-certificate authentication is part of the protocol. Do not put an HTTP TLS-terminating reverse proxy or CDN in front of it. A TCP/L4 proxy that passes TLS through unchanged is suitable. Internet-facing installations should also use the host firewall or provider firewall for volumetric denial-of-service protection; an application cannot fully absorb a link or TCP flood. -If a backend host or its private credential is compromised, run `/votingpluginbungee httprevoke ` on the proxy before generating a new connection code. Revocation takes effect on the next request and permits a replacement identity to enroll under that server name. Keep the proxy's `http` data directory backed up and private: it contains the transport authority. +If a backend host or its private credential is compromised, run `/votingpluginproxy httprevoke ` on the proxy before generating a new connection code. Revocation takes effect on the next request and permits a replacement identity to enroll under that server name. Keep the proxy's `http` data directory backed up and private: it contains the transport authority. ## Performance From 63277bcd88350b2ecb6c0999c9cc28c8ec27e437 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:19:40 -0600 Subject: [PATCH 042/136] Declare SimpleAPI HTTP dependency --- VotingPlugin/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/VotingPlugin/pom.xml b/VotingPlugin/pom.xml index 144bc03f9..bba653aa2 100644 --- a/VotingPlugin/pom.xml +++ b/VotingPlugin/pom.xml @@ -329,6 +329,12 @@ 3.8.2-SNAPSHOT compile + + com.bencodez + simpleapi + 1.0.2-SNAPSHOT + compile + org.junit.jupiter junit-jupiter-engine From e6ba72cf5936d611b97cd9c05a15aaac016241b3 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:45:33 -0600 Subject: [PATCH 043/136] Resume vote parties after effect retry --- .../votingplugin/proxy/VotingPluginProxy.java | 5 ++++- .../tests/VotingPluginProxyTest.java | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 5cad21d9f..d09af583a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -3307,7 +3307,10 @@ private void scheduleVotePartyDeliveryRetry() { try { getScheduler().schedule(() -> { synchronized (VotingPluginProxy.this) { votePartyDeliveryRetryScheduled = false; } - if (retryPendingVotePartyProxyEffects() && method == BungeeMethod.HTTP) retryPendingVotePartyRewards(); + if (retryPendingVotePartyProxyEffects()) { + if (method == BungeeMethod.HTTP) retryPendingVotePartyRewards(); + if (votePartyVotes >= currentVotePartyVotesRequired) checkVoteParty(); + } }, 5, TimeUnit.SECONDS); } catch (RuntimeException failure) { votePartyDeliveryRetryScheduled = false; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 5a1716760..f939579c4 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -149,6 +149,27 @@ void httpVotePartyRetriesJournaledProxyEffectsAfterExecutionFailure() { assertTrue(votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().isEmpty()); } + @Test + void scheduledVotePartyEffectRetryProcessesAnotherReachedThreshold() { + configureHttpVotePartyEffects("Party time", java.util.List.of()); + java.util.concurrent.ScheduledExecutorService scheduler = Mockito + .mock(java.util.concurrent.ScheduledExecutorService.class); + votingPluginProxy.setSchedulerForTest(scheduler); + votingPluginProxy.failNextBroadcast(); + + votingPluginProxy.checkVoteParty(); + votingPluginProxy.setVotePartyVotes(1); + org.mockito.ArgumentCaptor retry = org.mockito.ArgumentCaptor.forClass(Runnable.class); + verify(scheduler).schedule(retry.capture(), Mockito.eq(5L), + Mockito.eq(java.util.concurrent.TimeUnit.SECONDS)); + + retry.getValue().run(); + + assertEquals(java.util.List.of("Party time", "Party time"), votingPluginProxy.getBroadcasts()); + assertEquals(0, votingPluginProxy.getVotePartyVotes()); + assertTrue(votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().isEmpty()); + } + @Test void httpVotePartyRestoresExecutedEffectWhenProgressIsNotDurable() { configureHttpVotePartyEffects("Party time", java.util.List.of("reward all")); From 3e17e389193d80a1877b2d32035ecb83eb4db64d Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:31:37 -0600 Subject: [PATCH 044/136] Preserve HTTP messages during replacement --- .../votingplugin/VotingPluginMain.java | 1 + .../backendproxy/BackendProxyHandler.java | 6 +++ .../BackendProxyTransportManager.java | 30 +++++++++++-- .../transport/HttpBackendProxyTransport.java | 42 ++++++++++++++++++- .../BackendProxyHandlerLifecycleTest.java | 25 +++++++++++ .../HttpBackendProxyTransportTest.java | 18 +++++++- 6 files changed, 115 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index b5cda2d0b..107540fa6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1310,6 +1310,7 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { return; } if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); + if (restart.previous != null) restart.previous.completeHttpHandoff(restart.replacement); restart.replacement.activatePresenceReporting(); backendProxyHandler = restart.replacement; if (restart.previous != null) restart.previous.close(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index a314bb889..bb4d786f8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -159,6 +159,12 @@ public void completeRedisHandoff(BackendProxyHandler replacement) { replacement.transportManager.activateRedisAfterHandoff(); } + /** Forwards messages buffered while the previous HTTP credentials were released. */ + public void completeHttpHandoff(BackendProxyHandler replacement) { + if (method != BungeeMethod.HTTP || replacement.method != BungeeMethod.HTTP) return; + transportManager.completePreparedTransportHandoff(replacement.transportManager); + } + public void playerOnline(String playerName, String uuid) { if (presenceManager != null) { presenceManager.playerOnline(playerName, uuid); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 08e3dccd5..07f8dffc5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -15,11 +15,15 @@ * Selects and owns the active backend-to-proxy transport. */ public class BackendProxyTransportManager { + private static final int MAX_PREPARED_SENDS = 1024; private final VotingPluginMain plugin; private final ProcessedVoteCache processedVoteCache; private BackendProxyTransport transport; private BackendProxyTransport preparedTransport; + private BackendProxyTransportManager forwardingManager; + private final java.util.ArrayDeque preparedSends = new java.util.ArrayDeque<>(); + private boolean preparedQueueWarning; public BackendProxyTransportManager(VotingPluginMain plugin) { this(plugin, new ProcessedVoteCache()); @@ -57,9 +61,18 @@ public void start(BungeeMethod method, GlobalMessageHandler messageHandler) { transport.start(messageHandler); } - public void send(JsonEnvelope envelope) { + public synchronized void send(JsonEnvelope envelope) { if (transport != null) { transport.send(envelope); + } else if (forwardingManager != null) { + forwardingManager.send(envelope); + } else if (preparedTransport != null) { + if (preparedSends.size() < MAX_PREPARED_SENDS) { + preparedSends.addLast(envelope); + } else if (!preparedQueueWarning) { + preparedQueueWarning = true; + plugin.getLogger().severe("HTTP replacement handoff queue is full; delivery was not accepted"); + } } } @@ -67,7 +80,7 @@ public void activateAfterPublication() { if (transport != null) transport.activateAfterPublication(); } - public void close() { + public synchronized void close() { if (transport != null) { transport.close(); transport = null; @@ -76,6 +89,7 @@ public void close() { preparedTransport.close(); preparedTransport = null; } + if (forwardingManager == null) preparedSends.clear(); } public void validate() { @@ -88,7 +102,7 @@ public void validate(long deadlineNanos) { else transport.validate(); } - public void prepareForReplacement() { + public synchronized void prepareForReplacement() { if (transport != null) { transport.prepareForReplacement(); preparedTransport = transport; @@ -96,7 +110,13 @@ public void prepareForReplacement() { } } - public void restorePreparedTransport() { + public synchronized void completePreparedTransportHandoff(BackendProxyTransportManager replacement) { + if (preparedTransport == null) return; + forwardingManager = java.util.Objects.requireNonNull(replacement, "replacement"); + while (!preparedSends.isEmpty()) forwardingManager.send(preparedSends.removeFirst()); + } + + public synchronized void restorePreparedTransport() { if (transport != null || preparedTransport == null) return; if (preparedTransport instanceof HttpBackendProxyTransport http) { transport = http.recreatePrepared(); @@ -104,6 +124,8 @@ public void restorePreparedTransport() { throw new IllegalStateException("Prepared backend proxy transport cannot be restored"); } preparedTransport = null; + while (!preparedSends.isEmpty()) transport.send(preparedSends.removeFirst()); + preparedQueueWarning = false; } public void restoreAfterFailedReplacement() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index d6e3eb35b..314bf76ce 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -33,6 +33,8 @@ public final class HttpBackendProxyTransport implements BackendProxyTransport { private volatile RuntimeException credentialRestoreFailure; private volatile boolean started; private volatile boolean closed; + private volatile boolean restartAfterFailedFlush; + private final java.util.concurrent.atomic.AtomicBoolean flushRecoveryRunning = new java.util.concurrent.atomic.AtomicBoolean(); private Path configuredDirectory; private String configuredServerId; private String configuredConnectionCode; @@ -92,9 +94,42 @@ public void prepareForReplacement() { } catch (Exception failure) { throw new IllegalStateException("Could not preserve the active HTTP client credential", failure); } + flushForReplacement(connector, System.nanoTime() + TimeUnit.SECONDS.toNanos(SHUTDOWN_FLUSH_SECONDS)); close(); } + void flushForReplacement(HttpBackendTransportConnector connector, long deadlineNanos) { + if (connector.flushOutgoing(deadlineNanos)) return; + restartAfterFailedFlush = true; + connector.start(); + if (flushRecoveryRunning.compareAndSet(false, true)) { + Thread recovery = new Thread(() -> resumeAfterFailedFlush(connector), + "VotingPlugin-HTTP-Backend-Flush-Recovery"); + recovery.setDaemon(true); + recovery.start(); + } + throw new IllegalStateException("Could not drain the active HTTP transport before replacement"); + } + + private void resumeAfterFailedFlush(HttpBackendTransportConnector active) { + try { + // start() intentionally does nothing while the interrupted long-poll worker is + // still winding down. Keep a single recovery owner alive until this transport is + // closed or replaced so the connector's already-accepted queue cannot be stranded. + while (!closed && connector == active) { + active.start(); + synchronized (lifecycle) { + while (!startupQueue.isEmpty() && active.send(startupQueue.peekFirst())) startupQueue.removeFirst(); + } + try { TimeUnit.SECONDS.sleep(1); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); break; } + } + } finally { + flushRecoveryRunning.set(false); + if (closed || connector != active) restartAfterFailedFlush = false; + } + } + private void initialize(Path directory, String serverId, String configuredCode, GlobalMessageHandler messageHandler) { Path ownerKey = directory.toAbsolutePath().normalize(); @@ -187,8 +222,11 @@ public void send(JsonEnvelope envelope) { if (closed) return; HttpBackendTransportConnector active = connector; if (active != null) { - if (!active.send(envelope) && queueWarning.compareAndSet(false, true)) - plugin.getLogger().severe("Secure HTTP transport queue is full or rejected an oversized message; delivery was not accepted"); + if (!active.send(envelope)) { + if (restartAfterFailedFlush && startupQueue.size() < MAX_STARTUP_QUEUE) startupQueue.addLast(envelope); + else if (queueWarning.compareAndSet(false, true)) + plugin.getLogger().severe("Secure HTTP transport queue is full or rejected an oversized message; delivery was not accepted"); + } } else if (startupQueue.size() < MAX_STARTUP_QUEUE) { startupQueue.addLast(envelope); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index f41eea3d7..60c476cc4 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -309,6 +309,31 @@ void failedSameMysqlReplacementLeavesExistingTransportAvailable() throws Excepti verifyNoInteractions(messenger); } + @Test + void forwardsMessagesBufferedDuringPreparedHttpReplacement() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BackendProxyTransportManager previous = new BackendProxyTransportManager(plugin); + BackendProxyTransportManager replacement = new BackendProxyTransportManager(plugin); + BackendProxyTransport previousTransport = mock(BackendProxyTransport.class); + BackendProxyTransport replacementTransport = mock(BackendProxyTransport.class); + setField(previous, "transport", previousTransport); + setField(replacement, "transport", replacementTransport); + com.bencodez.simpleapi.servercomm.codec.JsonEnvelope duringValidation = + com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("during-validation").build(); + com.bencodez.simpleapi.servercomm.codec.JsonEnvelope afterPublication = + com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("after-publication").build(); + + previous.prepareForReplacement(); + previous.send(duringValidation); + verifyNoInteractions(replacementTransport); + + previous.completePreparedTransportHandoff(replacement); + previous.send(afterPublication); + + verify(replacementTransport).send(duringValidation); + verify(replacementTransport).send(afterPublication); + } + @Test void releasesMysqlSubscriberBeforeSameMethodReplacement() throws Exception { MysqlBackendProxyTransport handler = new MysqlBackendProxyTransport(null); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index f74f9ef5e..5ac674cab 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -7,8 +7,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.bencodez.simpleapi.servercomm.http.HttpBackendTransportConnector; import com.bencodez.simpleapi.servercomm.http.HttpClientCredentialStore; import com.bencodez.simpleapi.servercomm.http.HttpConnectionCode; import com.bencodez.simpleapi.servercomm.http.HttpTlsIdentity; @@ -68,6 +70,18 @@ void freshConnectionCodeOverridesAnExistingEnrollment() throws Exception { () -> HttpBackendProxyTransport.enrollmentCode(credentials, "lobby-1", "malformed")); } + @Test + void failedReplacementFlushResumesTheExistingConnector() { + HttpBackendTransportConnector connector = mock(HttpBackendTransportConnector.class); + when(connector.flushOutgoing(org.mockito.ArgumentMatchers.anyLong())).thenReturn(false); + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(mock(VotingPluginMain.class)); + + assertThrows(IllegalStateException.class, + () -> transport.flushForReplacement(connector, System.nanoTime())); + + verify(connector).start(); + } + @Test void closeNeverWaitsForSetupOnTheCallingThread() throws Exception { HttpBackendProxyTransport transport = new HttpBackendProxyTransport(mock(VotingPluginMain.class)); @@ -105,7 +119,9 @@ void prepareForReplacementSnapshotsThePersistedCredentialGeneration() throws Exc directoryField.set(transport, credentials); java.lang.reflect.Field connectorField = HttpBackendProxyTransport.class.getDeclaredField("connector"); connectorField.setAccessible(true); - connectorField.set(transport, mock(com.bencodez.simpleapi.servercomm.http.HttpBackendTransportConnector.class)); + HttpBackendTransportConnector connector = mock(HttpBackendTransportConnector.class); + when(connector.flushOutgoing(org.mockito.ArgumentMatchers.anyLong())).thenReturn(true); + connectorField.set(transport, connector); assertDoesNotThrow(transport::prepareForReplacement); java.lang.reflect.Field generationField = From 804271e24aaa61df626d1f1356516eecf6ac17e6 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:33:02 -0600 Subject: [PATCH 045/136] Keep HTTP transport retries active --- .../transport/HttpBackendProxyTransport.java | 59 ++++++++-- .../HttpBackendProxyTransportTest.java | 102 ++++++++++++++++++ 2 files changed, 155 insertions(+), 6 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 314bf76ce..62c0d52ef 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -7,6 +7,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicInteger; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; @@ -20,6 +22,7 @@ public final class HttpBackendProxyTransport implements BackendProxyTransport { private static final int MAX_STARTUP_QUEUE = 1024; private static final long DEFAULT_STARTUP_VALIDATION_SECONDS = 25L; + private static final long INCOMING_DISPATCH_SECONDS = 25L; private static final long SHUTDOWN_FLUSH_SECONDS = 5L; private static final ConcurrentHashMap DIRECTORY_OWNERS = new ConcurrentHashMap<>(); private final VotingPluginMain plugin; @@ -167,12 +170,10 @@ private void initialize(Path directory, String serverId, String configuredCode, HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); if (!enrolled.profile().serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); - replacement = new HttpBackendTransportConnector(directory, messageHandler::onMessage); + replacement = new HttpBackendTransportConnector(directory, + envelope -> dispatchIncoming(messageHandler, envelope, + System.nanoTime() + TimeUnit.SECONDS.toNanos(INCOMING_DISPATCH_SECONDS))); replacement.start(); - if (!replacement.awaitFirstResponse(System.nanoTime() - + TimeUnit.SECONDS.toNanos(DEFAULT_STARTUP_VALIDATION_SECONDS))) { - throw new IllegalStateException("HTTP backend could not authenticate with the proxy"); - } boolean discard = false; synchronized (lifecycle) { if (closed) { @@ -202,6 +203,44 @@ private void initialize(Path directory, String serverId, String configuredCode, } } + void dispatchIncoming(GlobalMessageHandler messageHandler, JsonEnvelope envelope, long deadlineNanos) { + CountDownLatch completed = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + AtomicInteger state = new AtomicInteger(0); // pending, running, cancelled, finished + try { + plugin.getBukkitScheduler().runTask(plugin, () -> { + if (!state.compareAndSet(0, 1)) { + completed.countDown(); + return; + } + try { + messageHandler.onMessage(envelope); + } catch (Throwable thrown) { + failure.set(thrown); + } finally { + state.set(3); + completed.countDown(); + } + }); + } catch (Throwable rejected) { + throw new IllegalStateException("Could not schedule an incoming HTTP message on the server thread", rejected); + } + try { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L || !completed.await(remaining, TimeUnit.NANOSECONDS)) { + state.compareAndSet(0, 2); + throw new IllegalStateException("Incoming HTTP message handling exceeded its delivery deadline"); + } + } catch (InterruptedException interrupted) { + state.compareAndSet(0, 2); + Thread.currentThread().interrupt(); + throw new IllegalStateException("Incoming HTTP message handling was interrupted", interrupted); + } + Throwable thrown = failure.get(); + if (thrown != null) + throw new IllegalStateException("Incoming HTTP message handling failed", thrown); + } + void awaitCredentialRestoration(long deadlineNanos) { if (credentialGenerationToRestore == null) return; try { @@ -256,7 +295,15 @@ void validate(long deadlineNanos) { } RuntimeException failure = startupFailure; if (failure != null) throw failure; - if (closed || connector == null) throw new IllegalStateException("Secure HTTP backend transport did not become ready"); + HttpBackendTransportConnector active = connector; + if (closed || active == null) throw new IllegalStateException("Secure HTTP backend transport did not initialize"); + try { + if (!active.awaitFirstResponse(deadlineNanos)) + throw new IllegalStateException("HTTP backend could not authenticate with the proxy"); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Secure HTTP backend readiness validation was interrupted", interrupted); + } } public static void validateConfiguration(Path directory, String serverId, String configuredCode) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 5ac674cab..2cb10bc37 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -17,17 +18,114 @@ import com.bencodez.votingplugin.VotingPluginMain; import com.bencodez.votingplugin.config.BungeeSettings; import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageListener; +import com.bencodez.simpleapi.scheduler.BukkitScheduler; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import java.net.URI; import java.nio.file.Path; import java.time.Instant; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; class HttpBackendProxyTransportTest { @TempDir Path directory; + @Test + void dispatchesIncomingMessagesOnTheServerSchedulerAndWaitsForCompletion() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + AtomicReference callbackThread = new AtomicReference<>(); + CountDownLatch callbackFinished = new CountDownLatch(1); + GlobalMessageHandler handler = new GlobalMessageHandler() { + @Override public void sendMessage(JsonEnvelope envelope) { } + }; + handler.addListener(new GlobalMessageListener("test") { + @Override public void onReceive(JsonEnvelope envelope) { + callbackThread.set(Thread.currentThread().getName()); + callbackFinished.countDown(); + } + }); + doAnswer(invocation -> { + Runnable task = invocation.getArgument(1); + Thread serverThread = new Thread(task, "test-server-thread"); + serverThread.start(); + return null; + }).when(scheduler).runTask(org.mockito.ArgumentMatchers.eq(plugin), org.mockito.ArgumentMatchers.any(Runnable.class)); + JsonEnvelope envelope = JsonEnvelope.builder("test").put("message", "payload").build(); + + new HttpBackendProxyTransport(plugin).dispatchIncoming(handler, envelope, + System.nanoTime() + TimeUnit.SECONDS.toNanos(1)); + + assertTrue(callbackFinished.await(1, TimeUnit.SECONDS)); + assertEquals("test-server-thread", callbackThread.get()); + } + + @Test + void timedOutScheduledMessageCannotExecuteLater() { + VotingPluginMain plugin = mock(VotingPluginMain.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + AtomicReference scheduled = new AtomicReference<>(); + doAnswer(invocation -> { + scheduled.set(invocation.getArgument(1)); + return null; + }).when(scheduler).runTask(org.mockito.ArgumentMatchers.eq(plugin), org.mockito.ArgumentMatchers.any(Runnable.class)); + java.util.concurrent.atomic.AtomicInteger deliveries = new java.util.concurrent.atomic.AtomicInteger(); + GlobalMessageHandler handler = new GlobalMessageHandler() { + @Override public void sendMessage(JsonEnvelope envelope) { } + }; + handler.addListener(new GlobalMessageListener("test") { + @Override public void onReceive(JsonEnvelope envelope) { deliveries.incrementAndGet(); } + }); + + assertThrows(IllegalStateException.class, + () -> new HttpBackendProxyTransport(plugin).dispatchIncoming(handler, + JsonEnvelope.builder("test").build(), System.nanoTime())); + assertTrue(scheduled.get() != null); + scheduled.get().run(); + assertEquals(0, deliveries.get(), "a delivery rejected at its deadline must stay fenced"); + } + + @Test + void interruptedScheduledMessageCannotExecuteLater() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + AtomicReference scheduled = new AtomicReference<>(); + doAnswer(invocation -> { + scheduled.set(invocation.getArgument(1)); + return null; + }).when(scheduler).runTask(org.mockito.ArgumentMatchers.eq(plugin), org.mockito.ArgumentMatchers.any(Runnable.class)); + java.util.concurrent.atomic.AtomicInteger deliveries = new java.util.concurrent.atomic.AtomicInteger(); + GlobalMessageHandler handler = new GlobalMessageHandler() { + @Override public void sendMessage(JsonEnvelope envelope) { } + }; + handler.addListener(new GlobalMessageListener("test") { + @Override public void onReceive(JsonEnvelope envelope) { deliveries.incrementAndGet(); } + }); + AtomicReference dispatchFailure = new AtomicReference<>(); + Thread dispatch = new Thread(() -> { + try { + new HttpBackendProxyTransport(plugin).dispatchIncoming(handler, + JsonEnvelope.builder("test").build(), System.nanoTime() + TimeUnit.SECONDS.toNanos(5)); + } catch (Throwable thrown) { + dispatchFailure.set(thrown); + } + }); + dispatch.start(); + while (scheduled.get() == null) Thread.onSpinWait(); + dispatch.interrupt(); + dispatch.join(TimeUnit.SECONDS.toMillis(1)); + + assertTrue(dispatchFailure.get() instanceof IllegalStateException); + scheduled.get().run(); + assertEquals(0, deliveries.get(), "an interrupted delivery must stay fenced"); + } + @Test void validatesInitialConnectionCodeSynchronously() { VotingPluginMain plugin = mock(VotingPluginMain.class); @@ -165,6 +263,10 @@ void validationWaitsForThePreviousDirectoryOwnerBeforeReadinessFailure() throws assertTrue(finished.await(3, TimeUnit.SECONDS)); validation.join(TimeUnit.SECONDS.toMillis(1)); assertTrue(failure.get() instanceof IllegalStateException); + java.lang.reflect.Field connectorField = HttpBackendProxyTransport.class.getDeclaredField("connector"); + connectorField.setAccessible(true); + assertTrue(connectorField.get(transport) instanceof HttpBackendTransportConnector, + "a readiness timeout must not tear down the connector's background retry loop"); transport.close(); } From f08dd1c3a3f1614081e9bce466f3ccbe56e8f029 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:33:43 -0600 Subject: [PATCH 046/136] fix: retry HTTP enrollment failures --- .../backendproxy/BackendProxyHandler.java | 2 +- .../transport/BackendProxyTransport.java | 9 ++++ .../BackendProxyTransportManager.java | 6 ++- .../transport/HttpBackendProxyTransport.java | 54 +++++++++++++++++-- .../proxy/bungee/VotingPluginBungee.java | 17 ++++-- .../HttpBackendProxyTransportTest.java | 42 +++++++++++++++ .../tests/VotingPluginProxyTest.java | 14 +++++ .../tests/VotingPluginProxyTestImpl.java | 10 ++++ 8 files changed, 143 insertions(+), 11 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index bb4d786f8..119f3c215 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -89,7 +89,7 @@ public void sendMessage(JsonEnvelope envelope) { messageRouter = new BackendProxyMessageRouter(plugin, presenceManager, globalDataSync, votePartySync, processedVoteCache); messageRouter.register(globalMessageHandler, method); - transportManager.start(method, globalMessageHandler); + transportManager.start(method, globalMessageHandler, activatePresenceReporting); if (plugin.getOptions().getServer().equalsIgnoreCase("pleaseset")) { plugin.getLogger().warning("Server name for bungee voting is not set, please set it"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java index 008affab1..f84f1c6fd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java @@ -10,6 +10,15 @@ public interface BackendProxyTransport { void start(GlobalMessageHandler messageHandler); + /** + * Starts the transport, optionally allowing startup-only transient failures to + * be retried. Replacement transports disable this so Control validation stays + * fail-fast; ordinary startup retains its recovery loop. + */ + default void start(GlobalMessageHandler messageHandler, boolean retryInitialization) { + start(messageHandler); + } + void send(JsonEnvelope envelope); default void validate() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 07f8dffc5..6d0a3120c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -35,6 +35,10 @@ public BackendProxyTransportManager(VotingPluginMain plugin, ProcessedVoteCache } public void start(BungeeMethod method, GlobalMessageHandler messageHandler) { + start(method, messageHandler, true); + } + + public void start(BungeeMethod method, GlobalMessageHandler messageHandler, boolean retryInitialization) { close(); switch (method) { case MYSQL: @@ -58,7 +62,7 @@ public void start(BungeeMethod method, GlobalMessageHandler messageHandler) { default: throw new IllegalArgumentException("Unsupported backend proxy method: " + method); } - transport.start(messageHandler); + transport.start(messageHandler, retryInitialization); } public synchronized void send(JsonEnvelope envelope) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 62c0d52ef..9e1f36c1f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -22,6 +22,8 @@ public final class HttpBackendProxyTransport implements BackendProxyTransport { private static final int MAX_STARTUP_QUEUE = 1024; private static final long DEFAULT_STARTUP_VALIDATION_SECONDS = 25L; + private static final long ENROLLMENT_RETRY_INITIAL_MILLIS = 1_000L; + private static final long ENROLLMENT_RETRY_MAX_MILLIS = 60_000L; private static final long INCOMING_DISPATCH_SECONDS = 25L; private static final long SHUTDOWN_FLUSH_SECONDS = 5L; private static final ConcurrentHashMap DIRECTORY_OWNERS = new ConcurrentHashMap<>(); @@ -53,20 +55,32 @@ public HttpBackendProxyTransport(VotingPluginMain plugin) { @Override public void start(GlobalMessageHandler messageHandler) { + start(messageHandler, true); + } + + @Override + public void start(GlobalMessageHandler messageHandler, boolean retryInitialization) { Path directory = plugin.getDataFolder().toPath().resolve("http"); String serverId = plugin.getBungeeSettings().getServer(); String connectionCode = plugin.getBungeeSettings().getHttpConnectionCode(); - start(directory, serverId, connectionCode, messageHandler); + start(directory, serverId, connectionCode, messageHandler, null, retryInitialization); } private void start(Path directory, String serverId, String connectionCode, GlobalMessageHandler messageHandler) { - start(directory, serverId, connectionCode, messageHandler, null); + start(directory, serverId, connectionCode, messageHandler, null, true); } private void start(Path directory, String serverId, String connectionCode, GlobalMessageHandler messageHandler, HttpClientCredentialStore.ActiveCredentialGeneration generationToRestore) { + start(directory, serverId, connectionCode, messageHandler, generationToRestore, false); + } + + private void start(Path directory, String serverId, String connectionCode, + GlobalMessageHandler messageHandler, + HttpClientCredentialStore.ActiveCredentialGeneration generationToRestore, + boolean retryInitialization) { if (generationToRestore == null) validateConfiguration(directory, serverId, connectionCode); else HttpTlsIdentity.canonicalServerId(serverId); configuredDirectory = directory; @@ -75,7 +89,7 @@ private void start(Path directory, String serverId, String connectionCode, configuredMessageHandler = messageHandler; credentialGenerationToRestore = generationToRestore; started = true; - worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler), + worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler, retryInitialization), "VotingPlugin-HTTP-Backend-Setup"); worker.setDaemon(true); worker.start(); @@ -134,7 +148,7 @@ private void resumeAfterFailedFlush(HttpBackendTransportConnector active) { } private void initialize(Path directory, String serverId, String configuredCode, - GlobalMessageHandler messageHandler) { + GlobalMessageHandler messageHandler, boolean retryEnrollment) { Path ownerKey = directory.toAbsolutePath().normalize(); Semaphore owner = DIRECTORY_OWNERS.computeIfAbsent(ownerKey, ignored -> new Semaphore(1)); boolean acquired = false, installed = false; @@ -166,7 +180,8 @@ private void initialize(Path directory, String serverId, String configuredCode, // renewal/re-enrollment retained the newer active generation. HttpConnectionCode code = enrollmentCode(directory, serverId, credentialGenerationToRestore == null ? configuredCode : null); - if (code != null) HttpBackendTransportConnector.enroll(code, serverId, directory); + if (code != null && !enrollForStartup(code, serverId, directory, retryEnrollment, + this::waitForEnrollmentRetry)) return; HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); if (!enrolled.profile().serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); @@ -203,6 +218,35 @@ private void initialize(Path directory, String serverId, String configuredCode, } } + private boolean waitForEnrollmentRetry(long delayMillis) { + if (closed) return false; + try { + TimeUnit.MILLISECONDS.sleep(delayMillis); + return !closed; + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return false; + } + } + + /** Performs initial enrollment with a bounded backoff for ordinary startup. */ + boolean enrollForStartup(HttpConnectionCode code, String serverId, Path directory, + boolean retryEnrollment, java.util.function.LongPredicate waitForRetry) throws Exception { + long retryDelayMillis = ENROLLMENT_RETRY_INITIAL_MILLIS; + while (true) { + if (closed) return false; + try { + HttpBackendTransportConnector.enroll(code, serverId, directory); + return true; + } catch (Exception failure) { + if (!retryEnrollment) throw failure; + plugin.getLogger().warning("Secure HTTP backend enrollment failed; retrying with bounded backoff"); + if (!waitForRetry.test(retryDelayMillis)) return false; + retryDelayMillis = Math.min(ENROLLMENT_RETRY_MAX_MILLIS, retryDelayMillis * 2L); + } + } + } + void dispatchIncoming(GlobalMessageHandler messageHandler, JsonEnvelope envelope, long deadlineNanos) { CountDownLatch completed = new CountDownLatch(1); AtomicReference failure = new AtomicReference<>(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java index 41cd4345c..2118c6c62 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java @@ -682,10 +682,19 @@ public void runAsync(Runnable run) { runAsyncNow(run); } - @Override - public void runConsoleCommand(String command) { - getProxy().getPluginManager().dispatchCommand(getProxy().getConsole(), command); - } + @Override + public void runConsoleCommand(String command) { + getProxy().getPluginManager().dispatchCommand(getProxy().getConsole(), command); + } + + @Override + protected java.util.concurrent.CompletableFuture runVotePartyConsoleCommand(String command) { + if (!getProxy().getPluginManager().dispatchCommand(getProxy().getConsole(), command)) { + return java.util.concurrent.CompletableFuture.failedFuture( + new IllegalStateException("Bungee declined the vote-party proxy command")); + } + return java.util.concurrent.CompletableFuture.completedFuture(null); + } @Override public void saveVoteCacheFile() { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 2cb10bc37..f7ca8b3ed 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -24,11 +24,15 @@ import java.net.URI; import java.nio.file.Path; import java.time.Instant; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.mockito.MockedStatic; class HttpBackendProxyTransportTest { @TempDir Path directory; @@ -150,6 +154,44 @@ void validatesInitialConnectionCodeSynchronously() { assertDoesNotThrow(() -> transport.validate()); } + @Test + void ordinaryStartupRetriesTransientInitialEnrollmentWithBoundedBackoff() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(plugin); + HttpConnectionCode enrollment = code("lobby-1", Instant.now().plusSeconds(60)); + List delays = new ArrayList<>(); + java.util.concurrent.atomic.AtomicInteger attempts = new java.util.concurrent.atomic.AtomicInteger(); + + try (MockedStatic connector = org.mockito.Mockito.mockStatic(HttpBackendTransportConnector.class)) { + connector.when(() -> HttpBackendTransportConnector.enroll(enrollment, "lobby-1", directory)) + .thenAnswer(ignored -> { + if (attempts.getAndIncrement() == 0) throw new IOException("proxy unavailable"); + return null; + }); + assertTrue(transport.enrollForStartup(enrollment, "lobby-1", directory, true, + delay -> { delays.add(delay); return true; })); + } + + assertEquals(2, attempts.get()); + assertEquals(List.of(1_000L), delays); + } + + @Test + void stagedEnrollmentRemainsFailFast() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(plugin); + HttpConnectionCode enrollment = code("lobby-1", Instant.now().plusSeconds(60)); + try (MockedStatic connector = org.mockito.Mockito.mockStatic(HttpBackendTransportConnector.class)) { + connector.when(() -> HttpBackendTransportConnector.enroll(enrollment, "lobby-1", directory)) + .thenThrow(new IOException("proxy unavailable")); + assertThrows(IOException.class, + () -> transport.enrollForStartup(enrollment, "lobby-1", directory, false, + delay -> { throw new AssertionError("staged enrollment must not retry"); })); + } + } + @Test void freshConnectionCodeOverridesAnExistingEnrollment() throws Exception { Path credentials = directory.resolve("http"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index f939579c4..f93a0fbd8 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -202,6 +202,20 @@ void httpVotePartyPersistsOrderedCommandProgress() { assertEquals(java.util.List.of("first", "second", "third"), votingPluginProxy.getConsoleCommands()); } + @Test + void declinedVotePartyCommandRemainsPendingForRetry() { + configureHttpVotePartyEffects("", java.util.List.of("not-yet-registered")); + votingPluginProxy.declineNextVotePartyCommand(); + + votingPluginProxy.checkVoteParty(); + + assertEquals(java.util.List.of("not-yet-registered"), + votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().commands()); + assertTrue(votingPluginProxy.retryPendingVotePartyProxyEffectsForTest()); + assertTrue(votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().isEmpty()); + assertEquals(java.util.List.of("not-yet-registered"), votingPluginProxy.getConsoleCommands()); + } + @Test void httpVotePartyKeepsCommandPendingUntilAsyncExecutionCompletesAndResumesReachedThreshold() { configureHttpVotePartyEffects("", java.util.List.of("async command")); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index 70a3a780a..4ceed32b2 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -43,6 +43,7 @@ public class VotingPluginProxyTestImpl extends VotingPluginProxy { private String failConsoleCommand; private boolean failSaveAfterNextBroadcast; private java.util.concurrent.CompletableFuture nextVotePartyCommandCompletion; + private boolean declineNextVotePartyCommand; private Runnable votePartyProxyCommandTimeout; public List getWarnings() { @@ -211,6 +212,11 @@ public void runConsoleCommand(String command) { protected java.util.concurrent.CompletableFuture runVotePartyConsoleCommand(String command) { java.util.concurrent.CompletableFuture completion = nextVotePartyCommandCompletion; nextVotePartyCommandCompletion = null; + if (declineNextVotePartyCommand) { + declineNextVotePartyCommand = false; + return java.util.concurrent.CompletableFuture.failedFuture( + new IllegalStateException("proxy declined the vote-party command")); + } runConsoleCommand(command); return completion == null ? java.util.concurrent.CompletableFuture.completedFuture(null) : completion; } @@ -245,6 +251,10 @@ public void failConsoleCommand(String command) { failConsoleCommand = command; } + public void declineNextVotePartyCommand() { + declineNextVotePartyCommand = true; + } + public void failSaveAfterNextBroadcast() { failSaveAfterNextBroadcast = true; } From 7c715cf95c8789b678eaf5e2a2e15e296d052621 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 7 Sep 2026 23:57:53 -0600 Subject: [PATCH 047/136] Fix HTTP transport replacement during enrollment --- .../BackendProxyTransportManager.java | 3 + .../transport/HttpBackendProxyTransport.java | 163 +++++++++++++++--- .../HttpBackendProxyTransportTest.java | 128 ++++++++++++++ 3 files changed, 267 insertions(+), 27 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 6d0a3120c..dafa994db 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -117,6 +117,9 @@ public synchronized void prepareForReplacement() { public synchronized void completePreparedTransportHandoff(BackendProxyTransportManager replacement) { if (preparedTransport == null) return; forwardingManager = java.util.Objects.requireNonNull(replacement, "replacement"); + if (preparedTransport instanceof HttpBackendProxyTransport http) { + for (JsonEnvelope envelope : http.drainPreparedMessages()) forwardingManager.send(envelope); + } while (!preparedSends.isEmpty()) forwardingManager.send(preparedSends.removeFirst()); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 9e1f36c1f..5f7e262ee 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -1,5 +1,7 @@ package com.bencodez.votingplugin.backendproxy.transport; +import java.nio.file.Files; +import java.nio.file.LinkOption; import java.nio.file.Path; import java.time.Clock; import java.util.ArrayDeque; @@ -17,6 +19,7 @@ import com.bencodez.simpleapi.servercomm.http.HttpConnectionCode; import com.bencodez.simpleapi.servercomm.http.HttpTlsIdentity; import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.util.DurableFiles; /** Backend adapter for the secure outbound-only HTTP proxy transport. */ public final class HttpBackendProxyTransport implements BackendProxyTransport { @@ -46,6 +49,8 @@ public final class HttpBackendProxyTransport implements BackendProxyTransport { private GlobalMessageHandler configuredMessageHandler; private HttpClientCredentialStore.ActiveCredentialGeneration configuredCredentialGeneration; private HttpClientCredentialStore.ActiveCredentialGeneration credentialGenerationToRestore; + private boolean retryInitialization; + private boolean restoreUnenrolledState; private Semaphore directoryOwner; private final java.util.concurrent.atomic.AtomicBoolean queueWarning = new java.util.concurrent.atomic.AtomicBoolean(); @@ -63,33 +68,37 @@ public void start(GlobalMessageHandler messageHandler, boolean retryInitializati Path directory = plugin.getDataFolder().toPath().resolve("http"); String serverId = plugin.getBungeeSettings().getServer(); String connectionCode = plugin.getBungeeSettings().getHttpConnectionCode(); - start(directory, serverId, connectionCode, messageHandler, null, retryInitialization); + start(directory, serverId, connectionCode, messageHandler, null, retryInitialization, false); } private void start(Path directory, String serverId, String connectionCode, GlobalMessageHandler messageHandler) { - start(directory, serverId, connectionCode, messageHandler, null, true); + start(directory, serverId, connectionCode, messageHandler, null, true, false); } private void start(Path directory, String serverId, String connectionCode, GlobalMessageHandler messageHandler, HttpClientCredentialStore.ActiveCredentialGeneration generationToRestore) { - start(directory, serverId, connectionCode, messageHandler, generationToRestore, false); + start(directory, serverId, connectionCode, messageHandler, generationToRestore, false, false); } private void start(Path directory, String serverId, String connectionCode, GlobalMessageHandler messageHandler, HttpClientCredentialStore.ActiveCredentialGeneration generationToRestore, - boolean retryInitialization) { - if (generationToRestore == null) validateConfiguration(directory, serverId, connectionCode); + boolean retryInitialization, boolean restoreUnenrolledState) { + if (generationToRestore == null && !restoreUnenrolledState) + validateConfiguration(directory, serverId, connectionCode); else HttpTlsIdentity.canonicalServerId(serverId); configuredDirectory = directory; configuredServerId = serverId; configuredConnectionCode = connectionCode; configuredMessageHandler = messageHandler; credentialGenerationToRestore = generationToRestore; + this.retryInitialization = retryInitialization; + this.restoreUnenrolledState = restoreUnenrolledState; started = true; - worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler, retryInitialization), + worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler, retryInitialization, + restoreUnenrolledState), "VotingPlugin-HTTP-Backend-Setup"); worker.setDaemon(true); worker.start(); @@ -97,22 +106,88 @@ private void start(Path directory, String serverId, String connectionCode, HttpBackendProxyTransport recreatePrepared() { HttpBackendProxyTransport restored = new HttpBackendProxyTransport(plugin); + synchronized (lifecycle) { + restored.startupQueue.addAll(startupQueue); + startupQueue.clear(); + } restored.start(configuredDirectory, configuredServerId, configuredConnectionCode, configuredMessageHandler, - configuredCredentialGeneration); + configuredCredentialGeneration, configuredCredentialGeneration == null && retryInitialization, + restoreUnenrolledState); return restored; } + java.util.List drainPreparedMessages() { + synchronized (lifecycle) { + java.util.List pending = java.util.List.copyOf(startupQueue); + startupQueue.clear(); + return pending; + } + } + @Override public void prepareForReplacement() { - if (connector == null || configuredDirectory == null) - throw new IllegalStateException("Could not preserve the active HTTP client credential before it became ready"); + HttpBackendTransportConnector active; + Thread cancelledInitialization = null; + /* + * Keep the check, credential snapshot, and cancellation in one lifecycle + * critical section. During ordinary first-time enrollment the setup worker + * owns DIRECTORY_OWNERS while it retries and connector is intentionally null. + * A Control replacement must be able to cancel that worker before claiming + * the same credential directory; there is no credential to preserve in that + * state. If enrollment has already published a credential but the connector + * has not yet been installed, retain the generation just as we do for an + * established connector. + */ + synchronized (lifecycle) { + if (configuredDirectory == null) + throw new IllegalStateException("Could not preserve the active HTTP client credential before it became ready"); + active = connector; + if (active == null && !HttpClientCredentialStore.hasEnrolledProfile(configuredDirectory)) { + restoreUnenrolledState = true; + cancelledInitialization = worker; + closeForReplacement(); + } else try { + configuredCredentialGeneration = HttpClientCredentialStore.snapshotActiveGeneration(configuredDirectory); + } catch (Exception failure) { + throw new IllegalStateException("Could not preserve the active HTTP client credential", failure); + } + if (cancelledInitialization == null && active == null) { + closeForReplacement(); + return; + } + } + if (cancelledInitialization != null) { + awaitCancelledInitialization(cancelledInitialization); + captureEnrollmentPublishedDuringCancellation(); + return; + } + flushForReplacement(active, System.nanoTime() + TimeUnit.SECONDS.toNanos(SHUTDOWN_FLUSH_SECONDS)); + closeForReplacement(); + } + + private void awaitCancelledInitialization(Thread setup) { + try { setup.join(TimeUnit.SECONDS.toMillis(SHUTDOWN_FLUSH_SECONDS)); } + catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while stopping HTTP client enrollment", interrupted); + } + if (setup.isAlive()) + throw new IllegalStateException("Could not stop HTTP client enrollment before replacement"); + } + + private void captureEnrollmentPublishedDuringCancellation() { + if (!HttpClientCredentialStore.hasEnrolledProfile(configuredDirectory)) return; try { + HttpConnectionCode original = HttpConnectionCode.parse(configuredConnectionCode); + if (!original.serverId().equals(HttpTlsIdentity.canonicalServerId(configuredServerId)) + || !HttpClientCredentialStore.matchesEnrollmentCode(configuredDirectory, original)) + throw new IllegalStateException("HTTP client enrollment changed during replacement preparation"); configuredCredentialGeneration = HttpClientCredentialStore.snapshotActiveGeneration(configuredDirectory); - } catch (Exception failure) { - throw new IllegalStateException("Could not preserve the active HTTP client credential", failure); + restoreUnenrolledState = false; + } catch (IllegalStateException failure) { throw failure; } + catch (Exception failure) { + throw new IllegalStateException("Could not preserve the completed HTTP client enrollment", failure); } - flushForReplacement(connector, System.nanoTime() + TimeUnit.SECONDS.toNanos(SHUTDOWN_FLUSH_SECONDS)); - close(); } void flushForReplacement(HttpBackendTransportConnector connector, long deadlineNanos) { @@ -148,7 +223,7 @@ private void resumeAfterFailedFlush(HttpBackendTransportConnector active) { } private void initialize(Path directory, String serverId, String configuredCode, - GlobalMessageHandler messageHandler, boolean retryEnrollment) { + GlobalMessageHandler messageHandler, boolean retryEnrollment, boolean restoreUnenrolledState) { Path ownerKey = directory.toAbsolutePath().normalize(); Semaphore owner = DIRECTORY_OWNERS.computeIfAbsent(ownerKey, ignored -> new Semaphore(1)); boolean acquired = false, installed = false; @@ -158,26 +233,28 @@ private void initialize(Path directory, String serverId, String configuredCode, acquired = true; synchronized (lifecycle) { if (closed) { - if (credentialGenerationToRestore != null) + if (credentialGenerationToRestore != null || restoreUnenrolledState) credentialRestoreFailure = new IllegalStateException( - "Previous HTTP client credential restoration was cancelled"); + "Previous HTTP client credential state restoration was cancelled"); return; } } - if (credentialGenerationToRestore != null) { - try { + try { + if (restoreUnenrolledState) + restoreUnenrolledCredentialState(directory, serverId, configuredCode); + if (credentialGenerationToRestore != null) { HttpClientCredentialStore.restoreActiveGenerationAfterReplacement(directory, credentialGenerationToRestore); - } catch (Exception failure) { - credentialRestoreFailure = new IllegalStateException( - "Could not restore the previous HTTP client credential", failure); - throw failure; } + } catch (Exception failure) { + credentialRestoreFailure = new IllegalStateException( + "Could not restore the previous HTTP client credential state", failure); + throw failure; } credentialRestoreComplete.countDown(); - // A rollback always resumes a validated enrolled generation. Replaying the - // previous temporary code is both unnecessary and unsafe after a same-endpoint - // renewal/re-enrollment retained the newer active generation. + // An enrolled rollback resumes the validated generation without replaying its + // temporary code. An un-enrolled rollback reuses the original startup code + // only after the staged credential has been made inactive. HttpConnectionCode code = enrollmentCode(directory, serverId, credentialGenerationToRestore == null ? configuredCode : null); if (code != null && !enrollForStartup(code, serverId, directory, retryEnrollment, @@ -218,6 +295,30 @@ private void initialize(Path directory, String serverId, String configuredCode, } } + static void restoreUnenrolledCredentialState(Path directory, String serverId, String configuredCode) throws Exception { + Path root = directory.toAbsolutePath().normalize(); + if (configuredCode != null && !configuredCode.isBlank() + && HttpClientCredentialStore.hasEnrolledProfile(root)) { + try { + HttpConnectionCode original = HttpConnectionCode.parse(configuredCode); + if (original.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId)) + && HttpClientCredentialStore.matchesEnrollmentCode(root, original)) return; + } catch (Exception ignored) { + // The pre-replacement state was un-enrolled; never retain a credential + // that cannot be tied to its already-validated original code. + } + } + Path current = root.resolve("http-transport-client-current").normalize(); + if (!current.getParent().equals(root) || Files.isSymbolicLink(current)) + throw new java.io.IOException("HTTP client credential pointer is unsafe"); + if (Files.exists(current, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(current, LinkOption.NOFOLLOW_LINKS)) + throw new java.io.IOException("HTTP client credential pointer is unsafe"); + DurableFiles.deleteIfExists(current); + if (HttpClientCredentialStore.hasEnrolledProfile(root)) + throw new java.io.IOException("HTTP client credential rollback did not restore the un-enrolled state"); + } + private boolean waitForEnrollmentRetry(long delayMillis) { if (closed) return false; try { @@ -286,7 +387,7 @@ void dispatchIncoming(GlobalMessageHandler messageHandler, JsonEnvelope envelope } void awaitCredentialRestoration(long deadlineNanos) { - if (credentialGenerationToRestore == null) return; + if (credentialGenerationToRestore == null && !restoreUnenrolledState) return; try { long remaining = deadlineNanos - System.nanoTime(); if (remaining <= 0L || !credentialRestoreComplete.await(remaining, TimeUnit.NANOSECONDS)) @@ -377,13 +478,21 @@ static HttpConnectionCode enrollmentCode(Path directory, String serverId, String @Override public void close() { + close(true); + } + + private void closeForReplacement() { + close(false); + } + + private void close(boolean discardQueuedMessages) { Thread setup; HttpBackendTransportConnector active; Semaphore owner; synchronized (lifecycle) { if (closed) return; closed = true; - startupQueue.clear(); + if (discardQueuedMessages) startupQueue.clear(); setup = worker; worker = null; active = connector; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index f7ca8b3ed..91906a8ab 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -177,6 +178,121 @@ void ordinaryStartupRetriesTransientInitialEnrollmentWithBoundedBackoff() throws assertEquals(List.of(1_000L), delays); } + @Test + void replacementCancelsRetryingInitialEnrollmentWithoutCredential() throws Exception { + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(mock(VotingPluginMain.class)); + Path credentials = directory.resolve("http"); + java.lang.reflect.Field directoryField = HttpBackendProxyTransport.class.getDeclaredField("configuredDirectory"); + directoryField.setAccessible(true); + directoryField.set(transport, credentials); + java.lang.reflect.Field ownersField = HttpBackendProxyTransport.class.getDeclaredField("DIRECTORY_OWNERS"); + ownersField.setAccessible(true); + @SuppressWarnings("unchecked") + var owners = (java.util.concurrent.ConcurrentHashMap) ownersField.get(null); + Path ownerKey = credentials.toAbsolutePath().normalize(); + java.util.concurrent.Semaphore owner = new java.util.concurrent.Semaphore(0); + owners.put(ownerKey, owner); + CountDownLatch workerStarted = new CountDownLatch(1); + Thread retryingEnrollment = new Thread(() -> { + workerStarted.countDown(); + try { Thread.sleep(Long.MAX_VALUE); } + catch (InterruptedException expected) { Thread.currentThread().interrupt(); } + finally { owner.release(); } + }); + java.lang.reflect.Field workerField = HttpBackendProxyTransport.class.getDeclaredField("worker"); + workerField.setAccessible(true); + workerField.set(transport, retryingEnrollment); + retryingEnrollment.start(); + assertTrue(workerStarted.await(1, TimeUnit.SECONDS)); + JsonEnvelope queued = JsonEnvelope.builder("queued-before-replacement").build(); + transport.send(queued); + + assertDoesNotThrow(transport::prepareForReplacement, + "a staged Control replacement must cancel a first-time retry loop without an active credential"); + java.lang.reflect.Field restoreAbsent = + HttpBackendProxyTransport.class.getDeclaredField("restoreUnenrolledState"); + restoreAbsent.setAccessible(true); + assertTrue(restoreAbsent.getBoolean(transport), "rollback must remember that no credential existed"); + assertEquals(List.of(queued), transport.drainPreparedMessages(), + "messages accepted before replacement must remain available for handoff"); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (owner.availablePermits() == 0 && System.nanoTime() < deadline) Thread.onSpinWait(); + assertEquals(1, owner.availablePermits(), + "cancelled enrollment must release the credential-directory ownership"); + owners.remove(ownerKey, owner); + } + + @Test + void replacementSnapshotsInitialEnrollmentPublishedWhileCancellationCompletes() throws Exception { + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(mock(VotingPluginMain.class)); + Path credentials = directory.resolve("http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("original-proxy"), + "proxy.example.test"); + HttpConnectionCode original = new HttpConnectionCode("lobby-1", URI.create("https://proxy.example.test:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "G".repeat(43)); + setField(transport, "configuredDirectory", credentials); + setField(transport, "configuredServerId", "lobby-1"); + setField(transport, "configuredConnectionCode", original.encode()); + CountDownLatch workerStarted = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference failure = new java.util.concurrent.atomic.AtomicReference<>(); + Thread finishingEnrollment = new Thread(() -> { + workerStarted.countDown(); + try { + Thread.sleep(Long.MAX_VALUE); + } catch (InterruptedException expected) { + try { + HttpClientCredentialStore.saveEnrolled(credentials, original, + identity.issueClientCertificate("lobby-1")); + } catch (Throwable thrown) { failure.set(thrown); } + } + }); + setField(transport, "worker", finishingEnrollment); + finishingEnrollment.start(); + assertTrue(workerStarted.await(1, TimeUnit.SECONDS)); + + transport.prepareForReplacement(); + + assertNull(failure.get()); + assertNotNull(field(transport, "configuredCredentialGeneration"), + "a credential published while cancellation finishes must be retained for rollback"); + assertFalse((boolean) field(transport, "restoreUnenrolledState")); + } + + @Test + void rollbackRestoresAnExplicitlyUnenrolledCredentialState() throws Exception { + Path credentials = directory.resolve("http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("replacement-proxy"), + "proxy.example.test"); + HttpConnectionCode stagedCode = new HttpConnectionCode("lobby-1", + URI.create("https://proxy.example.test:1297/"), identity.serverCertificatePin(), + identity.caCertificatePin(), Instant.now().plusSeconds(60), "E".repeat(43)); + HttpClientCredentialStore.saveEnrolled(credentials, stagedCode, + identity.issueClientCertificate("lobby-1")); + assertTrue(HttpClientCredentialStore.hasEnrolledProfile(credentials)); + + HttpConnectionCode original = code("lobby-1", Instant.now().plusSeconds(60)); + HttpBackendProxyTransport.restoreUnenrolledCredentialState(credentials, "lobby-1", original.encode()); + + assertFalse(HttpClientCredentialStore.hasEnrolledProfile(credentials), + "a failed staged first enrollment must not remain active during rollback"); + } + + @Test + void rollbackRetainsEnrollmentIssuedForTheOriginalOneTimeCode() throws Exception { + Path credentials = directory.resolve("http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("same-code-proxy"), + "proxy.example.test"); + HttpConnectionCode original = new HttpConnectionCode("lobby-1", URI.create("https://proxy.example.test:1297/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "F".repeat(43)); + HttpClientCredentialStore.saveEnrolled(credentials, original, identity.issueClientCertificate("lobby-1")); + + HttpBackendProxyTransport.restoreUnenrolledCredentialState(credentials, "lobby-1", original.encode()); + + assertTrue(HttpClientCredentialStore.hasEnrolledProfile(credentials), + "a credential issued for the consumed original code must remain recoverable"); + } + @Test void stagedEnrollmentRemainsFailFast() throws Exception { VotingPluginMain plugin = mock(VotingPluginMain.class); @@ -316,4 +432,16 @@ private static HttpConnectionCode code(String serverId, Instant expiry) { return new HttpConnectionCode(serverId, URI.create("https://proxy.example.test:1297/"), "a".repeat(64), "b".repeat(64), expiry, "A".repeat(43)); } + + private static void setField(Object target, String name, Object value) throws Exception { + java.lang.reflect.Field field = target.getClass().getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } + + private static Object field(Object target, String name) throws Exception { + java.lang.reflect.Field field = target.getClass().getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } } From bc36371e25ea5789c4e67ce881ddd15d199f2fee Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 8 Sep 2026 08:30:37 -0600 Subject: [PATCH 048/136] Defer HTTP callbacks until handler publication --- .../votingplugin/VotingPluginMain.java | 14 ++++++- .../transport/HttpBackendProxyTransport.java | 27 +++++++++--- ...PluginMainBackendProxyPublicationTest.java | 41 +++++++++++++++++++ .../HttpBackendProxyTransportTest.java | 11 +++++ 4 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 107540fa6..5f65781cf 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1311,8 +1311,7 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { } if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); if (restart.previous != null) restart.previous.completeHttpHandoff(restart.replacement); - restart.replacement.activatePresenceReporting(); - backendProxyHandler = restart.replacement; + publishBackendProxyHandler(restart.previous, restart.replacement); if (restart.previous != null) restart.previous.close(); restart.finished = true; restart.published = true; @@ -1324,6 +1323,17 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { } } + /** Publishes the handler before opening any transport callback or presence gate. */ + void publishBackendProxyHandler(BackendProxyHandler previous, BackendProxyHandler replacement) { + backendProxyHandler = replacement; + try { + replacement.activatePresenceReporting(); + } catch (RuntimeException activationFailure) { + backendProxyHandler = previous; + throw activationFailure; + } + } + /** Returns false once publication committed and can no longer be rolled back as a failed apply. */ public synchronized boolean requestBackendProxyHandlerRestartAbandonment(BackendProxyRestart restart) { if (restart == null) return true; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 5f7e262ee..55bbf2a1c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -51,6 +51,7 @@ public final class HttpBackendProxyTransport implements BackendProxyTransport { private HttpClientCredentialStore.ActiveCredentialGeneration credentialGenerationToRestore; private boolean retryInitialization; private boolean restoreUnenrolledState; + private boolean inboundActive; private Semaphore directoryOwner; private final java.util.concurrent.atomic.AtomicBoolean queueWarning = new java.util.concurrent.atomic.AtomicBoolean(); @@ -68,24 +69,25 @@ public void start(GlobalMessageHandler messageHandler, boolean retryInitializati Path directory = plugin.getDataFolder().toPath().resolve("http"); String serverId = plugin.getBungeeSettings().getServer(); String connectionCode = plugin.getBungeeSettings().getHttpConnectionCode(); - start(directory, serverId, connectionCode, messageHandler, null, retryInitialization, false); + start(directory, serverId, connectionCode, messageHandler, null, retryInitialization, false, + retryInitialization); } private void start(Path directory, String serverId, String connectionCode, GlobalMessageHandler messageHandler) { - start(directory, serverId, connectionCode, messageHandler, null, true, false); + start(directory, serverId, connectionCode, messageHandler, null, true, false, true); } private void start(Path directory, String serverId, String connectionCode, GlobalMessageHandler messageHandler, HttpClientCredentialStore.ActiveCredentialGeneration generationToRestore) { - start(directory, serverId, connectionCode, messageHandler, generationToRestore, false, false); + start(directory, serverId, connectionCode, messageHandler, generationToRestore, false, false, true); } private void start(Path directory, String serverId, String connectionCode, GlobalMessageHandler messageHandler, HttpClientCredentialStore.ActiveCredentialGeneration generationToRestore, - boolean retryInitialization, boolean restoreUnenrolledState) { + boolean retryInitialization, boolean restoreUnenrolledState, boolean inboundActive) { if (generationToRestore == null && !restoreUnenrolledState) validateConfiguration(directory, serverId, connectionCode); else HttpTlsIdentity.canonicalServerId(serverId); @@ -96,6 +98,7 @@ private void start(Path directory, String serverId, String connectionCode, credentialGenerationToRestore = generationToRestore; this.retryInitialization = retryInitialization; this.restoreUnenrolledState = restoreUnenrolledState; + this.inboundActive = inboundActive; started = true; worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler, retryInitialization, restoreUnenrolledState), @@ -112,7 +115,7 @@ HttpBackendProxyTransport recreatePrepared() { } restored.start(configuredDirectory, configuredServerId, configuredConnectionCode, configuredMessageHandler, configuredCredentialGeneration, configuredCredentialGeneration == null && retryInitialization, - restoreUnenrolledState); + restoreUnenrolledState, true); return restored; } @@ -265,7 +268,7 @@ private void initialize(Path directory, String serverId, String configuredCode, replacement = new HttpBackendTransportConnector(directory, envelope -> dispatchIncoming(messageHandler, envelope, System.nanoTime() + TimeUnit.SECONDS.toNanos(INCOMING_DISPATCH_SECONDS))); - replacement.start(); + replacement.startPaused(); boolean discard = false; synchronized (lifecycle) { if (closed) { @@ -279,6 +282,7 @@ private void initialize(Path directory, String serverId, String configuredCode, connector = replacement; directoryOwner = owner; installed = true; + if (inboundActive) replacement.activateIncoming(); } } if (discard) replacement.close(); @@ -295,6 +299,17 @@ private void initialize(Path directory, String serverId, String configuredCode, } } + @Override + public void activateAfterPublication() { + HttpBackendTransportConnector active; + synchronized (lifecycle) { + if (closed) return; + inboundActive = true; + active = connector; + } + if (active != null) active.activateIncoming(); + } + static void restoreUnenrolledCredentialState(Path directory, String serverId, String configuredCode) throws Exception { Path root = directory.toAbsolutePath().normalize(); if (configuredCode != null && !configuredCode.isBlank() diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java new file mode 100644 index 000000000..d2140c5af --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -0,0 +1,41 @@ +package com.bencodez.votingplugin; + +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; + +import org.junit.jupiter.api.Test; + +import com.bencodez.votingplugin.backendproxy.BackendProxyHandler; + +class VotingPluginMainBackendProxyPublicationTest { + @Test + void publishesReplacementBeforeOpeningItsTransportGate() { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + doAnswer(invocation -> { + assertSame(replacement, plugin.getBackendProxyHandler(), + "inbound callbacks must not open before the replacement is published"); + return null; + }).when(replacement).activatePresenceReporting(); + + plugin.publishBackendProxyHandler(null, replacement); + + assertSame(replacement, plugin.getBackendProxyHandler()); + } + + @Test + void restoresPreviousHandlerWhenActivationFails() { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + doAnswer(invocation -> { throw new IllegalStateException("activation failed"); }) + .when(replacement).activatePresenceReporting(); + + org.junit.jupiter.api.Assertions.assertThrows(IllegalStateException.class, + () -> plugin.publishBackendProxyHandler(previous, replacement)); + + assertSame(previous, plugin.getBackendProxyHandler()); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 91906a8ab..bef3b2645 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -69,6 +69,17 @@ void dispatchesIncomingMessagesOnTheServerSchedulerAndWaitsForCompletion() throw assertEquals("test-server-thread", callbackThread.get()); } + @Test + void publicationActivatesTheSimpleApiInboundBarrier() throws Exception { + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(mock(VotingPluginMain.class)); + HttpBackendTransportConnector connector = mock(HttpBackendTransportConnector.class); + setField(transport, "connector", connector); + + transport.activateAfterPublication(); + + verify(connector).activateIncoming(); + } + @Test void timedOutScheduledMessageCannotExecuteLater() { VotingPluginMain plugin = mock(VotingPluginMain.class); From d3fc00ea1e43e0abba4e3fd77d283cfd6a08a893 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:47:42 -0600 Subject: [PATCH 049/136] Make backend transport replacement failure-atomic --- .../votingplugin/VotingPluginMain.java | 20 +++- .../backendproxy/BackendProxyHandler.java | 12 +- .../BackendProxyTransportManager.java | 81 ++++++++++++- .../transport/HttpBackendProxyTransport.java | 85 +++++++++++++- .../transport/RedisBackendProxyTransport.java | 74 +++++++++++- ...PluginMainBackendProxyPublicationTest.java | 54 +++++++++ .../BackendProxyHandlerLifecycleTest.java | 60 ++++++++++ .../HttpBackendProxyTransportTest.java | 53 +++++++++ .../RedisBackendProxyTransportTest.java | 110 ++++++++++++++++++ 9 files changed, 531 insertions(+), 18 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 5f65781cf..d46d119e5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1284,9 +1284,13 @@ public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long validationDeadlineNanos) { if (restart == null) throw new IllegalArgumentException("Backend proxy restart is required"); if (restart.previousRequiresPreparation && !restart.previousPrepared) { + // Preparation can close a retrying enrollment transport before a bounded + // worker join fails. Mark the restart first so abort/await still owns the + // restoration path after a partially completed preparation. + restart.previousPrepared = true; + restart.replacement.beginPreparedHttpHandoff(); if (!restart.previous.prepareForReplacement(restart.replacement.getMethod())) throw new IllegalStateException("Previous HTTP proxy transport could not be prepared for replacement"); - restart.previousPrepared = true; } if (restart.replacement != null) restart.replacement.validateTransport(validationDeadlineNanos); } @@ -1309,9 +1313,19 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { restart.published = true; return; } - if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); - if (restart.previous != null) restart.previous.completeHttpHandoff(restart.replacement); publishBackendProxyHandler(restart.previous, restart.replacement); + try { + if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); + } catch (RuntimeException handoffFailure) { + // Redis promotion fails before the old listener is retired. Restore the + // published pointer so abort can close only the staged replacement. + backendProxyHandler = restart.previous; + throw handoffFailure; + } + // Keep the prepared HTTP queue owned by the previous handler until every + // fallible publication step succeeds. That makes activation rollback + // reversible even when an in-flight sender reaches the previous handler. + if (restart.previous != null) restart.previous.completeHttpHandoff(restart.replacement); if (restart.previous != null) restart.previous.close(); restart.finished = true; restart.published = true; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 119f3c215..ea528a0b9 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -99,11 +99,14 @@ public void sendMessage(JsonEnvelope envelope) { /** Starts presence only after a staged handler reaches the atomic publication boundary. */ public void activatePresenceReporting() { - transportManager.activateAfterPublication(); if (presenceManager != null && !presenceReportingActivated) { presenceManager.start(); presenceReportingActivated = true; } + // Presence startup can throw while scheduling its heartbeat. Keep inbound + // HTTP callbacks behind the publication barrier until every fallible part of + // the replacement is active, so rollback cannot race a queued callback. + transportManager.activateAfterPublication(); } /** @@ -130,6 +133,10 @@ public boolean prepareForReplacement(BungeeMethod replacementMethod) { return false; } + public void beginPreparedHttpHandoff() { + transportManager.beginPreparedHttpHandoff(); + } + /** Restores a prepared HTTP transport when its replacement fails validation. */ public void restoreAfterFailedReplacement() { transportManager.restoreAfterFailedReplacement(); @@ -155,8 +162,7 @@ public void validateTransport(long deadlineNanos) { /** Completes the no-loss/no-duplicate same-Redis subscriber handoff after validation. */ public void completeRedisHandoff(BackendProxyHandler replacement) { if (method != BungeeMethod.REDIS || replacement.method != BungeeMethod.REDIS) return; - transportManager.closeRedisForHandoff(); - replacement.transportManager.activateRedisAfterHandoff(); + transportManager.completeRedisHandoff(replacement.transportManager); } /** Forwards messages buffered while the previous HTTP credentials were released. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index dafa994db..4bb6b6211 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -21,6 +21,7 @@ public class BackendProxyTransportManager { private final ProcessedVoteCache processedVoteCache; private BackendProxyTransport transport; private BackendProxyTransport preparedTransport; + private BackendProxyTransport retiredTransport; private BackendProxyTransportManager forwardingManager; private final java.util.ArrayDeque preparedSends = new java.util.ArrayDeque<>(); private boolean preparedQueueWarning; @@ -93,6 +94,20 @@ public synchronized void close() { preparedTransport.close(); preparedTransport = null; } + if (retiredTransport != null) { + BackendProxyTransport retired = retiredTransport; + try { + retired.close(); + retiredTransport = null; + } catch (RuntimeException cleanupFailure) { + // This transport has already been fenced and replaced. Keep its handle + // for another cleanup attempt, but never fail or close the live replacement. + if (plugin != null) { + plugin.getLogger().warning("Retired backend proxy transport did not stop cleanly"); + plugin.debug(cleanupFailure); + } + } + } if (forwardingManager == null) preparedSends.clear(); } @@ -108,19 +123,51 @@ public void validate(long deadlineNanos) { public synchronized void prepareForReplacement() { if (transport != null) { - transport.prepareForReplacement(); preparedTransport = transport; transport = null; + try { + preparedTransport.prepareForReplacement(); + } catch (RuntimeException failure) { + HttpBackendProxyTransport http = preparedTransport instanceof HttpBackendProxyTransport candidate + ? candidate : null; + if (http != null && !http.isClosedForReplacement()) { + // A failed flush deliberately restarts the existing connector. Reinstall + // that live instance instead of creating a second directory owner. + transport = preparedTransport; + preparedTransport = null; + } else { + // Enrollment cancellation may already have closed this instance. Restore + // from its captured configuration before configuration rollback. + try { + restorePreparedTransport(); + } catch (RuntimeException restorationFailure) { + failure.addSuppressed(restorationFailure); + } + } + throw failure; + } } } public synchronized void completePreparedTransportHandoff(BackendProxyTransportManager replacement) { if (preparedTransport == null) return; forwardingManager = java.util.Objects.requireNonNull(replacement, "replacement"); + java.util.ArrayList pending = new java.util.ArrayList<>(); if (preparedTransport instanceof HttpBackendProxyTransport http) { - for (JsonEnvelope envelope : http.drainPreparedMessages()) forwardingManager.send(envelope); + pending.addAll(http.drainPreparedMessages()); } - while (!preparedSends.isEmpty()) forwardingManager.send(preparedSends.removeFirst()); + while (!preparedSends.isEmpty()) pending.add(preparedSends.removeFirst()); + if (forwardingManager.transport instanceof HttpBackendProxyTransport http) { + http.acceptHandoffMessages(pending); + } else { + for (JsonEnvelope envelope : pending) forwardingManager.send(envelope); + } + } + + public void beginPreparedHttpHandoff() { + if (!(transport instanceof HttpBackendProxyTransport http)) + throw new IllegalStateException("HTTP replacement transport is unavailable"); + http.beginPreparedHandoff(); } public synchronized void restorePreparedTransport() { @@ -147,12 +194,20 @@ public void awaitPreparedTransportRestoration(long deadlineNanos) { if (transport instanceof HttpBackendProxyTransport http) http.awaitCredentialRestoration(deadlineNanos); } - public void closeRedisForHandoff() { + public synchronized void closeRedisForHandoff() { if (!(transport instanceof RedisBackendProxyTransport)) { throw new IllegalStateException("Redis backend proxy transport is unavailable"); } - ((RedisBackendProxyTransport) transport).closeForHandoff(); + retiredTransport = transport; transport = null; + try { + ((RedisBackendProxyTransport) retiredTransport).closeForHandoff(); + retiredTransport = null; + } catch (RuntimeException failure) { + // Retain the fenced old listener so a later manager close can retry its + // cleanup without ever touching the promoted replacement. + throw failure; + } } public void activateRedisAfterHandoff() { @@ -162,6 +217,22 @@ public void activateRedisAfterHandoff() { ((RedisBackendProxyTransport) transport).activateAfterHandoff(); } + /** Promotes the validated standby before retiring the old Redis listener. */ + public void completeRedisHandoff(BackendProxyTransportManager replacement) { + java.util.Objects.requireNonNull(replacement, "replacement").activateRedisAfterHandoff(); + try { + closeRedisForHandoff(); + } catch (RuntimeException retirementFailure) { + // The replacement is already active and usable. Treat failure to join the + // closed old listener as cleanup degradation, not a reason to tear down + // the newly promoted subscriber and leave the backend disconnected. + if (plugin != null) { + plugin.getLogger().warning("Previous Redis backend listener did not stop cleanly after handoff"); + plugin.debug(retirementFailure); + } + } + } + public ClientHandler getClientHandler() { return transport instanceof SocketBackendProxyTransport ? ((SocketBackendProxyTransport) transport).getClientHandler() : null; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 55bbf2a1c..6c71f6117 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -24,6 +24,8 @@ /** Backend adapter for the secure outbound-only HTTP proxy transport. */ public final class HttpBackendProxyTransport implements BackendProxyTransport { private static final int MAX_STARTUP_QUEUE = 1024; + private static final int MAX_PREPUBLICATION_QUEUE = 2048; + private static final int MAX_HANDOFF_QUEUE = 4096; private static final long DEFAULT_STARTUP_VALIDATION_SECONDS = 25L; private static final long ENROLLMENT_RETRY_INITIAL_MILLIS = 1_000L; private static final long ENROLLMENT_RETRY_MAX_MILLIS = 60_000L; @@ -35,6 +37,9 @@ public final class HttpBackendProxyTransport implements BackendProxyTransport { private final CountDownLatch startupComplete = new CountDownLatch(1); private final CountDownLatch credentialRestoreComplete = new CountDownLatch(1); private final ArrayDeque startupQueue = new ArrayDeque<>(); + private final ArrayDeque handoffQueue = new ArrayDeque<>(); + private volatile Thread handoffWorker; + private boolean awaitingPreparedHandoff; private volatile HttpBackendTransportConnector connector; private volatile Thread worker; private volatile RuntimeException startupFailure; @@ -119,6 +124,10 @@ HttpBackendProxyTransport recreatePrepared() { return restored; } + boolean isClosedForReplacement() { + return closed; + } + java.util.List drainPreparedMessages() { synchronized (lifecycle) { java.util.List pending = java.util.List.copyOf(startupQueue); @@ -419,12 +428,17 @@ void awaitCredentialRestoration(long deadlineNanos) { public void send(JsonEnvelope envelope) { synchronized (lifecycle) { if (closed) return; + if (awaitingPreparedHandoff || !handoffQueue.isEmpty()) { + int capacity = awaitingPreparedHandoff ? MAX_PREPUBLICATION_QUEUE : MAX_HANDOFF_QUEUE; + if (handoffQueue.size() < capacity) handoffQueue.addLast(envelope); + else warnRejectedSend(); + return; + } HttpBackendTransportConnector active = connector; if (active != null) { if (!active.send(envelope)) { if (restartAfterFailedFlush && startupQueue.size() < MAX_STARTUP_QUEUE) startupQueue.addLast(envelope); - else if (queueWarning.compareAndSet(false, true)) - plugin.getLogger().severe("Secure HTTP transport queue is full or rejected an oversized message; delivery was not accepted"); + else warnRejectedSend(); } } else if (startupQueue.size() < MAX_STARTUP_QUEUE) { startupQueue.addLast(envelope); @@ -432,6 +446,67 @@ else if (queueWarning.compareAndSet(false, true)) } } + void beginPreparedHandoff() { + synchronized (lifecycle) { + if (closed) throw new IllegalStateException("HTTP replacement transport is closed"); + awaitingPreparedHandoff = true; + } + } + + void acceptHandoffMessages(java.util.List messages) { + Thread drain; + synchronized (lifecycle) { + if (closed) throw new IllegalStateException("HTTP replacement transport is closed"); + if (messages.size() > MAX_HANDOFF_QUEUE - handoffQueue.size()) + throw new IllegalStateException("HTTP handoff queue exceeded its fixed capacity"); + java.util.ArrayDeque newer = new java.util.ArrayDeque<>(handoffQueue); + handoffQueue.clear(); + handoffQueue.addAll(messages); + handoffQueue.addAll(newer); + awaitingPreparedHandoff = false; + if (handoffQueue.isEmpty()) return; + if (handoffWorker != null) return; + drain = new Thread(this::drainHandoffMessages, "VotingPlugin-HTTP-Backend-Handoff"); + drain.setDaemon(true); + handoffWorker = drain; + } + drain.start(); + } + + private void drainHandoffMessages() { + try { + while (!Thread.currentThread().isInterrupted()) { + boolean delivered = false; + synchronized (lifecycle) { + if (closed || handoffQueue.isEmpty()) return; + HttpBackendTransportConnector active = connector; + if (active != null && active.send(handoffQueue.peekFirst())) { + handoffQueue.removeFirst(); + delivered = true; + } + } + if (!delivered) TimeUnit.MILLISECONDS.sleep(25L); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } finally { + synchronized (lifecycle) { + if (handoffWorker == Thread.currentThread()) handoffWorker = null; + } + } + } + + java.util.List handoffMessagesSnapshot() { + synchronized (lifecycle) { + return java.util.List.copyOf(handoffQueue); + } + } + + private void warnRejectedSend() { + if (queueWarning.compareAndSet(false, true)) + plugin.getLogger().severe("Secure HTTP transport queue is full or rejected an oversized message; delivery was not accepted"); + } + @Override public void validate() { validate(System.nanoTime() + TimeUnit.SECONDS.toNanos(DEFAULT_STARTUP_VALIDATION_SECONDS)); @@ -502,14 +577,19 @@ private void closeForReplacement() { private void close(boolean discardQueuedMessages) { Thread setup; + Thread pendingHandoff; HttpBackendTransportConnector active; Semaphore owner; synchronized (lifecycle) { if (closed) return; closed = true; if (discardQueuedMessages) startupQueue.clear(); + if (discardQueuedMessages) handoffQueue.clear(); + awaitingPreparedHandoff = false; setup = worker; worker = null; + pendingHandoff = handoffWorker; + handoffWorker = null; active = connector; connector = null; owner = directoryOwner; @@ -517,6 +597,7 @@ private void close(boolean discardQueuedMessages) { } startupComplete.countDown(); if (setup != null) setup.interrupt(); + if (pendingHandoff != null) pendingHandoff.interrupt(); if (setup == null && active == null && owner == null) return; Thread cleanup = new Thread(() -> drain(setup, active, owner), "VotingPlugin-HTTP-Backend-Cleanup"); cleanup.setDaemon(true); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index eb2ce2539..60d12293e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -35,6 +35,9 @@ public class RedisBackendProxyTransport implements BackendProxyTransport { private final List bufferedLegacyDeliveries = new ArrayList<>(); private long bufferedLegacyDeliveryBytes; private boolean legacyHandoffDegraded; + private boolean retiredAfterHandoff; + private boolean replayingHandoff; + private final java.util.ArrayDeque deliveriesAfterReplay = new java.util.ArrayDeque<>(); private GlobalMessageHandler messageHandler; public RedisBackendProxyTransport(VotingPluginMain plugin) { @@ -49,6 +52,7 @@ public RedisBackendProxyTransport(VotingPluginMain plugin, ProcessedVoteCache pr @Override public void start(GlobalMessageHandler messageHandler) { this.messageHandler = messageHandler; + retiredAfterHandoff = false; processedVoteCache.registerRedisSubscriber(subscriberIdentity); redisHandler = new RedisHandler(plugin.getBungeeSettings().getRedisHost(), plugin.getBungeeSettings().getRedisPort(), plugin.getBungeeSettings().getRedisUsername(), @@ -71,7 +75,7 @@ public void debug(String message) { JsonEnvelope envelope = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.decode(payload); String deliveryId = envelope.getFields().get(VotingPluginWire.K_REDIS_DELIVERY_ID); if (deliveryId != null) { - if (processedVoteCache.reserveRedisDelivery(deliveryId)) messageHandler.onMessage(envelope); + dispatchIdentified(envelope, deliveryId); } else { dispatchLegacy(envelope); } @@ -89,12 +93,27 @@ public void onSubscribe(String channel, int subscribedChannels) { listenerThread.start(); } + void dispatchIdentified(JsonEnvelope envelope, String deliveryId) { + boolean accepted; + synchronized (legacyLifecycle) { + if (retiredAfterHandoff) return; + accepted = processedVoteCache.reserveRedisDelivery(deliveryId); + if (accepted && replayingHandoff) { + deliveriesAfterReplay.addLast(envelope); + accepted = false; + } + } + if (accepted) messageHandler.onMessage(envelope); + } + void dispatchLegacy(JsonEnvelope envelope) { String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(envelope); int encodedBytes = ProcessedVoteCache.legacyRedisDeliveryBytes(signature); + boolean dispatch = false; synchronized (legacyLifecycle) { + if (retiredAfterHandoff) return; if (processedVoteCache.reserveLegacyRedisDelivery(subscriberIdentity, signature)) { - messageHandler.onMessage(envelope); + dispatch = true; } else if (encodedBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_DELIVERY_BYTES && bufferedLegacyDeliveries.size() < MAX_LEGACY_HANDOFF_DELIVERIES && bufferedLegacyDeliveryBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES - encodedBytes) { @@ -107,31 +126,76 @@ void dispatchLegacy(JsonEnvelope envelope) { + " byte buffer; temporarily degrading duplicate suppression"); } legacyHandoffDegraded = true; - messageHandler.onMessage(envelope); + dispatch = true; + } + if (dispatch && replayingHandoff) { + deliveriesAfterReplay.addLast(envelope); + dispatch = false; } } + if (dispatch) messageHandler.onMessage(envelope); } /** Promotes a validated standby after the previous listener has completely stopped. */ public void activateAfterHandoff() { + java.util.ArrayList replay = new java.util.ArrayList<>(); synchronized (legacyLifecycle) { + replayingHandoff = true; processedVoteCache.activateRedisSubscriber(subscriberIdentity); for (JsonEnvelope envelope : bufferedLegacyDeliveries) { - String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(envelope); - if (!processedVoteCache.consumeLegacyRedisDelivery(signature)) messageHandler.onMessage(envelope); + try { + String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(envelope); + if (!processedVoteCache.consumeLegacyRedisDelivery(signature)) replay.add(envelope); + } catch (RuntimeException replayFailure) { + if (plugin != null) plugin.debug("Redis handoff replay failed: " + replayFailure.getMessage()); + } } bufferedLegacyDeliveries.clear(); bufferedLegacyDeliveryBytes = 0; legacyHandoffDegraded = false; processedVoteCache.finishRedisHandoff(); } + dispatchReplayBatch(replay); + while (true) { + synchronized (legacyLifecycle) { + if (deliveriesAfterReplay.isEmpty()) { + replayingHandoff = false; + break; + } + replay = new java.util.ArrayList<>(deliveriesAfterReplay); + deliveriesAfterReplay.clear(); + } + dispatchReplayBatch(replay); + } + } + + private void dispatchReplayBatch(java.util.List replay) { + for (JsonEnvelope envelope : replay) { + try { + messageHandler.onMessage(envelope); + } catch (RuntimeException replayFailure) { + if (plugin != null) plugin.debug("Redis handoff replay failed: " + replayFailure.getMessage()); + } + } } /** Stops the old listener while retaining its overlap accounting for standby promotion. */ public void closeForHandoff() { + fenceAfterHandoff(); closeListener(false); } + /** Prevents a listener that misses its shutdown deadline from dispatching duplicates. */ + void fenceAfterHandoff() { + synchronized (legacyLifecycle) { + retiredAfterHandoff = true; + replayingHandoff = false; + deliveriesAfterReplay.clear(); + bufferedLegacyDeliveries.clear(); + bufferedLegacyDeliveryBytes = 0; + } + } + @Override public void send(JsonEnvelope envelope) { if (redisHandler != null) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index d2140c5af..86fe9b9f5 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -4,6 +4,11 @@ import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; import org.junit.jupiter.api.Test; @@ -38,4 +43,53 @@ void restoresPreviousHandlerWhenActivationFails() { assertSame(previous, plugin.getBackendProxyHandler()); } + + @Test + void keepsPreparedHttpQueueWithPreviousHandlerUntilPublicationSucceeds() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + doAnswer(invocation -> { throw new IllegalStateException("activation failed"); }) + .when(replacement).activatePresenceReporting(); + VotingPluginMain.BackendProxyRestart restart = restart(previous, replacement); + + org.junit.jupiter.api.Assertions.assertThrows(IllegalStateException.class, + () -> plugin.completeBackendProxyHandlerRestart(restart)); + + verify(previous, never()).completeHttpHandoff(replacement); + verify(previous, never()).completeRedisHandoff(replacement); + assertSame(previous, plugin.getBackendProxyHandler()); + } + + @Test + void restoresPreviousHandlerWhenRedisPromotionFails() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + doAnswer(invocation -> { throw new IllegalStateException("promotion failed"); }) + .when(previous).completeRedisHandoff(replacement); + + org.junit.jupiter.api.Assertions.assertThrows(IllegalStateException.class, + () -> plugin.completeBackendProxyHandlerRestart(restart(previous, replacement))); + + assertSame(previous, plugin.getBackendProxyHandler()); + verify(previous, never()).completeHttpHandoff(replacement); + verify(previous, never()).close(); + } + + private VotingPluginMain.BackendProxyRestart restart(BackendProxyHandler previous, + BackendProxyHandler replacement) throws Exception { + Constructor constructor = VotingPluginMain.BackendProxyRestart.class + .getDeclaredConstructor(BackendProxyHandler.class, BackendProxyHandler.class, boolean.class, boolean.class); + constructor.setAccessible(true); + return constructor.newInstance(previous, replacement, false, true); + } + + private void setBackendProxyHandler(VotingPluginMain plugin, BackendProxyHandler handler) throws Exception { + Field field = VotingPluginMain.class.getDeclaredField("backendProxyHandler"); + field.setAccessible(true); + field.set(plugin, handler); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 60c476cc4..c58676f17 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -1,12 +1,15 @@ package com.bencodez.votingplugin.backendproxy; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -187,6 +190,29 @@ void stagedPresenceStartsOnlyAtExplicitPublication() throws Exception { verify(presence, times(1)).start(); } + @Test + void failedPresenceStartKeepsTransportBehindPublicationBarrier() throws Exception { + BackendProxyHandler handler = new BackendProxyHandler(null); + BackendPresenceManager presence = mock(BackendPresenceManager.class); + org.mockito.Mockito.doThrow(new java.util.concurrent.RejectedExecutionException()).when(presence).start(); + Field presenceField = BackendProxyHandler.class.getDeclaredField("presenceManager"); + presenceField.setAccessible(true); + presenceField.set(handler, presence); + + Field managerField = BackendProxyHandler.class.getDeclaredField("transportManager"); + managerField.setAccessible(true); + BackendProxyTransportManager manager = (BackendProxyTransportManager) managerField.get(handler); + BackendProxyTransport transport = mock(BackendProxyTransport.class); + Field transportField = BackendProxyTransportManager.class.getDeclaredField("transport"); + transportField.setAccessible(true); + transportField.set(manager, transport); + + assertThrows(java.util.concurrent.RejectedExecutionException.class, handler::activatePresenceReporting); + + verify(presence).start(); + verify(transport, never()).activateAfterPublication(); + } + @Test void sharesVoteDeduplicationAcrossHandlerReplacement() { ProcessedVoteCache cache = new ProcessedVoteCache(); @@ -348,6 +374,34 @@ void releasesMysqlSubscriberBeforeSameMethodReplacement() throws Exception { assertNull(handler.getMessenger()); } + @Test + void keepsPromotedRedisReplacementWhenOldListenerRetirementFails() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + java.util.logging.Logger logger = mock(java.util.logging.Logger.class); + when(plugin.getLogger()).thenReturn(logger); + BackendProxyTransportManager previous = new BackendProxyTransportManager(plugin); + BackendProxyTransportManager replacement = new BackendProxyTransportManager(plugin); + RedisBackendProxyTransport oldTransport = mock(RedisBackendProxyTransport.class); + RedisBackendProxyTransport newTransport = mock(RedisBackendProxyTransport.class); + setField(previous, "transport", oldTransport); + setField(replacement, "transport", newTransport); + doThrow(new IllegalStateException("listener did not stop")).when(oldTransport).closeForHandoff(); + doThrow(new IllegalStateException("connection still closing")).doNothing().when(oldTransport).close(); + + previous.completeRedisHandoff(replacement); + + org.mockito.InOrder order = inOrder(newTransport, oldTransport); + order.verify(newTransport).activateAfterHandoff(); + order.verify(oldTransport).closeForHandoff(); + assertSame(newTransport, transport(replacement)); + assertNull(transport(previous)); + + assertDoesNotThrow(previous::close); + assertDoesNotThrow(previous::close); + + verify(oldTransport, times(2)).close(); + } + @Test void stopsGlobalDataTimerWhenHandlerIsReplaced() throws Exception { BackendGlobalDataSync handler = new BackendGlobalDataSync(null, null); @@ -375,6 +429,12 @@ private void setTransport(BackendProxyHandler handler, BackendProxyTransport tra setField(manager, "transport", transport); } + private BackendProxyTransport transport(BackendProxyTransportManager manager) throws Exception { + Field field = BackendProxyTransportManager.class.getDeclaredField("transport"); + field.setAccessible(true); + return (BackendProxyTransport) field.get(manager); + } + private void setField(Object target, String name, Object value) throws Exception { Field field = target.getClass().getDeclaredField(name); field.setAccessible(true); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index bef3b2645..7ab632142 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -80,6 +80,40 @@ void publicationActivatesTheSimpleApiInboundBarrier() throws Exception { verify(connector).activateIncoming(); } + @Test + void failedPreparationAfterCloseRestoresCapturedHttpTransport() throws Exception { + BackendProxyTransportManager manager = new BackendProxyTransportManager(mock(VotingPluginMain.class)); + HttpBackendProxyTransport failed = mock(HttpBackendProxyTransport.class); + HttpBackendProxyTransport restored = mock(HttpBackendProxyTransport.class); + JsonEnvelope queued = JsonEnvelope.builder("queued-after-failure").build(); + setField(manager, "transport", failed); + org.mockito.Mockito.doThrow(new IllegalStateException("worker still stopping")) + .when(failed).prepareForReplacement(); + when(failed.isClosedForReplacement()).thenReturn(true); + when(failed.recreatePrepared()).thenReturn(restored); + + assertThrows(IllegalStateException.class, manager::prepareForReplacement); + manager.send(queued); + + verify(restored).send(queued); + } + + @Test + void failedFlushKeepsTheRestartedHttpTransport() throws Exception { + BackendProxyTransportManager manager = new BackendProxyTransportManager(mock(VotingPluginMain.class)); + HttpBackendProxyTransport active = mock(HttpBackendProxyTransport.class); + JsonEnvelope queued = JsonEnvelope.builder("queued-after-flush-timeout").build(); + setField(manager, "transport", active); + org.mockito.Mockito.doThrow(new IllegalStateException("flush timed out")) + .when(active).prepareForReplacement(); + + assertThrows(IllegalStateException.class, manager::prepareForReplacement); + manager.send(queued); + + verify(active).send(queued); + verify(active, org.mockito.Mockito.never()).recreatePrepared(); + } + @Test void timedOutScheduledMessageCannotExecuteLater() { VotingPluginMain plugin = mock(VotingPluginMain.class); @@ -439,6 +473,25 @@ void validationWaitsForThePreviousDirectoryOwnerBeforeReadinessFailure() throws transport.close(); } + @Test + void preparedHandoffMessagesStayAheadOfNewlyPublishedSendsWhenConnectorIsFull() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(plugin); + HttpBackendTransportConnector connector = mock(HttpBackendTransportConnector.class); + when(connector.send(org.mockito.ArgumentMatchers.any(JsonEnvelope.class))).thenReturn(false); + setField(transport, "connector", connector); + JsonEnvelope acceptedBeforePublication = JsonEnvelope.builder("old").build(); + JsonEnvelope sentAfterPublication = JsonEnvelope.builder("new").build(); + + transport.beginPreparedHandoff(); + transport.send(sentAfterPublication); + transport.acceptHandoffMessages(List.of(acceptedBeforePublication)); + + assertEquals(List.of(acceptedBeforePublication, sentAfterPublication), transport.handoffMessagesSnapshot()); + transport.close(); + } + private static HttpConnectionCode code(String serverId, Instant expiry) { return new HttpConnectionCode(serverId, URI.create("https://proxy.example.test:1297/"), "a".repeat(64), "b".repeat(64), expiry, "A".repeat(43)); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java index fc97bb834..0f5dd5b10 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java @@ -4,10 +4,16 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; import java.lang.reflect.Field; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.CopyOnWriteArrayList; import org.junit.jupiter.api.Test; @@ -65,4 +71,108 @@ void legacyHandoffBufferDegradesForAnOversizedEnvelope() throws Exception { verify(messages).onMessage(envelope); } + + @Test + void retiredListenerDropsLegacyDeliveriesAfterShutdownTimeout() throws Exception { + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, new ProcessedVoteCache()); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + Field handler = RedisBackendProxyTransport.class.getDeclaredField("messageHandler"); + handler.setAccessible(true); + handler.set(transport, messages); + transport.fenceAfterHandoff(); + + transport.dispatchLegacy(JsonEnvelope.builder(VotingPluginWire.SUB_VOTE_UPDATE).build()); + + verifyNoInteractions(messages); + } + + @Test + void retiredListenerDoesNotReserveOrDispatchDeliveryIds() throws Exception { + ProcessedVoteCache cache = new ProcessedVoteCache(); + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, cache); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + Field handler = RedisBackendProxyTransport.class.getDeclaredField("messageHandler"); + handler.setAccessible(true); + handler.set(transport, messages); + transport.fenceAfterHandoff(); + String deliveryId = "00000000-0000-0000-0000-000000000001"; + + transport.dispatchIdentified(JsonEnvelope.builder(VotingPluginWire.SUB_VOTE_UPDATE).build(), deliveryId); + + assertTrue(cache.reserveRedisDelivery(deliveryId), "retired listener must not consume shared deduplication state"); + verifyNoInteractions(messages); + } + + @Test + void fencingDoesNotWaitForAnIdentifiedMessageHandler() throws Exception { + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, new ProcessedVoteCache()); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + doAnswer(invocation -> { + entered.countDown(); + try { + release.await(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + return null; + }).when(messages).onMessage(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); + Field handler = RedisBackendProxyTransport.class.getDeclaredField("messageHandler"); + handler.setAccessible(true); + handler.set(transport, messages); + Thread delivery = new Thread(() -> transport.dispatchIdentified( + JsonEnvelope.builder(VotingPluginWire.SUB_VOTE_UPDATE).build(), "delivery")); + delivery.start(); + assertTrue(entered.await(1, TimeUnit.SECONDS)); + + CountDownLatch fenced = new CountDownLatch(1); + Thread retirement = new Thread(() -> { + transport.fenceAfterHandoff(); + fenced.countDown(); + }); + retirement.start(); + try { + assertTrue(fenced.await(1, TimeUnit.SECONDS)); + } finally { + release.countDown(); + } + delivery.join(TimeUnit.SECONDS.toMillis(1)); + retirement.join(TimeUnit.SECONDS.toMillis(1)); + assertFalse(delivery.isAlive()); + assertFalse(retirement.isAlive()); + } + + @Test + @SuppressWarnings("unchecked") + void replayCompletesBeforeNewlyAdmittedDeliveries() throws Exception { + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.reserveRedisDelivery("new-delivery")).thenReturn(true); + when(cache.consumeLegacyRedisDelivery(org.mockito.ArgumentMatchers.anyString())).thenReturn(false); + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, cache); + JsonEnvelope replayed = JsonEnvelope.builder("old").build(); + JsonEnvelope newer = JsonEnvelope.builder("new").build(); + Field buffered = RedisBackendProxyTransport.class.getDeclaredField("bufferedLegacyDeliveries"); + buffered.setAccessible(true); + ((java.util.List) buffered.get(transport)).add(replayed); + CopyOnWriteArrayList order = new CopyOnWriteArrayList<>(); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + doAnswer(invocation -> { + JsonEnvelope envelope = invocation.getArgument(0); + order.add(envelope); + if (envelope == replayed) { + Thread arrival = new Thread(() -> transport.dispatchIdentified(newer, "new-delivery")); + arrival.start(); + arrival.join(TimeUnit.SECONDS.toMillis(1)); + } + return null; + }).when(messages).onMessage(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); + Field handler = RedisBackendProxyTransport.class.getDeclaredField("messageHandler"); + handler.setAccessible(true); + handler.set(transport, messages); + + transport.activateAfterHandoff(); + + assertEquals(java.util.List.of(replayed, newer), order); + } } From 59e05f19e4f5b628b258eec312eeeb68cdca5467 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:07:45 -0600 Subject: [PATCH 050/136] Preserve HTTP delivery across publication and shutdown --- .../transport/HttpBackendProxyTransport.java | 88 ++++++++++++++++--- .../HttpBackendProxyTransportTest.java | 26 +++++- 2 files changed, 103 insertions(+), 11 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 6c71f6117..7b4c020eb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -274,10 +274,10 @@ private void initialize(Path directory, String serverId, String configuredCode, HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); if (!enrolled.profile().serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); - replacement = new HttpBackendTransportConnector(directory, - envelope -> dispatchIncoming(messageHandler, envelope, - System.nanoTime() + TimeUnit.SECONDS.toNanos(INCOMING_DISPATCH_SECONDS))); - replacement.startPaused(); + replacement = new HttpBackendTransportConnector(directory, envelope -> { + dispatchAfterPublication(messageHandler, envelope); + }); + invokeConnectorLifecycle(replacement, "startPaused"); boolean discard = false; synchronized (lifecycle) { if (closed) { @@ -291,7 +291,7 @@ private void initialize(Path directory, String serverId, String configuredCode, connector = replacement; directoryOwner = owner; installed = true; - if (inboundActive) replacement.activateIncoming(); + if (inboundActive) invokeConnectorLifecycle(replacement, "activateIncoming"); } } if (discard) replacement.close(); @@ -313,10 +313,55 @@ public void activateAfterPublication() { HttpBackendTransportConnector active; synchronized (lifecycle) { if (closed) return; - inboundActive = true; active = connector; } - if (active != null) active.activateIncoming(); + if (active != null) invokeConnectorLifecycle(active, "activateIncoming"); + synchronized (lifecycle) { + if (closed) return; + inboundActive = true; + lifecycle.notifyAll(); + } + } + + private boolean awaitInboundPublication() { + synchronized (lifecycle) { + while (!inboundActive && !closed) { + try { + lifecycle.wait(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return false; + } + } + return !closed; + } + } + + void dispatchAfterPublication(GlobalMessageHandler messageHandler, JsonEnvelope envelope) { + if (!awaitInboundPublication()) + throw new IllegalStateException("HTTP transport closed before inbound publication"); + dispatchIncoming(messageHandler, envelope, + System.nanoTime() + TimeUnit.SECONDS.toNanos(INCOMING_DISPATCH_SECONDS)); + } + + private static void invokeConnectorLifecycle(HttpBackendTransportConnector connector, String method) { + try { + connector.getClass().getMethod(method).invoke(connector); + } catch (java.lang.reflect.InvocationTargetException failure) { + Throwable cause = failure.getCause(); + if (cause instanceof RuntimeException runtime) throw runtime; + if (cause instanceof Error error) throw error; + throw new IllegalStateException("HTTP connector " + method + " failed", cause); + } catch (NoSuchMethodException unavailable) { + // Older published SimpleAPI snapshots do not yet expose the publication + // barrier. The wrapper callback above provides the same fence until #79 is + // deployed, while newer versions use the native connector barrier. + if ("startPaused".equals(method)) connector.start(); + else if (!"activateIncoming".equals(method)) + throw new IllegalStateException("SimpleAPI HTTP connector does not support " + method, unavailable); + } catch (ReflectiveOperationException failure) { + throw new IllegalStateException("SimpleAPI HTTP connector does not support " + method, failure); + } } static void restoreUnenrolledCredentialState(Path directory, String serverId, String configuredCode) throws Exception { @@ -580,10 +625,13 @@ private void close(boolean discardQueuedMessages) { Thread pendingHandoff; HttpBackendTransportConnector active; Semaphore owner; + java.util.List finalHandoff; synchronized (lifecycle) { if (closed) return; closed = true; + lifecycle.notifyAll(); if (discardQueuedMessages) startupQueue.clear(); + finalHandoff = discardQueuedMessages ? java.util.List.copyOf(handoffQueue) : java.util.List.of(); if (discardQueuedMessages) handoffQueue.clear(); awaitingPreparedHandoff = false; setup = worker; @@ -599,19 +647,39 @@ private void close(boolean discardQueuedMessages) { if (setup != null) setup.interrupt(); if (pendingHandoff != null) pendingHandoff.interrupt(); if (setup == null && active == null && owner == null) return; - Thread cleanup = new Thread(() -> drain(setup, active, owner), "VotingPlugin-HTTP-Backend-Cleanup"); + Thread cleanup = new Thread(() -> drain(setup, active, owner, finalHandoff), + "VotingPlugin-HTTP-Backend-Cleanup"); cleanup.setDaemon(true); cleanup.start(); } - private static void drain(Thread setup, HttpBackendTransportConnector active, Semaphore owner) { + private static void drain(Thread setup, HttpBackendTransportConnector active, Semaphore owner, + java.util.List finalHandoff) { try { if (setup != null) try { setup.join(TimeUnit.SECONDS.toMillis(5)); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } if (active != null) { - active.flushOutgoing(System.nanoTime() + TimeUnit.SECONDS.toNanos(SHUTDOWN_FLUSH_SECONDS)); + flushHandoffForShutdown(active, finalHandoff, + System.nanoTime() + TimeUnit.SECONDS.toNanos(SHUTDOWN_FLUSH_SECONDS)); active.close(); } } finally { if (owner != null) owner.release(); } } + + static boolean flushHandoffForShutdown(HttpBackendTransportConnector active, + java.util.List finalHandoff, long deadlineNanos) { + for (JsonEnvelope envelope : finalHandoff) { + while (!active.send(envelope)) { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) return false; + try { + TimeUnit.NANOSECONDS.sleep(Math.min(remaining, TimeUnit.MILLISECONDS.toNanos(25L))); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return false; + } + } + } + return active.flushOutgoing(deadlineNanos); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 7ab632142..81d319b23 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -77,7 +77,31 @@ void publicationActivatesTheSimpleApiInboundBarrier() throws Exception { transport.activateAfterPublication(); - verify(connector).activateIncoming(); + assertTrue((boolean) field(transport, "inboundActive")); + } + + @Test + void closedPublicationGateRejectsDeliveryForReplay() { + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(mock(VotingPluginMain.class)); + transport.close(); + + assertThrows(IllegalStateException.class, () -> transport.dispatchAfterPublication( + mock(GlobalMessageHandler.class), JsonEnvelope.builder("test").build())); + } + + @Test + void orderlyShutdownFlushesHandoffAfterMakingConnectorCapacity() { + HttpBackendTransportConnector connector = mock(HttpBackendTransportConnector.class); + JsonEnvelope pending = JsonEnvelope.builder("backend-stopped").build(); + when(connector.send(pending)).thenReturn(false, true); + when(connector.flushOutgoing(org.mockito.ArgumentMatchers.anyLong())).thenReturn(true); + + assertTrue(HttpBackendProxyTransport.flushHandoffForShutdown(connector, List.of(pending), + System.nanoTime() + TimeUnit.SECONDS.toNanos(1))); + + org.mockito.InOrder order = org.mockito.Mockito.inOrder(connector); + order.verify(connector, org.mockito.Mockito.times(2)).send(pending); + order.verify(connector).flushOutgoing(org.mockito.ArgumentMatchers.anyLong()); } @Test From e3e577402b48588cfb1dfc7424c28026be4bfa4a Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:28:19 -0600 Subject: [PATCH 051/136] Preserve proxy delivery across handoff and restart --- .../cache/ProcessedVoteCache.java | 15 +- .../BackendProxyTransportManager.java | 17 +- .../transport/HttpBackendProxyTransport.java | 20 +- .../transport/RedisBackendProxyTransport.java | 125 +++++++- .../votingplugin/proxy/OfflineBungeeVote.java | 186 ++++++++++-- .../votingplugin/proxy/VotingPluginProxy.java | 280 ++++++++++++++---- .../proxy/bungee/BungeeJsonVoteCache.java | 19 +- .../cache/ProxyOnlineVoteCacheTable.java | 49 ++- .../proxy/cache/ProxyTimedVoteCacheTable.java | 49 ++- .../proxy/cache/ProxyVoteCacheTable.java | 52 +++- .../proxy/cache/VoteCacheHandler.java | 37 ++- .../proxy/velocity/VelocityJsonVoteCache.java | 6 + .../votingplugin/timequeue/VoteTimeQueue.java | 71 +++++ .../BackendProxyHandlerLifecycleTest.java | 23 +- .../HttpBackendProxyTransportTest.java | 38 +++ .../RedisBackendProxyTransportTest.java | 189 +++++++++++- .../tests/VoteCacheHandlerVoteIdTest.java | 43 +++ .../tests/VotingPluginProxyTest.java | 257 ++++++++++++++++ .../tests/VotingPluginProxyTestImpl.java | 57 +++- .../cache/ProcessedVoteCacheTest.java | 2 + 20 files changed, 1381 insertions(+), 154 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java index 919fb8077..c137bf948 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java @@ -25,6 +25,7 @@ public class ProcessedVoteCache { private final LinkedHashMap processedRedisDeliveries = new LinkedHashMap<>(); private final LinkedHashMap legacyRedisDeliveries = new LinkedHashMap<>(); private long legacyRedisDeliveryBytes; + private boolean legacyRedisHandoffOverflowed; private Object activeRedisSubscriber; private Object standbyRedisSubscriber; @@ -82,14 +83,18 @@ public synchronized boolean reserveRedisDelivery(String deliveryId) { return true; } - public synchronized void registerRedisSubscriber(Object subscriber) { + public synchronized boolean registerRedisSubscriber(Object subscriber) { if (activeRedisSubscriber == null) { activeRedisSubscriber = subscriber; + return true; } else if (activeRedisSubscriber != subscriber) { standbyRedisSubscriber = subscriber; legacyRedisDeliveries.clear(); legacyRedisDeliveryBytes = 0; + legacyRedisHandoffOverflowed = false; + return false; } + return true; } /** Returns true only for the active subscriber and counts its legacy delivery during overlap. */ @@ -107,12 +112,16 @@ public synchronized boolean reserveLegacyRedisDelivery(Object subscriber, String && legacyRedisDeliveryBytes <= MAX_LEGACY_REDIS_TOTAL_BYTES - bytes) { legacyRedisDeliveries.put(signature, 1); legacyRedisDeliveryBytes += bytes; - } + } else legacyRedisHandoffOverflowed = true; } } return true; } + public synchronized boolean isLegacyRedisHandoffOverflowed() { + return legacyRedisHandoffOverflowed; + } + public synchronized void activateRedisSubscriber(Object subscriber) { if (standbyRedisSubscriber != subscriber) { throw new IllegalStateException("Redis replacement subscriber is not registered"); @@ -135,6 +144,7 @@ public synchronized boolean consumeLegacyRedisDelivery(String signature) { public synchronized void finishRedisHandoff() { legacyRedisDeliveries.clear(); legacyRedisDeliveryBytes = 0; + legacyRedisHandoffOverflowed = false; } public synchronized void unregisterRedisSubscriber(Object subscriber) { @@ -142,6 +152,7 @@ public synchronized void unregisterRedisSubscriber(Object subscriber) { standbyRedisSubscriber = null; legacyRedisDeliveries.clear(); legacyRedisDeliveryBytes = 0; + legacyRedisHandoffOverflowed = false; } if (activeRedisSubscriber == subscriber) activeRedisSubscriber = null; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 4bb6b6211..81d1fe6bd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -204,6 +204,10 @@ public synchronized void closeRedisForHandoff() { ((RedisBackendProxyTransport) retiredTransport).closeForHandoff(); retiredTransport = null; } catch (RuntimeException failure) { + if (failure instanceof RedisBackendProxyTransport.HandoffQuiescenceException) { + transport = retiredTransport; + retiredTransport = null; + } // Retain the fenced old listener so a later manager close can retry its // cleanup without ever touching the promoted replacement. throw failure; @@ -217,20 +221,23 @@ public void activateRedisAfterHandoff() { ((RedisBackendProxyTransport) transport).activateAfterHandoff(); } - /** Promotes the validated standby before retiring the old Redis listener. */ + /** Fences the old Redis listener before promoting the validated standby. */ public void completeRedisHandoff(BackendProxyTransportManager replacement) { - java.util.Objects.requireNonNull(replacement, "replacement").activateRedisAfterHandoff(); + java.util.Objects.requireNonNull(replacement, "replacement"); try { closeRedisForHandoff(); } catch (RuntimeException retirementFailure) { - // The replacement is already active and usable. Treat failure to join the - // closed old listener as cleanup degradation, not a reason to tear down - // the newly promoted subscriber and leave the backend disconnected. + if (retirementFailure instanceof RedisBackendProxyTransport.HandoffQuiescenceException) { + throw retirementFailure; + } + // The old listener is already fenced. Treat failure to join it as cleanup + // degradation; promotion remains safe because callbacks can no longer enter. if (plugin != null) { plugin.getLogger().warning("Previous Redis backend listener did not stop cleanly after handoff"); plugin.debug(retirementFailure); } } + replacement.activateRedisAfterHandoff(); } public ClientHandler getClientHandler() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 7b4c020eb..4de3fccb7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -46,6 +46,8 @@ public final class HttpBackendProxyTransport implements BackendProxyTransport { private volatile RuntimeException credentialRestoreFailure; private volatile boolean started; private volatile boolean closed; + /** True only after this transport has crossed the handler publication boundary. */ + private boolean published; private volatile boolean restartAfterFailedFlush; private final java.util.concurrent.atomic.AtomicBoolean flushRecoveryRunning = new java.util.concurrent.atomic.AtomicBoolean(); private Path configuredDirectory; @@ -104,6 +106,7 @@ private void start(Path directory, String serverId, String connectionCode, this.retryInitialization = retryInitialization; this.restoreUnenrolledState = restoreUnenrolledState; this.inboundActive = inboundActive; + this.published = inboundActive; started = true; worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler, retryInitialization, restoreUnenrolledState), @@ -319,6 +322,7 @@ public void activateAfterPublication() { synchronized (lifecycle) { if (closed) return; inboundActive = true; + published = true; lifecycle.notifyAll(); } } @@ -630,9 +634,19 @@ private void close(boolean discardQueuedMessages) { if (closed) return; closed = true; lifecycle.notifyAll(); - if (discardQueuedMessages) startupQueue.clear(); - finalHandoff = discardQueuedMessages ? java.util.List.copyOf(handoffQueue) : java.util.List.of(); - if (discardQueuedMessages) handoffQueue.clear(); + if (discardQueuedMessages) { + if (published) { + finalHandoff = new java.util.ArrayList<>(startupQueue.size() + handoffQueue.size()); + finalHandoff.addAll(startupQueue); + finalHandoff.addAll(handoffQueue); + } else { + // A staged replacement never became authoritative. Its presence and + // handoff messages must not be flushed after rollback. + finalHandoff = java.util.List.of(); + } + startupQueue.clear(); + handoffQueue.clear(); + } else finalHandoff = java.util.List.of(); awaitingPreparedHandoff = false; setup = worker; worker = null; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index 60d12293e..5e5ca4535 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -22,7 +22,16 @@ import lombok.Getter; public class RedisBackendProxyTransport implements BackendProxyTransport { + public static final class HandoffQuiescenceException extends IllegalStateException { + private static final long serialVersionUID = 1L; + + public HandoffQuiescenceException(String message) { + super(message); + } + } static final int MAX_LEGACY_HANDOFF_DELIVERIES = 4096; + static final int MAX_IDENTIFIED_HANDOFF_DELIVERIES = 4096; + private static final long HANDOFF_QUIESCE_TIMEOUT_NANOS = TimeUnit.SECONDS.toNanos(3); private final VotingPluginMain plugin; private final ProcessedVoteCache processedVoteCache; @@ -32,11 +41,18 @@ public class RedisBackendProxyTransport implements BackendProxyTransport { private Thread listenerThread; private final Object subscriberIdentity = new Object(); private final Object legacyLifecycle = new Object(); - private final List bufferedLegacyDeliveries = new ArrayList<>(); + private final List bufferedLegacyDeliveries = new ArrayList<>(); + private final List bufferedIdentifiedDeliveries = new ArrayList<>(); private long bufferedLegacyDeliveryBytes; + private long bufferedIdentifiedDeliveryBytes; + private long nextHandoffSequence; + private boolean standbySubscriber; + private boolean identifiedHandoffOverflowed; + private boolean legacyHandoffOverflowed; private boolean legacyHandoffDegraded; private boolean retiredAfterHandoff; private boolean replayingHandoff; + private int dispatchesInFlight; private final java.util.ArrayDeque deliveriesAfterReplay = new java.util.ArrayDeque<>(); private GlobalMessageHandler messageHandler; @@ -53,7 +69,7 @@ public RedisBackendProxyTransport(VotingPluginMain plugin, ProcessedVoteCache pr public void start(GlobalMessageHandler messageHandler) { this.messageHandler = messageHandler; retiredAfterHandoff = false; - processedVoteCache.registerRedisSubscriber(subscriberIdentity); + standbySubscriber = !processedVoteCache.registerRedisSubscriber(subscriberIdentity); redisHandler = new RedisHandler(plugin.getBungeeSettings().getRedisHost(), plugin.getBungeeSettings().getRedisPort(), plugin.getBungeeSettings().getRedisUsername(), plugin.getBungeeSettings().getRedisPassword(), plugin.getBungeeSettings().getRedisdbindex(), @@ -97,13 +113,18 @@ void dispatchIdentified(JsonEnvelope envelope, String deliveryId) { boolean accepted; synchronized (legacyLifecycle) { if (retiredAfterHandoff) return; + if (standbySubscriber) { + bufferIdentifiedDelivery(envelope, deliveryId); + return; + } accepted = processedVoteCache.reserveRedisDelivery(deliveryId); if (accepted && replayingHandoff) { deliveriesAfterReplay.addLast(envelope); accepted = false; } + if (accepted) dispatchesInFlight++; } - if (accepted) messageHandler.onMessage(envelope); + if (accepted) dispatchTracked(envelope); } void dispatchLegacy(JsonEnvelope envelope) { @@ -117,41 +138,75 @@ void dispatchLegacy(JsonEnvelope envelope) { } else if (encodedBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_DELIVERY_BYTES && bufferedLegacyDeliveries.size() < MAX_LEGACY_HANDOFF_DELIVERIES && bufferedLegacyDeliveryBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES - encodedBytes) { - bufferedLegacyDeliveries.add(envelope); + bufferedLegacyDeliveries.add(new BufferedHandoffDelivery( + nextHandoffSequence++, envelope, signature, false)); bufferedLegacyDeliveryBytes += encodedBytes; } else { - if (!legacyHandoffDegraded && plugin != null) { - plugin.getLogger().warning("Redis legacy handoff exceeded its " + MAX_LEGACY_HANDOFF_DELIVERIES - + " delivery / " + ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES - + " byte buffer; temporarily degrading duplicate suppression"); + if (standbySubscriber) { + if (!legacyHandoffOverflowed && plugin != null) + plugin.getLogger().warning("Redis legacy handoff buffer is full; aborting the staged handoff"); + legacyHandoffOverflowed = true; + } else { + if (!legacyHandoffDegraded && plugin != null) { + plugin.getLogger().warning("Redis legacy handoff exceeded its " + MAX_LEGACY_HANDOFF_DELIVERIES + + " delivery / " + ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES + + " byte buffer; temporarily degrading duplicate suppression"); + } + legacyHandoffDegraded = true; + dispatch = true; } - legacyHandoffDegraded = true; - dispatch = true; } if (dispatch && replayingHandoff) { deliveriesAfterReplay.addLast(envelope); dispatch = false; } + if (dispatch) dispatchesInFlight++; + } + if (dispatch) dispatchTracked(envelope); + } + + private void dispatchTracked(JsonEnvelope envelope) { + try { + messageHandler.onMessage(envelope); + } finally { + synchronized (legacyLifecycle) { + dispatchesInFlight--; + legacyLifecycle.notifyAll(); + } } - if (dispatch) messageHandler.onMessage(envelope); } /** Promotes a validated standby after the previous listener has completely stopped. */ public void activateAfterHandoff() { java.util.ArrayList replay = new java.util.ArrayList<>(); synchronized (legacyLifecycle) { + if (identifiedHandoffOverflowed || legacyHandoffOverflowed + || processedVoteCache.isLegacyRedisHandoffOverflowed()) + throw new IllegalStateException("Redis handoff buffer overflowed before publication"); replayingHandoff = true; processedVoteCache.activateRedisSubscriber(subscriberIdentity); - for (JsonEnvelope envelope : bufferedLegacyDeliveries) { + standbySubscriber = false; + java.util.ArrayList buffered = new java.util.ArrayList<>( + bufferedIdentifiedDeliveries.size() + bufferedLegacyDeliveries.size()); + buffered.addAll(bufferedIdentifiedDeliveries); + buffered.addAll(bufferedLegacyDeliveries); + buffered.sort(java.util.Comparator.comparingLong(BufferedHandoffDelivery::sequence)); + for (BufferedHandoffDelivery delivery : buffered) { try { - String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(envelope); - if (!processedVoteCache.consumeLegacyRedisDelivery(signature)) replay.add(envelope); + boolean dispatch = delivery.identified() + ? processedVoteCache.reserveRedisDelivery(delivery.identity()) + : !processedVoteCache.consumeLegacyRedisDelivery(delivery.identity()); + if (dispatch) replay.add(delivery.envelope()); } catch (RuntimeException replayFailure) { if (plugin != null) plugin.debug("Redis handoff replay failed: " + replayFailure.getMessage()); } } + bufferedIdentifiedDeliveries.clear(); + bufferedIdentifiedDeliveryBytes = 0; bufferedLegacyDeliveries.clear(); bufferedLegacyDeliveryBytes = 0; + identifiedHandoffOverflowed = false; + legacyHandoffOverflowed = false; legacyHandoffDegraded = false; processedVoteCache.finishRedisHandoff(); } @@ -187,15 +242,57 @@ public void closeForHandoff() { /** Prevents a listener that misses its shutdown deadline from dispatching duplicates. */ void fenceAfterHandoff() { + boolean interrupted = false; + long deadline = System.nanoTime() + HANDOFF_QUIESCE_TIMEOUT_NANOS; synchronized (legacyLifecycle) { retiredAfterHandoff = true; + while (dispatchesInFlight > 0) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) break; + try { + TimeUnit.NANOSECONDS.timedWait(legacyLifecycle, remaining); + } catch (InterruptedException waitInterrupted) { + interrupted = true; + break; + } + } + if (dispatchesInFlight > 0) { + retiredAfterHandoff = false; + if (interrupted) Thread.currentThread().interrupt(); + throw new HandoffQuiescenceException( + "Redis backend callbacks did not quiesce before handoff"); + } replayingHandoff = false; deliveriesAfterReplay.clear(); bufferedLegacyDeliveries.clear(); bufferedLegacyDeliveryBytes = 0; + bufferedIdentifiedDeliveries.clear(); + bufferedIdentifiedDeliveryBytes = 0; + identifiedHandoffOverflowed = false; + legacyHandoffOverflowed = false; } + if (interrupted) Thread.currentThread().interrupt(); } + private void bufferIdentifiedDelivery(JsonEnvelope envelope, String deliveryId) { + String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(envelope); + int encodedBytes = ProcessedVoteCache.legacyRedisDeliveryBytes(signature); + if (encodedBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_DELIVERY_BYTES + && bufferedIdentifiedDeliveries.size() < MAX_IDENTIFIED_HANDOFF_DELIVERIES + && bufferedIdentifiedDeliveryBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES - encodedBytes) { + bufferedIdentifiedDeliveries.add(new BufferedHandoffDelivery( + nextHandoffSequence++, envelope, deliveryId, true)); + bufferedIdentifiedDeliveryBytes += encodedBytes; + } else { + if (plugin != null && !identifiedHandoffOverflowed) + plugin.getLogger().warning("Redis identified handoff buffer is full; aborting the staged handoff"); + identifiedHandoffOverflowed = true; + } + } + + private record BufferedHandoffDelivery(long sequence, JsonEnvelope envelope, String identity, + boolean identified) {} + @Override public void send(JsonEnvelope envelope) { if (redisHandler != null) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java index 329681705..bc7cfcc20 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java @@ -1,9 +1,14 @@ package com.bencodez.votingplugin.proxy; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.Set; -import java.util.UUID; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Locale; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; @@ -46,6 +51,10 @@ public class OfflineBungeeVote { @Getter @Setter private boolean deliveryStateDirty; + /** Stable HTTP delivery IDs that must be reused for each target server. */ + private final Map httpDeliveryIds; + /** Stable HTTP standalone-broadcast IDs, separate from reward delivery IDs. */ + private final Map httpBroadcastDeliveryIds; /** * Constructor with UUID voteId. @@ -94,10 +103,29 @@ public OfflineBungeeVote(UUID voteId, String playerName, String uuid, String ser * @param broadcastForwardedServers targets that accepted standalone delivery * @param rewardDelivered whether the cached reward vote was already delivered */ - public OfflineBungeeVote(UUID voteId, String playerName, String uuid, String service, long time, boolean realVote, - String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets, - Set broadcastForwardedServers, boolean rewardDelivered) { - this.playerName = playerName; + public OfflineBungeeVote(UUID voteId, String playerName, String uuid, String service, long time, boolean realVote, + String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets, + Set broadcastForwardedServers, boolean rewardDelivered) { + this(voteId, playerName, uuid, service, time, realVote, text, broadcastForwarded, proxyBroadcastHandled, + broadcastTargets, broadcastForwardedServers, rewardDelivered, Collections.emptyMap()); + } + + /** + * Constructor with full proxy state and stable HTTP delivery IDs. + * @param httpDeliveryIds stable HTTP delivery ID by target server + */ + public OfflineBungeeVote(UUID voteId, String playerName, String uuid, String service, long time, boolean realVote, + String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets, + Set broadcastForwardedServers, boolean rewardDelivered, Map httpDeliveryIds) { + this(voteId, playerName, uuid, service, time, realVote, text, broadcastForwarded, proxyBroadcastHandled, + broadcastTargets, broadcastForwardedServers, rewardDelivered, httpDeliveryIds, Collections.emptyMap()); + } + + public OfflineBungeeVote(UUID voteId, String playerName, String uuid, String service, long time, boolean realVote, + String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets, + Set broadcastForwardedServers, boolean rewardDelivered, Map httpDeliveryIds, + Map httpBroadcastDeliveryIds) { + this.playerName = playerName; this.uuid = uuid; this.service = service; this.time = time; @@ -106,10 +134,16 @@ public OfflineBungeeVote(UUID voteId, String playerName, String uuid, String ser this.voteId = voteId; this.broadcastForwarded = broadcastForwarded; this.proxyBroadcastHandled = proxyBroadcastHandled; - setBroadcastTargets(broadcastTargets); - setBroadcastForwardedServers(broadcastForwardedServers); - this.rewardDelivered = rewardDelivered; - } + setBroadcastTargets(broadcastTargets); + setBroadcastForwardedServers(broadcastForwardedServers); + this.rewardDelivered = rewardDelivered; + this.httpDeliveryIds = new LinkedHashMap<>(); + if (httpDeliveryIds != null) { + httpDeliveryIds.forEach(this::setHttpDeliveryId); + } + this.httpBroadcastDeliveryIds = new LinkedHashMap<>(); + if (httpBroadcastDeliveryIds != null) httpBroadcastDeliveryIds.forEach(this::setHttpBroadcastDeliveryId); + } /** * Constructor with String voteId. @@ -157,12 +191,28 @@ public OfflineBungeeVote(String voteId, String playerName, String uuid, String s * @param broadcastForwardedServers targets that accepted standalone delivery * @param rewardDelivered whether the cached reward vote was already delivered */ - public OfflineBungeeVote(String voteId, String playerName, String uuid, String service, long time, boolean realVote, - String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets, - Set broadcastForwardedServers, boolean rewardDelivered) { - this(parseVoteId(voteId), playerName, uuid, service, time, realVote, text, broadcastForwarded, - proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, rewardDelivered); - } + public OfflineBungeeVote(String voteId, String playerName, String uuid, String service, long time, boolean realVote, + String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets, + Set broadcastForwardedServers, boolean rewardDelivered) { + this(parseVoteId(voteId), playerName, uuid, service, time, realVote, text, broadcastForwarded, + proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, rewardDelivered); + } + + public OfflineBungeeVote(String voteId, String playerName, String uuid, String service, long time, boolean realVote, + String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets, + Set broadcastForwardedServers, boolean rewardDelivered, Map httpDeliveryIds) { + this(parseVoteId(voteId), playerName, uuid, service, time, realVote, text, broadcastForwarded, + proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, rewardDelivered, httpDeliveryIds); + } + + public OfflineBungeeVote(String voteId, String playerName, String uuid, String service, long time, boolean realVote, + String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, Set broadcastTargets, + Set broadcastForwardedServers, boolean rewardDelivered, Map httpDeliveryIds, + Map httpBroadcastDeliveryIds) { + this(parseVoteId(voteId), playerName, uuid, service, time, realVote, text, broadcastForwarded, + proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, rewardDelivered, httpDeliveryIds, + httpBroadcastDeliveryIds); + } private static UUID parseVoteId(String voteId) { return voteId == null || voteId.isEmpty() ? null : UUID.fromString(voteId); @@ -198,12 +248,108 @@ public boolean isProxyBroadcastComplete() { * @param server backend server receiving the cached vote * @return true when this server is an original undelivered target */ - public boolean needsBroadcastOn(String server) { + public boolean needsBroadcastOn(String server) { if (!proxyBroadcastHandled) { return !broadcastForwarded; } return server != null && broadcastTargets.contains(server) && !broadcastForwardedServers.contains(server); - } + } + + /** + * Returns the stable HTTP delivery ID for a target, if one is pending. + * @param server target server + * @return stable delivery ID or null + */ + public String getHttpDeliveryId(String server) { + return server == null ? null : httpDeliveryIds.get(server.toLowerCase(Locale.ROOT)); + } + + /** + * Stores or removes a stable HTTP delivery ID for a target. + * @param server target server + * @param deliveryId stable delivery ID, or null to remove + */ + public void setHttpDeliveryId(String server, String deliveryId) { + if (server == null || server.isBlank()) return; + String key = server.toLowerCase(Locale.ROOT); + if (deliveryId == null || deliveryId.isBlank()) httpDeliveryIds.remove(key); + else httpDeliveryIds.put(key, deliveryId); + } + + public String getHttpBroadcastDeliveryId(String server) { + return server == null ? null : httpBroadcastDeliveryIds.get(server.toLowerCase(Locale.ROOT)); + } + + public void setHttpBroadcastDeliveryId(String server, String deliveryId) { + if (server == null || server.isBlank()) return; + String key = server.toLowerCase(Locale.ROOT); + if (deliveryId == null || deliveryId.isBlank()) httpBroadcastDeliveryIds.remove(key); + else httpBroadcastDeliveryIds.put(key, deliveryId); + } + + /** + * Returns a copy of pending standalone HTTP delivery IDs for cache handoff. + * @return pending standalone delivery IDs by target server + */ + public Map getHttpBroadcastDeliveryIds() { + return new LinkedHashMap<>(httpBroadcastDeliveryIds); + } + + /** + * Encodes stable delivery IDs for JSON/SQL cache storage. + * @return bounded delimiter-safe encoding + */ + public String encodeHttpDeliveryIds() { + return encodeDeliveryIds(httpDeliveryIds); + } + + public String encodeHttpBroadcastDeliveryIds() { + return encodeDeliveryIds(httpBroadcastDeliveryIds); + } + + private static String encodeDeliveryIds(Map values) { + StringBuilder encoded = new StringBuilder(); + for (Map.Entry entry : values.entrySet()) { + if (encoded.length() > 0) encoded.append('.'); + encoded.append(Base64.getUrlEncoder().withoutPadding() + .encodeToString(entry.getKey().getBytes(StandardCharsets.UTF_8))); + encoded.append('~'); + encoded.append(Base64.getUrlEncoder().withoutPadding() + .encodeToString(entry.getValue().getBytes(StandardCharsets.UTF_8))); + } + return encoded.toString(); + } + + /** + * Decodes stable delivery IDs from cache storage. Malformed entries are ignored. + * @param encoded encoded map + * @return decoded map + */ + public static Map decodeHttpDeliveryIds(String encoded) { + return decodeDeliveryIds(encoded); + } + + public static Map decodeHttpBroadcastDeliveryIds(String encoded) { + return decodeDeliveryIds(encoded); + } + + private static Map decodeDeliveryIds(String encoded) { + Map decoded = new LinkedHashMap<>(); + if (encoded == null || encoded.isBlank()) return decoded; + for (String entry : encoded.split("\\.", -1)) { + int separator = entry.indexOf('~'); + if (separator <= 0 || separator == entry.length() - 1) continue; + try { + String server = new String(Base64.getUrlDecoder().decode(entry.substring(0, separator)), StandardCharsets.UTF_8); + String deliveryId = new String(Base64.getUrlDecoder().decode(entry.substring(separator + 1)), StandardCharsets.UTF_8); + if (!server.isBlank() && deliveryId.matches("[0-9a-fA-F-]{36}")) + decoded.put(server.toLowerCase(Locale.ROOT), deliveryId); + } catch (IllegalArgumentException ignored) { + // Ignore corrupt optional delivery state and retain the vote itself. + } + } + return decoded; + } /** * Encodes the original broadcast targets for cache storage. diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index d09af583a..0d654e2a4 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -536,16 +536,106 @@ public void addVoteParty() { private Set sendProxyBroadcast(Set targets, String uuid, String player, String service, long time, String text, boolean wasOnline) { + return sendProxyBroadcast(targets, uuid, player, service, time, text, wasOnline, + (OfflineBungeeVote) null); + } + + private Set sendProxyBroadcast(Set targets, String uuid, String player, String service, long time, + String text, boolean wasOnline, OfflineBungeeVote cachedVote) { Set forwarded = new LinkedHashSet<>(); for (String targetServer : targets) { JsonEnvelope envelope = VotingPluginWire.voteBroadcast(uuid, player, service, time, text, wasOnline); - if (sendProxyBroadcastEnvelopeNow(targetServer, envelope)) { + boolean accepted = method == BungeeMethod.HTTP + ? sendHttpBroadcastEnvelopeWithRecovery(targetServer, envelope, cachedVote) + : sendProxyBroadcastEnvelopeNow(targetServer, envelope); + if (accepted) { forwarded.add(targetServer); } } return forwarded; } + private Set sendProxyBroadcast(Set targets, String uuid, String player, String service, long time, + String text, boolean wasOnline, VoteTimeQueue cachedVote) { + Set forwarded = new LinkedHashSet<>(); + for (String targetServer : targets) { + JsonEnvelope envelope = VotingPluginWire.voteBroadcast(uuid, player, service, time, text, wasOnline); + boolean accepted = method == BungeeMethod.HTTP + ? sendHttpBroadcastEnvelopeWithRecovery(targetServer, envelope, cachedVote) + : sendProxyBroadcastEnvelopeNow(targetServer, envelope); + if (accepted) forwarded.add(targetServer); + } + return forwarded; + } + + protected boolean sendHttpBroadcastEnvelopeWithRecovery(String server, JsonEnvelope envelope, + OfflineBungeeVote cachedVote) { + String stableId = cachedVote == null ? null : cachedVote.getHttpBroadcastDeliveryId(server); + try { + boolean accepted = stableId == null ? sendProxyBroadcastEnvelopeNow(server, envelope) + : sendHttpEnvelope(server, stableId, envelope); + if (accepted && cachedVote != null && stableId != null) { + cachedVote.setHttpBroadcastDeliveryId(server, null); + cachedVote.setDeliveryStateDirty(true); + } + return accepted; + } catch (HttpProxyTransportServer.DeliveryRetryException failure) { + if (cachedVote != null) { + cachedVote.setHttpBroadcastDeliveryId(server, failure.deliveryId()); + cachedVote.setDeliveryStateDirty(true); + } + try { + boolean accepted = sendHttpEnvelope(server, failure.deliveryId(), envelope); + if (accepted && cachedVote != null) { + cachedVote.setHttpBroadcastDeliveryId(server, null); + cachedVote.setDeliveryStateDirty(true); + } + return accepted; + } catch (RuntimeException retryFailure) { + debug("Unable to recover HTTP standalone delivery " + failure.deliveryId() + ": " + + retryFailure.getMessage()); + return false; + } + } catch (RuntimeException failure) { + debug("Unable to send HTTP standalone delivery: " + failure.getMessage()); + return false; + } + } + + private boolean sendHttpBroadcastEnvelopeWithRecovery(String server, JsonEnvelope envelope, + VoteTimeQueue cachedVote) { + String stableId = cachedVote == null ? null : cachedVote.getHttpBroadcastDeliveryId(server); + try { + boolean accepted = stableId == null ? sendProxyBroadcastEnvelopeNow(server, envelope) + : sendHttpEnvelope(server, stableId, envelope); + if (accepted && cachedVote != null && stableId != null) { + cachedVote.setHttpBroadcastDeliveryId(server, null); + cachedVote.setDeliveryStateDirty(true); + } + return accepted; + } catch (HttpProxyTransportServer.DeliveryRetryException failure) { + if (cachedVote != null) { + cachedVote.setHttpBroadcastDeliveryId(server, failure.deliveryId()); + cachedVote.setDeliveryStateDirty(true); + } + try { + boolean accepted = sendHttpEnvelope(server, failure.deliveryId(), envelope); + if (accepted && cachedVote != null) { + cachedVote.setHttpBroadcastDeliveryId(server, null); + cachedVote.setDeliveryStateDirty(true); + } + return accepted; + } catch (RuntimeException retryFailure) { + debug("Unable to recover HTTP timed broadcast delivery " + failure.deliveryId() + ": " + + retryFailure.getMessage()); + return false; + } + } catch (RuntimeException failure) { + debug("Unable to send HTTP timed broadcast delivery: " + failure.getMessage()); + return false; + } + } + /** * Sends a standalone proxy broadcast through the selected transport and reports * whether that transport accepted the message. @@ -592,8 +682,13 @@ protected boolean sendProxyBroadcastEnvelopeNow(String server, JsonEnvelope enve * when the queue is full. */ protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope) { + return sendVoteEnvelopeAccepted(server, delay, envelope, null); + } + + protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope, + OfflineBungeeVote cachedVote) { if (method == BungeeMethod.HTTP) { - return sendHttpEnvelope(server, envelope); + return sendHttpEnvelopeWithRecovery(server, envelope, cachedVote); } GlobalMessageProxyHandler handler = globalMessageProxyHandler; if (handler == null) { @@ -620,13 +715,18 @@ public synchronized void checkCachedVotes(String server) { if (cache.isProxyBroadcastHandled() && cache.needsBroadcastOn(server)) { Set forwarded = sendProxyBroadcast(Collections.singleton(server), cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), - cache.getText(), false); - if (cache.getBroadcastForwardedServers().addAll(forwarded)) { + cache.getText(), false, cache); + boolean broadcastChanged = cache.getBroadcastForwardedServers().addAll(forwarded); + if (broadcastChanged) { cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - if (!persistServerVoteDelivery(server, cache)) { - continue; - } } + if ((broadcastChanged || cache.isDeliveryStateDirty()) + && !persistServerVoteDelivery(server, cache)) continue; + } + if (cache.isRewardDelivered()) { + if (cache.isProxyBroadcastHandled() && !cache.isProxyBroadcastComplete()) continue; + removed.add(cache); + continue; } boolean toSend = true; @@ -639,7 +739,7 @@ public synchronized void checkCachedVotes(String server) { } } if (toSend) { - boolean broadcastHere = cache.needsBroadcastOn(server); + boolean broadcastHere = !cache.isProxyBroadcastHandled() && cache.needsBroadcastOn(server); if (!cache.isProxyBroadcastHandled() && broadcastHere && getConfig().getProxyBroadcastEnabled()) { boolean playerOnline = isPlayerOnlineForVoteRouting(cache.getPlayerName()); @@ -655,12 +755,18 @@ && getConfig().getProxyBroadcastEnabled()) { VotingPluginWire.vote(cache.getPlayerName(), cache.getUuid(), cache.getService(), cache.getTime(), false, cache.isRealVote(), cache.getText(), cache.getVoteId(), getConfig().getBungeeManageTotals(), - broadcastHere, num, numberOfVotes))) { + broadcastHere, num, numberOfVotes), cache)) { debug("Retaining cached vote because the transport rejected delivery for " + server); + persistServerVoteDelivery(server, cache); continue; } delay++; num++; + cache.setRewardDelivered(true); + if (cache.isProxyBroadcastHandled() && !cache.isProxyBroadcastComplete()) { + persistServerVoteDelivery(server, cache); + continue; + } removed.add(cache); } else { debug("Not sending vote because user isn't on server " + server + ": " @@ -703,10 +809,10 @@ public synchronized void checkOnlineVotes(String player, String uuid, String ser } cache.getBroadcastForwardedServers().addAll(sendProxyBroadcast(pendingTargets, cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), - cache.getText(), false)); + cache.getText(), false, cache)); cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); } - boolean broadcastHere = cache.needsBroadcastOn(server); + boolean broadcastHere = !cache.isProxyBroadcastHandled() && cache.needsBroadcastOn(server); if (!cache.isProxyBroadcastHandled() && broadcastHere && getConfig().getProxyBroadcastEnabled()) { String playerServer = (server != null) ? server : getCurrentPlayerServerForVoteRouting(player); @@ -719,18 +825,12 @@ && getConfig().getProxyBroadcastEnabled()) { if (!sendVoteEnvelopeAccepted(server, delay, VotingPluginWire.voteOnline(cache.getPlayerName(), cache.getUuid(), cache.getService(), cache.getTime(), false, cache.isRealVote(), cache.getText(), cache.getVoteId(), - getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes))) { + getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes), cache)) { debug("Retaining online vote because the transport rejected delivery for " + server); + persistOnlineVoteDelivery(uuid, cache); retained.add(cache); continue; } - // The normal envelope is also a valid broadcast delivery for the - // current target. Record it so a previously pending standalone - // retry cannot announce the same vote again later. - if (cache.isProxyBroadcastHandled() && broadcastHere) { - cache.getBroadcastForwardedServers().add(server); - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - } cache.setRewardDelivered(true); deliveredReward = true; delay++; @@ -776,7 +876,7 @@ protected synchronized void retryPendingOnlineBroadcasts(String server) { continue; } Set forwarded = sendProxyBroadcast(Collections.singleton(server), cache.getUuid(), - cache.getPlayerName(), cache.getService(), cache.getTime(), cache.getText(), false); + cache.getPlayerName(), cache.getService(), cache.getTime(), cache.getText(), false, cache); if (cache.getBroadcastForwardedServers().addAll(forwarded)) { cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { @@ -784,6 +884,8 @@ protected synchronized void retryPendingOnlineBroadcasts(String server) { } else { persistOnlineVoteDelivery(cachedUuid, cache); } + } else if (cache.isDeliveryStateDirty()) { + persistOnlineVoteDelivery(cachedUuid, cache); } } } @@ -806,8 +908,8 @@ protected synchronized void retryPendingTimeBroadcasts(String server) { continue; } Set forwarded = sendProxyBroadcast(Collections.singleton(server), vote.getUuid(), vote.getName(), - vote.getService(), vote.getTime(), vote.getTotals(), false); - if (vote.getBroadcastForwardedServers().addAll(forwarded)) { + vote.getService(), vote.getTime(), vote.getTotals(), false, vote); + if (vote.getBroadcastForwardedServers().addAll(forwarded) || vote.isDeliveryStateDirty()) { persistTimeVoteDelivery(vote); } } @@ -834,7 +936,7 @@ public synchronized void retryPendingOnlineBroadcasts() { pendingTargets.removeAll(blockedServers); } Set forwarded = sendProxyBroadcast(pendingTargets, cache.getUuid(), cache.getPlayerName(), - cache.getService(), cache.getTime(), cache.getText(), false); + cache.getService(), cache.getTime(), cache.getText(), false, cache); if (cache.getBroadcastForwardedServers().addAll(forwarded)) { cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { @@ -842,6 +944,8 @@ public synchronized void retryPendingOnlineBroadcasts() { } else { persistOnlineVoteDelivery(cachedUuid, cache); } + } else if (cache.isDeliveryStateDirty()) { + persistOnlineVoteDelivery(cachedUuid, cache); } } } @@ -866,8 +970,8 @@ public synchronized void retryPendingTimeBroadcasts() { pendingTargets.removeAll(blockedServers); } Set forwarded = sendProxyBroadcast(pendingTargets, vote.getUuid(), vote.getName(), vote.getService(), - vote.getTime(), vote.getTotals(), false); - if (vote.getBroadcastForwardedServers().addAll(forwarded)) { + vote.getTime(), vote.getTotals(), false, vote); + if (vote.getBroadcastForwardedServers().addAll(forwarded) || vote.isDeliveryStateDirty()) { persistTimeVoteDelivery(vote); } } @@ -2685,6 +2789,15 @@ private UUID parseUUIDFromString(String uuidAsString) { public synchronized void processQueue() { while (getVoteCacheHandler().getTimeChangeQueue().size() > 0) { VoteTimeQueue vote = getVoteCacheHandler().getTimeChangeQueue().element(); + if (vote.isProcessed() && vote.hasPendingHttpBroadcastDeliveryIds()) { + // The reward/totals work is already complete. Only retry the durable + // standalone broadcasts; removing this row would lose their stable IDs. + retryPendingTimeBroadcasts(); + if (vote.hasPendingHttpBroadcastDeliveryIds()) { + scheduleTimeVoteRetry(); + return; + } + } if (!vote.isProcessed()) { VoteTotalsSnapshot queuedTotals = vote.getTotals() == null || vote.getTotals().isEmpty() ? null : VoteTotalsSnapshot.parseStorage(vote.getTotals()); @@ -2783,11 +2896,50 @@ private synchronized boolean sendSocketEnvelope(String server, JsonEnvelope enve } } - private synchronized boolean sendHttpEnvelope(String server, JsonEnvelope envelope) { + protected synchronized boolean sendHttpEnvelope(String server, JsonEnvelope envelope) { HttpProxyTransportServer transport = httpTransportServer; return transport != null && transport.send(server, envelope); } + /** + * Sends a vote envelope while recovering the stable ID exposed when durable + * publication is indeterminate. The first attempt has already published a + * quarantine file, so retrying the identical envelope with a new ID could + * deliver the vote twice. + */ + protected boolean sendHttpEnvelopeWithRecovery(String server, JsonEnvelope envelope) { + return sendHttpEnvelopeWithRecovery(server, envelope, null); + } + + protected boolean sendHttpEnvelopeWithRecovery(String server, JsonEnvelope envelope, OfflineBungeeVote cachedVote) { + String stableId = cachedVote == null ? null : cachedVote.getHttpDeliveryId(server); + try { + boolean accepted = stableId == null ? sendHttpEnvelope(server, envelope) + : sendHttpEnvelope(server, stableId, envelope); + if (accepted && cachedVote != null && stableId != null) { + cachedVote.setHttpDeliveryId(server, null); + cachedVote.setDeliveryStateDirty(true); + } + return accepted; + } catch (HttpProxyTransportServer.DeliveryRetryException failure) { + if (cachedVote != null) { + cachedVote.setHttpDeliveryId(server, failure.deliveryId()); + cachedVote.setDeliveryStateDirty(true); + } + try { + boolean accepted = sendHttpEnvelope(server, failure.deliveryId(), envelope); + if (accepted && cachedVote != null) { + cachedVote.setHttpDeliveryId(server, null); + cachedVote.setDeliveryStateDirty(true); + } + return accepted; + } catch (RuntimeException retryFailure) { + debug("Unable to recover HTTP vote delivery " + failure.deliveryId() + ": " + retryFailure.getMessage()); + return false; + } + } + } + protected synchronized boolean sendHttpEnvelope(String server, String deliveryId, JsonEnvelope envelope) { HttpProxyTransportServer transport = httpTransportServer; return transport != null && transport.send(server, deliveryId, envelope); @@ -3741,9 +3893,10 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea if (proxyBroadcastHandled) { for (String target : broadcastTargets) { Set forwarded = sendProxyBroadcast(Collections.singleton(target), uuid, player, - service, time, projectedTotals == null ? "" : projectedTotals.toString(), false); - if (delayedVote.getBroadcastForwardedServers().addAll(forwarded)) { - broadcastForwardedServers.addAll(forwarded); + service, time, projectedTotals == null ? "" : projectedTotals.toString(), false, delayedVote); + if (delayedVote.getBroadcastForwardedServers().addAll(forwarded) + || delayedVote.isDeliveryStateDirty()) { + broadcastForwardedServers.addAll(forwarded); persistTimeVoteDelivery(delayedVote); } } @@ -3815,6 +3968,8 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea boolean standaloneProxyBroadcast = canValidateStandaloneBroadcast && (proxyBroadcastHandled || proxyBroadcastDecider.usesImmediateForwarding(playerOnline)); Set proxyBroadcastTargets = Collections.emptySet(); + OfflineBungeeVote standaloneBroadcastState = null; + boolean standaloneBroadcastStatePersisted = false; if (standaloneProxyBroadcast) { // A handled queued broadcast was necessarily sampled while the player was // offline. Retry only targets that did not previously accept delivery. @@ -3822,8 +3977,11 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea : proxyBroadcastDecider.resolveTargets(false, null); Set remainingTargets = new LinkedHashSet<>(proxyBroadcastTargets); remainingTargets.removeAll(broadcastForwardedServers); + standaloneBroadcastState = new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, + text == null ? "" : text.toString(), false, true, proxyBroadcastTargets, + broadcastForwardedServers, false); broadcastForwardedServers.addAll(sendProxyBroadcast(remainingTargets, uuid, player, service, time, - text == null ? "" : text.toString(), false)); + text == null ? "" : text.toString(), false, standaloneBroadcastState)); } // =========================== @@ -3844,9 +4002,12 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea boolean broadcastForwarded = standaloneProxyBroadcast && broadcastForwardedServers.containsAll(proxyBroadcastTargets); getVoteCacheHandler().addServerVote(s, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, - text.toString(), broadcastForwarded, standaloneProxyBroadcast, - proxyBroadcastTargets, broadcastForwardedServers, false)); + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, + text.toString(), broadcastForwarded, standaloneProxyBroadcast, + proxyBroadcastTargets, broadcastForwardedServers, false, Collections.emptyMap(), + standaloneBroadcastState == null ? Collections.emptyMap() + : standaloneBroadcastState.getHttpBroadcastDeliveryIds())); + standaloneBroadcastStatePersisted |= standaloneBroadcastState != null; debug("Caching vote for " + player + " on " + service + " for " + s); } else { boolean broadcastHere = !broadcastForwardedServers.contains(s); @@ -3856,16 +4017,19 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea broadcastHere = proxyBroadcastDecider.shouldBroadcast(s, targets); } + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + OfflineBungeeVote pendingVote = new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, + text.toString(), broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, + broadcastForwardedServers, false, Collections.emptyMap(), + standaloneBroadcastState == null ? Collections.emptyMap() + : standaloneBroadcastState.getHttpBroadcastDeliveryIds()); if (!sendVoteEnvelopeAccepted(s, 2, VotingPluginWire.vote(player, uuid, service, time, true, realVote, text.toString(), - voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1))) { + voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote)) { voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addServerVote(s, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, - text.toString(), broadcastForwarded, standaloneProxyBroadcast, - proxyBroadcastTargets, broadcastForwardedServers, false)); + getVoteCacheHandler().addServerVote(s, pendingVote); + standaloneBroadcastStatePersisted |= standaloneBroadcastState != null; debug("Caching vote after the transport rejected delivery for " + s); } } @@ -3883,17 +4047,20 @@ voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1))) { broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); } + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + OfflineBungeeVote pendingVote = new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, + text.toString(), broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, + broadcastForwardedServers, false, Collections.emptyMap(), + standaloneBroadcastState == null ? Collections.emptyMap() + : standaloneBroadcastState.getHttpBroadcastDeliveryIds()); boolean rewardAccepted = sendVoteEnvelopeAccepted(server, 1, VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(), - voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1)); + voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote); if (!rewardAccepted) { voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addOnlineVote(uuid, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), - broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, - broadcastForwardedServers, false)); + getVoteCacheHandler().addOnlineVote(uuid, pendingVote); + standaloneBroadcastStatePersisted |= standaloneBroadcastState != null; debug("Caching online vote after the transport rejected delivery for " + server); } @@ -3927,9 +4094,12 @@ voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1))) { boolean broadcastForwarded = standaloneProxyBroadcast && broadcastForwardedServers.containsAll(proxyBroadcastTargets); getVoteCacheHandler().addOnlineVote(uuid, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), - broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, - broadcastForwardedServers, false)); + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), + broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, + broadcastForwardedServers, false, Collections.emptyMap(), + standaloneBroadcastState == null ? Collections.emptyMap() + : standaloneBroadcastState.getHttpBroadcastDeliveryIds())); + standaloneBroadcastStatePersisted |= standaloneBroadcastState != null; debug("Caching online vote for " + player + " on " + service); } @@ -3941,6 +4111,8 @@ voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1))) { } } + persistUncachedStandaloneBroadcast(uuid, standaloneBroadcastState, standaloneBroadcastStatePersisted); + // Vote logging if (voteLogMysqlTable != null && getConfig().getVoteLoggingEnabled()) { voteLogMysqlTable.logVote(voteId, voteStatus, service, uuid, player, time, @@ -3989,6 +4161,14 @@ voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1))) { } } + protected void persistUncachedStandaloneBroadcast(String uuid, OfflineBungeeVote state, + boolean alreadyPersisted) { + if (alreadyPersisted || state == null || state.isProxyBroadcastComplete()) return; + state.setRewardDelivered(true); + state.setBroadcastForwarded(false); + getVoteCacheHandler().addOnlineVote(uuid, state); + } + private static final class PendingPresenceHandoff { private UUID requestId; private final UUID playerUuid; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java index 7bd462002..d0b829d62 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java @@ -50,9 +50,10 @@ public void addTimedVote(int num, VoteTimeQueue voteTimedQueue) { setBoolean(path + ".ProxyBroadcastHandled", voteTimedQueue.isProxyBroadcastHandled()); setString(path + ".Totals", voteTimedQueue.getTotals()); setBoolean(path + ".Processed", voteTimedQueue.isProcessed()); - setString(path + ".BroadcastTargets", voteTimedQueue.encodeBroadcastTargets()); - setString(path + ".BroadcastForwardedServers", voteTimedQueue.encodeBroadcastForwardedServers()); - } + setString(path + ".BroadcastTargets", voteTimedQueue.encodeBroadcastTargets()); + setString(path + ".BroadcastForwardedServers", voteTimedQueue.encodeBroadcastForwardedServers()); + setString(path + ".HttpBroadcastDeliveryIds", voteTimedQueue.encodeHttpBroadcastDeliveryIds()); + } public void addVote(String server, int num, OfflineBungeeVote voteData) { String path = "VoteCache." + server + "." + num; @@ -66,8 +67,10 @@ public void addVote(String server, int num, OfflineBungeeVote voteData) { setBoolean(path + ".BroadcastForwarded", voteData.isBroadcastForwarded()); setBoolean(path + ".ProxyBroadcastHandled", voteData.isProxyBroadcastHandled()); setString(path + ".BroadcastTargets", voteData.encodeBroadcastTargets()); - setString(path + ".BroadcastForwardedServers", voteData.encodeBroadcastForwardedServers()); - setBoolean(path + ".RewardDelivered", voteData.isRewardDelivered()); + setString(path + ".BroadcastForwardedServers", voteData.encodeBroadcastForwardedServers()); + setBoolean(path + ".RewardDelivered", voteData.isRewardDelivered()); + setString(path + ".HttpDeliveryIds", voteData.encodeHttpDeliveryIds()); + setString(path + ".HttpBroadcastDeliveryIds", voteData.encodeHttpBroadcastDeliveryIds()); } public void addVoteOnline(String player, int num, OfflineBungeeVote voteData) { @@ -82,8 +85,10 @@ public void addVoteOnline(String player, int num, OfflineBungeeVote voteData) { setBoolean(path + ".BroadcastForwarded", voteData.isBroadcastForwarded()); setBoolean(path + ".ProxyBroadcastHandled", voteData.isProxyBroadcastHandled()); setString(path + ".BroadcastTargets", voteData.encodeBroadcastTargets()); - setString(path + ".BroadcastForwardedServers", voteData.encodeBroadcastForwardedServers()); - setBoolean(path + ".RewardDelivered", voteData.isRewardDelivered()); + setString(path + ".BroadcastForwardedServers", voteData.encodeBroadcastForwardedServers()); + setBoolean(path + ".RewardDelivered", voteData.isRewardDelivered()); + setString(path + ".HttpDeliveryIds", voteData.encodeHttpDeliveryIds()); + setString(path + ".HttpBroadcastDeliveryIds", voteData.encodeHttpBroadcastDeliveryIds()); } public void clearData() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java index 2749a1cbb..8f7092ae0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java @@ -42,7 +42,8 @@ public String buildCreateTableSql(DbType dbType) { + " BOOLEAN NOT NULL DEFAULT FALSE, " + qi("proxyBroadcastHandled") + " BOOLEAN NOT NULL DEFAULT FALSE, " + qi("broadcastTargets") + " TEXT, " + qi("broadcastForwardedServers") + " TEXT, " + qi("rewardDelivered") - + " BOOLEAN NOT NULL DEFAULT FALSE" + ");"; + + " BOOLEAN NOT NULL DEFAULT FALSE, " + qi("httpDeliveryIds") + " TEXT, " + + qi("httpBroadcastDeliveryIds") + " TEXT" + ");"; } return "CREATE TABLE IF NOT EXISTS " + qi(getTableName()) + " (" + qi("id") + " INT AUTO_INCREMENT PRIMARY KEY," @@ -51,7 +52,8 @@ public String buildCreateTableSql(DbType dbType) { + qi("text") + " TEXT," + qi("broadcastForwarded") + " TINYINT(1) NOT NULL DEFAULT 0," + qi("proxyBroadcastHandled") + " TINYINT(1) NOT NULL DEFAULT 0," + qi("broadcastTargets") + " TEXT," + qi("broadcastForwardedServers") + " TEXT," + qi("rewardDelivered") - + " TINYINT(1) NOT NULL DEFAULT 0," + + " TINYINT(1) NOT NULL DEFAULT 0," + qi("httpDeliveryIds") + " TEXT," + + qi("httpBroadcastDeliveryIds") + " TEXT," + "INDEX idx_uuid (" + qi("uuid") + ")," + "INDEX idx_time (" + qi("time") + ")" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"; } @@ -163,7 +165,8 @@ public boolean updateProxyBroadcastState(OfflineBungeeVote vote) { boolean hasVoteId = vote.getVoteId() != null; String sql = "UPDATE " + qi(getTableName()) + " SET " + qi("broadcastForwarded") + " = ?, " + qi("proxyBroadcastHandled") + " = ?, " + qi("broadcastTargets") + " = ?, " - + qi("broadcastForwardedServers") + " = ?, " + qi("rewardDelivered") + " = ? WHERE " + + qi("broadcastForwardedServers") + " = ?, " + qi("rewardDelivered") + " = ?, " + + qi("httpDeliveryIds") + " = ?, " + qi("httpBroadcastDeliveryIds") + " = ? WHERE " + (hasVoteId ? qi("voteid") + " = ?;" : qi("uuid") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ?;"); @@ -180,16 +183,18 @@ public boolean updateProxyBroadcastState(OfflineBungeeVote vote) { } ps.setString(3, vote.encodeBroadcastTargets()); ps.setString(4, vote.encodeBroadcastForwardedServers()); + ps.setString(6, vote.encodeHttpDeliveryIds()); + ps.setString(7, vote.encodeHttpBroadcastDeliveryIds()); if (hasVoteId) { - ps.setString(6, vote.getVoteId().toString()); + ps.setString(8, vote.getVoteId().toString()); } else { if (getDbType() == DbType.POSTGRESQL) { - ps.setObject(6, UUID.fromString(vote.getUuid())); + ps.setObject(8, UUID.fromString(vote.getUuid())); } else { - ps.setString(6, vote.getUuid()); + ps.setString(8, vote.getUuid()); } - ps.setString(7, vote.getService()); - ps.setLong(8, vote.getTime()); + ps.setString(9, vote.getService()); + ps.setLong(10, vote.getTime()); } return ps.executeUpdate() > 0; } catch (SQLException | IllegalArgumentException e) { @@ -269,6 +274,8 @@ private void ensureProxyBroadcastStateColumns() { ensureColumn("broadcastTargets", "TEXT"); ensureColumn("broadcastForwardedServers", "TEXT"); ensureColumn("rewardDelivered", booleanType); + ensureColumn("httpDeliveryIds", "TEXT"); + ensureColumn("httpBroadcastDeliveryIds", "TEXT"); } private void ensureColumn(String column, String type) { @@ -317,13 +324,15 @@ private void ensureColumn(String column, String type) { */ public void insertVote(UUID voteId, String uuid, String playerName, String service, long time, boolean real, String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, String broadcastTargets, - String broadcastForwardedServers, boolean rewardDelivered) { + String broadcastForwardedServers, boolean rewardDelivered, String httpDeliveryIds, + String httpBroadcastDeliveryIds) { String sql = "INSERT INTO " + qi(getTableName()) + " (" + qi("uuid") + ", " + qi("voteid") + ", " + qi("playerName") + ", " + qi("service") + ", " + qi("time") + ", " + qi("realVote") + ", " + qi("text") + ", " + qi("broadcastForwarded") + ", " + qi("proxyBroadcastHandled") + ", " + qi("broadcastTargets") + ", " + qi("broadcastForwardedServers") + ", " + qi("rewardDelivered") - + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; + + ", " + qi("httpDeliveryIds") + ", " + qi("httpBroadcastDeliveryIds") + + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; try (Connection conn = mysql.getConnectionManager().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { @@ -357,6 +366,8 @@ public void insertVote(UUID voteId, String uuid, String playerName, String servi } ps.setString(10, broadcastTargets); ps.setString(11, broadcastForwardedServers); + ps.setString(13, httpDeliveryIds); + ps.setString(14, httpBroadcastDeliveryIds); ps.executeUpdate(); } catch (SQLException | IllegalArgumentException e) { @@ -463,7 +474,8 @@ private List selectVotes(String sql, Object[] params) { (getDbType() == DbType.POSTGRESQL ? rs.getBoolean("realVote") : rs.getInt("realVote") == 1), rs.getString("text"), rs.getBoolean("broadcastForwarded"), rs.getBoolean("proxyBroadcastHandled"), rs.getString("broadcastTargets"), - rs.getString("broadcastForwardedServers"), rs.getBoolean("rewardDelivered"))); + rs.getString("broadcastForwardedServers"), rs.getBoolean("rewardDelivered"), + rs.getString("httpDeliveryIds"), rs.getString("httpBroadcastDeliveryIds"))); } } } catch (SQLException e) { @@ -489,6 +501,8 @@ public static class VoteRow { private final String broadcastTargets; private final String broadcastForwardedServers; private final boolean rewardDelivered; + private final String httpDeliveryIds; + private final String httpBroadcastDeliveryIds; /** * Constructor for VoteRow. @@ -508,7 +522,8 @@ public static class VoteRow { */ public VoteRow(int id, String voteId, String uuid, String playerName, String service, long time, boolean realVote, String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, - String broadcastTargets, String broadcastForwardedServers, boolean rewardDelivered) { + String broadcastTargets, String broadcastForwardedServers, boolean rewardDelivered, String httpDeliveryIds, + String httpBroadcastDeliveryIds) { this.id = id; this.voteId = voteId; this.uuid = uuid; @@ -522,6 +537,8 @@ public VoteRow(int id, String voteId, String uuid, String playerName, String ser this.broadcastTargets = broadcastTargets; this.broadcastForwardedServers = broadcastForwardedServers; this.rewardDelivered = rewardDelivered; + this.httpDeliveryIds = httpDeliveryIds; + this.httpBroadcastDeliveryIds = httpBroadcastDeliveryIds; } /** @@ -611,5 +628,13 @@ public String getBroadcastForwardedServers() { public boolean isRewardDelivered() { return rewardDelivered; } + + public String getHttpDeliveryIds() { + return httpDeliveryIds; + } + + public String getHttpBroadcastDeliveryIds() { + return httpBroadcastDeliveryIds; + } } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java index b311494e0..fbd80ad54 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java @@ -40,7 +40,8 @@ public String buildCreateTableSql(DbType dbType) { + qi("totals") + " TEXT, " + qi("processed") + " BOOLEAN NOT NULL DEFAULT FALSE, " + qi("broadcastTargets") + " TEXT, " - + qi("broadcastForwardedServers") + " TEXT" + + qi("broadcastForwardedServers") + " TEXT, " + + qi("httpBroadcastDeliveryIds") + " TEXT" + ");"; } @@ -56,6 +57,7 @@ public String buildCreateTableSql(DbType dbType) { + qi("processed") + " TINYINT(1) NOT NULL DEFAULT 0," + qi("broadcastTargets") + " TEXT," + qi("broadcastForwardedServers") + " TEXT," + + qi("httpBroadcastDeliveryIds") + " TEXT," + "INDEX idx_time (" + qi("time") + ")" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"; } @@ -111,6 +113,7 @@ private void ensureProxyBroadcastColumns() { ensureColumn("processed", getDbType() == DbType.POSTGRESQL ? "BOOLEAN NOT NULL DEFAULT FALSE" : "TINYINT(1) NOT NULL DEFAULT 0"); + ensureColumn("httpBroadcastDeliveryIds", "TEXT"); } private void ensureColumn(String column, String type) { @@ -158,11 +161,11 @@ private void ensureIndexes() { */ public boolean insertTimedVote(UUID voteId, String uuid, String playerName, String service, long time, boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, - boolean processed) { + boolean processed, String httpBroadcastDeliveryIds) { String sql = "INSERT INTO " + qi(getTableName()) + " (" + qi("playerName") + ", " + qi("service") + ", " + qi("time") + ", " + qi("voteId") + ", " + qi("uuid") + ", " + qi("proxyBroadcastHandled") + ", " + qi("broadcastTargets") + ", " + qi("broadcastForwardedServers") + ", " + qi("totals") + ", " - + qi("processed") + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; + + qi("processed") + ", " + qi("httpBroadcastDeliveryIds") + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; try (Connection conn = mysql.getConnectionManager().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { ps.setString(1, playerName); @@ -183,6 +186,7 @@ public boolean insertTimedVote(UUID voteId, String uuid, String playerName, Stri } else { ps.setInt(10, processed ? 1 : 0); } + ps.setString(11, httpBroadcastDeliveryIds); ps.executeUpdate(); return true; } catch (SQLException e) { @@ -191,6 +195,14 @@ public boolean insertTimedVote(UUID voteId, String uuid, String playerName, Stri } } + /** Backward-compatible insert overload without HTTP delivery state. */ + public boolean insertTimedVote(UUID voteId, String uuid, String playerName, String service, long time, + boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, + boolean processed) { + return insertTimedVote(voteId, uuid, playerName, service, time, proxyBroadcastHandled, broadcastTargets, + broadcastForwardedServers, totals, processed, ""); + } + /** * Updates a queued vote's standalone broadcast delivery state. * @@ -201,7 +213,8 @@ public boolean updateTimedVote(VoteTimeQueue vote) { boolean hasVoteId = vote.getVoteId() != null; String sql = "UPDATE " + qi(getTableName()) + " SET " + qi("proxyBroadcastHandled") + " = ?, " + qi("broadcastTargets") + " = ?, " + qi("broadcastForwardedServers") + " = ?, " + qi("totals") - + " = ?, " + qi("processed") + " = ?, " + qi("uuid") + " = ? WHERE " + + " = ?, " + qi("processed") + " = ?, " + qi("uuid") + " = ?, " + qi("httpBroadcastDeliveryIds") + + " = ? WHERE " + (hasVoteId ? qi("voteId") + " = ?;" : qi("playerName") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ?;"); try (Connection conn = mysql.getConnectionManager().getConnection(); @@ -220,12 +233,13 @@ public boolean updateTimedVote(VoteTimeQueue vote) { ps.setInt(5, vote.isProcessed() ? 1 : 0); } ps.setString(6, vote.getUuid()); + ps.setString(7, vote.encodeHttpBroadcastDeliveryIds()); if (hasVoteId) { - ps.setString(7, vote.getVoteId().toString()); + ps.setString(8, vote.getVoteId().toString()); } else { - ps.setString(7, vote.getName()); - ps.setString(8, vote.getService()); - ps.setLong(9, vote.getTime()); + ps.setString(8, vote.getName()); + ps.setString(9, vote.getService()); + ps.setLong(10, vote.getTime()); } ps.executeUpdate(); return true; @@ -352,7 +366,8 @@ private List selectVotes(String sql, Object[] params) { rs.getString("broadcastTargets"), rs.getString("broadcastForwardedServers"), rs.getString("totals"), - rs.getBoolean("processed") + rs.getBoolean("processed"), + rs.getString("httpBroadcastDeliveryIds") )); } } @@ -388,6 +403,7 @@ public static class TimedVoteRow { private final String broadcastForwardedServers; private final String totals; private final boolean processed; + private final String httpBroadcastDeliveryIds; /** * Constructor for TimedVoteRow. @@ -404,7 +420,7 @@ public static class TimedVoteRow { */ public TimedVoteRow(int id, String playerName, String service, long time, UUID voteId, String uuid, boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, - boolean processed) { + boolean processed, String httpBroadcastDeliveryIds) { this.id = id; this.playerName = playerName; this.service = service; @@ -416,6 +432,15 @@ public TimedVoteRow(int id, String playerName, String service, long time, UUID v this.broadcastForwardedServers = broadcastForwardedServers; this.totals = totals; this.processed = processed; + this.httpBroadcastDeliveryIds = httpBroadcastDeliveryIds; + } + + /** Backward-compatible row constructor without HTTP delivery state. */ + public TimedVoteRow(int id, String playerName, String service, long time, UUID voteId, String uuid, + boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, + boolean processed) { + this(id, playerName, service, time, voteId, uuid, proxyBroadcastHandled, broadcastTargets, + broadcastForwardedServers, totals, processed, null); } /** @@ -490,5 +515,9 @@ public String getTotals() { public boolean isProcessed() { return processed; } + + public String getHttpBroadcastDeliveryIds() { + return httpBroadcastDeliveryIds; + } } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java index d9e48e993..9904df3f3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java @@ -58,6 +58,8 @@ public String buildCreateTableSql(DbType dbType) { .append(qi("proxyBroadcastHandled")).append(" BOOLEAN NOT NULL DEFAULT FALSE, ") .append(qi("broadcastTargets")).append(" TEXT, ").append(qi("broadcastForwardedServers")) .append(" TEXT, ").append(qi("rewardDelivered")).append(" BOOLEAN NOT NULL DEFAULT FALSE, ") + .append(qi("httpDeliveryIds")).append(" TEXT, ") + .append(qi("httpBroadcastDeliveryIds")).append(" TEXT, ") .append(qi("server")).append(" VARCHAR(100)").append(");"); } else { sb.append("CREATE TABLE IF NOT EXISTS ").append(qi(getTableName())).append(" (").append(qi("id")) @@ -69,6 +71,8 @@ public String buildCreateTableSql(DbType dbType) { .append(qi("proxyBroadcastHandled")).append(" TINYINT(1) NOT NULL DEFAULT 0,") .append(qi("broadcastTargets")).append(" TEXT,").append(qi("broadcastForwardedServers")) .append(" TEXT,").append(qi("rewardDelivered")).append(" TINYINT(1) NOT NULL DEFAULT 0,") + .append(qi("httpDeliveryIds")).append(" TEXT,") + .append(qi("httpBroadcastDeliveryIds")).append(" TEXT,") .append(qi("server")).append(" VARCHAR(100),").append("INDEX idx_server (").append(qi("server")) .append("),").append("INDEX idx_uuid (").append(qi("uuid")).append("),").append("INDEX idx_time (") .append(qi("time")).append(")").append(") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"); @@ -126,7 +130,8 @@ public boolean updateProxyBroadcastState(OfflineBungeeVote vote, String server) boolean hasVoteId = vote.getVoteId() != null; String sql = "UPDATE " + qi(getTableName()) + " SET " + qi("broadcastForwarded") + " = ?, " + qi("proxyBroadcastHandled") + " = ?, " + qi("broadcastTargets") + " = ?, " - + qi("broadcastForwardedServers") + " = ?, " + qi("rewardDelivered") + " = ? WHERE " + + qi("broadcastForwardedServers") + " = ?, " + qi("rewardDelivered") + " = ?, " + + qi("httpDeliveryIds") + " = ?, " + qi("httpBroadcastDeliveryIds") + " = ? WHERE " + (hasVoteId ? qi("voteid") + " = ? AND " + qi("server") + " = ?;" : qi("uuid") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ? AND " + qi("server") + " = ?;"); @@ -144,18 +149,20 @@ public boolean updateProxyBroadcastState(OfflineBungeeVote vote, String server) } ps.setString(3, vote.encodeBroadcastTargets()); ps.setString(4, vote.encodeBroadcastForwardedServers()); + ps.setString(6, vote.encodeHttpDeliveryIds()); + ps.setString(7, vote.encodeHttpBroadcastDeliveryIds()); if (hasVoteId) { - ps.setString(6, vote.getVoteId().toString()); - ps.setString(7, server); + ps.setString(8, vote.getVoteId().toString()); + ps.setString(9, server); } else { if (getDbType() == DbType.POSTGRESQL) { - ps.setObject(6, UUID.fromString(vote.getUuid())); + ps.setObject(8, UUID.fromString(vote.getUuid())); } else { - ps.setString(6, vote.getUuid()); + ps.setString(8, vote.getUuid()); } - ps.setString(7, vote.getService()); - ps.setLong(8, vote.getTime()); - ps.setString(9, server); + ps.setString(9, vote.getService()); + ps.setLong(10, vote.getTime()); + ps.setString(11, server); } return ps.executeUpdate() > 0; } catch (SQLException | IllegalArgumentException e) { @@ -284,6 +291,8 @@ private void ensureProxyBroadcastStateColumns() { ensureColumn("broadcastTargets", "TEXT"); ensureColumn("broadcastForwardedServers", "TEXT"); ensureColumn("rewardDelivered", booleanType); + ensureColumn("httpDeliveryIds", "TEXT"); + ensureColumn("httpBroadcastDeliveryIds", "TEXT"); } private void ensureColumn(String column, String type) { @@ -333,13 +342,15 @@ private void ensureColumn(String column, String type) { */ public void insertVote(UUID voteId, String uuid, String playerName, String service, long time, boolean real, String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, String broadcastTargets, - String broadcastForwardedServers, boolean rewardDelivered, String server) { + String broadcastForwardedServers, boolean rewardDelivered, String httpDeliveryIds, + String httpBroadcastDeliveryIds, String server) { String sql = "INSERT INTO " + qi(getTableName()) + " (" + qi("uuid") + ", " + qi("voteid") + ", " + qi("playerName") + ", " + qi("service") + ", " + qi("time") + ", " + qi("realVote") + ", " + qi("text") + ", " + qi("broadcastForwarded") + ", " + qi("proxyBroadcastHandled") + ", " + qi("broadcastTargets") + ", " + qi("broadcastForwardedServers") + ", " + qi("rewardDelivered") - + ", " + qi("server") + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; + + ", " + qi("httpDeliveryIds") + ", " + qi("httpBroadcastDeliveryIds") + ", " + qi("server") + + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; try (Connection conn = mysql.getConnectionManager().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { @@ -373,7 +384,9 @@ public void insertVote(UUID voteId, String uuid, String playerName, String servi } ps.setString(10, broadcastTargets); ps.setString(11, broadcastForwardedServers); - ps.setString(13, server); + ps.setString(13, httpDeliveryIds); + ps.setString(14, httpBroadcastDeliveryIds); + ps.setString(15, server); ps.executeUpdate(); } catch (SQLException | IllegalArgumentException e) { @@ -578,6 +591,8 @@ private List selectVotes(String sql, Object[] params) { rs.getString("text"), rs.getBoolean("broadcastForwarded"), rs.getBoolean("proxyBroadcastHandled"), rs.getString("broadcastTargets"), rs.getString("broadcastForwardedServers"), rs.getBoolean("rewardDelivered"), + rs.getString("httpDeliveryIds"), + rs.getString("httpBroadcastDeliveryIds"), rs.getString("server")); list.add(v); } @@ -605,6 +620,8 @@ public static class VoteRow { private final String broadcastTargets; private final String broadcastForwardedServers; private final boolean rewardDelivered; + private final String httpDeliveryIds; + private final String httpBroadcastDeliveryIds; private final String server; /** @@ -626,7 +643,8 @@ public static class VoteRow { */ public VoteRow(int id, String voteId, String uuid, String playerName, String service, long time, boolean realVote, String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, - String broadcastTargets, String broadcastForwardedServers, boolean rewardDelivered, String server) { + String broadcastTargets, String broadcastForwardedServers, boolean rewardDelivered, String httpDeliveryIds, + String httpBroadcastDeliveryIds, String server) { this.id = id; this.voteId = voteId; this.uuid = uuid; @@ -640,6 +658,8 @@ public VoteRow(int id, String voteId, String uuid, String playerName, String ser this.broadcastTargets = broadcastTargets; this.broadcastForwardedServers = broadcastForwardedServers; this.rewardDelivered = rewardDelivered; + this.httpDeliveryIds = httpDeliveryIds; + this.httpBroadcastDeliveryIds = httpBroadcastDeliveryIds; this.server = server; } @@ -731,6 +751,14 @@ public boolean isRewardDelivered() { return rewardDelivered; } + public String getHttpDeliveryIds() { + return httpDeliveryIds; + } + + public String getHttpBroadcastDeliveryIds() { + return httpBroadcastDeliveryIds; + } + /** * Gets the server name. * @return the server name diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index 01c079413..d51598406 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -104,7 +104,8 @@ public synchronized void addServerVote(String server, OfflineBungeeVote vote) { voteCacheTable.insertVote(vote.getVoteId(), vote.getUuid(), vote.getPlayerName(), vote.getService(), vote.getTime(), vote.isRealVote(), vote.getText(), vote.isBroadcastForwarded(), vote.isProxyBroadcastHandled(), vote.encodeBroadcastTargets(), - vote.encodeBroadcastForwardedServers(), vote.isRewardDelivered(), server); + vote.encodeBroadcastForwardedServers(), vote.isRewardDelivered(), vote.encodeHttpDeliveryIds(), + vote.encodeHttpBroadcastDeliveryIds(), server); } else { // IMPORTANT: index must come from JSON, not from cachedVotes (cache can be out // of sync with JSON) @@ -227,7 +228,8 @@ public synchronized void addOnlineVote(String uuid, OfflineBungeeVote vote) { onlineVoteCacheTable.insertVote(vote.getVoteId(), vote.getUuid(), vote.getPlayerName(), vote.getService(), vote.getTime(), vote.isRealVote(), vote.getText(), vote.isBroadcastForwarded(), vote.isProxyBroadcastHandled(), vote.encodeBroadcastTargets(), - vote.encodeBroadcastForwardedServers(), vote.isRewardDelivered()); + vote.encodeBroadcastForwardedServers(), vote.isRewardDelivered(), vote.encodeHttpDeliveryIds(), + vote.encodeHttpBroadcastDeliveryIds()); } else { // IMPORTANT: index must come from JSON, not from cachedOnlineVotes (cache can // be out of sync with JSON) @@ -501,7 +503,8 @@ public synchronized boolean addTimeVoteToCache(VoteTimeQueue vote) { if (useMySQL) { boolean stored = timedVoteCacheTable.insertTimedVote(vote.getVoteId(), vote.getUuid(), vote.getName(), vote.getService(), vote.getTime(), vote.isProxyBroadcastHandled(), vote.encodeBroadcastTargets(), - vote.encodeBroadcastForwardedServers(), vote.getTotals(), vote.isProcessed()); + vote.encodeBroadcastForwardedServers(), vote.getTotals(), vote.isProcessed(), + vote.encodeHttpBroadcastDeliveryIds()); if (!stored) { timeChangeQueue.remove(vote); } @@ -602,7 +605,8 @@ public void load() { voteRow.getText(), voteRow.isBroadcastForwarded(), voteRow.isProxyBroadcastHandled(), VoteTimeQueue.decodeBroadcastForwardedServers(voteRow.getBroadcastTargets()), VoteTimeQueue.decodeBroadcastForwardedServers(voteRow.getBroadcastForwardedServers()), - voteRow.isRewardDelivered()); + voteRow.isRewardDelivered(), OfflineBungeeVote.decodeHttpDeliveryIds(voteRow.getHttpDeliveryIds()), + OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(voteRow.getHttpBroadcastDeliveryIds())); String server = voteRow.getServer(); cachedVotes.putIfAbsent(server, new ArrayList<>()); cachedVotes.get(server).add(vote); @@ -615,7 +619,8 @@ public void load() { voteRow.getText(), voteRow.isBroadcastForwarded(), voteRow.isProxyBroadcastHandled(), VoteTimeQueue.decodeBroadcastForwardedServers(voteRow.getBroadcastTargets()), VoteTimeQueue.decodeBroadcastForwardedServers(voteRow.getBroadcastForwardedServers()), - voteRow.isRewardDelivered()); + voteRow.isRewardDelivered(), OfflineBungeeVote.decodeHttpDeliveryIds(voteRow.getHttpDeliveryIds()), + OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(voteRow.getHttpBroadcastDeliveryIds())); String player = vote.getUuid(); cachedOnlineVotes.putIfAbsent(player, new ArrayList<>()); cachedOnlineVotes.get(player).add(vote); @@ -628,7 +633,8 @@ public void load() { timedVoteRow.getService(), timedVoteRow.getTime(), timedVoteRow.isProxyBroadcastHandled(), VoteTimeQueue.decodeBroadcastForwardedServers(timedVoteRow.getBroadcastTargets()), VoteTimeQueue.decodeBroadcastForwardedServers(timedVoteRow.getBroadcastForwardedServers()), - timedVoteRow.getTotals(), timedVoteRow.isProcessed(), timedVoteRow.getUuid()); + timedVoteRow.getTotals(), timedVoteRow.isProcessed(), timedVoteRow.getUuid(), + VoteTimeQueue.decodeHttpBroadcastDeliveryIds(timedVoteRow.getHttpBroadcastDeliveryIds())); timedVotes.add(voteTimeQueue); }); timeChangeQueue.addAll(timedVotes); @@ -654,10 +660,13 @@ public void load() { : ""; String totals = data.has("Totals") ? data.get("Totals").asString() : ""; boolean processed = data.has("Processed") && data.get("Processed").asBoolean(); + String httpBroadcastDeliveryIds = data.has("HttpBroadcastDeliveryIds") + ? data.get("HttpBroadcastDeliveryIds").asString() : ""; getTimeChangeQueue().add(new VoteTimeQueue(voteId, name, service, time, proxyBroadcastHandled, VoteTimeQueue.decodeBroadcastForwardedServers(broadcastTargets), - VoteTimeQueue.decodeBroadcastForwardedServers(forwardedServers), totals, processed, uuid)); + VoteTimeQueue.decodeBroadcastForwardedServers(forwardedServers), totals, processed, uuid, + VoteTimeQueue.decodeHttpBroadcastDeliveryIds(httpBroadcastDeliveryIds))); } } @@ -690,6 +699,10 @@ public void load() { String broadcastForwardedServers = data.has("BroadcastForwardedServers") ? data.get("BroadcastForwardedServers").asString() : ""; + String httpDeliveryIds = data.has("HttpDeliveryIds") + ? data.get("HttpDeliveryIds").asString() : ""; + String httpBroadcastDeliveryIds = data.has("HttpBroadcastDeliveryIds") + ? data.get("HttpBroadcastDeliveryIds").asString() : ""; boolean rewardDelivered = data.has("RewardDelivered") && data.get("RewardDelivered").asBoolean(); @@ -697,7 +710,8 @@ public void load() { broadcastForwarded, proxyBroadcastHandled, VoteTimeQueue.decodeBroadcastForwardedServers(broadcastTargets), VoteTimeQueue.decodeBroadcastForwardedServers(broadcastForwardedServers), - rewardDelivered)); + rewardDelivered, OfflineBungeeVote.decodeHttpDeliveryIds(httpDeliveryIds), + OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(httpBroadcastDeliveryIds))); } } cachedVotes.put(server, votes); @@ -731,6 +745,10 @@ public void load() { String broadcastForwardedServers = data.has("BroadcastForwardedServers") ? data.get("BroadcastForwardedServers").asString() : ""; + String httpDeliveryIds = data.has("HttpDeliveryIds") + ? data.get("HttpDeliveryIds").asString() : ""; + String httpBroadcastDeliveryIds = data.has("HttpBroadcastDeliveryIds") + ? data.get("HttpBroadcastDeliveryIds").asString() : ""; boolean rewardDelivered = data.has("RewardDelivered") && data.get("RewardDelivered").asBoolean(); @@ -738,7 +756,8 @@ public void load() { broadcastForwarded, proxyBroadcastHandled, VoteTimeQueue.decodeBroadcastForwardedServers(broadcastTargets), VoteTimeQueue.decodeBroadcastForwardedServers(broadcastForwardedServers), - rewardDelivered)); + rewardDelivered, OfflineBungeeVote.decodeHttpDeliveryIds(httpDeliveryIds), + OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(httpBroadcastDeliveryIds))); } } cachedOnlineVotes.put(player, votes); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java index 32a0e23cc..5649ab3b0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java @@ -43,6 +43,8 @@ public void addTimedVote(int num, VoteTimeQueue voteTimedQueue) { setPath(voteTimedQueue.encodeBroadcastTargets(), "TimedVoteCache", String.valueOf(num), "BroadcastTargets"); setPath(voteTimedQueue.encodeBroadcastForwardedServers(), "TimedVoteCache", String.valueOf(num), "BroadcastForwardedServers"); + setPath(voteTimedQueue.encodeHttpBroadcastDeliveryIds(), "TimedVoteCache", String.valueOf(num), + "HttpBroadcastDeliveryIds"); } @Override @@ -61,6 +63,8 @@ public void addVote(String server, int num, OfflineBungeeVote voteData) { setPath(voteData.encodeBroadcastForwardedServers(), "VoteCache", server, String.valueOf(num), "BroadcastForwardedServers"); setPath(voteData.isRewardDelivered(), "VoteCache", server, String.valueOf(num), "RewardDelivered"); + setPath(voteData.encodeHttpDeliveryIds(), "VoteCache", server, String.valueOf(num), "HttpDeliveryIds"); + setPath(voteData.encodeHttpBroadcastDeliveryIds(), "VoteCache", server, String.valueOf(num), "HttpBroadcastDeliveryIds"); } @Override @@ -79,6 +83,8 @@ public void addVoteOnline(String player, int num, OfflineBungeeVote voteData) { setPath(voteData.encodeBroadcastForwardedServers(), "OnlineCache", player, String.valueOf(num), "BroadcastForwardedServers"); setPath(voteData.isRewardDelivered(), "OnlineCache", player, String.valueOf(num), "RewardDelivered"); + setPath(voteData.encodeHttpDeliveryIds(), "OnlineCache", player, String.valueOf(num), "HttpDeliveryIds"); + setPath(voteData.encodeHttpBroadcastDeliveryIds(), "OnlineCache", player, String.valueOf(num), "HttpBroadcastDeliveryIds"); } @Override diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java index 7f3f5cf9a..6c3df8256 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java @@ -4,6 +4,9 @@ import java.util.Base64; import java.util.Collections; import java.util.LinkedHashSet; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.UUID; @@ -45,6 +48,8 @@ public class VoteTimeQueue { private Set broadcastTargets; @Getter private Set broadcastForwardedServers; + /** Stable HTTP standalone-broadcast delivery IDs by target server. */ + private final Map httpBroadcastDeliveryIds; /** * Creates a legacy-compatible queued vote without an identifier. @@ -124,6 +129,14 @@ public VoteTimeQueue(UUID voteId, String name, String service, long time, boolea public VoteTimeQueue(UUID voteId, String name, String service, long time, boolean proxyBroadcastHandled, Set broadcastTargets, Set broadcastForwardedServers, String totals, boolean processed, String uuid) { + this(voteId, name, service, time, proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, totals, + processed, uuid, Collections.emptyMap()); + } + + /** Creates a queued vote with persisted HTTP standalone-broadcast IDs. */ + public VoteTimeQueue(UUID voteId, String name, String service, long time, boolean proxyBroadcastHandled, + Set broadcastTargets, Set broadcastForwardedServers, String totals, boolean processed, + String uuid, Map httpBroadcastDeliveryIds) { this.voteId = voteId; this.uuid = uuid == null ? "" : uuid; this.name = name; @@ -140,6 +153,64 @@ public VoteTimeQueue(UUID voteId, String name, String service, long time, boolea if (broadcastForwardedServers != null) { this.broadcastForwardedServers.addAll(broadcastForwardedServers); } + this.httpBroadcastDeliveryIds = new LinkedHashMap<>(); + if (httpBroadcastDeliveryIds != null) { + httpBroadcastDeliveryIds.forEach(this::setHttpBroadcastDeliveryId); + } + } + + public String getHttpBroadcastDeliveryId(String server) { + return server == null ? null : httpBroadcastDeliveryIds.get(server.toLowerCase(Locale.ROOT)); + } + + public void setHttpBroadcastDeliveryId(String server, String deliveryId) { + if (server == null || server.isBlank()) return; + String key = server.toLowerCase(Locale.ROOT); + if (deliveryId == null || deliveryId.isBlank()) httpBroadcastDeliveryIds.remove(key); + else httpBroadcastDeliveryIds.put(key, deliveryId); + } + + public Map getHttpBroadcastDeliveryIds() { + return new LinkedHashMap<>(httpBroadcastDeliveryIds); + } + + /** Returns whether any standalone HTTP broadcast still has to be delivered. */ + public boolean hasPendingHttpBroadcastDeliveryIds() { + return !httpBroadcastDeliveryIds.isEmpty(); + } + + public String encodeHttpBroadcastDeliveryIds() { + StringBuilder encoded = new StringBuilder(); + for (Map.Entry entry : httpBroadcastDeliveryIds.entrySet()) { + if (encoded.length() > 0) encoded.append('.'); + encoded.append(Base64.getUrlEncoder().withoutPadding() + .encodeToString(entry.getKey().getBytes(StandardCharsets.UTF_8))); + encoded.append('~'); + encoded.append(Base64.getUrlEncoder().withoutPadding() + .encodeToString(entry.getValue().getBytes(StandardCharsets.UTF_8))); + } + return encoded.toString(); + } + + public static Map decodeHttpBroadcastDeliveryIds(String encoded) { + Map decoded = new LinkedHashMap<>(); + if (encoded == null || encoded.isBlank()) return decoded; + for (String entry : encoded.split("\\.", -1)) { + int separator = entry.indexOf('~'); + if (separator <= 0 || separator == entry.length() - 1) continue; + try { + String server = new String(Base64.getUrlDecoder().decode(entry.substring(0, separator)), + StandardCharsets.UTF_8); + String deliveryId = new String(Base64.getUrlDecoder().decode(entry.substring(separator + 1)), + StandardCharsets.UTF_8); + if (!server.isBlank() && deliveryId.matches("[0-9a-fA-F-]{36}")) { + decoded.put(server.toLowerCase(Locale.ROOT), deliveryId); + } + } catch (IllegalArgumentException ignored) { + // Ignore corrupt optional delivery state and retain the queued vote. + } + } + return decoded; } /** diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index c58676f17..5be3eae3b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -375,7 +375,7 @@ void releasesMysqlSubscriberBeforeSameMethodReplacement() throws Exception { } @Test - void keepsPromotedRedisReplacementWhenOldListenerRetirementFails() throws Exception { + void promotesRedisReplacementAfterOldListenerIsFencedEvenWhenRetirementFails() throws Exception { com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); java.util.logging.Logger logger = mock(java.util.logging.Logger.class); when(plugin.getLogger()).thenReturn(logger); @@ -391,8 +391,8 @@ void keepsPromotedRedisReplacementWhenOldListenerRetirementFails() throws Except previous.completeRedisHandoff(replacement); org.mockito.InOrder order = inOrder(newTransport, oldTransport); - order.verify(newTransport).activateAfterHandoff(); order.verify(oldTransport).closeForHandoff(); + order.verify(newTransport).activateAfterHandoff(); assertSame(newTransport, transport(replacement)); assertNull(transport(previous)); @@ -402,6 +402,25 @@ void keepsPromotedRedisReplacementWhenOldListenerRetirementFails() throws Except verify(oldTransport, times(2)).close(); } + @Test + void doesNotPromoteRedisReplacementWhenOldCallbacksDoNotQuiesce() throws Exception { + BackendProxyTransportManager previous = new BackendProxyTransportManager(null); + BackendProxyTransportManager replacement = new BackendProxyTransportManager(null); + RedisBackendProxyTransport oldTransport = mock(RedisBackendProxyTransport.class); + RedisBackendProxyTransport newTransport = mock(RedisBackendProxyTransport.class); + setField(previous, "transport", oldTransport); + setField(replacement, "transport", newTransport); + doThrow(new RedisBackendProxyTransport.HandoffQuiescenceException("busy")) + .when(oldTransport).closeForHandoff(); + + assertThrows(RedisBackendProxyTransport.HandoffQuiescenceException.class, + () -> previous.completeRedisHandoff(replacement)); + + verify(newTransport, never()).activateAfterHandoff(); + assertSame(oldTransport, transport(previous)); + assertSame(newTransport, transport(replacement)); + } + @Test void stopsGlobalDataTimerWhenHandlerIsReplaced() throws Exception { BackendGlobalDataSync handler = new BackendGlobalDataSync(null, null); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 81d319b23..df139a7fd 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -104,6 +104,44 @@ void orderlyShutdownFlushesHandoffAfterMakingConnectorCapacity() { order.verify(connector).flushOutgoing(org.mockito.ArgumentMatchers.anyLong()); } + @Test + @SuppressWarnings("unchecked") + void unpublishedHandoffMessagesAreDiscardedOnRollback() throws Exception { + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(mock(VotingPluginMain.class)); + HttpBackendTransportConnector connector = mock(HttpBackendTransportConnector.class); + setField(transport, "connector", connector); + ((java.util.ArrayDeque) field(transport, "handoffQueue")) + .add(JsonEnvelope.builder("unpublished").build()); + + transport.close(); + + verify(connector, org.mockito.Mockito.timeout(1000)).close(); + verify(connector, org.mockito.Mockito.never()).send(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); + } + + @Test + @SuppressWarnings("unchecked") + void orderlyShutdownFlushesRecoveryQueueBeforeHandoffQueue() throws Exception { + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(mock(VotingPluginMain.class)); + HttpBackendTransportConnector connector = mock(HttpBackendTransportConnector.class); + JsonEnvelope recovery = JsonEnvelope.builder("recovery").build(); + JsonEnvelope handoff = JsonEnvelope.builder("handoff").build(); + setField(transport, "connector", connector); + setField(transport, "published", true); + ((java.util.ArrayDeque) field(transport, "startupQueue")).add(recovery); + ((java.util.ArrayDeque) field(transport, "handoffQueue")).add(handoff); + when(connector.send(org.mockito.ArgumentMatchers.any(JsonEnvelope.class))).thenReturn(true); + when(connector.flushOutgoing(org.mockito.ArgumentMatchers.anyLong())).thenReturn(true); + + transport.close(); + + verify(connector, org.mockito.Mockito.timeout(1000)).flushOutgoing(org.mockito.ArgumentMatchers.anyLong()); + org.mockito.InOrder order = org.mockito.Mockito.inOrder(connector); + order.verify(connector).send(recovery); + order.verify(connector).send(handoff); + order.verify(connector).flushOutgoing(org.mockito.ArgumentMatchers.anyLong()); + } + @Test void failedPreparationAfterCloseRestoresCapturedHttpTransport() throws Exception { BackendProxyTransportManager manager = new BackendProxyTransportManager(mock(VotingPluginMain.class)); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java index 0f5dd5b10..aef9eb9e5 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.doAnswer; @@ -104,7 +105,119 @@ void retiredListenerDoesNotReserveOrDispatchDeliveryIds() throws Exception { } @Test - void fencingDoesNotWaitForAnIdentifiedMessageHandler() throws Exception { + void standbyBuffersIdentifiedDeliveriesUntilItIsPublished() throws Exception { + ProcessedVoteCache cache = new ProcessedVoteCache(); + cache.registerRedisSubscriber(new Object()); + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, cache); + Field identity = RedisBackendProxyTransport.class.getDeclaredField("subscriberIdentity"); + identity.setAccessible(true); + cache.registerRedisSubscriber(identity.get(transport)); + Field standby = RedisBackendProxyTransport.class.getDeclaredField("standbySubscriber"); + standby.setAccessible(true); + standby.setBoolean(transport, true); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + Field handler = RedisBackendProxyTransport.class.getDeclaredField("messageHandler"); + handler.setAccessible(true); + handler.set(transport, messages); + String deliveryId = "00000000-0000-0000-0000-000000000002"; + JsonEnvelope envelope = JsonEnvelope.builder(VotingPluginWire.SUB_VOTE_UPDATE) + .put(VotingPluginWire.K_REDIS_DELIVERY_ID, deliveryId).build(); + + transport.dispatchIdentified(envelope, deliveryId); + + verifyNoInteractions(messages); + transport.activateAfterHandoff(); + verify(messages).onMessage(envelope); + assertFalse(cache.reserveRedisDelivery(deliveryId), + "publication must reserve the delivery exactly when its buffered callback is replayed"); + } + + @Test + void standbyReplaysMixedLegacyAndIdentifiedDeliveriesInArrivalOrder() throws Exception { + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.reserveRedisDelivery(org.mockito.ArgumentMatchers.anyString())).thenReturn(true); + when(cache.consumeLegacyRedisDelivery(org.mockito.ArgumentMatchers.anyString())).thenReturn(false); + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, cache); + setBoolean(transport, "standbySubscriber", true); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + setField(transport, "messageHandler", messages); + JsonEnvelope legacy = JsonEnvelope.builder("legacy").build(); + String deliveryId = "00000000-0000-0000-0000-000000000003"; + JsonEnvelope identified = JsonEnvelope.builder("identified") + .put(VotingPluginWire.K_REDIS_DELIVERY_ID, deliveryId).build(); + + transport.dispatchLegacy(legacy); + transport.dispatchIdentified(identified, deliveryId); + transport.activateAfterHandoff(); + + org.mockito.InOrder order = org.mockito.Mockito.inOrder(messages); + order.verify(messages).onMessage(legacy); + order.verify(messages).onMessage(identified); + } + + @Test + void identifiedStandbyOverflowAbortsPublication() throws Exception { + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, mock(ProcessedVoteCache.class)); + setBoolean(transport, "standbySubscriber", true); + setField(transport, "messageHandler", mock(GlobalMessageHandler.class)); + for (int index = 0; index <= RedisBackendProxyTransport.MAX_IDENTIFIED_HANDOFF_DELIVERIES; index++) { + String deliveryId = String.format("00000000-0000-0000-0000-%012d", index); + transport.dispatchIdentified(JsonEnvelope.builder("identified") + .put(VotingPluginWire.K_REDIS_DELIVERY_ID, deliveryId).build(), deliveryId); + } + + assertThrows(IllegalStateException.class, transport::activateAfterHandoff); + } + + @Test + void oversizedLegacyStandbyDeliveryAbortsPublicationWithoutDispatching() throws Exception { + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, mock(ProcessedVoteCache.class)); + setBoolean(transport, "standbySubscriber", true); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + setField(transport, "messageHandler", messages); + JsonEnvelope oversized = JsonEnvelope.builder("legacy") + .put("payload", "x".repeat(ProcessedVoteCache.MAX_LEGACY_REDIS_DELIVERY_BYTES + 1)).build(); + + transport.dispatchLegacy(oversized); + + verifyNoInteractions(messages); + assertThrows(IllegalStateException.class, transport::activateAfterHandoff); + } + + @Test + void legacyStandbyBufferOverflowAbortsPublicationWithoutDispatching() throws Exception { + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, mock(ProcessedVoteCache.class)); + setBoolean(transport, "standbySubscriber", true); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + setField(transport, "messageHandler", messages); + + for (int index = 0; index <= RedisBackendProxyTransport.MAX_LEGACY_HANDOFF_DELIVERIES; index++) + transport.dispatchLegacy(JsonEnvelope.builder("legacy-" + index).build()); + + verifyNoInteractions(messages); + assertThrows(IllegalStateException.class, transport::activateAfterHandoff); + } + + @Test + void activeLegacyAccountingOverflowAbortsLaggingStandbyPublication() throws Exception { + ProcessedVoteCache cache = new ProcessedVoteCache(); + Object previousIdentity = new Object(); + cache.registerRedisSubscriber(previousIdentity); + RedisBackendProxyTransport replacement = new RedisBackendProxyTransport(null, cache); + Field identity = RedisBackendProxyTransport.class.getDeclaredField("subscriberIdentity"); + identity.setAccessible(true); + cache.registerRedisSubscriber(identity.get(replacement)); + setBoolean(replacement, "standbySubscriber", true); + + for (int index = 0; index <= 4096; index++) { + cache.reserveLegacyRedisDelivery(previousIdentity, "old-only-" + index); + } + + assertThrows(IllegalStateException.class, replacement::activateAfterHandoff); + } + + @Test + void fencingWaitsForAnIdentifiedMessageHandler() throws Exception { RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, new ProcessedVoteCache()); CountDownLatch entered = new CountDownLatch(1); CountDownLatch release = new CountDownLatch(1); @@ -127,12 +240,21 @@ void fencingDoesNotWaitForAnIdentifiedMessageHandler() throws Exception { assertTrue(entered.await(1, TimeUnit.SECONDS)); CountDownLatch fenced = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference fenceFailure = + new java.util.concurrent.atomic.AtomicReference<>(); Thread retirement = new Thread(() -> { - transport.fenceAfterHandoff(); - fenced.countDown(); + try { + transport.fenceAfterHandoff(); + } catch (Throwable failure) { + fenceFailure.set(failure); + } finally { + fenced.countDown(); + } }); retirement.start(); try { + assertFalse(fenced.await(100, TimeUnit.MILLISECONDS)); + retirement.interrupt(); assertTrue(fenced.await(1, TimeUnit.SECONDS)); } finally { release.countDown(); @@ -141,10 +263,53 @@ void fencingDoesNotWaitForAnIdentifiedMessageHandler() throws Exception { retirement.join(TimeUnit.SECONDS.toMillis(1)); assertFalse(delivery.isAlive()); assertFalse(retirement.isAlive()); + assertTrue(fenceFailure.get() instanceof RedisBackendProxyTransport.HandoffQuiescenceException); + } + + @Test + void legacyPromotionWaitsForReservedDeliveryBeforeReplayingStandbyCopy() throws Exception { + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.reserveLegacyRedisDelivery(org.mockito.ArgumentMatchers.any(), + org.mockito.ArgumentMatchers.anyString())).thenReturn(true, false); + when(cache.consumeLegacyRedisDelivery(org.mockito.ArgumentMatchers.anyString())).thenReturn(true); + RedisBackendProxyTransport previous = new RedisBackendProxyTransport(null, cache); + RedisBackendProxyTransport replacement = new RedisBackendProxyTransport(null, cache); + setBoolean(replacement, "standbySubscriber", true); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + GlobalMessageHandler previousMessages = mock(GlobalMessageHandler.class); + doAnswer(invocation -> { + entered.countDown(); + release.await(); + return null; + }).when(previousMessages).onMessage(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); + GlobalMessageHandler replacementMessages = mock(GlobalMessageHandler.class); + setField(previous, "messageHandler", previousMessages); + setField(replacement, "messageHandler", replacementMessages); + JsonEnvelope envelope = JsonEnvelope.builder(VotingPluginWire.SUB_VOTE_UPDATE).build(); + + Thread delivery = new Thread(() -> previous.dispatchLegacy(envelope)); + delivery.start(); + assertTrue(entered.await(1, TimeUnit.SECONDS)); + replacement.dispatchLegacy(envelope); + CountDownLatch promoted = new CountDownLatch(1); + Thread promotion = new Thread(() -> { + previous.fenceAfterHandoff(); + replacement.activateAfterHandoff(); + promoted.countDown(); + }); + promotion.start(); + assertFalse(promoted.await(100, TimeUnit.MILLISECONDS)); + + release.countDown(); + assertTrue(promoted.await(1, TimeUnit.SECONDS)); + delivery.join(TimeUnit.SECONDS.toMillis(1)); + promotion.join(TimeUnit.SECONDS.toMillis(1)); + verify(previousMessages).onMessage(envelope); + verifyNoInteractions(replacementMessages); } @Test - @SuppressWarnings("unchecked") void replayCompletesBeforeNewlyAdmittedDeliveries() throws Exception { ProcessedVoteCache cache = mock(ProcessedVoteCache.class); when(cache.reserveRedisDelivery("new-delivery")).thenReturn(true); @@ -152,9 +317,6 @@ void replayCompletesBeforeNewlyAdmittedDeliveries() throws Exception { RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, cache); JsonEnvelope replayed = JsonEnvelope.builder("old").build(); JsonEnvelope newer = JsonEnvelope.builder("new").build(); - Field buffered = RedisBackendProxyTransport.class.getDeclaredField("bufferedLegacyDeliveries"); - buffered.setAccessible(true); - ((java.util.List) buffered.get(transport)).add(replayed); CopyOnWriteArrayList order = new CopyOnWriteArrayList<>(); GlobalMessageHandler messages = mock(GlobalMessageHandler.class); doAnswer(invocation -> { @@ -170,9 +332,22 @@ void replayCompletesBeforeNewlyAdmittedDeliveries() throws Exception { Field handler = RedisBackendProxyTransport.class.getDeclaredField("messageHandler"); handler.setAccessible(true); handler.set(transport, messages); + transport.dispatchLegacy(replayed); transport.activateAfterHandoff(); assertEquals(java.util.List.of(replayed, newer), order); } + + private static void setBoolean(Object target, String name, boolean value) throws Exception { + Field field = target.getClass().getDeclaredField(name); + field.setAccessible(true); + field.setBoolean(target, value); + } + + private static void setField(Object target, String name, Object value) throws Exception { + Field field = target.getClass().getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index d4d78e4d1..b99715447 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -16,6 +16,7 @@ import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.CountDownLatch; @@ -222,6 +223,19 @@ public void timedVoteBroadcastStateRoundTripsThroughStorageEncoding() { VoteTimeQueue.decodeBroadcastForwardedServers(vote.encodeBroadcastForwardedServers())); } + @Test + public void timedVoteHttpBroadcastDeliveryIdsRoundTripThroughStorageEncoding() { + String deliveryId = "00000000-0000-0000-0000-000000000164"; + VoteTimeQueue vote = new VoteTimeQueue(UUID.randomUUID(), "Player", "Service", 100L, true, + Set.of("Server1"), Set.of(), "totals", false, "uuid", + Map.of("Server1", deliveryId)); + + assertEquals(deliveryId, vote.getHttpBroadcastDeliveryId("server1")); + assertEquals(deliveryId, + VoteTimeQueue.decodeHttpBroadcastDeliveryIds(vote.encodeHttpBroadcastDeliveryIds()) + .get("server1")); + } + @Test public void cachedBroadcastStateSuppressesNonTargetsAndDeliveredTargets() { OfflineBungeeVote vote = new OfflineBungeeVote(UUID.randomUUID(), "Player", "player-uuid", "Service", 100L, @@ -370,6 +384,35 @@ public void timedVoteBroadcastStateLoadsFromJsonCache() { assertTrue(loaded.isProcessed()); } + @Test + public void timedVoteHttpBroadcastDeliveryIdLoadsFromJsonCache() { + IVoteCache stored = mock(IVoteCache.class); + DataNode timedNode = mock(DataNode.class); + when(stored.getTimedVoteCache()).thenReturn(List.of("0")); + when(stored.getTimedVoteCache("0")).thenReturn(timedNode); + when(stored.getServers()).thenReturn(Collections.emptyList()); + when(stored.getPlayers()).thenReturn(Collections.emptyList()); + when(timedNode.isObject()).thenReturn(true); + String deliveryId = "00000000-0000-0000-0000-000000000165"; + UUID voteId = UUID.randomUUID(); + stubString(timedNode, "Name", "Player"); + stubString(timedNode, "Service", "Service"); + stubLong(timedNode, "Time", 100L); + stubString(timedNode, "VoteId", voteId.toString()); + stubString(timedNode, "UUID", "uuid"); + stubBoolean(timedNode, "ProxyBroadcastHandled", true); + stubString(timedNode, "BroadcastTargets", VoteTimeQueue.encodeBroadcastServers(Set.of("Server1"))); + stubString(timedNode, "BroadcastForwardedServers", ""); + stubString(timedNode, "HttpBroadcastDeliveryIds", new VoteTimeQueue(voteId, "Player", "Service", 100L, + true, Set.of("Server1"), Set.of(), "", false, "uuid", Map.of("Server1", deliveryId)) + .encodeHttpBroadcastDeliveryIds()); + + handler = newHandler(stored); + handler.load(); + + assertEquals(deliveryId, handler.getTimeChangeQueue().element().getHttpBroadcastDeliveryId("SERVER1")); + } + @Test public void onlineVoteBroadcastTargetsAndRewardStateLoadFromJsonCache() { IVoteCache stored = mock(IVoteCache.class); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index f93a0fbd8..5f0cafff1 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -51,6 +51,98 @@ void setUp() { } + @Test + void indeterminateHttpVoteRetriesWithThePublishedStableDeliveryId() { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + String deliveryId = "00000000-0000-0000-0000-000000000159"; + votingPluginProxy.failNextGeneratedHttpSend(deliveryId); + + assertTrue(votingPluginProxy.sendHttpVoteEnvelopeWithRecoveryForTest("Server1", + JsonEnvelope.builder("vote").build())); + assertEquals(java.util.List.of(deliveryId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); + } + + @Test + void rejectedIndeterminateHttpVoteRetainsStableIdAcrossCacheSerialization() { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + String deliveryId = "00000000-0000-0000-0000-000000000160"; + votingPluginProxy.failNextGeneratedHttpSend(deliveryId); + OfflineBungeeVote pending = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "uuid", + "Service", 100L, false, "totals"); + + assertFalse(votingPluginProxy.sendHttpVoteEnvelopeWithRecoveryForTest("Server1", + JsonEnvelope.builder("vote").build(), pending)); + assertEquals(deliveryId, pending.getHttpDeliveryId("server1")); + + OfflineBungeeVote restored = new OfflineBungeeVote(pending.getVoteId(), pending.getPlayerName(), + pending.getUuid(), pending.getService(), pending.getTime(), pending.isRealVote(), pending.getText(), + pending.isBroadcastForwarded(), pending.isProxyBroadcastHandled(), pending.getBroadcastTargets(), + pending.getBroadcastForwardedServers(), pending.isRewardDelivered(), + OfflineBungeeVote.decodeHttpDeliveryIds(pending.encodeHttpDeliveryIds())); + assertEquals(deliveryId, restored.getHttpDeliveryId("SERVER1")); + assertFalse(votingPluginProxy.sendHttpVoteEnvelopeWithRecoveryForTest("Server1", + JsonEnvelope.builder("vote").build(), restored)); + assertEquals(java.util.List.of(deliveryId, deliveryId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); + } + + @Test + void standaloneHttpBroadcastUsesSeparateStableIdFromRewardDelivery() { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + String broadcastId = "00000000-0000-0000-0000-000000000161"; + OfflineBungeeVote pending = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "uuid", + "Service", 100L, true, "totals"); + pending.setHttpDeliveryId("Server1", "00000000-0000-0000-0000-000000000162"); + votingPluginProxy.failNextGeneratedHttpSend(broadcastId); + + assertTrue(votingPluginProxy.sendHttpBroadcastEnvelopeWithRecoveryForTest("Server1", + JsonEnvelope.builder("vote").build(), pending)); + assertEquals(java.util.List.of(broadcastId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); + assertEquals("00000000-0000-0000-0000-000000000162", pending.getHttpDeliveryId("Server1")); + assertEquals(null, pending.getHttpBroadcastDeliveryId("Server1")); + } + + @Test + void rejectedStandaloneHttpBroadcastRetainsStableIdAcrossCacheSerialization() { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + String deliveryId = "00000000-0000-0000-0000-000000000163"; + votingPluginProxy.failNextGeneratedHttpSend(deliveryId); + OfflineBungeeVote pending = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "uuid", + "Service", 100L, true, "totals"); + + assertFalse(votingPluginProxy.sendHttpBroadcastEnvelopeWithRecoveryForTest("Server1", + JsonEnvelope.builder("broadcast").build(), pending)); + assertEquals(deliveryId, pending.getHttpBroadcastDeliveryId("server1")); + OfflineBungeeVote restored = new OfflineBungeeVote(pending.getVoteId(), pending.getPlayerName(), pending.getUuid(), + pending.getService(), pending.getTime(), pending.isRealVote(), pending.getText(), false, true, + java.util.Set.of("Server1"), java.util.Set.of(), pending.isRewardDelivered(), + OfflineBungeeVote.decodeHttpDeliveryIds(pending.encodeHttpDeliveryIds()), + OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(pending.encodeHttpBroadcastDeliveryIds())); + assertFalse(votingPluginProxy.sendHttpBroadcastEnvelopeWithRecoveryForTest("Server1", + JsonEnvelope.builder("broadcast").build(), restored)); + assertEquals(java.util.List.of(deliveryId, deliveryId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); + } + + @Test + void acceptedRewardStillPersistsUnresolvedStandaloneBroadcast() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote state = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "player-uuid", + "Service", 100L, true, "totals", false, true, java.util.Set.of("Server1"), + java.util.Collections.emptySet(), false, java.util.Collections.emptyMap(), + java.util.Map.of("Server1", "00000000-0000-0000-0000-000000000169")); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + spyProxy.persistUncachedStandaloneBroadcastForTest("player-uuid", state, false); + + assertTrue(state.isRewardDelivered()); + assertFalse(state.isBroadcastForwarded()); + verify(voteCache).addOnlineVote("player-uuid", state); + } + @Test void testAddVoteParty() { // Initial votePartyVotes should be 0 @@ -871,6 +963,32 @@ void durablyProcessedRolloverVoteIsDeletedWithoutReplay() { verify(voteCache).removeTimeVote(processed); } + @Test + void processedRolloverVoteIsRetainedUntilStableHttpBroadcastDeliveryCompletes() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + String deliveryId = "00000000-0000-0000-0000-000000000167"; + VoteTimeQueue processed = new VoteTimeQueue(java.util.UUID.randomUUID(), "Player", "Service", 100L, true, + java.util.Set.of("Server1"), java.util.Collections.emptySet(), "totals", true, "player-uuid", + java.util.Map.of("Server1", deliveryId)); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + queue.add(processed); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.updateTimeVote(processed)).thenReturn(true); + Mockito.when(voteCache.removeTimeVote(processed)).thenAnswer(invocation -> queue.remove(processed)); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.processQueue(); + + assertTrue(queue.isEmpty()); + assertFalse(processed.hasPendingHttpBroadcastDeliveryIds()); + verify(voteCache).updateTimeVote(processed); + verify(voteCache).removeTimeVote(processed); + verify(spyProxy, never()).getUUID(Mockito.anyString()); + } + @Test void pendingServerBroadcastRetriesBeforeOfflineRewardDelivery() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); @@ -898,6 +1016,34 @@ void pendingServerBroadcastRetriesBeforeOfflineRewardDelivery() { verify(voteCache).updateServerVote("Server1", vote); } + @Test + void acceptedCachedRewardRetainsAmbiguousStandaloneBroadcastWithoutRebroadcasting() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote vote = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "player-uuid", + "Service", 100L, true, "totals", false, true, java.util.Set.of("Server1"), + java.util.Collections.emptySet(), false, java.util.Collections.emptyMap(), + java.util.Map.of("Server1", "00000000-0000-0000-0000-000000000170")); + Mockito.when(voteCache.hasVotes("Server1")).thenReturn(true); + Mockito.when(voteCache.getVotes("Server1")) + .thenReturn(new java.util.ArrayList<>(java.util.List.of(vote))); + Mockito.when(voteCache.updateServerVote("Server1", vote)).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getBlockedServers()).thenReturn(java.util.Collections.emptyList()); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + votingPluginProxy.setStableHttpDeliveryResult(false); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.checkCachedVotes("Server1"); + + assertTrue(vote.isRewardDelivered()); + assertFalse(vote.isProxyBroadcastComplete()); + assertEquals("false", spyProxy.getLastVoteEnvelope().getFields() + .get(VotingPluginWire.K_BUNGEE_BROADCAST)); + verify(voteCache).updateServerVote("Server1", vote); + verify(voteCache).removeServerVotes(Mockito.eq("Server1"), Mockito.argThat(java.util.List::isEmpty)); + } + @Test void rejectedHttpQueueDeliveryRetainsCachedServerVote() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); @@ -941,6 +1087,32 @@ void rejectedHttpQueueDeliveryRetainsCachedOnlineVote() { verify(multiProxyHandler, never()).sendClearVote(Mockito.anyString(), Mockito.anyString()); } + @Test + void acceptedOnlineRewardRetainsAmbiguousStandaloneBroadcastWithoutRebroadcasting() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote vote = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "player-uuid", + "Service", 100L, true, "totals", false, true, java.util.Set.of("Server1"), + java.util.Collections.emptySet(), false, java.util.Collections.emptyMap(), + java.util.Map.of("Server1", "00000000-0000-0000-0000-000000000171")); + Mockito.when(voteCache.hasOnlineVotes("player-uuid")).thenReturn(true); + Mockito.when(voteCache.getOnlineVotes("player-uuid")) + .thenReturn(new java.util.ArrayList<>(java.util.List.of(vote))); + Mockito.when(votingPluginProxy.getConfig().getBlockedServers()).thenReturn(java.util.Collections.emptyList()); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + votingPluginProxy.setStableHttpDeliveryResult(false); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.checkOnlineVotes("Player", "player-uuid", "Server1"); + + assertTrue(vote.isRewardDelivered()); + assertFalse(vote.isProxyBroadcastComplete()); + assertEquals("false", spyProxy.getLastVoteEnvelope().getFields() + .get(VotingPluginWire.K_BUNGEE_BROADCAST)); + verify(voteCache).addOnlineVote("player-uuid", vote); + } + @Test void pendingOnlineBroadcastRetriesWhenTargetGainsAnyCarrier() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); @@ -990,6 +1162,54 @@ void periodicRetryDeliversBrokerBackedOnlineCacheWithoutVoterLogin() { verify(voteCache).updateOnlineVote("voter-uuid", vote); } + @Test + void failedPeriodicHttpBroadcastPersistsGeneratedStableId() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote vote = new OfflineBungeeVote(java.util.UUID.randomUUID(), "OfflineVoter", "voter-uuid", + "Service", 100L, true, "totals", false, true, java.util.Set.of("Server1"), + java.util.Collections.emptySet(), false); + Mockito.when(voteCache.getOnlineVoteUUIDs()).thenReturn(java.util.Set.of("voter-uuid")); + Mockito.when(voteCache.getOnlineVotes("voter-uuid")) + .thenReturn(new java.util.ArrayList<>(java.util.List.of(vote))); + Mockito.when(voteCache.updateOnlineVote("voter-uuid", vote)).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getBlockedServers()).thenReturn(java.util.Collections.emptyList()); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + String deliveryId = "00000000-0000-0000-0000-000000000167"; + votingPluginProxy.failNextGeneratedHttpSend(deliveryId); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.retryPendingOnlineBroadcasts(); + + assertEquals(deliveryId, vote.getHttpBroadcastDeliveryId("Server1")); + verify(voteCache).updateOnlineVote("voter-uuid", vote); + } + + @Test + void failedCarrierHttpBroadcastPersistsGeneratedStableId() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote vote = new OfflineBungeeVote(java.util.UUID.randomUUID(), "OfflineVoter", "voter-uuid", + "Service", 100L, true, "totals", false, true, java.util.Set.of("Server1"), + java.util.Collections.emptySet(), false); + Mockito.when(voteCache.getOnlineVoteUUIDs()).thenReturn(java.util.Set.of("voter-uuid")); + Mockito.when(voteCache.getOnlineVotes("voter-uuid")) + .thenReturn(new java.util.ArrayList<>(java.util.List.of(vote))); + Mockito.when(voteCache.updateOnlineVote("voter-uuid", vote)).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getBlockedServers()).thenReturn(java.util.Collections.emptyList()); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + String deliveryId = "00000000-0000-0000-0000-000000000168"; + votingPluginProxy.failNextGeneratedHttpSend(deliveryId); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.retryPendingOnlineBroadcastsForTest("Server1"); + + assertEquals(deliveryId, vote.getHttpBroadcastDeliveryId("Server1")); + verify(voteCache).updateOnlineVote("voter-uuid", vote); + } + @Test void timedBroadcastRetriesWhileRolloverIsStillActive() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); @@ -1009,6 +1229,43 @@ void timedBroadcastRetriesWhileRolloverIsStillActive() { verify(voteCache).updateTimeVote(vote); } + @Test + void timedHttpBroadcastRetainsStableIdWhenPublicationIsAmbiguous() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + VoteTimeQueue vote = new VoteTimeQueue(java.util.UUID.randomUUID(), "OfflineVoter", "Service", 100L, true, + java.util.Set.of("Server1"), java.util.Collections.emptySet(), "totals", false, "voter-uuid"); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + queue.add(vote); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.updateTimeVote(vote)).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getBlockedServers()).thenReturn(java.util.Collections.emptyList()); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + String deliveryId = "00000000-0000-0000-0000-000000000166"; + votingPluginProxy.failNextGeneratedHttpSend(deliveryId); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.retryPendingTimeBroadcastsForTest("Server1"); + + assertEquals(deliveryId, vote.getHttpBroadcastDeliveryId("Server1")); + verify(voteCache).updateTimeVote(vote); + + VoteTimeQueue restored = new VoteTimeQueue(vote.getVoteId(), vote.getName(), vote.getService(), vote.getTime(), + vote.isProxyBroadcastHandled(), vote.getBroadcastTargets(), vote.getBroadcastForwardedServers(), + vote.getTotals(), vote.isProcessed(), vote.getUuid(), + VoteTimeQueue.decodeHttpBroadcastDeliveryIds(vote.encodeHttpBroadcastDeliveryIds())); + java.util.Queue restoredQueue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + restoredQueue.add(restored); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(restoredQueue); + spyProxy.setVoteEnvelopeDeliveryResult(true); + spyProxy.retryPendingTimeBroadcastsForTest("Server1"); + + assertTrue(restored.getBroadcastForwardedServers().contains("Server1")); + assertEquals(null, restored.getHttpBroadcastDeliveryId("Server1")); + assertEquals(java.util.List.of(deliveryId, deliveryId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); + } + @Test void failedTimedDeliveryStatePersistenceRemainsDirtyUntilRetrySucceeds() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index 4ceed32b2..7d5db0544 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -28,6 +28,8 @@ public class VotingPluginProxyTestImpl extends VotingPluginProxy { private VotingPluginProxyConfig config; private boolean pluginMessageDeliveryResult = true; private boolean voteEnvelopeDeliveryResult = true; + private Boolean stableHttpDeliveryResult; + private JsonEnvelope lastVoteEnvelope; private boolean communicationTestDeliveryResult = true; private JsonEnvelope lastCommunicationTestEnvelope; private boolean playerOnline = true; @@ -35,6 +37,8 @@ public class VotingPluginProxyTestImpl extends VotingPluginProxy { private boolean failNextVoteCacheSave; private final java.util.Map> pendingVotePartyRewards = new HashMap<>(); private final List attemptedVotePartyDeliveryIds = new ArrayList<>(); + private boolean failNextGeneratedHttpSend; + private String generatedHttpRetryId; private PendingVotePartyProxyEffects pendingVotePartyProxyEffects = PendingVotePartyProxyEffects.empty(); private PendingVotePartyProxyEffects quarantinedVotePartyProxyEffects = PendingVotePartyProxyEffects.empty(); private final List broadcasts = new ArrayList<>(); @@ -330,6 +334,7 @@ public boolean sendProxyBroadcastImmediately(String server, JsonEnvelope envelop @Override protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope) { + lastVoteEnvelope = envelope; if (getMethod() == com.bencodez.votingplugin.proxy.BungeeMethod.HTTP) { return voteEnvelopeDeliveryResult; } @@ -340,16 +345,61 @@ public void setVoteEnvelopeDeliveryResult(boolean voteEnvelopeDeliveryResult) { this.voteEnvelopeDeliveryResult = voteEnvelopeDeliveryResult; } + @Override + protected boolean sendHttpEnvelope(String server, JsonEnvelope envelope) { + lastVoteEnvelope = envelope; + if (failNextGeneratedHttpSend) { + failNextGeneratedHttpSend = false; + try { + java.lang.reflect.Constructor constructor = + HttpProxyTransportServer.DeliveryRetryException.class + .getDeclaredConstructor(String.class, Throwable.class); + constructor.setAccessible(true); + throw constructor.newInstance(generatedHttpRetryId, new IllegalStateException("publication ambiguous")); + } catch (ReflectiveOperationException failure) { + throw new AssertionError(failure); + } + } + return voteEnvelopeDeliveryResult; + } + @Override protected boolean sendHttpEnvelope(String server, String deliveryId, JsonEnvelope envelope) { attemptedVotePartyDeliveryIds.add(deliveryId); - return voteEnvelopeDeliveryResult; + return stableHttpDeliveryResult == null ? voteEnvelopeDeliveryResult : stableHttpDeliveryResult; + } + + public void setStableHttpDeliveryResult(Boolean stableHttpDeliveryResult) { + this.stableHttpDeliveryResult = stableHttpDeliveryResult; + } + + public JsonEnvelope getLastVoteEnvelope() { + return lastVoteEnvelope; } public List getAttemptedVotePartyDeliveryIds() { return attemptedVotePartyDeliveryIds; } + public void failNextGeneratedHttpSend(String deliveryId) { + failNextGeneratedHttpSend = true; + generatedHttpRetryId = deliveryId; + } + + public boolean sendHttpVoteEnvelopeWithRecoveryForTest(String server, JsonEnvelope envelope) { + return sendHttpEnvelopeWithRecovery(server, envelope); + } + + public boolean sendHttpVoteEnvelopeWithRecoveryForTest(String server, JsonEnvelope envelope, + OfflineBungeeVote cachedVote) { + return sendHttpEnvelopeWithRecovery(server, envelope, cachedVote); + } + + public boolean sendHttpBroadcastEnvelopeWithRecoveryForTest(String server, JsonEnvelope envelope, + OfflineBungeeVote cachedVote) { + return sendHttpBroadcastEnvelopeWithRecovery(server, envelope, cachedVote); + } + public void acknowledgeVotePartyDeliveryForTest(String server, String deliveryId) throws java.io.IOException { acknowledgeVotePartyDelivery(server, deliveryId); } @@ -410,6 +460,11 @@ public void retryPendingOnlineBroadcastsForTest(String server) { retryPendingOnlineBroadcasts(server); } + public void persistUncachedStandaloneBroadcastForTest(String uuid, OfflineBungeeVote state, + boolean alreadyPersisted) { + persistUncachedStandaloneBroadcast(uuid, state, alreadyPersisted); + } + public void retryPendingTimeBroadcastsForTest(String server) { retryPendingTimeBroadcasts(server); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java index 66bcd22df..b01650df1 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java @@ -82,6 +82,7 @@ public void legacyRedisHandoffAccountingIsBounded() throws Exception { Field deliveries = ProcessedVoteCache.class.getDeclaredField("legacyRedisDeliveries"); deliveries.setAccessible(true); assertEquals(4096, ((Map) deliveries.get(cache)).size()); + assertTrue(cache.isLegacyRedisHandoffOverflowed()); } @Test @@ -97,5 +98,6 @@ public void legacyRedisHandoffAccountingRejectsOversizedSignatures() throws Exce Field deliveries = ProcessedVoteCache.class.getDeclaredField("legacyRedisDeliveries"); deliveries.setAccessible(true); assertTrue(((Map) deliveries.get(cache)).isEmpty()); + assertTrue(cache.isLegacyRedisHandoffOverflowed()); } } From 17a5bd581f9422dbf376ae6a865ecebff675c10c Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:08:26 -0600 Subject: [PATCH 052/136] Restore Redis transport cleanly on failed handoff --- .../votingplugin/VotingPluginMain.java | 15 ++++++-- .../backendproxy/BackendProxyHandler.java | 8 +++++ .../BackendProxyTransportManager.java | 35 +++++++++++++++++-- .../transport/RedisBackendProxyTransport.java | 10 ++++++ ...PluginMainBackendProxyPublicationTest.java | 3 ++ .../BackendProxyHandlerLifecycleTest.java | 35 +++++++++++++++++++ 6 files changed, 102 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index d46d119e5..c3da6f96e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1317,9 +1317,20 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { try { if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); } catch (RuntimeException handoffFailure) { - // Redis promotion fails before the old listener is retired. Restore the - // published pointer so abort can close only the staged replacement. + // Stop the staged presence generation before reasserting the restored old + // handler. Otherwise the proxy rejects the old generation after rollback. backendProxyHandler = restart.previous; + try { + restart.replacement.close(); + } catch (RuntimeException closeFailure) { + handoffFailure.addSuppressed(closeFailure); + } + try { + restart.previous.refreshPresenceAfterFailedReplacement(); + } catch (RuntimeException refreshFailure) { + handoffFailure.addSuppressed(refreshFailure); + } + restart.finished = true; throw handoffFailure; } // Keep the prepared HTTP queue owned by the previous handler until every diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index ea528a0b9..2c6ff1c6f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -142,6 +142,14 @@ public void restoreAfterFailedReplacement() { transportManager.restoreAfterFailedReplacement(); } + /** Reasserts the old handler with a fresh presence generation after rollback. */ + public void refreshPresenceAfterFailedReplacement() { + if (presenceManager != null && presenceReportingActivated) { + presenceManager.stop(); + presenceManager.start(); + } + } + public void awaitRestoreAfterFailedReplacement(long deadlineNanos) { transportManager.awaitPreparedTransportRestoration(deadlineNanos); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 81d1fe6bd..24d950b6d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -202,7 +202,6 @@ public synchronized void closeRedisForHandoff() { transport = null; try { ((RedisBackendProxyTransport) retiredTransport).closeForHandoff(); - retiredTransport = null; } catch (RuntimeException failure) { if (failure instanceof RedisBackendProxyTransport.HandoffQuiescenceException) { transport = retiredTransport; @@ -237,7 +236,39 @@ public void completeRedisHandoff(BackendProxyTransportManager replacement) { plugin.debug(retirementFailure); } } - replacement.activateRedisAfterHandoff(); + try { + replacement.activateRedisAfterHandoff(); + } catch (RuntimeException activationFailure) { + restoreRetiredRedisAfterFailedHandoff(activationFailure); + throw activationFailure; + } + closeRetiredRedisAfterHandoff(); + } + + private synchronized void restoreRetiredRedisAfterFailedHandoff(RuntimeException activationFailure) { + if (!(retiredTransport instanceof RedisBackendProxyTransport redis)) return; + try { + redis.restoreAfterFailedHandoff(); + transport = redis; + retiredTransport = null; + } catch (RuntimeException restorationFailure) { + activationFailure.addSuppressed(restorationFailure); + } + } + + private synchronized void closeRetiredRedisAfterHandoff() { + if (retiredTransport == null) return; + BackendProxyTransport retired = retiredTransport; + retiredTransport = null; + try { + retired.close(); + } catch (RuntimeException cleanupFailure) { + retiredTransport = retired; + if (plugin != null) { + plugin.getLogger().warning("Retired Redis backend listener did not stop cleanly after handoff"); + plugin.debug(cleanupFailure); + } + } } public ClientHandler getClientHandler() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index 5e5ca4535..c95c0d7e1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -55,6 +55,7 @@ public HandoffQuiescenceException(String message) { private int dispatchesInFlight; private final java.util.ArrayDeque deliveriesAfterReplay = new java.util.ArrayDeque<>(); private GlobalMessageHandler messageHandler; + private GlobalMessageHandler handoffMessageHandler; public RedisBackendProxyTransport(VotingPluginMain plugin) { this(plugin, new ProcessedVoteCache()); @@ -236,10 +237,19 @@ private void dispatchReplayBatch(java.util.List replay) { /** Stops the old listener while retaining its overlap accounting for standby promotion. */ public void closeForHandoff() { + handoffMessageHandler = messageHandler; fenceAfterHandoff(); closeListener(false); } + /** Reopens a fenced active subscriber when standby promotion is rejected. */ + public void restoreAfterFailedHandoff() { + GlobalMessageHandler handler = handoffMessageHandler; + if (handler == null) throw new IllegalStateException("Redis handoff transport cannot be restored"); + start(handler); + handoffMessageHandler = null; + } + /** Prevents a listener that misses its shutdown deadline from dispatching duplicates. */ void fenceAfterHandoff() { boolean interrupted = false; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index 86fe9b9f5..1f27d5533 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -77,6 +77,9 @@ void restoresPreviousHandlerWhenRedisPromotionFails() throws Exception { assertSame(previous, plugin.getBackendProxyHandler()); verify(previous, never()).completeHttpHandoff(replacement); verify(previous, never()).close(); + org.mockito.InOrder rollback = org.mockito.Mockito.inOrder(replacement, previous); + rollback.verify(replacement).close(); + rollback.verify(previous).refreshPresenceAfterFailedReplacement(); } private VotingPluginMain.BackendProxyRestart restart(BackendProxyHandler previous, diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 5be3eae3b..a0d515862 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -421,6 +421,41 @@ void doesNotPromoteRedisReplacementWhenOldCallbacksDoNotQuiesce() throws Excepti assertSame(newTransport, transport(replacement)); } + @Test + void restoresOldRedisSubscriberWhenStandbyActivationFails() throws Exception { + BackendProxyTransportManager previous = new BackendProxyTransportManager(null); + BackendProxyTransportManager replacement = new BackendProxyTransportManager(null); + RedisBackendProxyTransport oldTransport = mock(RedisBackendProxyTransport.class); + RedisBackendProxyTransport newTransport = mock(RedisBackendProxyTransport.class); + setField(previous, "transport", oldTransport); + setField(replacement, "transport", newTransport); + doThrow(new IllegalStateException("handoff buffer overflowed")).when(newTransport).activateAfterHandoff(); + + assertThrows(IllegalStateException.class, () -> previous.completeRedisHandoff(replacement)); + + org.mockito.InOrder order = inOrder(oldTransport, newTransport); + order.verify(oldTransport).closeForHandoff(); + order.verify(newTransport).activateAfterHandoff(); + order.verify(oldTransport).restoreAfterFailedHandoff(); + assertSame(oldTransport, transport(previous)); + assertSame(newTransport, transport(replacement)); + } + + @Test + void redisRollbackForcesAFreshPresenceGeneration() throws Exception { + BackendProxyHandler handler = new BackendProxyHandler(null); + com.bencodez.votingplugin.backendproxy.presence.BackendPresenceManager presence = + mock(com.bencodez.votingplugin.backendproxy.presence.BackendPresenceManager.class); + setField(handler, "presenceManager", presence); + setField(handler, "presenceReportingActivated", true); + + handler.refreshPresenceAfterFailedReplacement(); + + org.mockito.InOrder order = inOrder(presence); + order.verify(presence).stop(); + order.verify(presence).start(); + } + @Test void stopsGlobalDataTimerWhenHandlerIsReplaced() throws Exception { BackendGlobalDataSync handler = new BackendGlobalDataSync(null, null); From a0ee827e63d7381f25940e1d24e18ab6e4d73b1c Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:39:38 -0600 Subject: [PATCH 053/136] Recover auxiliary HTTP vote deliveries --- .../votingplugin/proxy/VotingPluginProxy.java | 50 ++++++++++++++++--- .../tests/VotingPluginProxyTest.java | 36 +++++++++++++ .../tests/VotingPluginProxyTestImpl.java | 8 +++ 3 files changed, 87 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 0d654e2a4..4c7e37783 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -11,6 +11,7 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.time.Duration; import java.time.Instant; @@ -1616,7 +1617,7 @@ public void sendMessage(String server, int delay, JsonEnvelope envelope) { sendSocketEnvelope(server, envelope); break; case HTTP: - sendHttpEnvelope(server, envelope); + sendGenericHttpEnvelope(server, envelope); break; default: break; @@ -2911,6 +2912,34 @@ protected boolean sendHttpEnvelopeWithRecovery(String server, JsonEnvelope envel return sendHttpEnvelopeWithRecovery(server, envelope, null); } + /** + * Sends a non-reward HTTP message without allowing a transport handoff failure + * to retry the surrounding vote transaction. An ambiguous durable publication + * is recovered with its original ID; a definite rejection is reported and + * logged, rather than escaping through the void global-message adapter. + */ + protected boolean sendGenericHttpEnvelope(String server, JsonEnvelope envelope) { + try { + boolean accepted = sendHttpEnvelopeWithRecovery(server, envelope); + if (!accepted) debug("HTTP transport rejected auxiliary delivery for " + server); + return accepted; + } catch (RuntimeException failure) { + debug("Unable to send HTTP auxiliary delivery: " + failure.getMessage()); + return false; + } + } + + protected boolean sendStableHttpEnvelope(String server, String deliveryId, JsonEnvelope envelope) { + try { + boolean accepted = sendHttpEnvelope(server, deliveryId, envelope); + if (!accepted) debug("HTTP transport rejected auxiliary delivery for " + server); + return accepted; + } catch (RuntimeException failure) { + debug("Unable to send HTTP auxiliary delivery: " + failure.getMessage()); + return false; + } + } + protected boolean sendHttpEnvelopeWithRecovery(String server, JsonEnvelope envelope, OfflineBungeeVote cachedVote) { String stableId = cachedVote == null ? null : cachedVote.getHttpDeliveryId(server); try { @@ -3601,15 +3630,21 @@ private static CommunicationTestResult failure(String server, BungeeMethod metho private record PendingCommunicationTest(String server, BungeeMethod method, long startedAtNanos, CompletableFuture result) { } - private void sendVoteDelayRejected(String player, String uuid, String service, boolean playerOnline, - String playerServer) { + private boolean sendVoteDelayRejected(UUID voteId, String player, String uuid, String service, + boolean playerOnline, String playerServer) { if (!playerOnline || playerServer == null || !getAllAvailableServers().contains(playerServer)) { debug("Not sending vote delay rejection for " + player + " because the player is offline"); - return; + return true; } - globalMessageProxyHandler.sendMessage(playerServer, 1, - VotingPluginWire.voteDelayRejected(player, uuid, service, true)); + JsonEnvelope envelope = VotingPluginWire.voteDelayRejected(player, uuid, service, true); + if (method == BungeeMethod.HTTP) { + String key = voteId + "\u0000vote-delay-rejected\u0000" + playerServer.toLowerCase(Locale.ROOT); + String deliveryId = UUID.nameUUIDFromBytes(key.getBytes(StandardCharsets.UTF_8)).toString(); + return sendStableHttpEnvelope(playerServer, deliveryId, envelope); + } + globalMessageProxyHandler.sendMessage(playerServer, 1, envelope); + return true; } public String getWaitUntilDelaySiteFromService(String service) { @@ -3868,7 +3903,8 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea data = getProxyMySQL().getExactQuery(new Column("uuid", new DataValueString(uuid))); if (!checkVoteDelay(uuid, player, service, data, queuedVote == null)) { log("Vote delay is not met for " + player + "/" + service + ", skipping vote"); - sendVoteDelayRejected(player, uuid, service, playerOnline, playerServer); + if (!sendVoteDelayRejected(voteId, player, uuid, service, playerOnline, playerServer) + && queuedVote != null) return QueuedVoteResult.RETRY; return QueuedVoteResult.TERMINAL; } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 5f0cafff1..5f1844ea1 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -63,6 +63,42 @@ void indeterminateHttpVoteRetriesWithThePublishedStableDeliveryId() { assertEquals(java.util.List.of(deliveryId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); } + @Test + void indeterminateGenericHttpSendRecoversWithoutEscapingToRetryTheVote() { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + String deliveryId = "00000000-0000-0000-0000-00000000016a"; + votingPluginProxy.failNextGeneratedHttpSend(deliveryId); + + assertTrue(votingPluginProxy.sendGenericHttpEnvelopeForTest("Server1", + VotingPluginWire.voteUpdate("uuid", 1, 10, "Service", 100L, "totals"))); + assertEquals(java.util.List.of(deliveryId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); + } + + @Test + void rejectedGenericHttpSendIsHandledWithoutEscapingToRetryTheVote() { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + + assertFalse(votingPluginProxy.sendGenericHttpEnvelopeForTest("Server1", + VotingPluginWire.voteUpdate("uuid", 1, 10, "Service", 100L, "totals"))); + } + + @Test + void rejectedRewardBearingAuxiliarySendRetainsItsStableDeliveryIdForRetry() { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + String deliveryId = "00000000-0000-0000-0000-00000000016b"; + JsonEnvelope envelope = VotingPluginWire.voteDelayRejected("Player", + "00000000-0000-0000-0000-000000000001", "Service", true); + + assertFalse(votingPluginProxy.sendStableHttpEnvelopeForTest("Server1", deliveryId, envelope)); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + assertTrue(votingPluginProxy.sendStableHttpEnvelopeForTest("Server1", deliveryId, envelope)); + assertEquals(java.util.List.of(deliveryId, deliveryId), + votingPluginProxy.getAttemptedVotePartyDeliveryIds()); + } + @Test void rejectedIndeterminateHttpVoteRetainsStableIdAcrossCacheSerialization() { votingPluginProxy.setMethod(BungeeMethod.HTTP); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index 7d5db0544..ca4b12354 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -390,6 +390,14 @@ public boolean sendHttpVoteEnvelopeWithRecoveryForTest(String server, JsonEnvelo return sendHttpEnvelopeWithRecovery(server, envelope); } + public boolean sendGenericHttpEnvelopeForTest(String server, JsonEnvelope envelope) { + return sendGenericHttpEnvelope(server, envelope); + } + + public boolean sendStableHttpEnvelopeForTest(String server, String deliveryId, JsonEnvelope envelope) { + return sendStableHttpEnvelope(server, deliveryId, envelope); + } + public boolean sendHttpVoteEnvelopeWithRecoveryForTest(String server, JsonEnvelope envelope, OfflineBungeeVote cachedVote) { return sendHttpEnvelopeWithRecovery(server, envelope, cachedVote); From 158d4030fb62905efbde8c7fc5b8329848c757f3 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:26:22 -0600 Subject: [PATCH 054/136] Preserve HTTP delivery state across retries --- .../transport/HttpBackendProxyTransport.java | 20 +++- .../votingplugin/proxy/VotingPluginProxy.java | 93 +++++++++++-------- .../HttpBackendProxyTransportTest.java | 54 +++++++++++ .../tests/VotingPluginProxyTest.java | 61 ++++++++++++ .../tests/VotingPluginProxyTestImpl.java | 5 + 5 files changed, 190 insertions(+), 43 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 4de3fccb7..628606788 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -118,8 +118,10 @@ private void start(Path directory, String serverId, String connectionCode, HttpBackendProxyTransport recreatePrepared() { HttpBackendProxyTransport restored = new HttpBackendProxyTransport(plugin); synchronized (lifecycle) { - restored.startupQueue.addAll(startupQueue); - startupQueue.clear(); + // Startup and handoff queues are one FIFO from the caller's perspective. + // The handoff queue can still contain messages accepted by the previous + // replacement, so rollback must carry it into the restored transport too. + restored.startupQueue.addAll(takeQueuedMessages()); } restored.start(configuredDirectory, configuredServerId, configuredConnectionCode, configuredMessageHandler, configuredCredentialGeneration, configuredCredentialGeneration == null && retryInitialization, @@ -133,12 +135,20 @@ boolean isClosedForReplacement() { java.util.List drainPreparedMessages() { synchronized (lifecycle) { - java.util.List pending = java.util.List.copyOf(startupQueue); - startupQueue.clear(); - return pending; + return takeQueuedMessages(); } } + /** Takes both pending queues in their original FIFO order for replacement handoff. */ + private java.util.List takeQueuedMessages() { + java.util.List pending = new java.util.ArrayList<>(startupQueue.size() + handoffQueue.size()); + pending.addAll(startupQueue); + pending.addAll(handoffQueue); + startupQueue.clear(); + handoffQueue.clear(); + return pending; + } + @Override public void prepareForReplacement() { HttpBackendTransportConnector active; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 4c7e37783..1d9bacb5c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -4015,9 +4015,12 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea remainingTargets.removeAll(broadcastForwardedServers); standaloneBroadcastState = new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text == null ? "" : text.toString(), false, true, proxyBroadcastTargets, - broadcastForwardedServers, false); - broadcastForwardedServers.addAll(sendProxyBroadcast(remainingTargets, uuid, player, service, time, - text == null ? "" : text.toString(), false, standaloneBroadcastState)); + broadcastForwardedServers, false, Collections.emptyMap(), queuedVote == null + ? Collections.emptyMap() : queuedVote.getHttpBroadcastDeliveryIds()); + Set forwarded = sendProxyBroadcast(remainingTargets, uuid, player, service, time, + text == null ? "" : text.toString(), false, standaloneBroadcastState); + broadcastForwardedServers.addAll(forwarded); + standaloneBroadcastState.getBroadcastForwardedServers().addAll(forwarded); } // =========================== @@ -4035,15 +4038,8 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea if ((!isSomeoneOnlineServerForVoteRouting(s) && method.requiresPlayerOnline()) || forceCache) { voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addServerVote(s, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, - text.toString(), broadcastForwarded, standaloneProxyBroadcast, - proxyBroadcastTargets, broadcastForwardedServers, false, Collections.emptyMap(), - standaloneBroadcastState == null ? Collections.emptyMap() - : standaloneBroadcastState.getHttpBroadcastDeliveryIds())); - standaloneBroadcastStatePersisted |= standaloneBroadcastState != null; + getVoteCacheHandler().addServerVote(s, createCachedRewardVote(voteId, player, uuid, service, time, + realVote, text.toString(), standaloneProxyBroadcast)); debug("Caching vote for " + player + " on " + service + " for " + s); } else { boolean broadcastHere = !broadcastForwardedServers.contains(s); @@ -4053,20 +4049,21 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea broadcastHere = proxyBroadcastDecider.shouldBroadcast(s, targets); } - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - OfflineBungeeVote pendingVote = new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, - text.toString(), broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, - broadcastForwardedServers, false, Collections.emptyMap(), - standaloneBroadcastState == null ? Collections.emptyMap() - : standaloneBroadcastState.getHttpBroadcastDeliveryIds()); - if (!sendVoteEnvelopeAccepted(s, 2, + OfflineBungeeVote pendingVote = standaloneProxyBroadcast + ? new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), false, + true, proxyBroadcastTargets, broadcastForwardedServers, false, + Collections.emptyMap(), standaloneBroadcastState.getHttpBroadcastDeliveryIds()) + : createCachedRewardVote(voteId, player, uuid, service, time, realVote, text.toString(), false); + boolean rewardAccepted = sendVoteEnvelopeAccepted(s, 2, VotingPluginWire.vote(player, uuid, service, time, true, realVote, text.toString(), - voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote)) { + voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote); + if (!rewardAccepted) { voteStatus = VoteLogStatus.CACHED; - getVoteCacheHandler().addServerVote(s, pendingVote); - standaloneBroadcastStatePersisted |= standaloneBroadcastState != null; + getVoteCacheHandler().addServerVote(s, standaloneProxyBroadcast + ? createCachedRewardVote(pendingVote, true) : pendingVote); debug("Caching vote after the transport rejected delivery for " + s); + } else if (standaloneBroadcastState != null) { + standaloneBroadcastState.setRewardDelivered(true); } } } @@ -4083,13 +4080,11 @@ voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote)) broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); } - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - OfflineBungeeVote pendingVote = new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, - text.toString(), broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, - broadcastForwardedServers, false, Collections.emptyMap(), - standaloneBroadcastState == null ? Collections.emptyMap() - : standaloneBroadcastState.getHttpBroadcastDeliveryIds()); + OfflineBungeeVote pendingVote = standaloneProxyBroadcast + ? new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), false, + true, proxyBroadcastTargets, broadcastForwardedServers, false, Collections.emptyMap(), + standaloneBroadcastState.getHttpBroadcastDeliveryIds()) + : createCachedRewardVote(voteId, player, uuid, service, time, realVote, text.toString(), false); boolean rewardAccepted = sendVoteEnvelopeAccepted(server, 1, VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(), voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote); @@ -4098,6 +4093,8 @@ voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote)) getVoteCacheHandler().addOnlineVote(uuid, pendingVote); standaloneBroadcastStatePersisted |= standaloneBroadcastState != null; debug("Caching online vote after the transport rejected delivery for " + server); + } else if (standaloneBroadcastState != null) { + standaloneBroadcastState.setRewardDelivered(true); } if (rewardAccepted && canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled() @@ -4127,14 +4124,11 @@ voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote)) } } else { voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addOnlineVote(uuid, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), - broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, - broadcastForwardedServers, false, Collections.emptyMap(), - standaloneBroadcastState == null ? Collections.emptyMap() - : standaloneBroadcastState.getHttpBroadcastDeliveryIds())); + getVoteCacheHandler().addOnlineVote(uuid, standaloneProxyBroadcast + ? new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), false, + true, proxyBroadcastTargets, broadcastForwardedServers, false, Collections.emptyMap(), + standaloneBroadcastState.getHttpBroadcastDeliveryIds()) + : createCachedRewardVote(voteId, player, uuid, service, time, realVote, text.toString(), false)); standaloneBroadcastStatePersisted |= standaloneBroadcastState != null; debug("Caching online vote for " + player + " on " + service); } @@ -4197,9 +4191,32 @@ voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote)) } } + /** + * Creates the reward cache entry for a backend. Standalone proxy broadcast + * progress belongs to the single voter-keyed canonical state, not to every + * backend row. Marking the local row as already broadcast prevents it from + * emitting a second broadcast while its reward waits for the player. + */ + protected OfflineBungeeVote createCachedRewardVote(UUID voteId, String player, String uuid, String service, long time, + boolean realVote, String text, boolean standaloneProxyBroadcast) { + return new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text, + standaloneProxyBroadcast, false, Collections.emptySet(), Collections.emptySet(), false, + Collections.emptyMap(), Collections.emptyMap()); + } + + private OfflineBungeeVote createCachedRewardVote(OfflineBungeeVote deliveryState, + boolean standaloneProxyBroadcast) { + return new OfflineBungeeVote(deliveryState.getVoteId(), deliveryState.getPlayerName(), deliveryState.getUuid(), + deliveryState.getService(), deliveryState.getTime(), deliveryState.isRealVote(), deliveryState.getText(), + standaloneProxyBroadcast, false, Collections.emptySet(), Collections.emptySet(), false, + OfflineBungeeVote.decodeHttpDeliveryIds(deliveryState.encodeHttpDeliveryIds()), Collections.emptyMap()); + } + protected void persistUncachedStandaloneBroadcast(String uuid, OfflineBungeeVote state, boolean alreadyPersisted) { if (alreadyPersisted || state == null || state.isProxyBroadcastComplete()) return; + // This canonical row retries only the standalone broadcast. Reward delivery + // remains owned by the target-specific server/online cache entry. state.setRewardDelivered(true); state.setBroadcastForwarded(false); getVoteCacheHandler().addOnlineVote(uuid, state); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index df139a7fd..388f24bd9 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -554,6 +554,60 @@ void preparedHandoffMessagesStayAheadOfNewlyPublishedSendsWhenConnectorIsFull() transport.close(); } + @Test + @SuppressWarnings("unchecked") + void preparedDrainPreservesStartupAndExistingHandoffMessagesInFifoOrder() throws Exception { + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(mock(VotingPluginMain.class)); + JsonEnvelope startup = JsonEnvelope.builder("startup").build(); + JsonEnvelope handoff = JsonEnvelope.builder("handoff").build(); + ((java.util.ArrayDeque) field(transport, "startupQueue")).add(startup); + ((java.util.ArrayDeque) field(transport, "handoffQueue")).add(handoff); + + assertEquals(List.of(startup, handoff), transport.drainPreparedMessages()); + assertTrue(((java.util.ArrayDeque) field(transport, "startupQueue")).isEmpty()); + assertTrue(((java.util.ArrayDeque) field(transport, "handoffQueue")).isEmpty()); + } + + @Test + @SuppressWarnings("unchecked") + void rollbackRecreateCarriesExistingHandoffMessagesAfterStartupMessages() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(plugin); + Path credentials = directory.resolve("http"); + setField(transport, "configuredDirectory", credentials); + setField(transport, "configuredServerId", "lobby-1"); + setField(transport, "configuredConnectionCode", ""); + setField(transport, "configuredMessageHandler", mock(GlobalMessageHandler.class)); + setField(transport, "restoreUnenrolledState", true); + JsonEnvelope startup = JsonEnvelope.builder("startup").build(); + JsonEnvelope handoff = JsonEnvelope.builder("handoff").build(); + ((java.util.ArrayDeque) field(transport, "startupQueue")).add(startup); + ((java.util.ArrayDeque) field(transport, "handoffQueue")).add(handoff); + + java.lang.reflect.Field ownersField = HttpBackendProxyTransport.class.getDeclaredField("DIRECTORY_OWNERS"); + ownersField.setAccessible(true); + var owners = (java.util.concurrent.ConcurrentHashMap) ownersField.get(null); + Path ownerKey = credentials.toAbsolutePath().normalize(); + java.util.concurrent.Semaphore owner = new java.util.concurrent.Semaphore(0); + owners.put(ownerKey, owner); + HttpBackendProxyTransport restored = null; + Thread setup = null; + try { + restored = transport.recreatePrepared(); + assertEquals(List.of(startup, handoff), + new java.util.ArrayList<>((java.util.ArrayDeque) field(restored, "startupQueue"))); + assertTrue(((java.util.ArrayDeque) field(restored, "handoffQueue")).isEmpty()); + assertTrue(((java.util.ArrayDeque) field(transport, "startupQueue")).isEmpty()); + assertTrue(((java.util.ArrayDeque) field(transport, "handoffQueue")).isEmpty()); + setup = (Thread) field(restored, "worker"); + } finally { + if (restored != null) restored.close(); + if (setup != null) setup.join(TimeUnit.SECONDS.toMillis(1)); + owners.remove(ownerKey, owner); + } + } + private static HttpConnectionCode code(String serverId, Instant expiry) { return new HttpConnectionCode(serverId, URI.create("https://proxy.example.test:1297/"), "a".repeat(64), "b".repeat(64), expiry, "A".repeat(43)); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 5f1844ea1..9d2cb6e22 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -164,6 +164,23 @@ void rejectedStandaloneHttpBroadcastRetainsStableIdAcrossCacheSerialization() { @Test void acceptedRewardStillPersistsUnresolvedStandaloneBroadcast() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote state = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "player-uuid", + "Service", 100L, true, "totals", false, true, java.util.Set.of("Server1"), + java.util.Collections.emptySet(), true, java.util.Collections.emptyMap(), + java.util.Map.of("Server1", "00000000-0000-0000-0000-000000000169")); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + spyProxy.persistUncachedStandaloneBroadcastForTest("player-uuid", state, false); + + assertTrue(state.isRewardDelivered()); + assertFalse(state.isBroadcastForwarded()); + verify(voteCache).addOnlineVote("player-uuid", state); + } + + @Test + void uncachedStandaloneBroadcastPersistsAsBroadcastOnlyState() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); OfflineBungeeVote state = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "player-uuid", "Service", 100L, true, "totals", false, true, java.util.Set.of("Server1"), @@ -179,6 +196,50 @@ void acceptedRewardStillPersistsUnresolvedStandaloneBroadcast() { verify(voteCache).addOnlineVote("player-uuid", state); } + @Test + void standaloneBackendCacheRowDoesNotCopyCanonicalMultiTargetState() { + OfflineBungeeVote row = votingPluginProxy.createCachedRewardVoteForTest( + java.util.UUID.randomUUID(), "Player", "player-uuid", "Service", 100L, true, "totals", true); + + assertTrue(row.isBroadcastForwarded()); + assertFalse(row.isProxyBroadcastHandled()); + assertTrue(row.getBroadcastTargets().isEmpty()); + assertTrue(row.getBroadcastForwardedServers().isEmpty()); + assertTrue(row.getHttpBroadcastDeliveryIds().isEmpty()); + } + + @Test + void queuedRolloverReplayReusesStableStandaloneBroadcastDeliveryId() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + String deliveryId = "00000000-0000-0000-0000-000000000172"; + VoteTimeQueue queued = new VoteTimeQueue(java.util.UUID.randomUUID(), "Player", "Service", 100L, true, + java.util.Set.of("Server1", "Server2"), java.util.Set.of("Server1"), "totals", false, + "player-uuid", java.util.Map.of("Server2", deliveryId)); + queue.add(queued); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.updateTimeVote(queued)).thenReturn(true); + Mockito.when(voteCache.removeTimeVote(queued)).thenAnswer(invocation -> queue.remove(queued)); + Mockito.when(votingPluginProxy.getConfig().getPrimaryServer()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getBungeeManageTotals()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getProxyBroadcastEnabled()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getProxyBroadcastOfflineMode()).thenReturn("FORWARD"); + votingPluginProxy.setGlobalMessageProxyHandlerForTest( + Mockito.mock(com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler.class)); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.processQueue(); + + assertTrue(queue.isEmpty()); + assertEquals(java.util.List.of(deliveryId), spyProxy.getAttemptedVotePartyDeliveryIds()); + verify(voteCache).updateTimeVote(queued); + verify(voteCache).removeTimeVote(queued); + verify(voteCache, never()).addOnlineVote(Mockito.anyString(), Mockito.any(OfflineBungeeVote.class)); + } + @Test void testAddVoteParty() { // Initial votePartyVotes should be 0 diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index ca4b12354..2b2ade005 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -473,6 +473,11 @@ public void persistUncachedStandaloneBroadcastForTest(String uuid, OfflineBungee persistUncachedStandaloneBroadcast(uuid, state, alreadyPersisted); } + public OfflineBungeeVote createCachedRewardVoteForTest(UUID voteId, String player, String uuid, String service, + long time, boolean realVote, String text, boolean standaloneProxyBroadcast) { + return createCachedRewardVote(voteId, player, uuid, service, time, realVote, text, standaloneProxyBroadcast); + } + public void retryPendingTimeBroadcastsForTest(String server) { retryPendingTimeBroadcasts(server); } From ce7443955ba7d2062cf7305ad2cdb5e689f414f9 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:25:46 -0600 Subject: [PATCH 055/136] Fence vote-party work during transport changes --- .../votingplugin/proxy/VotingPluginProxy.java | 56 +++++++++++++++++-- .../tests/VotingPluginProxyTest.java | 31 ++++++++++ .../tests/VotingPluginProxyTestImpl.java | 4 ++ 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 1d9bacb5c..5b1575b2e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1473,6 +1473,7 @@ public void debug1(Throwable e) { } }; voteCacheHandler.load(); + method = retainHttpForPendingVotePartyRewards(method); nonVotedPlayersCache = new NonVotedPlayersCache(getNonVotedCacheMySQLConfig(), getConfig().getNonVotedCacheUseMySQL(), getConfig().getNonVotedCacheUseMainMySQL(), @@ -2624,6 +2625,9 @@ public void onDisable(boolean waitForHosted) { /** Fail-closed gate that must complete before a replacement proxy runtime is created. */ public void prepareForRuntimeReplacement() { + if (!quarantineInFlightVotePartyProxyCommandForReplacement()) { + throw new IllegalStateException("In-flight vote-party command must be durably quarantined before proxy runtime replacement"); + } controlServicesGeneration.incrementAndGet(); synchronized (controlLifecycleLock) { ControlConnector connector = controlConnector; @@ -2848,10 +2852,9 @@ public void reloadFromControl() { } private void reloadRuntime(boolean restartControlServices) { - method = BungeeMethod.getByName(getConfig().getBungeeMethod()); - if (getMethod() == null) { - method = BungeeMethod.PLUGINMESSAGING; - } + BungeeMethod configuredMethod = BungeeMethod.getByName(getConfig().getBungeeMethod()); + if (configuredMethod == null) configuredMethod = BungeeMethod.PLUGINMESSAGING; + method = retainHttpForPendingVotePartyRewards(configuredMethod); warnUnsupportedDedicatedVotingProxyMode(); if (!restartControlServices && method == BungeeMethod.SOCKETS) { rebuildSocketClients(); @@ -2865,6 +2868,21 @@ private void reloadRuntime(boolean restartControlServices) { } } + private synchronized BungeeMethod retainHttpForPendingVotePartyRewards(BungeeMethod configuredMethod) { + if (configuredMethod == BungeeMethod.HTTP) return configuredMethod; + Collection servers = getVoteCachePendingVotePartyServers(); + if (servers != null) { + for (String server : servers) { + Collection rewards = getVoteCachePendingVotePartyRewardIds(server); + if (rewards != null && !rewards.isEmpty()) { + logSevere("Retaining HTTP transport until pending vote-party rewards are acknowledged"); + return BungeeMethod.HTTP; + } + } + } + return configuredMethod; + } + private synchronized void rebuildSocketClients() { HashMap rebuilt = new HashMap<>(); try { @@ -3431,6 +3449,36 @@ private void quarantineTimedOutVotePartyProxyCommand(long attempt, PendingVotePa } } + protected synchronized boolean quarantineInFlightVotePartyProxyCommandForReplacement() { + if (votePartyProxyCommandInFlight == 0L) return true; + PendingVotePartyProxyEffects pending; + PendingVotePartyProxyEffects previousQuarantine; + try { + pending = getVoteCachePendingVotePartyProxyEffects(); + previousQuarantine = getVoteCacheQuarantinedVotePartyProxyEffects(); + if (pending.commands().isEmpty()) return false; + java.util.List commands = new java.util.ArrayList<>(previousQuarantine.commands()); + commands.add(pending.commands().get(0)); + PendingVotePartyProxyEffects quarantine = new PendingVotePartyProxyEffects(previousQuarantine.broadcast(), commands); + PendingVotePartyProxyEffects remaining = new PendingVotePartyProxyEffects("", + pending.commands().subList(1, pending.commands().size())); + setVoteCacheQuarantinedVotePartyProxyEffects(quarantine); + setVoteCachePendingVotePartyProxyEffects(remaining); + try { + saveVotePartyStateDurably(); + } catch (IOException | RuntimeException failure) { + setVoteCacheQuarantinedVotePartyProxyEffects(previousQuarantine); + setVoteCachePendingVotePartyProxyEffects(pending); + return false; + } + votePartyProxyCommandInFlight = 0L; + logSevere("An in-flight HTTP vote-party proxy command was durably quarantined for runtime replacement"); + return true; + } catch (RuntimeException invalid) { + return false; + } + } + private boolean persistVotePartyProxyEffectProgress(PendingVotePartyProxyEffects previous, PendingVotePartyProxyEffects remaining) { setVoteCachePendingVotePartyProxyEffects(remaining); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 9d2cb6e22..b90eb8862 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -447,6 +447,37 @@ void httpVotePartyQuarantinesHungCommandWithoutRetryingIt() { votingPluginProxy.getVoteCacheQuarantinedVotePartyProxyEffects().commands()); } + @Test + void runtimeReplacementQuarantinesAnInFlightVotePartyCommandBeforeHandoff() { + configureHttpVotePartyEffects("", java.util.List.of("in flight", "next command")); + java.util.concurrent.CompletableFuture completion = + votingPluginProxy.delayNextVotePartyCommandCompletion(); + votingPluginProxy.checkVoteParty(); + + assertTrue(votingPluginProxy.quarantineInFlightVotePartyProxyCommandForReplacementForTest()); + + assertEquals(java.util.List.of("in flight"), + votingPluginProxy.getVoteCacheQuarantinedVotePartyProxyEffects().commands()); + assertEquals(java.util.List.of("next command"), + votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().commands()); + completion.complete(null); + assertEquals(java.util.List.of("in flight"), votingPluginProxy.getConsoleCommands()); + } + + @Test + void transportReloadRetainsHttpUntilPendingVotePartyRewardsAreAcknowledged() { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteCachePendingVotePartyReward("Server1", "delivery", true); + Mockito.when(votingPluginProxy.getConfig().getBungeeMethod()).thenReturn("MYSQL"); + + votingPluginProxy.reloadFromControl(); + + assertEquals(BungeeMethod.HTTP, votingPluginProxy.getMethod()); + votingPluginProxy.setVoteCachePendingVotePartyReward("Server1", "delivery", false); + votingPluginProxy.reloadFromControl(); + assertEquals(BungeeMethod.MYSQL, votingPluginProxy.getMethod()); + } + @Test void httpVotePartyRetainsHungCommandFenceWhenQuarantineIsNotDurable() { configureHttpVotePartyEffects("", java.util.List.of("hung command", "next command")); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index 2b2ade005..4479fe8f8 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -490,6 +490,10 @@ public boolean retryPendingVotePartyProxyEffectsForTest() { return retryPendingVotePartyProxyEffects(); } + public boolean quarantineInFlightVotePartyProxyCommandForReplacementForTest() { + return quarantineInFlightVotePartyProxyCommandForReplacement(); + } + public int[] getProjectedVotePartyStateForTest(int acceptedVotes) { return getProjectedVotePartyState(acceptedVotes); } From 874d62fa0a5602c3c916b9fdcf5db0530ad92173 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:25:33 -0600 Subject: [PATCH 056/136] Harden HTTP handoff and broadcast recovery --- .../votingplugin/VotingPluginMain.java | 1 + .../backendproxy/BackendProxyHandler.java | 5 + .../BackendProxyTransportManager.java | 57 ++++++--- .../transport/HttpBackendProxyTransport.java | 40 +++++- .../votingplugin/proxy/VotingPluginProxy.java | 71 ++++++----- .../HttpBackendProxyTransportTest.java | 112 +++++++++++++++++ .../tests/VotingPluginProxyTest.java | 115 +++++++++++++++--- .../tests/VotingPluginProxyTestImpl.java | 4 + 8 files changed, 336 insertions(+), 69 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index aef9d3b7f..91ed2f648 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1300,6 +1300,7 @@ public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long restart.replacement.beginPreparedHttpHandoff(); if (!restart.previous.prepareForReplacement(restart.replacement.getMethod())) throw new IllegalStateException("Previous HTTP proxy transport could not be prepared for replacement"); + restart.previous.reservePreparedHttpHandoff(restart.replacement); } if (restart.replacement != null) restart.replacement.validateTransport(validationDeadlineNanos); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 2c6ff1c6f..f729f3030 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -137,6 +137,11 @@ public void beginPreparedHttpHandoff() { transportManager.beginPreparedHttpHandoff(); } + /** Reserves staged HTTP capacity before publication can admit replacement sends. */ + public void reservePreparedHttpHandoff(BackendProxyHandler replacement) { + transportManager.reservePreparedTransportHandoff(replacement.transportManager); + } + /** Restores a prepared HTTP transport when its replacement fails validation. */ public void restoreAfterFailedReplacement() { transportManager.restoreAfterFailedReplacement(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 24d950b6d..f10e2230a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -121,20 +121,30 @@ public void validate(long deadlineNanos) { else transport.validate(); } - public synchronized void prepareForReplacement() { - if (transport != null) { - preparedTransport = transport; + public void prepareForReplacement() { + BackendProxyTransport candidate; + synchronized (this) { + if (transport == null) return; + candidate = transport; + preparedTransport = candidate; transport = null; - try { - preparedTransport.prepareForReplacement(); - } catch (RuntimeException failure) { - HttpBackendProxyTransport http = preparedTransport instanceof HttpBackendProxyTransport candidate - ? candidate : null; + } + try { + // HTTP preparation can wait on a bounded network flush. Keep send() free to + // enqueue presence and vote messages while that I/O is in progress. + candidate.prepareForReplacement(); + } catch (RuntimeException failure) { + synchronized (this) { + if (preparedTransport != candidate) throw failure; + HttpBackendProxyTransport http = candidate instanceof HttpBackendProxyTransport prepared + ? prepared : null; if (http != null && !http.isClosedForReplacement()) { // A failed flush deliberately restarts the existing connector. Reinstall // that live instance instead of creating a second directory owner. - transport = preparedTransport; + transport = candidate; preparedTransport = null; + while (!preparedSends.isEmpty()) transport.send(preparedSends.removeFirst()); + preparedQueueWarning = false; } else { // Enrollment cancellation may already have closed this instance. Restore // from its captured configuration before configuration rollback. @@ -144,24 +154,41 @@ public synchronized void prepareForReplacement() { failure.addSuppressed(restorationFailure); } } - throw failure; } + throw failure; } } public synchronized void completePreparedTransportHandoff(BackendProxyTransportManager replacement) { if (preparedTransport == null) return; - forwardingManager = java.util.Objects.requireNonNull(replacement, "replacement"); + BackendProxyTransportManager target = java.util.Objects.requireNonNull(replacement, "replacement"); java.util.ArrayList pending = new java.util.ArrayList<>(); if (preparedTransport instanceof HttpBackendProxyTransport http) { - pending.addAll(http.drainPreparedMessages()); + pending.addAll(http.preparedMessagesSnapshot()); } - while (!preparedSends.isEmpty()) pending.add(preparedSends.removeFirst()); - if (forwardingManager.transport instanceof HttpBackendProxyTransport http) { + pending.addAll(preparedSends); + if (target.transport instanceof HttpBackendProxyTransport http) { http.acceptHandoffMessages(pending); } else { - for (JsonEnvelope envelope : pending) forwardingManager.send(envelope); + for (JsonEnvelope envelope : pending) target.send(envelope); } + // Do not consume the old queues or forward subsequent sends until the + // replacement has admitted every snapshot. A failed admission therefore + // leaves rollback with the complete original FIFO intact. + if (preparedTransport instanceof HttpBackendProxyTransport http) http.drainPreparedMessages(); + preparedSends.clear(); + forwardingManager = target; + } + + /** Reserves replacement capacity for every old queued message and future prepared send. */ + public synchronized void reservePreparedTransportHandoff(BackendProxyTransportManager replacement) { + if (!(preparedTransport instanceof HttpBackendProxyTransport previous)) return; + if (!(java.util.Objects.requireNonNull(replacement, "replacement").transport + instanceof HttpBackendProxyTransport target)) + throw new IllegalStateException("HTTP replacement transport is unavailable"); + // send() remains available while the previous credential is fenced. Reserve + // its whole remaining bounded allowance, not only the current queue size. + target.reservePreparedHandoffCapacity(previous.preparedMessageCount() + MAX_PREPARED_SENDS); } public void beginPreparedHttpHandoff() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 628606788..8b35a21d9 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -40,6 +40,8 @@ public final class HttpBackendProxyTransport implements BackendProxyTransport { private final ArrayDeque handoffQueue = new ArrayDeque<>(); private volatile Thread handoffWorker; private boolean awaitingPreparedHandoff; + /** Capacity reserved for the prepared predecessor before this transport is published. */ + private int preparedHandoffReservation; private volatile HttpBackendTransportConnector connector; private volatile Thread worker; private volatile RuntimeException startupFailure; @@ -139,6 +141,21 @@ java.util.List drainPreparedMessages() { } } + int preparedMessageCount() { + synchronized (lifecycle) { + return startupQueue.size() + handoffQueue.size(); + } + } + + java.util.List preparedMessagesSnapshot() { + synchronized (lifecycle) { + java.util.List pending = new java.util.ArrayList<>(startupQueue.size() + handoffQueue.size()); + pending.addAll(startupQueue); + pending.addAll(handoffQueue); + return pending; + } + } + /** Takes both pending queues in their original FIFO order for replacement handoff. */ private java.util.List takeQueuedMessages() { java.util.List pending = new java.util.ArrayList<>(startupQueue.size() + handoffQueue.size()); @@ -488,7 +505,9 @@ public void send(JsonEnvelope envelope) { synchronized (lifecycle) { if (closed) return; if (awaitingPreparedHandoff || !handoffQueue.isEmpty()) { - int capacity = awaitingPreparedHandoff ? MAX_PREPUBLICATION_QUEUE : MAX_HANDOFF_QUEUE; + int capacity = awaitingPreparedHandoff + ? Math.min(MAX_PREPUBLICATION_QUEUE, MAX_HANDOFF_QUEUE - preparedHandoffReservation) + : MAX_HANDOFF_QUEUE; if (handoffQueue.size() < capacity) handoffQueue.addLast(envelope); else warnRejectedSend(); return; @@ -512,6 +531,24 @@ void beginPreparedHandoff() { } } + /** + * Reserves enough of the bounded handoff queue for the predecessor before callers + * can send through this staged replacement. This turns an otherwise late, + * destructive capacity failure into a pre-publication validation failure. + */ + void reservePreparedHandoffCapacity(int messages) { + if (messages < 0 || messages > MAX_HANDOFF_QUEUE) + throw new IllegalStateException("HTTP prepared handoff exceeds its fixed capacity"); + synchronized (lifecycle) { + if (closed) throw new IllegalStateException("HTTP replacement transport is closed"); + if (!awaitingPreparedHandoff) + throw new IllegalStateException("HTTP replacement transport is not awaiting a prepared handoff"); + if (handoffQueue.size() > MAX_HANDOFF_QUEUE - messages) + throw new IllegalStateException("HTTP handoff queue exceeded its reserved capacity"); + preparedHandoffReservation = messages; + } + } + void acceptHandoffMessages(java.util.List messages) { Thread drain; synchronized (lifecycle) { @@ -523,6 +560,7 @@ void acceptHandoffMessages(java.util.List messages) { handoffQueue.addAll(messages); handoffQueue.addAll(newer); awaitingPreparedHandoff = false; + preparedHandoffReservation = 0; if (handoffQueue.isEmpty()) return; if (handoffWorker != null) return; drain = new Thread(this::drainHandoffMessages, "VotingPlugin-HTTP-Backend-Handoff"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 5b1575b2e..d1e6c6393 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -560,15 +560,37 @@ private Set sendProxyBroadcast(Set targets, String uuid, String String text, boolean wasOnline, VoteTimeQueue cachedVote) { Set forwarded = new LinkedHashSet<>(); for (String targetServer : targets) { + // A timed vote can survive a proxy restart. Persist its HTTP delivery ID before + // publication so a crash after the transport accepts it replays with the same + // ID rather than creating a second backend broadcast. + if (method == BungeeMethod.HTTP && !prepareTimedHttpBroadcastDelivery(targetServer, cachedVote)) { + continue; + } JsonEnvelope envelope = VotingPluginWire.voteBroadcast(uuid, player, service, time, text, wasOnline); boolean accepted = method == BungeeMethod.HTTP ? sendHttpBroadcastEnvelopeWithRecovery(targetServer, envelope, cachedVote) : sendProxyBroadcastEnvelopeNow(targetServer, envelope); - if (accepted) forwarded.add(targetServer); + if (accepted) { + forwarded.add(targetServer); + if (method == BungeeMethod.HTTP) { + cachedVote.getBroadcastForwardedServers().add(targetServer); + // Persist completion before preparing another target. Otherwise persisting + // that target's ID could leave this accepted target looking pending after a + // crash, and it would be replayed under a newly generated ID. + if (!persistTimeVoteDelivery(cachedVote)) break; + } + } } return forwarded; } + private boolean prepareTimedHttpBroadcastDelivery(String server, VoteTimeQueue vote) { + if (vote.getHttpBroadcastDeliveryId(server) != null) return true; + vote.setHttpBroadcastDeliveryId(server, UUID.randomUUID().toString()); + vote.setDeliveryStateDirty(true); + return persistTimeVoteDelivery(vote); + } + protected boolean sendHttpBroadcastEnvelopeWithRecovery(String server, JsonEnvelope envelope, OfflineBungeeVote cachedVote) { String stableId = cachedVote == null ? null : cachedVote.getHttpBroadcastDeliveryId(server); @@ -584,13 +606,12 @@ protected boolean sendHttpBroadcastEnvelopeWithRecovery(String server, JsonEnvel if (cachedVote != null) { cachedVote.setHttpBroadcastDeliveryId(server, failure.deliveryId()); cachedVote.setDeliveryStateDirty(true); + // Let the caller persist the recovered ID before retrying. Retrying here + // would create a crash window after acceptance but before durable cache state. + return false; } try { boolean accepted = sendHttpEnvelope(server, failure.deliveryId(), envelope); - if (accepted && cachedVote != null) { - cachedVote.setHttpBroadcastDeliveryId(server, null); - cachedVote.setDeliveryStateDirty(true); - } return accepted; } catch (RuntimeException retryFailure) { debug("Unable to recover HTTP standalone delivery " + failure.deliveryId() + ": " @@ -605,32 +626,18 @@ protected boolean sendHttpBroadcastEnvelopeWithRecovery(String server, JsonEnvel private boolean sendHttpBroadcastEnvelopeWithRecovery(String server, JsonEnvelope envelope, VoteTimeQueue cachedVote) { - String stableId = cachedVote == null ? null : cachedVote.getHttpBroadcastDeliveryId(server); + String stableId = cachedVote.getHttpBroadcastDeliveryId(server); + if (stableId == null) { + debug("Skipping HTTP timed broadcast without a persisted delivery ID for " + server); + return false; + } try { - boolean accepted = stableId == null ? sendProxyBroadcastEnvelopeNow(server, envelope) - : sendHttpEnvelope(server, stableId, envelope); - if (accepted && cachedVote != null && stableId != null) { + boolean accepted = sendHttpEnvelope(server, stableId, envelope); + if (accepted) { cachedVote.setHttpBroadcastDeliveryId(server, null); cachedVote.setDeliveryStateDirty(true); } return accepted; - } catch (HttpProxyTransportServer.DeliveryRetryException failure) { - if (cachedVote != null) { - cachedVote.setHttpBroadcastDeliveryId(server, failure.deliveryId()); - cachedVote.setDeliveryStateDirty(true); - } - try { - boolean accepted = sendHttpEnvelope(server, failure.deliveryId(), envelope); - if (accepted && cachedVote != null) { - cachedVote.setHttpBroadcastDeliveryId(server, null); - cachedVote.setDeliveryStateDirty(true); - } - return accepted; - } catch (RuntimeException retryFailure) { - debug("Unable to recover HTTP timed broadcast delivery " + failure.deliveryId() + ": " - + retryFailure.getMessage()); - return false; - } } catch (RuntimeException failure) { debug("Unable to send HTTP timed broadcast delivery: " + failure.getMessage()); return false; @@ -2972,13 +2979,11 @@ protected boolean sendHttpEnvelopeWithRecovery(String server, JsonEnvelope envel if (cachedVote != null) { cachedVote.setHttpDeliveryId(server, failure.deliveryId()); cachedVote.setDeliveryStateDirty(true); + // Retry only after the caller durably records the recovered transport ID. + return false; } try { boolean accepted = sendHttpEnvelope(server, failure.deliveryId(), envelope); - if (accepted && cachedVote != null) { - cachedVote.setHttpDeliveryId(server, null); - cachedVote.setDeliveryStateDirty(true); - } return accepted; } catch (RuntimeException retryFailure) { debug("Unable to recover HTTP vote delivery " + failure.deliveryId() + ": " + retryFailure.getMessage()); @@ -3978,9 +3983,9 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea for (String target : broadcastTargets) { Set forwarded = sendProxyBroadcast(Collections.singleton(target), uuid, player, service, time, projectedTotals == null ? "" : projectedTotals.toString(), false, delayedVote); - if (delayedVote.getBroadcastForwardedServers().addAll(forwarded) - || delayedVote.isDeliveryStateDirty()) { - broadcastForwardedServers.addAll(forwarded); + boolean newlyForwarded = delayedVote.getBroadcastForwardedServers().addAll(forwarded); + broadcastForwardedServers.addAll(forwarded); + if (newlyForwarded || delayedVote.isDeliveryStateDirty()) { persistTimeVoteDelivery(delayedVote); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 388f24bd9..4e84548e6 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -176,6 +176,46 @@ void failedFlushKeepsTheRestartedHttpTransport() throws Exception { verify(active, org.mockito.Mockito.never()).recreatePrepared(); } + @Test + void sendsRemainNonBlockingWhileReplacementFlushes() throws Exception { + BackendProxyTransportManager manager = new BackendProxyTransportManager(mock(VotingPluginMain.class)); + BackendProxyTransport active = mock(BackendProxyTransport.class); + BackendProxyTransportManager replacement = new BackendProxyTransportManager(mock(VotingPluginMain.class)); + BackendProxyTransport replacementTransport = mock(BackendProxyTransport.class); + JsonEnvelope queued = JsonEnvelope.builder("queued-during-flush").build(); + setField(manager, "transport", active); + setField(replacement, "transport", replacementTransport); + java.util.concurrent.CountDownLatch flushStarted = new java.util.concurrent.CountDownLatch(1); + java.util.concurrent.CountDownLatch releaseFlush = new java.util.concurrent.CountDownLatch(1); + doAnswer(invocation -> { + flushStarted.countDown(); + assertTrue(releaseFlush.await(2, java.util.concurrent.TimeUnit.SECONDS)); + return null; + }).when(active).prepareForReplacement(); + + java.util.concurrent.atomic.AtomicReference preparationFailure = new java.util.concurrent.atomic.AtomicReference<>(); + Thread preparation = new Thread(() -> { + try { + manager.prepareForReplacement(); + } catch (Throwable failure) { + preparationFailure.set(failure); + } + }); + preparation.start(); + assertTrue(flushStarted.await(1, java.util.concurrent.TimeUnit.SECONDS)); + + java.util.concurrent.CompletableFuture sent = java.util.concurrent.CompletableFuture.runAsync( + () -> manager.send(queued)); + assertDoesNotThrow(() -> sent.get(1, java.util.concurrent.TimeUnit.SECONDS)); + releaseFlush.countDown(); + preparation.join(2000L); + assertFalse(preparation.isAlive()); + assertNull(preparationFailure.get()); + + manager.completePreparedTransportHandoff(replacement); + verify(replacementTransport).send(queued); + } + @Test void timedOutScheduledMessageCannotExecuteLater() { VotingPluginMain plugin = mock(VotingPluginMain.class); @@ -554,6 +594,78 @@ void preparedHandoffMessagesStayAheadOfNewlyPublishedSendsWhenConnectorIsFull() transport.close(); } + @Test + @SuppressWarnings("unchecked") + void reservationKeepsPreparedHandoffWithinTheReplacementCapacity() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + BackendProxyTransportManager previous = new BackendProxyTransportManager(plugin); + BackendProxyTransportManager replacement = new BackendProxyTransportManager(plugin); + HttpBackendProxyTransport oldTransport = new HttpBackendProxyTransport(plugin); + HttpBackendProxyTransport newTransport = new HttpBackendProxyTransport(plugin); + setField(previous, "preparedTransport", oldTransport); + setField(replacement, "transport", newTransport); + java.util.ArrayDeque oldQueue = + (java.util.ArrayDeque) field(oldTransport, "handoffQueue"); + List oldMessages = new ArrayList<>(); + List preparedMessages = new ArrayList<>(); + List replacementMessages = new ArrayList<>(); + for (int index = 0; index < 2048; index++) { + JsonEnvelope envelope = JsonEnvelope.builder("old-" + index).build(); + oldQueue.addLast(envelope); + oldMessages.add(envelope); + } + for (int index = 0; index < 1024; index++) { + JsonEnvelope envelope = JsonEnvelope.builder("prepared-" + index).build(); + previous.send(envelope); + preparedMessages.add(envelope); + } + + replacement.beginPreparedHttpHandoff(); + previous.reservePreparedTransportHandoff(replacement); + for (int index = 0; index < 2048; index++) { + JsonEnvelope envelope = JsonEnvelope.builder("new-" + index).build(); + replacement.send(envelope); + replacementMessages.add(envelope); + } + + assertDoesNotThrow(() -> previous.completePreparedTransportHandoff(replacement)); + List queued = newTransport.handoffMessagesSnapshot(); + assertEquals(4096, queued.size()); + assertEquals(oldMessages, queued.subList(0, 2048)); + assertEquals(preparedMessages, queued.subList(2048, 3072)); + assertEquals(replacementMessages.subList(0, 1024), queued.subList(3072, 4096)); + assertEquals(0, oldTransport.preparedMessageCount()); + assertTrue(((java.util.ArrayDeque) field(previous, "preparedSends")).isEmpty()); + newTransport.close(); + } + + @Test + @SuppressWarnings("unchecked") + void failedHandoffAdmissionLeavesTheOldPreparedQueuesIntact() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + BackendProxyTransportManager previous = new BackendProxyTransportManager(plugin); + BackendProxyTransportManager replacement = new BackendProxyTransportManager(plugin); + HttpBackendProxyTransport oldTransport = new HttpBackendProxyTransport(plugin); + HttpBackendProxyTransport newTransport = new HttpBackendProxyTransport(plugin); + JsonEnvelope old = JsonEnvelope.builder("old").build(); + JsonEnvelope prepared = JsonEnvelope.builder("prepared").build(); + setField(previous, "preparedTransport", oldTransport); + setField(replacement, "transport", newTransport); + ((java.util.ArrayDeque) field(oldTransport, "startupQueue")).addLast(old); + previous.send(prepared); + java.util.ArrayDeque replacementQueue = + (java.util.ArrayDeque) field(newTransport, "handoffQueue"); + for (int index = 0; index < 4096; index++) + replacementQueue.addLast(JsonEnvelope.builder("replacement-" + index).build()); + + assertThrows(IllegalStateException.class, () -> previous.completePreparedTransportHandoff(replacement)); + + assertEquals(List.of(old), oldTransport.preparedMessagesSnapshot()); + assertEquals(List.of(prepared), new ArrayList<>((java.util.ArrayDeque) field(previous, "preparedSends"))); + assertNull(field(previous, "forwardingManager")); + } + @Test @SuppressWarnings("unchecked") void preparedDrainPreservesStartupAndExistingHandoffMessagesInFifoOrder() throws Exception { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index b90eb8862..b7e959af7 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -100,9 +100,9 @@ void rejectedRewardBearingAuxiliarySendRetainsItsStableDeliveryIdForRetry() { } @Test - void rejectedIndeterminateHttpVoteRetainsStableIdAcrossCacheSerialization() { + void indeterminateCachedHttpVotePersistsRecoveredIdBeforeRetry() { votingPluginProxy.setMethod(BungeeMethod.HTTP); - votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); String deliveryId = "00000000-0000-0000-0000-000000000160"; votingPluginProxy.failNextGeneratedHttpSend(deliveryId); OfflineBungeeVote pending = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "uuid", @@ -118,9 +118,10 @@ void rejectedIndeterminateHttpVoteRetainsStableIdAcrossCacheSerialization() { pending.getBroadcastForwardedServers(), pending.isRewardDelivered(), OfflineBungeeVote.decodeHttpDeliveryIds(pending.encodeHttpDeliveryIds())); assertEquals(deliveryId, restored.getHttpDeliveryId("SERVER1")); - assertFalse(votingPluginProxy.sendHttpVoteEnvelopeWithRecoveryForTest("Server1", + assertTrue(votingPluginProxy.sendHttpVoteEnvelopeWithRecoveryForTest("Server1", JsonEnvelope.builder("vote").build(), restored)); - assertEquals(java.util.List.of(deliveryId, deliveryId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); + assertEquals(null, restored.getHttpDeliveryId("Server1")); + assertEquals(java.util.List.of(deliveryId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); } @Test @@ -133,17 +134,17 @@ void standaloneHttpBroadcastUsesSeparateStableIdFromRewardDelivery() { pending.setHttpDeliveryId("Server1", "00000000-0000-0000-0000-000000000162"); votingPluginProxy.failNextGeneratedHttpSend(broadcastId); - assertTrue(votingPluginProxy.sendHttpBroadcastEnvelopeWithRecoveryForTest("Server1", + assertFalse(votingPluginProxy.sendHttpBroadcastEnvelopeWithRecoveryForTest("Server1", JsonEnvelope.builder("vote").build(), pending)); - assertEquals(java.util.List.of(broadcastId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); + assertTrue(votingPluginProxy.getAttemptedVotePartyDeliveryIds().isEmpty()); assertEquals("00000000-0000-0000-0000-000000000162", pending.getHttpDeliveryId("Server1")); - assertEquals(null, pending.getHttpBroadcastDeliveryId("Server1")); + assertEquals(broadcastId, pending.getHttpBroadcastDeliveryId("Server1")); } @Test - void rejectedStandaloneHttpBroadcastRetainsStableIdAcrossCacheSerialization() { + void indeterminateCachedStandaloneBroadcastPersistsRecoveredIdBeforeRetry() { votingPluginProxy.setMethod(BungeeMethod.HTTP); - votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); String deliveryId = "00000000-0000-0000-0000-000000000163"; votingPluginProxy.failNextGeneratedHttpSend(deliveryId); OfflineBungeeVote pending = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "uuid", @@ -157,9 +158,10 @@ void rejectedStandaloneHttpBroadcastRetainsStableIdAcrossCacheSerialization() { java.util.Set.of("Server1"), java.util.Set.of(), pending.isRewardDelivered(), OfflineBungeeVote.decodeHttpDeliveryIds(pending.encodeHttpDeliveryIds()), OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(pending.encodeHttpBroadcastDeliveryIds())); - assertFalse(votingPluginProxy.sendHttpBroadcastEnvelopeWithRecoveryForTest("Server1", + assertTrue(votingPluginProxy.sendHttpBroadcastEnvelopeWithRecoveryForTest("Server1", JsonEnvelope.builder("broadcast").build(), restored)); - assertEquals(java.util.List.of(deliveryId, deliveryId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); + assertEquals(null, restored.getHttpBroadcastDeliveryId("Server1")); + assertEquals(java.util.List.of(deliveryId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); } @Test @@ -1358,34 +1360,62 @@ void timedBroadcastRetriesWhileRolloverIsStillActive() { } @Test - void timedHttpBroadcastRetainsStableIdWhenPublicationIsAmbiguous() { + void timedHttpBroadcastDoesNotSendUntilItsNewDeliveryIdIsPersisted() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); VoteTimeQueue vote = new VoteTimeQueue(java.util.UUID.randomUUID(), "OfflineVoter", "Service", 100L, true, java.util.Set.of("Server1"), java.util.Collections.emptySet(), "totals", false, "voter-uuid"); java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); queue.add(vote); Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); - Mockito.when(voteCache.updateTimeVote(vote)).thenReturn(true); + Mockito.when(voteCache.updateTimeVote(vote)).thenReturn(false); Mockito.when(votingPluginProxy.getConfig().getBlockedServers()).thenReturn(java.util.Collections.emptyList()); votingPluginProxy.setMethod(BungeeMethod.HTTP); - votingPluginProxy.setVoteEnvelopeDeliveryResult(false); - String deliveryId = "00000000-0000-0000-0000-000000000166"; - votingPluginProxy.failNextGeneratedHttpSend(deliveryId); VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); spyProxy.retryPendingTimeBroadcastsForTest("Server1"); - assertEquals(deliveryId, vote.getHttpBroadcastDeliveryId("Server1")); - verify(voteCache).updateTimeVote(vote); + assertTrue(vote.getHttpBroadcastDeliveryId("Server1") != null); + assertTrue(spyProxy.getAttemptedVotePartyDeliveryIds().isEmpty()); + } + + @Test + void timedHttpBroadcastPersistsIdBeforeSendAndReusesItAfterPostAcceptCrash() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + VoteTimeQueue vote = new VoteTimeQueue(java.util.UUID.randomUUID(), "OfflineVoter", "Service", 100L, true, + java.util.Set.of("Server1"), java.util.Collections.emptySet(), "totals", false, "voter-uuid"); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + queue.add(vote); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(votingPluginProxy.getConfig().getBlockedServers()).thenReturn(java.util.Collections.emptyList()); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + java.util.concurrent.atomic.AtomicReference persistedDeliveryId = new java.util.concurrent.atomic.AtomicReference<>(); + Mockito.when(voteCache.updateTimeVote(vote)).thenAnswer(invocation -> { + if (persistedDeliveryId.get() == null) { + assertTrue(spyProxy.getAttemptedVotePartyDeliveryIds().isEmpty()); + persistedDeliveryId.set(vote.getHttpBroadcastDeliveryId("Server1")); + return true; + } + throw new IllegalStateException("simulated crash after HTTP acceptance"); + }); + + assertThrows(IllegalStateException.class, () -> spyProxy.retryPendingTimeBroadcastsForTest("Server1")); + String deliveryId = persistedDeliveryId.get(); + assertTrue(deliveryId != null && !deliveryId.isBlank()); + assertEquals(java.util.List.of(deliveryId), spyProxy.getAttemptedVotePartyDeliveryIds()); VoteTimeQueue restored = new VoteTimeQueue(vote.getVoteId(), vote.getName(), vote.getService(), vote.getTime(), - vote.isProxyBroadcastHandled(), vote.getBroadcastTargets(), vote.getBroadcastForwardedServers(), + vote.isProxyBroadcastHandled(), vote.getBroadcastTargets(), java.util.Collections.emptySet(), vote.getTotals(), vote.isProcessed(), vote.getUuid(), - VoteTimeQueue.decodeHttpBroadcastDeliveryIds(vote.encodeHttpBroadcastDeliveryIds())); + java.util.Map.of("Server1", deliveryId)); java.util.Queue restoredQueue = new java.util.concurrent.ConcurrentLinkedQueue<>(); restoredQueue.add(restored); Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(restoredQueue); + Mockito.when(voteCache.updateTimeVote(restored)).thenReturn(true); spyProxy.setVoteEnvelopeDeliveryResult(true); spyProxy.retryPendingTimeBroadcastsForTest("Server1"); @@ -1394,6 +1424,51 @@ void timedHttpBroadcastRetainsStableIdWhenPublicationIsAmbiguous() { assertEquals(java.util.List.of(deliveryId, deliveryId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); } + @Test + void timedHttpBroadcastPersistsEachAcceptedTargetBeforePreparingTheNext() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + VoteTimeQueue vote = new VoteTimeQueue(java.util.UUID.randomUUID(), "OfflineVoter", "Service", 100L, true, + new java.util.LinkedHashSet<>(java.util.List.of("Server1", "Server2")), + java.util.Collections.emptySet(), "totals", false, "voter-uuid"); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + queue.add(vote); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(votingPluginProxy.getConfig().getBlockedServers()).thenReturn(java.util.Collections.emptyList()); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + java.util.concurrent.atomic.AtomicReference persistedBeforeSecondSend = + new java.util.concurrent.atomic.AtomicReference<>(); + Mockito.when(voteCache.updateTimeVote(vote)).thenAnswer(invocation -> { + if (vote.getHttpBroadcastDeliveryId("Server2") != null) { + persistedBeforeSecondSend.set(new VoteTimeQueue(vote.getVoteId(), vote.getName(), vote.getService(), + vote.getTime(), vote.isProxyBroadcastHandled(), vote.getBroadcastTargets(), + vote.getBroadcastForwardedServers(), vote.getTotals(), vote.isProcessed(), vote.getUuid(), + vote.getHttpBroadcastDeliveryIds())); + throw new IllegalStateException("simulated crash after persisting the second target ID"); + } + return true; + }); + + assertThrows(IllegalStateException.class, spyProxy::retryPendingTimeBroadcastsForTest); + VoteTimeQueue restored = persistedBeforeSecondSend.get(); + assertTrue(restored.getBroadcastForwardedServers().contains("Server1")); + assertFalse(restored.getBroadcastForwardedServers().contains("Server2")); + String secondDeliveryId = restored.getHttpBroadcastDeliveryId("Server2"); + assertTrue(secondDeliveryId != null && !secondDeliveryId.isBlank()); + + java.util.Queue restoredQueue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + restoredQueue.add(restored); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(restoredQueue); + Mockito.when(voteCache.updateTimeVote(restored)).thenReturn(true); + spyProxy.retryPendingTimeBroadcastsForTest(); + + assertEquals(2, votingPluginProxy.getAttemptedVotePartyDeliveryIds().size()); + assertEquals(secondDeliveryId, votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(1)); + } + @Test void failedTimedDeliveryStatePersistenceRemainsDirtyUntilRetrySucceeds() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index 4479fe8f8..af1b449e9 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -482,6 +482,10 @@ public void retryPendingTimeBroadcastsForTest(String server) { retryPendingTimeBroadcasts(server); } + public void retryPendingTimeBroadcastsForTest() { + retryPendingTimeBroadcasts(); + } + public void retryPendingVotePartyRewardsForTest() { retryPendingVotePartyRewards(); } From c45f3ea0b2991719fb01269d5c70cb3abe8f5dbe Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:01:06 -0600 Subject: [PATCH 057/136] Preserve proxy deliveries across transport changes --- .../votingplugin/VotingPluginMain.java | 75 ++++- .../backendproxy/BackendProxyHandler.java | 32 +- .../BackendProxyTransportManager.java | 285 ++++++++++++++++-- .../transport/MqttBackendProxyTransport.java | 5 +- .../transport/RedisBackendProxyTransport.java | 4 +- .../votingplugin/proxy/OfflineBungeeVote.java | 5 + .../votingplugin/proxy/VotingPluginProxy.java | 63 +++- ...PluginMainBackendProxyPublicationTest.java | 104 ++++++- .../BackendProxyHandlerLifecycleTest.java | 182 ++++++++++- .../HttpBackendProxyTransportTest.java | 3 +- .../tests/VotingPluginProxyTest.java | 80 +++++ 11 files changed, 786 insertions(+), 52 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 91ed2f648..ef20e86f8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1275,11 +1275,14 @@ private BackendProxyRestartPreparationException(BackendProxyRestart restart, Run public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { BackendProxyHandler previous = backendProxyHandler; if (!bungeeSettings.isUseBungeecoord()) { - return new BackendProxyRestart(previous, null, true, false); + boolean previousRequiresPreparation = previous != null && previous.requiresPreparationForReplacement(); + return new BackendProxyRestart(previous, null, true, previousRequiresPreparation); } BungeeMethod replacementMethod = BungeeMethod.getByName(bungeeSettings.getBungeeMethod()); - boolean previousRequiresPreparation = previous != null && previous.getMethod() == replacementMethod - && replacementMethod == BungeeMethod.HTTP; + // Every transition away from an active HTTP transport must first drain its + // durable outgoing queue. Restricting preparation to HTTP-to-HTTP swaps can + // strand accepted deliveries when another transport is published. + boolean previousRequiresPreparation = previous != null && previous.requiresPreparationForReplacement(); BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); try { replacement.loadForReplacement(); @@ -1297,10 +1300,12 @@ public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long // worker join fails. Mark the restart first so abort/await still owns the // restoration path after a partially completed preparation. restart.previousPrepared = true; - restart.replacement.beginPreparedHttpHandoff(); - if (!restart.previous.prepareForReplacement(restart.replacement.getMethod())) - throw new IllegalStateException("Previous HTTP proxy transport could not be prepared for replacement"); - restart.previous.reservePreparedHttpHandoff(restart.replacement); + BungeeMethod replacementMethod = restart.replacement == null ? null : restart.replacement.getMethod(); + if (restart.replacement != null) restart.replacement.beginPreparedHttpHandoff(); + if (!restart.previous.prepareForReplacement(replacementMethod, validationDeadlineNanos)) + throw new IllegalStateException("Previous proxy transport could not be prepared for replacement"); + if (replacementMethod == BungeeMethod.HTTP) + restart.previous.reservePreparedHttpHandoff(restart.replacement); } if (restart.replacement != null) restart.replacement.validateTransport(validationDeadlineNanos); } @@ -1314,13 +1319,29 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { } if (backendProxyHandler != restart.previous) throw new IllegalStateException("Backend proxy handler changed during restart"); if (restart.disabled) { + if (restart.previous != null && !restart.previous.commitPreparedDisable()) + throw new IllegalStateException("Backend proxy transport accepted a delivery while disabling"); backendProxyHandler = null; - if (restart.previous != null) restart.previous.close(); BackendControlAutoEnrollment enrollment = backendControlAutoEnrollment; backendControlAutoEnrollment = null; - if (enrollment != null) enrollment.close(); restart.finished = true; restart.published = true; + if (restart.previous != null) { + try { + restart.previous.close(); + } catch (RuntimeException cleanupFailure) { + getLogger().warning("Previous backend proxy handler did not stop cleanly after disabling"); + debug(cleanupFailure); + } + } + if (enrollment != null) { + try { + enrollment.close(); + } catch (RuntimeException cleanupFailure) { + getLogger().warning("Backend Control enrollment did not stop cleanly after disabling"); + debug(cleanupFailure); + } + } return; } publishBackendProxyHandler(restart.previous, restart.replacement); @@ -1343,13 +1364,39 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { restart.finished = true; throw handoffFailure; } - // Keep the prepared HTTP queue owned by the previous handler until every - // fallible publication step succeeds. That makes activation rollback - // reversible even when an in-flight sender reaches the previous handler. - if (restart.previous != null) restart.previous.completeHttpHandoff(restart.replacement); - if (restart.previous != null) restart.previous.close(); + // Keep the prepared queue owned by the previous handler until every fallible + // publication step succeeds. Admission performs no network I/O. + try { + if (restart.previous != null) restart.previous.completeHttpHandoff(restart.replacement); + } catch (RuntimeException handoffFailure) { + backendProxyHandler = restart.previous; + try { + restart.replacement.close(); + } catch (RuntimeException closeFailure) { + handoffFailure.addSuppressed(closeFailure); + } + try { + restart.previous.restoreAfterFailedReplacement(); + restart.previous.refreshPresenceAfterFailedReplacement(); + } catch (RuntimeException restorationFailure) { + handoffFailure.addSuppressed(restorationFailure); + } + restart.finished = true; + throw handoffFailure; + } restart.finished = true; restart.published = true; + if (restart.previous != null) { + try { + restart.previous.close(); + } catch (RuntimeException cleanupFailure) { + // Publication and handoff are already committed. A predecessor that + // misses its shutdown deadline must not make Control roll back the + // replacement and close the only live handler. + getLogger().warning("Previous backend proxy handler did not stop cleanly after publication"); + debug(cleanupFailure); + } + } } try { refreshBackendControlAutoEnrollment(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index f729f3030..5323bf545 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -124,13 +124,35 @@ public void close() { globalDataSync.close(); } - /** Prepares the restorable HTTP transport before a same-method replacement starts. */ + /** Returns whether replacement preparation must preserve accepted deliveries. */ + public boolean requiresPreparationForReplacement() { + return method == BungeeMethod.HTTP || method == BungeeMethod.PLUGINMESSAGING + || transportManager.hasPendingAsyncHandoff(); + } + + /** Prepares HTTP state or waits off-thread for an earlier cross-transport handoff. */ public boolean prepareForReplacement(BungeeMethod replacementMethod) { - if (method == replacementMethod && method == BungeeMethod.HTTP) { + return prepareForReplacement(replacementMethod, + System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(25)); + } + + public boolean prepareForReplacement(BungeeMethod replacementMethod, long deadlineNanos) { + if (method == BungeeMethod.HTTP) { transportManager.prepareForReplacement(); return true; } - return false; + if (method == BungeeMethod.PLUGINMESSAGING) { + transportManager.prepareAsyncHandoffForReplacement(deadlineNanos); + return true; + } + if (!transportManager.hasPendingAsyncHandoff()) return false; + transportManager.awaitAsyncHandoff(deadlineNanos); + return true; + } + + /** Atomically fences new sends only when disabling cannot discard prepared HTTP messages. */ + public boolean commitPreparedDisable() { + return transportManager.commitPreparedDisable(); } public void beginPreparedHttpHandoff() { @@ -178,9 +200,9 @@ public void completeRedisHandoff(BackendProxyHandler replacement) { transportManager.completeRedisHandoff(replacement.transportManager); } - /** Forwards messages buffered while the previous HTTP credentials were released. */ + /** Forwards messages buffered while the previous transport was fenced. */ public void completeHttpHandoff(BackendProxyHandler replacement) { - if (method != BungeeMethod.HTTP || replacement.method != BungeeMethod.HTTP) return; + if (replacement == null) return; transportManager.completePreparedTransportHandoff(replacement.transportManager); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index f10e2230a..69f994769 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -16,6 +16,8 @@ */ public class BackendProxyTransportManager { private static final int MAX_PREPARED_SENDS = 1024; + private static final int MAX_ASYNC_HANDOFF_SENDS = 6144; + private static final int PLUGIN_MESSAGE_HANDOFF_BATCH_SIZE = 32; private final VotingPluginMain plugin; private final ProcessedVoteCache processedVoteCache; @@ -24,7 +26,14 @@ public class BackendProxyTransportManager { private BackendProxyTransport retiredTransport; private BackendProxyTransportManager forwardingManager; private final java.util.ArrayDeque preparedSends = new java.util.ArrayDeque<>(); + private final java.util.ArrayDeque asyncHandoffSends = new java.util.ArrayDeque<>(); + private Thread asyncHandoffWorker; + private boolean pluginMessageHandoffScheduled; + private long handoffGeneration; + private boolean preparedSendFence; + private boolean rejectPreparedSends; private boolean preparedQueueWarning; + private boolean rejectedSendWarning; public BackendProxyTransportManager(VotingPluginMain plugin) { this(plugin, new ProcessedVoteCache()); @@ -67,17 +76,32 @@ public void start(BungeeMethod method, GlobalMessageHandler messageHandler, bool } public synchronized void send(JsonEnvelope envelope) { - if (transport != null) { - transport.send(envelope); + if (rejectPreparedSends) { + if (!rejectedSendWarning) { + rejectedSendWarning = true; + plugin.getLogger().severe("Backend proxy transport is disabled; delivery was not accepted"); + } + } else if (preparedSendFence) { + acceptPreparedSend(envelope); } else if (forwardingManager != null) { forwardingManager.send(envelope); + } else if (hasPendingAsyncHandoff()) { + acceptQueuedTransportSend(envelope); + } else if (transport instanceof PluginMessagingBackendProxyTransport) { + acceptQueuedTransportSend(envelope); + } else if (transport != null) { + transport.send(envelope); } else if (preparedTransport != null) { - if (preparedSends.size() < MAX_PREPARED_SENDS) { - preparedSends.addLast(envelope); - } else if (!preparedQueueWarning) { - preparedQueueWarning = true; - plugin.getLogger().severe("HTTP replacement handoff queue is full; delivery was not accepted"); - } + acceptPreparedSend(envelope); + } + } + + private void acceptPreparedSend(JsonEnvelope envelope) { + if (preparedSends.size() < MAX_PREPARED_SENDS) { + preparedSends.addLast(envelope); + } else if (!preparedQueueWarning) { + preparedQueueWarning = true; + plugin.getLogger().severe("Backend proxy replacement handoff queue is full; delivery was not accepted"); } } @@ -86,6 +110,12 @@ public void activateAfterPublication() { } public synchronized void close() { + handoffGeneration++; + if (asyncHandoffWorker != null) asyncHandoffWorker.interrupt(); + asyncHandoffWorker = null; + pluginMessageHandoffScheduled = false; + asyncHandoffSends.clear(); + notifyAll(); if (transport != null) { transport.close(); transport = null; @@ -160,51 +190,263 @@ public void prepareForReplacement() { } public synchronized void completePreparedTransportHandoff(BackendProxyTransportManager replacement) { - if (preparedTransport == null) return; + if (preparedTransport == null && !preparedSendFence) return; BackendProxyTransportManager target = java.util.Objects.requireNonNull(replacement, "replacement"); java.util.ArrayList pending = new java.util.ArrayList<>(); if (preparedTransport instanceof HttpBackendProxyTransport http) { pending.addAll(http.preparedMessagesSnapshot()); } pending.addAll(preparedSends); - if (target.transport instanceof HttpBackendProxyTransport http) { - http.acceptHandoffMessages(pending); - } else { - for (JsonEnvelope envelope : pending) target.send(envelope); - } + target.acceptPreparedHandoffMessages(pending); // Do not consume the old queues or forward subsequent sends until the // replacement has admitted every snapshot. A failed admission therefore // leaves rollback with the complete original FIFO intact. if (preparedTransport instanceof HttpBackendProxyTransport http) http.drainPreparedMessages(); preparedSends.clear(); forwardingManager = target; + preparedSendFence = false; + } + + /** Prevents disabling from discarding a delivery accepted during HTTP preparation. */ + public synchronized boolean commitPreparedDisable() { + if (preparedTransport instanceof HttpBackendProxyTransport http && http.preparedMessageCount() != 0) return false; + if (!preparedSends.isEmpty()) return false; + rejectPreparedSends = true; + return true; + } + + public synchronized boolean hasPendingAsyncHandoff() { + return !asyncHandoffSends.isEmpty() || asyncHandoffWorker != null || pluginMessageHandoffScheduled; + } + + /** Drains an older handoff, then atomically buffers sends until publication. */ + public synchronized void prepareAsyncHandoffForReplacement(long deadlineNanos) { + awaitAsyncHandoff(deadlineNanos); + preparedSendFence = true; + } + + /** Waits off-thread so a later replacement cannot clear an unfinished admitted handoff. */ + public synchronized void awaitAsyncHandoff(long deadlineNanos) { + while (hasPendingAsyncHandoff()) { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0) + throw new IllegalStateException("Timed out waiting for pending backend proxy deliveries"); + try { + java.util.concurrent.TimeUnit.NANOSECONDS.timedWait(this, remaining); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "Interrupted while waiting for pending backend proxy deliveries", interrupted); + } + } + } + + /** Holds staged replacement sends until its predecessor FIFO can be prepended. */ + public synchronized void beginPreparedTransportHandoff() { + if (transport instanceof HttpBackendProxyTransport http) { + http.beginPreparedHandoff(); + } else { + preparedSendFence = true; + } + } + + /** Admits the predecessor before staged replacement messages without caller-thread I/O. */ + private void acceptPreparedHandoffMessages(java.util.List pending) { + if (transport instanceof HttpBackendProxyTransport http) { + http.acceptHandoffMessages(pending); + return; + } + Thread worker = null; + boolean schedulePluginMessages = false; + long generation = 0; + synchronized (this) { + if (!preparedSendFence) + throw new IllegalStateException("Replacement transport is not awaiting a prepared handoff"); + int admitted = pending.size() + preparedSends.size(); + if (admitted > MAX_ASYNC_HANDOFF_SENDS - asyncHandoffSends.size()) + throw new IllegalStateException("Backend proxy handoff queue exceeded its fixed capacity"); + asyncHandoffSends.addAll(pending); + asyncHandoffSends.addAll(preparedSends); + preparedSends.clear(); + preparedSendFence = false; + if (!asyncHandoffSends.isEmpty() && asyncHandoffWorker == null && !pluginMessageHandoffScheduled) { + if (transport instanceof PluginMessagingBackendProxyTransport) { + pluginMessageHandoffScheduled = true; + schedulePluginMessages = true; + generation = handoffGeneration; + } else { + worker = createAsyncHandoffWorker(); + } + } + } + if (schedulePluginMessages) schedulePluginMessageHandoff(generation); + else startAsyncHandoffWorker(worker); + } + + /** Preserves handoff FIFO and keeps plugin-message API access on the primary thread. */ + private void acceptQueuedTransportSend(JsonEnvelope envelope) { + if (asyncHandoffSends.size() >= MAX_ASYNC_HANDOFF_SENDS) { + if (!preparedQueueWarning) { + preparedQueueWarning = true; + plugin.getLogger().severe("Plugin-message delivery queue is full; delivery was not accepted"); + } + return; + } + asyncHandoffSends.addLast(envelope); + if (transport instanceof PluginMessagingBackendProxyTransport) { + if (pluginMessageHandoffScheduled) return; + pluginMessageHandoffScheduled = true; + long generation = handoffGeneration; + try { + schedulePluginMessageHandoff(generation); + } catch (RuntimeException failure) { + pluginMessageHandoffScheduled = false; + notifyAll(); + throw failure; + } + } else if (asyncHandoffWorker == null) { + startAsyncHandoffWorker(createAsyncHandoffWorker()); + } + } + + private Thread createAsyncHandoffWorker() { + Thread worker = new Thread(this::drainAsyncHandoffMessages, + "VotingPlugin-Backend-Transport-Handoff"); + worker.setDaemon(true); + asyncHandoffWorker = worker; + return worker; + } + + private void startAsyncHandoffWorker(Thread worker) { + if (worker == null) return; + try { + worker.start(); + } catch (RuntimeException failure) { + synchronized (this) { + if (asyncHandoffWorker == worker) asyncHandoffWorker = null; + notifyAll(); + } + throw failure; + } + } + + private void drainAsyncHandoffMessages() { + try { + while (!Thread.currentThread().isInterrupted()) { + JsonEnvelope envelope; + BackendProxyTransport target; + synchronized (this) { + envelope = asyncHandoffSends.peekFirst(); + if (envelope == null) { + if (asyncHandoffWorker == Thread.currentThread()) asyncHandoffWorker = null; + notifyAll(); + return; + } + target = transport; + } + if (target == null) return; + target.send(envelope); + synchronized (this) { + if (asyncHandoffSends.peekFirst() == envelope) asyncHandoffSends.removeFirst(); + notifyAll(); + } + } + } finally { + synchronized (this) { + if (asyncHandoffWorker == Thread.currentThread()) asyncHandoffWorker = null; + notifyAll(); + } + } + } + + private void schedulePluginMessageHandoff(long generation) { + try { + plugin.getBukkitScheduler().runTask(plugin, () -> drainPluginMessageHandoff(generation)); + } catch (RuntimeException failure) { + synchronized (this) { + if (handoffGeneration == generation) pluginMessageHandoffScheduled = false; + notifyAll(); + } + throw failure; + } + } + + private void drainPluginMessageHandoff(long generation) { + for (int sent = 0; sent < PLUGIN_MESSAGE_HANDOFF_BATCH_SIZE; sent++) { + JsonEnvelope envelope; + BackendProxyTransport target; + synchronized (this) { + if (handoffGeneration != generation) return; + envelope = asyncHandoffSends.peekFirst(); + if (envelope == null) { + pluginMessageHandoffScheduled = false; + preparedQueueWarning = false; + notifyAll(); + return; + } + target = transport; + } + if (!(target instanceof PluginMessagingBackendProxyTransport)) return; + try { + target.send(envelope); + } catch (RuntimeException failure) { + synchronized (this) { + if (handoffGeneration == generation) pluginMessageHandoffScheduled = false; + notifyAll(); + } + throw failure; + } + synchronized (this) { + if (handoffGeneration != generation) return; + if (asyncHandoffSends.peekFirst() == envelope) asyncHandoffSends.removeFirst(); + notifyAll(); + } + } + synchronized (this) { + if (handoffGeneration != generation) return; + if (asyncHandoffSends.isEmpty()) { + pluginMessageHandoffScheduled = false; + preparedQueueWarning = false; + notifyAll(); + return; + } + } + schedulePluginMessageHandoff(generation); } /** Reserves replacement capacity for every old queued message and future prepared send. */ public synchronized void reservePreparedTransportHandoff(BackendProxyTransportManager replacement) { - if (!(preparedTransport instanceof HttpBackendProxyTransport previous)) return; + if (preparedTransport == null && !preparedSendFence) return; if (!(java.util.Objects.requireNonNull(replacement, "replacement").transport instanceof HttpBackendProxyTransport target)) throw new IllegalStateException("HTTP replacement transport is unavailable"); // send() remains available while the previous credential is fenced. Reserve // its whole remaining bounded allowance, not only the current queue size. - target.reservePreparedHandoffCapacity(previous.preparedMessageCount() + MAX_PREPARED_SENDS); + int previousMessages = preparedTransport instanceof HttpBackendProxyTransport previous + ? previous.preparedMessageCount() : 0; + target.reservePreparedHandoffCapacity(previousMessages + MAX_PREPARED_SENDS); } public void beginPreparedHttpHandoff() { - if (!(transport instanceof HttpBackendProxyTransport http)) - throw new IllegalStateException("HTTP replacement transport is unavailable"); - http.beginPreparedHandoff(); + beginPreparedTransportHandoff(); } public synchronized void restorePreparedTransport() { - if (transport != null || preparedTransport == null) return; + if (preparedTransport == null) { + if (!preparedSendFence) return; + preparedSendFence = false; + while (!preparedSends.isEmpty()) send(preparedSends.removeFirst()); + preparedQueueWarning = false; + return; + } + if (transport != null) return; if (preparedTransport instanceof HttpBackendProxyTransport http) { transport = http.recreatePrepared(); } else { throw new IllegalStateException("Prepared backend proxy transport cannot be restored"); } preparedTransport = null; + preparedSendFence = false; while (!preparedSends.isEmpty()) transport.send(preparedSends.removeFirst()); preparedQueueWarning = false; } @@ -212,9 +454,8 @@ public synchronized void restorePreparedTransport() { public void restoreAfterFailedReplacement() { if (transport instanceof PluginMessagingBackendProxyTransport pluginMessaging) { pluginMessaging.restoreAfterFailedReplacement(); - } else { - restorePreparedTransport(); } + restorePreparedTransport(); } public void awaitPreparedTransportRestoration(long deadlineNanos) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java index 5b9bb44a0..9a8879554 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java @@ -15,6 +15,7 @@ public class MqttBackendProxyTransport implements BackendProxyTransport { private final VotingPluginMain plugin; @Getter private MqttHandler mqttHandler; + private String publishTopic; public MqttBackendProxyTransport(VotingPluginMain plugin) { this.plugin = plugin; @@ -23,6 +24,7 @@ public MqttBackendProxyTransport(VotingPluginMain plugin) { @Override public void start(GlobalMessageHandler messageHandler) { try { + publishTopic = plugin.getBungeeSettings().getMqttPrefix() + "votingplugin/servers/proxy"; String id = plugin.getBungeeSettings().getMqttClientID(); if (id.isEmpty()) { id = plugin.getOptions().getServer(); @@ -49,8 +51,7 @@ public void send(JsonEnvelope envelope) { return; } try { - mqttHandler.publishEnvelope(plugin.getBungeeSettings().getMqttPrefix() + "votingplugin/servers/proxy", - envelope); + mqttHandler.publishEnvelope(publishTopic, envelope); } catch (Exception e) { e.printStackTrace(); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index c95c0d7e1..0c13721d6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -56,6 +56,7 @@ public HandoffQuiescenceException(String message) { private final java.util.ArrayDeque deliveriesAfterReplay = new java.util.ArrayDeque<>(); private GlobalMessageHandler messageHandler; private GlobalMessageHandler handoffMessageHandler; + private String publishChannel; public RedisBackendProxyTransport(VotingPluginMain plugin) { this(plugin, new ProcessedVoteCache()); @@ -69,6 +70,7 @@ public RedisBackendProxyTransport(VotingPluginMain plugin, ProcessedVoteCache pr @Override public void start(GlobalMessageHandler messageHandler) { this.messageHandler = messageHandler; + publishChannel = plugin.getBungeeSettings().getRedisPrefix() + "VotingPlugin"; retiredAfterHandoff = false; standbySubscriber = !processedVoteCache.registerRedisSubscriber(subscriberIdentity); redisHandler = new RedisHandler(plugin.getBungeeSettings().getRedisHost(), @@ -306,7 +308,7 @@ private record BufferedHandoffDelivery(long sequence, JsonEnvelope envelope, Str @Override public void send(JsonEnvelope envelope) { if (redisHandler != null) { - redisHandler.publishEnvelope(plugin.getBungeeSettings().getRedisPrefix() + "VotingPlugin", + redisHandler.publishEnvelope(publishChannel, VotingPluginWire.withRedisDeliveryId(envelope)); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java index bc7cfcc20..965f8cedb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java @@ -295,6 +295,11 @@ public Map getHttpBroadcastDeliveryIds() { return new LinkedHashMap<>(httpBroadcastDeliveryIds); } + /** Returns whether any reward or standalone HTTP delivery is still pending. */ + public boolean hasPendingHttpDeliveryIds() { + return !httpDeliveryIds.isEmpty() || !httpBroadcastDeliveryIds.isEmpty(); + } + /** * Encodes stable delivery IDs for JSON/SQL cache storage. * @return bounded delimiter-safe encoding diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index d1e6c6393..d5b5adbcb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1480,7 +1480,7 @@ public void debug1(Throwable e) { } }; voteCacheHandler.load(); - method = retainHttpForPendingVotePartyRewards(method); + method = retainHttpForPendingDeliveries(method); nonVotedPlayersCache = new NonVotedPlayersCache(getNonVotedCacheMySQLConfig(), getConfig().getNonVotedCacheUseMySQL(), getConfig().getNonVotedCacheUseMainMySQL(), @@ -2861,7 +2861,7 @@ public void reloadFromControl() { private void reloadRuntime(boolean restartControlServices) { BungeeMethod configuredMethod = BungeeMethod.getByName(getConfig().getBungeeMethod()); if (configuredMethod == null) configuredMethod = BungeeMethod.PLUGINMESSAGING; - method = retainHttpForPendingVotePartyRewards(configuredMethod); + method = retainHttpForPendingDeliveries(configuredMethod); warnUnsupportedDedicatedVotingProxyMode(); if (!restartControlServices && method == BungeeMethod.SOCKETS) { rebuildSocketClients(); @@ -2875,8 +2875,31 @@ private void reloadRuntime(boolean restartControlServices) { } } - private synchronized BungeeMethod retainHttpForPendingVotePartyRewards(BungeeMethod configuredMethod) { + private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod configuredMethod) { if (configuredMethod == BungeeMethod.HTTP) return configuredMethod; + HttpProxyTransportServer transport = httpTransportServer; + if (transport != null && transport.hasPendingDeliveries()) { + logSevere("Retaining HTTP transport until durable deliveries are acknowledged"); + return BungeeMethod.HTTP; + } + if (transport == null) { + try { + if (HttpProxyTransportServer.hasPersistedDeliveries( + getDataFolderPlugin().toPath().resolve("http").resolve("outgoing-v1"))) { + logSevere("Retaining HTTP transport until persisted deliveries are acknowledged"); + return BungeeMethod.HTTP; + } + } catch (IOException unreadableQueue) { + // An unreadable durable queue is not proof that it is empty. Reopen HTTP so + // its normal bounded loader can validate or recover the state. + logSevere("Retaining HTTP transport because its persisted delivery queue could not be inspected"); + return BungeeMethod.HTTP; + } + } + if (hasPendingCachedHttpDeliveries()) { + logSevere("Retaining HTTP transport until cached deliveries are acknowledged"); + return BungeeMethod.HTTP; + } Collection servers = getVoteCachePendingVotePartyServers(); if (servers != null) { for (String server : servers) { @@ -2890,6 +2913,40 @@ private synchronized BungeeMethod retainHttpForPendingVotePartyRewards(BungeeMet return configuredMethod; } + private boolean hasPendingCachedHttpDeliveries() { + VoteCacheHandler cache = getVoteCacheHandler(); + if (cache == null) return false; + String[] cachedServers = cache.getCachedVotesServers(); + if (cachedServers != null) { + for (String server : cachedServers) { + Collection votes = cache.getVotes(server); + if (hasPendingHttpDelivery(votes)) return true; + } + } + Collection onlinePlayers = cache.getOnlineVoteUUIDs(); + if (onlinePlayers != null) { + for (String uuid : onlinePlayers) { + Collection votes = cache.getOnlineVotes(uuid); + if (hasPendingHttpDelivery(votes)) return true; + } + } + Collection timedVotes = cache.getTimeChangeQueue(); + if (timedVotes != null) { + for (VoteTimeQueue vote : timedVotes) { + if (vote != null && vote.hasPendingHttpBroadcastDeliveryIds()) return true; + } + } + return false; + } + + private static boolean hasPendingHttpDelivery(Collection votes) { + if (votes == null) return false; + for (OfflineBungeeVote vote : votes) { + if (vote != null && vote.hasPendingHttpDeliveryIds()) return true; + } + return false; + } + private synchronized void rebuildSocketClients() { HashMap rebuilt = new HashMap<>(); try { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index 1f27d5533..30b2350ff 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -1,18 +1,28 @@ package com.bencodez.votingplugin; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.lang.reflect.Constructor; import java.lang.reflect.Field; +import org.bukkit.configuration.file.FileConfiguration; import org.junit.jupiter.api.Test; import com.bencodez.votingplugin.backendproxy.BackendProxyHandler; +import com.bencodez.votingplugin.config.BungeeSettings; +import com.bencodez.votingplugin.config.Config; +import com.bencodez.votingplugin.proxy.BungeeMethod; class VotingPluginMainBackendProxyPublicationTest { @Test @@ -82,6 +92,92 @@ void restoresPreviousHandlerWhenRedisPromotionFails() throws Exception { rollback.verify(previous).refreshPresenceAfterFailedReplacement(); } + @Test + void restoresPreviousHandlerWhenPreparedDeliveryAdmissionFails() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + doThrow(new IllegalStateException("handoff admission failed")) + .when(previous).completeHttpHandoff(replacement); + + assertThrows(IllegalStateException.class, + () -> plugin.completeBackendProxyHandlerRestart(restart(previous, replacement))); + + assertSame(previous, plugin.getBackendProxyHandler()); + verify(replacement).close(); + verify(previous).restoreAfterFailedReplacement(); + verify(previous).refreshPresenceAfterFailedReplacement(); + verify(previous, never()).close(); + } + + @Test + void predecessorCleanupFailureDoesNotRollBackPublishedReplacement() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + when(plugin.getLogger()).thenReturn(mock(java.util.logging.Logger.class)); + Config config = mock(Config.class); + when(config.getData()).thenReturn(mock(FileConfiguration.class)); + when(plugin.getConfigFile()).thenReturn(config); + doAnswer(invocation -> null).when(plugin).debug(org.mockito.ArgumentMatchers.any(Throwable.class)); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + doThrow(new IllegalStateException("listener did not stop")).when(previous).close(); + VotingPluginMain.BackendProxyRestart restart = restart(previous, replacement); + + assertDoesNotThrow(() -> plugin.completeBackendProxyHandlerRestart(restart)); + + assertSame(replacement, plugin.getBackendProxyHandler()); + assertFalse(plugin.requestBackendProxyHandlerRestartAbandonment(restart), + "published runtime state must not be treated as rollbackable"); + verify(previous).completeHttpHandoff(replacement); + } + + @Test + void disablingBungeeCoordinationDrainsActiveHttpTransportBeforeClose() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BungeeSettings settings = mock(BungeeSettings.class); + when(settings.isUseBungeecoord()).thenReturn(false); + setField(plugin, "bungeeSettings", settings); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + when(previous.getMethod()).thenReturn(BungeeMethod.HTTP); + when(previous.requiresPreparationForReplacement()).thenReturn(true); + when(previous.prepareForReplacement(org.mockito.ArgumentMatchers.isNull(), + org.mockito.ArgumentMatchers.anyLong())).thenReturn(true); + when(previous.commitPreparedDisable()).thenReturn(true); + setBackendProxyHandler(plugin, previous); + VotingPluginMain.BackendProxyRestart restart = plugin.prepareBackendProxyHandlerRestart(); + + plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + 1_000_000_000L); + plugin.completeBackendProxyHandlerRestart(restart); + + verify(previous).prepareForReplacement(org.mockito.ArgumentMatchers.isNull(), + org.mockito.ArgumentMatchers.anyLong()); + verify(previous).close(); + assertNull(plugin.getBackendProxyHandler()); + } + + @Test + void disablingKeepsPreviousHandlerWhenPreparationAcceptedAnotherDelivery() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BungeeSettings settings = mock(BungeeSettings.class); + when(settings.isUseBungeecoord()).thenReturn(false); + setField(plugin, "bungeeSettings", settings); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + when(previous.requiresPreparationForReplacement()).thenReturn(true); + when(previous.prepareForReplacement(org.mockito.ArgumentMatchers.isNull(), + org.mockito.ArgumentMatchers.anyLong())).thenReturn(true); + when(previous.commitPreparedDisable()).thenReturn(false); + setBackendProxyHandler(plugin, previous); + VotingPluginMain.BackendProxyRestart restart = plugin.prepareBackendProxyHandlerRestart(); + + plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + 1_000_000_000L); + assertThrows(IllegalStateException.class, () -> plugin.completeBackendProxyHandlerRestart(restart)); + + assertSame(previous, plugin.getBackendProxyHandler()); + verify(previous, never()).close(); + } + private VotingPluginMain.BackendProxyRestart restart(BackendProxyHandler previous, BackendProxyHandler replacement) throws Exception { Constructor constructor = VotingPluginMain.BackendProxyRestart.class @@ -91,8 +187,12 @@ private VotingPluginMain.BackendProxyRestart restart(BackendProxyHandler previou } private void setBackendProxyHandler(VotingPluginMain plugin, BackendProxyHandler handler) throws Exception { - Field field = VotingPluginMain.class.getDeclaredField("backendProxyHandler"); + setField(plugin, "backendProxyHandler", handler); + } + + private void setField(VotingPluginMain plugin, String name, Object value) throws Exception { + Field field = VotingPluginMain.class.getDeclaredField(name); field.setAccessible(true); - field.set(plugin, handler); + field.set(plugin, value); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index a0d515862..4d5d97732 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; @@ -16,6 +17,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import java.lang.reflect.Field; import java.util.concurrent.atomic.AtomicReference; @@ -38,6 +41,7 @@ import com.bencodez.votingplugin.backendproxy.transport.PluginMessagingBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.BackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.BackendProxyTransportManager; +import com.bencodez.votingplugin.backendproxy.transport.HttpBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.RedisBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.SocketBackendProxyTransport; import com.bencodez.votingplugin.proxy.BungeeMethod; @@ -335,6 +339,51 @@ void failedSameMysqlReplacementLeavesExistingTransportAvailable() throws Excepti verifyNoInteractions(messenger); } + @Test + void preparesHttpDeliveryQueueWhenSwitchingToAnotherMethod() throws Exception { + BackendProxyHandler handler = handlerWithTransport(BungeeMethod.HTTP); + HttpBackendProxyTransport transport = mock(HttpBackendProxyTransport.class); + setTransport(handler, transport); + + assertTrue(handler.prepareForReplacement(BungeeMethod.REDIS)); + + verify(transport).prepareForReplacement(); + } + + @Test + void redisSendKeepsTheChannelCapturedByTheActiveTransport() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = + mock(com.bencodez.votingplugin.VotingPluginMain.class); + BungeeSettings settings = mock(BungeeSettings.class); + when(plugin.getBungeeSettings()).thenReturn(settings); + when(settings.getRedisPrefix()).thenReturn("new:"); + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(plugin); + RedisHandler redis = mock(RedisHandler.class); + setField(transport, "redisHandler", redis); + setField(transport, "publishChannel", "old:VotingPlugin"); + + transport.send(com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("vote").build()); + + verify(redis).publishEnvelope(eq("old:VotingPlugin"), any()); + } + + @Test + void mqttSendKeepsTheTopicCapturedByTheActiveTransport() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = + mock(com.bencodez.votingplugin.VotingPluginMain.class); + BungeeSettings settings = mock(BungeeSettings.class); + when(plugin.getBungeeSettings()).thenReturn(settings); + when(settings.getMqttPrefix()).thenReturn("new/"); + MqttBackendProxyTransport transport = new MqttBackendProxyTransport(plugin); + MqttHandler mqtt = mock(MqttHandler.class); + setField(transport, "mqttHandler", mqtt); + setField(transport, "publishTopic", "old/votingplugin/servers/proxy"); + + transport.send(com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("vote").build()); + + verify(mqtt).publishEnvelope(eq("old/votingplugin/servers/proxy"), any()); + } + @Test void forwardsMessagesBufferedDuringPreparedHttpReplacement() throws Exception { com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); @@ -348,16 +397,145 @@ void forwardsMessagesBufferedDuringPreparedHttpReplacement() throws Exception { com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("during-validation").build(); com.bencodez.simpleapi.servercomm.codec.JsonEnvelope afterPublication = com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("after-publication").build(); + com.bencodez.simpleapi.servercomm.codec.JsonEnvelope replacementStarted = + com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("replacement-started").build(); + replacement.beginPreparedTransportHandoff(); previous.prepareForReplacement(); previous.send(duringValidation); + replacement.send(replacementStarted); verifyNoInteractions(replacementTransport); previous.completePreparedTransportHandoff(replacement); previous.send(afterPublication); - verify(replacementTransport).send(duringValidation); - verify(replacementTransport).send(afterPublication); + org.mockito.InOrder order = inOrder(replacementTransport); + order.verify(replacementTransport, org.mockito.Mockito.timeout(1000)).send(duringValidation); + order.verify(replacementTransport).send(replacementStarted); + order.verify(replacementTransport).send(afterPublication); + } + + @Test + void crossTransportPreparedHandoffDoesNotPerformNetworkSendOnCallerThread() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BackendProxyTransportManager previous = new BackendProxyTransportManager(plugin); + BackendProxyTransportManager replacement = new BackendProxyTransportManager(plugin); + BackendProxyTransport previousTransport = mock(BackendProxyTransport.class); + BackendProxyTransport replacementTransport = mock(BackendProxyTransport.class); + setField(previous, "transport", previousTransport); + setField(replacement, "transport", replacementTransport); + java.util.concurrent.CountDownLatch sendStarted = new java.util.concurrent.CountDownLatch(1); + java.util.concurrent.CountDownLatch releaseSend = new java.util.concurrent.CountDownLatch(1); + doAnswer(invocation -> { + sendStarted.countDown(); + releaseSend.await(2, java.util.concurrent.TimeUnit.SECONDS); + return null; + }).when(replacementTransport).send(any()); + + replacement.beginPreparedTransportHandoff(); + previous.prepareForReplacement(); + previous.send(com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("buffered").build()); + org.junit.jupiter.api.Assertions.assertTimeoutPreemptively(java.time.Duration.ofSeconds(1), + () -> previous.completePreparedTransportHandoff(replacement)); + + org.junit.jupiter.api.Assertions.assertTrue(sendStarted.await(1, java.util.concurrent.TimeUnit.SECONDS)); + releaseSend.countDown(); + verify(replacementTransport, org.mockito.Mockito.timeout(1000)).send(any()); + } + + @Test + void preparedDisableRefusesToDiscardMessagesAcceptedDuringPreparation() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BackendProxyTransportManager manager = new BackendProxyTransportManager(plugin); + HttpBackendProxyTransport transport = mock(HttpBackendProxyTransport.class); + setField(manager, "transport", transport); + + manager.prepareForReplacement(); + manager.send(com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("during-disable").build()); + + assertFalse(manager.commitPreparedDisable()); + verify(transport, never()).close(); + } + + @Test + void laterReplacementWaitsForAnEarlierAsyncHandoff() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BackendProxyTransportManager source = new BackendProxyTransportManager(plugin); + BackendProxyHandler target = handlerWithTransport(BungeeMethod.REDIS); + Field managerField = BackendProxyHandler.class.getDeclaredField("transportManager"); + managerField.setAccessible(true); + BackendProxyTransportManager targetManager = (BackendProxyTransportManager) managerField.get(target); + BackendProxyTransport sourceTransport = mock(BackendProxyTransport.class); + BackendProxyTransport targetTransport = mock(BackendProxyTransport.class); + setField(source, "transport", sourceTransport); + setField(targetManager, "transport", targetTransport); + java.util.concurrent.CountDownLatch sendStarted = new java.util.concurrent.CountDownLatch(1); + java.util.concurrent.CountDownLatch releaseSend = new java.util.concurrent.CountDownLatch(1); + doAnswer(invocation -> { + sendStarted.countDown(); + releaseSend.await(2, java.util.concurrent.TimeUnit.SECONDS); + return null; + }).when(targetTransport).send(any()); + + targetManager.beginPreparedTransportHandoff(); + source.prepareForReplacement(); + source.send(com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("buffered").build()); + source.completePreparedTransportHandoff(targetManager); + assertTrue(sendStarted.await(1, java.util.concurrent.TimeUnit.SECONDS)); + assertTrue(target.requiresPreparationForReplacement()); + java.util.concurrent.CompletableFuture preparation = java.util.concurrent.CompletableFuture.supplyAsync( + () -> target.prepareForReplacement(BungeeMethod.MQTT, + System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(2))); + try { + Thread.sleep(50); + assertFalse(preparation.isDone(), "replacement must wait while an admitted send is in flight"); + } finally { + releaseSend.countDown(); + } + assertTrue(preparation.get(1, java.util.concurrent.TimeUnit.SECONDS)); + assertFalse(targetManager.hasPendingAsyncHandoff()); + } + + @Test + void pluginMessageHandoffRunsThroughTheBukkitScheduler() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + com.bencodez.simpleapi.scheduler.BukkitScheduler scheduler = + mock(com.bencodez.simpleapi.scheduler.BukkitScheduler.class); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + java.util.concurrent.atomic.AtomicReference scheduled = new java.util.concurrent.atomic.AtomicReference<>(); + doAnswer(invocation -> { + scheduled.set(invocation.getArgument(1)); + return null; + }).when(scheduler).runTask(eq(plugin), any(Runnable.class)); + BackendProxyTransportManager source = new BackendProxyTransportManager(plugin); + BackendProxyTransportManager target = new BackendProxyTransportManager(plugin); + BackendProxyTransport sourceTransport = mock(BackendProxyTransport.class); + PluginMessagingBackendProxyTransport targetTransport = mock(PluginMessagingBackendProxyTransport.class); + setField(source, "transport", sourceTransport); + setField(target, "transport", targetTransport); + + target.beginPreparedTransportHandoff(); + source.prepareForReplacement(); + source.send(com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("buffered").build()); + source.completePreparedTransportHandoff(target); + + verify(targetTransport, never()).send(any()); + assertTrue(scheduled.get() != null); + scheduled.get().run(); + verify(targetTransport).send(any()); + assertFalse(target.hasPendingAsyncHandoff()); + + target.prepareAsyncHandoffForReplacement( + System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(1)); + com.bencodez.simpleapi.servercomm.codec.JsonEnvelope afterPreparation = + com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("after-preparation").build(); + target.send(afterPreparation); + BackendProxyTransportManager next = new BackendProxyTransportManager(plugin); + BackendProxyTransport nextTransport = mock(BackendProxyTransport.class); + setField(next, "transport", nextTransport); + next.beginPreparedTransportHandoff(); + target.completePreparedTransportHandoff(next); + verify(nextTransport, org.mockito.Mockito.timeout(1000)).send(afterPreparation); } @Test diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 4e84548e6..8c4d1555f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -185,6 +185,7 @@ void sendsRemainNonBlockingWhileReplacementFlushes() throws Exception { JsonEnvelope queued = JsonEnvelope.builder("queued-during-flush").build(); setField(manager, "transport", active); setField(replacement, "transport", replacementTransport); + replacement.beginPreparedTransportHandoff(); java.util.concurrent.CountDownLatch flushStarted = new java.util.concurrent.CountDownLatch(1); java.util.concurrent.CountDownLatch releaseFlush = new java.util.concurrent.CountDownLatch(1); doAnswer(invocation -> { @@ -213,7 +214,7 @@ void sendsRemainNonBlockingWhileReplacementFlushes() throws Exception { assertNull(preparationFailure.get()); manager.completePreparedTransportHandoff(replacement); - verify(replacementTransport).send(queued); + verify(replacementTransport, org.mockito.Mockito.timeout(1000)).send(queued); } @Test diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index b7e959af7..7b6d4ef7b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -11,6 +11,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; @@ -18,6 +19,7 @@ import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalDataHandlerProxy; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.http.HttpProxyTransportServer; import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; import com.bencodez.votingplugin.proxy.BungeeMethod; import com.bencodez.votingplugin.proxy.OfflineBungeeVote; @@ -29,6 +31,8 @@ import com.bencodez.votingplugin.timequeue.VoteTimeQueue; public class VotingPluginProxyTest { + @TempDir + java.nio.file.Path temporaryDirectory; @InjectMocks private VotingPluginProxyTestImpl votingPluginProxy; @@ -480,6 +484,76 @@ void transportReloadRetainsHttpUntilPendingVotePartyRewardsAreAcknowledged() { assertEquals(BungeeMethod.MYSQL, votingPluginProxy.getMethod()); } + @Test + void transportReloadRetainsHttpForPendingServerAndOnlineDeliveries() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote serverVote = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "uuid", + "Service", 100L, true, "totals"); + serverVote.setHttpDeliveryId("Server1", "00000000-0000-0000-0000-000000000180"); + OfflineBungeeVote onlineVote = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "uuid", + "Service", 101L, true, "totals"); + onlineVote.setHttpBroadcastDeliveryId("Server2", "00000000-0000-0000-0000-000000000181"); + Mockito.when(voteCache.getCachedVotesServers()).thenReturn(new String[] { "Server1" }); + Mockito.when(voteCache.getVotes("Server1")) + .thenReturn(new java.util.ArrayList<>(java.util.List.of(serverVote))); + Mockito.when(voteCache.getOnlineVoteUUIDs()).thenReturn(java.util.Set.of("uuid")); + Mockito.when(voteCache.getOnlineVotes("uuid")) + .thenReturn(new java.util.ArrayList<>(java.util.List.of(onlineVote))); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.setMethod(BungeeMethod.HTTP); + Mockito.when(spyProxy.getConfig().getBungeeMethod()).thenReturn("MYSQL"); + + spyProxy.reloadFromControl(); + + assertEquals(BungeeMethod.HTTP, spyProxy.getMethod()); + } + + @Test + void transportReloadRetainsHttpForPendingTimedBroadcast() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + VoteTimeQueue timed = new VoteTimeQueue(java.util.UUID.randomUUID(), "Player", "Service", 100L); + timed.setHttpBroadcastDeliveryId("Server1", "00000000-0000-0000-0000-000000000182"); + Mockito.when(voteCache.getTimeChangeQueue()) + .thenReturn(new java.util.concurrent.ConcurrentLinkedQueue<>(java.util.List.of(timed))); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.setMethod(BungeeMethod.HTTP); + Mockito.when(spyProxy.getConfig().getBungeeMethod()).thenReturn("REDIS"); + + spyProxy.reloadFromControl(); + + assertEquals(BungeeMethod.HTTP, spyProxy.getMethod()); + } + + @Test + void transportReloadRetainsHttpForOrdinaryDurableQueueEntry() throws Exception { + HttpProxyTransportServer transport = Mockito.mock(HttpProxyTransportServer.class); + Mockito.when(transport.hasPendingDeliveries()).thenReturn(true); + setProxyField(votingPluginProxy, "httpTransportServer", transport); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + Mockito.when(votingPluginProxy.getConfig().getBungeeMethod()).thenReturn("MQTT"); + + votingPluginProxy.reloadFromControl(); + + assertEquals(BungeeMethod.HTTP, votingPluginProxy.getMethod()); + } + + @Test + void startupRetainsHttpForPersistedQueueWithoutLiveServer() throws Exception { + java.nio.file.Path backendQueue = temporaryDirectory.resolve("http/outgoing-v1/lobby-1"); + java.nio.file.Files.createDirectories(backendQueue); + java.nio.file.Files.writeString(backendQueue.resolve(".pending-delivery.json"), "pending"); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(temporaryDirectory.toFile()).when(spyProxy).getDataFolderPlugin(); + spyProxy.setMethod(BungeeMethod.HTTP); + Mockito.when(spyProxy.getConfig().getBungeeMethod()).thenReturn("REDIS"); + + spyProxy.reloadFromControl(); + + assertEquals(BungeeMethod.HTTP, spyProxy.getMethod()); + } + @Test void httpVotePartyRetainsHungCommandFenceWhenQuarantineIsNotDurable() { configureHttpVotePartyEffects("", java.util.List.of("hung command", "next command")); @@ -1534,4 +1608,10 @@ void completedBroadcastOnlyOnlineVoteIsRemovedAfterRetry() { verify(voteCache).removeOnlineVote("voter-uuid", vote); verify(voteCache, never()).updateOnlineVote("voter-uuid", vote); } + + private static void setProxyField(VotingPluginProxy target, String name, Object value) throws Exception { + java.lang.reflect.Field field = VotingPluginProxy.class.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } } From f1337fa6d4d77c995fd064c198721812dd17a674 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:55:42 -0600 Subject: [PATCH 058/136] Preserve deliveries across transport replacement --- .../votingplugin/VotingPluginMain.java | 9 +- .../backendproxy/BackendProxyHandler.java | 61 ++++++++- .../transport/BackendProxyTransport.java | 3 +- .../BackendProxyTransportManager.java | 36 +++++- .../transport/HttpBackendProxyTransport.java | 26 +++- .../transport/MqttBackendProxyTransport.java | 11 +- .../transport/MysqlBackendProxyTransport.java | 11 +- .../PluginMessagingBackendProxyTransport.java | 3 +- .../transport/RedisBackendProxyTransport.java | 4 +- .../SocketBackendProxyTransport.java | 4 +- .../votingplugin/proxy/VotingPluginProxy.java | 80 +++++++++++- ...PluginMainBackendProxyPublicationTest.java | 7 +- .../BackendProxyHandlerLifecycleTest.java | 121 +++++++++++++++++- .../HttpBackendProxyTransportTest.java | 1 + .../tests/VotingPluginProxyTest.java | 64 ++++----- .../tests/VotingPluginProxyTestImpl.java | 19 +++ 16 files changed, 388 insertions(+), 72 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index ef20e86f8..27f986a93 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1351,6 +1351,7 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { // Stop the staged presence generation before reasserting the restored old // handler. Otherwise the proxy rejects the old generation after rollback. backendProxyHandler = restart.previous; + restart.replacement.abortStagedInboundTo(restart.previous); try { restart.replacement.close(); } catch (RuntimeException closeFailure) { @@ -1370,6 +1371,7 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { if (restart.previous != null) restart.previous.completeHttpHandoff(restart.replacement); } catch (RuntimeException handoffFailure) { backendProxyHandler = restart.previous; + restart.replacement.abortStagedInboundTo(restart.previous); try { restart.replacement.close(); } catch (RuntimeException closeFailure) { @@ -1384,6 +1386,7 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { restart.finished = true; throw handoffFailure; } + restart.replacement.activateInboundMessages(); restart.finished = true; restart.published = true; if (restart.previous != null) { @@ -1412,6 +1415,7 @@ void publishBackendProxyHandler(BackendProxyHandler previous, BackendProxyHandle replacement.activatePresenceReporting(); } catch (RuntimeException activationFailure) { backendProxyHandler = previous; + replacement.abortStagedInboundTo(previous); throw activationFailure; } } @@ -1426,7 +1430,10 @@ public synchronized boolean requestBackendProxyHandlerRestartAbandonment(Backend public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart restart) { if (restart == null || restart.finished) return; - if (restart.replacement != null) restart.replacement.close(); + if (restart.replacement != null) { + restart.replacement.abortStagedInboundTo(restart.previous); + restart.replacement.close(); + } if (backendProxyHandler == restart.previous && restart.previous != null && (restart.previousPrepared || restart.previous.getMethod() == BungeeMethod.PLUGINMESSAGING)) { restart.previous.restoreAfterFailedReplacement(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 5323bf545..51d09cd46 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -40,6 +40,10 @@ public class BackendProxyHandler implements Listener { private BackendPresenceManager presenceManager; private boolean presenceReportingActivated; + private final Object inboundPublication = new Object(); + private boolean inboundPublished; + private boolean inboundAborted; + private BackendProxyHandler inboundRollbackTarget; private BackendVotePartySync votePartySync; private BackendProxyMessageRouter messageRouter; @@ -78,6 +82,11 @@ private void load(boolean activatePresenceReporting) { globalDataSync.load(); globalMessageHandler = new GlobalMessageHandler() { + @Override + public void onMessage(JsonEnvelope envelope) { + BackendProxyHandler.this.dispatchIncomingAfterPublication(envelope, () -> super.onMessage(envelope)); + } + @Override public void sendMessage(JsonEnvelope envelope) { transportManager.send(envelope); @@ -94,7 +103,10 @@ public void sendMessage(JsonEnvelope envelope) { if (plugin.getOptions().getServer().equalsIgnoreCase("pleaseset")) { plugin.getLogger().warning("Server name for bungee voting is not set, please set it"); } - if (activatePresenceReporting) activatePresenceReporting(); + if (activatePresenceReporting) { + activatePresenceReporting(); + activateInboundMessages(); + } } /** Starts presence only after a staged handler reaches the atomic publication boundary. */ @@ -113,6 +125,10 @@ public void activatePresenceReporting() { * Closes backend/proxy components and persists cached proxy state. */ public void close() { + synchronized (inboundPublication) { + if (!inboundPublished && !inboundAborted) inboundAborted = true; + inboundPublication.notifyAll(); + } if (presenceManager != null && presenceReportingActivated) { presenceManager.stop(); presenceReportingActivated = false; @@ -124,6 +140,47 @@ public void close() { globalDataSync.close(); } + /** Opens inbound dispatch only after the replacement and all handoffs are committed. */ + public void activateInboundMessages() { + synchronized (inboundPublication) { + if (inboundAborted) return; + inboundPublished = true; + inboundPublication.notifyAll(); + } + } + + /** Routes an already accepted staged callback through the restored predecessor on rollback. */ + public void abortStagedInboundTo(BackendProxyHandler previous) { + synchronized (inboundPublication) { + if (inboundPublished || inboundAborted) return; + inboundRollbackTarget = previous; + inboundAborted = true; + inboundPublication.notifyAll(); + } + } + + void dispatchIncomingAfterPublication(JsonEnvelope envelope, Runnable localDispatch) { + BackendProxyHandler rollbackTarget; + synchronized (inboundPublication) { + while (!inboundPublished && !inboundAborted) { + try { + inboundPublication.wait(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return; + } + } + rollbackTarget = inboundPublished ? null : inboundRollbackTarget; + if (!inboundPublished && rollbackTarget == null) return; + } + if (rollbackTarget != null) { + GlobalMessageHandler rollbackHandler = rollbackTarget.globalMessageHandler; + if (rollbackHandler != null) rollbackHandler.onMessage(envelope); + return; + } + plugin.getBukkitScheduler().executeOrScheduleSync(plugin, localDispatch); + } + /** Returns whether replacement preparation must preserve accepted deliveries. */ public boolean requiresPreparationForReplacement() { return method == BungeeMethod.HTTP || method == BungeeMethod.PLUGINMESSAGING @@ -146,7 +203,7 @@ public boolean prepareForReplacement(BungeeMethod replacementMethod, long deadli return true; } if (!transportManager.hasPendingAsyncHandoff()) return false; - transportManager.awaitAsyncHandoff(deadlineNanos); + transportManager.prepareAsyncHandoffForReplacement(deadlineNanos); return true; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java index f84f1c6fd..ab4a7f6ae 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransport.java @@ -19,7 +19,8 @@ default void start(GlobalMessageHandler messageHandler, boolean retryInitializat start(messageHandler); } - void send(JsonEnvelope envelope); + /** Returns true only when the delivery was accepted by the transport or its durable queue. */ + boolean send(JsonEnvelope envelope); default void validate() { } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 69f994769..3088d2a94 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -34,6 +34,7 @@ public class BackendProxyTransportManager { private boolean rejectPreparedSends; private boolean preparedQueueWarning; private boolean rejectedSendWarning; + private boolean asyncHandoffRetryWarning; public BackendProxyTransportManager(VotingPluginMain plugin) { this(plugin, new ProcessedVoteCache()); @@ -173,7 +174,8 @@ public void prepareForReplacement() { // that live instance instead of creating a second directory owner. transport = candidate; preparedTransport = null; - while (!preparedSends.isEmpty()) transport.send(preparedSends.removeFirst()); + while (!preparedSends.isEmpty() && transport.send(preparedSends.peekFirst())) + preparedSends.removeFirst(); preparedQueueWarning = false; } else { // Enrollment cancellation may already have closed this instance. Restore @@ -345,8 +347,28 @@ private void drainAsyncHandoffMessages() { target = transport; } if (target == null) return; - target.send(envelope); + boolean accepted; + try { + accepted = target.send(envelope); + } catch (RuntimeException sendFailure) { + accepted = false; + } synchronized (this) { + if (!accepted) { + if (!asyncHandoffRetryWarning && plugin != null && plugin.getLogger() != null) { + asyncHandoffRetryWarning = true; + plugin.getLogger().warning( + "Backend proxy handoff delivery was rejected; retaining it for retry"); + } + try { + wait(250L); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return; + } + continue; + } + asyncHandoffRetryWarning = false; if (asyncHandoffSends.peekFirst() == envelope) asyncHandoffSends.removeFirst(); notifyAll(); } @@ -388,7 +410,10 @@ private void drainPluginMessageHandoff(long generation) { } if (!(target instanceof PluginMessagingBackendProxyTransport)) return; try { - target.send(envelope); + if (!target.send(envelope)) { + schedulePluginMessageHandoff(generation); + return; + } } catch (RuntimeException failure) { synchronized (this) { if (handoffGeneration == generation) pluginMessageHandoffScheduled = false; @@ -434,8 +459,7 @@ public void beginPreparedHttpHandoff() { public synchronized void restorePreparedTransport() { if (preparedTransport == null) { if (!preparedSendFence) return; - preparedSendFence = false; - while (!preparedSends.isEmpty()) send(preparedSends.removeFirst()); + acceptPreparedHandoffMessages(java.util.Collections.emptyList()); preparedQueueWarning = false; return; } @@ -447,7 +471,7 @@ public synchronized void restorePreparedTransport() { } preparedTransport = null; preparedSendFence = false; - while (!preparedSends.isEmpty()) transport.send(preparedSends.removeFirst()); + while (!preparedSends.isEmpty() && transport.send(preparedSends.peekFirst())) preparedSends.removeFirst(); preparedQueueWarning = false; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 8b35a21d9..6d0584e3d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -501,26 +501,38 @@ void awaitCredentialRestoration(long deadlineNanos) { } @Override - public void send(JsonEnvelope envelope) { + public boolean send(JsonEnvelope envelope) { synchronized (lifecycle) { - if (closed) return; + if (closed) return false; if (awaitingPreparedHandoff || !handoffQueue.isEmpty()) { int capacity = awaitingPreparedHandoff ? Math.min(MAX_PREPUBLICATION_QUEUE, MAX_HANDOFF_QUEUE - preparedHandoffReservation) : MAX_HANDOFF_QUEUE; - if (handoffQueue.size() < capacity) handoffQueue.addLast(envelope); - else warnRejectedSend(); - return; + if (handoffQueue.size() < capacity) { + handoffQueue.addLast(envelope); + return true; + } + warnRejectedSend(); + return false; } HttpBackendTransportConnector active = connector; if (active != null) { if (!active.send(envelope)) { - if (restartAfterFailedFlush && startupQueue.size() < MAX_STARTUP_QUEUE) startupQueue.addLast(envelope); - else warnRejectedSend(); + if (restartAfterFailedFlush && startupQueue.size() < MAX_STARTUP_QUEUE) { + startupQueue.addLast(envelope); + return true; + } + warnRejectedSend(); + return false; } } else if (startupQueue.size() < MAX_STARTUP_QUEUE) { startupQueue.addLast(envelope); + return true; + } else { + warnRejectedSend(); + return false; } + return true; } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java index 9a8879554..5250c002c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java @@ -46,14 +46,19 @@ public void validate() { } @Override - public void send(JsonEnvelope envelope) { + public boolean send(JsonEnvelope envelope) { if (mqttHandler == null) { - return; + return false; } try { mqttHandler.publishEnvelope(publishTopic, envelope); + return true; } catch (Exception e) { - e.printStackTrace(); + if (plugin != null && plugin.getLogger() != null) { + plugin.getLogger().warning("MQTT backend proxy delivery failed"); + plugin.debug(e); + } + return false; } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MysqlBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MysqlBackendProxyTransport.java index dc61e9f70..0cbf34be2 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MysqlBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MysqlBackendProxyTransport.java @@ -43,14 +43,19 @@ public void validate() { } @Override - public void send(JsonEnvelope envelope) { + public boolean send(JsonEnvelope envelope) { if (messenger == null) { - return; + return false; } try { messenger.sendToProxy(envelope); + return true; } catch (SQLException e) { - e.printStackTrace(); + if (plugin != null && plugin.getLogger() != null) { + plugin.getLogger().warning("MySQL backend proxy delivery failed"); + plugin.debug(e); + } + return false; } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java index 2688eccbb..5979e0baf 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java @@ -57,8 +57,9 @@ private void publishSharedState() { } @Override - public void send(JsonEnvelope envelope) { + public boolean send(JsonEnvelope envelope) { plugin.getPluginMessaging().sendEnvelope(envelope); + return true; } @Override diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index 0c13721d6..01feed8e0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -306,11 +306,13 @@ private record BufferedHandoffDelivery(long sequence, JsonEnvelope envelope, Str boolean identified) {} @Override - public void send(JsonEnvelope envelope) { + public boolean send(JsonEnvelope envelope) { if (redisHandler != null) { redisHandler.publishEnvelope(publishChannel, VotingPluginWire.withRedisDeliveryId(envelope)); + return true; } + return false; } @Override diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java index a660c8874..f24abad98 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java @@ -48,10 +48,12 @@ public void onReceiveEnvelope(JsonEnvelope envelope) { } @Override - public void send(JsonEnvelope envelope) { + public boolean send(JsonEnvelope envelope) { if (clientHandler != null) { clientHandler.sendEnvelope(envelope); + return true; } + return false; } @Override diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index d5b5adbcb..c354240cd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -593,11 +593,13 @@ private boolean prepareTimedHttpBroadcastDelivery(String server, VoteTimeQueue v protected boolean sendHttpBroadcastEnvelopeWithRecovery(String server, JsonEnvelope envelope, OfflineBungeeVote cachedVote) { - String stableId = cachedVote == null ? null : cachedVote.getHttpBroadcastDeliveryId(server); + String persistedId = cachedVote == null ? null : cachedVote.getHttpBroadcastDeliveryId(server); + String stableId = persistedId != null ? persistedId + : stableCachedHttpDeliveryId("broadcast", server, envelope, cachedVote); try { boolean accepted = stableId == null ? sendProxyBroadcastEnvelopeNow(server, envelope) : sendHttpEnvelope(server, stableId, envelope); - if (accepted && cachedVote != null && stableId != null) { + if (accepted && persistedId != null) { cachedVote.setHttpBroadcastDeliveryId(server, null); cachedVote.setDeliveryStateDirty(true); } @@ -2878,13 +2880,13 @@ private void reloadRuntime(boolean restartControlServices) { private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod configuredMethod) { if (configuredMethod == BungeeMethod.HTTP) return configuredMethod; HttpProxyTransportServer transport = httpTransportServer; - if (transport != null && transport.hasPendingDeliveries()) { + if (transport != null && httpTransportHasPendingDeliveries(transport)) { logSevere("Retaining HTTP transport until durable deliveries are acknowledged"); return BungeeMethod.HTTP; } if (transport == null) { try { - if (HttpProxyTransportServer.hasPersistedDeliveries( + if (httpQueueHasPersistedDeliveries( getDataFolderPlugin().toPath().resolve("http").resolve("outgoing-v1"))) { logSevere("Retaining HTTP transport until persisted deliveries are acknowledged"); return BungeeMethod.HTTP; @@ -2913,6 +2915,58 @@ private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod co return configuredMethod; } + /** Uses SimpleAPI #80 when deployed while remaining safe with an older published snapshot. */ + protected boolean httpTransportHasPendingDeliveries(HttpProxyTransportServer transport) { + try { + return Boolean.TRUE.equals(transport.getClass().getMethod("hasPendingDeliveries").invoke(transport)); + } catch (NoSuchMethodException unavailable) { + // Without an authoritative query, the live durable server cannot be proven empty. + return true; + } catch (ReflectiveOperationException | SecurityException failure) { + logSevere("Retaining HTTP transport because its live delivery queue could not be inspected"); + return true; + } + } + + /** Invokes the additive SimpleAPI disk probe, with an equivalent bounded compatibility fallback. */ + protected boolean httpQueueHasPersistedDeliveries(java.nio.file.Path outgoingDirectory) throws IOException { + try { + Object result = HttpProxyTransportServer.class.getMethod("hasPersistedDeliveries", java.nio.file.Path.class) + .invoke(null, outgoingDirectory); + return Boolean.TRUE.equals(result); + } catch (NoSuchMethodException unavailable) { + return inspectPersistedHttpDeliveries(outgoingDirectory); + } catch (java.lang.reflect.InvocationTargetException failure) { + Throwable cause = failure.getCause(); + if (cause instanceof IOException io) throw io; + throw new IOException("HTTP outgoing queue inspection failed", cause); + } catch (ReflectiveOperationException | SecurityException failure) { + throw new IOException("HTTP outgoing queue inspection failed", failure); + } + } + + private boolean inspectPersistedHttpDeliveries(java.nio.file.Path outgoingDirectory) throws IOException { + java.nio.file.Path root = outgoingDirectory.toAbsolutePath().normalize(); + if (!java.nio.file.Files.exists(root, java.nio.file.LinkOption.NOFOLLOW_LINKS)) return false; + if (java.nio.file.Files.isSymbolicLink(root) + || !java.nio.file.Files.isDirectory(root, java.nio.file.LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue directory is invalid"); + int backendCount = 0; + try (java.nio.file.DirectoryStream backends = java.nio.file.Files.newDirectoryStream(root)) { + for (java.nio.file.Path backend : backends) { + if (java.nio.file.Files.isSymbolicLink(backend) + || !java.nio.file.Files.isDirectory(backend, java.nio.file.LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue contains an invalid entry"); + if (++backendCount > 128) throw new IOException("HTTP outgoing queue exceeds its backend bound"); + try (java.nio.file.DirectoryStream entries = + java.nio.file.Files.newDirectoryStream(backend)) { + if (entries.iterator().hasNext()) return true; + } + } + } + return false; + } + private boolean hasPendingCachedHttpDeliveries() { VoteCacheHandler cache = getVoteCacheHandler(); if (cache == null) return false; @@ -3023,11 +3077,13 @@ protected boolean sendStableHttpEnvelope(String server, String deliveryId, JsonE } protected boolean sendHttpEnvelopeWithRecovery(String server, JsonEnvelope envelope, OfflineBungeeVote cachedVote) { - String stableId = cachedVote == null ? null : cachedVote.getHttpDeliveryId(server); + String persistedId = cachedVote == null ? null : cachedVote.getHttpDeliveryId(server); + String stableId = persistedId != null ? persistedId + : stableCachedHttpDeliveryId("reward", server, envelope, cachedVote); try { boolean accepted = stableId == null ? sendHttpEnvelope(server, envelope) : sendHttpEnvelope(server, stableId, envelope); - if (accepted && cachedVote != null && stableId != null) { + if (accepted && persistedId != null) { cachedVote.setHttpDeliveryId(server, null); cachedVote.setDeliveryStateDirty(true); } @@ -3049,6 +3105,18 @@ protected boolean sendHttpEnvelopeWithRecovery(String server, JsonEnvelope envel } } + /** Derives the same transport identity after a crash before cache cleanup. */ + private String stableCachedHttpDeliveryId(String purpose, String server, JsonEnvelope envelope, + OfflineBungeeVote cachedVote) { + if (cachedVote == null || server == null || envelope == null) return null; + String voteIdentity = cachedVote.getVoteId() == null + ? cachedVote.getUuid() + "\u0000" + cachedVote.getService() + "\u0000" + cachedVote.getTime() + : cachedVote.getVoteId().toString(); + String key = "VotingPlugin:http-cache:v1\u0000" + purpose + "\u0000" + + server.toLowerCase(Locale.ROOT) + "\u0000" + envelope.getSubChannel() + "\u0000" + voteIdentity; + return UUID.nameUUIDFromBytes(key.getBytes(StandardCharsets.UTF_8)).toString(); + } + protected synchronized boolean sendHttpEnvelope(String server, String deliveryId, JsonEnvelope envelope) { HttpProxyTransportServer transport = httpTransportServer; return transport != null && transport.send(server, deliveryId, envelope); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index 30b2350ff..ca21524d1 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -52,6 +52,7 @@ void restoresPreviousHandlerWhenActivationFails() { () -> plugin.publishBackendProxyHandler(previous, replacement)); assertSame(previous, plugin.getBackendProxyHandler()); + verify(replacement).abortStagedInboundTo(previous); } @Test @@ -88,6 +89,7 @@ void restoresPreviousHandlerWhenRedisPromotionFails() throws Exception { verify(previous, never()).completeHttpHandoff(replacement); verify(previous, never()).close(); org.mockito.InOrder rollback = org.mockito.Mockito.inOrder(replacement, previous); + rollback.verify(replacement).abortStagedInboundTo(previous); rollback.verify(replacement).close(); rollback.verify(previous).refreshPresenceAfterFailedReplacement(); } @@ -130,7 +132,10 @@ void predecessorCleanupFailureDoesNotRollBackPublishedReplacement() throws Excep assertSame(replacement, plugin.getBackendProxyHandler()); assertFalse(plugin.requestBackendProxyHandlerRestartAbandonment(restart), "published runtime state must not be treated as rollbackable"); - verify(previous).completeHttpHandoff(replacement); + org.mockito.InOrder publication = org.mockito.Mockito.inOrder(previous, replacement); + publication.verify(previous).completeHttpHandoff(replacement); + publication.verify(replacement).activateInboundMessages(); + publication.verify(previous).close(); } @Test diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 4d5d97732..0730aef42 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -21,11 +21,16 @@ import static org.mockito.ArgumentMatchers.eq; import java.lang.reflect.Field; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; +import com.bencodez.simpleapi.scheduler.BukkitScheduler; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; import com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessageHandler; import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; @@ -47,6 +52,60 @@ import com.bencodez.votingplugin.proxy.BungeeMethod; class BackendProxyHandlerLifecycleTest { + @Test + void stagedInboundWaitsForPublicationAndUsesTheBukkitScheduler() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + BackendProxyHandler handler = new BackendProxyHandler(plugin); + JsonEnvelope envelope = JsonEnvelope.builder("staged").build(); + Runnable localDispatch = mock(Runnable.class); + AtomicReference scheduled = new AtomicReference<>(); + doAnswer(invocation -> { + scheduled.set(invocation.getArgument(1)); + return null; + }).when(scheduler).executeOrScheduleSync(eq(plugin), any(Runnable.class)); + CountDownLatch started = new CountDownLatch(1); + + CompletableFuture callback = CompletableFuture.runAsync(() -> { + started.countDown(); + handler.dispatchIncomingAfterPublication(envelope, localDispatch); + }); + assertTrue(started.await(1, TimeUnit.SECONDS)); + assertFalse(callback.isDone()); + verifyNoInteractions(scheduler); + + handler.activateInboundMessages(); + callback.get(1, TimeUnit.SECONDS); + verify(scheduler).executeOrScheduleSync(eq(plugin), any(Runnable.class)); + verify(localDispatch, never()).run(); + scheduled.get().run(); + verify(localDispatch).run(); + } + + @Test + void stagedInboundRoutesThroughRestoredPredecessorOnRollback() throws Exception { + BackendProxyHandler previous = new BackendProxyHandler(null); + BackendProxyHandler replacement = new BackendProxyHandler(null); + GlobalMessageHandler previousMessages = mock(GlobalMessageHandler.class); + setField(previous, "globalMessageHandler", previousMessages); + JsonEnvelope envelope = JsonEnvelope.builder("rollback").build(); + Runnable replacementDispatch = mock(Runnable.class); + CountDownLatch started = new CountDownLatch(1); + + CompletableFuture callback = CompletableFuture.runAsync(() -> { + started.countDown(); + replacement.dispatchIncomingAfterPublication(envelope, replacementDispatch); + }); + assertTrue(started.await(1, TimeUnit.SECONDS)); + assertFalse(callback.isDone()); + + replacement.abortStagedInboundTo(previous); + callback.get(1, TimeUnit.SECONDS); + verify(previousMessages).onMessage(envelope); + verify(replacementDispatch, never()).run(); + } + @Test void failedPluginMessagePublicationRestoresPreviousSharedState() { com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); @@ -379,11 +438,54 @@ void mqttSendKeepsTheTopicCapturedByTheActiveTransport() throws Exception { setField(transport, "mqttHandler", mqtt); setField(transport, "publishTopic", "old/votingplugin/servers/proxy"); - transport.send(com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("vote").build()); + assertTrue(transport.send(com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("vote").build())); verify(mqtt).publishEnvelope(eq("old/votingplugin/servers/proxy"), any()); } + @Test + void mqttAndMysqlReportRejectedHandoffDeliveries() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + MqttBackendProxyTransport mqttTransport = new MqttBackendProxyTransport(plugin); + MqttHandler mqtt = mock(MqttHandler.class); + setField(mqttTransport, "mqttHandler", mqtt); + setField(mqttTransport, "publishTopic", "votingplugin/servers/proxy"); + doThrow(new IllegalStateException("publish failed")).when(mqtt).publishEnvelope(any(), any()); + JsonEnvelope mqttEnvelope = JsonEnvelope.builder("mqtt").build(); + + assertFalse(mqttTransport.send(mqttEnvelope)); + + MysqlBackendProxyTransport mysqlTransport = new MysqlBackendProxyTransport(plugin); + MySqlMessenger mysql = mock(MySqlMessenger.class); + setField(mysqlTransport, "messenger", mysql); + doThrow(new java.sql.SQLException("write failed")).when(mysql).sendToProxy(any()); + + assertFalse(mysqlTransport.send(JsonEnvelope.builder("mysql").build())); + } + + @Test + void asyncHandoffRetainsARejectedEntryUntilAcceptance() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + BackendProxyTransportManager source = new BackendProxyTransportManager(plugin); + BackendProxyTransportManager replacement = new BackendProxyTransportManager(plugin); + BackendProxyTransport sourceTransport = mock(BackendProxyTransport.class); + BackendProxyTransport replacementTransport = mock(BackendProxyTransport.class); + JsonEnvelope envelope = JsonEnvelope.builder("retry").build(); + when(replacementTransport.send(envelope)).thenReturn(false, true); + setField(source, "transport", sourceTransport); + setField(replacement, "transport", replacementTransport); + + replacement.beginPreparedTransportHandoff(); + source.prepareForReplacement(); + source.send(envelope); + source.completePreparedTransportHandoff(replacement); + + verify(replacementTransport, org.mockito.Mockito.timeout(1500).times(2)).send(envelope); + replacement.awaitAsyncHandoff(System.nanoTime() + TimeUnit.SECONDS.toNanos(1)); + assertFalse(replacement.hasPendingAsyncHandoff()); + } + @Test void forwardsMessagesBufferedDuringPreparedHttpReplacement() throws Exception { com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); @@ -391,6 +493,7 @@ void forwardsMessagesBufferedDuringPreparedHttpReplacement() throws Exception { BackendProxyTransportManager replacement = new BackendProxyTransportManager(plugin); BackendProxyTransport previousTransport = mock(BackendProxyTransport.class); BackendProxyTransport replacementTransport = mock(BackendProxyTransport.class); + when(replacementTransport.send(any())).thenReturn(true); setField(previous, "transport", previousTransport); setField(replacement, "transport", replacementTransport); com.bencodez.simpleapi.servercomm.codec.JsonEnvelope duringValidation = @@ -429,7 +532,7 @@ void crossTransportPreparedHandoffDoesNotPerformNetworkSendOnCallerThread() thro doAnswer(invocation -> { sendStarted.countDown(); releaseSend.await(2, java.util.concurrent.TimeUnit.SECONDS); - return null; + return true; }).when(replacementTransport).send(any()); replacement.beginPreparedTransportHandoff(); @@ -474,7 +577,7 @@ void laterReplacementWaitsForAnEarlierAsyncHandoff() throws Exception { doAnswer(invocation -> { sendStarted.countDown(); releaseSend.await(2, java.util.concurrent.TimeUnit.SECONDS); - return null; + return true; }).when(targetTransport).send(any()); targetManager.beginPreparedTransportHandoff(); @@ -494,6 +597,16 @@ void laterReplacementWaitsForAnEarlierAsyncHandoff() throws Exception { } assertTrue(preparation.get(1, java.util.concurrent.TimeUnit.SECONDS)); assertFalse(targetManager.hasPendingAsyncHandoff()); + com.bencodez.simpleapi.servercomm.codec.JsonEnvelope afterPreparation = + com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("after-preparation").build(); + targetManager.send(afterPreparation); + BackendProxyTransportManager next = new BackendProxyTransportManager(plugin); + BackendProxyTransport nextTransport = mock(BackendProxyTransport.class); + when(nextTransport.send(any())).thenReturn(true); + setField(next, "transport", nextTransport); + next.beginPreparedTransportHandoff(); + targetManager.completePreparedTransportHandoff(next); + verify(nextTransport, org.mockito.Mockito.timeout(1000)).send(afterPreparation); } @Test @@ -511,6 +624,7 @@ void pluginMessageHandoffRunsThroughTheBukkitScheduler() throws Exception { BackendProxyTransportManager target = new BackendProxyTransportManager(plugin); BackendProxyTransport sourceTransport = mock(BackendProxyTransport.class); PluginMessagingBackendProxyTransport targetTransport = mock(PluginMessagingBackendProxyTransport.class); + when(targetTransport.send(any())).thenReturn(true); setField(source, "transport", sourceTransport); setField(target, "transport", targetTransport); @@ -532,6 +646,7 @@ void pluginMessageHandoffRunsThroughTheBukkitScheduler() throws Exception { target.send(afterPreparation); BackendProxyTransportManager next = new BackendProxyTransportManager(plugin); BackendProxyTransport nextTransport = mock(BackendProxyTransport.class); + when(nextTransport.send(any())).thenReturn(true); setField(next, "transport", nextTransport); next.beginPreparedTransportHandoff(); target.completePreparedTransportHandoff(next); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 8c4d1555f..aa7a0ab7f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -182,6 +182,7 @@ void sendsRemainNonBlockingWhileReplacementFlushes() throws Exception { BackendProxyTransport active = mock(BackendProxyTransport.class); BackendProxyTransportManager replacement = new BackendProxyTransportManager(mock(VotingPluginMain.class)); BackendProxyTransport replacementTransport = mock(BackendProxyTransport.class); + when(replacementTransport.send(org.mockito.ArgumentMatchers.any(JsonEnvelope.class))).thenReturn(true); JsonEnvelope queued = JsonEnvelope.builder("queued-during-flush").build(); setField(manager, "transport", active); setField(replacement, "transport", replacementTransport); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 7b6d4ef7b..0bafd6ad5 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -104,59 +104,52 @@ void rejectedRewardBearingAuxiliarySendRetainsItsStableDeliveryIdForRetry() { } @Test - void indeterminateCachedHttpVotePersistsRecoveredIdBeforeRetry() { + void cachedHttpVoteReusesItsDeterministicIdAfterRestart() { votingPluginProxy.setMethod(BungeeMethod.HTTP); votingPluginProxy.setVoteEnvelopeDeliveryResult(true); - String deliveryId = "00000000-0000-0000-0000-000000000160"; - votingPluginProxy.failNextGeneratedHttpSend(deliveryId); OfflineBungeeVote pending = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "uuid", "Service", 100L, false, "totals"); - assertFalse(votingPluginProxy.sendHttpVoteEnvelopeWithRecoveryForTest("Server1", + assertTrue(votingPluginProxy.sendHttpVoteEnvelopeWithRecoveryForTest("Server1", JsonEnvelope.builder("vote").build(), pending)); - assertEquals(deliveryId, pending.getHttpDeliveryId("server1")); OfflineBungeeVote restored = new OfflineBungeeVote(pending.getVoteId(), pending.getPlayerName(), pending.getUuid(), pending.getService(), pending.getTime(), pending.isRealVote(), pending.getText(), pending.isBroadcastForwarded(), pending.isProxyBroadcastHandled(), pending.getBroadcastTargets(), pending.getBroadcastForwardedServers(), pending.isRewardDelivered(), OfflineBungeeVote.decodeHttpDeliveryIds(pending.encodeHttpDeliveryIds())); - assertEquals(deliveryId, restored.getHttpDeliveryId("SERVER1")); assertTrue(votingPluginProxy.sendHttpVoteEnvelopeWithRecoveryForTest("Server1", JsonEnvelope.builder("vote").build(), restored)); - assertEquals(null, restored.getHttpDeliveryId("Server1")); - assertEquals(java.util.List.of(deliveryId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); + assertEquals(2, votingPluginProxy.getAttemptedVotePartyDeliveryIds().size()); + assertEquals(votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(0), + votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(1)); } @Test void standaloneHttpBroadcastUsesSeparateStableIdFromRewardDelivery() { votingPluginProxy.setMethod(BungeeMethod.HTTP); votingPluginProxy.setVoteEnvelopeDeliveryResult(true); - String broadcastId = "00000000-0000-0000-0000-000000000161"; + String rewardId = "00000000-0000-0000-0000-000000000162"; OfflineBungeeVote pending = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "uuid", "Service", 100L, true, "totals"); - pending.setHttpDeliveryId("Server1", "00000000-0000-0000-0000-000000000162"); - votingPluginProxy.failNextGeneratedHttpSend(broadcastId); + pending.setHttpDeliveryId("Server1", rewardId); - assertFalse(votingPluginProxy.sendHttpBroadcastEnvelopeWithRecoveryForTest("Server1", + assertTrue(votingPluginProxy.sendHttpBroadcastEnvelopeWithRecoveryForTest("Server1", JsonEnvelope.builder("vote").build(), pending)); - assertTrue(votingPluginProxy.getAttemptedVotePartyDeliveryIds().isEmpty()); - assertEquals("00000000-0000-0000-0000-000000000162", pending.getHttpDeliveryId("Server1")); - assertEquals(broadcastId, pending.getHttpBroadcastDeliveryId("Server1")); + assertEquals(rewardId, pending.getHttpDeliveryId("Server1")); + assertEquals(1, votingPluginProxy.getAttemptedVotePartyDeliveryIds().size()); + assertFalse(rewardId.equals(votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(0))); } @Test - void indeterminateCachedStandaloneBroadcastPersistsRecoveredIdBeforeRetry() { + void cachedStandaloneBroadcastReusesItsDeterministicIdAfterRestart() { votingPluginProxy.setMethod(BungeeMethod.HTTP); votingPluginProxy.setVoteEnvelopeDeliveryResult(true); - String deliveryId = "00000000-0000-0000-0000-000000000163"; - votingPluginProxy.failNextGeneratedHttpSend(deliveryId); OfflineBungeeVote pending = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "uuid", "Service", 100L, true, "totals"); - assertFalse(votingPluginProxy.sendHttpBroadcastEnvelopeWithRecoveryForTest("Server1", + assertTrue(votingPluginProxy.sendHttpBroadcastEnvelopeWithRecoveryForTest("Server1", JsonEnvelope.builder("broadcast").build(), pending)); - assertEquals(deliveryId, pending.getHttpBroadcastDeliveryId("server1")); OfflineBungeeVote restored = new OfflineBungeeVote(pending.getVoteId(), pending.getPlayerName(), pending.getUuid(), pending.getService(), pending.getTime(), pending.isRealVote(), pending.getText(), false, true, java.util.Set.of("Server1"), java.util.Set.of(), pending.isRewardDelivered(), @@ -164,8 +157,9 @@ void indeterminateCachedStandaloneBroadcastPersistsRecoveredIdBeforeRetry() { OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(pending.encodeHttpBroadcastDeliveryIds())); assertTrue(votingPluginProxy.sendHttpBroadcastEnvelopeWithRecoveryForTest("Server1", JsonEnvelope.builder("broadcast").build(), restored)); - assertEquals(null, restored.getHttpBroadcastDeliveryId("Server1")); - assertEquals(java.util.List.of(deliveryId), votingPluginProxy.getAttemptedVotePartyDeliveryIds()); + assertEquals(2, votingPluginProxy.getAttemptedVotePartyDeliveryIds().size()); + assertEquals(votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(0), + votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(1)); } @Test @@ -240,7 +234,9 @@ void queuedRolloverReplayReusesStableStandaloneBroadcastDeliveryId() { spyProxy.processQueue(); assertTrue(queue.isEmpty()); - assertEquals(java.util.List.of(deliveryId), spyProxy.getAttemptedVotePartyDeliveryIds()); + assertEquals(deliveryId, spyProxy.getAttemptedVotePartyDeliveryIds().get(0)); + assertEquals(2, spyProxy.getAttemptedVotePartyDeliveryIds().size()); + assertFalse(deliveryId.equals(spyProxy.getAttemptedVotePartyDeliveryIds().get(1))); verify(voteCache).updateTimeVote(queued); verify(voteCache).removeTimeVote(queued); verify(voteCache, never()).addOnlineVote(Mockito.anyString(), Mockito.any(OfflineBungeeVote.class)); @@ -529,8 +525,8 @@ void transportReloadRetainsHttpForPendingTimedBroadcast() { @Test void transportReloadRetainsHttpForOrdinaryDurableQueueEntry() throws Exception { HttpProxyTransportServer transport = Mockito.mock(HttpProxyTransportServer.class); - Mockito.when(transport.hasPendingDeliveries()).thenReturn(true); setProxyField(votingPluginProxy, "httpTransportServer", transport); + votingPluginProxy.setPendingHttpTransportDeliveries(true); votingPluginProxy.setMethod(BungeeMethod.HTTP); Mockito.when(votingPluginProxy.getConfig().getBungeeMethod()).thenReturn("MQTT"); @@ -1234,7 +1230,7 @@ void acceptedCachedRewardRetainsAmbiguousStandaloneBroadcastWithoutRebroadcastin Mockito.when(votingPluginProxy.getConfig().getBlockedServers()).thenReturn(java.util.Collections.emptyList()); votingPluginProxy.setMethod(BungeeMethod.HTTP); votingPluginProxy.setVoteEnvelopeDeliveryResult(true); - votingPluginProxy.setStableHttpDeliveryResult(false); + votingPluginProxy.setStableHttpDeliveryResults(false, true); VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); @@ -1304,7 +1300,7 @@ void acceptedOnlineRewardRetainsAmbiguousStandaloneBroadcastWithoutRebroadcastin Mockito.when(votingPluginProxy.getConfig().getBlockedServers()).thenReturn(java.util.Collections.emptyList()); votingPluginProxy.setMethod(BungeeMethod.HTTP); votingPluginProxy.setVoteEnvelopeDeliveryResult(true); - votingPluginProxy.setStableHttpDeliveryResult(false); + votingPluginProxy.setStableHttpDeliveryResults(false, true); VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); @@ -1367,7 +1363,7 @@ void periodicRetryDeliversBrokerBackedOnlineCacheWithoutVoterLogin() { } @Test - void failedPeriodicHttpBroadcastPersistsGeneratedStableId() { + void rejectedPeriodicHttpBroadcastKeepsItsDeterministicIdentity() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); OfflineBungeeVote vote = new OfflineBungeeVote(java.util.UUID.randomUUID(), "OfflineVoter", "voter-uuid", "Service", 100L, true, "totals", false, true, java.util.Set.of("Server1"), @@ -1379,19 +1375,17 @@ void failedPeriodicHttpBroadcastPersistsGeneratedStableId() { Mockito.when(votingPluginProxy.getConfig().getBlockedServers()).thenReturn(java.util.Collections.emptyList()); votingPluginProxy.setMethod(BungeeMethod.HTTP); votingPluginProxy.setVoteEnvelopeDeliveryResult(false); - String deliveryId = "00000000-0000-0000-0000-000000000167"; - votingPluginProxy.failNextGeneratedHttpSend(deliveryId); VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); spyProxy.retryPendingOnlineBroadcasts(); - assertEquals(deliveryId, vote.getHttpBroadcastDeliveryId("Server1")); - verify(voteCache).updateOnlineVote("voter-uuid", vote); + assertEquals(null, vote.getHttpBroadcastDeliveryId("Server1")); + verify(voteCache, never()).updateOnlineVote("voter-uuid", vote); } @Test - void failedCarrierHttpBroadcastPersistsGeneratedStableId() { + void rejectedCarrierHttpBroadcastKeepsItsDeterministicIdentity() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); OfflineBungeeVote vote = new OfflineBungeeVote(java.util.UUID.randomUUID(), "OfflineVoter", "voter-uuid", "Service", 100L, true, "totals", false, true, java.util.Set.of("Server1"), @@ -1403,15 +1397,13 @@ void failedCarrierHttpBroadcastPersistsGeneratedStableId() { Mockito.when(votingPluginProxy.getConfig().getBlockedServers()).thenReturn(java.util.Collections.emptyList()); votingPluginProxy.setMethod(BungeeMethod.HTTP); votingPluginProxy.setVoteEnvelopeDeliveryResult(false); - String deliveryId = "00000000-0000-0000-0000-000000000168"; - votingPluginProxy.failNextGeneratedHttpSend(deliveryId); VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); spyProxy.retryPendingOnlineBroadcastsForTest("Server1"); - assertEquals(deliveryId, vote.getHttpBroadcastDeliveryId("Server1")); - verify(voteCache).updateOnlineVote("voter-uuid", vote); + assertEquals(null, vote.getHttpBroadcastDeliveryId("Server1")); + verify(voteCache, never()).updateOnlineVote("voter-uuid", vote); } @Test diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index af1b449e9..c0dede3c2 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -29,6 +29,7 @@ public class VotingPluginProxyTestImpl extends VotingPluginProxy { private boolean pluginMessageDeliveryResult = true; private boolean voteEnvelopeDeliveryResult = true; private Boolean stableHttpDeliveryResult; + private final java.util.ArrayDeque stableHttpDeliveryResults = new java.util.ArrayDeque<>(); private JsonEnvelope lastVoteEnvelope; private boolean communicationTestDeliveryResult = true; private JsonEnvelope lastCommunicationTestEnvelope; @@ -49,11 +50,22 @@ public class VotingPluginProxyTestImpl extends VotingPluginProxy { private java.util.concurrent.CompletableFuture nextVotePartyCommandCompletion; private boolean declineNextVotePartyCommand; private Runnable votePartyProxyCommandTimeout; + private Boolean pendingHttpTransportDeliveries; public List getWarnings() { return warnings; } + public void setPendingHttpTransportDeliveries(Boolean pending) { + pendingHttpTransportDeliveries = pending; + } + + @Override + protected boolean httpTransportHasPendingDeliveries(HttpProxyTransportServer transport) { + return pendingHttpTransportDeliveries != null ? pendingHttpTransportDeliveries + : super.httpTransportHasPendingDeliveries(transport); + } + @Override public void addNonVotedPlayer(String uuid, String playerName) { // Implementation for testing purposes @@ -365,7 +377,9 @@ protected boolean sendHttpEnvelope(String server, JsonEnvelope envelope) { @Override protected boolean sendHttpEnvelope(String server, String deliveryId, JsonEnvelope envelope) { + lastVoteEnvelope = envelope; attemptedVotePartyDeliveryIds.add(deliveryId); + if (!stableHttpDeliveryResults.isEmpty()) return stableHttpDeliveryResults.removeFirst(); return stableHttpDeliveryResult == null ? voteEnvelopeDeliveryResult : stableHttpDeliveryResult; } @@ -373,6 +387,11 @@ public void setStableHttpDeliveryResult(Boolean stableHttpDeliveryResult) { this.stableHttpDeliveryResult = stableHttpDeliveryResult; } + public void setStableHttpDeliveryResults(Boolean... results) { + stableHttpDeliveryResults.clear(); + java.util.Collections.addAll(stableHttpDeliveryResults, results); + } + public JsonEnvelope getLastVoteEnvelope() { return lastVoteEnvelope; } From ba2c091f693b2fcf2e1534f4569e6daa2c4b4581 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:06:50 -0600 Subject: [PATCH 059/136] Replay Redis handoff after inbound activation --- .../java/com/bencodez/votingplugin/VotingPluginMain.java | 1 + .../votingplugin/backendproxy/BackendProxyHandler.java | 5 +++++ .../transport/BackendProxyTransportManager.java | 4 ++++ .../transport/RedisBackendProxyTransport.java | 9 ++++++--- .../VotingPluginMainBackendProxyPublicationTest.java | 1 + .../transport/RedisBackendProxyTransportTest.java | 4 ++++ 6 files changed, 21 insertions(+), 3 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 27f986a93..1a6247b2c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1387,6 +1387,7 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { throw handoffFailure; } restart.replacement.activateInboundMessages(); + restart.replacement.replayRedisAfterHandoffPublication(); restart.finished = true; restart.published = true; if (restart.previous != null) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 51d09cd46..9a4fd35aa 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -257,6 +257,11 @@ public void completeRedisHandoff(BackendProxyHandler replacement) { transportManager.completeRedisHandoff(replacement.transportManager); } + /** Replays Redis handoff deliveries only after inbound publication is open. */ + public void replayRedisAfterHandoffPublication() { + transportManager.replayRedisAfterHandoffPublication(); + } + /** Forwards messages buffered while the previous transport was fenced. */ public void completeHttpHandoff(BackendProxyHandler replacement) { if (replacement == null) return; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 3088d2a94..d33e41aeb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -512,6 +512,10 @@ public void activateRedisAfterHandoff() { ((RedisBackendProxyTransport) transport).activateAfterHandoff(); } + public void replayRedisAfterHandoffPublication() { + if (transport instanceof RedisBackendProxyTransport redis) redis.replayAfterHandoffPublication(); + } + /** Fences the old Redis listener before promoting the validated standby. */ public void completeRedisHandoff(BackendProxyTransportManager replacement) { java.util.Objects.requireNonNull(replacement, "replacement"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index 01feed8e0..3d11756f4 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -181,7 +181,6 @@ private void dispatchTracked(JsonEnvelope envelope) { /** Promotes a validated standby after the previous listener has completely stopped. */ public void activateAfterHandoff() { - java.util.ArrayList replay = new java.util.ArrayList<>(); synchronized (legacyLifecycle) { if (identifiedHandoffOverflowed || legacyHandoffOverflowed || processedVoteCache.isLegacyRedisHandoffOverflowed()) @@ -199,7 +198,7 @@ public void activateAfterHandoff() { boolean dispatch = delivery.identified() ? processedVoteCache.reserveRedisDelivery(delivery.identity()) : !processedVoteCache.consumeLegacyRedisDelivery(delivery.identity()); - if (dispatch) replay.add(delivery.envelope()); + if (dispatch) deliveriesAfterReplay.addLast(delivery.envelope()); } catch (RuntimeException replayFailure) { if (plugin != null) plugin.debug("Redis handoff replay failed: " + replayFailure.getMessage()); } @@ -213,7 +212,11 @@ public void activateAfterHandoff() { legacyHandoffDegraded = false; processedVoteCache.finishRedisHandoff(); } - dispatchReplayBatch(replay); + } + + /** Replays buffered deliveries after the owning handler opens its publication gate. */ + public void replayAfterHandoffPublication() { + java.util.ArrayList replay; while (true) { synchronized (legacyLifecycle) { if (deliveriesAfterReplay.isEmpty()) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index ca21524d1..e4feb74a2 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -135,6 +135,7 @@ void predecessorCleanupFailureDoesNotRollBackPublishedReplacement() throws Excep org.mockito.InOrder publication = org.mockito.Mockito.inOrder(previous, replacement); publication.verify(previous).completeHttpHandoff(replacement); publication.verify(replacement).activateInboundMessages(); + publication.verify(replacement).replayRedisAfterHandoffPublication(); publication.verify(previous).close(); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java index aef9eb9e5..bcc21e138 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java @@ -127,6 +127,8 @@ void standbyBuffersIdentifiedDeliveriesUntilItIsPublished() throws Exception { verifyNoInteractions(messages); transport.activateAfterHandoff(); + verifyNoInteractions(messages); + transport.replayAfterHandoffPublication(); verify(messages).onMessage(envelope); assertFalse(cache.reserveRedisDelivery(deliveryId), "publication must reserve the delivery exactly when its buffered callback is replayed"); @@ -149,6 +151,7 @@ void standbyReplaysMixedLegacyAndIdentifiedDeliveriesInArrivalOrder() throws Exc transport.dispatchLegacy(legacy); transport.dispatchIdentified(identified, deliveryId); transport.activateAfterHandoff(); + transport.replayAfterHandoffPublication(); org.mockito.InOrder order = org.mockito.Mockito.inOrder(messages); order.verify(messages).onMessage(legacy); @@ -335,6 +338,7 @@ void replayCompletesBeforeNewlyAdmittedDeliveries() throws Exception { transport.dispatchLegacy(replayed); transport.activateAfterHandoff(); + transport.replayAfterHandoffPublication(); assertEquals(java.util.List.of(replayed, newer), order); } From f74a228a2e194f433024974c8021d4be1dfb9c2b Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:32:47 -0600 Subject: [PATCH 060/136] Avoid Redis handoff callback lock inversion --- .../BackendProxyTransportManager.java | 31 +++++--- .../RedisBackendProxyTransportTest.java | 75 +++++++++++++++++++ 2 files changed, 97 insertions(+), 9 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index d33e41aeb..788ca140e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -486,23 +486,36 @@ public void awaitPreparedTransportRestoration(long deadlineNanos) { if (transport instanceof HttpBackendProxyTransport http) http.awaitCredentialRestoration(deadlineNanos); } - public synchronized void closeRedisForHandoff() { - if (!(transport instanceof RedisBackendProxyTransport)) { - throw new IllegalStateException("Redis backend proxy transport is unavailable"); + public void closeRedisForHandoff() { + RedisBackendProxyTransport retiring; + synchronized (this) { + if (!(transport instanceof RedisBackendProxyTransport)) { + throw new IllegalStateException("Redis backend proxy transport is unavailable"); + } + retiring = (RedisBackendProxyTransport) transport; + retiredTransport = retiring; } - retiredTransport = transport; - transport = null; try { - ((RedisBackendProxyTransport) retiredTransport).closeForHandoff(); + // closeForHandoff() waits for already-running Redis callbacks. Those callbacks + // may publish a reply through send(), so never retain this manager's monitor + // while waiting for them to drain. + retiring.closeForHandoff(); } catch (RuntimeException failure) { - if (failure instanceof RedisBackendProxyTransport.HandoffQuiescenceException) { - transport = retiredTransport; - retiredTransport = null; + synchronized (this) { + if (failure instanceof RedisBackendProxyTransport.HandoffQuiescenceException + && retiredTransport == retiring && transport == retiring) { + retiredTransport = null; + } else if (transport == retiring) { + transport = null; + } } // Retain the fenced old listener so a later manager close can retry its // cleanup without ever touching the promoted replacement. throw failure; } + synchronized (this) { + if (transport == retiring) transport = null; + } } public void activateRedisAfterHandoff() { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java index bcc21e138..d08f9eb1e 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java @@ -2,14 +2,17 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.spy; import java.lang.reflect.Field; import java.util.concurrent.CountDownLatch; @@ -269,6 +272,62 @@ void fencingWaitsForAnIdentifiedMessageHandler() throws Exception { assertTrue(fenceFailure.get() instanceof RedisBackendProxyTransport.HandoffQuiescenceException); } + @Test + void managerDoesNotHoldItsMonitorWhileWaitingForRedisCallbackReplies() throws Exception { + BackendProxyTransportManager manager = new BackendProxyTransportManager(null); + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.reserveLegacyRedisDelivery(org.mockito.ArgumentMatchers.any(), + org.mockito.ArgumentMatchers.anyString())).thenReturn(true); + RedisBackendProxyTransport transport = spy(new RedisBackendProxyTransport(null, cache)); + doReturn(true).when(transport).send(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); + setField(manager, "transport", transport); + CountDownLatch callbackEntered = new CountDownLatch(1); + CountDownLatch allowReply = new CountDownLatch(1); + CountDownLatch callbackCompleted = new CountDownLatch(1); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + doAnswer(invocation -> { + callbackEntered.countDown(); + allowReply.await(); + manager.send(JsonEnvelope.builder("reply").build()); + callbackCompleted.countDown(); + return null; + }).when(messages).onMessage(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); + setField(transport, "messageHandler", messages); + + Thread delivery = new Thread(() -> transport.dispatchLegacy( + JsonEnvelope.builder(VotingPluginWire.SUB_VOTE_UPDATE).build())); + delivery.start(); + assertTrue(callbackEntered.await(1, TimeUnit.SECONDS)); + + java.util.concurrent.atomic.AtomicReference closeFailure = + new java.util.concurrent.atomic.AtomicReference<>(); + Thread retirement = new Thread(() -> { + try { + manager.closeRedisForHandoff(); + } catch (Throwable failure) { + closeFailure.set(failure); + } + }); + retirement.start(); + try { + assertTrue(awaitRetiredAfterHandoff(transport)); + + allowReply.countDown(); + assertTrue(callbackCompleted.await(1, TimeUnit.SECONDS)); + retirement.join(TimeUnit.SECONDS.toMillis(1)); + delivery.join(TimeUnit.SECONDS.toMillis(1)); + assertFalse(retirement.isAlive()); + assertFalse(delivery.isAlive()); + assertNull(closeFailure.get()); + verify(transport).send(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); + } finally { + allowReply.countDown(); + if (retirement.isAlive()) retirement.interrupt(); + delivery.join(TimeUnit.SECONDS.toMillis(1)); + retirement.join(TimeUnit.SECONDS.toMillis(1)); + } + } + @Test void legacyPromotionWaitsForReservedDeliveryBeforeReplayingStandbyCopy() throws Exception { ProcessedVoteCache cache = mock(ProcessedVoteCache.class); @@ -354,4 +413,20 @@ private static void setField(Object target, String name, Object value) throws Ex field.setAccessible(true); field.set(target, value); } + + private static boolean awaitRetiredAfterHandoff(RedisBackendProxyTransport transport) throws Exception { + Field lifecycleField = RedisBackendProxyTransport.class.getDeclaredField("legacyLifecycle"); + lifecycleField.setAccessible(true); + Object lifecycle = lifecycleField.get(transport); + Field retiredField = RedisBackendProxyTransport.class.getDeclaredField("retiredAfterHandoff"); + retiredField.setAccessible(true); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + while (System.nanoTime() < deadline) { + synchronized (lifecycle) { + if (retiredField.getBoolean(transport)) return true; + } + Thread.sleep(10L); + } + return false; + } } From 38ae6f676c477674d14db2a2d070571c365f4a0b Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Thu, 10 Sep 2026 00:49:21 -0600 Subject: [PATCH 061/136] Complete bounded proxy transport handoffs --- .../votingplugin/VotingPluginMain.java | 124 +++- .../backendproxy/BackendProxyHandler.java | 27 +- .../cache/ProcessedVoteCache.java | 9 + .../BackendProxyTransportManager.java | 169 +++++- .../transport/RedisBackendProxyTransport.java | 571 +++++++++++++++++- .../votingplugin/proxy/VotingPluginProxy.java | 134 +++- .../proxy/bungee/VotingPluginBungee.java | 36 +- .../proxy/velocity/VotingPluginVelocity.java | 14 + ...PluginMainBackendProxyPublicationTest.java | 98 ++- .../BackendProxyHandlerLifecycleTest.java | 54 +- .../RedisBackendProxyTransportTest.java | 448 +++++++++++++- .../tests/VotingPluginProxyTest.java | 89 +++ .../tests/VotingPluginProxyTestImpl.java | 11 +- 13 files changed, 1687 insertions(+), 97 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 1a6247b2c..ab51a51cb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1247,6 +1247,11 @@ public static final class BackendProxyRestart { private final boolean disabled; private final boolean previousRequiresPreparation; private volatile boolean previousPrepared; + // Redis listener retirement can wait for callbacks and listener shutdown. It is + // completed by the Control worker during validation, before the final Bukkit + // publication callback, and must be restored if that publication is abandoned. + private volatile boolean redisHandoffCompleted; + private boolean redisHandoffInProgress; private boolean finished; private boolean abandonmentRequested; private volatile boolean published; @@ -1282,7 +1287,10 @@ public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { // Every transition away from an active HTTP transport must first drain its // durable outgoing queue. Restricting preparation to HTTP-to-HTTP swaps can // strand accepted deliveries when another transport is published. - boolean previousRequiresPreparation = previous != null && previous.requiresPreparationForReplacement(); + boolean sameRedisHandoff = previous != null && previous.getMethod() == BungeeMethod.REDIS + && replacementMethod == BungeeMethod.REDIS; + boolean previousRequiresPreparation = previous != null + && (previous.requiresPreparationForReplacement() || sameRedisHandoff); BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); try { replacement.loadForReplacement(); @@ -1308,12 +1316,51 @@ public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long restart.previous.reservePreparedHttpHandoff(restart.replacement); } if (restart.replacement != null) restart.replacement.validateTransport(validationDeadlineNanos); + // Same-Redis handoff fences callbacks and joins the retiring listener. Those + // waits are bounded, but they must never run on the Bukkit publication task. + // The staged replacement still keeps all inbound callbacks behind its + // publication gate, so doing this on the Control worker cannot expose it early. + if (restart.previous != null && restart.replacement != null + && restart.previous.requiresRedisHandoff(restart.replacement)) { + synchronized (restart) { + if (restart.redisHandoffCompleted) return; + if (restart.redisHandoffInProgress) + throw new IllegalStateException("Redis handoff validation is already in progress"); + restart.redisHandoffInProgress = true; + } + boolean abandonAfterHandoff; + try { + restart.previous.completeRedisHandoff(restart.replacement); + } catch (RuntimeException handoffFailure) { + synchronized (restart) { + restart.redisHandoffInProgress = false; + restart.notifyAll(); + } + throw handoffFailure; + } + synchronized (restart) { + restart.redisHandoffInProgress = false; + restart.redisHandoffCompleted = true; + abandonAfterHandoff = restart.abandonmentRequested; + restart.notifyAll(); + } + if (abandonAfterHandoff) abortBackendProxyHandlerRestart(restart); + } } public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { synchronized (this) { if (restart == null || restart.finished) throw new IllegalStateException("Backend proxy restart is no longer active"); - if (restart.abandonmentRequested) { + boolean requiresRedisHandoff = restart.previous != null && restart.replacement != null + && restart.previous.requiresRedisHandoff(restart.replacement); + boolean abandonmentRequested; + synchronized (restart) { + if (requiresRedisHandoff && (!restart.redisHandoffCompleted || restart.redisHandoffInProgress)) { + throw new IllegalStateException("Redis handoff must complete during validation before publication"); + } + abandonmentRequested = restart.abandonmentRequested; + } + if (abandonmentRequested) { abortBackendProxyHandlerRestart(restart); return; } @@ -1345,26 +1392,6 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { return; } publishBackendProxyHandler(restart.previous, restart.replacement); - try { - if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); - } catch (RuntimeException handoffFailure) { - // Stop the staged presence generation before reasserting the restored old - // handler. Otherwise the proxy rejects the old generation after rollback. - backendProxyHandler = restart.previous; - restart.replacement.abortStagedInboundTo(restart.previous); - try { - restart.replacement.close(); - } catch (RuntimeException closeFailure) { - handoffFailure.addSuppressed(closeFailure); - } - try { - restart.previous.refreshPresenceAfterFailedReplacement(); - } catch (RuntimeException refreshFailure) { - handoffFailure.addSuppressed(refreshFailure); - } - restart.finished = true; - throw handoffFailure; - } // Keep the prepared queue owned by the previous handler until every fallible // publication step succeeds. Admission performs no network I/O. try { @@ -1372,16 +1399,33 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { } catch (RuntimeException handoffFailure) { backendProxyHandler = restart.previous; restart.replacement.abortStagedInboundTo(restart.previous); + // A validated Redis promotion may already own accepted, deduplicated + // replay envelopes even though inbound publication has not opened. Move + // those envelopes back before closing the staged replacement, whose + // normal close path deliberately clears its replay queue. + if (restart.redisHandoffCompleted && restart.previous != null) { + try { + restart.previous.restoreAfterFailedReplacement(restart.replacement); + restart.previous.refreshPresenceAfterFailedReplacement(); + } catch (RuntimeException restorationFailure) { + handoffFailure.addSuppressed(restorationFailure); + // Retain the staged replacement and its replay queue for the caller's + // subsequent rollback retry rather than clearing accepted envelopes. + throw handoffFailure; + } + } try { restart.replacement.close(); } catch (RuntimeException closeFailure) { handoffFailure.addSuppressed(closeFailure); } - try { - restart.previous.restoreAfterFailedReplacement(); - restart.previous.refreshPresenceAfterFailedReplacement(); - } catch (RuntimeException restorationFailure) { - handoffFailure.addSuppressed(restorationFailure); + if (!restart.redisHandoffCompleted) { + try { + restart.previous.restoreAfterFailedReplacement(); + restart.previous.refreshPresenceAfterFailedReplacement(); + } catch (RuntimeException restorationFailure) { + handoffFailure.addSuppressed(restorationFailure); + } } restart.finished = true; throw handoffFailure; @@ -1424,20 +1468,38 @@ void publishBackendProxyHandler(BackendProxyHandler previous, BackendProxyHandle /** Returns false once publication committed and can no longer be rolled back as a failed apply. */ public synchronized boolean requestBackendProxyHandlerRestartAbandonment(BackendProxyRestart restart) { if (restart == null) return true; - if (restart.published) return false; - restart.abandonmentRequested = true; + synchronized (restart) { + if (restart.published) return false; + restart.abandonmentRequested = true; + restart.notifyAll(); + } return true; } public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart restart) { if (restart == null || restart.finished) return; + synchronized (restart) { + // Validation owns the retiring Redis listener until its handoff either + // succeeds or fails. Deferring rollback closes the race where abort could + // observe a false completion flag and leave that listener fenced. + if (restart.redisHandoffInProgress) { + restart.abandonmentRequested = true; + return; + } + } + if (restart.redisHandoffCompleted && restart.previous != null && restart.replacement != null) { + // Preserve the promoted replacement's pre-publication replay queue before + // its close fences and clears it, then retire that staged listener. + restart.previous.restoreAfterFailedReplacement(restart.replacement); + } if (restart.replacement != null) { restart.replacement.abortStagedInboundTo(restart.previous); restart.replacement.close(); } if (backendProxyHandler == restart.previous && restart.previous != null - && (restart.previousPrepared || restart.previous.getMethod() == BungeeMethod.PLUGINMESSAGING)) { - restart.previous.restoreAfterFailedReplacement(); + && (restart.previousPrepared || restart.redisHandoffCompleted + || restart.previous.getMethod() == BungeeMethod.PLUGINMESSAGING)) { + if (!restart.redisHandoffCompleted) restart.previous.restoreAfterFailedReplacement(); } restart.finished = true; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 9a4fd35aa..db7532fc0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -184,7 +184,7 @@ void dispatchIncomingAfterPublication(JsonEnvelope envelope, Runnable localDispa /** Returns whether replacement preparation must preserve accepted deliveries. */ public boolean requiresPreparationForReplacement() { return method == BungeeMethod.HTTP || method == BungeeMethod.PLUGINMESSAGING - || transportManager.hasPendingAsyncHandoff(); + || transportManager.hasPendingAsyncHandoff() || transportManager.hasPendingRedisReplay(); } /** Prepares HTTP state or waits off-thread for an earlier cross-transport handoff. */ @@ -202,6 +202,18 @@ public boolean prepareForReplacement(BungeeMethod replacementMethod, long deadli transportManager.prepareAsyncHandoffForReplacement(deadlineNanos); return true; } + if (method == BungeeMethod.REDIS && replacementMethod != BungeeMethod.REDIS) { + if (transportManager.hasPendingAsyncHandoff()) + transportManager.prepareAsyncHandoffForReplacement(deadlineNanos); + return transportManager.prepareRedisReplayTransition(replacementMethod, deadlineNanos); + } + if (method == BungeeMethod.REDIS) { + // Same-Redis retirement installs its send fence during the bounded + // off-thread handoff. Returning true here lets the staged replacement + // buffer its own sends until the predecessor FIFO is admitted at Bukkit + // publication. + return true; + } if (!transportManager.hasPendingAsyncHandoff()) return false; transportManager.prepareAsyncHandoffForReplacement(deadlineNanos); return true; @@ -226,6 +238,12 @@ public void restoreAfterFailedReplacement() { transportManager.restoreAfterFailedReplacement(); } + /** Restores a failed same-Redis predecessor without discarding its replacement's replay FIFO. */ + public void restoreAfterFailedReplacement(BackendProxyHandler failedReplacement) { + transportManager.restoreAfterFailedReplacement( + failedReplacement == null ? null : failedReplacement.transportManager); + } + /** Reasserts the old handler with a fresh presence generation after rollback. */ public void refreshPresenceAfterFailedReplacement() { if (presenceManager != null && presenceReportingActivated) { @@ -253,10 +271,15 @@ public void validateTransport(long deadlineNanos) { /** Completes the no-loss/no-duplicate same-Redis subscriber handoff after validation. */ public void completeRedisHandoff(BackendProxyHandler replacement) { - if (method != BungeeMethod.REDIS || replacement.method != BungeeMethod.REDIS) return; + if (!requiresRedisHandoff(replacement)) return; transportManager.completeRedisHandoff(replacement.transportManager); } + /** Returns whether this replacement needs the bounded same-Redis retirement path. */ + public boolean requiresRedisHandoff(BackendProxyHandler replacement) { + return replacement != null && method == BungeeMethod.REDIS && replacement.method == BungeeMethod.REDIS; + } + /** Replays Redis handoff deliveries only after inbound publication is open. */ public void replayRedisAfterHandoffPublication() { transportManager.replayRedisAfterHandoffPublication(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java index c137bf948..5029b7e4d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java @@ -130,6 +130,15 @@ public synchronized void activateRedisSubscriber(Object subscriber) { standbyRedisSubscriber = null; } + /** Restores the retired active listener after a promoted replacement is rolled back. */ + public synchronized void restoreRedisSubscriber(Object subscriber) { + activeRedisSubscriber = subscriber; + standbyRedisSubscriber = null; + legacyRedisDeliveries.clear(); + legacyRedisDeliveryBytes = 0; + legacyRedisHandoffOverflowed = false; + } + /** Consumes one matching delivery processed by the previous active subscriber. */ public synchronized boolean consumeLegacyRedisDelivery(String signature) { Integer count = legacyRedisDeliveries.get(signature); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 788ca140e..e382e02b7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -24,6 +24,10 @@ public class BackendProxyTransportManager { private BackendProxyTransport transport; private BackendProxyTransport preparedTransport; private BackendProxyTransport retiredTransport; + // The old Redis instance is already stopped after a successful worker-side + // handoff, but keeps its captured handler long enough to restore it if Bukkit + // publication subsequently fails or is abandoned. + private RedisBackendProxyTransport completedRedisHandoffTransport; private BackendProxyTransportManager forwardingManager; private final java.util.ArrayDeque preparedSends = new java.util.ArrayDeque<>(); private final java.util.ArrayDeque asyncHandoffSends = new java.util.ArrayDeque<>(); @@ -127,7 +131,13 @@ public synchronized void close() { } if (retiredTransport != null) { BackendProxyTransport retired = retiredTransport; - try { + if (retired instanceof RedisBackendProxyTransport redis) { + // A failed Redis listener shutdown can spend its bounded join timeout in + // close(). Publication calls the predecessor's close on Bukkit, so retry + // that fenced cleanup independently instead of stalling publication. + retiredTransport = null; + closeRetiredRedisAsync(redis); + } else try { retired.close(); retiredTransport = null; } catch (RuntimeException cleanupFailure) { @@ -139,9 +149,38 @@ public synchronized void close() { } } } + completedRedisHandoffTransport = null; if (forwardingManager == null) preparedSends.clear(); } + /** Retries only a fenced retired Redis listener off the Bukkit publication callback. */ + private void closeRetiredRedisAsync(RedisBackendProxyTransport retired) { + Thread cleanup = new Thread(() -> { + RuntimeException cleanupFailure = null; + for (int attempt = 0; attempt < 3; attempt++) { + try { + retired.close(); + return; + } catch (RuntimeException failure) { + cleanupFailure = failure; + if (attempt == 2) break; + try { + Thread.sleep(250L); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + break; + } + } + } + if (cleanupFailure != null && plugin != null) { + plugin.getLogger().warning("Retired Redis backend listener did not stop cleanly during async cleanup"); + plugin.debug(cleanupFailure); + } + }, "VotingPlugin-Retired-Redis-Cleanup"); + cleanup.setDaemon(true); + cleanup.start(); + } + public void validate() { validate(System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(25)); } @@ -476,17 +515,45 @@ public synchronized void restorePreparedTransport() { } public void restoreAfterFailedReplacement() { + restoreAfterFailedReplacement(null); + } + + /** Restores an old Redis listener together with a promoted replacement's unplayed replay FIFO. */ + public void restoreAfterFailedReplacement(BackendProxyTransportManager failedReplacement) { + java.util.List replacementReplay = failedReplacement == null + ? java.util.Collections.emptyList() : failedReplacement.detachRedisReplayForFailedHandoff(); + restoreRetiredRedisAfterFailedHandoff(replacementReplay); + restoreCompletedRedisAfterFailedHandoff(replacementReplay); if (transport instanceof PluginMessagingBackendProxyTransport pluginMessaging) { pluginMessaging.restoreAfterFailedReplacement(); } restorePreparedTransport(); } + private synchronized java.util.List detachRedisReplayForFailedHandoff() { + if (!(transport instanceof RedisBackendProxyTransport redis)) return java.util.Collections.emptyList(); + return redis.detachReplayForFailedHandoff(); + } + public void awaitPreparedTransportRestoration(long deadlineNanos) { if (transport instanceof HttpBackendProxyTransport http) http.awaitCredentialRestoration(deadlineNanos); } - public void closeRedisForHandoff() { + /** Requires an off-thread drain before abandoning active Redis replay for another transport. */ + public synchronized boolean hasPendingRedisReplay() { + return transport instanceof RedisBackendProxyTransport redis && redis.hasPendingReplayForReplacement(); + } + + public boolean prepareRedisReplayTransition(BungeeMethod replacementMethod, long deadlineNanos) { + RedisBackendProxyTransport redis; + synchronized (this) { + if (replacementMethod == BungeeMethod.REDIS || !(transport instanceof RedisBackendProxyTransport)) return true; + redis = (RedisBackendProxyTransport) transport; + } + return redis.awaitReplayDrainForNonRedisReplacement(deadlineNanos); + } + + public java.util.List closeRedisForHandoff(BackendProxyTransportManager replacement) { RedisBackendProxyTransport retiring; synchronized (this) { if (!(transport instanceof RedisBackendProxyTransport)) { @@ -494,6 +561,23 @@ public void closeRedisForHandoff() { } retiring = (RedisBackendProxyTransport) transport; retiredTransport = retiring; + // The Redis listener is fenced off-thread before Bukkit publishes the + // staged replacement. Keep sends accepted in that interval in the same + // bounded predecessor FIFO instead of dropping them once transport is + // detached below. Publication transfers this queue ahead of messages the + // staged replacement accepted, while rollback drains it through the + // restored Redis transport. + preparedSendFence = true; + } + java.util.List replay = retiring.freezeReplayForSuccessiveHandoff(); + try { + replacement.acceptRedisReplayFromPreviousHandoff(replay); + } catch (RuntimeException admissionFailure) { + retiring.restoreFrozenReplayAfterFailedSuccessiveHandoff(replay); + synchronized (this) { + if (retiredTransport == retiring) retiredTransport = null; + } + throw admissionFailure; } try { // closeForHandoff() waits for already-running Redis callbacks. Those callbacks @@ -504,18 +588,34 @@ public void closeRedisForHandoff() { synchronized (this) { if (failure instanceof RedisBackendProxyTransport.HandoffQuiescenceException && retiredTransport == retiring && transport == retiring) { + replacement.removeRedisReplayFromPreviousHandoff(replay); + retiring.restoreFrozenReplayAfterFailedSuccessiveHandoff(replay); retiredTransport = null; } else if (transport == retiring) { transport = null; } } + if (failure instanceof RedisBackendProxyTransport.HandoffQuiescenceException) throw failure; // Retain the fenced old listener so a later manager close can retry its // cleanup without ever touching the promoted replacement. - throw failure; + if (plugin != null) { + plugin.getLogger().warning("Previous Redis backend listener did not stop cleanly after handoff"); + plugin.debug(failure); + } } synchronized (this) { if (transport == retiring) transport = null; } + return replay; + } + + private void acceptRedisReplayFromPreviousHandoff(java.util.List replay) { + if (transport instanceof RedisBackendProxyTransport redis) redis.acceptReplayFromPreviousHandoff(replay); + else if (!replay.isEmpty()) throw new IllegalStateException("Redis replacement transport is unavailable"); + } + + private void removeRedisReplayFromPreviousHandoff(java.util.List replay) { + if (transport instanceof RedisBackendProxyTransport redis) redis.removeReplayFromPreviousHandoff(replay); } public void activateRedisAfterHandoff() { @@ -532,36 +632,51 @@ public void replayRedisAfterHandoffPublication() { /** Fences the old Redis listener before promoting the validated standby. */ public void completeRedisHandoff(BackendProxyTransportManager replacement) { java.util.Objects.requireNonNull(replacement, "replacement"); - try { - closeRedisForHandoff(); - } catch (RuntimeException retirementFailure) { - if (retirementFailure instanceof RedisBackendProxyTransport.HandoffQuiescenceException) { - throw retirementFailure; - } - // The old listener is already fenced. Treat failure to join it as cleanup - // degradation; promotion remains safe because callbacks can no longer enter. - if (plugin != null) { - plugin.getLogger().warning("Previous Redis backend listener did not stop cleanly after handoff"); - plugin.debug(retirementFailure); - } - } + java.util.List replay = closeRedisForHandoff(replacement); try { replacement.activateRedisAfterHandoff(); } catch (RuntimeException activationFailure) { - restoreRetiredRedisAfterFailedHandoff(activationFailure); + java.util.List rollbackReplay = replacement.detachRedisReplayForFailedHandoff(); + restoreRetiredRedisAfterFailedHandoff(activationFailure, + mergeRedisRollbackReplay(replay, rollbackReplay)); throw activationFailure; } closeRetiredRedisAfterHandoff(); } - private synchronized void restoreRetiredRedisAfterFailedHandoff(RuntimeException activationFailure) { + private static java.util.List mergeRedisRollbackReplay(java.util.List predecessorReplay, + java.util.List standbyReplay) { + if (standbyReplay.isEmpty()) return predecessorReplay; + if (predecessorReplay.isEmpty()) return standbyReplay; + if (standbyReplay.size() >= predecessorReplay.size() + && standbyReplay.subList(0, predecessorReplay.size()).equals(predecessorReplay)) return standbyReplay; + java.util.ArrayList merged = new java.util.ArrayList<>( + predecessorReplay.size() + standbyReplay.size()); + merged.addAll(predecessorReplay); + merged.addAll(standbyReplay); + if (merged.size() > RedisBackendProxyTransport.MAX_REPLAY_HANDOFF_DELIVERIES) + throw new IllegalStateException("Redis rollback replay exceeds its bounded handoff capacity"); + return merged; + } + + private synchronized void restoreRetiredRedisAfterFailedHandoff(RuntimeException activationFailure, + java.util.List replacementReplay) { + try { + restoreRetiredRedisAfterFailedHandoff(replacementReplay); + } catch (RuntimeException restorationFailure) { + activationFailure.addSuppressed(restorationFailure); + } + } + + private synchronized void restoreRetiredRedisAfterFailedHandoff(java.util.List replacementReplay) { + if (transport != null) return; if (!(retiredTransport instanceof RedisBackendProxyTransport redis)) return; try { - redis.restoreAfterFailedHandoff(); + redis.restoreAfterFailedHandoff(replacementReplay); transport = redis; retiredTransport = null; } catch (RuntimeException restorationFailure) { - activationFailure.addSuppressed(restorationFailure); + throw new IllegalStateException("Retired Redis backend listener could not be restored", restorationFailure); } } @@ -571,6 +686,7 @@ private synchronized void closeRetiredRedisAfterHandoff() { retiredTransport = null; try { retired.close(); + if (retired instanceof RedisBackendProxyTransport redis) completedRedisHandoffTransport = redis; } catch (RuntimeException cleanupFailure) { retiredTransport = retired; if (plugin != null) { @@ -580,6 +696,19 @@ private synchronized void closeRetiredRedisAfterHandoff() { } } + /** Restores a worker-retired Redis listener when publication did not commit. */ + private synchronized void restoreCompletedRedisAfterFailedHandoff(java.util.List replacementReplay) { + if (transport != null || completedRedisHandoffTransport == null) return; + RedisBackendProxyTransport retired = completedRedisHandoffTransport; + try { + retired.restoreAfterFailedHandoff(replacementReplay); + transport = retired; + completedRedisHandoffTransport = null; + } catch (RuntimeException restorationFailure) { + throw new IllegalStateException("Retired Redis backend listener could not be restored", restorationFailure); + } + } + public ClientHandler getClientHandler() { return transport instanceof SocketBackendProxyTransport ? ((SocketBackendProxyTransport) transport).getClientHandler() : null; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index 3d11756f4..070359dbb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -29,9 +29,22 @@ public HandoffQuiescenceException(String message) { super(message); } } + + public static final class HandoffReplayBackpressureException extends IllegalStateException { + private static final long serialVersionUID = 1L; + + public HandoffReplayBackpressureException(String message) { + super(message); + } + } static final int MAX_LEGACY_HANDOFF_DELIVERIES = 4096; static final int MAX_IDENTIFIED_HANDOFF_DELIVERIES = 4096; + static final int MAX_REPLAY_HANDOFF_DELIVERIES = MAX_LEGACY_HANDOFF_DELIVERIES + + MAX_IDENTIFIED_HANDOFF_DELIVERIES; + static final int REPLAY_BATCH_SIZE = 32; private static final long HANDOFF_QUIESCE_TIMEOUT_NANOS = TimeUnit.SECONDS.toNanos(3); + private static final long HANDOFF_REPLAY_BACKPRESSURE_TIMEOUT_NANOS = TimeUnit.SECONDS.toNanos(3); + private static final long REPLAY_BATCH_EXECUTION_TIMEOUT_NANOS = TimeUnit.SECONDS.toNanos(3); private final VotingPluginMain plugin; private final ProcessedVoteCache processedVoteCache; @@ -52,6 +65,17 @@ public HandoffQuiescenceException(String message) { private boolean legacyHandoffDegraded; private boolean retiredAfterHandoff; private boolean replayingHandoff; + private boolean replayTransferFrozen; + private boolean replayBackpressureFailureLogged; + private long replayGeneration; + private Thread replayWorker; + private boolean replayTaskOutstanding; + private long replayTaskGeneration = -1L; + private int replayDeliveriesInFlight; + // Counts only handlers which have already crossed the Bukkit dispatch fence. + // It is deliberately independent of replayGeneration so cancellation cannot + // erase accounting for a callback that was already executing. + private int replayCallbacksInFlight; private int dispatchesInFlight; private final java.util.ArrayDeque deliveriesAfterReplay = new java.util.ArrayDeque<>(); private GlobalMessageHandler messageHandler; @@ -93,11 +117,7 @@ public void debug(String message) { try { JsonEnvelope envelope = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.decode(payload); String deliveryId = envelope.getFields().get(VotingPluginWire.K_REDIS_DELIVERY_ID); - if (deliveryId != null) { - dispatchIdentified(envelope, deliveryId); - } else { - dispatchLegacy(envelope); - } + dispatchReceivedSubscriberEnvelope(envelope, deliveryId); } catch (Exception e) { plugin.debug("Redis decode failed: " + e.getMessage()); } @@ -112,9 +132,34 @@ public void onSubscribe(String channel, int subscribedChannels) { listenerThread.start(); } + /** + * Redis Pub/Sub has already consumed this payload when its callback is invoked. + * A bounded queue wait therefore cannot discard it: retry the same callback in + * place, applying TCP/Pub/Sub backpressure until FIFO replay capacity returns or + * this subscriber is deliberately fenced during shutdown/rollback. + */ + void dispatchReceivedSubscriberEnvelope(JsonEnvelope envelope, String deliveryId) { + while (true) { + try { + if (deliveryId != null) dispatchIdentified(envelope, deliveryId); + else dispatchLegacy(envelope); + return; + } catch (HandoffReplayBackpressureException retry) { + synchronized (legacyLifecycle) { + if (retiredAfterHandoff) return; + } + if (plugin != null) + plugin.debug("Redis handoff replay remains full; retaining the received Pub/Sub payload for retry"); + } + } + } + void dispatchIdentified(JsonEnvelope envelope, String deliveryId) { boolean accepted; synchronized (legacyLifecycle) { + if (!awaitReplayTransferResolution()) return; + if (retiredAfterHandoff) return; + awaitReplayCapacity(); if (retiredAfterHandoff) return; if (standbySubscriber) { bufferIdentifiedDelivery(envelope, deliveryId); @@ -122,7 +167,7 @@ void dispatchIdentified(JsonEnvelope envelope, String deliveryId) { } accepted = processedVoteCache.reserveRedisDelivery(deliveryId); if (accepted && replayingHandoff) { - deliveriesAfterReplay.addLast(envelope); + enqueueReplayDelivery(envelope); accepted = false; } if (accepted) dispatchesInFlight++; @@ -135,11 +180,15 @@ void dispatchLegacy(JsonEnvelope envelope) { int encodedBytes = ProcessedVoteCache.legacyRedisDeliveryBytes(signature); boolean dispatch = false; synchronized (legacyLifecycle) { + if (!awaitReplayTransferResolution()) return; + if (retiredAfterHandoff) return; + awaitReplayCapacity(); if (retiredAfterHandoff) return; if (processedVoteCache.reserveLegacyRedisDelivery(subscriberIdentity, signature)) { dispatch = true; } else if (encodedBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_DELIVERY_BYTES && bufferedLegacyDeliveries.size() < MAX_LEGACY_HANDOFF_DELIVERIES + && handoffDeliveryCount() < MAX_REPLAY_HANDOFF_DELIVERIES && bufferedLegacyDeliveryBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES - encodedBytes) { bufferedLegacyDeliveries.add(new BufferedHandoffDelivery( nextHandoffSequence++, envelope, signature, false)); @@ -160,7 +209,7 @@ void dispatchLegacy(JsonEnvelope envelope) { } } if (dispatch && replayingHandoff) { - deliveriesAfterReplay.addLast(envelope); + enqueueReplayDelivery(envelope); dispatch = false; } if (dispatch) dispatchesInFlight++; @@ -168,6 +217,22 @@ void dispatchLegacy(JsonEnvelope envelope) { if (dispatch) dispatchTracked(envelope); } + /** Holds a consumed subscriber callback until a same-Redis transfer commits or rolls back. */ + private boolean awaitReplayTransferResolution() { + boolean interrupted = false; + while (replayTransferFrozen) { + try { + legacyLifecycle.wait(); + } catch (InterruptedException waitInterrupted) { + // Do not drop a Pub/Sub payload merely because the listener was nudged + // during transfer. Its final owner will explicitly release this wait. + interrupted = true; + } + } + if (interrupted) Thread.currentThread().interrupt(); + return !retiredAfterHandoff; + } + private void dispatchTracked(JsonEnvelope envelope) { try { messageHandler.onMessage(envelope); @@ -186,6 +251,7 @@ public void activateAfterHandoff() { || processedVoteCache.isLegacyRedisHandoffOverflowed()) throw new IllegalStateException("Redis handoff buffer overflowed before publication"); replayingHandoff = true; + replayBackpressureFailureLogged = false; processedVoteCache.activateRedisSubscriber(subscriberIdentity); standbySubscriber = false; java.util.ArrayList buffered = new java.util.ArrayList<>( @@ -194,14 +260,16 @@ public void activateAfterHandoff() { buffered.addAll(bufferedLegacyDeliveries); buffered.sort(java.util.Comparator.comparingLong(BufferedHandoffDelivery::sequence)); for (BufferedHandoffDelivery delivery : buffered) { + boolean dispatch; try { - boolean dispatch = delivery.identified() + dispatch = delivery.identified() ? processedVoteCache.reserveRedisDelivery(delivery.identity()) : !processedVoteCache.consumeLegacyRedisDelivery(delivery.identity()); - if (dispatch) deliveriesAfterReplay.addLast(delivery.envelope()); } catch (RuntimeException replayFailure) { if (plugin != null) plugin.debug("Redis handoff replay failed: " + replayFailure.getMessage()); + continue; } + if (dispatch) enqueueReplayDelivery(delivery.envelope()); } bufferedIdentifiedDeliveries.clear(); bufferedIdentifiedDeliveryBytes = 0; @@ -216,26 +284,295 @@ public void activateAfterHandoff() { /** Replays buffered deliveries after the owning handler opens its publication gate. */ public void replayAfterHandoffPublication() { - java.util.ArrayList replay; + if (plugin == null) { + // Unit tests use a transport without a Bukkit scheduler. Production always + // takes the bounded worker path below. + drainReplayWithoutScheduler(); + return; + } + startReplayWorkerIfNeeded(); + } + + /** True while a same-Redis replay still owns accepted envelopes or Bukkit work. */ + public boolean hasPendingReplayForReplacement() { + synchronized (legacyLifecycle) { + return replayingHandoff || replayTransferFrozen || !deliveriesAfterReplay.isEmpty() + || replayDeliveriesInFlight != 0 || replayCallbacksInFlight != 0 || replayTaskOutstanding; + } + } + + /** + * Waits only on the Control/validation worker for active replay to finish + * before a non-Redis replacement can retire this transport. It never moves + * Redis envelopes into a transport that cannot preserve Redis semantics. + */ + public boolean awaitReplayDrainForNonRedisReplacement(long deadlineNanos) { + boolean interrupted = false; + synchronized (legacyLifecycle) { + while (hasPendingReplayForReplacement()) { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) { + if (interrupted) Thread.currentThread().interrupt(); + return false; + } + try { + TimeUnit.NANOSECONDS.timedWait(legacyLifecycle, remaining); + } catch (InterruptedException waitInterrupted) { + interrupted = true; + break; + } + } + } + if (interrupted) Thread.currentThread().interrupt(); + return !interrupted; + } + + private void startReplayWorkerIfNeeded() { + Thread worker; + synchronized (legacyLifecycle) { + if (!replayingHandoff || replayTransferFrozen || replayWorker != null || replayTaskOutstanding) return; + worker = new Thread(this::drainReplayOnWorker, "VotingPlugin-Redis-Backend-Replay"); + worker.setDaemon(true); + replayWorker = worker; + } + try { + worker.start(); + } catch (RuntimeException failed) { + synchronized (legacyLifecycle) { + if (replayWorker == worker) replayWorker = null; + legacyLifecycle.notifyAll(); + } + throw failed; + } + } + + private void enqueueReplayDelivery(JsonEnvelope envelope) { + if (deliveriesAfterReplay.size() + replayDeliveriesInFlight >= MAX_REPLAY_HANDOFF_DELIVERIES) + throw new IllegalStateException("Redis handoff replay capacity was not reserved"); + deliveriesAfterReplay.addLast(envelope); + } + + /** + * Preserves handoff FIFO without allowing the replay queue to grow without + * bound. A timeout is explicit and happens before the identified ID is + * reserved, so an undeliverable callback is never silently marked processed. + */ + private void awaitReplayCapacity() { + if (!replayingHandoff) return; + boolean interrupted = false; + long deadline = System.nanoTime() + HANDOFF_REPLAY_BACKPRESSURE_TIMEOUT_NANOS; + while (replayingHandoff && deliveriesAfterReplay.size() + replayDeliveriesInFlight + >= MAX_REPLAY_HANDOFF_DELIVERIES) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0L) { + if (!replayBackpressureFailureLogged && plugin != null) { + plugin.getLogger().severe("Redis handoff replay is not draining; rejected an unreserved callback"); + replayBackpressureFailureLogged = true; + } + if (interrupted) Thread.currentThread().interrupt(); + throw new HandoffReplayBackpressureException("Redis handoff replay queue did not drain before its deadline"); + } + try { + TimeUnit.NANOSECONDS.timedWait(legacyLifecycle, remaining); + } catch (InterruptedException waitInterrupted) { + interrupted = true; + break; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + throw new HandoffReplayBackpressureException("Interrupted while waiting for Redis handoff replay capacity"); + } + } + + private void drainReplayWithoutScheduler() { while (true) { + java.util.ArrayList replay; + GlobalMessageHandler handler; + long generation; synchronized (legacyLifecycle) { - if (deliveriesAfterReplay.isEmpty()) { + if (!replayingHandoff || deliveriesAfterReplay.isEmpty()) { replayingHandoff = false; - break; + legacyLifecycle.notifyAll(); + return; + } + replay = takeReplayBatch(); + handler = messageHandler; + generation = replayGeneration; + } + dispatchReplayBatch(replay, handler, generation); + completeReplayBatch(replay.size()); + } + } + + /** Coordinates one bounded Bukkit batch at a time without blocking publication. */ + private void drainReplayOnWorker() { + try { + while (!Thread.currentThread().isInterrupted()) { + java.util.ArrayList replay; + GlobalMessageHandler handler; + long generation; + synchronized (legacyLifecycle) { + if (!replayingHandoff || deliveriesAfterReplay.isEmpty()) { + replayingHandoff = false; + legacyLifecycle.notifyAll(); + return; + } + if (replayTransferFrozen) return; + replay = takeReplayBatch(); + handler = messageHandler; + generation = replayGeneration; } - replay = new java.util.ArrayList<>(deliveriesAfterReplay); - deliveriesAfterReplay.clear(); + ReplayBatch batch = new ReplayBatch(generation); + java.util.concurrent.CountDownLatch completed = new java.util.concurrent.CountDownLatch(1); + try { + synchronized (legacyLifecycle) { + replayTaskOutstanding = true; + replayTaskGeneration = batch.generation; + } + plugin.getBukkitScheduler().runTask(plugin, () -> { + try { + synchronized (legacyLifecycle) { + if (batch.generation != replayGeneration || batch.returned || !replayingHandoff) return; + if (replayTransferFrozen) return; + batch.started = true; + } + dispatchReplayBatch(replay, handler, batch.generation); + } finally { + synchronized (legacyLifecycle) { + if (batch.generation == replayGeneration && !batch.returned) { + if (!batch.started && replayTransferFrozen) { + batch.returned = true; + requeueReplayBatch(replay); + } else completeReplayBatch(replay.size()); + } + if (replayTaskOutstanding && replayTaskGeneration == batch.generation) { + replayTaskOutstanding = false; + replayTaskGeneration = -1L; + } + legacyLifecycle.notifyAll(); + } + completed.countDown(); + startReplayWorkerIfNeeded(); + } + }); + if (!completed.await(REPLAY_BATCH_EXECUTION_TIMEOUT_NANOS, TimeUnit.NANOSECONDS)) { + synchronized (legacyLifecycle) { + if (!batch.started && !batch.returned && batch.generation == replayGeneration) { + batch.returned = true; + replayGeneration++; + requeueReplayBatch(replay); + // Do not leave replay owned by a Bukkit task that was accepted but + // never began. A task already running must retain ownership until its + // finally block completes; otherwise a second worker could overlap it + // and the older task could clear the newer task's shared state. + if (replayTaskOutstanding && replayTaskGeneration == batch.generation) { + replayTaskOutstanding = false; + replayTaskGeneration = -1L; + } + } + legacyLifecycle.notifyAll(); + } + return; + } + } catch (InterruptedException interrupted) { + synchronized (legacyLifecycle) { + // A batch is removed before it can be scheduled. If the worker is + // interrupted while waiting for Bukkit, put an unstarted batch back + // at the front; otherwise the accepted deliveries would vanish. + if (!batch.started && !batch.returned && batch.generation == replayGeneration) { + batch.returned = true; + requeueReplayBatch(replay); + if (replayTaskOutstanding && replayTaskGeneration == batch.generation) { + replayTaskOutstanding = false; + replayTaskGeneration = -1L; + } + } + legacyLifecycle.notifyAll(); + } + Thread.currentThread().interrupt(); + return; + } catch (RuntimeException schedulingFailure) { + synchronized (legacyLifecycle) { + if (!batch.returned && batch.generation == replayGeneration) { + batch.returned = true; + requeueReplayBatch(replay); + } + if (replayTaskOutstanding && replayTaskGeneration == batch.generation) { + replayTaskOutstanding = false; + replayTaskGeneration = -1L; + } + legacyLifecycle.notifyAll(); + } + if (plugin != null) plugin.debug("Redis handoff replay scheduling failed: " + schedulingFailure.getMessage()); + try { + Thread.sleep(250L); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return; + } + } + } + } finally { + synchronized (legacyLifecycle) { + if (replayWorker == Thread.currentThread()) replayWorker = null; + legacyLifecycle.notifyAll(); } - dispatchReplayBatch(replay); + startReplayWorkerIfNeeded(); } } - private void dispatchReplayBatch(java.util.List replay) { + private java.util.ArrayList takeReplayBatch() { + java.util.ArrayList replay = new java.util.ArrayList<>(REPLAY_BATCH_SIZE); + while (!deliveriesAfterReplay.isEmpty() && replay.size() < REPLAY_BATCH_SIZE) + replay.add(deliveriesAfterReplay.removeFirst()); + replayDeliveriesInFlight += replay.size(); + return replay; + } + + private void completeReplayBatch(int size) { + synchronized (legacyLifecycle) { + replayDeliveriesInFlight -= size; + if (replayDeliveriesInFlight < 0) replayDeliveriesInFlight = 0; + legacyLifecycle.notifyAll(); + } + } + + private void requeueReplayBatch(java.util.List replay) { + for (int index = replay.size() - 1; index >= 0; index--) deliveriesAfterReplay.addFirst(replay.get(index)); + completeReplayBatch(replay.size()); + } + + private static final class ReplayBatch { + private final long generation; + private boolean started; + private boolean returned; + + private ReplayBatch(long generation) { + this.generation = generation; + } + } + + private void dispatchReplayBatch(java.util.List replay, GlobalMessageHandler handler, + long generation) { + if (handler == null) return; for (JsonEnvelope envelope : replay) { + synchronized (legacyLifecycle) { + // A close/fence may happen after Bukkit accepts the batch. Never let a + // stale generation start another callback after that point. + if (!replayingHandoff || generation != replayGeneration) return; + replayCallbacksInFlight++; + } try { - messageHandler.onMessage(envelope); + handler.onMessage(envelope); } catch (RuntimeException replayFailure) { if (plugin != null) plugin.debug("Redis handoff replay failed: " + replayFailure.getMessage()); + } finally { + synchronized (legacyLifecycle) { + replayCallbacksInFlight--; + if (replayCallbacksInFlight < 0) replayCallbacksInFlight = 0; + legacyLifecycle.notifyAll(); + } } } } @@ -244,23 +581,193 @@ private void dispatchReplayBatch(java.util.List replay) { public void closeForHandoff() { handoffMessageHandler = messageHandler; fenceAfterHandoff(); + // A successful fence guarantees this listener will not dispatch the held + // callback. Release it before listener shutdown so the Redis listener thread + // can return and join promptly; the overlapping standby owns its duplicate. + completeFrozenReplayTransfer(); closeListener(false); } /** Reopens a fenced active subscriber when standby promotion is rejected. */ public void restoreAfterFailedHandoff() { + restoreAfterFailedHandoff(java.util.Collections.emptyList()); + } + + /** + * Reinstates the retired subscriber and prepends deliveries accepted by a + * promoted replacement before publication later failed. Standby overlap IDs + * are resolved while detaching them, so this FIFO can replay directly exactly + * once after the old subscriber is restored. + */ + public void restoreAfterFailedHandoff(java.util.List replacementReplay) { GlobalMessageHandler handler = handoffMessageHandler; if (handler == null) throw new IllegalStateException("Redis handoff transport cannot be restored"); + synchronized (legacyLifecycle) { + if (replacementReplay.size() > MAX_REPLAY_HANDOFF_DELIVERIES) + throw new IllegalStateException("Redis rollback replay exceeds its bounded handoff capacity"); + deliveriesAfterReplay.clear(); + deliveriesAfterReplay.addAll(replacementReplay); + replayDeliveriesInFlight = 0; + replayingHandoff = !replacementReplay.isEmpty(); + replayGeneration++; + retiredAfterHandoff = false; + } + processedVoteCache.restoreRedisSubscriber(subscriberIdentity); start(handler); + if (!replacementReplay.isEmpty()) replayAfterHandoffPublication(); handoffMessageHandler = null; } + /** Transfers pre-publication replay ownership to a restored predecessor. */ + public java.util.List detachReplayForFailedHandoff() { + synchronized (legacyLifecycle) { + if (replayDeliveriesInFlight != 0 || replayCallbacksInFlight != 0) + throw new IllegalStateException("Redis rollback cannot detach a replay batch already executing"); + java.util.ArrayList buffered = new java.util.ArrayList<>( + bufferedIdentifiedDeliveries.size() + bufferedLegacyDeliveries.size()); + buffered.addAll(bufferedIdentifiedDeliveries); + buffered.addAll(bufferedLegacyDeliveries); + buffered.sort(java.util.Comparator.comparingLong(BufferedHandoffDelivery::sequence)); + if (deliveriesAfterReplay.size() + buffered.size() > MAX_REPLAY_HANDOFF_DELIVERIES) + throw new IllegalStateException("Redis rollback replay exceeds its bounded handoff capacity"); + java.util.ArrayList pending = new java.util.ArrayList<>( + deliveriesAfterReplay.size() + buffered.size()); + pending.addAll(deliveriesAfterReplay); + for (BufferedHandoffDelivery delivery : buffered) { + boolean replay; + try { + replay = delivery.identified() + ? processedVoteCache.reserveRedisDelivery(delivery.identity()) + : !processedVoteCache.consumeLegacyRedisDelivery(delivery.identity()); + } catch (RuntimeException cacheFailure) { + // A rollback must keep an accepted staged callback. Retain it for the + // restored subscriber rather than letting a transient dedupe failure + // turn replacement close into data loss. + replay = true; + if (plugin != null) plugin.debug("Redis rollback dedupe failed: " + cacheFailure.getMessage()); + } + if (replay) pending.add(delivery.envelope()); + } + retiredAfterHandoff = true; + replayingHandoff = false; + replayTransferFrozen = false; + replayGeneration++; + deliveriesAfterReplay.clear(); + bufferedIdentifiedDeliveries.clear(); + bufferedIdentifiedDeliveryBytes = 0; + bufferedLegacyDeliveries.clear(); + bufferedLegacyDeliveryBytes = 0; + identifiedHandoffOverflowed = false; + legacyHandoffOverflowed = false; + legacyHandoffDegraded = false; + replayTaskOutstanding = false; + replayTaskGeneration = -1L; + legacyLifecycle.notifyAll(); + return pending; + } + } + + /** + * Stops admitting new callbacks and transfers the remaining active replay FIFO + * to the next same-Redis standby. An already-started Bukkit batch is allowed + * to finish; an unstarted scheduled batch is returned to the deque first. + */ + public java.util.List freezeReplayForSuccessiveHandoff() { + boolean interrupted = false; + long deadline = System.nanoTime() + HANDOFF_QUIESCE_TIMEOUT_NANOS; + synchronized (legacyLifecycle) { + replayTransferFrozen = true; + retiredAfterHandoff = true; + Thread worker = replayWorker; + if (worker != null) worker.interrupt(); + while (replayDeliveriesInFlight != 0 || replayCallbacksInFlight != 0 || replayTaskOutstanding) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0L) { + resumeReplayAfterFailedSuccessiveHandoffLocked(); + if (interrupted) Thread.currentThread().interrupt(); + throw new HandoffQuiescenceException("Redis replay did not quiesce before successive handoff"); + } + try { + TimeUnit.NANOSECONDS.timedWait(legacyLifecycle, remaining); + } catch (InterruptedException waitInterrupted) { + interrupted = true; + resumeReplayAfterFailedSuccessiveHandoffLocked(); + Thread.currentThread().interrupt(); + throw new HandoffQuiescenceException("Interrupted while freezing Redis replay for successive handoff"); + } + } + java.util.ArrayList pending = new java.util.ArrayList<>(deliveriesAfterReplay); + deliveriesAfterReplay.clear(); + replayingHandoff = false; + replayGeneration++; + legacyLifecycle.notifyAll(); + if (interrupted) Thread.currentThread().interrupt(); + return pending; + } + } + + /** Prepends a predecessor's still-unplayed replay FIFO before this standby's own overlap buffer. */ + public void acceptReplayFromPreviousHandoff(java.util.List predecessorReplay) { + if (predecessorReplay == null || predecessorReplay.isEmpty()) return; + synchronized (legacyLifecycle) { + if (deliveriesAfterReplay.size() + replayDeliveriesInFlight + bufferedLegacyDeliveries.size() + + bufferedIdentifiedDeliveries.size() + predecessorReplay.size() + > MAX_REPLAY_HANDOFF_DELIVERIES) + throw new IllegalStateException("Redis replacement replay capacity is exhausted"); + for (JsonEnvelope envelope : predecessorReplay) deliveriesAfterReplay.addLast(envelope); + } + } + + /** Rolls back a not-yet-published predecessor transfer when its old listener cannot retire. */ + public void removeReplayFromPreviousHandoff(java.util.List predecessorReplay) { + if (predecessorReplay == null || predecessorReplay.isEmpty()) return; + synchronized (legacyLifecycle) { + if (deliveriesAfterReplay.size() < predecessorReplay.size()) + throw new IllegalStateException("Redis replacement replay transfer is incomplete"); + for (JsonEnvelope expected : predecessorReplay) { + JsonEnvelope actual = deliveriesAfterReplay.removeFirst(); + if (!java.util.Objects.equals(actual, expected)) + throw new IllegalStateException("Redis replacement replay FIFO changed during handoff rollback"); + } + } + } + + /** Finalizes a successful transfer and releases callbacks to the promoted standby overlap. */ + public void completeFrozenReplayTransfer() { + synchronized (legacyLifecycle) { + replayTransferFrozen = false; + legacyLifecycle.notifyAll(); + } + } + + /** Restores the frozen active FIFO when successor admission or retirement fails. */ + public void restoreFrozenReplayAfterFailedSuccessiveHandoff(java.util.List replay) { + synchronized (legacyLifecycle) { + if (!deliveriesAfterReplay.isEmpty()) + throw new IllegalStateException("Redis replay ownership changed while successive handoff was aborted"); + deliveriesAfterReplay.addAll(replay); + resumeReplayAfterFailedSuccessiveHandoffLocked(); + } + if (plugin != null) startReplayWorkerIfNeeded(); + } + + private void resumeReplayAfterFailedSuccessiveHandoffLocked() { + replayTransferFrozen = false; + retiredAfterHandoff = false; + if (!deliveriesAfterReplay.isEmpty()) replayingHandoff = true; + legacyLifecycle.notifyAll(); + } + /** Prevents a listener that misses its shutdown deadline from dispatching duplicates. */ void fenceAfterHandoff() { boolean interrupted = false; long deadline = System.nanoTime() + HANDOFF_QUIESCE_TIMEOUT_NANOS; synchronized (legacyLifecycle) { retiredAfterHandoff = true; + replayGeneration++; + Thread worker = replayWorker; + if (worker != null) worker.interrupt(); + replayWorker = null; while (dispatchesInFlight > 0) { long remaining = deadline - System.nanoTime(); if (remaining <= 0) break; @@ -279,6 +786,10 @@ void fenceAfterHandoff() { } replayingHandoff = false; deliveriesAfterReplay.clear(); + replayBackpressureFailureLogged = false; + replayTaskOutstanding = false; + replayTaskGeneration = -1L; + replayDeliveriesInFlight = 0; bufferedLegacyDeliveries.clear(); bufferedLegacyDeliveryBytes = 0; bufferedIdentifiedDeliveries.clear(); @@ -294,6 +805,7 @@ private void bufferIdentifiedDelivery(JsonEnvelope envelope, String deliveryId) int encodedBytes = ProcessedVoteCache.legacyRedisDeliveryBytes(signature); if (encodedBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_DELIVERY_BYTES && bufferedIdentifiedDeliveries.size() < MAX_IDENTIFIED_HANDOFF_DELIVERIES + && handoffDeliveryCount() < MAX_REPLAY_HANDOFF_DELIVERIES && bufferedIdentifiedDeliveryBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES - encodedBytes) { bufferedIdentifiedDeliveries.add(new BufferedHandoffDelivery( nextHandoffSequence++, envelope, deliveryId, true)); @@ -305,6 +817,12 @@ private void bufferIdentifiedDelivery(JsonEnvelope envelope, String deliveryId) } } + /** Includes inherited replay plus staged overlap entries so rollback remains bounded. */ + private int handoffDeliveryCount() { + return deliveriesAfterReplay.size() + replayDeliveriesInFlight + + bufferedIdentifiedDeliveries.size() + bufferedLegacyDeliveries.size(); + } + private record BufferedHandoffDelivery(long sequence, JsonEnvelope envelope, String identity, boolean identified) {} @@ -357,9 +875,28 @@ static DefaultJedisClientConfig buildValidationClientConfig(int database, String @Override public void close() { + cancelReplay(); closeListener(true); } + private void cancelReplay() { + synchronized (legacyLifecycle) { + retiredAfterHandoff = true; + replayGeneration++; + replayingHandoff = false; + replayTransferFrozen = false; + deliveriesAfterReplay.clear(); + replayBackpressureFailureLogged = false; + replayTaskOutstanding = false; + replayTaskGeneration = -1L; + replayDeliveriesInFlight = 0; + Thread worker = replayWorker; + replayWorker = null; + if (worker != null) worker.interrupt(); + legacyLifecycle.notifyAll(); + } + } + private void closeListener(boolean unregister) { Thread thread = listenerThread; if (redisHandler != null) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index c354240cd..247635968 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -108,6 +108,11 @@ public abstract class VotingPluginProxy { private static final long PRESENCE_BACKEND_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(90); private static final long CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10); private static final int MAX_PENDING_VOTE_PARTY_REWARDS = 1024; + private static final long HTTP_TRANSPORT_RECONCILIATION_DELAY_MILLIS = 100L; + // Acks run before SimpleAPI removes an entry. Keep one bounded, single-flight + // poll armed while a replacement is deferred so state cleared after an ack (or + // from the durable cache) cannot strand the old HTTP runtime indefinitely. + private static final long HTTP_TRANSPORT_RECONCILIATION_POLL_MILLIS = 1_000L; @Getter @Setter @@ -148,6 +153,10 @@ public abstract class VotingPluginProxy { private boolean timeVoteDeliveryRetryScheduled; private boolean cachedVoteDeliveryRetryScheduled; private boolean votePartyDeliveryRetryScheduled; + private boolean deferredHttpTransportReconciliation; + private boolean httpTransportReconciliationScheduled; + private boolean httpTransportReconciliationRunning; + private long httpTransportReconciliationGeneration; private long votePartyProxyCommandAttemptSequence; private long votePartyProxyCommandInFlight; @@ -2622,6 +2631,7 @@ public void onDisable() { /** Full runtime replacement waits for hosted workers; final proxy stop remains non-blocking. */ public void onDisable(boolean waitForHosted) { + invalidateDeferredHttpTransportReconciliation(); if (waitForHosted) { prepareForRuntimeReplacement(); } else { @@ -2861,9 +2871,14 @@ public void reloadFromControl() { } private void reloadRuntime(boolean restartControlServices) { + // A manual reload supersedes any nested deferred reload scheduled by an old + // runtime. The generation check prevents that task from rebuilding after + // shutdown or racing this explicit replacement. + invalidateDeferredHttpTransportReconciliation(); BungeeMethod configuredMethod = BungeeMethod.getByName(getConfig().getBungeeMethod()); if (configuredMethod == null) configuredMethod = BungeeMethod.PLUGINMESSAGING; method = retainHttpForPendingDeliveries(configuredMethod); + scheduleDeferredHttpTransportReconciliation(); warnUnsupportedDedicatedVotingProxyMode(); if (!restartControlServices && method == BungeeMethod.SOCKETS) { rebuildSocketClients(); @@ -2878,9 +2893,13 @@ private void reloadRuntime(boolean restartControlServices) { } private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod configuredMethod) { - if (configuredMethod == BungeeMethod.HTTP) return configuredMethod; + if (configuredMethod == BungeeMethod.HTTP) { + deferredHttpTransportReconciliation = false; + return configuredMethod; + } HttpProxyTransportServer transport = httpTransportServer; if (transport != null && httpTransportHasPendingDeliveries(transport)) { + deferredHttpTransportReconciliation = true; logSevere("Retaining HTTP transport until durable deliveries are acknowledged"); return BungeeMethod.HTTP; } @@ -2888,17 +2907,20 @@ private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod co try { if (httpQueueHasPersistedDeliveries( getDataFolderPlugin().toPath().resolve("http").resolve("outgoing-v1"))) { + deferredHttpTransportReconciliation = true; logSevere("Retaining HTTP transport until persisted deliveries are acknowledged"); return BungeeMethod.HTTP; } } catch (IOException unreadableQueue) { // An unreadable durable queue is not proof that it is empty. Reopen HTTP so // its normal bounded loader can validate or recover the state. + deferredHttpTransportReconciliation = true; logSevere("Retaining HTTP transport because its persisted delivery queue could not be inspected"); return BungeeMethod.HTTP; } } if (hasPendingCachedHttpDeliveries()) { + deferredHttpTransportReconciliation = true; logSevere("Retaining HTTP transport until cached deliveries are acknowledged"); return BungeeMethod.HTTP; } @@ -2907,11 +2929,13 @@ private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod co for (String server : servers) { Collection rewards = getVoteCachePendingVotePartyRewardIds(server); if (rewards != null && !rewards.isEmpty()) { + deferredHttpTransportReconciliation = true; logSevere("Retaining HTTP transport until pending vote-party rewards are acknowledged"); return BungeeMethod.HTTP; } } } + deferredHttpTransportReconciliation = false; return configuredMethod; } @@ -3137,7 +3161,7 @@ private void startHttpTransport() { httpTransportServer = new HttpProxyTransportServer( new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope, - this::acknowledgeVotePartyDelivery); + this::acknowledgeHttpDelivery); httpTransportServer.start(); logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" + httpTransportServer.port() + "; use /votingpluginproxy httpcode for each backend"); @@ -3179,6 +3203,102 @@ private synchronized void closeHttpTransport() { if (transport != null) transport.close(); } + /** + * Receives every durable HTTP acknowledgement, including non-vote-party + * deliveries. The SimpleAPI queue removes the acknowledged entry immediately + * after this callback returns, so defer inspection to the proxy scheduler. + */ + protected void acknowledgeHttpDelivery(String server, String deliveryId) throws IOException { + acknowledgeVotePartyDelivery(server, deliveryId); + scheduleDeferredHttpTransportReconciliation(); + } + + /** Rebuilds the proxy runtime only after the retained HTTP transport is proven empty. */ + private void invalidateDeferredHttpTransportReconciliation() { + synchronized (this) { + httpTransportReconciliationGeneration++; + httpTransportReconciliationScheduled = false; + // Older work is fenced by its generation and must not suppress the + // replacement runtime's own single-flight probe. + httpTransportReconciliationRunning = false; + } + } + + private void scheduleDeferredHttpTransportReconciliation() { + scheduleDeferredHttpTransportReconciliation(HTTP_TRANSPORT_RECONCILIATION_DELAY_MILLIS); + } + + private void scheduleDeferredHttpTransportReconciliation(long delayMillis) { + scheduleDeferredHttpTransportReconciliation(delayMillis, -1L); + } + + private void scheduleDeferredHttpTransportReconciliation(long delayMillis, long requiredGeneration) { + ScheduledExecutorService scheduler = getScheduler(); + if (scheduler == null) return; + long generation; + synchronized (this) { + if (requiredGeneration >= 0L && requiredGeneration != httpTransportReconciliationGeneration) return; + if (!deferredHttpTransportReconciliation || method != BungeeMethod.HTTP + || httpTransportReconciliationScheduled || httpTransportReconciliationRunning) return; + httpTransportReconciliationScheduled = true; + generation = httpTransportReconciliationGeneration; + } + try { + scheduler.schedule(() -> reconcileDeferredHttpTransport(generation), delayMillis, TimeUnit.MILLISECONDS); + } catch (RuntimeException unavailable) { + synchronized (this) { + if (generation == httpTransportReconciliationGeneration) + httpTransportReconciliationScheduled = false; + } + debug("Unable to schedule deferred HTTP transport reconciliation: " + unavailable.getMessage()); + } + } + + private void reconcileDeferredHttpTransport(long generation) { + BungeeMethod configuredMethod; + boolean stillPending; + synchronized (this) { + if (generation != httpTransportReconciliationGeneration) return; + httpTransportReconciliationScheduled = false; + if (!enabled || !deferredHttpTransportReconciliation || method != BungeeMethod.HTTP) return; + configuredMethod = BungeeMethod.getByName(getConfig().getBungeeMethod()); + if (configuredMethod == null) configuredMethod = BungeeMethod.PLUGINMESSAGING; + stillPending = retainHttpForPendingDeliveries(configuredMethod) == BungeeMethod.HTTP; + if (!stillPending) httpTransportReconciliationRunning = true; + } + if (stillPending) { + // The acknowledgement callback happens before queue removal. Re-arm the + // same single-flight probe so the final removal/cached-state clear is + // observed without relying on another inbound message. + scheduleDeferredHttpTransportReconciliation(HTTP_TRANSPORT_RECONCILIATION_POLL_MILLIS, generation); + return; + } + synchronized (this) { + if (generation != httpTransportReconciliationGeneration || !enabled) { + httpTransportReconciliationRunning = false; + return; + } + } + try { + // The concrete platform checks this generation again while holding its + // reload lock. That closes the gap between this scheduler callback and a + // manual platform reload/shutdown without introducing lock inversion. + reloadDeferredHttpTransportCore(generation); + } catch (RuntimeException reconciliationFailure) { + debug("Deferred HTTP transport reconciliation failed: " + reconciliationFailure.getMessage()); + } finally { + boolean retry; + synchronized (this) { + retry = generation == httpTransportReconciliationGeneration && enabled + && deferredHttpTransportReconciliation && method == BungeeMethod.HTTP; + if (generation == httpTransportReconciliationGeneration) + httpTransportReconciliationRunning = false; + } + if (retry) + scheduleDeferredHttpTransportReconciliation(HTTP_TRANSPORT_RECONCILIATION_POLL_MILLIS, generation); + } + } + public String createHttpConnectionCode(String serverId) { HttpEnrollmentAuthority authority = httpEnrollmentAuthority; if (method != BungeeMethod.HTTP || authority == null) { @@ -3241,6 +3361,16 @@ protected void scheduleVotePartyProxyCommandTimeout(Runnable timeout) { public abstract void reloadCore(boolean mysql); + /** Platform implementations recheck this under their reload lock before replacing the runtime. */ + protected void reloadDeferredHttpTransportCore(long generation) { + if (isDeferredHttpTransportGenerationCurrent(generation)) reloadCore(true); + } + + protected synchronized boolean isDeferredHttpTransportGenerationCurrent(long generation) { + return enabled && generation == httpTransportReconciliationGeneration + && httpTransportReconciliationRunning && method == BungeeMethod.HTTP; + } + /** Strict Control reload path; failures propagate so the caller can restore its backup. */ public abstract void reloadControlConfiguration() throws Exception; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java index 2118c6c62..727d7242b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java @@ -13,11 +13,12 @@ import java.util.Map.Entry; import java.util.Queue; import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -313,8 +314,8 @@ public void onPluginMessage(PluginMessageEvent ev) { * @param loadMysql true to fully reinitialize MySQL and rebuild the proxy * runtime, false for a config/runtime-only reload */ - public void reloadPlugin(boolean loadMysql) { - synchronized (reloadLock) { + public void reloadPlugin(boolean loadMysql) { + synchronized (reloadLock) { reloading = true; final String oldChannel = (config != null) ? config.getPluginMessageChannel() : null; @@ -486,8 +487,16 @@ public void reloadPlugin(boolean loadMysql) { try { getVotingPluginProxy().sendServerNameMessage(); } catch (Exception ignored) { - } - } + } + } + + /** Runs a scheduled deferred replacement only if it is still current while holding reloadLock. */ + private void reloadPluginIfCurrent(boolean loadMysql, BooleanSupplier stillCurrent) { + synchronized (reloadLock) { + if (!stillCurrent.getAsBoolean()) return; + reloadPlugin(loadMysql); + } + } /** * Creates a new VotingPluginProxy instance wired to this platform. @@ -777,12 +786,17 @@ public void warn(String message) { getLogger().warning(message); } - @Override + @Override public void reloadCore(boolean mysql) { - // mysql==true should do full reloadall behavior on the platform + // mysql==true should do full reloadall behavior on the platform reloadPlugin(mysql); } + @Override + protected void reloadDeferredHttpTransportCore(long generation) { + reloadPluginIfCurrent(true, () -> isDeferredHttpTransportGenerationCurrent(generation)); + } + @Override public void reloadControlConfiguration() throws Exception { config.loadControlConfiguration(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java index 6af9ac551..9a65c9b8c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java @@ -24,6 +24,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -570,6 +571,14 @@ public void reloadAllInternal(boolean loadMysql) { } } + /** Runs a scheduled deferred replacement only if it is still current while holding reloadLock. */ + private void reloadAllInternalIfCurrent(boolean loadMysql, BooleanSupplier stillCurrent) { + synchronized (reloadLock) { + if (!stillCurrent.getAsBoolean()) return; + reloadAllInternal(loadMysql); + } + } + /** * Create proxy runtime. * @@ -878,6 +887,11 @@ public void reloadCore(boolean mysql) { reloadAllInternal(mysql); } + @Override + protected void reloadDeferredHttpTransportCore(long generation) { + reloadAllInternalIfCurrent(true, () -> isDeferredHttpTransportGenerationCurrent(generation)); + } + @Override public void reloadControlConfiguration() throws Exception { config.loadControlConfiguration(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index e4feb74a2..8db7bd032 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; @@ -15,6 +16,8 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Field; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.bukkit.configuration.file.FileConfiguration; import org.junit.jupiter.api.Test; @@ -74,24 +77,107 @@ void keepsPreparedHttpQueueWithPreviousHandlerUntilPublicationSucceeds() throws } @Test - void restoresPreviousHandlerWhenRedisPromotionFails() throws Exception { + void failsRedisRetirementDuringWorkerValidationBeforeBukkitPublication() throws Exception { VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); BackendProxyHandler previous = mock(BackendProxyHandler.class); BackendProxyHandler replacement = mock(BackendProxyHandler.class); setBackendProxyHandler(plugin, previous); + when(previous.requiresRedisHandoff(replacement)).thenReturn(true); doAnswer(invocation -> { throw new IllegalStateException("promotion failed"); }) .when(previous).completeRedisHandoff(replacement); org.junit.jupiter.api.Assertions.assertThrows(IllegalStateException.class, - () -> plugin.completeBackendProxyHandlerRestart(restart(previous, replacement))); + () -> plugin.validateBackendProxyHandlerRestart(restart(previous, replacement), + System.nanoTime() + 1_000_000_000L)); assertSame(previous, plugin.getBackendProxyHandler()); verify(previous, never()).completeHttpHandoff(replacement); verify(previous, never()).close(); - org.mockito.InOrder rollback = org.mockito.Mockito.inOrder(replacement, previous); - rollback.verify(replacement).abortStagedInboundTo(previous); + verify(replacement, never()).abortStagedInboundTo(previous); + verify(replacement, never()).close(); + verify(previous, never()).refreshPresenceAfterFailedReplacement(); + } + + @Test + void restoresWorkerRetiredRedisListenerWhenPublicationIsAborted() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + when(previous.requiresRedisHandoff(replacement)).thenReturn(true); + VotingPluginMain.BackendProxyRestart restart = restart(previous, replacement); + + plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + 1_000_000_000L); + plugin.abortBackendProxyHandlerRestart(restart); + + verify(previous).completeRedisHandoff(replacement); + verify(replacement).close(); + verify(previous).restoreAfterFailedReplacement(replacement); + } + + @Test + void rollbackRestoresPromotedRedisReplayBeforeClosingReplacement() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + when(previous.requiresRedisHandoff(replacement)).thenReturn(true); + doThrow(new IllegalStateException("publication admission failed")) + .when(previous).completeHttpHandoff(replacement); + VotingPluginMain.BackendProxyRestart restart = restart(previous, replacement); + plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + TimeUnit.SECONDS.toNanos(1)); + + assertThrows(IllegalStateException.class, () -> plugin.completeBackendProxyHandlerRestart(restart)); + + org.mockito.InOrder rollback = org.mockito.Mockito.inOrder(previous, replacement); + rollback.verify(previous).restoreAfterFailedReplacement(replacement); rollback.verify(replacement).close(); - rollback.verify(previous).refreshPresenceAfterFailedReplacement(); + } + + @Test + void refusesRedisPublicationUntilWorkerHandoffWasValidated() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + when(previous.requiresRedisHandoff(replacement)).thenReturn(true); + + assertThrows(IllegalStateException.class, + () -> plugin.completeBackendProxyHandlerRestart(restart(previous, replacement))); + + assertSame(previous, plugin.getBackendProxyHandler()); + verify(previous, never()).completeRedisHandoff(replacement); + verify(replacement, never()).activateInboundMessages(); + } + + @Test + void abortWaitsForInProgressRedisHandoffAndRestoresItAfterValidation() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + when(previous.requiresRedisHandoff(replacement)).thenReturn(true); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + doAnswer(invocation -> { + entered.countDown(); + assertTrue(release.await(1, TimeUnit.SECONDS)); + return null; + }).when(previous).completeRedisHandoff(replacement); + VotingPluginMain.BackendProxyRestart restart = restart(previous, replacement); + Thread validation = new Thread(() -> plugin.validateBackendProxyHandlerRestart(restart, + System.nanoTime() + TimeUnit.SECONDS.toNanos(1))); + validation.start(); + assertTrue(entered.await(1, TimeUnit.SECONDS)); + + plugin.abortBackendProxyHandlerRestart(restart); + verify(replacement, never()).close(); + release.countDown(); + validation.join(TimeUnit.SECONDS.toMillis(1)); + assertFalse(validation.isAlive(), "validation must finish and perform the requested rollback"); + + verify(replacement).close(); + verify(previous).restoreAfterFailedReplacement(replacement); } @Test @@ -189,7 +275,7 @@ private VotingPluginMain.BackendProxyRestart restart(BackendProxyHandler previou Constructor constructor = VotingPluginMain.BackendProxyRestart.class .getDeclaredConstructor(BackendProxyHandler.class, BackendProxyHandler.class, boolean.class, boolean.class); constructor.setAccessible(true); - return constructor.newInstance(previous, replacement, false, true); + return constructor.newInstance(previous, replacement, false, false); } private void setBackendProxyHandler(VotingPluginMain plugin, BackendProxyHandler handler) throws Exception { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 0730aef42..b39a1cc4d 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -14,6 +14,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -692,7 +693,56 @@ void promotesRedisReplacementAfterOldListenerIsFencedEvenWhenRetirementFails() t assertDoesNotThrow(previous::close); assertDoesNotThrow(previous::close); - verify(oldTransport, times(2)).close(); + verify(oldTransport, timeout(1_000).times(2)).close(); + } + + @Test + void sameRedisHandoffBuffersOutboundSendsUntilPublicationAdmission() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + BackendProxyTransportManager previous = new BackendProxyTransportManager(plugin); + BackendProxyTransportManager replacement = new BackendProxyTransportManager(plugin); + RedisBackendProxyTransport oldTransport = mock(RedisBackendProxyTransport.class); + RedisBackendProxyTransport newTransport = mock(RedisBackendProxyTransport.class); + JsonEnvelope envelope = JsonEnvelope.builder("outbound-during-redis-handoff").build(); + when(newTransport.send(envelope)).thenReturn(true); + setField(previous, "transport", oldTransport); + setField(replacement, "transport", newTransport); + + // Validation fences the staged replacement before the old Redis listener is + // retired. Its send FIFO must be admitted only after publication succeeds. + replacement.beginPreparedTransportHandoff(); + previous.completeRedisHandoff(replacement); + previous.send(envelope); + + verify(oldTransport, never()).send(envelope); + verify(newTransport, never()).send(envelope); + + previous.completePreparedTransportHandoff(replacement); + + verify(newTransport, timeout(1_000).times(1)).send(envelope); + } + + @Test + void sameRedisHandoffRollbackDrainsBufferedOutboundSendsThroughRestoredListener() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + BackendProxyTransportManager previous = new BackendProxyTransportManager(plugin); + BackendProxyTransportManager replacement = new BackendProxyTransportManager(plugin); + RedisBackendProxyTransport oldTransport = mock(RedisBackendProxyTransport.class); + RedisBackendProxyTransport newTransport = mock(RedisBackendProxyTransport.class); + JsonEnvelope envelope = JsonEnvelope.builder("outbound-redis-rollback").build(); + when(oldTransport.send(envelope)).thenReturn(true); + setField(previous, "transport", oldTransport); + setField(replacement, "transport", newTransport); + + replacement.beginPreparedTransportHandoff(); + previous.completeRedisHandoff(replacement); + previous.send(envelope); + previous.restoreAfterFailedReplacement(replacement); + + verify(newTransport, never()).send(envelope); + verify(oldTransport, timeout(1_000).times(1)).send(envelope); } @Test @@ -729,7 +779,7 @@ void restoresOldRedisSubscriberWhenStandbyActivationFails() throws Exception { org.mockito.InOrder order = inOrder(oldTransport, newTransport); order.verify(oldTransport).closeForHandoff(); order.verify(newTransport).activateAfterHandoff(); - order.verify(oldTransport).restoreAfterFailedHandoff(); + order.verify(oldTransport).restoreAfterFailedHandoff(java.util.Collections.emptyList()); assertSame(oldTransport, transport(previous)); assertSame(newTransport, transport(replacement)); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java index d08f9eb1e..1d0b39d34 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java @@ -7,7 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -21,9 +21,12 @@ import org.junit.jupiter.api.Test; +import com.bencodez.simpleapi.scheduler.BukkitScheduler; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; +import com.bencodez.votingplugin.VotingPluginMain; import com.bencodez.votingplugin.backendproxy.cache.ProcessedVoteCache; +import com.bencodez.votingplugin.proxy.BungeeMethod; import com.bencodez.votingplugin.proxy.VotingPluginWire; import redis.clients.jedis.DefaultJedisClientConfig; @@ -279,16 +282,16 @@ void managerDoesNotHoldItsMonitorWhileWaitingForRedisCallbackReplies() throws Ex when(cache.reserveLegacyRedisDelivery(org.mockito.ArgumentMatchers.any(), org.mockito.ArgumentMatchers.anyString())).thenReturn(true); RedisBackendProxyTransport transport = spy(new RedisBackendProxyTransport(null, cache)); - doReturn(true).when(transport).send(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); setField(manager, "transport", transport); CountDownLatch callbackEntered = new CountDownLatch(1); CountDownLatch allowReply = new CountDownLatch(1); CountDownLatch callbackCompleted = new CountDownLatch(1); + JsonEnvelope reply = JsonEnvelope.builder("reply").build(); GlobalMessageHandler messages = mock(GlobalMessageHandler.class); doAnswer(invocation -> { callbackEntered.countDown(); allowReply.await(); - manager.send(JsonEnvelope.builder("reply").build()); + manager.send(reply); callbackCompleted.countDown(); return null; }).when(messages).onMessage(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); @@ -301,9 +304,14 @@ void managerDoesNotHoldItsMonitorWhileWaitingForRedisCallbackReplies() throws Ex java.util.concurrent.atomic.AtomicReference closeFailure = new java.util.concurrent.atomic.AtomicReference<>(); + BackendProxyTransportManager replacementManager = new BackendProxyTransportManager(null, cache); + RedisBackendProxyTransport replacementTransport = mock(RedisBackendProxyTransport.class); + when(replacementTransport.send(reply)).thenReturn(true); + setField(replacementManager, "transport", replacementTransport); + replacementManager.beginPreparedTransportHandoff(); Thread retirement = new Thread(() -> { try { - manager.closeRedisForHandoff(); + manager.closeRedisForHandoff(replacementManager); } catch (Throwable failure) { closeFailure.set(failure); } @@ -319,7 +327,13 @@ void managerDoesNotHoldItsMonitorWhileWaitingForRedisCallbackReplies() throws Ex assertFalse(retirement.isAlive()); assertFalse(delivery.isAlive()); assertNull(closeFailure.get()); - verify(transport).send(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); + // The callback could re-enter manager.send because retirement did not hold + // its monitor. The same-Redis fence retains that reply until the Bukkit + // publication path admits it to the staged replacement. + verify(transport, never()).send(reply); + verify(replacementTransport, never()).send(reply); + manager.completePreparedTransportHandoff(replacementManager); + verify(replacementTransport, org.mockito.Mockito.timeout(1_000).times(1)).send(reply); } finally { allowReply.countDown(); if (retirement.isAlive()) retirement.interrupt(); @@ -328,6 +342,41 @@ void managerDoesNotHoldItsMonitorWhileWaitingForRedisCallbackReplies() throws Ex } } + @Test + void activationFailureTransfersEveryBufferedStandbyOverlapBeforeReplacementClose() throws Exception { + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.reserveLegacyRedisDelivery(org.mockito.ArgumentMatchers.any(), + org.mockito.ArgumentMatchers.anyString())).thenReturn(false); + when(cache.reserveRedisDelivery("00000000-0000-0000-0000-000000001594")).thenReturn(true); + BackendProxyTransportManager previous = new BackendProxyTransportManager(null, cache); + BackendProxyTransportManager replacement = new BackendProxyTransportManager(null, cache); + RedisBackendProxyTransport oldTransport = mock(RedisBackendProxyTransport.class); + RedisBackendProxyTransport standby = spy(new RedisBackendProxyTransport(null, cache)); + JsonEnvelope legacy = JsonEnvelope.builder("legacy-overlap").build(); + JsonEnvelope identified = JsonEnvelope.builder("identified-overlap").build(); + setBoolean(standby, "standbySubscriber", true); + setField(standby, "messageHandler", mock(GlobalMessageHandler.class)); + standby.dispatchLegacy(legacy); + standby.dispatchIdentified(identified, "00000000-0000-0000-0000-000000001594"); + org.mockito.Mockito.doThrow(new IllegalStateException("standby promotion rejected")) + .when(standby).activateAfterHandoff(); + setField(previous, "transport", oldTransport); + setField(replacement, "transport", standby); + + assertThrows(IllegalStateException.class, () -> previous.completeRedisHandoff(replacement)); + + @SuppressWarnings("unchecked") + org.mockito.ArgumentCaptor> restored = + org.mockito.ArgumentCaptor.forClass(java.util.List.class); + verify(oldTransport, times(1)).restoreAfterFailedHandoff(restored.capture()); + assertEquals(java.util.List.of(legacy, identified), restored.getValue(), + "rollback must restore every staged overlap delivery in arrival order"); + + standby.close(); + assertTrue(standby.detachReplayForFailedHandoff().isEmpty(), + "closing the failed staged transport must not discard an untransferred delivery"); + } + @Test void legacyPromotionWaitsForReservedDeliveryBeforeReplayingStandbyCopy() throws Exception { ProcessedVoteCache cache = mock(ProcessedVoteCache.class); @@ -402,6 +451,363 @@ void replayCompletesBeforeNewlyAdmittedDeliveries() throws Exception { assertEquals(java.util.List.of(replayed, newer), order); } + @Test + void replayUsesBoundedBukkitBatchesFromItsDedicatedWorker() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + java.util.concurrent.atomic.AtomicInteger scheduled = new java.util.concurrent.atomic.AtomicInteger(); + doAnswer(invocation -> { + scheduled.incrementAndGet(); + ((Runnable) invocation.getArgument(1)).run(); + return null; + }).when(scheduler).runTask(org.mockito.ArgumentMatchers.same(plugin), + org.mockito.ArgumentMatchers.any(Runnable.class)); + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.consumeLegacyRedisDelivery(org.mockito.ArgumentMatchers.anyString())).thenReturn(false); + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(plugin, cache); + setBoolean(transport, "standbySubscriber", true); + CountDownLatch replayed = new CountDownLatch(RedisBackendProxyTransport.REPLAY_BATCH_SIZE + 1); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + doAnswer(invocation -> { + replayed.countDown(); + return null; + }).when(messages).onMessage(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); + setField(transport, "messageHandler", messages); + for (int index = 0; index <= RedisBackendProxyTransport.REPLAY_BATCH_SIZE; index++) + transport.dispatchLegacy(JsonEnvelope.builder("replay-" + index).build()); + + transport.activateAfterHandoff(); + transport.replayAfterHandoffPublication(); + + assertTrue(replayed.await(1, TimeUnit.SECONDS)); + assertEquals(2, scheduled.get(), "a replay larger than one batch must not run in one Bukkit task"); + } + + @Test + void fullReplayQueueBackpressuresNewCallbacksAndCloseWakesThemWithoutReserving() throws Exception { + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.reserveRedisDelivery("new-delivery")).thenReturn(true); + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, cache); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + setField(transport, "messageHandler", messages); + setBoolean(transport, "replayingHandoff", true); + Field queueField = RedisBackendProxyTransport.class.getDeclaredField("deliveriesAfterReplay"); + queueField.setAccessible(true); + @SuppressWarnings("unchecked") + java.util.ArrayDeque queue = (java.util.ArrayDeque) queueField.get(transport); + for (int index = 0; index < RedisBackendProxyTransport.MAX_REPLAY_HANDOFF_DELIVERIES; index++) + queue.addLast(JsonEnvelope.builder("queued-" + index).build()); + JsonEnvelope newer = JsonEnvelope.builder("new").build(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch returned = new CountDownLatch(1); + Thread callback = new Thread(() -> { + entered.countDown(); + transport.dispatchIdentified(newer, "new-delivery"); + returned.countDown(); + }); + callback.start(); + assertTrue(entered.await(1, TimeUnit.SECONDS)); + assertFalse(returned.await(100, TimeUnit.MILLISECONDS), + "a full replay queue must apply backpressure instead of overtaking replay FIFO"); + verify(cache, never()).reserveRedisDelivery("new-delivery"); + + transport.close(); + assertTrue(returned.await(1, TimeUnit.SECONDS), "close must wake a blocked Redis callback"); + callback.join(TimeUnit.SECONDS.toMillis(1)); + + verify(cache, never()).reserveRedisDelivery("new-delivery"); + verifyNoInteractions(messages); + } + + @Test + void timedOutPubSubCallbackIsRetriedUntilReplayCapacityReturns() throws Exception { + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.reserveRedisDelivery("received-delivery")).thenReturn(true); + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, cache); + setField(transport, "messageHandler", mock(GlobalMessageHandler.class)); + setBoolean(transport, "replayingHandoff", true); + java.util.ArrayDeque queue = replayQueue(transport); + for (int index = 0; index < RedisBackendProxyTransport.MAX_REPLAY_HANDOFF_DELIVERIES; index++) + queue.addLast(JsonEnvelope.builder("queued-" + index).build()); + JsonEnvelope received = JsonEnvelope.builder("received").build(); + CountDownLatch completed = new CountDownLatch(1); + Thread subscriber = new Thread(() -> { + transport.dispatchReceivedSubscriberEnvelope(received, "received-delivery"); + completed.countDown(); + }); + subscriber.start(); + // The first bounded wait must time out without returning the consumed + // Pub/Sub payload to Redis (which would lose it); the callback retries it. + Thread.sleep(TimeUnit.SECONDS.toMillis(3) + 200L); + verify(cache, never()).reserveRedisDelivery("received-delivery"); + Object lifecycle = lifecycle(transport); + synchronized (lifecycle) { + queue.removeFirst(); + lifecycle.notifyAll(); + } + + assertTrue(completed.await(1, TimeUnit.SECONDS)); + subscriber.join(TimeUnit.SECONDS.toMillis(1)); + verify(cache).reserveRedisDelivery("received-delivery"); + assertEquals(received, queue.getLast(), "the received payload must retain FIFO replay ownership"); + transport.close(); + } + + @Test + void prePublicationRollbackDetachesPromotedReplayInArrivalOrder() throws Exception { + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.reserveRedisDelivery("before-publication")).thenReturn(true); + when(cache.reserveRedisDelivery("during-publication")).thenReturn(true); + when(cache.consumeLegacyRedisDelivery(org.mockito.ArgumentMatchers.anyString())).thenReturn(false); + RedisBackendProxyTransport replacement = new RedisBackendProxyTransport(null, cache); + setBoolean(replacement, "standbySubscriber", true); + setField(replacement, "messageHandler", mock(GlobalMessageHandler.class)); + JsonEnvelope before = JsonEnvelope.builder("before").build(); + JsonEnvelope during = JsonEnvelope.builder("during").build(); + + replacement.dispatchIdentified(before, "before-publication"); + replacement.activateAfterHandoff(); + // This is the narrow interval after worker-side promotion but before Bukkit + // opens publication/replay. It has already reserved its Redis identity. + replacement.dispatchIdentified(during, "during-publication"); + + assertEquals(java.util.List.of(before, during), replacement.detachReplayForFailedHandoff()); + assertTrue(replayQueue(replacement).isEmpty()); + verify(cache).reserveRedisDelivery("during-publication"); + } + + @Test + void consecutiveRedisHandoffTransfersActiveReplayBeforeNewStandbyArrivals() throws Exception { + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.reserveRedisDelivery("new-overlap")).thenReturn(true); + RedisBackendProxyTransport previous = new RedisBackendProxyTransport(null, cache); + JsonEnvelope first = JsonEnvelope.builder("first").build(); + JsonEnvelope second = JsonEnvelope.builder("second").build(); + replayQueue(previous).addLast(first); + replayQueue(previous).addLast(second); + setBoolean(previous, "replayingHandoff", true); + + java.util.List transferred = previous.freezeReplayForSuccessiveHandoff(); + RedisBackendProxyTransport replacement = new RedisBackendProxyTransport(null, cache); + setBoolean(replacement, "standbySubscriber", true); + CopyOnWriteArrayList delivered = new CopyOnWriteArrayList<>(); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + doAnswer(invocation -> { + delivered.add(invocation.getArgument(0)); + return null; + }).when(messages).onMessage(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); + setField(replacement, "messageHandler", messages); + replacement.acceptReplayFromPreviousHandoff(transferred); + JsonEnvelope newer = JsonEnvelope.builder("newer").build(); + replacement.dispatchIdentified(newer, "new-overlap"); + replacement.activateAfterHandoff(); + replacement.replayAfterHandoffPublication(); + + assertEquals(java.util.List.of(first, second, newer), delivered, + "a later same-Redis handoff must preserve the active replay FIFO ahead of new overlap arrivals"); + } + + @Test + void callbackArrivingDuringFrozenTransferIsReleasedIntoRollbackReplay() throws Exception { + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.reserveRedisDelivery("during-freeze")).thenReturn(true); + RedisBackendProxyTransport previous = new RedisBackendProxyTransport(null, cache); + JsonEnvelope inherited = JsonEnvelope.builder("inherited").build(); + replayQueue(previous).addLast(inherited); + setBoolean(previous, "replayingHandoff", true); + java.util.List transferred = previous.freezeReplayForSuccessiveHandoff(); + JsonEnvelope duringFreeze = JsonEnvelope.builder("during-freeze").build(); + CountDownLatch completed = new CountDownLatch(1); + Thread callback = new Thread(() -> { + previous.dispatchIdentified(duringFreeze, "during-freeze"); + completed.countDown(); + }); + callback.start(); + assertFalse(completed.await(100, TimeUnit.MILLISECONDS), + "the old subscriber must retain a consumed callback until transfer resolves"); + + previous.restoreFrozenReplayAfterFailedSuccessiveHandoff(transferred); + assertTrue(completed.await(1, TimeUnit.SECONDS)); + callback.join(TimeUnit.SECONDS.toMillis(1)); + assertEquals(java.util.List.of(inherited, duringFreeze), new java.util.ArrayList<>(replayQueue(previous))); + verify(cache).reserveRedisDelivery("during-freeze"); + } + + @Test + void successfulFrozenHandoffReleasesBlockedCallbackBeforeListenerClose() throws Exception { + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + RedisBackendProxyTransport previous = new RedisBackendProxyTransport(null, cache); + setField(previous, "messageHandler", mock(GlobalMessageHandler.class)); + replayQueue(previous).addLast(JsonEnvelope.builder("inherited").build()); + setBoolean(previous, "replayingHandoff", true); + previous.freezeReplayForSuccessiveHandoff(); + CountDownLatch callbackDone = new CountDownLatch(1); + Thread callback = new Thread(() -> { + previous.dispatchIdentified(JsonEnvelope.builder("during-close").build(), "during-close"); + callbackDone.countDown(); + }); + callback.start(); + assertFalse(callbackDone.await(100, TimeUnit.MILLISECONDS)); + + Thread close = new Thread(previous::closeForHandoff); + close.start(); + close.join(TimeUnit.SECONDS.toMillis(1)); + assertFalse(close.isAlive(), "listener close must not wait for the frozen callback timeout"); + assertTrue(callbackDone.await(1, TimeUnit.SECONDS)); + callback.join(TimeUnit.SECONDS.toMillis(1)); + verify(cache, never()).reserveRedisDelivery("during-close"); + } + + @Test + void redisToNonRedisTransitionDefersUntilActiveReplayIsEmpty() throws Exception { + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + RedisBackendProxyTransport redis = new RedisBackendProxyTransport(null, cache); + JsonEnvelope pending = JsonEnvelope.builder("pending-replay").build(); + replayQueue(redis).addLast(pending); + setBoolean(redis, "replayingHandoff", true); + BackendProxyTransportManager manager = new BackendProxyTransportManager(null, cache); + setField(manager, "transport", redis); + + assertFalse(manager.prepareRedisReplayTransition(BungeeMethod.HTTP, System.nanoTime()), + "Redis-to-HTTP must defer rather than clearing active Redis replay"); + assertFalse(manager.prepareRedisReplayTransition(BungeeMethod.PLUGINMESSAGING, System.nanoTime()), + "Redis-to-plugin-messaging must apply the same replay ownership gate"); + assertEquals(java.util.List.of(pending), new java.util.ArrayList<>(replayQueue(redis))); + + setBoolean(redis, "replayingHandoff", false); + replayQueue(redis).clear(); + assertTrue(manager.prepareRedisReplayTransition(BungeeMethod.HTTP, + System.nanoTime() + TimeUnit.SECONDS.toNanos(1))); + assertTrue(manager.prepareRedisReplayTransition(BungeeMethod.PLUGINMESSAGING, + System.nanoTime() + TimeUnit.SECONDS.toNanos(1))); + } + + @Test + void retiredRedisCleanupDoesNotBlockManagerCloseOnPublicationThread() throws Exception { + BackendProxyTransportManager manager = new BackendProxyTransportManager(null, new ProcessedVoteCache()); + RedisBackendProxyTransport retired = mock(RedisBackendProxyTransport.class); + CountDownLatch closeEntered = new CountDownLatch(1); + CountDownLatch releaseClose = new CountDownLatch(1); + doAnswer(invocation -> { + closeEntered.countDown(); + releaseClose.await(1, TimeUnit.SECONDS); + return null; + }).when(retired).close(); + setField(manager, "retiredTransport", retired); + + long started = System.nanoTime(); + manager.close(); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started); + + assertTrue(elapsedMillis < 100L, + "publication-facing manager close must not join a retired Redis listener"); + assertTrue(closeEntered.await(1, TimeUnit.SECONDS)); + releaseClose.countDown(); + } + + @Test + void replayBatchIsRequeuedWhenBukkitSchedulingRejectsIt() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + CountDownLatch rejected = new CountDownLatch(1); + doAnswer(invocation -> { + rejected.countDown(); + throw new IllegalStateException("scheduler stopped"); + }).when(scheduler).runTask(org.mockito.ArgumentMatchers.same(plugin), + org.mockito.ArgumentMatchers.any(Runnable.class)); + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.consumeLegacyRedisDelivery(org.mockito.ArgumentMatchers.anyString())).thenReturn(false); + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(plugin, cache); + setBoolean(transport, "standbySubscriber", true); + setField(transport, "messageHandler", mock(GlobalMessageHandler.class)); + transport.dispatchLegacy(JsonEnvelope.builder("replay").build()); + transport.activateAfterHandoff(); + transport.replayAfterHandoffPublication(); + + assertTrue(rejected.await(1, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + while (replayTrackedDeliveries(transport) != 1 && System.nanoTime() < deadline) + Thread.sleep(10L); + assertEquals(1, replayTrackedDeliveries(transport), + "a rejected Bukkit task must return its removed batch to replay ownership"); + transport.close(); + } + + @Test + void interruptedReplayWorkerReturnsItsUnstartedBatch() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + CountDownLatch submitted = new CountDownLatch(1); + doAnswer(invocation -> { + submitted.countDown(); + return null; // emulate a Bukkit task that has not run yet + }).when(scheduler).runTask(org.mockito.ArgumentMatchers.same(plugin), + org.mockito.ArgumentMatchers.any(Runnable.class)); + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.consumeLegacyRedisDelivery(org.mockito.ArgumentMatchers.anyString())).thenReturn(false); + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(plugin, cache); + setBoolean(transport, "standbySubscriber", true); + setField(transport, "messageHandler", mock(GlobalMessageHandler.class)); + transport.dispatchLegacy(JsonEnvelope.builder("replay").build()); + transport.activateAfterHandoff(); + transport.replayAfterHandoffPublication(); + assertTrue(submitted.await(1, TimeUnit.SECONDS)); + + Field workerField = RedisBackendProxyTransport.class.getDeclaredField("replayWorker"); + workerField.setAccessible(true); + Thread worker = (Thread) workerField.get(transport); + assertTrue(worker != null); + worker.interrupt(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + while (replayTrackedDeliveries(transport) != 1 && System.nanoTime() < deadline) + Thread.sleep(10L); + assertEquals(1, replayTrackedDeliveries(transport), + "interrupting a worker before Bukkit executes must not lose its batch"); + transport.close(); + } + + @Test + void closeFencesAlreadyScheduledReplayAfterItsCurrentBukkitCallback() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + doAnswer(invocation -> { + ((Runnable) invocation.getArgument(1)).run(); + return null; + }).when(scheduler).runTask(org.mockito.ArgumentMatchers.same(plugin), + org.mockito.ArgumentMatchers.any(Runnable.class)); + ProcessedVoteCache cache = mock(ProcessedVoteCache.class); + when(cache.consumeLegacyRedisDelivery(org.mockito.ArgumentMatchers.anyString())).thenReturn(false); + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(plugin, cache); + setBoolean(transport, "standbySubscriber", true); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + doAnswer(invocation -> { + entered.countDown(); + release.await(1, TimeUnit.SECONDS); + return null; + }).when(messages).onMessage(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); + setField(transport, "messageHandler", messages); + transport.dispatchLegacy(JsonEnvelope.builder("first").build()); + transport.dispatchLegacy(JsonEnvelope.builder("second").build()); + transport.activateAfterHandoff(); + transport.replayAfterHandoffPublication(); + assertTrue(entered.await(1, TimeUnit.SECONDS)); + + transport.close(); + release.countDown(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + while (replayCallbacksInFlight(transport) != 0 && System.nanoTime() < deadline) + Thread.sleep(10L); + assertEquals(0, replayCallbacksInFlight(transport), + "the executing callback must remain accounted for after its generation is cancelled"); + verify(messages, times(1)).onMessage(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); + } + private static void setBoolean(Object target, String name, boolean value) throws Exception { Field field = target.getClass().getDeclaredField(name); field.setAccessible(true); @@ -414,6 +820,38 @@ private static void setField(Object target, String name, Object value) throws Ex field.set(target, value); } + private static int replayTrackedDeliveries(RedisBackendProxyTransport transport) throws Exception { + java.util.ArrayDeque queue = replayQueue(transport); + Field inFlightField = RedisBackendProxyTransport.class.getDeclaredField("replayDeliveriesInFlight"); + inFlightField.setAccessible(true); + Object lifecycle = lifecycle(transport); + synchronized (lifecycle) { + return queue.size() + inFlightField.getInt(transport); + } + } + + @SuppressWarnings("unchecked") + private static java.util.ArrayDeque replayQueue(RedisBackendProxyTransport transport) throws Exception { + Field queueField = RedisBackendProxyTransport.class.getDeclaredField("deliveriesAfterReplay"); + queueField.setAccessible(true); + return (java.util.ArrayDeque) queueField.get(transport); + } + + private static Object lifecycle(RedisBackendProxyTransport transport) throws Exception { + Field lifecycleField = RedisBackendProxyTransport.class.getDeclaredField("legacyLifecycle"); + lifecycleField.setAccessible(true); + return lifecycleField.get(transport); + } + + private static int replayCallbacksInFlight(RedisBackendProxyTransport transport) throws Exception { + Field callbacksField = RedisBackendProxyTransport.class.getDeclaredField("replayCallbacksInFlight"); + callbacksField.setAccessible(true); + Object lifecycle = lifecycle(transport); + synchronized (lifecycle) { + return callbacksField.getInt(transport); + } + } + private static boolean awaitRetiredAfterHandoff(RedisBackendProxyTransport transport) throws Exception { Field lifecycleField = RedisBackendProxyTransport.class.getDeclaredField("legacyLifecycle"); lifecycleField.setAccessible(true); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 0bafd6ad5..0016fd7dd 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -535,6 +535,95 @@ void transportReloadRetainsHttpForOrdinaryDurableQueueEntry() throws Exception { assertEquals(BungeeMethod.HTTP, votingPluginProxy.getMethod()); } + @Test + void deferredHttpTransportChangeReconcilesAfterTheFinalAcknowledgement() throws Exception { + HttpProxyTransportServer transport = Mockito.mock(HttpProxyTransportServer.class); + setProxyField(votingPluginProxy, "httpTransportServer", transport); + java.util.concurrent.ScheduledExecutorService scheduler = java.util.concurrent.Executors + .newSingleThreadScheduledExecutor(); + try { + votingPluginProxy.setSchedulerForTest(scheduler); + votingPluginProxy.setPendingHttpTransportDeliveries(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + Mockito.when(votingPluginProxy.getConfig().getBungeeMethod()).thenReturn("REDIS"); + + votingPluginProxy.reloadFromControl(); + assertEquals(BungeeMethod.HTTP, votingPluginProxy.getMethod()); + + votingPluginProxy.setPendingHttpTransportDeliveries(false); + votingPluginProxy.acknowledgeHttpDeliveryForTest("Server1", "delivery"); + long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(2); + while (votingPluginProxy.getReloadCoreCalls() == 0 && System.nanoTime() < deadline) + Thread.sleep(10L); + + assertEquals(1, votingPluginProxy.getReloadCoreCalls(), + "the retained HTTP runtime must be rebuilt after its durable queue drains"); + } finally { + scheduler.shutdownNow(); + } + } + + @Test + void deferredHttpTransportChangePollsWhenAckPrecedesQueueRemoval() throws Exception { + HttpProxyTransportServer transport = Mockito.mock(HttpProxyTransportServer.class); + setProxyField(votingPluginProxy, "httpTransportServer", transport); + java.util.concurrent.ScheduledExecutorService scheduler = java.util.concurrent.Executors + .newSingleThreadScheduledExecutor(); + try { + votingPluginProxy.setSchedulerForTest(scheduler); + votingPluginProxy.setPendingHttpTransportDeliveries(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + Mockito.when(votingPluginProxy.getConfig().getBungeeMethod()).thenReturn("REDIS"); + + votingPluginProxy.reloadFromControl(); + // The acknowledgement callback is intentionally observed before SimpleAPI + // removes its entry, so the first reconciliation still sees it as pending. + votingPluginProxy.acknowledgeHttpDeliveryForTest("Server1", "delivery"); + Thread.sleep(200L); + assertEquals(0, votingPluginProxy.getReloadCoreCalls()); + + votingPluginProxy.setPendingHttpTransportDeliveries(false); + long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(3); + while (votingPluginProxy.getReloadCoreCalls() == 0 && System.nanoTime() < deadline) + Thread.sleep(10L); + + assertEquals(1, votingPluginProxy.getReloadCoreCalls(), + "the deferred runtime must observe queue removal even without another acknowledgement"); + } finally { + scheduler.shutdownNow(); + } + } + + @Test + void concurrentManualReloadInvalidatesQueuedDeferredHttpReload() throws Exception { + HttpProxyTransportServer transport = Mockito.mock(HttpProxyTransportServer.class); + setProxyField(votingPluginProxy, "httpTransportServer", transport); + java.util.concurrent.ScheduledExecutorService scheduler = java.util.concurrent.Executors + .newSingleThreadScheduledExecutor(); + try { + votingPluginProxy.setSchedulerForTest(scheduler); + votingPluginProxy.setPendingHttpTransportDeliveries(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + Mockito.when(votingPluginProxy.getConfig().getBungeeMethod()).thenReturn("REDIS"); + votingPluginProxy.reloadFromControl(); + + // A direct platform reload changes the configured transport while the old + // acknowledgement task is queued. It must invalidate that task before it + // can enter the platform's reload coordination path. + Mockito.when(votingPluginProxy.getConfig().getBungeeMethod()).thenReturn("HTTP"); + Thread manualReload = new Thread(votingPluginProxy::reloadFromControl); + manualReload.start(); + manualReload.join(java.util.concurrent.TimeUnit.SECONDS.toMillis(1)); + assertFalse(manualReload.isAlive()); + Thread.sleep(300L); + + assertEquals(0, votingPluginProxy.getReloadCoreCalls(), + "an invalidated deferred generation must not perform a stale full replacement"); + } finally { + scheduler.shutdownNow(); + } + } + @Test void startupRetainsHttpForPersistedQueueWithoutLiveServer() throws Exception { java.nio.file.Path backendQueue = temporaryDirectory.resolve("http/outgoing-v1/lobby-1"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index c0dede3c2..454ec190a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -51,6 +51,7 @@ public class VotingPluginProxyTestImpl extends VotingPluginProxy { private boolean declineNextVotePartyCommand; private Runnable votePartyProxyCommandTimeout; private Boolean pendingHttpTransportDeliveries; + private volatile int reloadCoreCalls; public List getWarnings() { return warnings; @@ -296,7 +297,11 @@ public void runVotePartyProxyCommandTimeoutForTest() { @Override public void reloadCore(boolean mysql) { - // Mocked for testing + reloadCoreCalls++; + } + + public int getReloadCoreCalls() { + return reloadCoreCalls; } @Override @@ -431,6 +436,10 @@ public void acknowledgeVotePartyDeliveryForTest(String server, String deliveryId acknowledgeVotePartyDelivery(server, deliveryId); } + public void acknowledgeHttpDeliveryForTest(String server, String deliveryId) throws java.io.IOException { + acknowledgeHttpDelivery(server, deliveryId); + } + @Override protected boolean sendCommunicationTestEnvelopeNow(String server, JsonEnvelope envelope) { lastCommunicationTestEnvelope = envelope; From bf8de86a7778ecaaab7bb72edb01e19651e896ea Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Thu, 10 Sep 2026 01:53:22 -0600 Subject: [PATCH 062/136] Preserve proxy runtime during deferred handoffs --- .../votingplugin/VotingPluginMain.java | 32 +++++++++- .../transport/RedisBackendProxyTransport.java | 3 + .../control/BackendControlConnector.java | 17 ++++-- .../votingplugin/proxy/VotingPluginProxy.java | 34 ++++++++++- .../proxy/bungee/VotingPluginBungee.java | 36 +++++++++-- .../proxy/velocity/VotingPluginVelocity.java | 25 ++++++++ ...PluginMainBackendProxyPublicationTest.java | 60 ++++++++++++++++++- .../RedisBackendProxyTransportTest.java | 21 +++++++ .../tests/VotingPluginProxyTest.java | 31 ++++++++++ 9 files changed, 242 insertions(+), 17 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index ab51a51cb..e4f92c7dd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1414,7 +1414,8 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { throw handoffFailure; } } - try { + if (restart.redisHandoffCompleted) closeStagedRedisReplacementAsync(restart.replacement); + else try { restart.replacement.close(); } catch (RuntimeException closeFailure) { handoffFailure.addSuppressed(closeFailure); @@ -1494,7 +1495,8 @@ public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart res } if (restart.replacement != null) { restart.replacement.abortStagedInboundTo(restart.previous); - restart.replacement.close(); + if (restart.redisHandoffCompleted) closeStagedRedisReplacementAsync(restart.replacement); + else restart.replacement.close(); } if (backendProxyHandler == restart.previous && restart.previous != null && (restart.previousPrepared || restart.redisHandoffCompleted @@ -1504,6 +1506,20 @@ public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart res restart.finished = true; } + /** A staged Redis listener can spend its bounded join timeout in close(); abort runs on Bukkit. */ + private void closeStagedRedisReplacementAsync(BackendProxyHandler replacement) { + Thread cleanup = new Thread(() -> { + try { + replacement.close(); + } catch (RuntimeException cleanupFailure) { + getLogger().warning("Staged Redis backend proxy handler did not stop cleanly after rollback"); + debug(cleanupFailure); + } + }, "VotingPlugin-Staged-Redis-Rollback"); + cleanup.setDaemon(true); + cleanup.start(); + } + public void awaitBackendProxyHandlerRollback(BackendProxyRestart restart, long deadlineNanos) { if (restart != null && restart.previousPrepared) { restart.previous.awaitRestoreAfterFailedReplacement(deadlineNanos); @@ -1512,10 +1528,20 @@ public void awaitBackendProxyHandlerRollback(BackendProxyRestart restart, long d /** Applies only proxy communication settings without reloading unrelated Bukkit configuration. */ public synchronized void reloadBackendProxyMethodFromControl() { + reloadBackendProxyMethodSettingsFromControl(); + restartBackendProxyHandler(); + } + + /** Bukkit-side narrow preparation for a proxy-method Control APPLY. */ + public synchronized BackendProxyRestart prepareBackendProxyMethodRestartFromControl() { + reloadBackendProxyMethodSettingsFromControl(); + return prepareBackendProxyHandlerRestart(); + } + + private void reloadBackendProxyMethodSettingsFromControl() { bungeeSettings.reloadData(); getOptions().setServer(bungeeSettings.getServer()); updateAdvancedCoreHook(); - restartBackendProxyHandler(); } /** Keeps one plugin-message listener for the plugin lifetime and atomically swaps its active backend handler. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index 070359dbb..e105d6e4c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -602,6 +602,9 @@ public void restoreAfterFailedHandoff() { public void restoreAfterFailedHandoff(java.util.List replacementReplay) { GlobalMessageHandler handler = handoffMessageHandler; if (handler == null) throw new IllegalStateException("Redis handoff transport cannot be restored"); + Thread retiredListener = listenerThread; + if (retiredListener != null && retiredListener.isAlive()) + throw new IllegalStateException("Redis handoff listener did not stop before rollback restart"); synchronized (legacyLifecycle) { if (replacementReplay.size() > MAX_REPLAY_HANDOFF_DELIVERIES) throw new IllegalStateException("Redis rollback replay exceeds its bounded handoff capacity"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index e8296a4fd..48738ef52 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -115,6 +115,11 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set } private void reloadConfiguration(String fileName) throws Exception { + reloadConfiguration(fileName, false); + } + + /** Reuses the split restart lifecycle while retaining proxy-method's narrow reload scope. */ + private void reloadConfiguration(String fileName, boolean proxyMethodOnly) throws Exception { finishPendingBackendProxyRollback(); long validationDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(29); AtomicBoolean preparationAbandoned = new AtomicBoolean(); @@ -128,11 +133,12 @@ private void reloadConfiguration(String fileName) throws Exception { preparation = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { try { if (!preparationState.compareAndSet(0, 1)) return null; - plugin.reloadFromControl(); + if (!proxyMethodOnly) plugin.reloadFromControl(); VotingPluginMain.BackendProxyRestart prepared; try { - prepared = "BungeeSettings.yml".equals(fileName) - ? plugin.prepareBackendProxyHandlerRestart() : null; + if (!"BungeeSettings.yml".equals(fileName)) prepared = null; + else if (proxyMethodOnly) prepared = plugin.prepareBackendProxyMethodRestartFromControl(); + else prepared = plugin.prepareBackendProxyHandlerRestart(); } catch (VotingPluginMain.BackendProxyRestartPreparationException failure) { preparedRestart.set(failure.restart()); throw failure; @@ -228,7 +234,10 @@ private void reloadConfiguration(String fileName) throws Exception { } private void reloadProxyMethod(String ignored) throws Exception { - reloadOnServerThread(plugin::reloadBackendProxyMethodFromControl); + // Proxy-method APPLY can select HTTP or Redis. Reuse the BungeeSettings + // split lifecycle so Bukkit only prepares/publishes while validation and + // bounded transport handoff waits remain on this connector worker. + reloadConfiguration("BungeeSettings.yml", true); } private void reloadOnServerThread(Runnable action) throws Exception { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 247635968..e6d1550d4 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -133,6 +133,9 @@ public abstract class VotingPluginProxy { private SocketHandler socketHandler; private HttpProxyTransportServer httpTransportServer; private HttpEnrollmentAuthority httpEnrollmentAuthority; + private String liveHttpHost; + private String liveHttpPublicEndpoint; + private int liveHttpPort; @Getter @Setter @@ -2893,7 +2896,7 @@ private void reloadRuntime(boolean restartControlServices) { } private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod configuredMethod) { - if (configuredMethod == BungeeMethod.HTTP) { + if (configuredMethod == BungeeMethod.HTTP && !hasChangedLiveHttpConfiguration()) { deferredHttpTransportReconciliation = false; return configuredMethod; } @@ -2939,6 +2942,12 @@ private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod co return configuredMethod; } + private boolean hasChangedLiveHttpConfiguration() { + return httpTransportServer != null && (!java.util.Objects.equals(liveHttpHost, getConfig().getHttpHost()) + || liveHttpPort != getConfig().getHttpPort() + || !java.util.Objects.equals(liveHttpPublicEndpoint, getConfig().getHttpPublicEndpoint())); + } + /** Uses SimpleAPI #80 when deployed while remaining safe with an older published snapshot. */ protected boolean httpTransportHasPendingDeliveries(HttpProxyTransportServer transport) { try { @@ -3163,6 +3172,9 @@ private void startHttpTransport() { httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope, this::acknowledgeHttpDelivery); httpTransportServer.start(); + liveHttpHost = getConfig().getHttpHost(); + liveHttpPort = getConfig().getHttpPort(); + liveHttpPublicEndpoint = getConfig().getHttpPublicEndpoint(); logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" + httpTransportServer.port() + "; use /votingpluginproxy httpcode for each backend"); } catch (Exception failure) { @@ -3200,6 +3212,9 @@ private synchronized void closeHttpTransport() { HttpProxyTransportServer transport = httpTransportServer; httpTransportServer = null; httpEnrollmentAuthority = null; + liveHttpHost = null; + liveHttpPort = 0; + liveHttpPublicEndpoint = null; if (transport != null) transport.close(); } @@ -3304,7 +3319,9 @@ public String createHttpConnectionCode(String serverId) { if (method != BungeeMethod.HTTP || authority == null) { throw new IllegalStateException("The HTTP transport is not running"); } - return authority.createConnectionCode(serverId, URI.create(getConfig().getHttpPublicEndpoint()), Duration.ofMinutes(15)) + String publicEndpoint = liveHttpPublicEndpoint != null + ? liveHttpPublicEndpoint : getConfig().getHttpPublicEndpoint(); + return authority.createConnectionCode(serverId, URI.create(publicEndpoint), Duration.ofMinutes(15)) .encode(); } @@ -3371,6 +3388,19 @@ protected synchronized boolean isDeferredHttpTransportGenerationCurrent(long gen && httpTransportReconciliationRunning && method == BungeeMethod.HTTP; } + /** True when a changed configured transport must not retire this runtime's live HTTP queue yet. */ + public synchronized boolean isRetainingHttpTransportForDeferredReconciliation() { + return deferredHttpTransportReconciliation && method == BungeeMethod.HTTP; + } + + /** An active HTTP runtime changing method or endpoint needs a pre-teardown retention probe. */ + public synchronized boolean requiresHttpRetentionCheckBeforeRuntimeReplacement() { + BungeeMethod configuredMethod = BungeeMethod.getByName(getConfig().getBungeeMethod()); + if (configuredMethod == null) configuredMethod = BungeeMethod.PLUGINMESSAGING; + return method == BungeeMethod.HTTP + && (configuredMethod != BungeeMethod.HTTP || hasChangedLiveHttpConfiguration()); + } + /** Strict Control reload path; failures propagate so the caller can restore its backup. */ public abstract void reloadControlConfiguration() throws Exception; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java index 727d7242b..111aaf689 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java @@ -378,11 +378,25 @@ public void reloadPlugin(boolean loadMysql) { return; } - // ========================= - // FULL RELOAD (WITH MYSQL) - // ========================= - - // Save caches best-effort before teardown + // ========================= + // FULL RELOAD (WITH MYSQL) + // ========================= + // Keep the old listener and its original runtime alive when a changed + // config selects a non-HTTP transport while its durable HTTP queue is not + // empty. Recreating first would make the new (possibly blank/changed) + // HTTP endpoint own that queue and can prevent it from draining. + if (votingPluginProxy != null + && votingPluginProxy.requiresHttpRetentionCheckBeforeRuntimeReplacement()) { + votingPluginProxy.reload(); + if (votingPluginProxy.isRetainingHttpTransportForDeferredReconciliation()) { + schedulePlatformTasks(); + reloading = false; + drainQueuedPluginMessagesAfterReloadLock(); + return; + } + } + + // Save caches best-effort before teardown try { if (voteCacheFile != null) { voteCacheFile.save(); @@ -490,6 +504,18 @@ public void reloadPlugin(boolean loadMysql) { } } + /** The retention branch returns from inside reloadLock; drain only after that lock is released. */ + private void drainQueuedPluginMessagesAfterReloadLock() { + Thread drain = new Thread(() -> { + synchronized (reloadLock) { + // Acquire/release establishes that the returning reload has left its lock. + } + drainQueuedPluginMessages(); + }, "VotingPlugin-Bungee-Reload-Queue-Drain"); + drain.setDaemon(true); + drain.start(); + } + /** Runs a scheduled deferred replacement only if it is still current while holding reloadLock. */ private void reloadPluginIfCurrent(boolean loadMysql, BooleanSupplier stillCurrent) { synchronized (reloadLock) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java index 9a65c9b8c..cad7d0f46 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java @@ -466,6 +466,19 @@ public void reloadAllInternal(boolean loadMysql) { // ========================= // FULL RELOAD (WITH MYSQL) // ========================= + // Preserve the current live HTTP listener while it owns retained durable + // deliveries. A fresh runtime would read the newly loaded endpoint, which + // may have been removed or changed before that FIFO is acknowledged. + if (votingPluginProxy != null + && votingPluginProxy.requiresHttpRetentionCheckBeforeRuntimeReplacement()) { + votingPluginProxy.reload(); + if (votingPluginProxy.isRetainingHttpTransportForDeferredReconciliation()) { + scheduleTasks(); + reloading = false; + drainQueuedPluginMessagesAfterReloadLock(); + return; + } + } // Best-effort save caches before shutdown try { @@ -571,6 +584,18 @@ public void reloadAllInternal(boolean loadMysql) { } } + /** The retention branch returns from inside reloadLock; drain only after that lock is released. */ + private void drainQueuedPluginMessagesAfterReloadLock() { + Thread drain = new Thread(() -> { + synchronized (reloadLock) { + // Acquire/release establishes that the returning reload has left its lock. + } + drainQueuedPluginMessages(); + }, "VotingPlugin-Velocity-Reload-Queue-Drain"); + drain.setDaemon(true); + drain.start(); + } + /** Runs a scheduled deferred replacement only if it is still current while holding reloadLock. */ private void reloadAllInternalIfCurrent(boolean loadMysql, BooleanSupplier stillCurrent) { synchronized (reloadLock) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index 8db7bd032..231e1fc19 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -22,12 +23,34 @@ import org.bukkit.configuration.file.FileConfiguration; import org.junit.jupiter.api.Test; +import com.bencodez.advancedcore.AdvancedCoreConfigOptions; import com.bencodez.votingplugin.backendproxy.BackendProxyHandler; import com.bencodez.votingplugin.config.BungeeSettings; import com.bencodez.votingplugin.config.Config; import com.bencodez.votingplugin.proxy.BungeeMethod; class VotingPluginMainBackendProxyPublicationTest { + @Test + void proxyMethodControlPreparationReloadsOnlyNarrowSettingsBeforeReturningTheStagedRestart() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + doCallRealMethod().when(plugin).prepareBackendProxyMethodRestartFromControl(); + BungeeSettings settings = mock(BungeeSettings.class); + AdvancedCoreConfigOptions options = mock(AdvancedCoreConfigOptions.class); + VotingPluginMain.BackendProxyRestart restart = mock(VotingPluginMain.BackendProxyRestart.class); + setField(plugin, "bungeeSettings", settings); + when(settings.getServer()).thenReturn("backend-1"); + when(plugin.getOptions()).thenReturn(options); + when(plugin.prepareBackendProxyHandlerRestart()).thenReturn(restart); + + assertSame(restart, plugin.prepareBackendProxyMethodRestartFromControl()); + + verify(settings).reloadData(); + verify(options).setServer("backend-1"); + verify(plugin).updateAdvancedCoreHook(); + verify(plugin).prepareBackendProxyHandlerRestart(); + verify(plugin, never()).reloadFromControl(); + } + @Test void publishesReplacementBeforeOpeningItsTransportGate() { VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); @@ -111,7 +134,7 @@ void restoresWorkerRetiredRedisListenerWhenPublicationIsAborted() throws Excepti plugin.abortBackendProxyHandlerRestart(restart); verify(previous).completeRedisHandoff(replacement); - verify(replacement).close(); + verify(replacement, org.mockito.Mockito.timeout(1_000).times(1)).close(); verify(previous).restoreAfterFailedReplacement(replacement); } @@ -131,7 +154,7 @@ void rollbackRestoresPromotedRedisReplayBeforeClosingReplacement() throws Except org.mockito.InOrder rollback = org.mockito.Mockito.inOrder(previous, replacement); rollback.verify(previous).restoreAfterFailedReplacement(replacement); - rollback.verify(replacement).close(); + rollback.verify(replacement, org.mockito.Mockito.timeout(1_000).times(1)).close(); } @Test @@ -176,10 +199,41 @@ void abortWaitsForInProgressRedisHandoffAndRestoresItAfterValidation() throws Ex validation.join(TimeUnit.SECONDS.toMillis(1)); assertFalse(validation.isAlive(), "validation must finish and perform the requested rollback"); - verify(replacement).close(); + verify(replacement, org.mockito.Mockito.timeout(1_000).times(1)).close(); verify(previous).restoreAfterFailedReplacement(replacement); } + @Test + void stagedRedisAbortFencesOnBukkitButClosesTheReplacementOffThread() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + VotingPluginMain.BackendProxyRestart restart = restart(previous, replacement); + Field completed = VotingPluginMain.BackendProxyRestart.class.getDeclaredField("redisHandoffCompleted"); + completed.setAccessible(true); + completed.setBoolean(restart, true); + CountDownLatch closeStarted = new CountDownLatch(1); + CountDownLatch releaseClose = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference closeThread = new java.util.concurrent.atomic.AtomicReference<>(); + doAnswer(invocation -> { + closeThread.set(Thread.currentThread().getName()); + closeStarted.countDown(); + releaseClose.await(1, TimeUnit.SECONDS); + return null; + }).when(replacement).close(); + + try { + org.junit.jupiter.api.Assertions.assertTimeoutPreemptively(java.time.Duration.ofMillis(250), + () -> plugin.abortBackendProxyHandlerRestart(restart)); + assertTrue(closeStarted.await(1, TimeUnit.SECONDS)); + assertTrue(closeThread.get().startsWith("VotingPlugin-Staged-Redis-Rollback")); + } finally { + releaseClose.countDown(); + } + verify(replacement, org.mockito.Mockito.timeout(1_000)).close(); + } + @Test void restoresPreviousHandlerWhenPreparedDeliveryAdmissionFails() throws Exception { VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java index 1d0b39d34..c937e9b33 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java @@ -377,6 +377,21 @@ void activationFailureTransfersEveryBufferedStandbyOverlapBeforeReplacementClose "closing the failed staged transport must not discard an untransferred delivery"); } + @Test + void rollbackDoesNotRestartAListenerThatMissedItsHandoffShutdownDeadline() throws Exception { + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, mock(ProcessedVoteCache.class)); + Thread retiredListener = mock(Thread.class); + when(retiredListener.isAlive()).thenReturn(true); + setField(transport, "listenerThread", retiredListener); + setField(transport, "handoffMessageHandler", mock(GlobalMessageHandler.class)); + setField(transport, "retiredAfterHandoff", true); + + assertThrows(IllegalStateException.class, () -> transport.restoreAfterFailedHandoff(java.util.List.of())); + verify(retiredListener).isAlive(); + assertTrue(getBoolean(transport, "retiredAfterHandoff"), + "a listener that missed shutdown must remain fenced after rollback is rejected"); + } + @Test void legacyPromotionWaitsForReservedDeliveryBeforeReplayingStandbyCopy() throws Exception { ProcessedVoteCache cache = mock(ProcessedVoteCache.class); @@ -814,6 +829,12 @@ private static void setBoolean(Object target, String name, boolean value) throws field.setBoolean(target, value); } + private static boolean getBoolean(Object target, String name) throws Exception { + Field field = target.getClass().getDeclaredField(name); + field.setAccessible(true); + return field.getBoolean(target); + } + private static void setField(Object target, String name, Object value) throws Exception { Field field = target.getClass().getDeclaredField(name); field.setAccessible(true); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 0016fd7dd..2fc3faae4 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -533,6 +533,37 @@ void transportReloadRetainsHttpForOrdinaryDurableQueueEntry() throws Exception { votingPluginProxy.reloadFromControl(); assertEquals(BungeeMethod.HTTP, votingPluginProxy.getMethod()); + assertTrue(votingPluginProxy.isRetainingHttpTransportForDeferredReconciliation(), + "a full platform reload must keep this runtime's live HTTP listener until its queue drains"); + assertTrue(votingPluginProxy.requiresHttpRetentionCheckBeforeRuntimeReplacement()); + } + + @Test + void ordinaryFullReloadsDoNotProbeHttpRetention() { + votingPluginProxy.setMethod(BungeeMethod.REDIS); + Mockito.when(votingPluginProxy.getConfig().getBungeeMethod()).thenReturn("HTTP"); + assertFalse(votingPluginProxy.requiresHttpRetentionCheckBeforeRuntimeReplacement()); + + votingPluginProxy.setMethod(BungeeMethod.HTTP); + Mockito.when(votingPluginProxy.getConfig().getBungeeMethod()).thenReturn("HTTP"); + assertFalse(votingPluginProxy.requiresHttpRetentionCheckBeforeRuntimeReplacement()); + } + + @Test + void changedHttpEndpointProbesRetentionButAnUnchangedListenerDoesNot() throws Exception { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + setProxyField(votingPluginProxy, "httpTransportServer", Mockito.mock(HttpProxyTransportServer.class)); + setProxyField(votingPluginProxy, "liveHttpHost", "127.0.0.1"); + setProxyField(votingPluginProxy, "liveHttpPort", 8080); + setProxyField(votingPluginProxy, "liveHttpPublicEndpoint", "https://old.example:8080"); + Mockito.when(votingPluginProxy.getConfig().getHttpHost()).thenReturn("127.0.0.1"); + Mockito.when(votingPluginProxy.getConfig().getHttpPort()).thenReturn(8080); + Mockito.when(votingPluginProxy.getConfig().getHttpPublicEndpoint()).thenReturn("https://old.example:8080"); + Mockito.when(votingPluginProxy.getConfig().getBungeeMethod()).thenReturn("HTTP"); + assertFalse(votingPluginProxy.requiresHttpRetentionCheckBeforeRuntimeReplacement()); + + Mockito.when(votingPluginProxy.getConfig().getHttpPublicEndpoint()).thenReturn("https://new.example:8443"); + assertTrue(votingPluginProxy.requiresHttpRetentionCheckBeforeRuntimeReplacement()); } @Test From 058e8e7aa9ffecdd2026faff2ce1d1a3e3445f80 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Thu, 10 Sep 2026 06:24:59 -0600 Subject: [PATCH 063/136] Fix deferred transport retirement --- .../votingplugin/VotingPluginMain.java | 51 +++++++++-------- .../backendproxy/BackendProxyHandler.java | 5 ++ .../BackendProxyTransportManager.java | 7 ++- .../votingplugin/proxy/VotingPluginProxy.java | 3 +- ...PluginMainBackendProxyPublicationTest.java | 57 ++++++++++++++++++- .../RedisBackendProxyTransportTest.java | 12 ++++ .../tests/VotingPluginProxyTest.java | 35 ++++++++++++ 7 files changed, 144 insertions(+), 26 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index e4f92c7dd..1020ab156 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1280,17 +1280,16 @@ private BackendProxyRestartPreparationException(BackendProxyRestart restart, Run public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { BackendProxyHandler previous = backendProxyHandler; if (!bungeeSettings.isUseBungeecoord()) { - boolean previousRequiresPreparation = previous != null && previous.requiresPreparationForReplacement(); + boolean previousRequiresPreparation = previous != null + && (previous.requiresPreparationForReplacement() || previous.requiresRedisRetirement()); return new BackendProxyRestart(previous, null, true, previousRequiresPreparation); } BungeeMethod replacementMethod = BungeeMethod.getByName(bungeeSettings.getBungeeMethod()); // Every transition away from an active HTTP transport must first drain its // durable outgoing queue. Restricting preparation to HTTP-to-HTTP swaps can // strand accepted deliveries when another transport is published. - boolean sameRedisHandoff = previous != null && previous.getMethod() == BungeeMethod.REDIS - && replacementMethod == BungeeMethod.REDIS; boolean previousRequiresPreparation = previous != null - && (previous.requiresPreparationForReplacement() || sameRedisHandoff); + && (previous.requiresPreparationForReplacement() || previous.requiresRedisRetirement()); BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); try { replacement.loadForReplacement(); @@ -1373,14 +1372,7 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { backendControlAutoEnrollment = null; restart.finished = true; restart.published = true; - if (restart.previous != null) { - try { - restart.previous.close(); - } catch (RuntimeException cleanupFailure) { - getLogger().warning("Previous backend proxy handler did not stop cleanly after disabling"); - debug(cleanupFailure); - } - } + closePublishedPreviousBackendProxyHandler(restart.previous, "after disabling"); if (enrollment != null) { try { enrollment.close(); @@ -1435,17 +1427,7 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { restart.replacement.replayRedisAfterHandoffPublication(); restart.finished = true; restart.published = true; - if (restart.previous != null) { - try { - restart.previous.close(); - } catch (RuntimeException cleanupFailure) { - // Publication and handoff are already committed. A predecessor that - // misses its shutdown deadline must not make Control roll back the - // replacement and close the only live handler. - getLogger().warning("Previous backend proxy handler did not stop cleanly after publication"); - debug(cleanupFailure); - } - } + closePublishedPreviousBackendProxyHandler(restart.previous, "after publication"); } try { refreshBackendControlAutoEnrollment(); @@ -1454,6 +1436,29 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { } } + /** Retires Redis predecessors asynchronously so listener joins never block Bukkit publication. */ + void closePublishedPreviousBackendProxyHandler(BackendProxyHandler previous, String phase) { + if (previous == null) return; + if (previous.requiresRedisRetirement()) { + Thread cleanup = new Thread(() -> closePublishedPreviousBackendProxyHandlerNow(previous, phase), + "VotingPlugin-Retired-Redis-Backend"); + cleanup.setDaemon(true); + cleanup.start(); + return; + } + closePublishedPreviousBackendProxyHandlerNow(previous, phase); + } + + private void closePublishedPreviousBackendProxyHandlerNow(BackendProxyHandler previous, String phase) { + try { + previous.close(); + } catch (RuntimeException cleanupFailure) { + // Publication is committed. Cleanup failure must not roll back the only live handler. + getLogger().warning("Previous backend proxy handler did not stop cleanly " + phase); + debug(cleanupFailure); + } + } + /** Publishes the handler before opening any transport callback or presence gate. */ void publishBackendProxyHandler(BackendProxyHandler previous, BackendProxyHandler replacement) { backendProxyHandler = replacement; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index db7532fc0..df65d80b0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -275,6 +275,11 @@ public void completeRedisHandoff(BackendProxyHandler replacement) { transportManager.completeRedisHandoff(replacement.transportManager); } + /** Returns whether this handler owns a Redis listener whose shutdown can block. */ + public boolean requiresRedisRetirement() { + return method == BungeeMethod.REDIS; + } + /** Returns whether this replacement needs the bounded same-Redis retirement path. */ public boolean requiresRedisHandoff(BackendProxyHandler replacement) { return replacement != null && method == BungeeMethod.REDIS && replacement.method == BungeeMethod.REDIS; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index e382e02b7..146061e52 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -550,7 +550,12 @@ public boolean prepareRedisReplayTransition(BungeeMethod replacementMethod, long if (replacementMethod == BungeeMethod.REDIS || !(transport instanceof RedisBackendProxyTransport)) return true; redis = (RedisBackendProxyTransport) transport; } - return redis.awaitReplayDrainForNonRedisReplacement(deadlineNanos); + if (!redis.awaitReplayDrainForNonRedisReplacement(deadlineNanos)) return false; + synchronized (this) { + if (transport != redis) return false; + preparedSendFence = true; + } + return true; } public java.util.List closeRedisForHandoff(BackendProxyTransportManager replacement) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index e6d1550d4..4acef33ea 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -3278,7 +3278,8 @@ private void reconcileDeferredHttpTransport(long generation) { if (!enabled || !deferredHttpTransportReconciliation || method != BungeeMethod.HTTP) return; configuredMethod = BungeeMethod.getByName(getConfig().getBungeeMethod()); if (configuredMethod == null) configuredMethod = BungeeMethod.PLUGINMESSAGING; - stillPending = retainHttpForPendingDeliveries(configuredMethod) == BungeeMethod.HTTP; + retainHttpForPendingDeliveries(configuredMethod); + stillPending = deferredHttpTransportReconciliation; if (!stillPending) httpTransportReconciliationRunning = true; } if (stillPending) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index 231e1fc19..22f25c480 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -121,6 +121,56 @@ void failsRedisRetirementDuringWorkerValidationBeforeBukkitPublication() throws verify(previous, never()).refreshPresenceAfterFailedReplacement(); } + @Test + void preparesRedisBeforePublishingAndRetiresItOffTheBukkitThread() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + when(previous.requiresRedisRetirement()).thenReturn(true); + when(previous.requiresRedisHandoff(replacement)).thenReturn(false); + when(replacement.getMethod()).thenReturn(BungeeMethod.HTTP); + when(previous.prepareForReplacement(org.mockito.ArgumentMatchers.eq(BungeeMethod.HTTP), + org.mockito.ArgumentMatchers.anyLong())).thenReturn(true); + CountDownLatch closed = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference closeThread = new java.util.concurrent.atomic.AtomicReference<>(); + doAnswer(invocation -> { + closeThread.set(Thread.currentThread().getName()); + closed.countDown(); + return null; + }).when(previous).close(); + VotingPluginMain.BackendProxyRestart restart = restart(previous, replacement, true); + + plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + TimeUnit.SECONDS.toNanos(1)); + + org.mockito.InOrder retirement = org.mockito.Mockito.inOrder(previous); + retirement.verify(previous).prepareForReplacement(org.mockito.ArgumentMatchers.eq(BungeeMethod.HTTP), + org.mockito.ArgumentMatchers.anyLong()); + verify(previous, never()).close(); + + plugin.closePublishedPreviousBackendProxyHandler(previous, "after publication"); + assertTrue(closed.await(1, TimeUnit.SECONDS)); + assertTrue(closeThread.get().startsWith("VotingPlugin-Retired-Redis-Backend")); + } + + @Test + void preparesRedisBeforePublishingDisabledState() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + when(previous.requiresRedisRetirement()).thenReturn(true); + when(previous.prepareForReplacement(org.mockito.ArgumentMatchers.isNull(), + org.mockito.ArgumentMatchers.anyLong())).thenReturn(true); + VotingPluginMain.BackendProxyRestart restart = restart(previous, null, true); + + plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + TimeUnit.SECONDS.toNanos(1)); + + org.mockito.InOrder retirement = org.mockito.Mockito.inOrder(previous); + retirement.verify(previous).prepareForReplacement(org.mockito.ArgumentMatchers.isNull(), + org.mockito.ArgumentMatchers.anyLong()); + verify(previous, never()).close(); + } + @Test void restoresWorkerRetiredRedisListenerWhenPublicationIsAborted() throws Exception { VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); @@ -326,10 +376,15 @@ void disablingKeepsPreviousHandlerWhenPreparationAcceptedAnotherDelivery() throw private VotingPluginMain.BackendProxyRestart restart(BackendProxyHandler previous, BackendProxyHandler replacement) throws Exception { + return restart(previous, replacement, false); + } + + private VotingPluginMain.BackendProxyRestart restart(BackendProxyHandler previous, + BackendProxyHandler replacement, boolean previousRequiresPreparation) throws Exception { Constructor constructor = VotingPluginMain.BackendProxyRestart.class .getDeclaredConstructor(BackendProxyHandler.class, BackendProxyHandler.class, boolean.class, boolean.class); constructor.setAccessible(true); - return constructor.newInstance(previous, replacement, false, false); + return constructor.newInstance(previous, replacement, replacement == null, previousRequiresPreparation); } private void setBackendProxyHandler(VotingPluginMain plugin, BackendProxyHandler handler) throws Exception { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java index c937e9b33..ffe3d9f06 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java @@ -696,6 +696,10 @@ void redisToNonRedisTransitionDefersUntilActiveReplayIsEmpty() throws Exception System.nanoTime() + TimeUnit.SECONDS.toNanos(1))); assertTrue(manager.prepareRedisReplayTransition(BungeeMethod.PLUGINMESSAGING, System.nanoTime() + TimeUnit.SECONDS.toNanos(1))); + JsonEnvelope duringCutover = JsonEnvelope.builder("during-cutover").build(); + manager.send(duringCutover); + assertEquals(java.util.List.of(duringCutover), new java.util.ArrayList<>(preparedSendQueue(manager)), + "outbound sends must be fenced until publication can transfer them to the replacement"); } @Test @@ -841,6 +845,14 @@ private static void setField(Object target, String name, Object value) throws Ex field.set(target, value); } + @SuppressWarnings("unchecked") + private static java.util.ArrayDeque preparedSendQueue(BackendProxyTransportManager manager) + throws Exception { + Field field = BackendProxyTransportManager.class.getDeclaredField("preparedSends"); + field.setAccessible(true); + return (java.util.ArrayDeque) field.get(manager); + } + private static int replayTrackedDeliveries(RedisBackendProxyTransport transport) throws Exception { java.util.ArrayDeque queue = replayQueue(transport); Field inFlightField = RedisBackendProxyTransport.class.getDeclaredField("replayDeliveriesInFlight"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 2fc3faae4..4439b69dc 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -594,6 +594,41 @@ void deferredHttpTransportChangeReconcilesAfterTheFinalAcknowledgement() throws } } + @Test + void deferredHttpEndpointChangeReconcilesAfterTheFinalAcknowledgement() throws Exception { + HttpProxyTransportServer transport = Mockito.mock(HttpProxyTransportServer.class); + setProxyField(votingPluginProxy, "httpTransportServer", transport); + setProxyField(votingPluginProxy, "liveHttpHost", "old.example"); + setProxyField(votingPluginProxy, "liveHttpPort", 8080); + setProxyField(votingPluginProxy, "liveHttpPublicEndpoint", "https://old.example:8080"); + java.util.concurrent.ScheduledExecutorService scheduler = java.util.concurrent.Executors + .newSingleThreadScheduledExecutor(); + try { + votingPluginProxy.setSchedulerForTest(scheduler); + votingPluginProxy.setPendingHttpTransportDeliveries(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + Mockito.when(votingPluginProxy.getConfig().getBungeeMethod()).thenReturn("HTTP"); + Mockito.when(votingPluginProxy.getConfig().getHttpHost()).thenReturn("new.example"); + Mockito.when(votingPluginProxy.getConfig().getHttpPort()).thenReturn(8443); + Mockito.when(votingPluginProxy.getConfig().getHttpPublicEndpoint()) + .thenReturn("https://new.example:8443"); + + votingPluginProxy.reloadFromControl(); + assertEquals(BungeeMethod.HTTP, votingPluginProxy.getMethod()); + + votingPluginProxy.setPendingHttpTransportDeliveries(false); + votingPluginProxy.acknowledgeHttpDeliveryForTest("Server1", "delivery"); + long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(2); + while (votingPluginProxy.getReloadCoreCalls() == 0 && System.nanoTime() < deadline) + Thread.sleep(10L); + + assertEquals(1, votingPluginProxy.getReloadCoreCalls(), + "a changed HTTP endpoint must replace the retained runtime after its queue drains"); + } finally { + scheduler.shutdownNow(); + } + } + @Test void deferredHttpTransportChangePollsWhenAckPrecedesQueueRemoval() throws Exception { HttpProxyTransportServer transport = Mockito.mock(HttpProxyTransportServer.class); From 59b5e75c9d82a616ae9509b4eafa496134c0d365 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:39:57 -0600 Subject: [PATCH 064/136] Preserve stopped presence during transport disable --- .../votingplugin/VotingPluginMain.java | 13 ++++ .../backendproxy/BackendProxyHandler.java | 17 +++++ .../presence/BackendPresenceManager.java | 15 +++- .../BackendProxyTransportManager.java | 23 ++++++ ...PluginMainBackendProxyPublicationTest.java | 31 +++++++- .../BackendProxyHandlerLifecycleTest.java | 71 +++++++++++++++++++ 6 files changed, 166 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 1020ab156..3c067b27d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1247,6 +1247,7 @@ public static final class BackendProxyRestart { private final boolean disabled; private final boolean previousRequiresPreparation; private volatile boolean previousPrepared; + private boolean presenceStoppedForDisablePreparation; // Redis listener retirement can wait for callbacks and listener shutdown. It is // completed by the Control worker during validation, before the final Bukkit // publication callback, and must be restored if that publication is abandoned. @@ -1307,6 +1308,10 @@ public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long // worker join fails. Mark the restart first so abort/await still owns the // restoration path after a partially completed preparation. restart.previousPrepared = true; + if (restart.disabled) { + restart.presenceStoppedForDisablePreparation = true; + restart.previous.preparePresenceForDisable(); + } BungeeMethod replacementMethod = restart.replacement == null ? null : restart.replacement.getMethod(); if (restart.replacement != null) restart.replacement.beginPreparedHttpHandoff(); if (!restart.previous.prepareForReplacement(replacementMethod, validationDeadlineNanos)) @@ -1365,6 +1370,10 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { } if (backendProxyHandler != restart.previous) throw new IllegalStateException("Backend proxy handler changed during restart"); if (restart.disabled) { + if (restart.previous != null && !restart.presenceStoppedForDisablePreparation) { + restart.presenceStoppedForDisablePreparation = true; + restart.previous.preparePresenceForDisable(); + } if (restart.previous != null && !restart.previous.commitPreparedDisable()) throw new IllegalStateException("Backend proxy transport accepted a delivery while disabling"); backendProxyHandler = null; @@ -1508,6 +1517,10 @@ public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart res || restart.previous.getMethod() == BungeeMethod.PLUGINMESSAGING)) { if (!restart.redisHandoffCompleted) restart.previous.restoreAfterFailedReplacement(); } + if (backendProxyHandler == restart.previous && restart.previous != null + && restart.presenceStoppedForDisablePreparation) { + restart.previous.restorePresenceAfterFailedDisablePreparation(); + } restart.finished = true; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index df65d80b0..64d464e2c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -224,6 +224,23 @@ public boolean commitPreparedDisable() { return transportManager.commitPreparedDisable(); } + /** Publishes the final presence update while fencing other sends before transport preparation. */ + public void preparePresenceForDisable() { + transportManager.beginPreparedDisable(); + if (presenceManager != null && presenceReportingActivated) { + // stopForDisable may reject when the transport cannot accept the final + // presence update. Mark this inactive first so rollback can start it again. + presenceReportingActivated = false; + presenceManager.stopForDisable(); + } + } + + /** Restores delivery and presence when a prepared disable is abandoned. */ + public void restorePresenceAfterFailedDisablePreparation() { + transportManager.cancelPreparedDisable(); + activatePresenceReporting(); + } + public void beginPreparedHttpHandoff() { transportManager.beginPreparedHttpHandoff(); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java index 9b560a3c3..6d479f3d8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java @@ -92,6 +92,15 @@ public void run() { } public void stop() { + stop(false); + } + + /** Stops presence and propagates rejection when a configuration disable must be transactional. */ + public void stopForDisable() { + stop(true); + } + + private void stop(boolean requireStoppedDelivery) { synchronized (lifecycleLock) { String activeServer = server; UUID activeIncarnationId = incarnationId; @@ -104,8 +113,10 @@ public void stop() { heartbeatTask = null; } if (wasReporting && activeServer != null && activeIncarnationId != null) { - send(VotingPluginWire.backendStopped(activeServer, activeIncarnationId, activeStartedAt, - nextTimestamp())); + JsonEnvelope stopped = VotingPluginWire.backendStopped(activeServer, activeIncarnationId, + activeStartedAt, nextTimestamp()); + if (requireStoppedDelivery) globalMessageHandler.sendMessage(stopped); + else send(stopped); } incarnationId = null; lastResyncRequestId = null; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 146061e52..3571fbd25 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -10,6 +10,7 @@ import com.bencodez.votingplugin.VotingPluginMain; import com.bencodez.votingplugin.backendproxy.cache.ProcessedVoteCache; import com.bencodez.votingplugin.proxy.BungeeMethod; +import com.bencodez.votingplugin.proxy.VotingPluginWire; /** * Selects and owns the active backend-to-proxy transport. @@ -36,6 +37,7 @@ public class BackendProxyTransportManager { private long handoffGeneration; private boolean preparedSendFence; private boolean rejectPreparedSends; + private boolean allowStoppedPresenceDuringDisable; private boolean preparedQueueWarning; private boolean rejectedSendWarning; private boolean asyncHandoffRetryWarning; @@ -82,6 +84,14 @@ public void start(BungeeMethod method, GlobalMessageHandler messageHandler, bool public synchronized void send(JsonEnvelope envelope) { if (rejectPreparedSends) { + if (allowStoppedPresenceDuringDisable && envelope != null + && VotingPluginWire.SUB_BACKEND_STOPPED.equals(envelope.getSubChannel())) { + allowStoppedPresenceDuringDisable = false; + BackendProxyTransport stoppingTransport = transport != null ? transport : preparedTransport; + if (stoppingTransport == null || !stoppingTransport.send(envelope)) + throw new IllegalStateException("Backend stopped presence was not accepted before disabling transport"); + return; + } if (!rejectedSendWarning) { rejectedSendWarning = true; plugin.getLogger().severe("Backend proxy transport is disabled; delivery was not accepted"); @@ -256,6 +266,19 @@ public synchronized boolean commitPreparedDisable() { return true; } + /** Fences ordinary sends while allowing one final stopped-presence envelope. */ + public synchronized void beginPreparedDisable() { + rejectPreparedSends = true; + allowStoppedPresenceDuringDisable = true; + } + + /** Reopens delivery when a prepared disable is rolled back. */ + public synchronized void cancelPreparedDisable() { + rejectPreparedSends = false; + allowStoppedPresenceDuringDisable = false; + rejectedSendWarning = false; + } + public synchronized boolean hasPendingAsyncHandoff() { return !asyncHandoffSends.isEmpty() || asyncHandoffWorker != null || pluginMessageHandoffScheduled; } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index 22f25c480..480982af0 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -347,9 +347,12 @@ void disablingBungeeCoordinationDrainsActiveHttpTransportBeforeClose() throws Ex plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + 1_000_000_000L); plugin.completeBackendProxyHandlerRestart(restart); - verify(previous).prepareForReplacement(org.mockito.ArgumentMatchers.isNull(), + org.mockito.InOrder disable = org.mockito.Mockito.inOrder(previous); + disable.verify(previous).preparePresenceForDisable(); + disable.verify(previous).prepareForReplacement(org.mockito.ArgumentMatchers.isNull(), org.mockito.ArgumentMatchers.anyLong()); - verify(previous).close(); + disable.verify(previous).commitPreparedDisable(); + disable.verify(previous).close(); assertNull(plugin.getBackendProxyHandler()); } @@ -369,9 +372,33 @@ void disablingKeepsPreviousHandlerWhenPreparationAcceptedAnotherDelivery() throw plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + 1_000_000_000L); assertThrows(IllegalStateException.class, () -> plugin.completeBackendProxyHandlerRestart(restart)); + plugin.abortBackendProxyHandlerRestart(restart); assertSame(previous, plugin.getBackendProxyHandler()); verify(previous, never()).close(); + verify(previous).restoreAfterFailedReplacement(); + verify(previous).restorePresenceAfterFailedDisablePreparation(); + } + + @Test + void disablingUnpreparedTransportStopsPresenceBeforeRejectingFurtherSends() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BungeeSettings settings = mock(BungeeSettings.class); + when(settings.isUseBungeecoord()).thenReturn(false); + setField(plugin, "bungeeSettings", settings); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + when(previous.commitPreparedDisable()).thenReturn(true); + setBackendProxyHandler(plugin, previous); + VotingPluginMain.BackendProxyRestart restart = plugin.prepareBackendProxyHandlerRestart(); + + plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + 1_000_000_000L); + plugin.completeBackendProxyHandlerRestart(restart); + + org.mockito.InOrder disable = org.mockito.Mockito.inOrder(previous); + disable.verify(previous).preparePresenceForDisable(); + disable.verify(previous).commitPreparedDisable(); + disable.verify(previous).close(); + assertNull(plugin.getBackendProxyHandler()); } private VotingPluginMain.BackendProxyRestart restart(BackendProxyHandler previous, diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index b39a1cc4d..237b53d57 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; import org.junit.jupiter.api.Test; @@ -239,6 +240,37 @@ void failedPresenceActivationDoesNotAnnounceReplacementGeneration() { verifyNoInteractions(messages); } + @Test + void preparedDisablePropagatesRejectedStoppedPresence() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + BackendPresenceManager presence = new BackendPresenceManager(plugin, BungeeMethod.HTTP, messages); + setField(presence, "reporting", true); + setField(presence, "server", "backend-1"); + setField(presence, "incarnationId", java.util.UUID.randomUUID()); + setField(presence, "startedAt", 1L); + doThrow(new IllegalStateException("outbound queue full")).when(messages).sendMessage(any()); + + assertThrows(IllegalStateException.class, presence::stopForDisable); + assertFalse(presence.isReporting()); + } + + @Test + void failedPreparedDisableCanRestartPresenceDuringRollback() throws Exception { + BackendProxyHandler handler = new BackendProxyHandler(null); + BackendPresenceManager presence = mock(BackendPresenceManager.class); + setField(handler, "presenceManager", presence); + setField(handler, "presenceReportingActivated", true); + doThrow(new IllegalStateException("outbound queue full")).when(presence).stopForDisable(); + + assertThrows(IllegalStateException.class, handler::preparePresenceForDisable); + handler.restorePresenceAfterFailedDisablePreparation(); + + org.mockito.InOrder rollback = inOrder(presence); + rollback.verify(presence).stopForDisable(); + rollback.verify(presence).start(); + } + @Test void stagedPresenceStartsOnlyAtExplicitPublication() throws Exception { BackendProxyHandler handler = new BackendProxyHandler(null); @@ -561,6 +593,45 @@ void preparedDisableRefusesToDiscardMessagesAcceptedDuringPreparation() throws E verify(transport, never()).close(); } + @Test + void preparedDisableRejectsOrdinarySendsButDeliversOnlyOneFinalStoppedPresence() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + when(plugin.getLogger()).thenReturn(Logger.getLogger(getClass().getName())); + BackendProxyTransportManager manager = new BackendProxyTransportManager(plugin); + BackendProxyTransport transport = mock(BackendProxyTransport.class); + setField(manager, "transport", transport); + JsonEnvelope ordinary = JsonEnvelope.builder("ordinary").build(); + JsonEnvelope stopped = com.bencodez.votingplugin.proxy.VotingPluginWire.backendStopped("backend-1"); + when(transport.send(stopped)).thenReturn(true); + + manager.beginPreparedDisable(); + manager.send(ordinary); + manager.send(stopped); + manager.send(stopped); + + verify(transport, never()).send(ordinary); + verify(transport, times(1)).send(stopped); + } + + @Test + void preparedHttpDisableDeliversTheFinalStoppedPresenceBeforeClosingForReplacement() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BackendProxyTransportManager manager = new BackendProxyTransportManager(plugin); + HttpBackendProxyTransport transport = mock(HttpBackendProxyTransport.class); + setField(manager, "transport", transport); + JsonEnvelope stopped = com.bencodez.votingplugin.proxy.VotingPluginWire.backendStopped("backend-1"); + when(transport.send(stopped)).thenReturn(true); + + manager.beginPreparedDisable(); + manager.send(stopped); + manager.prepareForReplacement(); + assertTrue(manager.commitPreparedDisable()); + + org.mockito.InOrder disable = inOrder(transport); + disable.verify(transport).send(stopped); + disable.verify(transport).prepareForReplacement(); + } + @Test void laterReplacementWaitsForAnEarlierAsyncHandoff() throws Exception { com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); From 18ff986162aad15d4fa8443d4cfb47d7e6a06496 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:36:45 -0600 Subject: [PATCH 065/136] Harden HTTP transition failure handling --- .../votingplugin/proxy/VotingPluginProxy.java | 142 +++++++- .../cache/ProxyOnlineVoteCacheTable.java | 13 +- .../proxy/cache/ProxyVoteCacheTable.java | 13 +- .../proxy/cache/VoteCacheHandler.java | 313 ++++++++++++++---- .../ProxyMethodConfigurationService.java | 11 +- .../ProxyMethodConfigurationServiceTest.java | 47 +++ .../tests/VoteCacheHandlerVoteIdTest.java | 16 + .../tests/VotingPluginProxyTest.java | 53 +++ .../tests/VotingPluginProxyTestImpl.java | 7 +- 9 files changed, 532 insertions(+), 83 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 4acef33ea..cdab46d6c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -12,6 +12,7 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.sql.SQLException; import java.time.Duration; import java.time.Instant; @@ -37,6 +38,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import javax.net.ssl.SSLParameters; @@ -113,6 +115,7 @@ public abstract class VotingPluginProxy { // poll armed while a replacement is deferred so state cleared after an ack (or // from the durable cache) cannot strand the old HTTP runtime indefinitely. private static final long HTTP_TRANSPORT_RECONCILIATION_POLL_MILLIS = 1_000L; + private static final Map PREPARED_HTTP_TRANSPORTS = new ConcurrentHashMap<>(); @Getter @Setter @@ -137,6 +140,35 @@ public abstract class VotingPluginProxy { private String liveHttpPublicEndpoint; private int liveHttpPort; + private static final class PreparedHttpTransport { + private final HttpProxyTransportServer server; + private final HttpEnrollmentAuthority authority; + private final AtomicReference owner; + private final String host; + private final int port; + private final String publicEndpoint; + + private PreparedHttpTransport(HttpProxyTransportServer server, HttpEnrollmentAuthority authority, + AtomicReference owner, String host, int port, String publicEndpoint) { + this.server = server; + this.authority = authority; + this.owner = owner; + this.host = host; + this.port = port; + this.publicEndpoint = publicEndpoint; + } + + private boolean matches(VotingPluginProxyConfig config) { + return java.util.Objects.equals(host, config.getHttpHost()) && port == config.getHttpPort() + && java.util.Objects.equals(publicEndpoint, config.getHttpPublicEndpoint()); + } + + private void close() { + owner.set(null); + server.close(); + } + } + @Getter @Setter private boolean votifierEnabled = true; @@ -2638,6 +2670,10 @@ public void onDisable(boolean waitForHosted) { if (waitForHosted) { prepareForRuntimeReplacement(); } else { + if (!quarantineInFlightVotePartyProxyCommandForReplacement()) { + logSevere("Unable to durably quarantine an in-flight HTTP vote-party command during final shutdown; the attempt remains fenced"); + } + cancelPreparedHttpTransportChange(); controlServicesGeneration.incrementAndGet(); controlLifecycleExecutor.shutdownNow(); stopControlServices(false); @@ -3157,21 +3193,26 @@ protected synchronized boolean sendHttpEnvelope(String server, String deliveryId private void startHttpTransport() { try { - URI endpoint = URI.create(getConfig().getHttpPublicEndpoint()); - if (!"https".equalsIgnoreCase(endpoint.getScheme()) || endpoint.getHost() == null - || endpoint.getPort() == 0 || endpoint.getPort() > 65535 - || endpoint.getUserInfo() != null || endpoint.getQuery() != null || endpoint.getFragment() != null - || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) { - throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); + PreparedHttpTransport prepared = PREPARED_HTTP_TRANSPORTS.remove(httpTransportPreparationKey()); + if (prepared != null) { + if (!prepared.matches(getConfig())) { + prepared.close(); + throw new IllegalStateException("Prepared HTTP transport does not match the installed configuration"); + } + httpTransportServer = prepared.server; + httpEnrollmentAuthority = prepared.authority; + prepared.owner.set(this); + } else { + URI endpoint = validatedHttpEndpoint(getConfig().getHttpPublicEndpoint()); + File directory = new File(getDataFolderPlugin(), "http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.toPath(), endpoint.getHost()); + httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); + httpTransportServer = new HttpProxyTransportServer( + new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, + httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), + this::handleHttpTransportEnvelope, this::acknowledgeHttpDelivery); + httpTransportServer.start(); } - File directory = new File(getDataFolderPlugin(), "http"); - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.toPath(), endpoint.getHost()); - httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); - httpTransportServer = new HttpProxyTransportServer( - new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, - httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope, - this::acknowledgeHttpDelivery); - httpTransportServer.start(); liveHttpHost = getConfig().getHttpHost(); liveHttpPort = getConfig().getHttpPort(); liveHttpPublicEndpoint = getConfig().getHttpPublicEndpoint(); @@ -3218,6 +3259,66 @@ private synchronized void closeHttpTransport() { if (transport != null) transport.close(); } + /** + * Starts the candidate HTTP listener before Control publishes an HTTP method + * change. The replacement runtime adopts this listener, avoiding a bind/TLS/ + * queue failure after the old runtime has already been torn down. + */ + public synchronized void prepareHttpTransportChange(VotingPluginProxyConfig candidate) { + cancelPreparedHttpTransportChange(); + PreparedHttpTransport prepared = createPreparedHttpTransport(candidate); + PREPARED_HTTP_TRANSPORTS.put(httpTransportPreparationKey(), prepared); + } + + /** Cancels a candidate listener when configuration publication fails. */ + public synchronized void cancelPreparedHttpTransportChange() { + PreparedHttpTransport prepared = PREPARED_HTTP_TRANSPORTS.remove(httpTransportPreparationKey()); + if (prepared != null) prepared.close(); + } + + private PreparedHttpTransport createPreparedHttpTransport(VotingPluginProxyConfig candidate) { + HttpProxyTransportServer server = null; + try { + URI endpoint = validatedHttpEndpoint(candidate.getHttpPublicEndpoint()); + File directory = new File(getDataFolderPlugin(), "http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.toPath(), endpoint.getHost()); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.toPath()); + AtomicReference owner = new AtomicReference<>(); + server = new HttpProxyTransportServer( + new InetSocketAddress(candidate.getHttpHost(), candidate.getHttpPort()), identity, authority, + directory.toPath().resolve("outgoing-v1"), received -> { + VotingPluginProxy active = owner.get(); + if (active == null) throw new IllegalStateException("HTTP runtime replacement is not active"); + active.handleHttpTransportEnvelope(received); + }, (backend, deliveryId) -> { + VotingPluginProxy active = owner.get(); + if (active == null) throw new IOException("HTTP runtime replacement is not active"); + active.acknowledgeHttpDelivery(backend, deliveryId); + }); + server.start(); + return new PreparedHttpTransport(server, authority, owner, candidate.getHttpHost(), candidate.getHttpPort(), + candidate.getHttpPublicEndpoint()); + } catch (Exception failure) { + if (server != null) server.close(); + throw new IllegalStateException("HTTP transport could not be prepared securely", failure); + } + } + + private Path httpTransportPreparationKey() { + return getDataFolderPlugin().toPath().toAbsolutePath().normalize(); + } + + private URI validatedHttpEndpoint(String publicEndpoint) { + URI endpoint = URI.create(publicEndpoint); + if (!"https".equalsIgnoreCase(endpoint.getScheme()) || endpoint.getHost() == null + || endpoint.getPort() == 0 || endpoint.getPort() > 65535 + || endpoint.getUserInfo() != null || endpoint.getQuery() != null || endpoint.getFragment() != null + || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) { + throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); + } + return endpoint; + } + /** * Receives every durable HTTP acknowledgement, including non-vote-party * deliveries. The SimpleAPI queue removes the acknowledged entry immediately @@ -4397,9 +4498,13 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea VotingPluginWire.vote(player, uuid, service, time, true, realVote, text.toString(), voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote); if (!rewardAccepted) { + OfflineBungeeVote fallbackVote = standaloneProxyBroadcast + ? createCachedRewardVote(pendingVote, true) : pendingVote; + if (!getVoteCacheHandler().addServerVoteDurably(s, fallbackVote)) { + logSevere("Unable to durably cache the rejected vote delivery for " + s); + return QueuedVoteResult.RETRY; + } voteStatus = VoteLogStatus.CACHED; - getVoteCacheHandler().addServerVote(s, standaloneProxyBroadcast - ? createCachedRewardVote(pendingVote, true) : pendingVote); debug("Caching vote after the transport rejected delivery for " + s); } else if (standaloneBroadcastState != null) { standaloneBroadcastState.setRewardDelivered(true); @@ -4428,8 +4533,11 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(), voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote); if (!rewardAccepted) { + if (!getVoteCacheHandler().addOnlineVoteDurably(uuid, pendingVote)) { + logSevere("Unable to durably cache the rejected online vote delivery for " + uuid); + return QueuedVoteResult.RETRY; + } voteStatus = VoteLogStatus.CACHED; - getVoteCacheHandler().addOnlineVote(uuid, pendingVote); standaloneBroadcastStatePersisted |= standaloneBroadcastState != null; debug("Caching online vote after the transport rejected delivery for " + server); } else if (standaloneBroadcastState != null) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java index 8f7092ae0..d8dd31b8d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java @@ -326,6 +326,16 @@ public void insertVote(UUID voteId, String uuid, String playerName, String servi String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, boolean rewardDelivered, String httpDeliveryIds, String httpBroadcastDeliveryIds) { + tryInsertVote(voteId, uuid, playerName, service, time, real, text, broadcastForwarded, + proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, rewardDelivered, + httpDeliveryIds, httpBroadcastDeliveryIds); + } + + /** Inserts a vote and reports whether the database accepted it. */ + public boolean tryInsertVote(UUID voteId, String uuid, String playerName, String service, long time, boolean real, + String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, String broadcastTargets, + String broadcastForwardedServers, boolean rewardDelivered, String httpDeliveryIds, + String httpBroadcastDeliveryIds) { String sql = "INSERT INTO " + qi(getTableName()) + " (" + qi("uuid") + ", " + qi("voteid") + ", " + qi("playerName") + ", " + qi("service") + ", " + qi("time") + ", " + qi("realVote") + ", " @@ -369,9 +379,10 @@ public void insertVote(UUID voteId, String uuid, String playerName, String servi ps.setString(13, httpDeliveryIds); ps.setString(14, httpBroadcastDeliveryIds); - ps.executeUpdate(); + return ps.executeUpdate() > 0; } catch (SQLException | IllegalArgumentException e) { debug(e); + return false; } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java index 9904df3f3..f2bd62c9d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java @@ -344,6 +344,16 @@ public void insertVote(UUID voteId, String uuid, String playerName, String servi String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, boolean rewardDelivered, String httpDeliveryIds, String httpBroadcastDeliveryIds, String server) { + tryInsertVote(voteId, uuid, playerName, service, time, real, text, broadcastForwarded, + proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, rewardDelivered, + httpDeliveryIds, httpBroadcastDeliveryIds, server); + } + + /** Inserts a vote and reports whether the database accepted it. */ + public boolean tryInsertVote(UUID voteId, String uuid, String playerName, String service, long time, boolean real, + String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, String broadcastTargets, + String broadcastForwardedServers, boolean rewardDelivered, String httpDeliveryIds, + String httpBroadcastDeliveryIds, String server) { String sql = "INSERT INTO " + qi(getTableName()) + " (" + qi("uuid") + ", " + qi("voteid") + ", " + qi("playerName") + ", " + qi("service") + ", " + qi("time") + ", " + qi("realVote") + ", " @@ -388,9 +398,10 @@ public void insertVote(UUID voteId, String uuid, String playerName, String servi ps.setString(14, httpBroadcastDeliveryIds); ps.setString(15, server); - ps.executeUpdate(); + return ps.executeUpdate() > 0; } catch (SQLException | IllegalArgumentException e) { debug(e); + return false; } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index d51598406..b6684664f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -92,27 +92,44 @@ public int getProxyCachedTotal(String uuid) { * @param vote the vote to add */ public synchronized void addServerVote(String server, OfflineBungeeVote vote) { - if (containsServerVote(server, vote.getVoteId())) { + addServerVoteDurably(server, vote); + } + + /** + * Adds a server vote only when it has been durably stored. MySQL failures use + * the JSON cache as an emergency journal when it is available. + * + * @param server the server name + * @param vote the vote to add + * @return true when the vote is durable (including an already cached vote) + */ + public synchronized boolean addServerVoteDurably(String server, OfflineBungeeVote vote) { + if (server == null || vote == null) { + return false; + } + if (containsServerVote(server, vote)) { debug1("Not caching duplicate vote " + vote.getVoteId() + " for server " + server); - return; + return true; } - cachedVotes.putIfAbsent(server, new ArrayList<>()); - cachedVotes.get(server).add(vote); - + boolean stored; if (useMySQL) { - voteCacheTable.insertVote(vote.getVoteId(), vote.getUuid(), vote.getPlayerName(), vote.getService(), + stored = voteCacheTable.tryInsertVote(vote.getVoteId(), vote.getUuid(), vote.getPlayerName(), vote.getService(), vote.getTime(), vote.isRealVote(), vote.getText(), vote.isBroadcastForwarded(), vote.isProxyBroadcastHandled(), vote.encodeBroadcastTargets(), vote.encodeBroadcastForwardedServers(), vote.isRewardDelivered(), vote.encodeHttpDeliveryIds(), vote.encodeHttpBroadcastDeliveryIds(), server); + if (!stored) { + stored = persistServerVoteToJson(server, vote); + } } else { - // IMPORTANT: index must come from JSON, not from cachedVotes (cache can be out - // of sync with JSON) - int idx = jsonStorage.getServerVotes(server).size(); - jsonStorage.addVote(server, idx, vote); - jsonStorage.save(); + stored = persistServerVoteToJson(server, vote); } + if (stored) { + cachedVotes.putIfAbsent(server, new ArrayList<>()); + cachedVotes.get(server).add(vote); + } + return stored; } /** @@ -123,7 +140,10 @@ public synchronized void addServerVote(String server, OfflineBungeeVote vote) { */ public synchronized boolean updateServerVote(String server, OfflineBungeeVote vote) { if (useMySQL) { - return voteCacheTable.updateProxyBroadcastState(vote, server); + if (voteCacheTable.updateProxyBroadcastState(vote, server)) { + return true; + } + return updateServerVoteJson(server, vote); } Collection keys = jsonStorage.getServerVotes(server); @@ -159,13 +179,13 @@ public void removeVote(String server, String uuid) { if (cachedVotes.containsKey(server)) { ArrayList votes = cachedVotes.get(server); votes.removeIf(vote -> vote.getUuid().equals(uuid)); - - if (useMySQL) { - voteCacheTable.removeVotesByServerAndUUID(server, uuid); - } else { - jsonStorage.removeServerVote(server, uuid); - jsonStorage.save(); - } + } + if (useMySQL) { + voteCacheTable.removeVotesByServerAndUUID(server, uuid); + } + if (jsonStorage != null) { + jsonStorage.removeServerVote(server, uuid); + jsonStorage.save(); } } @@ -177,7 +197,8 @@ public void removeVotes(String server) { cachedVotes.remove(server); if (useMySQL) { voteCacheTable.removeVotesByServer(server); - } else { + } + if (jsonStorage != null) { jsonStorage.removeServerVotes(server); jsonStorage.save(); } @@ -216,27 +237,44 @@ public Set getOnlineVoteUUIDs() { * @param vote the vote to add */ public synchronized void addOnlineVote(String uuid, OfflineBungeeVote vote) { - if (containsOnlineVote(uuid, vote.getVoteId())) { + addOnlineVoteDurably(uuid, vote); + } + + /** + * Adds an online vote only when it has been durably stored. MySQL failures use + * the JSON cache as an emergency journal when it is available. + * + * @param uuid voter UUID + * @param vote the vote to add + * @return true when the vote is durable (including an already cached vote) + */ + public synchronized boolean addOnlineVoteDurably(String uuid, OfflineBungeeVote vote) { + if (uuid == null || vote == null) { + return false; + } + if (containsOnlineVote(uuid, vote)) { debug1("Not caching duplicate online vote " + vote.getVoteId() + " for " + uuid); - return; + return true; } - cachedOnlineVotes.putIfAbsent(uuid, new ArrayList<>()); - cachedOnlineVotes.get(uuid).add(vote); - + boolean stored; if (useMySQL) { - onlineVoteCacheTable.insertVote(vote.getVoteId(), vote.getUuid(), vote.getPlayerName(), vote.getService(), + stored = onlineVoteCacheTable.tryInsertVote(vote.getVoteId(), vote.getUuid(), vote.getPlayerName(), vote.getService(), vote.getTime(), vote.isRealVote(), vote.getText(), vote.isBroadcastForwarded(), vote.isProxyBroadcastHandled(), vote.encodeBroadcastTargets(), vote.encodeBroadcastForwardedServers(), vote.isRewardDelivered(), vote.encodeHttpDeliveryIds(), vote.encodeHttpBroadcastDeliveryIds()); + if (!stored) { + stored = persistOnlineVoteToJson(uuid, vote); + } } else { - // IMPORTANT: index must come from JSON, not from cachedOnlineVotes (cache can - // be out of sync with JSON) - int idx = jsonStorage.getOnlineVotes(uuid).size(); - jsonStorage.addVoteOnline(uuid, idx, vote); - jsonStorage.save(); + stored = persistOnlineVoteToJson(uuid, vote); + } + if (stored) { + cachedOnlineVotes.putIfAbsent(uuid, new ArrayList<>()); + cachedOnlineVotes.get(uuid).add(vote); } + return stored; } /** @@ -247,7 +285,10 @@ public synchronized void addOnlineVote(String uuid, OfflineBungeeVote vote) { */ public synchronized boolean updateOnlineVote(String uuid, OfflineBungeeVote vote) { if (useMySQL) { - return onlineVoteCacheTable.updateProxyBroadcastState(vote); + if (onlineVoteCacheTable.updateProxyBroadcastState(vote)) { + return true; + } + return updateOnlineVoteJson(uuid, vote); } Collection keys = jsonStorage.getOnlineVotes(uuid); @@ -333,7 +374,8 @@ public void removeOnlineVotes(String uuid) { cachedOnlineVotes.remove(uuid); if (useMySQL) { onlineVoteCacheTable.removeVotesByUuid(uuid); - } else { + } + if (jsonStorage != null) { jsonStorage.removeOnlineVotes(uuid); jsonStorage.save(); } @@ -376,38 +418,18 @@ public void checkVoteCacheTime(int voteCacheTime) { } } - /** - * Checks whether a vote is already cached for a server. - * - * @param server target server - * @param voteId unique vote identifier - * @return true if the vote is already cached for the server - */ - private boolean containsServerVote(String server, UUID voteId) { - if (voteId == null) { - return false; - } + private boolean containsServerVote(String server, OfflineBungeeVote candidate) { for (OfflineBungeeVote vote : getVotes(server)) { - if (voteId.equals(vote.getVoteId())) { + if (sameVoteIdentity(vote, candidate)) { return true; } } return false; } - /** - * Checks whether a vote is already cached for an online player. - * - * @param uuid player UUID - * @param voteId unique vote identifier - * @return true if the vote is already cached for the player - */ - private boolean containsOnlineVote(String uuid, UUID voteId) { - if (voteId == null) { - return false; - } + private boolean containsOnlineVote(String uuid, OfflineBungeeVote candidate) { for (OfflineBungeeVote vote : getOnlineVotes(uuid)) { - if (voteId.equals(vote.getVoteId())) { + if (sameVoteIdentity(vote, candidate)) { return true; } } @@ -462,6 +484,96 @@ private boolean matchesStoredVote(DataNode data, OfflineBungeeVote vote) { && vote.getTime() == data.get("Time").asLong(); } + /** Persists a server vote in the JSON emergency journal. */ + private boolean persistServerVoteToJson(String server, OfflineBungeeVote vote) { + if (jsonStorage == null) { + return false; + } + try { + Collection keys = jsonStorage.getServerVotes(server); + int index = nextCacheIndex(keys); + jsonStorage.addVote(server, index, vote); + jsonStorage.save(); + return true; + } catch (RuntimeException e) { + debug1(e); + return false; + } + } + + /** Persists an online vote in the JSON emergency journal. */ + private boolean persistOnlineVoteToJson(String uuid, OfflineBungeeVote vote) { + if (jsonStorage == null) { + return false; + } + try { + Collection keys = jsonStorage.getOnlineVotes(uuid); + int index = nextCacheIndex(keys); + jsonStorage.addVoteOnline(uuid, index, vote); + jsonStorage.save(); + return true; + } catch (RuntimeException e) { + debug1(e); + return false; + } + } + + private int nextCacheIndex(Collection keys) { + int index = 0; + while (keys != null && keys.contains(String.valueOf(index))) { + index++; + } + return index; + } + + private boolean updateServerVoteJson(String server, OfflineBungeeVote vote) { + if (jsonStorage == null) { + return false; + } + Collection keys = jsonStorage.getServerVotes(server); + if (keys == null) { + return false; + } + for (String key : keys) { + DataNode data = jsonStorage.getServerVotes(server, key); + if (data != null && data.isObject() && matchesStoredVote(data, vote)) { + try { + jsonStorage.addVote(server, Integer.parseInt(key), vote); + jsonStorage.save(); + return true; + } catch (RuntimeException e) { + debug1(e); + return false; + } + } + } + return false; + } + + private boolean updateOnlineVoteJson(String uuid, OfflineBungeeVote vote) { + if (jsonStorage == null) { + return false; + } + Collection keys = jsonStorage.getOnlineVotes(uuid); + if (keys == null) { + return false; + } + for (String key : keys) { + DataNode data = jsonStorage.getOnlineVotes(uuid, key); + if (data != null && data.isObject() && matchesStoredVote(data, vote)) { + try { + jsonStorage.addVoteOnline(uuid, Integer.parseInt(key), vote); + jsonStorage.save(); + return true; + } catch (RuntimeException e) { + debug1(e); + return false; + } + } + } + return false; + } + private boolean sameVoteIdentity(OfflineBungeeVote first, OfflineBungeeVote second) { if (first.getVoteId() != null && second.getVoteId() != null) { return first.getVoteId().equals(second.getVoteId()); @@ -485,7 +597,7 @@ private boolean matchesStoredTimeVote(DataNode data, VoteTimeQueue vote) { * Saves the vote cache to storage. */ public void saveVoteCache() { - if (!useMySQL) { + if (jsonStorage != null) { jsonStorage.save(); } } @@ -638,6 +750,7 @@ public void load() { timedVotes.add(voteTimeQueue); }); timeChangeQueue.addAll(timedVotes); + loadJsonEmergencyVotes(); } else { try { @@ -782,6 +895,79 @@ public void load() { } + /** + * Loads entries written to the JSON emergency journal while MySQL was + * unavailable. Entries already present in SQL are de-duplicated by vote ID (or + * the legacy UUID/service/time identity). + */ + private void loadJsonEmergencyVotes() { + if (jsonStorage == null) { + return; + } + try { + Collection servers = jsonStorage.getServers(); + if (servers != null) { + for (String server : servers) { + Collection keys = jsonStorage.getServerVotes(server); + if (keys == null) { + continue; + } + for (String key : keys) { + OfflineBungeeVote vote = decodeJsonVote(jsonStorage.getServerVotes(server, key)); + if (vote != null && !containsServerVote(server, vote)) { + cachedVotes.computeIfAbsent(server, ignored -> new ArrayList<>()).add(vote); + } + } + } + } + Collection players = jsonStorage.getPlayers(); + if (players != null) { + for (String player : players) { + Collection keys = jsonStorage.getOnlineVotes(player); + if (keys == null) { + continue; + } + for (String key : keys) { + OfflineBungeeVote vote = decodeJsonVote(jsonStorage.getOnlineVotes(player, key)); + if (vote != null && !containsOnlineVote(player, vote)) { + cachedOnlineVotes.computeIfAbsent(player, ignored -> new ArrayList<>()).add(vote); + } + } + } + } + } catch (RuntimeException e) { + debug1(e); + } + } + + private OfflineBungeeVote decodeJsonVote(DataNode data) { + if (data == null || !data.isObject()) { + return null; + } + String name = data.has("Name") ? data.get("Name").asString() : ""; + String uuid = data.has("UUID") ? data.get("UUID").asString() : ""; + String service = data.has("Service") ? data.get("Service").asString() : ""; + long time = data.has("Time") ? data.get("Time").asLong() : 0L; + boolean real = data.has("Real") && data.get("Real").asBoolean(); + String text = data.has("Text") ? data.get("Text").asString() : ""; + String voteId = readVoteId(data); + boolean broadcastForwarded = data.has("BroadcastForwarded") && data.get("BroadcastForwarded").asBoolean(); + boolean proxyBroadcastHandled = data.has("ProxyBroadcastHandled") + && data.get("ProxyBroadcastHandled").asBoolean(); + String broadcastTargets = data.has("BroadcastTargets") ? data.get("BroadcastTargets").asString() : ""; + String broadcastForwardedServers = data.has("BroadcastForwardedServers") + ? data.get("BroadcastForwardedServers").asString() : ""; + String httpDeliveryIds = data.has("HttpDeliveryIds") ? data.get("HttpDeliveryIds").asString() : ""; + String httpBroadcastDeliveryIds = data.has("HttpBroadcastDeliveryIds") + ? data.get("HttpBroadcastDeliveryIds").asString() : ""; + boolean rewardDelivered = data.has("RewardDelivered") && data.get("RewardDelivered").asBoolean(); + return new OfflineBungeeVote(voteId, name, uuid, service, time, real, text, broadcastForwarded, + proxyBroadcastHandled, VoteTimeQueue.decodeBroadcastForwardedServers(broadcastTargets), + VoteTimeQueue.decodeBroadcastForwardedServers(broadcastForwardedServers), rewardDelivered, + OfflineBungeeVote.decodeHttpDeliveryIds(httpDeliveryIds), + OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(httpBroadcastDeliveryIds)); + } + private final boolean useMySQL; private ProxyVoteCacheTable voteCacheTable; private ProxyTimedVoteCacheTable timedVoteCacheTable; @@ -831,6 +1017,9 @@ public void load() { public VoteCacheHandler(MysqlConfig mysqlConfig, boolean useMySQL, boolean useExistingConnection, MySQL mysql, boolean debug, IVoteCache jsonStorage) { this.useMySQL = useMySQL; + // Keep the JSON store available as a durable emergency journal even when + // MySQL is the primary cache backend. + this.jsonStorage = jsonStorage; if (useMySQL) { if (useExistingConnection) { @@ -974,8 +1163,6 @@ public void debug(String text) { } }; } - } else { - this.jsonStorage = jsonStorage; } } @@ -1002,7 +1189,8 @@ public void removeServerVotes(String server, ArrayList remove } if (useMySQL) { voteCacheTable.removeVote(vote, server); - } else { + } + if (jsonStorage != null) { jsonStorage.removeVote(server, vote); jsonStorage.save(); } @@ -1021,7 +1209,8 @@ public void removeOnlineVotes(ArrayList removed) { } if (useMySQL) { onlineVoteCacheTable.removeVote(vote); - } else { + } + if (jsonStorage != null) { jsonStorage.removeOnlineVote(vote); jsonStorage.save(); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java index d58041214..0975bcb1e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java @@ -86,19 +86,28 @@ private void validate(ProxyMethodConfiguration proposal, VotingPluginProxyConfig public void apply(ProxyMethodConfiguration proposal, String expectedRevision) throws IOException { if (expectedRevision == null) throw new StaleRevisionException(); + boolean prepareHttp = proposal.method() == BungeeMethod.HTTP && read().method() != BungeeMethod.HTTP; try { proxy.getConfig().persistControlProxyMethod(proposal.method().name(), expectedRevision, - latest -> validate(proposal, latest)); + latest -> { + validate(proposal, latest); + if (prepareHttp) proxy.prepareHttpTransportChange(latest); + }); proxy.getConfig().verifyControlProxyRoutingInstalled(); } catch (VotingPluginProxyConfig.StaleControlRevisionException e) { + if (prepareHttp) proxy.cancelPreparedHttpTransportChange(); throw new StaleRevisionException(); } catch (IllegalArgumentException validation) { + if (prepareHttp) proxy.cancelPreparedHttpTransportChange(); throw validation; } catch (DurableFiles.PublishedException published) { + if (prepareHttp) proxy.cancelPreparedHttpTransportChange(); throw rollbackAfterFailure(published); } catch (IOException failure) { + if (prepareHttp) proxy.cancelPreparedHttpTransportChange(); throw failure; } catch (RuntimeException failure) { + if (prepareHttp) proxy.cancelPreparedHttpTransportChange(); throw rollbackAfterFailure(failure); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java index f49600dce..bb34e5700 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java @@ -4,9 +4,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Set; @@ -142,4 +145,48 @@ void applyDoesNotRejectAValidFreshSnapshotUsingStaleCachedSettings() throws Exce ProxyMethodConfiguration proposal = new ProxyMethodConfiguration(BungeeMethod.REDIS); assertDoesNotThrow(() -> service.apply(proposal, service.read().revision())); } + + @Test + void preparesHttpListenerFromFreshSnapshotBeforePublishingMethod() throws Exception { + when(config.getBungeeMethod()).thenReturn("PLUGINMESSAGING"); + VotingPluginProxyConfig fresh = validHttpConfig(); + doAnswer(invocation -> { + VotingPluginProxyConfig.ControlProxyMethodValidator validator = invocation.getArgument(2); + validator.validate(fresh); + return null; + }).when(config).persistControlProxyMethod(org.mockito.ArgumentMatchers.eq("HTTP"), + org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); + + service.apply(new ProxyMethodConfiguration(BungeeMethod.HTTP), service.read().revision()); + + verify(proxy).prepareHttpTransportChange(fresh); + verify(config).verifyControlProxyRoutingInstalled(); + } + + @Test + void closesPreparedHttpListenerWhenPublicationVerificationFails() throws Exception { + when(config.getBungeeMethod()).thenReturn("PLUGINMESSAGING"); + VotingPluginProxyConfig fresh = validHttpConfig(); + doAnswer(invocation -> { + VotingPluginProxyConfig.ControlProxyMethodValidator validator = invocation.getArgument(2); + validator.validate(fresh); + return null; + }).when(config).persistControlProxyMethod(org.mockito.ArgumentMatchers.eq("HTTP"), + org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); + doThrow(new IOException("verification failed")).when(config).verifyControlProxyRoutingInstalled(); + + assertThrows(IOException.class, + () -> service.apply(new ProxyMethodConfiguration(BungeeMethod.HTTP), service.read().revision())); + + verify(proxy).prepareHttpTransportChange(fresh); + verify(proxy).cancelPreparedHttpTransportChange(); + } + + private VotingPluginProxyConfig validHttpConfig() { + VotingPluginProxyConfig fresh = mock(VotingPluginProxyConfig.class); + when(fresh.getHttpHost()).thenReturn("127.0.0.1"); + when(fresh.getHttpPort()).thenReturn(1297); + when(fresh.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test:1297"); + return fresh; + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index b99715447..718635586 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -112,6 +112,22 @@ public void missingVoteIdsRemainLegacyCompatible() { assertEquals(2, handler.getVotes("server").size()); } + @Test + public void serverVoteIsNotExposedWhenJsonJournalCannotSave() { + doThrow(new RuntimeException("save failed")).when(storage).save(); + + assertFalse(handler.addServerVoteDurably("server", vote(UUID.randomUUID(), 100L))); + assertTrue(handler.getVotes("server").isEmpty()); + } + + @Test + public void onlineVoteIsNotExposedWhenJsonJournalCannotSave() { + doThrow(new RuntimeException("save failed")).when(storage).save(); + + assertFalse(handler.addOnlineVoteDurably("player-uuid", vote(UUID.randomUUID(), 100L))); + assertTrue(handler.getOnlineVotes("player-uuid").isEmpty()); + } + @Test public void currentVoteIdKeyLoadsFromJsonCache() { UUID voteId = UUID.randomUUID(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 4439b69dc..865e9d330 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -52,9 +52,26 @@ void setUp() { votingPluginProxy.setProxyMySQL(proxyMySQL); votingPluginProxy.setGlobalDataHandler(globalDataHandler); votingPluginProxy.setMultiProxyHandler(multiProxyHandler); + votingPluginProxy.setDataFolder(temporaryDirectory.toFile()); } + @Test + void httpMethodPreparationRejectsAnOccupiedPortBeforeConfigurationPublication() throws Exception { + try (java.net.ServerSocket occupied = new java.net.ServerSocket(0, 1, + java.net.InetAddress.getByName("127.0.0.1"))) { + com.bencodez.votingplugin.proxy.VotingPluginProxyConfig candidate = + Mockito.mock(com.bencodez.votingplugin.proxy.VotingPluginProxyConfig.class); + Mockito.when(candidate.getHttpHost()).thenReturn("127.0.0.1"); + Mockito.when(candidate.getHttpPort()).thenReturn(occupied.getLocalPort()); + Mockito.when(candidate.getHttpPublicEndpoint()) + .thenReturn("https://proxy.example.test:" + occupied.getLocalPort()); + + assertThrows(IllegalStateException.class, + () -> votingPluginProxy.prepareHttpTransportChange(candidate)); + } + } + @Test void indeterminateHttpVoteRetriesWithThePublishedStableDeliveryId() { votingPluginProxy.setMethod(BungeeMethod.HTTP); @@ -466,6 +483,42 @@ void runtimeReplacementQuarantinesAnInFlightVotePartyCommandBeforeHandoff() { assertEquals(java.util.List.of("in flight"), votingPluginProxy.getConsoleCommands()); } + @Test + void finalShutdownQuarantinesAnInFlightVotePartyCommandBeforeTeardown() { + configureHttpVotePartyEffects("", java.util.List.of("in flight", "next command")); + java.util.concurrent.CompletableFuture completion = + votingPluginProxy.delayNextVotePartyCommandCompletion(); + votingPluginProxy.checkVoteParty(); + + votingPluginProxy.onDisable(false); + + assertEquals(java.util.List.of("in flight"), + votingPluginProxy.getVoteCacheQuarantinedVotePartyProxyEffects().commands()); + assertEquals(java.util.List.of("next command"), + votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().commands()); + completion.complete(null); + assertEquals(java.util.List.of("in flight"), votingPluginProxy.getConsoleCommands()); + } + + @Test + void finalShutdownLeavesInFlightCommandFencedWhenQuarantineIsNotDurable() { + configureHttpVotePartyEffects("", java.util.List.of("in flight", "next command")); + java.util.concurrent.CompletableFuture completion = + votingPluginProxy.delayNextVotePartyCommandCompletion(); + votingPluginProxy.checkVoteParty(); + votingPluginProxy.failNextVoteCacheSave(); + + votingPluginProxy.onDisable(false); + + assertTrue(votingPluginProxy.getVoteCacheQuarantinedVotePartyProxyEffects().isEmpty()); + assertEquals(java.util.List.of("in flight", "next command"), + votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().commands()); + completion.complete(null); + assertEquals(java.util.List.of("in flight"), votingPluginProxy.getConsoleCommands()); + assertEquals(java.util.List.of("in flight", "next command"), + votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().commands()); + } + @Test void transportReloadRetainsHttpUntilPendingVotePartyRewardsAreAcknowledged() { votingPluginProxy.setMethod(BungeeMethod.HTTP); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index 454ec190a..894444518 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -52,6 +52,7 @@ public class VotingPluginProxyTestImpl extends VotingPluginProxy { private Runnable votePartyProxyCommandTimeout; private Boolean pendingHttpTransportDeliveries; private volatile int reloadCoreCalls; + private File dataFolder = new File("."); public List getWarnings() { return warnings; @@ -113,7 +114,11 @@ public String getCurrentPlayerServer(String player) { @Override public File getDataFolderPlugin() { - return new File("."); + return dataFolder; + } + + public void setDataFolder(File dataFolder) { + this.dataFolder = dataFolder; } @Override From b117c36b08aacba7311a62a7218dece01afe7f8a Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:44:43 -0600 Subject: [PATCH 066/136] Harden HTTP shutdown and cache durability --- .../transport/HttpBackendProxyTransport.java | 16 ++-- .../votingplugin/proxy/VotingPluginProxy.java | 48 ++++++++++-- .../proxy/bungee/BungeeJsonVoteCache.java | 32 +++++++- .../votingplugin/proxy/cache/IVoteCache.java | 26 +++++++ .../proxy/cache/VoteCacheDurability.java | 75 +++++++++++++++++++ .../proxy/cache/VoteCacheHandler.java | 53 +++++++------ .../proxy/cache/VotePartyCacheDurability.java | 11 ++- .../proxy/velocity/VelocityJsonVoteCache.java | 26 +++++++ .../votingplugin/util/DurableFiles.java | 12 +++ .../HttpBackendProxyTransportTest.java | 13 ++++ .../tests/VoteCacheHandlerVoteIdTest.java | 67 +++++++++++++++++ .../tests/VotingPluginProxyTest.java | 26 ++++++- .../tests/VotingPluginProxyTestImpl.java | 5 ++ 13 files changed, 367 insertions(+), 43 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheDurability.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 6d0584e3d..1c26a697c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -313,11 +313,7 @@ private void initialize(Path directory, String serverId, String configuredCode, if (closed) { discard = true; } else { - while (!startupQueue.isEmpty()) { - if (!replacement.send(startupQueue.removeFirst())) { - throw new IllegalStateException("HTTP startup queue could not be transferred"); - } - } + transferStartupQueue(startupQueue, replacement); connector = replacement; directoryOwner = owner; installed = true; @@ -338,6 +334,16 @@ private void initialize(Path directory, String serverId, String configuredCode, } } + static void transferStartupQueue(ArrayDeque queue, HttpBackendTransportConnector connector) { + while (!queue.isEmpty()) { + JsonEnvelope envelope = queue.peekFirst(); + if (!connector.send(envelope)) { + throw new IllegalStateException("HTTP startup queue could not be transferred"); + } + queue.removeFirst(); + } + } + @Override public void activateAfterPublication() { HttpBackendTransportConnector active; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index cdab46d6c..e5c0e78cb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -194,6 +194,8 @@ private void close() { private long httpTransportReconciliationGeneration; private long votePartyProxyCommandAttemptSequence; private long votePartyProxyCommandInFlight; + private volatile CompletableFuture votePartyProxyCommandExecution; + private boolean votePartyProxyCommandCompletedUnpersisted; private boolean enabled; @@ -2671,7 +2673,10 @@ public void onDisable(boolean waitForHosted) { prepareForRuntimeReplacement(); } else { if (!quarantineInFlightVotePartyProxyCommandForReplacement()) { - logSevere("Unable to durably quarantine an in-flight HTTP vote-party command during final shutdown; the attempt remains fenced"); + awaitInFlightVotePartyProxyCommand(); + if (!quarantineInFlightVotePartyProxyCommandForReplacement()) { + logSevere("Unable to durably quarantine an in-flight HTTP vote-party command after the bounded final-shutdown wait; operator reconciliation may be required after restart"); + } } cancelPreparedHttpTransportChange(); controlServicesGeneration.incrementAndGet(); @@ -3718,6 +3723,11 @@ protected synchronized void retryPendingVotePartyRewards() { protected synchronized boolean retryPendingVotePartyProxyEffects() { if (!enabled) return true; if (votePartyProxyCommandInFlight != 0L) return false; + if (votePartyProxyCommandCompletedUnpersisted + && !quarantineInFlightVotePartyProxyCommandForReplacement()) { + scheduleVotePartyDeliveryRetry(); + return false; + } PendingVotePartyProxyEffects pending; try { pending = getVoteCachePendingVotePartyProxyEffects(); @@ -3739,6 +3749,8 @@ protected synchronized boolean retryPendingVotePartyProxyEffects() { if (attempt == 0L) attempt = ++votePartyProxyCommandAttemptSequence; long commandAttempt = attempt; votePartyProxyCommandInFlight = commandAttempt; + votePartyProxyCommandExecution = execution; + votePartyProxyCommandCompletedUnpersisted = false; PendingVotePartyProxyEffects expected = pending; PendingVotePartyProxyEffects completed = remaining; execution.whenComplete((ignored, failure) -> @@ -3771,10 +3783,10 @@ private void completeVotePartyProxyCommand(long attempt, PendingVotePartyProxyEf synchronized (this) { if (votePartyProxyCommandInFlight != attempt) return; votePartyProxyCommandInFlight = 0L; - if (!enabled) return; + votePartyProxyCommandExecution = null; if (failure != null) { logSevere("A committed HTTP vote-party proxy command failed and remains pending for retry"); - scheduleVotePartyDeliveryRetry(); + if (enabled) scheduleVotePartyDeliveryRetry(); return; } PendingVotePartyProxyEffects current; @@ -3788,8 +3800,13 @@ private void completeVotePartyProxyCommand(long attempt, PendingVotePartyProxyEf logSevere("Pending HTTP vote-party proxy effects changed while a command was running; progress was not advanced"); return; } - if (persistVotePartyProxyEffectProgress(expected, remaining) - && retryPendingVotePartyProxyEffects()) { + if (!persistVotePartyProxyEffectProgress(expected, remaining)) { + votePartyProxyCommandCompletedUnpersisted = true; + if (!enabled) quarantineInFlightVotePartyProxyCommandForReplacement(); + return; + } + votePartyProxyCommandCompletedUnpersisted = false; + if (enabled && retryPendingVotePartyProxyEffects()) { if (method == BungeeMethod.HTTP) retryPendingVotePartyRewards(); if (votePartyVotes >= currentVotePartyVotesRequired) checkVoteParty(); } @@ -3833,6 +3850,7 @@ private void quarantineTimedOutVotePartyProxyCommand(long attempt, PendingVotePa return; } votePartyProxyCommandInFlight = 0L; + votePartyProxyCommandExecution = null; logSevere("An HTTP vote-party proxy command did not complete within 60 seconds and was durably quarantined without retry"); if (retryPendingVotePartyProxyEffects()) { if (method == BungeeMethod.HTTP) retryPendingVotePartyRewards(); @@ -3842,7 +3860,7 @@ private void quarantineTimedOutVotePartyProxyCommand(long attempt, PendingVotePa } protected synchronized boolean quarantineInFlightVotePartyProxyCommandForReplacement() { - if (votePartyProxyCommandInFlight == 0L) return true; + if (votePartyProxyCommandInFlight == 0L && !votePartyProxyCommandCompletedUnpersisted) return true; PendingVotePartyProxyEffects pending; PendingVotePartyProxyEffects previousQuarantine; try { @@ -3864,6 +3882,8 @@ protected synchronized boolean quarantineInFlightVotePartyProxyCommandForReplace return false; } votePartyProxyCommandInFlight = 0L; + votePartyProxyCommandExecution = null; + votePartyProxyCommandCompletedUnpersisted = false; logSevere("An in-flight HTTP vote-party proxy command was durably quarantined for runtime replacement"); return true; } catch (RuntimeException invalid) { @@ -3871,6 +3891,22 @@ protected synchronized boolean quarantineInFlightVotePartyProxyCommandForReplace } } + /** Gives an executing platform command a bounded chance to finish and persist progress before final teardown. */ + protected void awaitInFlightVotePartyProxyCommand() { + CompletableFuture execution = votePartyProxyCommandExecution; + if (execution == null || execution.isDone()) return; + try { + execution.get(5, TimeUnit.SECONDS); + } catch (java.util.concurrent.ExecutionException ignored) { + // The completion callback retains failed commands for retry. + } catch (java.util.concurrent.TimeoutException timeout) { + logSevere("Timed out waiting for an in-flight HTTP vote-party command during final shutdown"); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + logSevere("Interrupted while waiting for an in-flight HTTP vote-party command during final shutdown"); + } + } + private boolean persistVotePartyProxyEffectProgress(PendingVotePartyProxyEffects previous, PendingVotePartyProxyEffects remaining) { setVoteCachePendingVotePartyProxyEffects(remaining); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java index d0b829d62..f6cfcab72 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java @@ -2,6 +2,9 @@ import java.io.File; import java.nio.charset.StandardCharsets; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Base64; import java.util.Collection; @@ -13,12 +16,14 @@ import com.bencodez.votingplugin.proxy.cache.GsonDataNode; import com.bencodez.votingplugin.proxy.cache.IVoteCache; import com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects; -import com.bencodez.votingplugin.timequeue.VoteTimeQueue; +import com.bencodez.votingplugin.timequeue.VoteTimeQueue; +import com.bencodez.votingplugin.util.DurableFiles; +import com.google.gson.GsonBuilder; /** * JSON file-based vote cache for Bungee. */ -public class BungeeJsonVoteCache extends BungeeJsonFile implements IVoteCache { +public class BungeeJsonVoteCache extends BungeeJsonFile implements IVoteCache { private VotingPluginBungee bungee; /** @@ -32,7 +37,7 @@ public BungeeJsonVoteCache(VotingPluginBungee bungee) { initialize(); } - private void initialize() { + private void initialize() { if (!bungee.getDataFolder().exists()) { bungee.getDataFolder().mkdir(); } @@ -54,6 +59,27 @@ public void addTimedVote(int num, VoteTimeQueue voteTimedQueue) { setString(path + ".BroadcastForwardedServers", voteTimedQueue.encodeBroadcastForwardedServers()); setString(path + ".HttpBroadcastDeliveryIds", voteTimedQueue.encodeHttpBroadcastDeliveryIds()); } + + @Override + public java.nio.file.Path getStoragePath() { + return getFile().toPath(); + } + + @Override + public synchronized void saveDurably() throws IOException { + Path target = getStoragePath().toAbsolutePath().normalize(); + Path parent = target.getParent(); + if (parent == null) throw new IOException("Vote cache has no parent directory"); + Files.createDirectories(parent); + Path staged = Files.createTempFile(parent, target.getFileName().toString(), ".tmp"); + try { + Files.writeString(staged, new GsonBuilder().setPrettyPrinting().create().toJson(getConf()), + StandardCharsets.UTF_8); + DurableFiles.publishStagedFile(staged, target); + } finally { + Files.deleteIfExists(staged); + } + } public void addVote(String server, int num, OfflineBungeeVote voteData) { String path = "VoteCache." + server + "." + num; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java index 12cf1921a..aa5da8667 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java @@ -1,6 +1,10 @@ package com.bencodez.votingplugin.proxy.cache; import java.util.Collection; +import java.nio.file.Path; +import java.io.IOException; + +import com.bencodez.votingplugin.util.DurableFiles; import com.bencodez.votingplugin.proxy.OfflineBungeeVote; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; @@ -10,6 +14,28 @@ */ public interface IVoteCache { + /** + * Returns the file backing this JSON cache, when one exists. + * + * @return backing file path, or {@code null} for non-file implementations + */ + default Path getStoragePath() { + return null; + } + + /** Persists the complete cache and forces it to stable storage. */ + default void saveDurably() throws IOException { + Path path = getStoragePath(); + if (path == null) throw new IOException("Vote cache has no durable storage path"); + save(); + DurableFiles.forceFile(path); + try { + DurableFiles.forceDirectory(path.toAbsolutePath().normalize().getParent()); + } catch (IOException failure) { + throw new DurableFiles.PublishedException(failure); + } + } + /** * Adds a timed vote to the cache. * diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheDurability.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheDurability.java new file mode 100644 index 000000000..32fd6884c --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheDurability.java @@ -0,0 +1,75 @@ +package com.bencodez.votingplugin.proxy.cache; + +import java.nio.file.Path; + +import com.bencodez.votingplugin.proxy.OfflineBungeeVote; +import com.bencodez.votingplugin.timequeue.VoteTimeQueue; +import com.bencodez.votingplugin.util.DurableFiles; + +/** Forces and reads back emergency JSON journal entries before they are exposed in memory. */ +public final class VoteCacheDurability { + private VoteCacheDurability() { } + + public static boolean saveAndVerifyServerVote(IVoteCache cache, String server, int index, + OfflineBungeeVote expected) { + return saveAndVerify(cache, () -> cache.getServerVotes(server, String.valueOf(index)), expected); + } + + public static boolean saveAndVerifyOnlineVote(IVoteCache cache, String uuid, int index, + OfflineBungeeVote expected) { + return saveAndVerify(cache, () -> cache.getOnlineVotes(uuid, String.valueOf(index)), expected); + } + + public static boolean saveAndVerifyTimeVote(IVoteCache cache, int index, VoteTimeQueue expected) { + return saveAndVerify(cache, () -> cache.getTimedVoteCache(String.valueOf(index)), + data -> matches(data, expected)); + } + + private static boolean saveAndVerify(IVoteCache cache, java.util.function.Supplier read, + OfflineBungeeVote expected) { + return saveAndVerify(cache, read, data -> matches(data, expected)); + } + + private static boolean saveAndVerify(IVoteCache cache, java.util.function.Supplier read, + java.util.function.Predicate verify) { + if (cache == null || verify == null) return false; + Path path = cache.getStoragePath(); + if (path == null) return false; + try { + try { + cache.saveDurably(); + } catch (DurableFiles.PublishedException published) { + // The new file is already active. Verify and keep its matching in-memory + // state rather than rolling back and later overwriting the publication. + } + cache.reload(); + return verify.test(read.get()); + } catch (RuntimeException | java.io.IOException failure) { + return false; + } + } + + private static boolean matches(DataNode data, VoteTimeQueue expected) { + if (expected == null || data == null || !data.isObject() || !data.has("UUID") + || !data.has("Service") || !data.has("Time")) return false; + if (!expected.getUuid().equals(data.get("UUID").asString()) + || !expected.getService().equals(data.get("Service").asString()) + || expected.getTime() != data.get("Time").asLong()) return false; + if (expected.getVoteId() == null) return true; + String voteId = data.has("VoteId") ? data.get("VoteId").asString() + : data.has("VoteID") ? data.get("VoteID").asString() : ""; + return expected.getVoteId().toString().equals(voteId); + } + + private static boolean matches(DataNode data, OfflineBungeeVote expected) { + if (data == null || !data.isObject() || !data.has("UUID") || !data.has("Service") || !data.has("Time")) + return false; + if (!expected.getUuid().equals(data.get("UUID").asString()) + || !expected.getService().equals(data.get("Service").asString()) + || expected.getTime() != data.get("Time").asLong()) return false; + if (expected.getVoteId() == null) return true; + String voteId = data.has("VoteId") ? data.get("VoteId").asString() + : data.has("VoteID") ? data.get("VoteID").asString() : ""; + return expected.getVoteId().toString().equals(voteId); + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index b6684664f..022063186 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -158,9 +158,9 @@ public synchronized boolean updateServerVote(String server, OfflineBungeeVote vo } if (matchesStoredVote(data, vote)) { try { - jsonStorage.addVote(server, Integer.parseInt(key), vote); - jsonStorage.save(); - return true; + int index = Integer.parseInt(key); + jsonStorage.addVote(server, index, vote); + return verifyJsonServerVote(server, index, vote); } catch (RuntimeException e) { debug1(e); return false; @@ -303,9 +303,9 @@ public synchronized boolean updateOnlineVote(String uuid, OfflineBungeeVote vote } if (matchesStoredVote(data, vote)) { try { - jsonStorage.addVoteOnline(uuid, Integer.parseInt(key), vote); - jsonStorage.save(); - return true; + int index = Integer.parseInt(key); + jsonStorage.addVoteOnline(uuid, index, vote); + return verifyJsonOnlineVote(uuid, index, vote); } catch (RuntimeException e) { debug1(e); return false; @@ -493,8 +493,7 @@ private boolean persistServerVoteToJson(String server, OfflineBungeeVote vote) { Collection keys = jsonStorage.getServerVotes(server); int index = nextCacheIndex(keys); jsonStorage.addVote(server, index, vote); - jsonStorage.save(); - return true; + return verifyJsonServerVote(server, index, vote); } catch (RuntimeException e) { debug1(e); return false; @@ -510,14 +509,25 @@ private boolean persistOnlineVoteToJson(String uuid, OfflineBungeeVote vote) { Collection keys = jsonStorage.getOnlineVotes(uuid); int index = nextCacheIndex(keys); jsonStorage.addVoteOnline(uuid, index, vote); - jsonStorage.save(); - return true; + return verifyJsonOnlineVote(uuid, index, vote); } catch (RuntimeException e) { debug1(e); return false; } } + protected boolean verifyJsonServerVote(String server, int index, OfflineBungeeVote vote) { + return VoteCacheDurability.saveAndVerifyServerVote(jsonStorage, server, index, vote); + } + + protected boolean verifyJsonOnlineVote(String uuid, int index, OfflineBungeeVote vote) { + return VoteCacheDurability.saveAndVerifyOnlineVote(jsonStorage, uuid, index, vote); + } + + protected boolean verifyJsonTimeVote(int index, VoteTimeQueue vote) { + return VoteCacheDurability.saveAndVerifyTimeVote(jsonStorage, index, vote); + } + private int nextCacheIndex(Collection keys) { int index = 0; while (keys != null && keys.contains(String.valueOf(index))) { @@ -538,9 +548,9 @@ private boolean updateServerVoteJson(String server, OfflineBungeeVote vote) { DataNode data = jsonStorage.getServerVotes(server, key); if (data != null && data.isObject() && matchesStoredVote(data, vote)) { try { - jsonStorage.addVote(server, Integer.parseInt(key), vote); - jsonStorage.save(); - return true; + int index = Integer.parseInt(key); + jsonStorage.addVote(server, index, vote); + return verifyJsonServerVote(server, index, vote); } catch (RuntimeException e) { debug1(e); return false; @@ -562,9 +572,9 @@ private boolean updateOnlineVoteJson(String uuid, OfflineBungeeVote vote) { DataNode data = jsonStorage.getOnlineVotes(uuid, key); if (data != null && data.isObject() && matchesStoredVote(data, vote)) { try { - jsonStorage.addVoteOnline(uuid, Integer.parseInt(key), vote); - jsonStorage.save(); - return true; + int index = Integer.parseInt(key); + jsonStorage.addVoteOnline(uuid, index, vote); + return verifyJsonOnlineVote(uuid, index, vote); } catch (RuntimeException e) { debug1(e); return false; @@ -630,8 +640,9 @@ public synchronized boolean addTimeVoteToCache(VoteTimeQueue vote) { index++; } jsonStorage.addTimedVote(index, vote); - jsonStorage.save(); - return true; + if (verifyJsonTimeVote(index, vote)) return true; + timeChangeQueue.remove(vote); + return false; } catch (RuntimeException e) { timeChangeQueue.remove(vote); debug1(e); @@ -658,9 +669,9 @@ public synchronized boolean updateTimeVote(VoteTimeQueue vote) { DataNode data = jsonStorage.getTimedVoteCache(key); if (data != null && data.isObject() && matchesStoredTimeVote(data, vote)) { try { - jsonStorage.addTimedVote(Integer.parseInt(key), vote); - jsonStorage.save(); - return true; + int index = Integer.parseInt(key); + jsonStorage.addTimedVote(index, vote); + return verifyJsonTimeVote(index, vote); } catch (NumberFormatException e) { debug1(e); return false; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java index d0c855bba..8fb999c4e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java @@ -10,10 +10,10 @@ import java.util.Map; import java.util.Set; -import com.bencodez.votingplugin.util.DurableFiles; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.bencodez.votingplugin.util.DurableFiles; /** Verifies that the cache library actually persisted the vote-party transaction. */ public final class VotePartyCacheDurability { @@ -25,9 +25,12 @@ public static void saveAndVerify(Path file, IVoteCache cache) throws IOException int expectedIncrease = cache.getVotePartyInreaseVotesRequired(); PendingVotePartyProxyEffects expectedProxyEffects = cache.getPendingVotePartyProxyEffects(); PendingVotePartyProxyEffects expectedQuarantinedEffects = cache.getQuarantinedVotePartyProxyEffects(); - cache.save(); - DurableFiles.forceFile(file); - DurableFiles.forceDirectory(file.toAbsolutePath().normalize().getParent()); + try { + cache.saveDurably(); + } catch (DurableFiles.PublishedException published) { + // The replacement is installed; validate it instead of letting callers + // restore stale in-memory state over the published file. + } try { JsonObject root = JsonParser.parseString(Files.readString(file, StandardCharsets.UTF_8)).getAsJsonObject(); JsonObject voteParty = object(root.get("VoteParty")); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java index 5649ab3b0..50859ae6b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java @@ -2,6 +2,9 @@ import java.io.File; import java.nio.charset.StandardCharsets; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Base64; import java.util.Collection; @@ -14,6 +17,9 @@ import com.bencodez.votingplugin.proxy.cache.IVoteCache; import com.bencodez.votingplugin.proxy.cache.PendingVotePartyProxyEffects; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; +import com.bencodez.votingplugin.util.DurableFiles; + +import org.spongepowered.configurate.gson.GsonConfigurationLoader; /** * JSON-based vote cache implementation for Velocity proxy. @@ -28,6 +34,26 @@ public VelocityJsonVoteCache(File file) { super(file); } + @Override + public java.nio.file.Path getStoragePath() { + return getPath(); + } + + @Override + public synchronized void saveDurably() throws IOException { + Path target = getStoragePath().toAbsolutePath().normalize(); + Path parent = target.getParent(); + if (parent == null) throw new IOException("Vote cache has no parent directory"); + Files.createDirectories(parent); + Path staged = Files.createTempFile(parent, target.getFileName().toString(), ".tmp"); + try { + GsonConfigurationLoader.builder().path(staged).build().save(getConf()); + DurableFiles.publishStagedFile(staged, target); + } finally { + Files.deleteIfExists(staged); + } + } + @Override public void addTimedVote(int num, VoteTimeQueue voteTimedQueue) { setPath(voteTimedQueue.getName(), "TimedVoteCache", String.valueOf(num), "Name"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/DurableFiles.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/DurableFiles.java index b57fdb72f..9122b1e61 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/DurableFiles.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/DurableFiles.java @@ -7,6 +7,7 @@ import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.nio.file.StandardCopyOption; import java.util.Locale; /** Cross-platform helpers for forcing file contents and published directory entries. */ @@ -34,6 +35,17 @@ public static void forceDirectory(Path directory) throws IOException { } } + /** Forces a staged file, atomically publishes it, then forces the directory entry. */ + public static void publishStagedFile(Path staged, Path target) throws IOException { + forceFile(staged); + Files.move(staged, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + try { + forceDirectory(target.toAbsolutePath().normalize().getParent()); + } catch (IOException failure) { + throw new PublishedException(failure); + } + } + public static boolean deleteIfExists(Path target) throws IOException { boolean deleted = Files.deleteIfExists(target); if (deleted) forceDirectory(target.toAbsolutePath().normalize().getParent()); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index aa7a0ab7f..4da906bd2 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -119,6 +119,19 @@ void unpublishedHandoffMessagesAreDiscardedOnRollback() throws Exception { verify(connector, org.mockito.Mockito.never()).send(org.mockito.ArgumentMatchers.any(JsonEnvelope.class)); } + @Test + void failedStartupTransferRetainsTheUnacceptedEnvelope() { + java.util.ArrayDeque queue = new java.util.ArrayDeque<>(); + JsonEnvelope pending = JsonEnvelope.builder("pending").build(); + queue.add(pending); + HttpBackendTransportConnector connector = mock(HttpBackendTransportConnector.class); + when(connector.send(pending)).thenReturn(false); + + assertThrows(IllegalStateException.class, + () -> HttpBackendProxyTransport.transferStartupQueue(queue, connector)); + assertEquals(java.util.List.of(pending), new java.util.ArrayList<>(queue)); + } + @Test @SuppressWarnings("unchecked") void orderlyShutdownFlushesRecoveryQueueBeforeHandoffQueue() throws Exception { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 718635586..5ca004bf3 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -23,6 +23,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.nio.file.Files; +import java.nio.file.Path; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -128,6 +130,43 @@ public void onlineVoteIsNotExposedWhenJsonJournalCannotSave() { assertTrue(handler.getOnlineVotes("player-uuid").isEmpty()); } + @Test + public void serverVoteIsNotExposedWhenJsonSaveDoesNotReachDisk() throws Exception { + Path journal = Files.createTempFile("votingplugin-empty-journal", ".json"); + try { + when(storage.getStoragePath()).thenReturn(journal); + handler = newVerifyingHandler(storage); + + assertFalse(handler.addServerVoteDurably("server", vote(UUID.randomUUID(), 100L))); + assertTrue(handler.getVotes("server").isEmpty()); + verify(storage).reload(); + } finally { + Files.deleteIfExists(journal); + } + } + + @Test + public void serverVoteUpdateIsRejectedWhenJsonSaveDoesNotReachDisk() throws Exception { + Path journal = Files.createTempFile("votingplugin-empty-update-journal", ".json"); + try { + DataNode voteNode = mock(DataNode.class); + when(voteNode.isObject()).thenReturn(true); + stubString(voteNode, "UUID", "player-uuid"); + stubString(voteNode, "Service", "Service"); + stubLong(voteNode, "Time", 100L); + when(storage.getStoragePath()).thenReturn(journal); + when(storage.getServerVotes("server")).thenReturn(List.of("4")); + when(storage.getServerVotes("server", "4")).thenReturn(voteNode, (DataNode) null); + handler = newVerifyingHandler(storage); + OfflineBungeeVote updated = vote(null, 100L); + + assertFalse(handler.updateServerVote("server", updated)); + verify(storage).reload(); + } finally { + Files.deleteIfExists(journal); + } + } + @Test public void currentVoteIdKeyLoadsFromJsonCache() { UUID voteId = UUID.randomUUID(); @@ -565,6 +604,24 @@ private static OfflineBungeeVote vote(UUID voteId, long time) { private static VoteCacheHandler newHandler(IVoteCache storage) { return new VoteCacheHandler(null, false, false, null, false, storage) { + @Override + protected boolean verifyJsonServerVote(String server, int index, OfflineBungeeVote vote) { + storage.save(); + return true; + } + + @Override + protected boolean verifyJsonOnlineVote(String uuid, int index, OfflineBungeeVote vote) { + storage.save(); + return true; + } + + @Override + protected boolean verifyJsonTimeVote(int index, VoteTimeQueue vote) { + storage.save(); + return true; + } + @Override public void logInfo1(String msg) { } @@ -586,4 +643,14 @@ public void debug1(String msg) { } }; } + + private static VoteCacheHandler newVerifyingHandler(IVoteCache storage) { + return new VoteCacheHandler(null, false, false, null, false, storage) { + @Override public void logInfo1(String msg) { } + @Override public void logSevere1(String msg) { } + @Override public void debug1(Exception e) { } + @Override public void debug1(Throwable e) { } + @Override public void debug1(String msg) { } + }; + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 865e9d330..6f535d5ba 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -446,6 +446,23 @@ void httpVotePartyKeepsCommandPendingUntilAsyncExecutionCompletesAndResumesReach assertEquals(0, votingPluginProxy.getVotePartyVotes()); } + @Test + void completedCommandWithUnpersistedProgressIsQuarantinedInsteadOfReplayed() { + configureHttpVotePartyEffects("", java.util.List.of("async command", "next command")); + java.util.concurrent.CompletableFuture completion = + votingPluginProxy.delayNextVotePartyCommandCompletion(); + + votingPluginProxy.checkVoteParty(); + votingPluginProxy.failNextVoteCacheSave(); + completion.complete(null); + + assertTrue(votingPluginProxy.retryPendingVotePartyProxyEffectsForTest()); + assertEquals(java.util.List.of("async command", "next command"), votingPluginProxy.getConsoleCommands()); + assertEquals(java.util.List.of("async command"), + votingPluginProxy.getVoteCacheQuarantinedVotePartyProxyEffects().commands()); + assertTrue(votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().isEmpty()); + } + @Test void httpVotePartyQuarantinesHungCommandWithoutRetryingIt() { configureHttpVotePartyEffects("", java.util.List.of("hung command", "next command")); @@ -501,7 +518,7 @@ void finalShutdownQuarantinesAnInFlightVotePartyCommandBeforeTeardown() { } @Test - void finalShutdownLeavesInFlightCommandFencedWhenQuarantineIsNotDurable() { + void finalShutdownRetriesTheDurableFenceBeforeTeardown() { configureHttpVotePartyEffects("", java.util.List.of("in flight", "next command")); java.util.concurrent.CompletableFuture completion = votingPluginProxy.delayNextVotePartyCommandCompletion(); @@ -510,12 +527,13 @@ void finalShutdownLeavesInFlightCommandFencedWhenQuarantineIsNotDurable() { votingPluginProxy.onDisable(false); - assertTrue(votingPluginProxy.getVoteCacheQuarantinedVotePartyProxyEffects().isEmpty()); - assertEquals(java.util.List.of("in flight", "next command"), + assertEquals(java.util.List.of("in flight"), + votingPluginProxy.getVoteCacheQuarantinedVotePartyProxyEffects().commands()); + assertEquals(java.util.List.of("next command"), votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().commands()); completion.complete(null); assertEquals(java.util.List.of("in flight"), votingPluginProxy.getConsoleCommands()); - assertEquals(java.util.List.of("in flight", "next command"), + assertEquals(java.util.List.of("next command"), votingPluginProxy.getVoteCachePendingVotePartyProxyEffects().commands()); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index 894444518..29a2d7a8a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -248,6 +248,11 @@ protected void scheduleVotePartyProxyCommandTimeout(Runnable timeout) { votePartyProxyCommandTimeout = timeout; } + @Override + protected void awaitInFlightVotePartyProxyCommand() { + // Keep shutdown-path tests deterministic; production performs the bounded wait. + } + @Override public void saveVoteCacheFile() { if (failNextVoteCacheSave) { From 3e5c168005869300e01a6d996a5228a6a6632f49 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:51:15 -0600 Subject: [PATCH 067/136] Make proxy vote delivery crash durable --- .../votingplugin/proxy/OfflineBungeeVote.java | 5 + .../votingplugin/proxy/VotingPluginProxy.java | 382 +++++++++++++++--- .../proxy/VotingPluginProxyCommand.java | 8 +- .../proxy/bungee/BungeeJsonVoteCache.java | 36 +- .../proxy/bungee/VoteEventBungee.java | 56 ++- .../cache/ProxyOnlineVoteCacheTable.java | 30 +- .../proxy/cache/ProxyVoteCacheTable.java | 34 +- .../proxy/cache/VoteCacheDurability.java | 39 ++ .../proxy/cache/VoteCacheHandler.java | 238 ++++++++--- .../proxy/velocity/VelocityJsonVoteCache.java | 16 + .../proxy/velocity/VoteEventVelocity.java | 49 ++- .../tests/JsonVoteCacheLegacyRemovalTest.java | 72 ++++ .../tests/VoteCacheHandlerVoteIdTest.java | 182 +++++++++ .../tests/VotingPluginProxyCommandTest.java | 24 ++ .../tests/VotingPluginProxyTest.java | 252 +++++++++++- .../tests/VotingPluginProxyTestImpl.java | 49 ++- 16 files changed, 1293 insertions(+), 179 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/JsonVoteCacheLegacyRemovalTest.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyCommandTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java index 965f8cedb..3923ef710 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java @@ -264,6 +264,11 @@ public String getHttpDeliveryId(String server) { return server == null ? null : httpDeliveryIds.get(server.toLowerCase(Locale.ROOT)); } + /** Returns a copy of stable reward delivery IDs by normalized target key. */ + public Map getHttpDeliveryIds() { + return new LinkedHashMap<>(httpDeliveryIds); + } + /** * Stores or removes a stable HTTP delivery ID for a target. * @param server target server diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index e5c0e78cb..5837045aa 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -104,8 +104,33 @@ import lombok.Setter; public abstract class VotingPluginProxy { + public static final class VoteRetryException extends IllegalStateException { + private static final long serialVersionUID = 1L; + + private VoteRetryException() { + super("Vote processing could not be made durable; retry is required"); + } + } private static final long PRESENCE_HANDOFF_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(2); private static final long PRESENCE_STARTUP_RESYNC_DELAY_SECONDS = 5L; + private static final int MAX_LIVE_VOTE_RETRIES = 1024; + private static final String REWARD_JOURNAL_TARGET_PREFIX = "__vp_reward_target__:"; + private final Map liveVoteRetries = new LinkedHashMap<>(); + + private static final class LiveVoteRetryState { + private VoteTotalsSnapshot totals; + private ArrayList totalsInput; + private boolean votePartyApplied; + private boolean totalsApplied; + private final Set broadcastForwardedServers = new LinkedHashSet<>(); + private final Set deliveredRewardServers = new LinkedHashSet<>(); + private final Map rewardStates = new LinkedHashMap<>(); + private Set rewardServers; + private boolean rewardJournalsDurable; + private OfflineBungeeVote standaloneBroadcastState; + private OfflineBungeeVote rewardJournalOwner; + + } private static final long PRESENCE_MAINTENANCE_INTERVAL_SECONDS = 30L; private static final long PRESENCE_BACKEND_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(90); private static final long CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10); @@ -819,8 +844,10 @@ && getConfig().getProxyBroadcastEnabled()) { delay++; num++; cache.setRewardDelivered(true); + if (!persistServerVoteDelivery(server, cache)) { + continue; + } if (cache.isProxyBroadcastHandled() && !cache.isProxyBroadcastComplete()) { - persistServerVoteDelivery(server, cache); continue; } removed.add(cache); @@ -854,8 +881,17 @@ public synchronized void checkOnlineVotes(String player, String uuid, String ser int num = 1; int numberOfVotes = (int) c.stream().filter(vote -> !vote.isRewardDelivered()).count(); boolean deliveredReward = false; - ArrayList retained = new ArrayList<>(); - for (OfflineBungeeVote cache : c) { + for (OfflineBungeeVote cache : new ArrayList<>(c)) { + if (isIncompleteRewardJournalOwner(cache)) { + if (!materializeRewardJournalOwner(uuid, cache)) continue; + if (!cache.isProxyBroadcastHandled() || cache.isProxyBroadcastComplete()) { + getVoteCacheHandler().removeOnlineVote(uuid, cache); + } + continue; + } + if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(uuid, cache)) { + continue; + } if (cache.isProxyBroadcastHandled()) { Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); pendingTargets.removeAll(cache.getBroadcastForwardedServers()); @@ -863,10 +899,12 @@ public synchronized void checkOnlineVotes(String player, String uuid, String ser if (blockedServers != null) { pendingTargets.removeAll(blockedServers); } - cache.getBroadcastForwardedServers().addAll(sendProxyBroadcast(pendingTargets, + boolean broadcastChanged = cache.getBroadcastForwardedServers().addAll(sendProxyBroadcast(pendingTargets, cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), cache.getText(), false, cache)); cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + if ((broadcastChanged || cache.isDeliveryStateDirty()) + && !persistOnlineVoteDelivery(uuid, cache)) continue; } boolean broadcastHere = !cache.isProxyBroadcastHandled() && cache.needsBroadcastOn(server); if (!cache.isProxyBroadcastHandled() && broadcastHere @@ -884,23 +922,19 @@ && getConfig().getProxyBroadcastEnabled()) { getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes), cache)) { debug("Retaining online vote because the transport rejected delivery for " + server); persistOnlineVoteDelivery(uuid, cache); - retained.add(cache); continue; } cache.setRewardDelivered(true); + if (!persistOnlineVoteDelivery(uuid, cache)) continue; deliveredReward = true; delay++; num++; } - if (cache.isProxyBroadcastHandled() && !cache.isProxyBroadcastComplete()) { - retained.add(cache); + if (!cache.isProxyBroadcastHandled() || cache.isProxyBroadcastComplete()) { + getVoteCacheHandler().removeOnlineVote(uuid, cache); } } - getVoteCacheHandler().removeOnlineVotes(uuid); - for (OfflineBungeeVote pending : retained) { - getVoteCacheHandler().addOnlineVote(uuid, pending); - } // multiproxy: envelope-only if (deliveredReward && getConfig().getMultiProxySupport() @@ -925,6 +959,9 @@ protected synchronized void retryPendingOnlineBroadcasts(String server) { } for (String cachedUuid : getVoteCacheHandler().getOnlineVoteUUIDs()) { for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { + if (isIncompleteRewardJournalOwner(cache) && !materializeRewardJournalOwner(cachedUuid, cache)) { + continue; + } if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { continue; } @@ -977,12 +1014,23 @@ protected synchronized void retryPendingTimeBroadcasts(String server) { * carrier event. */ public synchronized void retryPendingOnlineBroadcasts() { + if (!getVoteCacheHandler().retryPendingVotePersistence()) { + scheduleCachedVoteDeliveryRetry(); + } for (String cachedUuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { + if (isIncompleteRewardJournalOwner(cache) && !materializeRewardJournalOwner(cachedUuid, cache)) { + continue; + } if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { continue; } - if (!cache.isProxyBroadcastHandled() || cache.isProxyBroadcastComplete()) { + if (cache.isProxyBroadcastHandled() && cache.isRewardDelivered() + && cache.isProxyBroadcastComplete()) { + getVoteCacheHandler().removeOnlineVote(cachedUuid, cache); + continue; + } + if (!cache.isProxyBroadcastHandled()) { continue; } Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); @@ -1100,6 +1148,9 @@ private void scheduleCachedVoteDeliveryRetry() { } private synchronized void retryCachedVoteDeliveryPersistence() { + if (!getVoteCacheHandler().retryPendingVotePersistence()) { + scheduleCachedVoteDeliveryRetry(); + } for (String server : getVoteCacheHandler().getCachedVotesServers()) { for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getVotes(server))) { if (vote.isDeliveryStateDirty()) { @@ -4248,7 +4299,23 @@ public boolean checkVoteDelay(String uuid, String player, String service, ArrayL public synchronized void vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, VoteTotalsSnapshot text, String uuid) { - vote(player, service, realVote, timeQueue, queueTime, text, uuid, null); + UUID voteId = UUID.randomUUID(); + if (vote(player, service, realVote, timeQueue, queueTime, text, uuid, null, voteId) == QueuedVoteResult.RETRY) { + liveVoteRetries.remove(voteId); + throw new VoteRetryException(); + } + } + + public synchronized void vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid, UUID voteId) { + if (vote(player, service, realVote, timeQueue, queueTime, text, uuid, null, voteId) == QueuedVoteResult.RETRY) { + throw new VoteRetryException(); + } + } + + /** Releases retry-only state after the bounded event-listener retries are exhausted. */ + public synchronized void abandonLiveVoteRetry(UUID voteId) { + if (voteId != null) liveVoteRetries.remove(voteId); } private enum QueuedVoteResult { @@ -4257,6 +4324,11 @@ private enum QueuedVoteResult { private synchronized QueuedVoteResult vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, VoteTotalsSnapshot text, String uuid, VoteTimeQueue queuedVote) { + return vote(player, service, realVote, timeQueue, queueTime, text, uuid, queuedVote, null); + } + + private synchronized QueuedVoteResult vote(String player, String service, boolean realVote, boolean timeQueue, + long queueTime, VoteTotalsSnapshot text, String uuid, VoteTimeQueue queuedVote, UUID requestedVoteId) { try { if (!ServiceSiteValidator.isValid(service)) { warn("Rejected vote with invalid service site '" + ServiceSiteValidator.sanitizeForLog(service) + "'"); @@ -4271,8 +4343,10 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea UUID voteId = queuedVote == null ? null : queuedVote.getVoteId(); if (voteId == null) { - voteId = UUID.randomUUID(); + voteId = requestedVoteId == null ? UUID.randomUUID() : requestedVoteId; } + LiveVoteRetryState retryState = liveVoteRetries.get(voteId); + boolean resumingAfterTotals = retryState != null; // UUID resolution if (!getConfig().getOnlineMode()) { @@ -4347,17 +4421,21 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea : new LinkedHashSet<>(queuedVote.getBroadcastTargets()); Set broadcastForwardedServers = queuedVote == null ? new LinkedHashSet<>() : new LinkedHashSet<>(queuedVote.getBroadcastForwardedServers()); + if (retryState != null) { + broadcastForwardedServers.addAll(retryState.broadcastForwardedServers); + } boolean proxyBroadcastHandled = queuedVote != null && queuedVote.isProxyBroadcastHandled(); boolean processesTotals = getConfig().getPrimaryServer() || !getConfig().getMultiProxySupport(); boolean managesTotals = processesTotals && getConfig().getBungeeManageTotals(); boolean canValidateStandaloneBroadcast = canForwardStandaloneBroadcast(managesTotals); - ArrayList data = null; + ArrayList data = retryState == null ? null : retryState.totalsInput; boolean queueForTimeChange = false; // A completion callback can wipe totals and replay older queued votes. Run it // before loading this vote's database snapshot so the calculations below use // the post-rollover state. - if (getConfig().getGlobalDataEnabled() && getGlobalDataHandler().isTimeChangedHappened()) { + if (!resumingAfterTotals && getConfig().getGlobalDataEnabled() + && getGlobalDataHandler().isTimeChangedHappened()) { getGlobalDataHandler().checkForFinishedTimeChanges(); queueForTimeChange = timeQueue && getGlobalDataHandler().isTimeChangedHappened(); } @@ -4365,7 +4443,7 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea // Validate the vote before any immediate announcement. This keeps duplicate // votes rejected by the delay check out of the GlobalData rollover queue and // prevents announcing a vote that will not be processed. - if (managesTotals) { + if (!resumingAfterTotals && managesTotals) { if (getProxyMySQL() == null) { logSevere("Mysql is not loaded correctly, stopping vote processing"); return QueuedVoteResult.RETRY; @@ -4418,10 +4496,22 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea return QueuedVoteResult.SUCCESS; } - addVoteParty(); + if (retryState == null) { + if (liveVoteRetries.size() >= MAX_LIVE_VOTE_RETRIES) return QueuedVoteResult.RETRY; + retryState = new LiveVoteRetryState(); + retryState.totalsInput = data; + liveVoteRetries.put(voteId, retryState); + } + if (!retryState.votePartyApplied) { + // Fence the side effect before invoking it. If the call reports an + // indeterminate failure, a listener retry must not increment the party again. + retryState.votePartyApplied = true; + addVoteParty(); + } - // Totals processing (primary server OR no multiproxy) - if (processesTotals) { + if (!retryState.totalsApplied) { + // Totals processing (primary server OR no multiproxy) + if (processesTotals) { if (managesTotals) { int allTimeTotal = getValue(data, "AllTimeTotal", 1); int monthTotal = getValue(data, "MonthTotal", 1); @@ -4467,10 +4557,20 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea debug("Setting totals " + text.toString() + ", voteId=" + voteId + " for " + player + "/" + service); + retryState.totals = text; + retryState.totalsApplied = true; getProxyMySQL().update(uuid, update); } else { text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); } + } + if (text == null) { + text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); + } + retryState.totals = text; + retryState.totalsApplied = true; + } else { + text = retryState.totals; } if (text == null) { text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); @@ -4479,6 +4579,11 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea VoteLogStatus voteStatus = VoteLogStatus.IMMEDIATE; boolean standaloneProxyBroadcast = canValidateStandaloneBroadcast && (proxyBroadcastHandled || proxyBroadcastDecider.usesImmediateForwarding(playerOnline)); + if (getConfig().getSendVotesToAllServers() && retryState.rewardServers == null) { + retryState.rewardServers = new LinkedHashSet<>(getAllAvailableServers()); + } + Set rewardServers = retryState.rewardServers == null + ? Collections.emptySet() : retryState.rewardServers; Set proxyBroadcastTargets = Collections.emptySet(); OfflineBungeeVote standaloneBroadcastState = null; boolean standaloneBroadcastStatePersisted = false; @@ -4489,21 +4594,81 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea : proxyBroadcastDecider.resolveTargets(false, null); Set remainingTargets = new LinkedHashSet<>(proxyBroadcastTargets); remainingTargets.removeAll(broadcastForwardedServers); - standaloneBroadcastState = new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, - text == null ? "" : text.toString(), false, true, proxyBroadcastTargets, - broadcastForwardedServers, false, Collections.emptyMap(), queuedVote == null - ? Collections.emptyMap() : queuedVote.getHttpBroadcastDeliveryIds()); - Set forwarded = sendProxyBroadcast(remainingTargets, uuid, player, service, time, - text == null ? "" : text.toString(), false, standaloneBroadcastState); - broadcastForwardedServers.addAll(forwarded); - standaloneBroadcastState.getBroadcastForwardedServers().addAll(forwarded); + standaloneBroadcastState = retryState.standaloneBroadcastState; + if (standaloneBroadcastState == null) { + standaloneBroadcastState = new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, + text == null ? "" : text.toString(), false, true, proxyBroadcastTargets, + broadcastForwardedServers, !getConfig().getSendVotesToAllServers(), Collections.emptyMap(), queuedVote == null + ? Collections.emptyMap() : queuedVote.getHttpBroadcastDeliveryIds()); + if (getConfig().getSendVotesToAllServers()) markRewardJournalTargets(standaloneBroadcastState, rewardServers); + retryState.standaloneBroadcastState = standaloneBroadcastState; + retryState.rewardJournalOwner = standaloneBroadcastState; + } else { + proxyBroadcastTargets = new LinkedHashSet<>(standaloneBroadcastState.getBroadcastTargets()); + broadcastForwardedServers.addAll(standaloneBroadcastState.getBroadcastForwardedServers()); + remainingTargets = new LinkedHashSet<>(proxyBroadcastTargets); + remainingTargets.removeAll(broadcastForwardedServers); + } + // The canonical broadcast journal must reach durable storage before the + // first target sees the message. Otherwise a proxy crash after one accepted + // send can permanently lose every remaining target. + standaloneBroadcastStatePersisted = persistAndSendStandaloneBroadcast(uuid, + standaloneBroadcastState, remainingTargets, broadcastForwardedServers); + retryState.broadcastForwardedServers.addAll(broadcastForwardedServers); + if (!standaloneBroadcastStatePersisted) { + logSevere("Unable to durably journal standalone broadcast for " + uuid); + return QueuedVoteResult.RETRY; + } } // =========================== // Send vote(s) to backend(s) // =========================== if (getConfig().getSendVotesToAllServers()) { - for (String s : getAllAvailableServers()) { + OfflineBungeeVote rewardJournalOwner = retryState.rewardJournalOwner; + if (rewardJournalOwner == null) { + rewardJournalOwner = createCachedRewardVote(voteId, player, uuid, service, time, realVote, + text.toString(), false); + markRewardJournalTargets(rewardJournalOwner, rewardServers); + retryState.rewardJournalOwner = rewardJournalOwner; + if (!getVoteCacheHandler().addOnlineVoteDurably(uuid, rewardJournalOwner)) { + return QueuedVoteResult.RETRY; + } + } + for (String server : rewardServers) { + if (retryState.deliveredRewardServers.contains(server)) continue; + OfflineBungeeVote rewardState = retryState.rewardStates.get(server.toLowerCase(Locale.ROOT)); + if (rewardState == null) { + rewardState = createCachedRewardVote(voteId, player, uuid, service, time, + realVote, text.toString(), standaloneProxyBroadcast); + retryState.rewardStates.put(server.toLowerCase(Locale.ROOT), rewardState); + } + // Every target begins as pending so a crash before its send cannot lose + // the reward. Accepted deliveries are marked complete below. + rewardState.setRewardDelivered(false); + if (rewardState.getHttpDeliveryId(server) == null) { + rewardState.setHttpDeliveryId(server, rewardJournalDeliveryId(rewardJournalOwner, server)); + } + if (!getVoteCacheHandler().addServerVoteDurably(server, rewardState)) { + if (getVoteCacheHandler().retainServerVoteForPersistenceRetry(server, rewardState)) { + scheduleCachedVoteDeliveryRetry(); + } + logSevere("Unable to durably journal vote reward for " + server); + return QueuedVoteResult.RETRY; + } + } + if (!retryState.rewardJournalsDurable) { + // The source owner is removable only after every target-specific row is durable. + rewardJournalOwner.setRewardDelivered(true); + rewardJournalOwner.setDeliveryStateDirty(true); + if (!persistOnlineVoteDelivery(uuid, rewardJournalOwner)) return QueuedVoteResult.RETRY; + retryState.rewardJournalsDurable = true; + if (!rewardJournalOwner.isProxyBroadcastHandled() || rewardJournalOwner.isProxyBroadcastComplete()) { + getVoteCacheHandler().removeOnlineVote(uuid, rewardJournalOwner); + } + } + for (String s : rewardServers) { + if (retryState.deliveredRewardServers.contains(s)) continue; boolean forceCache = getConfig().getWaitForUserOnline() && (!playerOnline || playerServer == null || !playerServer.equalsIgnoreCase(s)); @@ -4514,8 +4679,6 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea if ((!isSomeoneOnlineServerForVoteRouting(s) && method.requiresPlayerOnline()) || forceCache) { voteStatus = VoteLogStatus.CACHED; - getVoteCacheHandler().addServerVote(s, createCachedRewardVote(voteId, player, uuid, service, time, - realVote, text.toString(), standaloneProxyBroadcast)); debug("Caching vote for " + player + " on " + service + " for " + s); } else { boolean broadcastHere = !broadcastForwardedServers.contains(s); @@ -4525,25 +4688,26 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea broadcastHere = proxyBroadcastDecider.shouldBroadcast(s, targets); } - OfflineBungeeVote pendingVote = standaloneProxyBroadcast - ? new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), false, - true, proxyBroadcastTargets, broadcastForwardedServers, false, - Collections.emptyMap(), standaloneBroadcastState.getHttpBroadcastDeliveryIds()) - : createCachedRewardVote(voteId, player, uuid, service, time, realVote, text.toString(), false); + OfflineBungeeVote pendingVote = retryState.rewardStates.get(s.toLowerCase(Locale.ROOT)); boolean rewardAccepted = sendVoteEnvelopeAccepted(s, 2, VotingPluginWire.vote(player, uuid, service, time, true, realVote, text.toString(), voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote); if (!rewardAccepted) { - OfflineBungeeVote fallbackVote = standaloneProxyBroadcast - ? createCachedRewardVote(pendingVote, true) : pendingVote; - if (!getVoteCacheHandler().addServerVoteDurably(s, fallbackVote)) { - logSevere("Unable to durably cache the rejected vote delivery for " + s); + pendingVote.setRewardDelivered(false); + pendingVote.setDeliveryStateDirty(true); + if (!persistServerVoteDelivery(s, pendingVote)) { + logSevere("Unable to persist the rejected vote delivery for " + s); return QueuedVoteResult.RETRY; } voteStatus = VoteLogStatus.CACHED; debug("Caching vote after the transport rejected delivery for " + s); - } else if (standaloneBroadcastState != null) { - standaloneBroadcastState.setRewardDelivered(true); + } else { + pendingVote.setRewardDelivered(true); + retryState.deliveredRewardServers.add(s); + pendingVote.setDeliveryStateDirty(true); + if (!persistServerVoteDelivery(s, pendingVote)) return QueuedVoteResult.RETRY; + getVoteCacheHandler().removeServerVotes(s, + new ArrayList<>(Collections.singletonList(pendingVote))); } } } @@ -4560,24 +4724,43 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); } - OfflineBungeeVote pendingVote = standaloneProxyBroadcast + OfflineBungeeVote pendingVote = retryState.rewardStates.get(server.toLowerCase(Locale.ROOT)); + if (pendingVote == null) { + pendingVote = standaloneProxyBroadcast ? new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), false, true, proxyBroadcastTargets, broadcastForwardedServers, false, Collections.emptyMap(), standaloneBroadcastState.getHttpBroadcastDeliveryIds()) : createCachedRewardVote(voteId, player, uuid, service, time, realVote, text.toString(), false); - boolean rewardAccepted = sendVoteEnvelopeAccepted(server, 1, - VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(), - voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote); + retryState.rewardStates.put(server.toLowerCase(Locale.ROOT), pendingVote); + } + boolean rewardAccepted = retryState.deliveredRewardServers.contains(server); + if (!rewardAccepted) { + rewardAccepted = sendVoteEnvelopeAccepted(server, 1, + VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(), + voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1), pendingVote); + if (rewardAccepted) retryState.deliveredRewardServers.add(server); + } if (!rewardAccepted) { + if (standaloneBroadcastState != null) { + standaloneBroadcastState.setRewardDelivered(false); + standaloneBroadcastState.setDeliveryStateDirty(true); + if (!persistOnlineVoteDelivery(uuid, standaloneBroadcastState)) { + return QueuedVoteResult.RETRY; + } + } if (!getVoteCacheHandler().addOnlineVoteDurably(uuid, pendingVote)) { + if (getVoteCacheHandler().retainOnlineVoteForPersistenceRetry(uuid, pendingVote)) { + scheduleCachedVoteDeliveryRetry(); + } logSevere("Unable to durably cache the rejected online vote delivery for " + uuid); return QueuedVoteResult.RETRY; } voteStatus = VoteLogStatus.CACHED; standaloneBroadcastStatePersisted |= standaloneBroadcastState != null; debug("Caching online vote after the transport rejected delivery for " + server); - } else if (standaloneBroadcastState != null) { - standaloneBroadcastState.setRewardDelivered(true); + } else if (standaloneBroadcastState != null + && standaloneBroadcastState.isProxyBroadcastComplete()) { + getVoteCacheHandler().removeOnlineVote(uuid, standaloneBroadcastState); } if (rewardAccepted && canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled() @@ -4607,11 +4790,27 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea } } else { voteStatus = VoteLogStatus.CACHED; - getVoteCacheHandler().addOnlineVote(uuid, standaloneProxyBroadcast + if (standaloneBroadcastState != null) { + standaloneBroadcastState.setRewardDelivered(false); + standaloneBroadcastState.setDeliveryStateDirty(true); + if (!persistOnlineVoteDelivery(uuid, standaloneBroadcastState)) { + return QueuedVoteResult.RETRY; + } + } + OfflineBungeeVote cachedReward = standaloneProxyBroadcast ? new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), false, true, proxyBroadcastTargets, broadcastForwardedServers, false, Collections.emptyMap(), standaloneBroadcastState.getHttpBroadcastDeliveryIds()) - : createCachedRewardVote(voteId, player, uuid, service, time, realVote, text.toString(), false)); + : createCachedRewardVote(voteId, player, uuid, service, time, realVote, text.toString(), false); + boolean cachedDurably = getVoteCacheHandler().addOnlineVoteDurably(uuid, cachedReward); + if (!cachedDurably) { + if (getVoteCacheHandler().retainOnlineVoteForPersistenceRetry(uuid, cachedReward)) { + scheduleCachedVoteDeliveryRetry(); + } + logSevere("Unable to durably cache online vote for " + uuid + + "; retaining it for persistence retry"); + return QueuedVoteResult.RETRY; + } standaloneBroadcastStatePersisted |= standaloneBroadcastState != null; debug("Caching online vote for " + player + " on " + service); } @@ -4624,7 +4823,11 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea } } - persistUncachedStandaloneBroadcast(uuid, standaloneBroadcastState, standaloneBroadcastStatePersisted); + if (!persistUncachedStandaloneBroadcast(uuid, standaloneBroadcastState, + standaloneBroadcastStatePersisted)) { + logSevere("Unable to durably cache the pending standalone broadcast for " + uuid); + return QueuedVoteResult.RETRY; + } // Vote logging if (voteLogMysqlTable != null && getConfig().getVoteLoggingEnabled()) { @@ -4667,6 +4870,7 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea + "; attempting durable removal immediately"); } } + liveVoteRetries.remove(voteId); return QueuedVoteResult.SUCCESS; } catch (Exception e) { e.printStackTrace(); @@ -4687,6 +4891,55 @@ protected OfflineBungeeVote createCachedRewardVote(UUID voteId, String player, S Collections.emptyMap(), Collections.emptyMap()); } + private void markRewardJournalTargets(OfflineBungeeVote owner, Set targets) { + for (String server : targets) { + String key = owner.getVoteId() + ":reward:" + server; + owner.setHttpDeliveryId(REWARD_JOURNAL_TARGET_PREFIX + server, + UUID.nameUUIDFromBytes(key.getBytes(StandardCharsets.UTF_8)).toString()); + } + } + + private boolean isIncompleteRewardJournalOwner(OfflineBungeeVote vote) { + if (vote == null || vote.isRewardDelivered()) return false; + for (String key : vote.getHttpDeliveryIds().keySet()) { + if (key.startsWith(REWARD_JOURNAL_TARGET_PREFIX)) return true; + } + return false; + } + + private String rewardJournalDeliveryId(OfflineBungeeVote owner, String server) { + String encoded = owner.getHttpDeliveryId(REWARD_JOURNAL_TARGET_PREFIX + server); + return encoded; + } + + private boolean materializeRewardJournalOwner(String uuid, OfflineBungeeVote owner) { + for (Map.Entry entry : owner.getHttpDeliveryIds().entrySet()) { + if (!entry.getKey().startsWith(REWARD_JOURNAL_TARGET_PREFIX)) continue; + String normalizedServer = entry.getKey().substring(REWARD_JOURNAL_TARGET_PREFIX.length()); + if (normalizedServer.isEmpty()) continue; + String server = normalizedServer; + for (String configured : getAllConfiguredServers()) { + if (configured.equalsIgnoreCase(normalizedServer)) { + server = configured; + break; + } + } + String deliveryId = entry.getValue(); + OfflineBungeeVote reward = createCachedRewardVote(owner.getVoteId(), owner.getPlayerName(), owner.getUuid(), + owner.getService(), owner.getTime(), owner.isRealVote(), owner.getText(), + owner.isProxyBroadcastHandled()); + reward.setHttpDeliveryId(server, deliveryId); + if (!getVoteCacheHandler().addServerVoteDurably(server, reward)) return false; + } + owner.setRewardDelivered(true); + owner.setDeliveryStateDirty(true); + if (!persistOnlineVoteDelivery(uuid, owner)) return false; + if (!owner.isProxyBroadcastHandled() || owner.isProxyBroadcastComplete()) { + getVoteCacheHandler().removeOnlineVote(uuid, owner); + } + return true; + } + private OfflineBungeeVote createCachedRewardVote(OfflineBungeeVote deliveryState, boolean standaloneProxyBroadcast) { return new OfflineBungeeVote(deliveryState.getVoteId(), deliveryState.getPlayerName(), deliveryState.getUuid(), @@ -4695,16 +4948,37 @@ private OfflineBungeeVote createCachedRewardVote(OfflineBungeeVote deliveryState OfflineBungeeVote.decodeHttpDeliveryIds(deliveryState.encodeHttpDeliveryIds()), Collections.emptyMap()); } - protected void persistUncachedStandaloneBroadcast(String uuid, OfflineBungeeVote state, + protected boolean persistUncachedStandaloneBroadcast(String uuid, OfflineBungeeVote state, boolean alreadyPersisted) { - if (alreadyPersisted || state == null || state.isProxyBroadcastComplete()) return; + if (alreadyPersisted || state == null || state.isProxyBroadcastComplete()) return true; // This canonical row retries only the standalone broadcast. Reward delivery // remains owned by the target-specific server/online cache entry. state.setRewardDelivered(true); state.setBroadcastForwarded(false); - getVoteCacheHandler().addOnlineVote(uuid, state); + if (getVoteCacheHandler().addOnlineVoteDurably(uuid, state)) return true; + boolean retained = getVoteCacheHandler().retainOnlineVoteForPersistenceRetry(uuid, state); + if (retained) scheduleCachedVoteDeliveryRetry(); + return false; } + protected boolean persistAndSendStandaloneBroadcast(String uuid, OfflineBungeeVote state, + Set remainingTargets, Set forwardedServers) { + if (!getVoteCacheHandler().addOnlineVoteDurably(uuid, state)) { + boolean retained = getVoteCacheHandler().retainOnlineVoteForPersistenceRetry(uuid, state); + if (retained) scheduleCachedVoteDeliveryRetry(); + return false; + } + for (String target : remainingTargets) { + Set forwarded = sendProxyBroadcast(Collections.singleton(target), uuid, state.getPlayerName(), + state.getService(), state.getTime(), state.getText(), false, state); + forwardedServers.addAll(forwarded); + if (state.getBroadcastForwardedServers().addAll(forwarded) || state.isDeliveryStateDirty()) { + state.setBroadcastForwarded(state.isProxyBroadcastComplete()); + if (!persistOnlineVoteDelivery(uuid, state)) return false; + } + } + return true; + } private static final class PendingPresenceHandoff { private UUID requestId; private final UUID playerUuid; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java index 871a86fd7..d631b7abc 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java @@ -32,8 +32,12 @@ public String execute(String[] args) { if (args.length >= 3) { String user = args[1]; String site = args[2]; - plugin.vote(user, site, false, true, 0, null, null); - return "&aVote sent for " + user + " on " + site; + try { + plugin.vote(user, site, false, true, 0, null, null); + return "&aVote sent for " + user + " on " + site; + } catch (VotingPluginProxy.VoteRetryException retryable) { + return "&cVote could not be stored safely. Please retry shortly."; + } } return "&cUsage: vote "; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java index f6cfcab72..636c50aa0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java @@ -278,12 +278,12 @@ public void removeServerVote(String server, String uuid) { return; } // search for vote with uuid and remove it - for (String num : votes) { - GsonDataNode node = getServerVotes(server, num); + for (String num : votes) { + GsonDataNode node = getServerVotes(server, num); if (node == null) { continue; } - DataNode uuidNode = node.get("UUID"); + DataNode uuidNode = node.get("UUID"); if (uuidNode == null) { continue; } @@ -302,10 +302,18 @@ public void removeVote(String server, OfflineBungeeVote vote) { } for (String num : votes) { GsonDataNode node = getServerVotes(server, num); - if (node == null) { - continue; - } - DataNode uuidNode = node.get("UUID"); + if (node == null) { + continue; + } + DataNode voteIdNode = node.has("VoteId") ? node.get("VoteId") + : node.has("VoteID") ? node.get("VoteID") : null; + if (vote.getVoteId() != null && voteIdNode != null) { + if (vote.getVoteId().toString().equals(voteIdNode.asString())) { + setString("VoteCache." + server + "." + num, null); + } + continue; + } + DataNode uuidNode = node.get("UUID"); DataNode serviceNode = node.get("Service"); DataNode timeNode = node.get("Time"); if (uuidNode == null || serviceNode == null || timeNode == null) { @@ -332,12 +340,20 @@ public void removeOnlineVote(OfflineBungeeVote vote) { if (onlineVotes == null) { continue; } - for (String num : onlineVotes) { - GsonDataNode node = getOnlineVotes(player, num); + for (String num : onlineVotes) { + GsonDataNode node = getOnlineVotes(player, num); if (node == null) { continue; } - DataNode uuidNode = node.get("UUID"); + DataNode voteIdNode = node.has("VoteId") ? node.get("VoteId") + : node.has("VoteID") ? node.get("VoteID") : null; + if (vote.getVoteId() != null && voteIdNode != null) { + if (vote.getVoteId().toString().equals(voteIdNode.asString())) { + setString("OnlineCache." + player + "." + num, null); + } + continue; + } + DataNode uuidNode = node.get("UUID"); DataNode serviceNode = node.get("Service"); DataNode timeNode = node.get("Time"); if (uuidNode == null || serviceNode == null || timeNode == null) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VoteEventBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VoteEventBungee.java index 79c41a464..fa826804a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VoteEventBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VoteEventBungee.java @@ -1,6 +1,10 @@ package com.bencodez.votingplugin.proxy.bungee; +import java.util.concurrent.TimeUnit; +import java.util.UUID; + import com.bencodez.votingplugin.util.MinecraftUsernameValidator; +import com.bencodez.votingplugin.proxy.VotingPluginProxy.VoteRetryException; import com.vexsoftware.votifier.bungee.events.VotifierEvent; import com.vexsoftware.votifier.model.Vote; @@ -26,25 +30,43 @@ public VoteEventBungee(VotingPluginBungee plugin) { */ @EventHandler public void onVote(VotifierEvent event) { - plugin.getProxy().getScheduler().runAsync(plugin, new Runnable() { - - @SuppressWarnings("deprecation") - @Override - public void run() { - Vote vote = event.getVote(); - String serviceSite = vote.getServiceName(); - plugin.getLogger().info("Vote received " + MinecraftUsernameValidator.sanitizeForLog(vote.getUsername()) - + " from service site " + MinecraftUsernameValidator.sanitizeForLog(serviceSite)); - - if (serviceSite.isEmpty()) { - serviceSite = "Empty"; - vote.setServiceName(serviceSite); - } + Vote vote = event.getVote(); + String serviceSite = vote.getServiceName().isEmpty() ? "Empty" : vote.getServiceName(); + plugin.getProxy().getScheduler().runAsync(plugin, + new RetryingVote(vote.getUsername(), serviceSite)); - plugin.getVotingPluginProxy().vote(vote.getUsername(), serviceSite, true, true, 0, null, null); - } - }); + } + private final class RetryingVote implements Runnable { + private static final int MAX_ATTEMPTS = 12; + private final String player; + private final String service; + private final UUID voteId = UUID.randomUUID(); + private int attempts; + + private RetryingVote(String player, String service) { + this.player = player; + this.service = service; + } + + @Override + public void run() { + plugin.getLogger().info("Vote received " + MinecraftUsernameValidator.sanitizeForLog(player) + + " from service site " + MinecraftUsernameValidator.sanitizeForLog(service)); + try { + plugin.getVotingPluginProxy().vote(player, service, true, true, 0, null, null, voteId); + } catch (VoteRetryException retryable) { + attempts++; + if (attempts < MAX_ATTEMPTS) { + plugin.getLogger().warning("Vote processing is waiting for durable storage; retrying shortly"); + plugin.getProxy().getScheduler().schedule(plugin, this, 5, TimeUnit.SECONDS); + } else { + plugin.getVotingPluginProxy().abandonLiveVoteRetry(voteId); + plugin.getLogger().severe("Vote processing exhausted bounded durable-storage retries for " + + MinecraftUsernameValidator.sanitizeForLog(player)); + } + } + } } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java index d8dd31b8d..fcdd4dea1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java @@ -429,24 +429,34 @@ public void removeVoteById(int id) { * @param vote the vote to remove */ public void removeVote(OfflineBungeeVote vote) { - String sql = "DELETE FROM " + qi(getTableName()) + " WHERE " + qi("uuid") + " = ? AND " + qi("service") - + " = ? AND " + qi("time") + " = ?;"; + tryRemoveVote(vote); + } + + /** Removes one stable vote identity and reports whether the statement completed. */ + public boolean tryRemoveVote(OfflineBungeeVote vote) { + boolean byVoteId = vote.getVoteId() != null; + String sql = "DELETE FROM " + qi(getTableName()) + " WHERE " + + (byVoteId ? qi("voteid") + " = ?" : qi("uuid") + " = ? AND " + qi("service") + + " = ? AND " + qi("time") + " = ?") + ";"; try (Connection conn = mysql.getConnectionManager().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { - - if (getDbType() == DbType.POSTGRESQL) { - ps.setObject(1, UUID.fromString(vote.getUuid())); + if (byVoteId) { + ps.setString(1, vote.getVoteId().toString()); } else { - ps.setString(1, vote.getUuid()); + if (getDbType() == DbType.POSTGRESQL) { + ps.setObject(1, UUID.fromString(vote.getUuid())); + } else { + ps.setString(1, vote.getUuid()); + } + ps.setString(2, vote.getService()); + ps.setLong(3, vote.getTime()); } - - ps.setString(2, vote.getService()); - ps.setLong(3, vote.getTime()); - ps.executeUpdate(); + return true; } catch (SQLException | IllegalArgumentException e) { debug(e); + return false; } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java index f2bd62c9d..1af57ebd7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java @@ -529,25 +529,37 @@ public void removeVotesByServerAndUUID(String server, String uuid) { * @param server the server name */ public void removeVote(OfflineBungeeVote vote, String server) { - String sql = "DELETE FROM " + qi(getTableName()) + " WHERE " + qi("uuid") + " = ? AND " + qi("service") - + " = ? AND " + qi("time") + " = ? AND " + qi("server") + " = ?;"; + tryRemoveVote(vote, server); + } + + /** Removes one stable vote identity and reports whether the statement completed. */ + public boolean tryRemoveVote(OfflineBungeeVote vote, String server) { + boolean byVoteId = vote.getVoteId() != null; + String sql = "DELETE FROM " + qi(getTableName()) + " WHERE " + + (byVoteId ? qi("voteid") + " = ?" : qi("uuid") + " = ? AND " + qi("service") + + " = ? AND " + qi("time") + " = ?") + + " AND " + qi("server") + " = ?;"; try (Connection conn = mysql.getConnectionManager().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { - - if (getDbType() == DbType.POSTGRESQL) { - ps.setObject(1, UUID.fromString(vote.getUuid())); + if (byVoteId) { + ps.setString(1, vote.getVoteId().toString()); + ps.setString(2, server); } else { - ps.setString(1, vote.getUuid()); + if (getDbType() == DbType.POSTGRESQL) { + ps.setObject(1, UUID.fromString(vote.getUuid())); + } else { + ps.setString(1, vote.getUuid()); + } + ps.setString(2, vote.getService()); + ps.setLong(3, vote.getTime()); + ps.setString(4, server); } - - ps.setString(2, vote.getService()); - ps.setLong(3, vote.getTime()); - ps.setString(4, server); - ps.executeUpdate(); + return true; } catch (SQLException | IllegalArgumentException e) { debug(e); + return false; } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheDurability.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheDurability.java index 32fd6884c..7b61ccb59 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheDurability.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheDurability.java @@ -1,6 +1,7 @@ package com.bencodez.votingplugin.proxy.cache; import java.nio.file.Path; +import java.util.function.BooleanSupplier; import com.bencodez.votingplugin.proxy.OfflineBungeeVote; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; @@ -8,6 +9,15 @@ /** Forces and reads back emergency JSON journal entries before they are exposed in memory. */ public final class VoteCacheDurability { + public static final class ReloadFailedException extends IllegalStateException { + private static final long serialVersionUID = 1L; + + private ReloadFailedException(Throwable saveFailure, Throwable reloadFailure) { + super("Unable to restore the JSON vote journal after a failed durable save", saveFailure); + addSuppressed(reloadFailure); + } + } + private VoteCacheDurability() { } public static boolean saveAndVerifyServerVote(IVoteCache cache, String server, int index, @@ -25,6 +35,27 @@ public static boolean saveAndVerifyTimeVote(IVoteCache cache, int index, VoteTim data -> matches(data, expected)); } + /** Publishes a cache deletion and verifies the removed identity remains absent after reload. */ + public static boolean saveAndVerifyRemoval(IVoteCache cache, BooleanSupplier isAbsent) { + if (cache == null || isAbsent == null || cache.getStoragePath() == null) return false; + try { + try { + cache.saveDurably(); + } catch (DurableFiles.PublishedException published) { + // The replacement is active; verification below determines the result. + } + cache.reload(); + return isAbsent.getAsBoolean(); + } catch (RuntimeException | java.io.IOException failure) { + try { + cache.reload(); + } catch (RuntimeException reloadFailure) { + throw new ReloadFailedException(failure, reloadFailure); + } + return false; + } + } + private static boolean saveAndVerify(IVoteCache cache, java.util.function.Supplier read, OfflineBungeeVote expected) { return saveAndVerify(cache, read, data -> matches(data, expected)); @@ -45,6 +76,14 @@ private static boolean saveAndVerify(IVoteCache cache, java.util.function.Suppli cache.reload(); return verify.test(read.get()); } catch (RuntimeException | java.io.IOException failure) { + // addVote* mutates the JSON object before saveDurably runs. Discard that + // uncommitted mutation so a later retry reuses the same slot instead of + // eventually publishing both the failed attempt and its retry. + try { + cache.reload(); + } catch (RuntimeException reloadFailure) { + throw new ReloadFailedException(failure, reloadFailure); + } return false; } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index 022063186..434caa6f9 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -24,6 +24,7 @@ * Handles caching of votes for proxy servers. */ public abstract class VoteCacheHandler { + private static final int MAX_PENDING_PERSISTENCE_VOTES = 1024; /** * Queue of timed votes for time change processing. @@ -37,6 +38,16 @@ public abstract class VoteCacheHandler { // server based private ConcurrentHashMap> cachedVotes = new ConcurrentHashMap<>(); + // Entries whose durable insert failed remain isolated from delivery until a + // retry succeeds. This prevents both silent loss and dispatch from memory + // before the cache/outbox record is durable. + private final ConcurrentHashMap> pendingOnlineVotePersistence = + new ConcurrentHashMap<>(); + private final ConcurrentHashMap> pendingServerVotePersistence = + new ConcurrentHashMap<>(); + private int pendingPersistenceVoteCount; + private boolean jsonStorageQuarantined; + /** * Checks if a server has cached votes. * @param server the server name @@ -132,6 +143,19 @@ public synchronized boolean addServerVoteDurably(String server, OfflineBungeeVot return stored; } + /** Retains a failed server-cache insert for a later durability retry. */ + public synchronized boolean retainServerVoteForPersistenceRetry(String server, OfflineBungeeVote vote) { + if (server == null || vote == null) return false; + ArrayList pending = pendingServerVotePersistence.get(server); + if (containsVoteId(pending, vote.getVoteId())) return true; + if (pendingPersistenceVoteCount >= MAX_PENDING_PERSISTENCE_VOTES) return false; + pendingServerVotePersistence.putIfAbsent(server, new ArrayList<>()); + pending = pendingServerVotePersistence.get(server); + pending.add(vote); + pendingPersistenceVoteCount++; + return true; + } + /** * Persists updated delivery state for an existing server-cached vote. * @@ -145,6 +169,7 @@ public synchronized boolean updateServerVote(String server, OfflineBungeeVote vo } return updateServerVoteJson(server, vote); } + if (jsonStorage == null || jsonStorageQuarantined) return false; Collection keys = jsonStorage.getServerVotes(server); if (keys == null) { @@ -183,7 +208,7 @@ public void removeVote(String server, String uuid) { if (useMySQL) { voteCacheTable.removeVotesByServerAndUUID(server, uuid); } - if (jsonStorage != null) { + if (jsonStorage != null && !jsonStorageQuarantined) { jsonStorage.removeServerVote(server, uuid); jsonStorage.save(); } @@ -198,7 +223,7 @@ public void removeVotes(String server) { if (useMySQL) { voteCacheTable.removeVotesByServer(server); } - if (jsonStorage != null) { + if (jsonStorage != null && !jsonStorageQuarantined) { jsonStorage.removeServerVotes(server); jsonStorage.save(); } @@ -277,6 +302,53 @@ public synchronized boolean addOnlineVoteDurably(String uuid, OfflineBungeeVote return stored; } + /** Retains a failed online-cache insert for a later durability retry. */ + public synchronized boolean retainOnlineVoteForPersistenceRetry(String uuid, OfflineBungeeVote vote) { + if (uuid == null || vote == null) return false; + ArrayList pending = pendingOnlineVotePersistence.get(uuid); + if (containsVoteId(pending, vote.getVoteId())) return true; + if (pendingPersistenceVoteCount >= MAX_PENDING_PERSISTENCE_VOTES) return false; + pendingOnlineVotePersistence.putIfAbsent(uuid, new ArrayList<>()); + pending = pendingOnlineVotePersistence.get(uuid); + pending.add(vote); + pendingPersistenceVoteCount++; + return true; + } + + /** + * Retries failed cache inserts. Entries become visible to normal delivery only + * after their ordinary durable insert succeeds. + */ + public synchronized boolean retryPendingVotePersistence() { + for (Map.Entry> entry + : new ArrayList<>(pendingServerVotePersistence.entrySet())) { + for (OfflineBungeeVote vote : new ArrayList<>(entry.getValue())) { + if (addServerVoteDurably(entry.getKey(), vote) && entry.getValue().remove(vote)) { + pendingPersistenceVoteCount--; + } + } + if (entry.getValue().isEmpty()) pendingServerVotePersistence.remove(entry.getKey()); + } + for (Map.Entry> entry + : new ArrayList<>(pendingOnlineVotePersistence.entrySet())) { + for (OfflineBungeeVote vote : new ArrayList<>(entry.getValue())) { + if (addOnlineVoteDurably(entry.getKey(), vote) && entry.getValue().remove(vote)) { + pendingPersistenceVoteCount--; + } + } + if (entry.getValue().isEmpty()) pendingOnlineVotePersistence.remove(entry.getKey()); + } + return pendingServerVotePersistence.isEmpty() && pendingOnlineVotePersistence.isEmpty(); + } + + private boolean containsVoteId(Collection votes, UUID voteId) { + if (votes == null || voteId == null) return false; + for (OfflineBungeeVote candidate : votes) { + if (candidate != null && voteId.equals(candidate.getVoteId())) return true; + } + return false; + } + /** * Persists updated delivery state for an existing voter-keyed cached vote. * @@ -290,6 +362,7 @@ public synchronized boolean updateOnlineVote(String uuid, OfflineBungeeVote vote } return updateOnlineVoteJson(uuid, vote); } + if (jsonStorage == null || jsonStorageQuarantined) return false; Collection keys = jsonStorage.getOnlineVotes(uuid); if (keys == null) { @@ -327,17 +400,16 @@ public synchronized void clearOnlineVoteRewards(String uuid) { return; } - ArrayList retained = new ArrayList<>(); - for (OfflineBungeeVote vote : votes) { + for (OfflineBungeeVote vote : new ArrayList<>(votes)) { if (vote.isProxyBroadcastHandled() && !vote.isProxyBroadcastComplete()) { - vote.setRewardDelivered(true); - retained.add(vote); + if (!vote.isRewardDelivered()) { + vote.setRewardDelivered(true); + vote.setDeliveryStateDirty(true); + } + if (vote.isDeliveryStateDirty() && updateOnlineVote(uuid, vote)) vote.setDeliveryStateDirty(false); + continue; } - } - - removeOnlineVotes(uuid); - for (OfflineBungeeVote vote : retained) { - addOnlineVote(uuid, vote); + removeOnlineVote(uuid, vote); } } @@ -348,22 +420,47 @@ public synchronized void clearOnlineVoteRewards(String uuid) { * @param removedVote vote to remove */ public synchronized void removeOnlineVote(String uuid, OfflineBungeeVote removedVote) { + tryRemoveOnlineVote(uuid, removedVote); + } + + /** Removes one voter-keyed cached vote after every configured store confirms deletion. */ + public synchronized boolean tryRemoveOnlineVote(String uuid, OfflineBungeeVote removedVote) { ArrayList votes = cachedOnlineVotes.get(uuid); if (votes == null || votes.isEmpty()) { - return; + return true; } + boolean mysqlRemoved = !useMySQL || onlineVoteCacheTable.tryRemoveVote(removedVote); + boolean jsonRemoved = jsonStorage == null; + if (jsonStorage != null && !jsonStorageQuarantined) { + jsonRemoved = removeJsonOnlineVoteDurably(uuid, removedVote); + } + boolean removedDurably = mysqlRemoved && jsonRemoved; + if (!removedDurably) return false; + votes.removeIf(vote -> sameVoteIdentity(vote, removedVote)); + if (votes.isEmpty()) cachedOnlineVotes.remove(uuid); + return true; + } - ArrayList retained = new ArrayList<>(); - for (OfflineBungeeVote vote : votes) { - if (!sameVoteIdentity(vote, removedVote)) { - retained.add(vote); - } + protected boolean removeJsonOnlineVoteDurably(String uuid, OfflineBungeeVote removedVote) { + jsonStorage.removeOnlineVote(removedVote); + try { + return VoteCacheDurability.saveAndVerifyRemoval(jsonStorage, + () -> !containsStoredOnlineVote(uuid, removedVote)); + } catch (VoteCacheDurability.ReloadFailedException failure) { + jsonStorageQuarantined = true; + debug1(failure); + return false; } + } - removeOnlineVotes(uuid); - for (OfflineBungeeVote vote : retained) { - addOnlineVote(uuid, vote); + private boolean containsStoredOnlineVote(String uuid, OfflineBungeeVote expected) { + Collection keys = jsonStorage.getOnlineVotes(uuid); + if (keys == null) return false; + for (String key : keys) { + DataNode data = jsonStorage.getOnlineVotes(uuid, key); + if (data != null && matchesStoredVote(data, expected)) return true; } + return false; } /** @@ -375,7 +472,7 @@ public void removeOnlineVotes(String uuid) { if (useMySQL) { onlineVoteCacheTable.removeVotesByUuid(uuid); } - if (jsonStorage != null) { + if (jsonStorage != null && !jsonStorageQuarantined) { jsonStorage.removeOnlineVotes(uuid); jsonStorage.save(); } @@ -486,7 +583,7 @@ private boolean matchesStoredVote(DataNode data, OfflineBungeeVote vote) { /** Persists a server vote in the JSON emergency journal. */ private boolean persistServerVoteToJson(String server, OfflineBungeeVote vote) { - if (jsonStorage == null) { + if (jsonStorage == null || jsonStorageQuarantined) { return false; } try { @@ -502,7 +599,7 @@ private boolean persistServerVoteToJson(String server, OfflineBungeeVote vote) { /** Persists an online vote in the JSON emergency journal. */ private boolean persistOnlineVoteToJson(String uuid, OfflineBungeeVote vote) { - if (jsonStorage == null) { + if (jsonStorage == null || jsonStorageQuarantined) { return false; } try { @@ -517,15 +614,33 @@ private boolean persistOnlineVoteToJson(String uuid, OfflineBungeeVote vote) { } protected boolean verifyJsonServerVote(String server, int index, OfflineBungeeVote vote) { - return VoteCacheDurability.saveAndVerifyServerVote(jsonStorage, server, index, vote); + try { + return VoteCacheDurability.saveAndVerifyServerVote(jsonStorage, server, index, vote); + } catch (VoteCacheDurability.ReloadFailedException failure) { + jsonStorageQuarantined = true; + debug1(failure); + return false; + } } protected boolean verifyJsonOnlineVote(String uuid, int index, OfflineBungeeVote vote) { - return VoteCacheDurability.saveAndVerifyOnlineVote(jsonStorage, uuid, index, vote); + try { + return VoteCacheDurability.saveAndVerifyOnlineVote(jsonStorage, uuid, index, vote); + } catch (VoteCacheDurability.ReloadFailedException failure) { + jsonStorageQuarantined = true; + debug1(failure); + return false; + } } protected boolean verifyJsonTimeVote(int index, VoteTimeQueue vote) { - return VoteCacheDurability.saveAndVerifyTimeVote(jsonStorage, index, vote); + try { + return VoteCacheDurability.saveAndVerifyTimeVote(jsonStorage, index, vote); + } catch (VoteCacheDurability.ReloadFailedException failure) { + jsonStorageQuarantined = true; + debug1(failure); + return false; + } } private int nextCacheIndex(Collection keys) { @@ -537,7 +652,7 @@ private int nextCacheIndex(Collection keys) { } private boolean updateServerVoteJson(String server, OfflineBungeeVote vote) { - if (jsonStorage == null) { + if (jsonStorage == null || jsonStorageQuarantined) { return false; } Collection keys = jsonStorage.getServerVotes(server); @@ -561,7 +676,7 @@ private boolean updateServerVoteJson(String server, OfflineBungeeVote vote) { } private boolean updateOnlineVoteJson(String uuid, OfflineBungeeVote vote) { - if (jsonStorage == null) { + if (jsonStorage == null || jsonStorageQuarantined) { return false; } Collection keys = jsonStorage.getOnlineVotes(uuid); @@ -607,7 +722,7 @@ private boolean matchesStoredTimeVote(DataNode data, VoteTimeQueue vote) { * Saves the vote cache to storage. */ public void saveVoteCache() { - if (jsonStorage != null) { + if (jsonStorage != null && !jsonStorageQuarantined) { jsonStorage.save(); } } @@ -632,6 +747,10 @@ public synchronized boolean addTimeVoteToCache(VoteTimeQueue vote) { } return stored; } + if (jsonStorage == null || jsonStorageQuarantined) { + timeChangeQueue.remove(vote); + return false; + } try { Collection keys = jsonStorage.getTimedVoteCache(); @@ -660,6 +779,7 @@ public synchronized boolean updateTimeVote(VoteTimeQueue vote) { if (useMySQL) { return timedVoteCacheTable.updateTimedVote(vote); } + if (jsonStorage == null || jsonStorageQuarantined) return false; Collection keys = jsonStorage.getTimedVoteCache(); if (keys == null) { @@ -701,6 +821,7 @@ public synchronized boolean removeTimeVote(VoteTimeQueue vote) { ArrayList remaining = new ArrayList<>(timeChangeQueue); remaining.remove(vote); + if (jsonStorage == null || jsonStorageQuarantined) return false; try { jsonStorage.removeTimedVotes(); int index = 0; @@ -1190,41 +1311,52 @@ public String[] getCachedVotesServers() { * @param server the server name * @param removed list of votes to remove */ - public void removeServerVotes(String server, ArrayList removed) { + public synchronized void removeServerVotes(String server, ArrayList removed) { for (OfflineBungeeVote vote : removed) { - for (Map.Entry> entry : cachedVotes.entrySet()) { - if (entry.getKey().equals(server)) { - entry.getValue().removeIf(v -> v.getUuid().equals(vote.getUuid()) - && v.getService().equals(vote.getService()) && v.getTime() == vote.getTime()); - } + boolean mysqlRemoved = !useMySQL || voteCacheTable.tryRemoveVote(vote, server); + boolean jsonRemoved = jsonStorage == null; + if (jsonStorage != null && !jsonStorageQuarantined) { + jsonRemoved = removeJsonServerVoteDurably(server, vote); } - if (useMySQL) { - voteCacheTable.removeVote(vote, server); - } - if (jsonStorage != null) { - jsonStorage.removeVote(server, vote); - jsonStorage.save(); + boolean removedDurably = mysqlRemoved && jsonRemoved; + if (!removedDurably) continue; + ArrayList serverVotes = cachedVotes.get(server); + if (serverVotes != null) { + serverVotes.removeIf(candidate -> sameVoteIdentity(candidate, vote)); + if (serverVotes.isEmpty()) cachedVotes.remove(server); } } } + protected boolean removeJsonServerVoteDurably(String server, OfflineBungeeVote vote) { + jsonStorage.removeVote(server, vote); + try { + return VoteCacheDurability.saveAndVerifyRemoval(jsonStorage, + () -> !containsStoredServerVote(server, vote)); + } catch (VoteCacheDurability.ReloadFailedException failure) { + jsonStorageQuarantined = true; + debug1(failure); + return false; + } + } + + private boolean containsStoredServerVote(String server, OfflineBungeeVote expected) { + Collection keys = jsonStorage.getServerVotes(server); + if (keys == null) return false; + for (String key : keys) { + DataNode data = jsonStorage.getServerVotes(server, key); + if (data != null && matchesStoredVote(data, expected)) return true; + } + return false; + } + /** * Removes specific votes from the online vote cache. * @param removed list of votes to remove */ public void removeOnlineVotes(ArrayList removed) { - for (OfflineBungeeVote vote : removed) { - for (Map.Entry> entry : cachedOnlineVotes.entrySet()) { - entry.getValue().removeIf(v -> v.getUuid().equals(vote.getUuid()) - && v.getService().equals(vote.getService()) && v.getTime() == vote.getTime()); - } - if (useMySQL) { - onlineVoteCacheTable.removeVote(vote); - } - if (jsonStorage != null) { - jsonStorage.removeOnlineVote(vote); - jsonStorage.save(); - } + for (OfflineBungeeVote vote : new ArrayList<>(removed)) { + tryRemoveOnlineVote(vote.getUuid(), vote); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java index 50859ae6b..a6f629dc6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java @@ -308,6 +308,14 @@ public void removeVote(String server, OfflineBungeeVote vote) { if (node == null) { continue; } + DataNode voteIdNode = node.has("VoteId") ? node.get("VoteId") + : node.has("VoteID") ? node.get("VoteID") : null; + if (vote.getVoteId() != null && voteIdNode != null) { + if (vote.getVoteId().toString().equals(voteIdNode.asString())) { + remove("VoteCache", server, num); + } + continue; + } DataNode uuidNode = node.get("UUID"); DataNode serviceNode = node.get("Service"); DataNode timeNode = node.get("Time"); @@ -341,6 +349,14 @@ public void removeOnlineVote(OfflineBungeeVote vote) { if (node == null) { continue; } + DataNode voteIdNode = node.has("VoteId") ? node.get("VoteId") + : node.has("VoteID") ? node.get("VoteID") : null; + if (vote.getVoteId() != null && voteIdNode != null) { + if (vote.getVoteId().toString().equals(voteIdNode.asString())) { + remove("OnlineCache", player, num); + } + continue; + } DataNode uuidNode = node.get("UUID"); DataNode serviceNode = node.get("Service"); DataNode timeNode = node.get("Time"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VoteEventVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VoteEventVelocity.java index b0199f28f..cc3a15e28 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VoteEventVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VoteEventVelocity.java @@ -1,6 +1,10 @@ package com.bencodez.votingplugin.proxy.velocity; +import java.util.concurrent.TimeUnit; +import java.util.UUID; + import com.bencodez.votingplugin.util.MinecraftUsernameValidator; +import com.bencodez.votingplugin.proxy.VotingPluginProxy.VoteRetryException; import com.velocitypowered.api.event.Subscribe; import com.vexsoftware.votifier.velocity.event.VotifierEvent; @@ -26,20 +30,39 @@ public VoteEventVelocity(VotingPluginVelocity plugin) { public void onVotifierEvent(VotifierEvent event) { final String serviceSiteVote = event.getVote().getServiceName(); final String name = event.getVote().getUsername(); - plugin.getTimer().execute(new Runnable() { - - @Override - public void run() { - String serviceSite = serviceSiteVote; - plugin.getLogger().info("Vote received " + MinecraftUsernameValidator.sanitizeForLog(name) - + " from service site " + MinecraftUsernameValidator.sanitizeForLog(serviceSite)); - if (serviceSite.isEmpty()) { - serviceSite = "Empty"; - } + plugin.getTimer().execute(new RetryingVote(name, serviceSiteVote.isEmpty() ? "Empty" : serviceSiteVote)); - plugin.getVotingPluginProxy().vote(name, serviceSite, true, false, 0, null, null); - } - }); + } + private final class RetryingVote implements Runnable { + private static final int MAX_ATTEMPTS = 12; + private final String player; + private final String service; + private final UUID voteId = UUID.randomUUID(); + private int attempts; + + private RetryingVote(String player, String service) { + this.player = player; + this.service = service; + } + + @Override + public void run() { + plugin.getLogger().info("Vote received " + MinecraftUsernameValidator.sanitizeForLog(player) + + " from service site " + MinecraftUsernameValidator.sanitizeForLog(service)); + try { + plugin.getVotingPluginProxy().vote(player, service, true, false, 0, null, null, voteId); + } catch (VoteRetryException retryable) { + attempts++; + if (attempts < MAX_ATTEMPTS) { + plugin.getLogger().warn("Vote processing is waiting for durable storage; retrying shortly"); + plugin.getTimer().schedule(this, 5, TimeUnit.SECONDS); + } else { + plugin.getVotingPluginProxy().abandonLiveVoteRetry(voteId); + plugin.getLogger().error("Vote processing exhausted bounded durable-storage retries for {}", + MinecraftUsernameValidator.sanitizeForLog(player)); + } + } + } } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/JsonVoteCacheLegacyRemovalTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/JsonVoteCacheLegacyRemovalTest.java new file mode 100644 index 000000000..5ce869259 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/JsonVoteCacheLegacyRemovalTest.java @@ -0,0 +1,72 @@ +package com.bencodez.votingplugin.tests; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.nio.file.Path; +import java.util.UUID; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.bencodez.votingplugin.proxy.OfflineBungeeVote; +import com.bencodez.votingplugin.proxy.bungee.BungeeJsonVoteCache; +import com.bencodez.votingplugin.proxy.bungee.VotingPluginBungee; +import com.bencodez.votingplugin.proxy.velocity.VelocityJsonVoteCache; + +class JsonVoteCacheLegacyRemovalTest { + @TempDir + Path temporaryDirectory; + + @Test + void bungeeExactRemovalHonorsLegacyVoteId() { + VotingPluginBungee plugin = mock(VotingPluginBungee.class); + when(plugin.getDataFolder()).thenReturn(temporaryDirectory.toFile()); + BungeeJsonVoteCache cache = new BungeeJsonVoteCache(plugin); + UUID removedId = UUID.randomUUID(); + UUID retainedId = UUID.randomUUID(); + putLegacyBungeeVote(cache, "0", removedId); + putLegacyBungeeVote(cache, "1", retainedId); + + cache.removeVote("server", vote(removedId)); + + assertTrue(cache.getConf().getAsJsonObject("VoteCache").getAsJsonObject("server").get("0").isJsonNull()); + assertTrue(cache.getConf().getAsJsonObject("VoteCache").getAsJsonObject("server").get("1").isJsonObject()); + } + + @Test + void velocityExactRemovalHonorsLegacyVoteId() { + VelocityJsonVoteCache cache = new VelocityJsonVoteCache( + temporaryDirectory.resolve("velocity-vote-cache.json").toFile()); + UUID removedId = UUID.randomUUID(); + UUID retainedId = UUID.randomUUID(); + putLegacyVelocityVote(cache, "0", removedId); + putLegacyVelocityVote(cache, "1", retainedId); + + cache.removeVote("server", vote(removedId)); + + assertFalse(cache.contains("VoteCache", "server", "0")); + assertTrue(cache.contains("VoteCache", "server", "1")); + } + + private static OfflineBungeeVote vote(UUID voteId) { + return new OfflineBungeeVote(voteId, "Player", "player-uuid", "Service", 100L, true, "totals"); + } + + private static void putLegacyBungeeVote(BungeeJsonVoteCache cache, String index, UUID voteId) { + String root = "VoteCache.server." + index; + cache.setString(root + ".VoteID", voteId.toString()); + cache.setString(root + ".UUID", "player-uuid"); + cache.setString(root + ".Service", "Service"); + cache.setLong(root + ".Time", 100L); + } + + private static void putLegacyVelocityVote(VelocityJsonVoteCache cache, String index, UUID voteId) { + cache.set(new Object[] { "VoteCache", "server", index, "VoteID" }, voteId.toString()); + cache.set(new Object[] { "VoteCache", "server", index, "UUID" }, "player-uuid"); + cache.set(new Object[] { "VoteCache", "server", index, "Service" }, "Service"); + cache.set(new Object[] { "VoteCache", "server", index, "Time" }, 100L); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 5ca004bf3..07e0b24ad 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -14,6 +14,7 @@ import static org.mockito.Mockito.when; import java.util.Collections; +import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -130,6 +131,157 @@ public void onlineVoteIsNotExposedWhenJsonJournalCannotSave() { assertTrue(handler.getOnlineVotes("player-uuid").isEmpty()); } + @Test + public void failedServerInsertIsPublishedOnlyAfterPersistenceRetrySucceeds() { + OfflineBungeeVote pending = vote(UUID.randomUUID(), 100L); + doThrow(new RuntimeException("save failed")).doNothing().when(storage).save(); + + assertFalse(handler.addServerVoteDurably("server", pending)); + assertTrue(handler.retainServerVoteForPersistenceRetry("server", pending)); + assertTrue(handler.getVotes("server").isEmpty()); + + assertTrue(handler.retryPendingVotePersistence()); + assertEquals(List.of(pending), handler.getVotes("server")); + } + + @Test + public void failedOnlineInsertIsPublishedOnlyAfterPersistenceRetrySucceeds() { + OfflineBungeeVote pending = vote(UUID.randomUUID(), 100L); + doThrow(new RuntimeException("save failed")).doNothing().when(storage).save(); + + assertFalse(handler.addOnlineVoteDurably("player-uuid", pending)); + assertTrue(handler.retainOnlineVoteForPersistenceRetry("player-uuid", pending)); + assertTrue(handler.getOnlineVotes("player-uuid").isEmpty()); + + assertTrue(handler.retryPendingVotePersistence()); + assertEquals(List.of(pending), handler.getOnlineVotes("player-uuid")); + } + + @Test + public void failedPersistenceRetryAdmissionIsBoundedAcrossCacheTypes() { + OfflineBungeeVote first = vote(UUID.randomUUID(), 0L); + assertTrue(handler.retainServerVoteForPersistenceRetry("server", first)); + for (int i = 1; i < 1024; i++) { + assertTrue(handler.retainServerVoteForPersistenceRetry("server", vote(UUID.randomUUID(), i))); + } + + assertFalse(handler.retainOnlineVoteForPersistenceRetry("another-player", vote(UUID.randomUUID(), 1025L))); + assertTrue(handler.retainServerVoteForPersistenceRetry("server", first)); + } + + @Test + public void expiredOnlineVoteRemovalKeepsCollidingVoteId() { + UUID retainedId = UUID.randomUUID(); + OfflineBungeeVote expired = vote(UUID.randomUUID(), 100L); + OfflineBungeeVote retained = vote(retainedId, 100L); + handler.addOnlineVote("player-uuid", expired); + handler.addOnlineVote("player-uuid", retained); + + handler.removeOnlineVotes(new ArrayList<>(List.of(expired))); + + assertEquals(List.of(retained), handler.getOnlineVotes("player-uuid")); + verify(storage).removeOnlineVote(same(expired)); + } + + @Test + public void failedExpiredOnlineVoteRemovalKeepsInMemoryEntries() { + OfflineBungeeVote expired = vote(UUID.randomUUID(), 100L); + OfflineBungeeVote retained = vote(UUID.randomUUID(), 101L); + handler = newHandler(storage, false); + handler.addOnlineVote("player-uuid", expired); + handler.addOnlineVote("player-uuid", retained); + + handler.removeOnlineVotes(new ArrayList<>(List.of(expired))); + + assertEquals(List.of(expired, retained), handler.getOnlineVotes("player-uuid")); + } + + @Test + public void failedDurableServerSaveRollsBackJsonMutationBeforeRetry() throws Exception { + Path journal = Files.createTempFile("votingplugin-server-rollback", ".json"); + try { + ArrayList keys = new ArrayList<>(); + when(storage.getStoragePath()).thenReturn(journal); + when(storage.getServerVotes("server")).thenAnswer(invocation -> new ArrayList<>(keys)); + org.mockito.Mockito.doAnswer(invocation -> { + keys.add(String.valueOf(invocation.getArgument(1, Integer.class))); + return null; + }).when(storage).addVote(eq("server"), org.mockito.ArgumentMatchers.anyInt(), + org.mockito.ArgumentMatchers.any(OfflineBungeeVote.class)); + doThrow(new java.io.IOException("save failed")).when(storage).saveDurably(); + org.mockito.Mockito.doAnswer(invocation -> { keys.clear(); return null; }).when(storage).reload(); + handler = newVerifyingHandler(storage); + + assertFalse(handler.addServerVoteDurably("server", vote(UUID.randomUUID(), 100L))); + assertFalse(handler.addServerVoteDurably("server", vote(UUID.randomUUID(), 101L))); + + verify(storage, times(2)).addVote(eq("server"), eq(0), + org.mockito.ArgumentMatchers.any(OfflineBungeeVote.class)); + verify(storage, never()).addVote(eq("server"), eq(1), + org.mockito.ArgumentMatchers.any(OfflineBungeeVote.class)); + verify(storage, times(2)).reload(); + } finally { + Files.deleteIfExists(journal); + } + } + + @Test + public void failedDurableOnlineSaveRollsBackJsonMutationBeforeRetry() throws Exception { + Path journal = Files.createTempFile("votingplugin-online-rollback", ".json"); + try { + ArrayList keys = new ArrayList<>(); + when(storage.getStoragePath()).thenReturn(journal); + when(storage.getOnlineVotes("player-uuid")).thenAnswer(invocation -> new ArrayList<>(keys)); + org.mockito.Mockito.doAnswer(invocation -> { + keys.add(String.valueOf(invocation.getArgument(1, Integer.class))); + return null; + }).when(storage).addVoteOnline(eq("player-uuid"), org.mockito.ArgumentMatchers.anyInt(), + org.mockito.ArgumentMatchers.any(OfflineBungeeVote.class)); + doThrow(new java.io.IOException("save failed")).when(storage).saveDurably(); + org.mockito.Mockito.doAnswer(invocation -> { keys.clear(); return null; }).when(storage).reload(); + handler = newVerifyingHandler(storage); + + assertFalse(handler.addOnlineVoteDurably("player-uuid", vote(UUID.randomUUID(), 100L))); + assertFalse(handler.addOnlineVoteDurably("player-uuid", vote(UUID.randomUUID(), 101L))); + + verify(storage, times(2)).addVoteOnline(eq("player-uuid"), eq(0), + org.mockito.ArgumentMatchers.any(OfflineBungeeVote.class)); + verify(storage, never()).addVoteOnline(eq("player-uuid"), eq(1), + org.mockito.ArgumentMatchers.any(OfflineBungeeVote.class)); + verify(storage, times(2)).reload(); + } finally { + Files.deleteIfExists(journal); + } + } + + @Test + public void failedRollbackReloadQuarantinesJsonJournal() throws Exception { + Path journal = Files.createTempFile("votingplugin-quarantined-journal", ".json"); + try { + when(storage.getStoragePath()).thenReturn(journal); + doThrow(new java.io.IOException("save failed")).when(storage).saveDurably(); + doThrow(new IllegalStateException("reload failed")).when(storage).reload(); + handler = newVerifyingHandler(storage); + + assertFalse(handler.addServerVoteDurably("server", vote(UUID.randomUUID(), 100L))); + assertFalse(handler.addServerVoteDurably("server", vote(UUID.randomUUID(), 101L))); + handler.removeVote("server", "player-uuid"); + handler.removeVotes("server"); + handler.removeOnlineVotes("player-uuid"); + handler.saveVoteCache(); + + verify(storage, times(1)).addVote(eq("server"), eq(0), + org.mockito.ArgumentMatchers.any(OfflineBungeeVote.class)); + verify(storage, times(1)).saveDurably(); + verify(storage, never()).removeServerVote("server", "player-uuid"); + verify(storage, never()).removeServerVotes("server"); + verify(storage, never()).removeOnlineVotes("player-uuid"); + verify(storage, never()).save(); + } finally { + Files.deleteIfExists(journal); + } + } + @Test public void serverVoteIsNotExposedWhenJsonSaveDoesNotReachDisk() throws Exception { Path journal = Files.createTempFile("votingplugin-empty-journal", ".json"); @@ -401,6 +553,18 @@ public void completedOnlineVoteRemovalKeepsCollidingVoteId() { assertEquals(List.of(retained), handler.getOnlineVotes("player-uuid")); } + @Test + public void completedServerVoteRemovalKeepsCollidingVoteId() { + OfflineBungeeVote removed = vote(UUID.randomUUID(), 100L); + OfflineBungeeVote retained = vote(UUID.randomUUID(), 100L); + handler.addServerVote("server", removed); + handler.addServerVote("server", retained); + + handler.removeServerVotes("server", new ArrayList<>(List.of(removed))); + + assertEquals(List.of(retained), handler.getVotes("server")); + } + @Test public void timedVoteBroadcastStateLoadsFromJsonCache() { IVoteCache stored = mock(IVoteCache.class); @@ -603,6 +767,10 @@ private static OfflineBungeeVote vote(UUID voteId, long time) { } private static VoteCacheHandler newHandler(IVoteCache storage) { + return newHandler(storage, true); + } + + private static VoteCacheHandler newHandler(IVoteCache storage, boolean removalSucceeds) { return new VoteCacheHandler(null, false, false, null, false, storage) { @Override protected boolean verifyJsonServerVote(String server, int index, OfflineBungeeVote vote) { @@ -622,6 +790,20 @@ protected boolean verifyJsonTimeVote(int index, VoteTimeQueue vote) { return true; } + @Override + protected boolean removeJsonOnlineVoteDurably(String uuid, OfflineBungeeVote vote) { + storage.removeOnlineVote(vote); + storage.save(); + return removalSucceeds; + } + + @Override + protected boolean removeJsonServerVoteDurably(String server, OfflineBungeeVote vote) { + storage.removeVote(server, vote); + storage.save(); + return true; + } + @Override public void logInfo1(String msg) { } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyCommandTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyCommandTest.java new file mode 100644 index 000000000..8b4e3816e --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyCommandTest.java @@ -0,0 +1,24 @@ +package com.bencodez.votingplugin.tests; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; + +import org.junit.jupiter.api.Test; + +import com.bencodez.votingplugin.proxy.VotingPluginProxy; +import com.bencodez.votingplugin.proxy.VotingPluginProxyCommand; + +public class VotingPluginProxyCommandTest { + @Test + void manualVoteReportsRetryableDurabilityFailure() { + VotingPluginProxy proxy = mock(VotingPluginProxy.class); + doThrow(VotingPluginProxy.VoteRetryException.class).when(proxy) + .vote("Player", "Service", false, true, 0, null, null); + + String result = new VotingPluginProxyCommand(proxy) + .execute(new String[] { "vote", "Player", "Service" }); + + assertEquals("&cVote could not be stored safely. Please retry shortly.", result); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 6f535d5ba..8f73649ff 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -188,12 +188,13 @@ void acceptedRewardStillPersistsUnresolvedStandaloneBroadcast() { java.util.Map.of("Server1", "00000000-0000-0000-0000-000000000169")); VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.when(voteCache.addOnlineVoteDurably("player-uuid", state)).thenReturn(true); - spyProxy.persistUncachedStandaloneBroadcastForTest("player-uuid", state, false); + assertTrue(spyProxy.persistUncachedStandaloneBroadcastForTest("player-uuid", state, false)); assertTrue(state.isRewardDelivered()); assertFalse(state.isBroadcastForwarded()); - verify(voteCache).addOnlineVote("player-uuid", state); + verify(voteCache).addOnlineVoteDurably("player-uuid", state); } @Test @@ -205,12 +206,247 @@ void uncachedStandaloneBroadcastPersistsAsBroadcastOnlyState() { java.util.Map.of("Server1", "00000000-0000-0000-0000-000000000169")); VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.when(voteCache.addOnlineVoteDurably("player-uuid", state)).thenReturn(true); - spyProxy.persistUncachedStandaloneBroadcastForTest("player-uuid", state, false); + assertTrue(spyProxy.persistUncachedStandaloneBroadcastForTest("player-uuid", state, false)); assertTrue(state.isRewardDelivered()); assertFalse(state.isBroadcastForwarded()); - verify(voteCache).addOnlineVote("player-uuid", state); + verify(voteCache).addOnlineVoteDurably("player-uuid", state); + } + + @Test + void uncachedStandaloneBroadcastRetainsFailedDurabilityForRetry() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote state = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "player-uuid", + "Service", 100L, true, "totals", false, true, java.util.Set.of("Server1"), + java.util.Collections.emptySet(), false, java.util.Collections.emptyMap(), + java.util.Map.of("Server1", "00000000-0000-0000-0000-000000000169")); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.when(voteCache.addOnlineVoteDurably("player-uuid", state)).thenReturn(false); + Mockito.when(voteCache.retainOnlineVoteForPersistenceRetry("player-uuid", state)).thenReturn(true); + + assertFalse(spyProxy.persistUncachedStandaloneBroadcastForTest("player-uuid", state, false)); + verify(voteCache).addOnlineVoteDurably("player-uuid", state); + verify(voteCache).retainOnlineVoteForPersistenceRetry("player-uuid", state); + } + + @Test + void standaloneBroadcastDoesNotSendBeforeItsCanonicalStateIsDurable() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote state = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "player-uuid", + "Service", 100L, true, "totals", false, true, java.util.Set.of("Server1"), + java.util.Collections.emptySet(), true); + Mockito.when(voteCache.addOnlineVoteDurably("player-uuid", state)).thenReturn(false); + Mockito.when(voteCache.retainOnlineVoteForPersistenceRetry("player-uuid", state)).thenReturn(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + assertFalse(spyProxy.persistAndSendStandaloneBroadcastForTest("player-uuid", state, + java.util.Set.of("Server1"), new java.util.LinkedHashSet<>())); + + assertTrue(spyProxy.getAttemptedVotePartyDeliveryIds().isEmpty()); + verify(voteCache).retainOnlineVoteForPersistenceRetry("player-uuid", state); + verify(voteCache, never()).updateOnlineVote("player-uuid", state); + } + + @Test + void standaloneBroadcastPersistsCanonicalStateBeforeSending() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote state = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "player-uuid", + "Service", 100L, true, "totals", false, true, java.util.Set.of("Server1"), + java.util.Collections.emptySet(), true); + Mockito.when(voteCache.addOnlineVoteDurably("player-uuid", state)).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote("player-uuid", state)).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + assertTrue(spyProxy.persistAndSendStandaloneBroadcastForTest("player-uuid", state, + java.util.Set.of("Server1"), new java.util.LinkedHashSet<>())); + + assertEquals(1, spyProxy.getAttemptedVotePartyDeliveryIds().size()); + org.mockito.InOrder order = Mockito.inOrder(voteCache); + order.verify(voteCache).addOnlineVoteDurably("player-uuid", state); + order.verify(voteCache).updateOnlineVote("player-uuid", state); + verify(voteCache, never()).removeOnlineVote("player-uuid", state); + } + + @Test + void liveVotePropagatesRetryWhenProcessingCannotBeMadeDurable() { + Mockito.when(votingPluginProxy.getConfig().getBungeeManageTotals()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getPrimaryServer()).thenReturn(true); + votingPluginProxy.setProxyMySQL(null); + + com.bencodez.votingplugin.proxy.VotingPluginProxy.VoteRetryException failure = assertThrows( + com.bencodez.votingplugin.proxy.VotingPluginProxy.VoteRetryException.class, + () -> votingPluginProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001")); + + assertTrue(failure.getMessage().contains("retry is required")); + } + + @Test + void liveVoteRetryDoesNotApplyVotePartyTwice() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())) + .thenReturn(false, true, true); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + + assertThrows(VotingPluginProxy.VoteRetryException.class, + () -> spyProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId)); + spyProxy.setAvailableServers("Server1", "Server2", "Server3"); + spyProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId); + + verify(spyProxy).addVoteParty(); + verify(voteCache, Mockito.times(3)).addServerVoteDurably(Mockito.anyString(), Mockito.any()); + } + + @Test + void legacyVoteEntryPointClearsUnownedRetryState() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(false); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + + assertThrows(VotingPluginProxy.VoteRetryException.class, + () -> spyProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001")); + + java.lang.reflect.Field retries = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); + retries.setAccessible(true); + assertTrue(((java.util.Map) retries.get(spyProxy)).isEmpty()); + } + + @Test + void durableRewardJournalOwnerRecreatesMissingServerOutbox() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote persistedOwner = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "player-uuid", + "Service", 100L, true, "totals"); + persistedOwner.setHttpDeliveryId("__vp_reward_target__:Server1", + "00000000-0000-0000-0000-000000000199"); + OfflineBungeeVote owner = new OfflineBungeeVote(persistedOwner.getVoteId(), "Player", "player-uuid", + "Service", 100L, true, "totals", false, false, java.util.Collections.emptySet(), + java.util.Collections.emptySet(), false, + OfflineBungeeVote.decodeHttpDeliveryIds(persistedOwner.encodeHttpDeliveryIds()), + java.util.Collections.emptyMap()); + owner.setDeliveryStateDirty(false); + Mockito.when(voteCache.retryPendingVotePersistence()).thenReturn(true); + Mockito.when(voteCache.getOnlineVoteUUIDs()).thenReturn(java.util.Set.of("player-uuid")); + Mockito.when(voteCache.getOnlineVotes("player-uuid")) + .thenReturn(new java.util.ArrayList<>(java.util.List.of(owner))); + Mockito.when(voteCache.addServerVoteDurably(Mockito.eq("Server1"), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote("player-uuid", owner)).thenReturn(true); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + spyProxy.retryPendingOnlineBroadcasts(); + + assertTrue(owner.isRewardDelivered()); + verify(voteCache).addServerVoteDurably(Mockito.eq("Server1"), Mockito.argThat(vote -> + "00000000-0000-0000-0000-000000000199".equals(vote.getHttpDeliveryId("Server1")))); + verify(voteCache).updateOnlineVote("player-uuid", owner); + verify(voteCache).removeOnlineVote("player-uuid", owner); + } + + @Test + void liveVoteRetryKeepsTotalsInputAfterVotePartyFailure() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getPrimaryServer()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getBungeeManageTotals()).thenReturn(true); + Mockito.when(proxyMySQL.containsKeyQuery(Mockito.anyString())).thenReturn(true); + Mockito.when(proxyMySQL.getExactQuery(Mockito.any())).thenReturn(new java.util.ArrayList<>()); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + votingPluginProxy.setStandaloneBroadcastForwarding(false); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doThrow(new IllegalStateException("party persistence failed")).when(spyProxy).addVoteParty(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + + assertThrows(VotingPluginProxy.VoteRetryException.class, + () -> spyProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId)); + spyProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId); + + verify(spyProxy).addVoteParty(); + verify(proxyMySQL).update(Mockito.anyString(), Mockito.any(java.util.ArrayList.class)); + } + + @Test + void liveVoteRetryDoesNotResendCompletedServerReward() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true, false, true); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setStableHttpDeliveryResults(true, false, true); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + + assertThrows(VotingPluginProxy.VoteRetryException.class, + () -> spyProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId)); + spyProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId); + + assertEquals(3, spyProxy.getAttemptedVotePartyDeliveryIds().size()); + verify(spyProxy).addVoteParty(); + } + + @Test + void liveVoteRetryRetainsRecoveredHttpDeliveryId() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true, false, true); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setStableHttpDeliveryResults(true, true); + String recoveredId = "00000000-0000-0000-0000-000000000198"; + votingPluginProxy.failNextStableHttpSend(recoveredId); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + + assertThrows(VotingPluginProxy.VoteRetryException.class, + () -> spyProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId)); + spyProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId); + + assertEquals(recoveredId, spyProxy.getAttemptedVotePartyDeliveryIds().get(2)); } @Test @@ -235,6 +471,10 @@ void queuedRolloverReplayReusesStableStandaloneBroadcastDeliveryId() { "player-uuid", java.util.Map.of("Server2", deliveryId)); queue.add(queued); Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any(OfflineBungeeVote.class))) + .thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any(OfflineBungeeVote.class))) + .thenReturn(true); Mockito.when(voteCache.updateTimeVote(queued)).thenReturn(true); Mockito.when(voteCache.removeTimeVote(queued)).thenAnswer(invocation -> queue.remove(queued)); Mockito.when(votingPluginProxy.getConfig().getPrimaryServer()).thenReturn(true); @@ -1509,7 +1749,7 @@ void rejectedHttpQueueDeliveryRetainsCachedOnlineVote() { spyProxy.checkOnlineVotes("Player", "player-uuid", "Server1"); assertFalse(vote.isRewardDelivered()); - verify(voteCache).addOnlineVote("player-uuid", vote); + verify(voteCache, never()).removeOnlineVote("player-uuid", vote); verify(multiProxyHandler, never()).sendClearVote(Mockito.anyString(), Mockito.anyString()); } @@ -1536,7 +1776,7 @@ void acceptedOnlineRewardRetainsAmbiguousStandaloneBroadcastWithoutRebroadcastin assertFalse(vote.isProxyBroadcastComplete()); assertEquals("false", spyProxy.getLastVoteEnvelope().getFields() .get(VotingPluginWire.K_BUNGEE_BROADCAST)); - verify(voteCache).addOnlineVote("player-uuid", vote); + verify(voteCache, never()).removeOnlineVote("player-uuid", vote); } @Test diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index 29a2d7a8a..0f7e362ad 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -34,11 +34,14 @@ public class VotingPluginProxyTestImpl extends VotingPluginProxy { private boolean communicationTestDeliveryResult = true; private JsonEnvelope lastCommunicationTestEnvelope; private boolean playerOnline = true; + private Set availableServers = new HashSet<>(Arrays.asList("Server1", "Server2")); + private Boolean standaloneBroadcastForwarding; private ScheduledExecutorService scheduler; private boolean failNextVoteCacheSave; private final java.util.Map> pendingVotePartyRewards = new HashMap<>(); private final List attemptedVotePartyDeliveryIds = new ArrayList<>(); private boolean failNextGeneratedHttpSend; + private boolean failNextStableHttpSend; private String generatedHttpRetryId; private PendingVotePartyProxyEffects pendingVotePartyProxyEffects = PendingVotePartyProxyEffects.empty(); private PendingVotePartyProxyEffects quarantinedVotePartyProxyEffects = PendingVotePartyProxyEffects.empty(); @@ -88,7 +91,21 @@ public void broadcast(String message) { @Override public Set getAllAvailableServers() { - return new HashSet<>(Arrays.asList("Server1", "Server2")); + return new HashSet<>(availableServers); + } + + public void setAvailableServers(String... servers) { + availableServers = new HashSet<>(Arrays.asList(servers)); + } + + public void setStandaloneBroadcastForwarding(Boolean enabled) { + standaloneBroadcastForwarding = enabled; + } + + @Override + protected boolean canForwardStandaloneBroadcast(boolean managesTotals) { + return standaloneBroadcastForwarding == null + ? super.canForwardStandaloneBroadcast(managesTotals) : standaloneBroadcastForwarding.booleanValue(); } @Override @@ -394,6 +411,10 @@ protected boolean sendHttpEnvelope(String server, JsonEnvelope envelope) { protected boolean sendHttpEnvelope(String server, String deliveryId, JsonEnvelope envelope) { lastVoteEnvelope = envelope; attemptedVotePartyDeliveryIds.add(deliveryId); + if (failNextStableHttpSend) { + failNextStableHttpSend = false; + throw deliveryRetryException(generatedHttpRetryId); + } if (!stableHttpDeliveryResults.isEmpty()) return stableHttpDeliveryResults.removeFirst(); return stableHttpDeliveryResult == null ? voteEnvelopeDeliveryResult : stableHttpDeliveryResult; } @@ -420,6 +441,23 @@ public void failNextGeneratedHttpSend(String deliveryId) { generatedHttpRetryId = deliveryId; } + public void failNextStableHttpSend(String deliveryId) { + failNextStableHttpSend = true; + generatedHttpRetryId = deliveryId; + } + + private static RuntimeException deliveryRetryException(String deliveryId) { + try { + java.lang.reflect.Constructor constructor = + HttpProxyTransportServer.DeliveryRetryException.class + .getDeclaredConstructor(String.class, Throwable.class); + constructor.setAccessible(true); + return constructor.newInstance(deliveryId, new IllegalStateException("publication ambiguous")); + } catch (ReflectiveOperationException failure) { + throw new AssertionError(failure); + } + } + public boolean sendHttpVoteEnvelopeWithRecoveryForTest(String server, JsonEnvelope envelope) { return sendHttpEnvelopeWithRecovery(server, envelope); } @@ -506,9 +544,14 @@ public void retryPendingOnlineBroadcastsForTest(String server) { retryPendingOnlineBroadcasts(server); } - public void persistUncachedStandaloneBroadcastForTest(String uuid, OfflineBungeeVote state, + public boolean persistUncachedStandaloneBroadcastForTest(String uuid, OfflineBungeeVote state, boolean alreadyPersisted) { - persistUncachedStandaloneBroadcast(uuid, state, alreadyPersisted); + return persistUncachedStandaloneBroadcast(uuid, state, alreadyPersisted); + } + + public boolean persistAndSendStandaloneBroadcastForTest(String uuid, OfflineBungeeVote state, + java.util.Set remainingTargets, java.util.Set forwardedServers) { + return persistAndSendStandaloneBroadcast(uuid, state, remainingTargets, forwardedServers); } public OfflineBungeeVote createCachedRewardVoteForTest(UUID voteId, String player, String uuid, String service, From cbe604683d5a4c68da728db964679506220d64c6 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Thu, 10 Sep 2026 21:24:48 -0600 Subject: [PATCH 068/136] Preserve live vote retries across reload --- .../votingplugin/proxy/VotingPluginProxy.java | 19 ++++++++++- .../tests/VotingPluginProxyTest.java | 33 +++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 5837045aa..02d373bde 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -116,6 +116,9 @@ private VoteRetryException() { private static final int MAX_LIVE_VOTE_RETRIES = 1024; private static final String REWARD_JOURNAL_TARGET_PREFIX = "__vp_reward_target__:"; private final Map liveVoteRetries = new LinkedHashMap<>(); + // Set only after all replacement gates have succeeded. Vote entry points are + // synchronized, so no new side-effecting vote can race the handoff window. + private boolean runtimeReplacementPrepared; private static final class LiveVoteRetryState { private VoteTotalsSnapshot totals; @@ -2738,7 +2741,15 @@ public void onDisable(boolean waitForHosted) { } /** Fail-closed gate that must complete before a replacement proxy runtime is created. */ - public void prepareForRuntimeReplacement() { + public synchronized void prepareForRuntimeReplacement() { + // Live retry markers fence side effects (including vote-party and totals) that + // have already run but whose durable outbox write has not. They are owned by + // this runtime only, so replacing it would let the scheduled listener retry + // start with an empty map and reapply those effects. Keep this runtime alive + // until the retry settles instead of dropping or replaying an uncertain vote. + if (!liveVoteRetries.isEmpty()) { + throw new IllegalStateException("Live vote retries must settle before proxy runtime replacement"); + } if (!quarantineInFlightVotePartyProxyCommandForReplacement()) { throw new IllegalStateException("In-flight vote-party command must be durably quarantined before proxy runtime replacement"); } @@ -2751,6 +2762,7 @@ public void prepareForRuntimeReplacement() { controlLifecycleExecutor.shutdown(); stopControlServicesLocked(true); } + runtimeReplacementPrepared = true; } /** Best-effort remainder of runtime teardown after the Control overlap gate has succeeded. */ @@ -4340,6 +4352,11 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea + MinecraftUsernameValidator.sanitizeForLog(service) + "'"); return QueuedVoteResult.TERMINAL; } + // A platform listener can receive a vote in the small interval after the + // replacement gate succeeds and before the platform publishes its fresh + // runtime. Tell its bounded retry wrapper to retry against that fresh + // runtime rather than creating state that would be lost with this one. + if (runtimeReplacementPrepared) return QueuedVoteResult.RETRY; UUID voteId = queuedVote == null ? null : queuedVote.getVoteId(); if (voteId == null) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 8f73649ff..61c601116 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -319,19 +319,46 @@ void liveVoteRetryDoesNotApplyVotePartyTwice() { } @Test - void legacyVoteEntryPointClearsUnownedRetryState() throws Exception { + void runtimeReplacementWaitsForLiveVoteRetryToSettle() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); - Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(false); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())) + .thenReturn(false, true, true); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true); votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); Mockito.doNothing().when(spyProxy).addVoteParty(); + java.util.UUID voteId = java.util.UUID.randomUUID(); assertThrows(VotingPluginProxy.VoteRetryException.class, () -> spyProxy.vote("Player", "Service", true, false, 100L, null, - "00000000-0000-0000-0000-000000000001")); + "00000000-0000-0000-0000-000000000001", voteId)); + IllegalStateException blocked = assertThrows(IllegalStateException.class, + spyProxy::prepareForRuntimeReplacement); + assertTrue(blocked.getMessage().contains("Live vote retries")); + spyProxy.setAvailableServers("Server1", "Server2", "Server3"); + spyProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId); + + verify(spyProxy).addVoteParty(); + } + + @Test + void preparedRuntimeDoesNotStartAReplacementWindowVote() throws Exception { + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doNothing().when(spyProxy).addVoteParty(); + + spyProxy.prepareForRuntimeReplacement(); + + assertThrows(VotingPluginProxy.VoteRetryException.class, + () -> spyProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", java.util.UUID.randomUUID())); + verify(spyProxy, never()).addVoteParty(); java.lang.reflect.Field retries = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); retries.setAccessible(true); assertTrue(((java.util.Map) retries.get(spyProxy)).isEmpty()); From d8fbbed94ecc85a13f4bee397f58cb9d55ad21ce Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:01:26 -0600 Subject: [PATCH 069/136] Keep vote completion retries idempotent --- .../votingplugin/proxy/VotingPluginProxy.java | 52 +++++++++++- .../proxy/VotingPluginProxyCommand.java | 16 +++- .../proxy/cache/VoteCacheHandler.java | 85 ++++++++++++++++++- .../tests/VoteCacheHandlerVoteIdTest.java | 23 ++++- .../tests/VotingPluginProxyCommandTest.java | 23 ++++- .../tests/VotingPluginProxyTest.java | 46 ++++++++-- 6 files changed, 226 insertions(+), 19 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 02d373bde..9462954a3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -121,6 +121,7 @@ private VoteRetryException() { private boolean runtimeReplacementPrepared; private static final class LiveVoteRetryState { + private String requestIdentity; private VoteTotalsSnapshot totals; private ArrayList totalsInput; private boolean votePartyApplied; @@ -652,6 +653,9 @@ private Set sendProxyBroadcast(Set targets, String uuid, String // that target's ID could leave this accepted target looking pending after a // crash, and it would be replayed under a newly generated ID. if (!persistTimeVoteDelivery(cachedVote)) break; + } else if (cachedVote.getHttpBroadcastDeliveryId(targetServer) != null) { + cachedVote.setHttpBroadcastDeliveryId(targetServer, null); + cachedVote.setDeliveryStateDirty(true); } } } @@ -2920,6 +2924,10 @@ private UUID parseUUIDFromString(String uuidAsString) { public synchronized void processQueue() { while (getVoteCacheHandler().getTimeChangeQueue().size() > 0) { VoteTimeQueue vote = getVoteCacheHandler().getTimeChangeQueue().element(); + if (!vote.isProcessed() && getVoteCacheHandler().hasTimeVoteCompletion(vote)) { + vote.setProcessed(true); + vote.setDeliveryStateDirty(true); + } if (vote.isProcessed() && vote.hasPendingHttpBroadcastDeliveryIds()) { // The reward/totals work is already complete. Only retry the durable // standalone broadcasts; removing this row would lose their stable IDs. @@ -2929,6 +2937,11 @@ public synchronized void processQueue() { return; } } + if (vote.isProcessed() && vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { + scheduleTimeVoteRetry(); + return; + } + if (vote.isProcessed()) liveVoteRetries.remove(vote.getVoteId()); if (!vote.isProcessed()) { VoteTotalsSnapshot queuedTotals = vote.getTotals() == null || vote.getTotals().isEmpty() ? null : VoteTotalsSnapshot.parseStorage(vote.getTotals()); @@ -2942,11 +2955,16 @@ public synchronized void processQueue() { warn("Removing terminal rollover vote " + vote.getVoteId() + " for " + vote.getName() + "/" + ServiceSiteValidator.sanitizeForLog(vote.getService())); } + if (!getVoteCacheHandler().getTimeChangeQueue().contains(vote)) { + getVoteCacheHandler().clearTimeVoteCompletion(vote); + continue; + } } if (!getVoteCacheHandler().removeTimeVote(vote)) { scheduleTimeVoteRetry(); return; } + getVoteCacheHandler().clearTimeVoteCompletion(vote); } } @@ -4352,6 +4370,7 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea + MinecraftUsernameValidator.sanitizeForLog(service) + "'"); return QueuedVoteResult.TERMINAL; } + String requestIdentity = player.toLowerCase(Locale.ROOT) + "\u0000" + service.toLowerCase(Locale.ROOT); // A platform listener can receive a vote in the small interval after the // replacement gate succeeds and before the platform publishes its fresh // runtime. Tell its bounded retry wrapper to retry against that fresh @@ -4363,6 +4382,9 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea voteId = requestedVoteId == null ? UUID.randomUUID() : requestedVoteId; } LiveVoteRetryState retryState = liveVoteRetries.get(voteId); + if (retryState != null && !requestIdentity.equals(retryState.requestIdentity)) { + throw new IllegalArgumentException("Retry ID does not match the original vote"); + } boolean resumingAfterTotals = retryState != null; // UUID resolution @@ -4516,6 +4538,7 @@ && getGlobalDataHandler().isTimeChangedHappened()) { if (retryState == null) { if (liveVoteRetries.size() >= MAX_LIVE_VOTE_RETRIES) return QueuedVoteResult.RETRY; retryState = new LiveVoteRetryState(); + retryState.requestIdentity = requestIdentity; retryState.totalsInput = data; liveVoteRetries.put(voteId, retryState); } @@ -4636,6 +4659,16 @@ && getGlobalDataHandler().isTimeChangedHappened()) { logSevere("Unable to durably journal standalone broadcast for " + uuid); return QueuedVoteResult.RETRY; } + if (queuedVote != null) { + // Keep the source row's in-memory completion state aligned with the + // canonical broadcast journal. If persisting the final processed marker + // fails, processQueue can retry that marker without publishing an already + // accepted broadcast again. + queuedVote.getBroadcastForwardedServers().addAll(broadcastForwardedServers); + for (String forwardedServer : broadcastForwardedServers) { + queuedVote.setHttpBroadcastDeliveryId(forwardedServer, null); + } + } } // =========================== @@ -4882,13 +4915,26 @@ && getGlobalDataHandler().isTimeChangedHappened()) { } if (queuedVote != null) { queuedVote.setProcessed(true); - if (!getVoteCacheHandler().updateTimeVote(queuedVote)) { - warn("Unable to persist completed rollover vote " + queuedVote.getVoteId() - + "; attempting durable removal immediately"); + if (!persistTimeVoteDelivery(queuedVote)) { + // Deleting the completed source row is itself a durable completion + // record. Prefer that fallback when updating the marker is unavailable. + if (getVoteCacheHandler().removeTimeVote(queuedVote)) { + liveVoteRetries.remove(voteId); + return QueuedVoteResult.SUCCESS; + } + if (getVoteCacheHandler().markTimeVoteCompletedDurably(queuedVote)) { + liveVoteRetries.remove(voteId); + return QueuedVoteResult.SUCCESS; + } + warn("Unable to persist or remove completed rollover vote " + queuedVote.getVoteId() + + "; retaining its live completion fence for retry"); + return QueuedVoteResult.RETRY; } } liveVoteRetries.remove(voteId); return QueuedVoteResult.SUCCESS; + } catch (IllegalArgumentException e) { + throw e; } catch (Exception e) { e.printStackTrace(); return QueuedVoteResult.RETRY; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java index d631b7abc..adf9db1ad 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java @@ -1,5 +1,7 @@ package com.bencodez.votingplugin.proxy; +import java.util.UUID; + import com.bencodez.advancedcore.api.time.TimeType; public class VotingPluginProxyCommand { @@ -32,14 +34,22 @@ public String execute(String[] args) { if (args.length >= 3) { String user = args[1]; String site = args[2]; + UUID voteId; + try { + voteId = args.length >= 4 ? UUID.fromString(args[3]) : UUID.randomUUID(); + } catch (IllegalArgumentException invalidId) { + return "&cUsage: vote [retry-id]"; + } try { - plugin.vote(user, site, false, true, 0, null, null); + plugin.vote(user, site, false, true, 0, null, null, voteId); return "&aVote sent for " + user + " on " + site; + } catch (IllegalArgumentException mismatchedRetry) { + return "&cRetry ID does not match that player and site."; } catch (VotingPluginProxy.VoteRetryException retryable) { - return "&cVote could not be stored safely. Please retry shortly."; + return "&cVote could not be stored safely. Retry with: vote " + user + " " + site + " " + voteId; } } - return "&cUsage: vote "; + return "&cUsage: vote [retry-id]"; case "forcetimechange": if (args.length >= 2) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index 434caa6f9..a7f39327f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -1,5 +1,9 @@ package com.bencodez.votingplugin.proxy.cache; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; @@ -17,6 +21,7 @@ import com.bencodez.simpleapi.sql.mysql.config.MysqlConfig; import com.bencodez.votingplugin.proxy.OfflineBungeeVote; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; +import com.bencodez.votingplugin.util.DurableFiles; import lombok.Getter; @@ -828,15 +833,93 @@ public synchronized boolean removeTimeVote(VoteTimeQueue vote) { for (VoteTimeQueue queued : remaining) { jsonStorage.addTimedVote(index++, queued); } - jsonStorage.save(); + boolean removedDurably = VoteCacheDurability.saveAndVerifyRemoval(jsonStorage, () -> { + Collection keys = jsonStorage.getTimedVoteCache(); + if (keys == null) return true; + for (String key : keys) { + DataNode data = jsonStorage.getTimedVoteCache(key); + if (data != null && data.isObject() && matchesStoredTimeVote(data, vote)) return false; + } + return true; + }); + if (!removedDurably) return false; timeChangeQueue.remove(vote); return true; + } catch (VoteCacheDurability.ReloadFailedException failure) { + jsonStorageQuarantined = true; + debug1(failure); + return false; } catch (RuntimeException e) { debug1(e); return false; } } + /** + * Records completed timed-vote side effects outside the mutable queue file. + * This tombstone closes the crash window where both updating and deleting the + * queue row fail after those side effects have already committed. + */ + public synchronized boolean markTimeVoteCompletedDurably(VoteTimeQueue vote) { + Path target = timeVoteCompletionPath(vote); + if (target == null) return false; + Path staged = null; + try { + Files.createDirectories(target.getParent()); + staged = Files.createTempFile(target.getParent(), ".completion-", ".tmp"); + Files.writeString(staged, timeVoteCompletionIdentity(vote), StandardCharsets.UTF_8); + try { + DurableFiles.publishStagedFile(staged, target); + } catch (DurableFiles.PublishedException published) { + // The tombstone is already visible; the read-back below is authoritative. + } + return hasTimeVoteCompletion(vote); + } catch (IOException | RuntimeException failure) { + debug1(failure); + return false; + } finally { + if (staged != null) { + try { Files.deleteIfExists(staged); } catch (IOException ignored) { } + } + } + } + + public synchronized boolean hasTimeVoteCompletion(VoteTimeQueue vote) { + Path target = timeVoteCompletionPath(vote); + if (target == null || !Files.isRegularFile(target)) return false; + try { + return timeVoteCompletionIdentity(vote).equals(Files.readString(target, StandardCharsets.UTF_8)); + } catch (IOException | RuntimeException failure) { + debug1(failure); + return false; + } + } + + public synchronized void clearTimeVoteCompletion(VoteTimeQueue vote) { + Path target = timeVoteCompletionPath(vote); + if (target == null) return; + try { + DurableFiles.deleteIfExists(target); + } catch (IOException | RuntimeException failure) { + debug1(failure); + } + } + + private Path timeVoteCompletionPath(VoteTimeQueue vote) { + if (vote == null || jsonStorage == null || jsonStorage.getStoragePath() == null) return null; + Path storage = jsonStorage.getStoragePath().toAbsolutePath().normalize(); + Path name = storage.getFileName(); + if (name == null) return null; + UUID key = vote.getVoteId() != null ? vote.getVoteId() + : UUID.nameUUIDFromBytes(timeVoteCompletionIdentity(vote).getBytes(StandardCharsets.UTF_8)); + return storage.resolveSibling(name + ".completed-timed-votes").resolve(key.toString()); + } + + private String timeVoteCompletionIdentity(VoteTimeQueue vote) { + return (vote.getVoteId() == null ? "" : vote.getVoteId()) + "\n" + vote.getUuid() + "\n" + + vote.getName() + "\n" + vote.getService() + "\n" + vote.getTime(); + } + /** * Loads vote cache from storage. */ diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 07e0b24ad..14f36d535 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -29,6 +29,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import com.bencodez.votingplugin.proxy.OfflineBungeeVote; import com.bencodez.votingplugin.proxy.cache.DataNode; @@ -40,6 +41,8 @@ * Regression tests for proxy vote identity across caches and delayed processing. */ public class VoteCacheHandlerVoteIdTest { + @TempDir + Path tempDir; private IVoteCache storage; private VoteCacheHandler handler; @@ -47,6 +50,7 @@ public class VoteCacheHandlerVoteIdTest { @BeforeEach public void setUp() { storage = mock(IVoteCache.class); + when(storage.getStoragePath()).thenReturn(tempDir.resolve("vote-cache.json")); when(storage.getServerVotes("server")).thenReturn(Collections.emptyList()); when(storage.getOnlineVotes("player-uuid")).thenReturn(Collections.emptyList()); handler = newHandler(storage); @@ -388,26 +392,39 @@ public void timedVoteDeliveryStateIsUpdatedByVoteId() { } @Test - public void timedVoteIsRemovedFromMemoryOnlyAfterJsonSaveSucceeds() { + public void timedVoteIsRemovedFromMemoryOnlyAfterJsonSaveSucceeds() throws Exception { VoteTimeQueue queued = new VoteTimeQueue(UUID.randomUUID(), "Player", "Service", 100L); assertTrue(handler.addTimeVoteToCache(queued)); assertTrue(handler.removeTimeVote(queued)); assertTrue(handler.getTimeChangeQueue().isEmpty()); + verify(storage).saveDurably(); } @Test - public void timedVoteRemainsQueuedWhenJsonDeleteCannotBeSaved() { + public void timedVoteRemainsQueuedWhenJsonDeleteCannotBeSaved() throws Exception { VoteTimeQueue queued = new VoteTimeQueue(UUID.randomUUID(), "Player", "Service", 100L); assertTrue(handler.addTimeVoteToCache(queued)); - doThrow(new RuntimeException("save failed")).when(storage).save(); + doThrow(new java.io.IOException("save failed")).when(storage).saveDurably(); assertFalse(handler.removeTimeVote(queued)); assertTrue(handler.getTimeChangeQueue().contains(queued)); } + @Test + public void timedVoteCompletionTombstoneSurvivesIndependentCacheFailure() { + VoteTimeQueue queued = new VoteTimeQueue(UUID.randomUUID(), "Player", "Service", 100L, false, + Set.of(), Set.of(), "totals", false, "player-uuid"); + + assertTrue(handler.markTimeVoteCompletedDurably(queued)); + assertTrue(handler.hasTimeVoteCompletion(queued)); + + handler.clearTimeVoteCompletion(queued); + assertFalse(handler.hasTimeVoteCompletion(queued)); + } + @Test public void legacyTimedVoteConstructorHasNoId() { VoteTimeQueue vote = new VoteTimeQueue("Player", "Service", 100L); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyCommandTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyCommandTest.java index 8b4e3816e..9c6d79782 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyCommandTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyCommandTest.java @@ -1,10 +1,17 @@ package com.bencodez.votingplugin.tests; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.UUID; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import com.bencodez.votingplugin.proxy.VotingPluginProxy; import com.bencodez.votingplugin.proxy.VotingPluginProxyCommand; @@ -14,11 +21,19 @@ public class VotingPluginProxyCommandTest { void manualVoteReportsRetryableDurabilityFailure() { VotingPluginProxy proxy = mock(VotingPluginProxy.class); doThrow(VotingPluginProxy.VoteRetryException.class).when(proxy) - .vote("Player", "Service", false, true, 0, null, null); + .vote(eq("Player"), eq("Service"), eq(false), eq(true), eq(0L), eq(null), eq(null), any(UUID.class)); + VotingPluginProxyCommand command = new VotingPluginProxyCommand(proxy); - String result = new VotingPluginProxyCommand(proxy) - .execute(new String[] { "vote", "Player", "Service" }); + String firstResult = command.execute(new String[] { "vote", "Player", "Service" }); + String retryIdText = firstResult.substring(firstResult.lastIndexOf(' ') + 1); + UUID retryId = UUID.fromString(retryIdText); + String secondResult = command.execute(new String[] { "vote", "Player", "Service", retryIdText }); - assertEquals("&cVote could not be stored safely. Please retry shortly.", result); + assertEquals("&cVote could not be stored safely. Retry with: vote Player Service " + retryId, firstResult); + assertEquals(firstResult, secondResult); + ArgumentCaptor ids = ArgumentCaptor.forClass(UUID.class); + verify(proxy, times(2)).vote(eq("Player"), eq("Service"), eq(false), eq(true), eq(0L), eq(null), eq(null), + ids.capture()); + assertEquals(java.util.List.of(retryId, retryId), ids.getAllValues()); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 61c601116..4cd3fcf46 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -418,6 +418,9 @@ void liveVoteRetryKeepsTotalsInputAfterVotePartyFailure() { assertThrows(VotingPluginProxy.VoteRetryException.class, () -> spyProxy.vote("Player", "Service", true, false, 100L, null, "00000000-0000-0000-0000-000000000001", voteId)); + assertThrows(IllegalArgumentException.class, + () -> spyProxy.vote("OtherPlayer", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000002", voteId)); spyProxy.vote("Player", "Service", true, false, 100L, null, "00000000-0000-0000-0000-000000000001", voteId); @@ -502,8 +505,10 @@ void queuedRolloverReplayReusesStableStandaloneBroadcastDeliveryId() { .thenReturn(true); Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any(OfflineBungeeVote.class))) .thenReturn(true); - Mockito.when(voteCache.updateTimeVote(queued)).thenReturn(true); - Mockito.when(voteCache.removeTimeVote(queued)).thenAnswer(invocation -> queue.remove(queued)); + Mockito.when(voteCache.updateTimeVote(queued)).thenReturn(false, true); + java.util.concurrent.atomic.AtomicInteger removeAttempts = new java.util.concurrent.atomic.AtomicInteger(); + Mockito.when(voteCache.removeTimeVote(queued)).thenAnswer(invocation -> + removeAttempts.getAndIncrement() == 0 ? false : queue.remove(queued)); Mockito.when(votingPluginProxy.getConfig().getPrimaryServer()).thenReturn(true); Mockito.when(votingPluginProxy.getConfig().getBungeeManageTotals()).thenReturn(true); Mockito.when(votingPluginProxy.getConfig().getProxyBroadcastEnabled()).thenReturn(true); @@ -515,17 +520,46 @@ void queuedRolloverReplayReusesStableStandaloneBroadcastDeliveryId() { VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + spyProxy.processQueue(); + + assertFalse(queue.isEmpty()); + assertThrows(IllegalStateException.class, spyProxy::prepareForRuntimeReplacement); + verify(voteCache).removeTimeVote(queued); + spyProxy.processQueue(); assertTrue(queue.isEmpty()); assertEquals(deliveryId, spyProxy.getAttemptedVotePartyDeliveryIds().get(0)); assertEquals(2, spyProxy.getAttemptedVotePartyDeliveryIds().size()); assertFalse(deliveryId.equals(spyProxy.getAttemptedVotePartyDeliveryIds().get(1))); - verify(voteCache).updateTimeVote(queued); - verify(voteCache).removeTimeVote(queued); + verify(voteCache, Mockito.times(2)).updateTimeVote(queued); + verify(voteCache, Mockito.times(2)).removeTimeVote(queued); + verify(spyProxy).addVoteParty(); verify(voteCache, never()).addOnlineVote(Mockito.anyString(), Mockito.any(OfflineBungeeVote.class)); } + @Test + void durableCompletionTombstoneSkipsTimedVoteSideEffectsAfterRestart() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + VoteTimeQueue queued = new VoteTimeQueue(java.util.UUID.randomUUID(), "Player", "Service", 100L, false, + java.util.Set.of(), java.util.Set.of(), "totals", false, "player-uuid"); + queue.add(queued); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.hasTimeVoteCompletion(queued)).thenReturn(true); + Mockito.when(voteCache.updateTimeVote(queued)).thenReturn(true); + Mockito.when(voteCache.removeTimeVote(queued)).thenAnswer(invocation -> queue.remove(queued)); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + spyProxy.processQueue(); + + assertTrue(queue.isEmpty()); + verify(spyProxy, never()).addVoteParty(); + verify(voteCache).clearTimeVoteCompletion(queued); + } + @Test void testAddVoteParty() { // Initial votePartyVotes should be 0 @@ -1903,7 +1937,8 @@ void rejectedCarrierHttpBroadcastKeepsItsDeterministicIdentity() { void timedBroadcastRetriesWhileRolloverIsStillActive() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); VoteTimeQueue vote = new VoteTimeQueue(java.util.UUID.randomUUID(), "OfflineVoter", "Service", 100L, true, - java.util.Set.of("Server1"), java.util.Collections.emptySet(), "totals", false, "voter-uuid"); + java.util.Set.of("Server1"), java.util.Collections.emptySet(), "totals", false, "voter-uuid", + java.util.Map.of("Server1", "stale-http-delivery-id")); java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); queue.add(vote); Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); @@ -1915,6 +1950,7 @@ void timedBroadcastRetriesWhileRolloverIsStillActive() { spyProxy.retryPendingTimeBroadcastsForTest("Server1"); assertEquals(java.util.Set.of("Server1"), vote.getBroadcastForwardedServers()); + assertEquals(null, vote.getHttpBroadcastDeliveryId("Server1")); verify(voteCache).updateTimeVote(vote); } From a404248d82e67599b8f0475d7924c7f81dcf0cc6 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:34:51 -0600 Subject: [PATCH 070/136] Settle live vote retries before shutdown --- .../votingplugin/proxy/VotingPluginProxy.java | 66 +++++++++++++++++ .../tests/VotingPluginProxyTest.java | 71 +++++++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 9462954a3..ff494190b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -114,6 +114,7 @@ private VoteRetryException() { private static final long PRESENCE_HANDOFF_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(2); private static final long PRESENCE_STARTUP_RESYNC_DELAY_SECONDS = 5L; private static final int MAX_LIVE_VOTE_RETRIES = 1024; + private static final int FINAL_SHUTDOWN_PERSISTENCE_ATTEMPTS = 3; private static final String REWARD_JOURNAL_TARGET_PREFIX = "__vp_reward_target__:"; private final Map liveVoteRetries = new LinkedHashMap<>(); // Set only after all replacement gates have succeeded. Vote entry points are @@ -133,6 +134,13 @@ private static final class LiveVoteRetryState { private boolean rewardJournalsDurable; private OfflineBungeeVote standaloneBroadcastState; private OfflineBungeeVote rewardJournalOwner; + private OfflineBungeeVote pendingOnlineRewardState; + private VoteTimeQueue queuedVote; + private String player; + private String service; + private String uuid; + private long time; + private boolean realVote; } private static final long PRESENCE_MAINTENANCE_INTERVAL_SECONDS = 30L; @@ -2730,6 +2738,15 @@ public void onDisable(boolean waitForHosted) { if (waitForHosted) { prepareForRuntimeReplacement(); } else { + boolean liveRetriesSettled; + try { + liveRetriesSettled = settleLiveVoteRetriesForFinalShutdown(); + } catch (RuntimeException failure) { + liveRetriesSettled = false; + } + if (!liveRetriesSettled) { + logSevere("Unable to durably quarantine live vote retries after bounded final-shutdown persistence attempts; operator reconciliation may be required after restart"); + } if (!quarantineInFlightVotePartyProxyCommandForReplacement()) { awaitInFlightVotePartyProxyCommand(); if (!quarantineInFlightVotePartyProxyCommandForReplacement()) { @@ -2744,6 +2761,43 @@ public void onDisable(boolean waitForHosted) { completeRuntimeReplacementShutdown(); } + /** Moves every in-memory live retry into the ordinary durable vote outbox before final teardown. */ + protected synchronized boolean settleLiveVoteRetriesForFinalShutdown() { + if (liveVoteRetries.isEmpty()) return true; + VoteCacheHandler cache = getVoteCacheHandler(); + boolean retained = true; + for (LiveVoteRetryState retry : liveVoteRetries.values()) { + Set onlineStates = Collections.newSetFromMap(new java.util.IdentityHashMap<>()); + if (retry.rewardJournalOwner != null) onlineStates.add(retry.rewardJournalOwner); + if (retry.standaloneBroadcastState != null) onlineStates.add(retry.standaloneBroadcastState); + if (retry.pendingOnlineRewardState != null) onlineStates.add(retry.pendingOnlineRewardState); + for (OfflineBungeeVote state : onlineStates) { + retained &= cache.retainOnlineVoteForPersistenceRetry(state.getUuid(), state); + } + for (Map.Entry entry : retry.rewardStates.entrySet()) { + String server = entry.getKey(); + for (String configured : getAllConfiguredServers()) { + if (configured.equalsIgnoreCase(server)) { + server = configured; + break; + } + } + retained &= cache.retainServerVoteForPersistenceRetry(server, entry.getValue()); + } + } + for (int attempt = 0; attempt < FINAL_SHUTDOWN_PERSISTENCE_ATTEMPTS; attempt++) { + if (!retained || !cache.retryPendingVotePersistence()) continue; + for (Map.Entry entry : new ArrayList<>(liveVoteRetries.entrySet())) { + LiveVoteRetryState retry = entry.getValue(); + QueuedVoteResult result = vote(retry.player, retry.service, retry.realVote, false, retry.time, + retry.totals, retry.uuid, retry.queuedVote, entry.getKey()); + if (result == QueuedVoteResult.TERMINAL) liveVoteRetries.remove(entry.getKey()); + } + if (liveVoteRetries.isEmpty()) return true; + } + return false; + } + /** Fail-closed gate that must complete before a replacement proxy runtime is created. */ public synchronized void prepareForRuntimeReplacement() { // Live retry markers fence side effects (including vote-party and totals) that @@ -4360,6 +4414,7 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea private synchronized QueuedVoteResult vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, VoteTotalsSnapshot text, String uuid, VoteTimeQueue queuedVote, UUID requestedVoteId) { try { + String requestPlayer = player; if (!ServiceSiteValidator.isValid(service)) { warn("Rejected vote with invalid service site '" + ServiceSiteValidator.sanitizeForLog(service) + "'"); return QueuedVoteResult.TERMINAL; @@ -4540,8 +4595,14 @@ && getGlobalDataHandler().isTimeChangedHappened()) { retryState = new LiveVoteRetryState(); retryState.requestIdentity = requestIdentity; retryState.totalsInput = data; + retryState.player = requestPlayer; + retryState.service = service; + retryState.uuid = uuid; + retryState.time = time; + retryState.realVote = realVote; liveVoteRetries.put(voteId, retryState); } + if (queuedVote != null) retryState.queuedVote = queuedVote; if (!retryState.votePartyApplied) { // Fence the side effect before invoking it. If the call reports an // indeterminate failure, a listener retry must not increment the party again. @@ -4682,6 +4743,9 @@ && getGlobalDataHandler().isTimeChangedHappened()) { markRewardJournalTargets(rewardJournalOwner, rewardServers); retryState.rewardJournalOwner = rewardJournalOwner; if (!getVoteCacheHandler().addOnlineVoteDurably(uuid, rewardJournalOwner)) { + if (getVoteCacheHandler().retainOnlineVoteForPersistenceRetry(uuid, rewardJournalOwner)) { + scheduleCachedVoteDeliveryRetry(); + } return QueuedVoteResult.RETRY; } } @@ -4852,6 +4916,7 @@ && getGlobalDataHandler().isTimeChangedHappened()) { true, proxyBroadcastTargets, broadcastForwardedServers, false, Collections.emptyMap(), standaloneBroadcastState.getHttpBroadcastDeliveryIds()) : createCachedRewardVote(voteId, player, uuid, service, time, realVote, text.toString(), false); + retryState.pendingOnlineRewardState = cachedReward; boolean cachedDurably = getVoteCacheHandler().addOnlineVoteDurably(uuid, cachedReward); if (!cachedDurably) { if (getVoteCacheHandler().retainOnlineVoteForPersistenceRetry(uuid, cachedReward)) { @@ -4861,6 +4926,7 @@ && getGlobalDataHandler().isTimeChangedHappened()) { + "; retaining it for persistence retry"); return QueuedVoteResult.RETRY; } + retryState.pendingOnlineRewardState = null; standaloneBroadcastStatePersisted |= standaloneBroadcastState != null; debug("Caching online vote for " + player + " on " + service); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 4cd3fcf46..b57bc3af5 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -26,6 +26,7 @@ import com.bencodez.votingplugin.proxy.ProxyMysqlUserTable; import com.bencodez.votingplugin.proxy.VotingPluginProxy; import com.bencodez.votingplugin.proxy.VotingPluginWire; +import com.bencodez.votingplugin.proxy.broadcast.ProxyBroadcastDecider; import com.bencodez.votingplugin.proxy.cache.VoteCacheHandler; import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyHandler; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; @@ -348,6 +349,76 @@ void runtimeReplacementWaitsForLiveVoteRetryToSettle() { verify(spyProxy).addVoteParty(); } + @Test + void finalShutdownMovesLiveVoteRetryIntoDurableOutbox() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())) + .thenReturn(false, true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.retainOnlineVoteForPersistenceRetry(Mockito.anyString(), Mockito.any())) + .thenReturn(true); + Mockito.when(voteCache.retainServerVoteForPersistenceRetry(Mockito.anyString(), Mockito.any())) + .thenReturn(true); + Mockito.when(voteCache.retryPendingVotePersistence()).thenReturn(true); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + + assertThrows(VotingPluginProxy.VoteRetryException.class, + () -> spyProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId)); + spyProxy.onDisable(false); + + verify(voteCache).retryPendingVotePersistence(); + java.lang.reflect.Field retries = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); + retries.setAccessible(true); + assertTrue(((java.util.Map) retries.get(spyProxy)).isEmpty()); + } + + @Test + void finalShutdownFencesQueuedVoteBeforeClearingLiveRetry() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + VoteTimeQueue queued = new VoteTimeQueue(java.util.UUID.randomUUID(), "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", false, + "00000000-0000-0000-0000-000000000001"); + queue.add(queued); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(false, true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.retainOnlineVoteForPersistenceRetry(Mockito.anyString(), Mockito.any())) + .thenReturn(true); + Mockito.when(voteCache.retainServerVoteForPersistenceRetry(Mockito.anyString(), Mockito.any())) + .thenReturn(true); + Mockito.when(voteCache.retryPendingVotePersistence()).thenReturn(true); + Mockito.when(voteCache.updateTimeVote(queued)).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getPrimaryServer()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getBungeeManageTotals()).thenReturn(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + java.lang.reflect.Field decider = VotingPluginProxy.class.getDeclaredField("proxyBroadcastDecider"); + decider.setAccessible(true); + decider.set(spyProxy, new ProxyBroadcastDecider(spyProxy::getConfig, spyProxy::getAllAvailableServers, + server -> true, server -> false)); + + spyProxy.processQueue(); + assertFalse(queued.isProcessed()); + spyProxy.onDisable(false); + + assertTrue(queued.isProcessed()); + verify(voteCache).updateTimeVote(queued); + verify(spyProxy).addVoteParty(); + } + @Test void preparedRuntimeDoesNotStartAReplacementWindowVote() throws Exception { VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); From 0da1c46084bc0aaf5345af00e3e54696d9b2a830 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 11 Sep 2026 00:37:12 -0600 Subject: [PATCH 071/136] Make multi-proxy vote delivery durable --- .../votingplugin/proxy/VotingPluginProxy.java | 521 +++++++++++++++++- .../votingplugin/proxy/VotingPluginWire.java | 52 ++ .../proxy/bungee/BungeeJsonVoteCache.java | 11 +- .../proxy/cache/ProxyTimedVoteCacheTable.java | 169 +++++- .../proxy/cache/VoteCacheHandler.java | 132 ++++- .../proxy/multiproxy/MultiProxyHandler.java | 179 +++++- .../proxy/velocity/VelocityJsonVoteCache.java | 12 + .../votingplugin/timequeue/VoteTimeQueue.java | 99 ++++ .../MultiProxyHandlerLifecycleTest.java | 97 ++++ .../tests/VoteCacheHandlerVoteIdTest.java | 41 ++ .../tests/VotingPluginProxyTest.java | 356 +++++++++++- 11 files changed, 1617 insertions(+), 52 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index ff494190b..b701332fc 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -13,6 +13,7 @@ import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import java.util.Base64; import java.sql.SQLException; import java.time.Duration; import java.time.Instant; @@ -114,9 +115,15 @@ private VoteRetryException() { private static final long PRESENCE_HANDOFF_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(2); private static final long PRESENCE_STARTUP_RESYNC_DELAY_SECONDS = 5L; private static final int MAX_LIVE_VOTE_RETRIES = 1024; + private static final int MAX_MULTI_PROXY_VOTE_ATTEMPTS = 12; + private static final int MAX_MULTI_PROXY_VOTE_RETRIES = 1024; + private static final int MAX_COMPLETED_MULTI_PROXY_VOTES = 4096; + private static final String FORWARDED_QUEUE_TOTALS_PREFIX = "\u0000VP-FWD:"; private static final int FINAL_SHUTDOWN_PERSISTENCE_ATTEMPTS = 3; private static final String REWARD_JOURNAL_TARGET_PREFIX = "__vp_reward_target__:"; private final Map liveVoteRetries = new LinkedHashMap<>(); + private final Map multiProxyVoteRetries = new LinkedHashMap<>(); + private final LinkedHashMap completedMultiProxyVotes = new LinkedHashMap<>(); // Set only after all replacement gates have succeeded. Vote entry points are // synchronized, so no new side-effecting vote can race the handoff window. private boolean runtimeReplacementPrepared; @@ -132,6 +139,7 @@ private static final class LiveVoteRetryState { private final Map rewardStates = new LinkedHashMap<>(); private Set rewardServers; private boolean rewardJournalsDurable; + private boolean multiProxyForwardingHandled; private OfflineBungeeVote standaloneBroadcastState; private OfflineBungeeVote rewardJournalOwner; private OfflineBungeeVote pendingOnlineRewardState; @@ -143,6 +151,45 @@ private static final class LiveVoteRetryState { private boolean realVote; } + + private final class MultiProxyVoteRetry implements Runnable { + private enum Phase { + EXECUTE, + PERSIST_DEFERRED_RECEIPT, + PERSIST_COMPLETION + } + + private final String player; + private final String service; + private final boolean realVote; + private final boolean timeQueue; + private final long queueTime; + private final VoteTotalsSnapshot totals; + private final String uuid; + private final UUID voteId; + private final String origin; + private int attempts; + private boolean scheduled; + private Phase phase = Phase.EXECUTE; + + private MultiProxyVoteRetry(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot totals, String uuid, UUID voteId, String origin) { + this.player = player; + this.service = service; + this.realVote = realVote; + this.timeQueue = timeQueue; + this.queueTime = queueTime; + this.totals = totals; + this.uuid = uuid; + this.voteId = voteId; + this.origin = origin == null ? "" : origin; + } + + @Override + public void run() { + attemptMultiProxyVote(this); + } + } private static final long PRESENCE_MAINTENANCE_INTERVAL_SECONDS = 30L; private static final long PRESENCE_BACKEND_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(90); private static final long CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10); @@ -2342,10 +2389,207 @@ public void triggerVote(String player, String service, boolean realVote, boolean VoteTotalsSnapshot text, String uuid) { vote(player, service, realVote, timeQueue, queueTime, text, uuid); } + + @Override + public void triggerVote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid, UUID voteId) { + receiveMultiProxyVote(player, service, realVote, timeQueue, queueTime, text, uuid, voteId); + } + + @Override + public void triggerVote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid, UUID voteId, String origin) { + receiveMultiProxyVote(player, service, realVote, timeQueue, queueTime, text, uuid, voteId, origin); + } + + @Override + public void onMultiProxyVoteAcknowledged(UUID voteId, String recipient) { + handleMultiProxyVoteAcknowledgement(voteId, recipient); + } }; multiProxyHandler.loadMultiProxySupport(); } + /** Receives and locally retries a forwarded vote under its wire-stable identity. */ + protected synchronized void receiveMultiProxyVote(String player, String service, boolean realVote, + boolean timeQueue, long queueTime, VoteTotalsSnapshot totals, String uuid, UUID wireVoteId) { + receiveMultiProxyVote(player, service, realVote, timeQueue, queueTime, totals, uuid, wireVoteId, ""); + } + + /** Receives a reliable envelope and retains its sender only for a later ACK. */ + protected synchronized void receiveMultiProxyVote(String player, String service, boolean realVote, + boolean timeQueue, long queueTime, VoteTotalsSnapshot totals, String uuid, UUID wireVoteId, String origin) { + UUID voteId = wireVoteId == null ? UUID.randomUUID() : wireVoteId; + if (completedMultiProxyVotes.containsKey(voteId)) { + acknowledgeCompletedMultiProxyVote(voteId, origin); + return; + } + if (getVoteCacheHandler().hasMultiProxyVoteCompletion(voteId)) { + rememberCompletedMultiProxyVote(voteId); + acknowledgeCompletedMultiProxyVote(voteId, origin); + return; + } + MultiProxyVoteRetry retry = multiProxyVoteRetries.get(voteId); + if (retry == null) { + if (multiProxyVoteRetries.size() >= MAX_MULTI_PROXY_VOTE_RETRIES) { + // Do not drop a sender delivery when the in-memory retry fence is full. + // The ordinary timed-vote cache is bounded by durable storage rather than + // this process heap and is loaded again after a restart. It also lets the + // normal queue processor drain the spill as retry slots become available. + if (retainForwardedVoteOverflow(player, service, realVote, totals, uuid, voteId, queueTime, origin)) { + scheduleTimeVoteRetry(); + return; + } + logSevere("Unable to durably retain forwarded multi-proxy vote while the bounded retry queue is full"); + return; + } + retry = new MultiProxyVoteRetry(player, service, realVote, timeQueue, queueTime, totals, uuid, voteId, origin); + multiProxyVoteRetries.put(voteId, retry); + } + if (!retry.scheduled) attemptMultiProxyVote(retry); + } + + /** Durably spills an over-capacity forwarded vote into the normal replay queue. */ + private boolean retainForwardedVoteOverflow(String player, String service, boolean realVote, + VoteTotalsSnapshot totals, String uuid, UUID voteId, long queueTime, String origin) { + if (voteId == null || player == null || service == null || uuid == null) return false; + for (VoteTimeQueue queued : getVoteCacheHandler().getTimeChangeQueue()) { + if (voteId.equals(queued.getVoteId())) return true; + } + long time = queueTime == 0L ? System.currentTimeMillis() : queueTime; + VoteTimeQueue queued = new VoteTimeQueue(voteId, player, service, time, false, + Collections.emptySet(), Collections.emptySet(), totals == null ? "" : totals.toString(), false, uuid); + queued.setRealVote(realVote); + queued.setMultiProxyOrigin(origin == null ? "" : origin); + return getVoteCacheHandler().addTimeVoteToCache(queued); + } + + private static String[] decodeForwardedQueueTotals(String encoded) { + if (encoded == null || !encoded.startsWith(FORWARDED_QUEUE_TOTALS_PREFIX)) { + return new String[] { "true", encoded == null ? "" : encoded }; + } + int flagIndex = FORWARDED_QUEUE_TOTALS_PREFIX.length(); + int separator = flagIndex + 1; + if (encoded.length() <= separator || (encoded.charAt(flagIndex) != '0' && encoded.charAt(flagIndex) != '1') + || encoded.charAt(separator) != ':') { + return new String[] { "true", encoded }; + } + char realVote = encoded.charAt(FORWARDED_QUEUE_TOTALS_PREFIX.length()); + try { + String decoded = new String(Base64.getUrlDecoder().decode(encoded.substring(separator + 1)), StandardCharsets.UTF_8); + return new String[] { realVote == '1' ? "true" : "false", decoded }; + } catch (IllegalArgumentException invalidEncoding) { + return new String[] { "true", encoded }; + } + } + + private synchronized void attemptMultiProxyVote(MultiProxyVoteRetry retry) { + attemptMultiProxyVote(retry, true); + } + + private synchronized void attemptMultiProxyVote(MultiProxyVoteRetry retry, boolean allowSchedule) { + if (multiProxyVoteRetries.get(retry.voteId) != retry) return; + retry.scheduled = false; + if (!enabled) return; + if (retry.phase == MultiProxyVoteRetry.Phase.PERSIST_DEFERRED_RECEIPT) { + if (persistDeferredMultiProxyReceipt(retry)) { + multiProxyVoteRetries.remove(retry.voteId); + } else { + scheduleMultiProxyVoteRetry(retry, allowSchedule, "deferred-receipt persistence"); + } + return; + } + if (retry.phase == MultiProxyVoteRetry.Phase.PERSIST_COMPLETION) { + if (getVoteCacheHandler().hasMultiProxyVoteCompletion(retry.voteId) + || getVoteCacheHandler().markMultiProxyVoteCompletedDurably(retry.voteId)) { + completeMultiProxyVote(retry); + } else { + scheduleMultiProxyVoteRetry(retry, allowSchedule, "completion-record persistence"); + } + return; + } + try { + vote(retry.player, retry.service, retry.realVote, retry.timeQueue, retry.queueTime, retry.totals, + retry.uuid, retry.voteId); + if (retry.timeQueue && findUnprocessedQueuedVote(retry.voteId) != null) { + retry.phase = MultiProxyVoteRetry.Phase.PERSIST_DEFERRED_RECEIPT; + retry.attempts = 0; + if (persistDeferredMultiProxyReceipt(retry)) multiProxyVoteRetries.remove(retry.voteId); + else scheduleMultiProxyVoteRetry(retry, allowSchedule, "deferred-receipt persistence"); + return; + } + retry.phase = MultiProxyVoteRetry.Phase.PERSIST_COMPLETION; + retry.attempts = 0; + if (getVoteCacheHandler().markMultiProxyVoteCompletedDurably(retry.voteId)) { + completeMultiProxyVote(retry); + } else { + scheduleMultiProxyVoteRetry(retry, allowSchedule, "completion-record persistence"); + } + } catch (VoteRetryException retryable) { + scheduleMultiProxyVoteRetry(retry, allowSchedule, "durable-storage"); + } + } + + /** Binds a deferred receiver queue to its origin before dropping its live retry fence. */ + private boolean persistDeferredMultiProxyReceipt(MultiProxyVoteRetry retry) { + VoteTimeQueue queued = findUnprocessedQueuedVote(retry.voteId); + if (queued == null) return false; + queued.setMultiProxyOrigin(retry.origin); + queued.setRealVote(retry.realVote); + queued.setDeliveryStateDirty(true); + return persistTimeVoteDelivery(queued); + } + + private VoteTimeQueue findUnprocessedQueuedVote(UUID voteId) { + if (voteId == null) return null; + for (VoteTimeQueue queued : getVoteCacheHandler().getTimeChangeQueue()) { + if (voteId.equals(queued.getVoteId()) && !queued.isProcessed()) return queued; + } + return null; + } + + private void completeMultiProxyVote(MultiProxyVoteRetry retry) { + multiProxyVoteRetries.remove(retry.voteId); + rememberCompletedMultiProxyVote(retry.voteId); + acknowledgeCompletedMultiProxyVote(retry.voteId, retry.origin); + } + + private void acknowledgeCompletedMultiProxyVote(UUID voteId, String origin) { + if (origin == null || origin.isBlank() || multiProxyHandler == null) return; + multiProxyHandler.acknowledgeMultiProxyVote(voteId, origin); + } + + private void scheduleMultiProxyVoteRetry(MultiProxyVoteRetry retry, boolean allowSchedule, String reason) { + retry.attempts++; + ScheduledExecutorService scheduler = getScheduler(); + if (!allowSchedule || retry.attempts >= MAX_MULTI_PROXY_VOTE_ATTEMPTS + || scheduler == null || scheduler.isShutdown()) { + if (!allowSchedule) return; + logSevere("Forwarded multi-proxy vote remains fenced after bounded " + reason + " retries for " + + MinecraftUsernameValidator.sanitizeForLog(retry.player)); + return; + } + retry.scheduled = true; + try { + scheduler.schedule(retry, 5, TimeUnit.SECONDS); + } catch (RuntimeException schedulingFailure) { + retry.scheduled = false; + logSevere("Unable to schedule a forwarded multi-proxy vote retry; its live side-effect fence was retained"); + } + } + + private void rememberCompletedMultiProxyVote(UUID voteId) { + completedMultiProxyVotes.put(voteId, Boolean.TRUE); + while (completedMultiProxyVotes.size() > MAX_COMPLETED_MULTI_PROXY_VOTES) { + UUID oldest = completedMultiProxyVotes.keySet().iterator().next(); + removeCompletedMultiProxyVote(oldest); + } + } + + private void removeCompletedMultiProxyVote(UUID voteId) { + completedMultiProxyVotes.remove(voteId); + } + public abstract void log(String message); /** @@ -2740,7 +2984,7 @@ public void onDisable(boolean waitForHosted) { } else { boolean liveRetriesSettled; try { - liveRetriesSettled = settleLiveVoteRetriesForFinalShutdown(); + liveRetriesSettled = settleMultiProxyAndLiveVotesForFinalShutdown(); } catch (RuntimeException failure) { liveRetriesSettled = false; } @@ -2761,6 +3005,73 @@ public void onDisable(boolean waitForHosted) { completeRuntimeReplacementShutdown(); } + /** Settles consumed multi-proxy envelopes together with any live phase fences they own. */ + protected synchronized boolean settleMultiProxyAndLiveVotesForFinalShutdown() { + for (int attempt = 0; attempt < FINAL_SHUTDOWN_PERSISTENCE_ATTEMPTS; attempt++) { + for (MultiProxyVoteRetry retry : new ArrayList<>(multiProxyVoteRetries.values())) { + retry.scheduled = false; + attemptMultiProxyVote(retry, false); + } + Set activeForwardedVotes = new LinkedHashSet<>(); + for (UUID voteId : multiProxyVoteRetries.keySet()) { + if (liveVoteRetries.containsKey(voteId)) activeForwardedVotes.add(voteId); + } + boolean liveSettled = settleLiveVoteRetriesForFinalShutdown(); + for (UUID voteId : activeForwardedVotes) { + if (!liveVoteRetries.containsKey(voteId)) { + MultiProxyVoteRetry retry = multiProxyVoteRetries.get(voteId); + if (retry != null) { + retry.phase = MultiProxyVoteRetry.Phase.PERSIST_COMPLETION; + retry.attempts = 0; + attemptMultiProxyVote(retry, false); + } + } + } + if (liveSettled && multiProxyVoteRetries.isEmpty()) return true; + } + // Platform disable callers always continue teardown. A receiver whose vote + // side effects completed but whose separate tombstone could not be written + // must therefore be represented by the ordinary durable queue, not retained + // only by this process's retry map. + if (liveVoteRetries.isEmpty() && quarantineCompletedMultiProxyRetriesForShutdown()) return true; + return liveVoteRetries.isEmpty() && multiProxyVoteRetries.isEmpty(); + } + + private boolean quarantineCompletedMultiProxyRetriesForShutdown() { + VoteCacheHandler cache = getVoteCacheHandler(); + for (MultiProxyVoteRetry retry : new ArrayList<>(multiProxyVoteRetries.values())) { + if (retry.phase != MultiProxyVoteRetry.Phase.PERSIST_COMPLETION + || !retainCompletedMultiProxyRetry(cache, retry)) return false; + multiProxyVoteRetries.remove(retry.voteId); + } + return true; + } + + /** Persists a completed receiver phase without acknowledging until its tombstone succeeds. */ + private boolean retainCompletedMultiProxyRetry(VoteCacheHandler cache, MultiProxyVoteRetry retry) { + if (retry.voteId == null) return false; + for (VoteTimeQueue queued : cache.getTimeChangeQueue()) { + if (!retry.voteId.equals(queued.getVoteId())) continue; + // An unprocessed row owns deferred vote effects. Do not convert it into a + // completion fence: its origin is persisted by PERSIST_DEFERRED_RECEIPT. + if (!queued.isProcessed()) return false; + queued.setProcessed(true); + queued.setRealVote(retry.realVote); + queued.setMultiProxyOrigin(retry.origin); + queued.setMultiProxyCompletionPending(true); + queued.setDeliveryStateDirty(true); + return cache.updateTimeVote(queued); + } + VoteTimeQueue quarantine = new VoteTimeQueue(retry.voteId, retry.player, retry.service, + retry.queueTime == 0L ? System.currentTimeMillis() : retry.queueTime, false, + Collections.emptySet(), Collections.emptySet(), retry.totals == null ? "" : retry.totals.toString(), true, + retry.uuid); + quarantine.setRealVote(retry.realVote); + quarantine.setMultiProxyOrigin(retry.origin); + quarantine.setMultiProxyCompletionPending(true); + return cache.addTimeVoteToCache(quarantine); + } + /** Moves every in-memory live retry into the ordinary durable vote outbox before final teardown. */ protected synchronized boolean settleLiveVoteRetriesForFinalShutdown() { if (liveVoteRetries.isEmpty()) return true; @@ -2805,8 +3116,9 @@ public synchronized void prepareForRuntimeReplacement() { // this runtime only, so replacing it would let the scheduled listener retry // start with an empty map and reapply those effects. Keep this runtime alive // until the retry settles instead of dropping or replaying an uncertain vote. - if (!liveVoteRetries.isEmpty()) { - throw new IllegalStateException("Live vote retries must settle before proxy runtime replacement"); + if (!liveVoteRetries.isEmpty() || !multiProxyVoteRetries.isEmpty()) { + throw new IllegalStateException( + "Live vote retries or forwarded multi-proxy vote retries must settle before proxy runtime replacement"); } if (!quarantineInFlightVotePartyProxyCommandForReplacement()) { throw new IllegalStateException("In-flight vote-party command must be durably quarantined before proxy runtime replacement"); @@ -2978,6 +3290,19 @@ private UUID parseUUIDFromString(String uuidAsString) { public synchronized void processQueue() { while (getVoteCacheHandler().getTimeChangeQueue().size() > 0) { VoteTimeQueue vote = getVoteCacheHandler().getTimeChangeQueue().element(); + if (vote.isMultiProxyCompletionPending()) { + if (!getVoteCacheHandler().hasMultiProxyVoteCompletion(vote.getVoteId()) + && !getVoteCacheHandler().markMultiProxyVoteCompletedDurably(vote.getVoteId())) { + scheduleTimeVoteRetry(); + return; + } + acknowledgeCompletedMultiProxyVote(vote.getVoteId(), vote.getMultiProxyOrigin()); + if (!getVoteCacheHandler().removeTimeVote(vote)) { + scheduleTimeVoteRetry(); + return; + } + continue; + } if (!vote.isProcessed() && getVoteCacheHandler().hasTimeVoteCompletion(vote)) { vote.setProcessed(true); vote.setDeliveryStateDirty(true); @@ -2991,20 +3316,46 @@ public synchronized void processQueue() { return; } } + if (vote.isProcessed() && vote.isMultiProxyForwardingRequired() + && !vote.isMultiProxyForwardingHandled()) { + if (!retryDurableMultiProxyOutbox(vote)) { + scheduleTimeVoteRetry(); + return; + } + } if (vote.isProcessed() && vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { scheduleTimeVoteRetry(); return; } if (vote.isProcessed()) liveVoteRetries.remove(vote.getVoteId()); if (!vote.isProcessed()) { - VoteTotalsSnapshot queuedTotals = vote.getTotals() == null || vote.getTotals().isEmpty() ? null - : VoteTotalsSnapshot.parseStorage(vote.getTotals()); - QueuedVoteResult result = vote(vote.getName(), vote.getService(), true, false, vote.getTime(), queuedTotals, + String[] forwardedTotals = decodeForwardedQueueTotals(vote.getTotals()); + boolean queuedRealVote = vote.isRealVote(); + // Existing NUL-prefixed emergency rows predate the explicit realVote + // column. Decode them once for compatibility, while new rows preserve + // PostgreSQL-safe plain totals. + if (vote.getTotals() != null && vote.getTotals().startsWith(FORWARDED_QUEUE_TOTALS_PREFIX)) { + queuedRealVote = Boolean.parseBoolean(forwardedTotals[0]); + vote.setTotals(forwardedTotals[1]); + vote.setRealVote(queuedRealVote); + vote.setDeliveryStateDirty(true); + } + VoteTotalsSnapshot queuedTotals = forwardedTotals[1].isEmpty() ? null + : VoteTotalsSnapshot.parseStorage(forwardedTotals[1]); + QueuedVoteResult result = vote(vote.getName(), vote.getService(), queuedRealVote, false, vote.getTime(), queuedTotals, vote.getUuid(), vote); if (result == QueuedVoteResult.RETRY) { scheduleTimeVoteRetry(); return; } + if ((result == QueuedVoteResult.SUCCESS || result == QueuedVoteResult.TERMINAL) + && !vote.getMultiProxyOrigin().isBlank()) { + if (!getVoteCacheHandler().markMultiProxyVoteCompletedDurably(vote.getVoteId())) { + scheduleTimeVoteRetry(); + return; + } + acknowledgeCompletedMultiProxyVote(vote.getVoteId(), vote.getMultiProxyOrigin()); + } if (result == QueuedVoteResult.TERMINAL) { warn("Removing terminal rollover vote " + vote.getVoteId() + " for " + vote.getName() + "/" + ServiceSiteValidator.sanitizeForLog(vote.getService())); @@ -4569,6 +4920,7 @@ && getGlobalDataHandler().isTimeChangedHappened()) { VoteTimeQueue delayedVote = new VoteTimeQueue(voteId, player, service, time, proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, projectedTotals == null ? "" : projectedTotals.toString(), false, uuid); + delayedVote.setRealVote(realVote); if (!getVoteCacheHandler().addTimeVoteToCache(delayedVote)) { logSevere("Unable to persist queued rollover vote for " + player + "/" + service + "; skipping proxy broadcast"); @@ -4602,7 +4954,10 @@ && getGlobalDataHandler().isTimeChangedHappened()) { retryState.realVote = realVote; liveVoteRetries.put(voteId, retryState); } - if (queuedVote != null) retryState.queuedVote = queuedVote; + if (queuedVote != null) { + retryState.queuedVote = queuedVote; + retryState.multiProxyForwardingHandled |= queuedVote.isMultiProxyForwardingHandled(); + } if (!retryState.votePartyApplied) { // Fence the side effect before invoking it. If the call reports an // indeterminate failure, a listener retry must not increment the party again. @@ -4954,11 +5309,17 @@ && getGlobalDataHandler().isTimeChangedHappened()) { // =========================== // Multiproxy forwarding // =========================== - if (getConfig().getMultiProxySupport() && getConfig().getPrimaryServer()) { + if (!retryState.multiProxyForwardingHandled && getConfig().getMultiProxySupport() + && getConfig().getPrimaryServer() + // An overflowed received envelope carries its sender in this field. It + // must be acknowledged after local completion, never forwarded again. + && (queuedVote == null || queuedVote.getMultiProxyOrigin().isBlank())) { if (!getConfig().getMultiProxyOneGlobalReward()) { debug("Sending global proxy vote envelope"); - multiProxyHandler.sendMultiProxyEnvelope(VotingPluginWire.vote(player, uuid, service, time, false, - realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); + if (!beginMultiProxyForwarding(retryState, + queuedVote == null ? retryState.queuedVote : queuedVote, player, uuid, service, time, realVote, text)) { + return QueuedVoteResult.RETRY; + } } else { // Only send to other proxies if the player DID NOT already receive reward on a // backend @@ -4971,11 +5332,13 @@ && getGlobalDataHandler().isTimeChangedHappened()) { if (shouldSend) { debug("Sending global proxy voteonline envelope"); - multiProxyHandler - .sendMultiProxyEnvelope(VotingPluginWire.voteOnline(player, uuid, service, time, false, - realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); + if (!beginMultiProxyForwarding(retryState, + queuedVote == null ? retryState.queuedVote : queuedVote, player, uuid, service, time, realVote, text)) { + return QueuedVoteResult.RETRY; + } } else { debug("Not sending global proxy message for voteonline, player already got reward"); + if (!markMultiProxyForwardingHandled(retryState, queuedVote)) return QueuedVoteResult.RETRY; } } } @@ -5007,6 +5370,138 @@ && getGlobalDataHandler().isTimeChangedHappened()) { } } + /** + * Creates/persists the sender outbox before the first publish. Redis and socket + * publish APIs are fire-and-forget, so their return value is never a delivery + * acknowledgement. The record stays until every configured peer confirms its + * own durable completion by stable vote ID. + */ + private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTimeQueue queuedVote, String player, + String uuid, String service, long time, boolean realVote, VoteTotalsSnapshot totals) { + if (multiProxyHandler == null) return false; + multiProxyHandler.announceMultiProxyVoteCapability(); + Set recipients = multiProxyHandler.getMultiProxyVoteRecipients(); + Set configuredRecipients = multiProxyHandler.getConfiguredMultiProxyVoteRecipients(); + // Keep custom MultiProxyHandler integrations that override only the + // established recipient method source-compatible while the base handler + // learns capabilities. + if (configuredRecipients.isEmpty() && !recipients.isEmpty()) configuredRecipients.addAll(recipients); + if (configuredRecipients.isEmpty()) return true; + Set legacyRecipients = new LinkedHashSet<>(configuredRecipients); + legacyRecipients.removeAll(recipients); + if (recipients.isEmpty()) { + // Older peers do not understand acknowledgements. Preserve their historical + // fire-and-forget route instead of creating an outbox they can never ACK. + multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(player, uuid, service, time, + false, realVote, totals == null ? "" : totals.toString(), findLiveVoteId(retryState), false, false, + 1, 1, getConfig().getProxyServerName()), legacyRecipients); + return true; + } + VoteTimeQueue outbox = queuedVote; + boolean newlyCreated = false; + if (outbox == null) { + outbox = new VoteTimeQueue(null, player, service, time, false, Collections.emptySet(), + Collections.emptySet(), totals == null ? "" : totals.toString(), true, uuid); + // The live retry key is the stable vote ID; copy it from the enclosing state + // by locating its identity rather than creating a new duplicate record. + outbox.setVoteId(findLiveVoteId(retryState)); + outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), recipients); + outbox.setRealVote(realVote); + outbox.setDeliveryStateDirty(true); + if (outbox.getVoteId() == null || !getVoteCacheHandler().addTimeVoteToCache(outbox)) return false; + retryState.queuedVote = outbox; + newlyCreated = true; + } + if (!outbox.isMultiProxyForwardingRequired()) { + outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), recipients); + outbox.setRealVote(realVote); + outbox.setProcessed(true); + outbox.setDeliveryStateDirty(true); + if (!persistTimeVoteDelivery(outbox)) return false; + } + if (outbox.hasCompletedMultiProxyAcknowledgements()) { + return markMultiProxyForwardingHandled(retryState, outbox); + } + // A successful publish only means the transport accepted the invocation. + // Keep the durable row and wait for an acknowledgement before continuing. + if (newlyCreated && !legacyRecipients.isEmpty()) { + // Never retry this legacy copy as part of the ACK outbox: a legacy peer has + // no receiver dedupe/ACK contract, while capable peers stay fully durable. + multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(player, uuid, service, time, + false, realVote, totals == null ? "" : totals.toString(), outbox.getVoteId(), false, false, 1, 1, + outbox.getMultiProxyOrigin()), legacyRecipients); + } + if (!sendDurableMultiProxyOutbox(outbox)) return false; + scheduleTimeVoteRetry(); + return false; + } + + private UUID findLiveVoteId(LiveVoteRetryState retryState) { + for (Map.Entry entry : liveVoteRetries.entrySet()) { + if (entry.getValue() == retryState) return entry.getKey(); + } + return null; + } + + /** Re-sends an unacknowledged outbox using the exact persisted stable ID. */ + private boolean retryDurableMultiProxyOutbox(VoteTimeQueue outbox) { + if (outbox.hasCompletedMultiProxyAcknowledgements()) { + outbox.setMultiProxyForwardingHandled(true); + outbox.setDeliveryStateDirty(true); + return persistTimeVoteDelivery(outbox); + } + sendDurableMultiProxyOutbox(outbox); + return false; + } + + private boolean sendDurableMultiProxyOutbox(VoteTimeQueue outbox) { + if (multiProxyHandler == null || outbox.getVoteId() == null || outbox.getMultiProxyOrigin().isBlank()) return false; + return multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(outbox.getName(), + outbox.getUuid(), outbox.getService(), outbox.getTime(), false, outbox.isRealVote(), outbox.getTotals(), + outbox.getVoteId(), false, false, 1, 1, outbox.getMultiProxyOrigin()), + outbox.getMultiProxyRecipients()); + } + + /** Handles an ACK only after confirming it belongs to a configured recipient. */ + private synchronized void handleMultiProxyVoteAcknowledgement(UUID voteId, String recipient) { + if (voteId == null || recipient == null || recipient.isBlank()) return; + LiveVoteRetryState state = liveVoteRetries.get(voteId); + VoteTimeQueue outbox = state == null ? null : state.queuedVote; + if (outbox == null) { + for (VoteTimeQueue candidate : getVoteCacheHandler().getTimeChangeQueue()) { + if (voteId.equals(candidate.getVoteId()) && candidate.isMultiProxyForwardingRequired()) { + outbox = candidate; + break; + } + } + } + if (outbox == null || !outbox.acknowledgeMultiProxyRecipient(recipient)) return; + outbox.setDeliveryStateDirty(true); + if (!persistTimeVoteDelivery(outbox)) { + scheduleTimeVoteRetry(); + return; + } + if (outbox.hasCompletedMultiProxyAcknowledgements()) { + outbox.setMultiProxyForwardingHandled(true); + outbox.setDeliveryStateDirty(true); + if (!persistTimeVoteDelivery(outbox)) { + scheduleTimeVoteRetry(); + return; + } + if (state != null) state.multiProxyForwardingHandled = true; + } + scheduleTimeVoteRetry(); + } + + /** Persists a queued vote's multi-proxy side-effect fence before later retryable work. */ + private boolean markMultiProxyForwardingHandled(LiveVoteRetryState retryState, VoteTimeQueue queuedVote) { + retryState.multiProxyForwardingHandled = true; + if (queuedVote == null) return true; + queuedVote.setMultiProxyForwardingHandled(true); + queuedVote.setDeliveryStateDirty(true); + return persistTimeVoteDelivery(queuedVote); + } + /** * Creates the reward cache entry for a backend. Standalone proxy broadcast * progress belongs to the single voter-keyed canonical state, not to every diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java index 29becc646..7d86fc5c7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java @@ -68,6 +68,10 @@ private VotingPluginWire() { // ========================= public static final String SUB_CLEAR_VOTE = "ClearVote"; public static final String SUB_CLEAR_VOTE_PRIMARY = "ClearVotePrimary"; + /** Additive acknowledgement for the reliable multi-proxy vote envelope. */ + public static final String SUB_MULTI_PROXY_VOTE_ACK = "MultiProxyVoteAck"; + /** Additive capability advertisement for durable multi-proxy acknowledgements. */ + public static final String SUB_MULTI_PROXY_CAPABILITIES = "MultiProxyCapabilities"; // ========================= // Field keys (public where referenced externally) @@ -102,6 +106,11 @@ private VotingPluginWire() { public static final String K_BUNGEE_BROADCAST = "bungeeBroadcast"; public static final String K_NUM = "num"; public static final String K_NUMBER_OF_VOTES = "numberOfVotes"; + /** Origin and receiving proxy names for reliable multi-proxy delivery. */ + public static final String K_MULTI_PROXY_ORIGIN = "multiProxyOrigin"; + public static final String K_MULTI_PROXY_RECIPIENT = "multiProxyRecipient"; + public static final String K_MULTI_PROXY_ACK_VERSION = "multiProxyAckVersion"; + public static final String K_MULTI_PROXY_CAPABILITY_REPLY = "multiProxyCapabilityReply"; // VoteUpdate extras public static final String K_PLAYER_UUID = "playerUuid"; @@ -140,6 +149,31 @@ public static JsonEnvelope voteOnline(String player, String uuid, String service .put(K_NUM, num).put(K_NUMBER_OF_VOTES, numberOfVotes).build(); } + /** + * Builds a multi-proxy vote with an additive sender identity for its durable + * acknowledgement. Older receivers safely ignore the extra field. + */ + public static JsonEnvelope multiProxyVote(String player, String uuid, String service, long time, boolean wasOnline, + boolean realVote, String totals, UUID voteId, boolean manageTotals, boolean bungeeBroadcast, int num, + int numberOfVotes, String origin) { + return base(SUB_VOTE).put(K_PLAYER, safe(player)).put(K_UUID, safe(uuid)).put(K_SERVICE, safe(service)) + .put(K_TIME, time).put(K_WAS_ONLINE, wasOnline).put(K_REAL_VOTE, realVote).put(K_TOTALS, safe(totals)) + .put(K_VOTE_ID, voteId == null ? "" : voteId.toString()).put(K_SET_TOTALS, true) + .put(K_MANAGE_TOTALS, manageTotals).put(K_BUNGEE_BROADCAST, bungeeBroadcast).put(K_NUM, num) + .put(K_NUMBER_OF_VOTES, numberOfVotes).put(K_MULTI_PROXY_ORIGIN, safe(origin)).build(); + } + + /** Reliable multi-proxy variant of {@link #voteOnline}. */ + public static JsonEnvelope multiProxyVoteOnline(String player, String uuid, String service, long time, + boolean wasOnline, boolean realVote, String totals, UUID voteId, boolean manageTotals, + boolean bungeeBroadcast, int num, int numberOfVotes, String origin) { + return base(SUB_VOTE_ONLINE).put(K_PLAYER, safe(player)).put(K_UUID, safe(uuid)).put(K_SERVICE, safe(service)) + .put(K_TIME, time).put(K_WAS_ONLINE, wasOnline).put(K_REAL_VOTE, realVote).put(K_TOTALS, safe(totals)) + .put(K_VOTE_ID, voteId == null ? "" : voteId.toString()).put(K_SET_TOTALS, true) + .put(K_MANAGE_TOTALS, manageTotals).put(K_BUNGEE_BROADCAST, bungeeBroadcast).put(K_NUM, num) + .put(K_NUMBER_OF_VOTES, numberOfVotes).put(K_MULTI_PROXY_ORIGIN, safe(origin)).build(); + } + public static JsonEnvelope voteDelayRejected(String player, String uuid, String service, boolean wasOnline) { return base(SUB_VOTE_DELAY_REJECTED).put(K_PLAYER, safe(player)).put(K_UUID, safe(uuid)) .put(K_SERVICE, safe(service)).put(K_WAS_ONLINE, wasOnline).build(); @@ -366,6 +400,24 @@ public static JsonEnvelope clearVotePrimary(String uuid, String player, String s .put(K_SERVER, safe(server)).build(); } + /** Acknowledges durable completion to the originating proxy. */ + public static JsonEnvelope multiProxyVoteAck(UUID voteId, String origin, String recipient) { + return base(SUB_MULTI_PROXY_VOTE_ACK).put(K_VOTE_ID, voteId == null ? "" : voteId.toString()) + .put(K_MULTI_PROXY_ORIGIN, safe(origin)).put(K_MULTI_PROXY_RECIPIENT, safe(recipient)).build(); + } + + /** Advertises support for the additive durable multi-proxy acknowledgement protocol. */ + public static JsonEnvelope multiProxyCapabilities(String recipient, int acknowledgementVersion) { + return multiProxyCapabilities(recipient, acknowledgementVersion, false); + } + + /** Capability response used to complete a bounded bidirectional handshake. */ + public static JsonEnvelope multiProxyCapabilities(String recipient, int acknowledgementVersion, boolean reply) { + return base(SUB_MULTI_PROXY_CAPABILITIES).put(K_MULTI_PROXY_RECIPIENT, safe(recipient)) + .put(K_MULTI_PROXY_ACK_VERSION, acknowledgementVersion) + .put(K_MULTI_PROXY_CAPABILITY_REPLY, reply).build(); + } + // ========================= // Readers (decode) // ========================= diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java index 636c50aa0..65a8064b5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java @@ -53,8 +53,15 @@ public void addTimedVote(int num, VoteTimeQueue voteTimedQueue) { setString(path + ".VoteId", voteTimedQueue.getVoteId() == null ? null : voteTimedQueue.getVoteId().toString()); setString(path + ".UUID", voteTimedQueue.getUuid()); setBoolean(path + ".ProxyBroadcastHandled", voteTimedQueue.isProxyBroadcastHandled()); - setString(path + ".Totals", voteTimedQueue.getTotals()); - setBoolean(path + ".Processed", voteTimedQueue.isProcessed()); + setString(path + ".Totals", voteTimedQueue.getTotals()); + setBoolean(path + ".Processed", voteTimedQueue.isProcessed()); + setBoolean(path + ".MultiProxyForwardingHandled", voteTimedQueue.isMultiProxyForwardingHandled()); + setBoolean(path + ".MultiProxyForwardingRequired", voteTimedQueue.isMultiProxyForwardingRequired()); + setBoolean(path + ".RealVote", voteTimedQueue.isRealVote()); + setString(path + ".MultiProxyOrigin", voteTimedQueue.getMultiProxyOrigin()); + setBoolean(path + ".MultiProxyCompletionPending", voteTimedQueue.isMultiProxyCompletionPending()); + setString(path + ".MultiProxyRecipients", voteTimedQueue.encodeMultiProxyRecipients()); + setString(path + ".MultiProxyAcknowledgedServers", voteTimedQueue.encodeMultiProxyAcknowledgedServers()); setString(path + ".BroadcastTargets", voteTimedQueue.encodeBroadcastTargets()); setString(path + ".BroadcastForwardedServers", voteTimedQueue.encodeBroadcastForwardedServers()); setString(path + ".HttpBroadcastDeliveryIds", voteTimedQueue.encodeHttpBroadcastDeliveryIds()); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java index fbd80ad54..81576b24e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java @@ -39,6 +39,13 @@ public String buildCreateTableSql(DbType dbType) { + qi("proxyBroadcastHandled") + " BOOLEAN NOT NULL DEFAULT FALSE, " + qi("totals") + " TEXT, " + qi("processed") + " BOOLEAN NOT NULL DEFAULT FALSE, " + + qi("multiProxyForwardingHandled") + " BOOLEAN NOT NULL DEFAULT FALSE, " + + qi("multiProxyForwardingRequired") + " BOOLEAN NOT NULL DEFAULT FALSE, " + + qi("realVote") + " BOOLEAN NOT NULL DEFAULT TRUE, " + + qi("multiProxyOrigin") + " VARCHAR(100), " + + qi("multiProxyCompletionPending") + " BOOLEAN NOT NULL DEFAULT FALSE, " + + qi("multiProxyRecipients") + " TEXT, " + + qi("multiProxyAcknowledgedServers") + " TEXT, " + qi("broadcastTargets") + " TEXT, " + qi("broadcastForwardedServers") + " TEXT, " + qi("httpBroadcastDeliveryIds") + " TEXT" @@ -55,6 +62,13 @@ public String buildCreateTableSql(DbType dbType) { + qi("proxyBroadcastHandled") + " TINYINT(1) NOT NULL DEFAULT 0," + qi("totals") + " TEXT," + qi("processed") + " TINYINT(1) NOT NULL DEFAULT 0," + + qi("multiProxyForwardingHandled") + " TINYINT(1) NOT NULL DEFAULT 0," + + qi("multiProxyForwardingRequired") + " TINYINT(1) NOT NULL DEFAULT 0," + + qi("realVote") + " TINYINT(1) NOT NULL DEFAULT 1," + + qi("multiProxyOrigin") + " VARCHAR(100)," + + qi("multiProxyCompletionPending") + " TINYINT(1) NOT NULL DEFAULT 0," + + qi("multiProxyRecipients") + " TEXT," + + qi("multiProxyAcknowledgedServers") + " TEXT," + qi("broadcastTargets") + " TEXT," + qi("broadcastForwardedServers") + " TEXT," + qi("httpBroadcastDeliveryIds") + " TEXT," @@ -113,6 +127,21 @@ private void ensureProxyBroadcastColumns() { ensureColumn("processed", getDbType() == DbType.POSTGRESQL ? "BOOLEAN NOT NULL DEFAULT FALSE" : "TINYINT(1) NOT NULL DEFAULT 0"); + ensureColumn("multiProxyForwardingHandled", getDbType() == DbType.POSTGRESQL + ? "BOOLEAN NOT NULL DEFAULT FALSE" + : "TINYINT(1) NOT NULL DEFAULT 0"); + ensureColumn("multiProxyForwardingRequired", getDbType() == DbType.POSTGRESQL + ? "BOOLEAN NOT NULL DEFAULT FALSE" + : "TINYINT(1) NOT NULL DEFAULT 0"); + ensureColumn("realVote", getDbType() == DbType.POSTGRESQL + ? "BOOLEAN NOT NULL DEFAULT TRUE" + : "TINYINT(1) NOT NULL DEFAULT 1"); + ensureColumn("multiProxyOrigin", "VARCHAR(100)"); + ensureColumn("multiProxyCompletionPending", getDbType() == DbType.POSTGRESQL + ? "BOOLEAN NOT NULL DEFAULT FALSE" + : "TINYINT(1) NOT NULL DEFAULT 0"); + ensureColumn("multiProxyRecipients", "TEXT"); + ensureColumn("multiProxyAcknowledgedServers", "TEXT"); ensureColumn("httpBroadcastDeliveryIds", "TEXT"); } @@ -159,13 +188,19 @@ private void ensureIndexes() { * @param processed whether normal replay processing completed * @return true when the row was inserted */ - public boolean insertTimedVote(UUID voteId, String uuid, String playerName, String service, long time, + private boolean insertTimedVote(UUID voteId, String uuid, String playerName, String service, long time, boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, - boolean processed, String httpBroadcastDeliveryIds) { + boolean processed, boolean multiProxyForwardingHandled, boolean multiProxyForwardingRequired, + boolean realVote, String multiProxyOrigin, boolean multiProxyCompletionPending, String multiProxyRecipients, + String multiProxyAcknowledgedServers, String httpBroadcastDeliveryIds) { String sql = "INSERT INTO " + qi(getTableName()) + " (" + qi("playerName") + ", " + qi("service") + ", " + qi("time") + ", " + qi("voteId") + ", " + qi("uuid") + ", " + qi("proxyBroadcastHandled") + ", " + qi("broadcastTargets") + ", " + qi("broadcastForwardedServers") + ", " + qi("totals") + ", " - + qi("processed") + ", " + qi("httpBroadcastDeliveryIds") + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; + + qi("processed") + ", " + qi("multiProxyForwardingHandled") + ", " + + qi("multiProxyForwardingRequired") + ", " + qi("realVote") + ", " + qi("multiProxyOrigin") + + ", " + qi("multiProxyCompletionPending") + ", " + qi("multiProxyRecipients") + ", " + + qi("multiProxyAcknowledgedServers") + ", " + qi("httpBroadcastDeliveryIds") + + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; try (Connection conn = mysql.getConnectionManager().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { ps.setString(1, playerName); @@ -183,10 +218,24 @@ public boolean insertTimedVote(UUID voteId, String uuid, String playerName, Stri ps.setString(9, totals); if (getDbType() == DbType.POSTGRESQL) { ps.setBoolean(10, processed); + ps.setBoolean(11, multiProxyForwardingHandled); } else { ps.setInt(10, processed ? 1 : 0); + ps.setInt(11, multiProxyForwardingHandled ? 1 : 0); } - ps.setString(11, httpBroadcastDeliveryIds); + if (getDbType() == DbType.POSTGRESQL) { + ps.setBoolean(12, multiProxyForwardingRequired); + ps.setBoolean(13, realVote); + } else { + ps.setInt(12, multiProxyForwardingRequired ? 1 : 0); + ps.setInt(13, realVote ? 1 : 0); + } + ps.setString(14, multiProxyOrigin); + if (getDbType() == DbType.POSTGRESQL) ps.setBoolean(15, multiProxyCompletionPending); + else ps.setInt(15, multiProxyCompletionPending ? 1 : 0); + ps.setString(16, multiProxyRecipients); + ps.setString(17, multiProxyAcknowledgedServers); + ps.setString(18, httpBroadcastDeliveryIds); ps.executeUpdate(); return true; } catch (SQLException e) { @@ -195,12 +244,41 @@ public boolean insertTimedVote(UUID voteId, String uuid, String playerName, Stri } } + /** Inserts the complete durable replay state without a post-insert crash window. */ + public boolean insertTimedVote(VoteTimeQueue vote) { + if (vote == null) return false; + return insertTimedVote(vote.getVoteId(), vote.getUuid(), vote.getName(), vote.getService(), vote.getTime(), + vote.isProxyBroadcastHandled(), vote.encodeBroadcastTargets(), vote.encodeBroadcastForwardedServers(), + vote.getTotals(), vote.isProcessed(), vote.isMultiProxyForwardingHandled(), + vote.isMultiProxyForwardingRequired(), vote.isRealVote(), vote.getMultiProxyOrigin(), + vote.isMultiProxyCompletionPending(), + vote.encodeMultiProxyRecipients(), vote.encodeMultiProxyAcknowledgedServers(), + vote.encodeHttpBroadcastDeliveryIds()); + } + + /** Backward-compatible insert overload. */ + public boolean insertTimedVote(UUID voteId, String uuid, String playerName, String service, long time, + boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, + boolean processed, boolean multiProxyForwardingHandled, String httpBroadcastDeliveryIds) { + return insertTimedVote(voteId, uuid, playerName, service, time, proxyBroadcastHandled, broadcastTargets, + broadcastForwardedServers, totals, processed, multiProxyForwardingHandled, false, true, "", false, "", "", + httpBroadcastDeliveryIds); + } + /** Backward-compatible insert overload without HTTP delivery state. */ public boolean insertTimedVote(UUID voteId, String uuid, String playerName, String service, long time, boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, boolean processed) { return insertTimedVote(voteId, uuid, playerName, service, time, proxyBroadcastHandled, broadcastTargets, - broadcastForwardedServers, totals, processed, ""); + broadcastForwardedServers, totals, processed, false, ""); + } + + /** Backward-compatible insert overload without the durable multi-proxy forwarding fence. */ + public boolean insertTimedVote(UUID voteId, String uuid, String playerName, String service, long time, + boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, + boolean processed, String httpBroadcastDeliveryIds) { + return insertTimedVote(voteId, uuid, playerName, service, time, proxyBroadcastHandled, broadcastTargets, + broadcastForwardedServers, totals, processed, false, httpBroadcastDeliveryIds); } /** @@ -213,8 +291,12 @@ public boolean updateTimedVote(VoteTimeQueue vote) { boolean hasVoteId = vote.getVoteId() != null; String sql = "UPDATE " + qi(getTableName()) + " SET " + qi("proxyBroadcastHandled") + " = ?, " + qi("broadcastTargets") + " = ?, " + qi("broadcastForwardedServers") + " = ?, " + qi("totals") - + " = ?, " + qi("processed") + " = ?, " + qi("uuid") + " = ?, " + qi("httpBroadcastDeliveryIds") - + " = ? WHERE " + + " = ?, " + qi("processed") + " = ?, " + qi("multiProxyForwardingHandled") + " = ?, " + + qi("multiProxyForwardingRequired") + " = ?, " + qi("realVote") + " = ?, " + + qi("multiProxyOrigin") + " = ?, " + qi("multiProxyCompletionPending") + " = ?, " + + qi("multiProxyRecipients") + " = ?, " + + qi("multiProxyAcknowledgedServers") + " = ?, " + qi("uuid") + " = ?, " + + qi("httpBroadcastDeliveryIds") + " = ? WHERE " + (hasVoteId ? qi("voteId") + " = ?;" : qi("playerName") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ?;"); try (Connection conn = mysql.getConnectionManager().getConnection(); @@ -229,17 +311,31 @@ public boolean updateTimedVote(VoteTimeQueue vote) { ps.setString(4, vote.getTotals()); if (getDbType() == DbType.POSTGRESQL) { ps.setBoolean(5, vote.isProcessed()); + ps.setBoolean(6, vote.isMultiProxyForwardingHandled()); } else { ps.setInt(5, vote.isProcessed() ? 1 : 0); + ps.setInt(6, vote.isMultiProxyForwardingHandled() ? 1 : 0); } - ps.setString(6, vote.getUuid()); - ps.setString(7, vote.encodeHttpBroadcastDeliveryIds()); + if (getDbType() == DbType.POSTGRESQL) { + ps.setBoolean(7, vote.isMultiProxyForwardingRequired()); + ps.setBoolean(8, vote.isRealVote()); + } else { + ps.setInt(7, vote.isMultiProxyForwardingRequired() ? 1 : 0); + ps.setInt(8, vote.isRealVote() ? 1 : 0); + } + ps.setString(9, vote.getMultiProxyOrigin()); + if (getDbType() == DbType.POSTGRESQL) ps.setBoolean(10, vote.isMultiProxyCompletionPending()); + else ps.setInt(10, vote.isMultiProxyCompletionPending() ? 1 : 0); + ps.setString(11, vote.encodeMultiProxyRecipients()); + ps.setString(12, vote.encodeMultiProxyAcknowledgedServers()); + ps.setString(13, vote.getUuid()); + ps.setString(14, vote.encodeHttpBroadcastDeliveryIds()); if (hasVoteId) { - ps.setString(8, vote.getVoteId().toString()); + ps.setString(15, vote.getVoteId().toString()); } else { - ps.setString(8, vote.getName()); - ps.setString(9, vote.getService()); - ps.setLong(10, vote.getTime()); + ps.setString(15, vote.getName()); + ps.setString(16, vote.getService()); + ps.setLong(17, vote.getTime()); } ps.executeUpdate(); return true; @@ -367,6 +463,13 @@ private List selectVotes(String sql, Object[] params) { rs.getString("broadcastForwardedServers"), rs.getString("totals"), rs.getBoolean("processed"), + rs.getBoolean("multiProxyForwardingHandled"), + rs.getBoolean("multiProxyForwardingRequired"), + rs.getBoolean("realVote"), + rs.getString("multiProxyOrigin"), + rs.getBoolean("multiProxyCompletionPending"), + rs.getString("multiProxyRecipients"), + rs.getString("multiProxyAcknowledgedServers"), rs.getString("httpBroadcastDeliveryIds") )); } @@ -403,6 +506,13 @@ public static class TimedVoteRow { private final String broadcastForwardedServers; private final String totals; private final boolean processed; + private final boolean multiProxyForwardingHandled; + private final boolean multiProxyForwardingRequired; + private final boolean realVote; + private final String multiProxyOrigin; + private final boolean multiProxyCompletionPending; + private final String multiProxyRecipients; + private final String multiProxyAcknowledgedServers; private final String httpBroadcastDeliveryIds; /** @@ -420,7 +530,9 @@ public static class TimedVoteRow { */ public TimedVoteRow(int id, String playerName, String service, long time, UUID voteId, String uuid, boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, - boolean processed, String httpBroadcastDeliveryIds) { + boolean processed, boolean multiProxyForwardingHandled, boolean multiProxyForwardingRequired, + boolean realVote, String multiProxyOrigin, boolean multiProxyCompletionPending, String multiProxyRecipients, + String multiProxyAcknowledgedServers, String httpBroadcastDeliveryIds) { this.id = id; this.playerName = playerName; this.service = service; @@ -432,6 +544,13 @@ public TimedVoteRow(int id, String playerName, String service, long time, UUID v this.broadcastForwardedServers = broadcastForwardedServers; this.totals = totals; this.processed = processed; + this.multiProxyForwardingHandled = multiProxyForwardingHandled; + this.multiProxyForwardingRequired = multiProxyForwardingRequired; + this.realVote = realVote; + this.multiProxyOrigin = multiProxyOrigin; + this.multiProxyCompletionPending = multiProxyCompletionPending; + this.multiProxyRecipients = multiProxyRecipients; + this.multiProxyAcknowledgedServers = multiProxyAcknowledgedServers; this.httpBroadcastDeliveryIds = httpBroadcastDeliveryIds; } @@ -440,7 +559,16 @@ public TimedVoteRow(int id, String playerName, String service, long time, UUID v boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, boolean processed) { this(id, playerName, service, time, voteId, uuid, proxyBroadcastHandled, broadcastTargets, - broadcastForwardedServers, totals, processed, null); + broadcastForwardedServers, totals, processed, false, false, true, "", false, "", "", null); + } + + /** Backward-compatible row constructor without the durable multi-proxy forwarding fence. */ + public TimedVoteRow(int id, String playerName, String service, long time, UUID voteId, String uuid, + boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, + boolean processed, String httpBroadcastDeliveryIds) { + this(id, playerName, service, time, voteId, uuid, proxyBroadcastHandled, broadcastTargets, + broadcastForwardedServers, totals, processed, false, false, true, "", false, "", "", + httpBroadcastDeliveryIds); } /** @@ -516,6 +644,17 @@ public boolean isProcessed() { return processed; } + public boolean isMultiProxyForwardingHandled() { + return multiProxyForwardingHandled; + } + + public boolean isMultiProxyForwardingRequired() { return multiProxyForwardingRequired; } + public boolean isRealVote() { return realVote; } + public String getMultiProxyOrigin() { return multiProxyOrigin; } + public boolean isMultiProxyCompletionPending() { return multiProxyCompletionPending; } + public String getMultiProxyRecipients() { return multiProxyRecipients; } + public String getMultiProxyAcknowledgedServers() { return multiProxyAcknowledgedServers; } + public String getHttpBroadcastDeliveryIds() { return httpBroadcastDeliveryIds; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index a7f39327f..d8b6b35f3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -8,6 +8,7 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; import java.util.LinkedHashSet; import java.util.Map; import java.util.Map.Entry; @@ -16,6 +17,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Stream; import com.bencodez.simpleapi.sql.mysql.MySQL; import com.bencodez.simpleapi.sql.mysql.config.MysqlConfig; @@ -30,6 +32,7 @@ */ public abstract class VoteCacheHandler { private static final int MAX_PENDING_PERSISTENCE_VOTES = 1024; + private static final int MAX_MULTI_PROXY_COMPLETIONS = 4096; /** * Queue of timed votes for time change processing. @@ -743,10 +746,7 @@ public synchronized boolean addTimeVoteToCache(VoteTimeQueue vote) { } timeChangeQueue.add(vote); if (useMySQL) { - boolean stored = timedVoteCacheTable.insertTimedVote(vote.getVoteId(), vote.getUuid(), vote.getName(), vote.getService(), - vote.getTime(), vote.isProxyBroadcastHandled(), vote.encodeBroadcastTargets(), - vote.encodeBroadcastForwardedServers(), vote.getTotals(), vote.isProcessed(), - vote.encodeHttpBroadcastDeliveryIds()); + boolean stored = timedVoteCacheTable.insertTimedVote(vote); if (!stored) { timeChangeQueue.remove(vote); } @@ -905,6 +905,88 @@ public synchronized void clearTimeVoteCompletion(VoteTimeQueue vote) { } } + /** + * Records a consumed multi-proxy vote before the receiver drops its in-memory + * retry fence. This deliberately uses the JSON cache's sibling directory even + * when SQL is the primary cache, so it remains available during SQL recovery. + * + * @param voteId stable multi-proxy vote identity + * @return true when the completion record is durably readable + */ + public synchronized boolean markMultiProxyVoteCompletedDurably(UUID voteId) { + Path target = multiProxyVoteCompletionPath(voteId); + if (target == null) return false; + Path staged = null; + try { + Files.createDirectories(target.getParent()); + if (hasMultiProxyVoteCompletion(voteId)) return true; + staged = Files.createTempFile(target.getParent(), ".multiproxy-completion-", ".tmp"); + Files.writeString(staged, voteId.toString(), StandardCharsets.UTF_8); + try { + DurableFiles.publishStagedFile(staged, target); + } catch (DurableFiles.PublishedException published) { + // The record is visible; the read-back below decides whether it is usable. + } + if (!hasMultiProxyVoteCompletion(voteId)) return false; + // Publish the new fence before pruning old fences. A pruning failure may + // leave the bounded journal temporarily one entry over capacity, but it + // must never remove an existing fence and then lose the new one. + if (!pruneMultiProxyVoteCompletions(target.getParent())) { + debug1("Unable to prune completed multi-proxy vote fences after publishing " + voteId); + } + return true; + } catch (IOException | RuntimeException failure) { + debug1(failure); + return false; + } finally { + if (staged != null) { + try { Files.deleteIfExists(staged); } catch (IOException ignored) { } + } + } + } + + /** Checks whether a multi-proxy receiver completion survived this runtime. */ + public synchronized boolean hasMultiProxyVoteCompletion(UUID voteId) { + Path target = multiProxyVoteCompletionPath(voteId); + if (target == null || !Files.isRegularFile(target)) return false; + try { + return voteId.toString().equals(Files.readString(target, StandardCharsets.UTF_8)); + } catch (IOException | RuntimeException failure) { + debug1(failure); + return false; + } + } + + private boolean pruneMultiProxyVoteCompletions(Path directory) { + ArrayList records = new ArrayList<>(); + try (Stream files = Files.list(directory)) { + files.filter(Files::isRegularFile).forEach(records::add); + } catch (IOException | RuntimeException failure) { + debug1(failure); + return false; + } + records.sort(Comparator.comparingLong(this::multiProxyCompletionModifiedTime)); + while (records.size() > MAX_MULTI_PROXY_COMPLETIONS) { + Path oldest = records.remove(0); + try { + DurableFiles.deleteIfExists(oldest); + } catch (IOException | RuntimeException failure) { + debug1(failure); + return false; + } + } + return true; + } + + private long multiProxyCompletionModifiedTime(Path record) { + try { + return Files.getLastModifiedTime(record).toMillis(); + } catch (IOException | RuntimeException failure) { + debug1(failure); + return Long.MIN_VALUE; + } + } + private Path timeVoteCompletionPath(VoteTimeQueue vote) { if (vote == null || jsonStorage == null || jsonStorage.getStoragePath() == null) return null; Path storage = jsonStorage.getStoragePath().toAbsolutePath().normalize(); @@ -915,6 +997,14 @@ private Path timeVoteCompletionPath(VoteTimeQueue vote) { return storage.resolveSibling(name + ".completed-timed-votes").resolve(key.toString()); } + private Path multiProxyVoteCompletionPath(UUID voteId) { + if (voteId == null || jsonStorage == null || jsonStorage.getStoragePath() == null) return null; + Path storage = jsonStorage.getStoragePath().toAbsolutePath().normalize(); + Path name = storage.getFileName(); + if (name == null) return null; + return storage.resolveSibling(name + ".completed-multiproxy-votes").resolve(voteId.toString()); + } + private String timeVoteCompletionIdentity(VoteTimeQueue vote) { return (vote.getVoteId() == null ? "" : vote.getVoteId()) + "\n" + vote.getUuid() + "\n" + vote.getName() + "\n" + vote.getService() + "\n" + vote.getTime(); @@ -960,8 +1050,18 @@ public void load() { timedVoteRow.getService(), timedVoteRow.getTime(), timedVoteRow.isProxyBroadcastHandled(), VoteTimeQueue.decodeBroadcastForwardedServers(timedVoteRow.getBroadcastTargets()), VoteTimeQueue.decodeBroadcastForwardedServers(timedVoteRow.getBroadcastForwardedServers()), - timedVoteRow.getTotals(), timedVoteRow.isProcessed(), timedVoteRow.getUuid(), + timedVoteRow.getTotals(), timedVoteRow.isProcessed(), + timedVoteRow.isMultiProxyForwardingHandled(), timedVoteRow.getUuid(), VoteTimeQueue.decodeHttpBroadcastDeliveryIds(timedVoteRow.getHttpBroadcastDeliveryIds())); + voteTimeQueue.setMultiProxyForwardingRequired(timedVoteRow.isMultiProxyForwardingRequired()); + voteTimeQueue.setRealVote(timedVoteRow.isRealVote()); + voteTimeQueue.setMultiProxyOrigin(timedVoteRow.getMultiProxyOrigin() == null ? "" + : timedVoteRow.getMultiProxyOrigin()); + voteTimeQueue.setMultiProxyCompletionPending(timedVoteRow.isMultiProxyCompletionPending()); + voteTimeQueue.setMultiProxyRecipients( + VoteTimeQueue.decodeBroadcastForwardedServers(timedVoteRow.getMultiProxyRecipients())); + voteTimeQueue.setMultiProxyAcknowledgedServers( + VoteTimeQueue.decodeBroadcastForwardedServers(timedVoteRow.getMultiProxyAcknowledgedServers())); timedVotes.add(voteTimeQueue); }); timeChangeQueue.addAll(timedVotes); @@ -988,13 +1088,29 @@ public void load() { : ""; String totals = data.has("Totals") ? data.get("Totals").asString() : ""; boolean processed = data.has("Processed") && data.get("Processed").asBoolean(); + boolean multiProxyForwardingHandled = data.has("MultiProxyForwardingHandled") + && data.get("MultiProxyForwardingHandled").asBoolean(); String httpBroadcastDeliveryIds = data.has("HttpBroadcastDeliveryIds") ? data.get("HttpBroadcastDeliveryIds").asString() : ""; - getTimeChangeQueue().add(new VoteTimeQueue(voteId, name, service, time, proxyBroadcastHandled, + VoteTimeQueue queuedVote = new VoteTimeQueue(voteId, name, service, time, proxyBroadcastHandled, VoteTimeQueue.decodeBroadcastForwardedServers(broadcastTargets), - VoteTimeQueue.decodeBroadcastForwardedServers(forwardedServers), totals, processed, uuid, - VoteTimeQueue.decodeHttpBroadcastDeliveryIds(httpBroadcastDeliveryIds))); + VoteTimeQueue.decodeBroadcastForwardedServers(forwardedServers), totals, processed, + multiProxyForwardingHandled, uuid, + VoteTimeQueue.decodeHttpBroadcastDeliveryIds(httpBroadcastDeliveryIds)); + queuedVote.setMultiProxyForwardingRequired(data.has("MultiProxyForwardingRequired") + && data.get("MultiProxyForwardingRequired").asBoolean()); + queuedVote.setRealVote(!data.has("RealVote") || data.get("RealVote").asBoolean()); + queuedVote.setMultiProxyOrigin(data.has("MultiProxyOrigin") + ? data.get("MultiProxyOrigin").asString() : ""); + queuedVote.setMultiProxyCompletionPending(data.has("MultiProxyCompletionPending") + && data.get("MultiProxyCompletionPending").asBoolean()); + queuedVote.setMultiProxyRecipients(VoteTimeQueue.decodeBroadcastForwardedServers( + data.has("MultiProxyRecipients") ? data.get("MultiProxyRecipients").asString() : "")); + queuedVote.setMultiProxyAcknowledgedServers(VoteTimeQueue.decodeBroadcastForwardedServers( + data.has("MultiProxyAcknowledgedServers") + ? data.get("MultiProxyAcknowledgedServers").asString() : "")); + getTimeChangeQueue().add(queuedVote); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java index be793efa3..a69c7900c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java @@ -4,8 +4,12 @@ import java.io.File; import java.util.Collection; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.UUID; import com.bencodez.simpleapi.encryption.EncryptionHandler; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; @@ -25,6 +29,8 @@ public abstract class MultiProxyHandler { private HashMap multiproxyClientHandles; private SocketHandler multiproxySocketHandler; + /** Peers that have explicitly advertised the additive durable-ACK protocol. */ + private final Set acknowledgedVoteCapabilityPeers = new LinkedHashSet<>(); @Getter private RedisHandler multiProxyRedis; @@ -64,6 +70,7 @@ public synchronized void close() { multiProxyRedis = null; stopSocketClients(multiproxyClientHandles); multiproxyClientHandles = null; + acknowledgedVoteCapabilityPeers.clear(); } /** @@ -249,6 +256,64 @@ public synchronized void close() { public abstract void triggerVote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, VoteTotalsSnapshot text, String uuid); + /** + * Triggers a vote using the stable ID carried by a multi-proxy envelope. + * Existing implementations retain the legacy callback contract; implementations + * that support durable retries should override this overload. + */ + public void triggerVote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid, UUID voteId) { + triggerVote(player, service, realVote, timeQueue, queueTime, text, uuid); + } + + /** + * Additive reliable-delivery overload. Existing integrations retain the + * stable-ID behavior without needing to understand acknowledgements. + */ + public void triggerVote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid, UUID voteId, String origin) { + triggerVote(player, service, realVote, timeQueue, queueTime, text, uuid, voteId); + } + + /** Called when a receiver acknowledges this proxy's stable vote ID. */ + public void onMultiProxyVoteAcknowledged(UUID voteId, String recipient) { + // Optional for legacy implementations. + } + + /** + * Publishes an acknowledgement after receiver completion is durable. The + * broadcast route keeps sockets and Redis compatible; recipients filter it by + * the origin field, and duplicate votes cause the receiver to acknowledge again. + */ + public void acknowledgeMultiProxyVote(UUID voteId, String origin) { + if (voteId == null || origin == null || origin.isBlank()) return; + sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVoteAck(voteId, origin, getMultiProxyServerName())); + } + + /** Returns every configured remote proxy recipient, independent of version. */ + public synchronized Set getConfiguredMultiProxyVoteRecipients() { + Collection source = getMultiProxyMethod().equals(MultiProxyMethod.SOCKETS) + ? getMultiProxyServers() : getProxyServers(); + Set recipients = new LinkedHashSet<>(); + if (source == null) return recipients; + for (String server : source) { + if (server != null && !server.isBlank()) recipients.add(server.toLowerCase(Locale.ROOT)); + } + return recipients; + } + + /** Returns only configured peers that explicitly support durable acknowledgements. */ + public synchronized Set getMultiProxyVoteRecipients() { + Set recipients = getConfiguredMultiProxyVoteRecipients(); + recipients.retainAll(acknowledgedVoteCapabilityPeers); + return recipients; + } + + /** Broadcasts this node's durable-ACK capability to configured peers. */ + public synchronized void announceMultiProxyVoteCapability() { + sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyCapabilities(getMultiProxyServerName(), 1)); + } + /** * Loads multi-proxy support. */ @@ -315,6 +380,7 @@ public void debug(String message) { } logInfo("Loaded multi-proxy support: " + getMultiProxyMethod().toString()); + announceMultiProxyVoteCapability(); } /** @@ -359,24 +425,73 @@ public void sendStatus() { * @param envelope the envelope to send */ public synchronized void sendMultiProxyEnvelope(JsonEnvelope envelope) { - if (envelope == null) { - return; + sendMultiProxyEnvelopeAccepted(envelope); + } + + /** + * Sends an envelope and reports whether the configured transport accepted it. + * + *

The legacy send method is intentionally retained for callers that do not + * need delivery fencing. A vote producer must use this result: a missing + * client, an unavailable Redis connection, an empty destination list, or a + * transport exception must leave the vote retryable instead of claiming that + * forwarding completed. Socket clients are fire-and-forget in SimpleAPI, so + * acceptance means that the configured client accepted the send invocation; + * the stable wire vote ID remains the receiver-side duplicate fence.

+ * + * @param envelope the envelope to send + * @return true only when every configured destination accepted the envelope + */ + public synchronized boolean sendMultiProxyEnvelopeAccepted(JsonEnvelope envelope) { + return sendMultiProxyEnvelopeAccepted(envelope, getConfiguredMultiProxyVoteRecipients()); + } + + /** + * Sends an envelope to a selected subset of configured peers. Reliable senders + * use this to retry only ACK-capable peers, avoiding duplicate legacy delivery. + */ + public synchronized boolean sendMultiProxyEnvelopeAccepted(JsonEnvelope envelope, Collection recipients) { + if (envelope == null) return false; + if (recipients == null || recipients.isEmpty()) return false; + Set requested = new LinkedHashSet<>(); + for (String recipient : recipients) { + if (recipient != null && !recipient.isBlank()) requested.add(recipient.toLowerCase(Locale.ROOT)); } + if (requested.isEmpty()) return false; if (getMultiProxyMethod().equals(MultiProxyMethod.SOCKETS)) { - if (multiproxyClientHandles == null) { - return; - } - for (ClientHandler h : multiproxyClientHandles.values()) { - h.sendEnvelope(envelope); + if (multiproxyClientHandles == null || multiproxyClientHandles.isEmpty()) return false; + boolean accepted = true; + int destinations = 0; + for (Map.Entry entry : multiproxyClientHandles.entrySet()) { + if (entry.getKey() == null || !requested.contains(entry.getKey().toLowerCase(Locale.ROOT))) continue; + ClientHandler h = entry.getValue(); + if (h == null) { + accepted = false; + continue; + } + destinations++; + try { + h.sendEnvelope(envelope); + } catch (RuntimeException failure) { + accepted = false; + } } + return accepted && destinations == requested.size(); } else if (getMultiProxyMethod().equals(MultiProxyMethod.REDIS)) { - if (multiProxyRedis == null) { - return; - } - for (String server : getProxyServers()) { - multiProxyRedis.publishEnvelope("VotingPluginProxy_" + server, envelope); + if (multiProxyRedis == null) return false; + boolean accepted = true; + int destinations = 0; + for (String server : requested) { + destinations++; + try { + multiProxyRedis.publishEnvelope("VotingPluginProxy_" + server, envelope); + } catch (RuntimeException failure) { + accepted = false; + } } + return accepted && destinations == requested.size(); } + return false; } static void stopSocketClients(Map clients) { @@ -439,15 +554,53 @@ private void handleEnvelope(JsonEnvelope envelope) { return; } + if (sub.equalsIgnoreCase(VotingPluginWire.SUB_MULTI_PROXY_VOTE_ACK)) { + String origin = f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_ORIGIN, ""); + String recipient = f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_RECIPIENT, ""); + if (!origin.equalsIgnoreCase(getMultiProxyServerName())) return; + try { + onMultiProxyVoteAcknowledged(UUID.fromString(f.getOrDefault(VotingPluginWire.K_VOTE_ID, "")), recipient); + } catch (IllegalArgumentException ignored) { + // Ignore malformed acknowledgements; they must never clear a sender fence. + } + return; + } + + if (sub.equalsIgnoreCase(VotingPluginWire.SUB_MULTI_PROXY_CAPABILITIES)) { + String recipient = f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_RECIPIENT, ""); + boolean reply = Boolean.parseBoolean(f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_CAPABILITY_REPLY, "false")); + boolean replyRequired = false; + try { + int version = Integer.parseInt(f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_ACK_VERSION, "0")); + String normalized = recipient.toLowerCase(Locale.ROOT); + synchronized (this) { + if (version >= 1 && getConfiguredMultiProxyVoteRecipients().contains(normalized)) { + acknowledgedVoteCapabilityPeers.add(normalized); + replyRequired = !reply; + } + } + } catch (IllegalArgumentException ignored) { + // A malformed capability must leave the peer on the legacy route. + } + if (replyRequired) { + sendMultiProxyEnvelopeAccepted( + VotingPluginWire.multiProxyCapabilities(getMultiProxyServerName(), 1, true)); + } + return; + } + if (sub.equalsIgnoreCase(VotingPluginWire.SUB_VOTE) || sub.equalsIgnoreCase(VotingPluginWire.SUB_VOTE_ONLINE)) { + final VotingPluginWire.Vote wireVote = VotingPluginWire.readVote(envelope); final String player = f.getOrDefault(VotingPluginWire.K_PLAYER, ""); final String uuid = f.getOrDefault(VotingPluginWire.K_UUID, ""); final String service = f.getOrDefault(VotingPluginWire.K_SERVICE, ""); final String totals = f.getOrDefault(VotingPluginWire.K_TOTALS, ""); final boolean realVote = Boolean.parseBoolean(f.getOrDefault(VotingPluginWire.K_REAL_VOTE, "false")); + final String origin = f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_ORIGIN, ""); if (!player.isEmpty() && !uuid.isEmpty() && !service.isEmpty()) { - triggerVote(player, service, realVote, true, 0L, VoteTotalsSnapshot.parseStorage(totals), uuid); + triggerVote(player, service, realVote, true, 0L, VoteTotalsSnapshot.parseStorage(totals), uuid, + wireVote.voteId, origin); } return; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java index a6f629dc6..ab5a13194 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java @@ -66,6 +66,18 @@ public void addTimedVote(int num, VoteTimeQueue voteTimedQueue) { "ProxyBroadcastHandled"); setPath(voteTimedQueue.getTotals(), "TimedVoteCache", String.valueOf(num), "Totals"); setPath(voteTimedQueue.isProcessed(), "TimedVoteCache", String.valueOf(num), "Processed"); + setPath(voteTimedQueue.isMultiProxyForwardingHandled(), "TimedVoteCache", String.valueOf(num), + "MultiProxyForwardingHandled"); + setPath(voteTimedQueue.isMultiProxyForwardingRequired(), "TimedVoteCache", String.valueOf(num), + "MultiProxyForwardingRequired"); + setPath(voteTimedQueue.isRealVote(), "TimedVoteCache", String.valueOf(num), "RealVote"); + setPath(voteTimedQueue.getMultiProxyOrigin(), "TimedVoteCache", String.valueOf(num), "MultiProxyOrigin"); + setPath(voteTimedQueue.isMultiProxyCompletionPending(), "TimedVoteCache", String.valueOf(num), + "MultiProxyCompletionPending"); + setPath(voteTimedQueue.encodeMultiProxyRecipients(), "TimedVoteCache", String.valueOf(num), + "MultiProxyRecipients"); + setPath(voteTimedQueue.encodeMultiProxyAcknowledgedServers(), "TimedVoteCache", String.valueOf(num), + "MultiProxyAcknowledgedServers"); setPath(voteTimedQueue.encodeBroadcastTargets(), "TimedVoteCache", String.valueOf(num), "BroadcastTargets"); setPath(voteTimedQueue.encodeBroadcastForwardedServers(), "TimedVoteCache", String.valueOf(num), "BroadcastForwardedServers"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java index 6c3df8256..02a507321 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java @@ -41,6 +41,26 @@ public class VoteTimeQueue { @Getter @Setter private boolean processed; + /** Whether multi-proxy forwarding was durably handled for this queued vote. */ + @Getter + @Setter + private boolean multiProxyForwardingHandled; + /** True when this row is a durable multi-proxy delivery outbox. */ + @Getter + @Setter + private boolean multiProxyForwardingRequired; + /** Original vote type; legacy rows default to a real vote. */ + @Getter + @Setter + private boolean realVote = true; + /** Sender identity used to route acknowledgement envelopes. */ + @Getter + @Setter + private String multiProxyOrigin = ""; + /** Receiver completion is durable in this row but still needs a completion tombstone/ACK. */ + @Getter + @Setter + private boolean multiProxyCompletionPending; @Getter @Setter private boolean deliveryStateDirty; @@ -48,6 +68,12 @@ public class VoteTimeQueue { private Set broadcastTargets; @Getter private Set broadcastForwardedServers; + /** Configured recipient proxy names for the durable multi-proxy outbox. */ + @Getter + private Set multiProxyRecipients; + /** Recipients whose durable completion acknowledgement was received. */ + @Getter + private Set multiProxyAcknowledgedServers; /** Stable HTTP standalone-broadcast delivery IDs by target server. */ private final Map httpBroadcastDeliveryIds; @@ -137,6 +163,18 @@ public VoteTimeQueue(UUID voteId, String name, String service, long time, boolea public VoteTimeQueue(UUID voteId, String name, String service, long time, boolean proxyBroadcastHandled, Set broadcastTargets, Set broadcastForwardedServers, String totals, boolean processed, String uuid, Map httpBroadcastDeliveryIds) { + this(voteId, name, service, time, proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, totals, + processed, false, uuid, httpBroadcastDeliveryIds); + } + + /** + * Creates a queued vote with all durable delivery fences. + * + * @param multiProxyForwardingHandled whether multi-proxy forwarding already completed + */ + public VoteTimeQueue(UUID voteId, String name, String service, long time, boolean proxyBroadcastHandled, + Set broadcastTargets, Set broadcastForwardedServers, String totals, boolean processed, + boolean multiProxyForwardingHandled, String uuid, Map httpBroadcastDeliveryIds) { this.voteId = voteId; this.uuid = uuid == null ? "" : uuid; this.name = name; @@ -145,6 +183,7 @@ public VoteTimeQueue(UUID voteId, String name, String service, long time, boolea this.proxyBroadcastHandled = proxyBroadcastHandled; this.totals = totals == null ? "" : totals; this.processed = processed; + this.multiProxyForwardingHandled = multiProxyForwardingHandled; this.broadcastTargets = new LinkedHashSet<>(); if (broadcastTargets != null) { this.broadcastTargets.addAll(broadcastTargets); @@ -153,12 +192,72 @@ public VoteTimeQueue(UUID voteId, String name, String service, long time, boolea if (broadcastForwardedServers != null) { this.broadcastForwardedServers.addAll(broadcastForwardedServers); } + this.multiProxyRecipients = new LinkedHashSet<>(); + this.multiProxyAcknowledgedServers = new LinkedHashSet<>(); this.httpBroadcastDeliveryIds = new LinkedHashMap<>(); if (httpBroadcastDeliveryIds != null) { httpBroadcastDeliveryIds.forEach(this::setHttpBroadcastDeliveryId); } } + /** Configures the durable acknowledgement fence before the first send. */ + public void requireMultiProxyAcknowledgements(String origin, Set recipients) { + multiProxyForwardingRequired = true; + multiProxyOrigin = origin == null ? "" : origin; + multiProxyRecipients.clear(); + if (recipients != null) { + for (String recipient : recipients) { + if (recipient != null && !recipient.isBlank()) { + multiProxyRecipients.add(recipient.toLowerCase(Locale.ROOT)); + } + } + } + } + + /** Adds an acknowledgement only for a configured recipient. */ + public boolean acknowledgeMultiProxyRecipient(String recipient) { + if (recipient == null || recipient.isBlank()) return false; + String normalized = recipient.toLowerCase(Locale.ROOT); + if (!multiProxyRecipients.contains(normalized)) return false; + return multiProxyAcknowledgedServers.add(normalized); + } + + /** Returns whether every intended receiver durably acknowledged the vote. */ + public boolean hasCompletedMultiProxyAcknowledgements() { + return multiProxyForwardingRequired && !multiProxyRecipients.isEmpty() + && multiProxyAcknowledgedServers.containsAll(multiProxyRecipients); + } + + public String encodeMultiProxyRecipients() { + return encodeBroadcastServers(multiProxyRecipients); + } + + public String encodeMultiProxyAcknowledgedServers() { + return encodeBroadcastServers(multiProxyAcknowledgedServers); + } + + public void setMultiProxyRecipients(Set recipients) { + multiProxyRecipients.clear(); + if (recipients != null) { + for (String recipient : recipients) { + if (recipient != null && !recipient.isBlank()) { + multiProxyRecipients.add(recipient.toLowerCase(Locale.ROOT)); + } + } + } + } + + public void setMultiProxyAcknowledgedServers(Set recipients) { + multiProxyAcknowledgedServers.clear(); + if (recipients != null) { + for (String recipient : recipients) { + if (recipient != null && !recipient.isBlank()) { + multiProxyAcknowledgedServers.add(recipient.toLowerCase(Locale.ROOT)); + } + } + } + } + public String getHttpBroadcastDeliveryId(String server) { return server == null ? null : httpBroadcastDeliveryIds.get(server.toLowerCase(Locale.ROOT)); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java index d1fbadaf9..9febf5c1e 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java @@ -3,16 +3,65 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import java.lang.reflect.Method; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.UUID; import org.junit.jupiter.api.Test; import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.votingplugin.proxy.VoteTotalsSnapshot; +import com.bencodez.votingplugin.proxy.VotingPluginWire; class MultiProxyHandlerLifecycleTest { + @Test + void doesNotAcceptAnEnvelopeWhenNoSocketDestinationExists() { + MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); + org.mockito.Mockito.when(handler.getMultiProxyMethod()).thenReturn(MultiProxyMethod.SOCKETS); + + assertFalse(handler.sendMultiProxyEnvelopeAccepted(JsonEnvelope.builder("vote").build())); + } + + @Test + void rejectsSocketSendWhenAnyConfiguredDestinationFails() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); + org.mockito.Mockito.when(handler.getMultiProxyMethod()).thenReturn(MultiProxyMethod.SOCKETS); + org.mockito.Mockito.when(handler.getMultiProxyServers()).thenReturn(List.of("backend")); + ClientHandler failing = mock(ClientHandler.class); + doThrow(new IllegalStateException("socket unavailable")).when(failing).sendEnvelope(org.mockito.ArgumentMatchers.any()); + java.lang.reflect.Field clients = MultiProxyHandler.class.getDeclaredField("multiproxyClientHandles"); + clients.setAccessible(true); + Map configured = new LinkedHashMap<>(); + configured.put("backend", failing); + clients.set(handler, configured); + + assertFalse(handler.sendMultiProxyEnvelopeAccepted(JsonEnvelope.builder("vote").build())); + } + + @Test + void acceptsAQueuedSocketSendWhenTheConfiguredDestinationReturnsNormally() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); + org.mockito.Mockito.when(handler.getMultiProxyMethod()).thenReturn(MultiProxyMethod.SOCKETS); + org.mockito.Mockito.when(handler.getMultiProxyServers()).thenReturn(List.of("backend")); + ClientHandler healthy = mock(ClientHandler.class); + java.lang.reflect.Field clients = MultiProxyHandler.class.getDeclaredField("multiproxyClientHandles"); + clients.setAccessible(true); + Map configured = new LinkedHashMap<>(); + configured.put("backend", healthy); + clients.set(handler, configured); + + assertTrue(handler.sendMultiProxyEnvelopeAccepted(JsonEnvelope.builder("vote").build())); + verify(healthy).sendEnvelope(org.mockito.ArgumentMatchers.any()); + } + @Test void stopsEveryReplacedSocketClientEvenWhenOneStopFails() { ClientHandler failing = mock(ClientHandler.class); @@ -27,4 +76,52 @@ void stopsEveryReplacedSocketClientEvenWhenOneStopFails() { verify(failing).stopConnection(); verify(healthy).stopConnection(); } + + @Test + void forwardsStableWireVoteIdToDurableTrigger() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); + UUID voteId = UUID.randomUUID(); + Method handleEnvelope = MultiProxyHandler.class.getDeclaredMethod("handleEnvelope", + com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.class); + handleEnvelope.setAccessible(true); + + handleEnvelope.invoke(handler, VotingPluginWire.vote("Player", + "00000000-0000-0000-0000-000000000001", "Service", 100L, false, true, "totals", voteId, + false, false, 1, 1)); + + verify(handler).triggerVote(org.mockito.ArgumentMatchers.eq("Player"), + org.mockito.ArgumentMatchers.eq("Service"), org.mockito.ArgumentMatchers.eq(true), + org.mockito.ArgumentMatchers.eq(true), org.mockito.ArgumentMatchers.eq(0L), + org.mockito.ArgumentMatchers.any(VoteTotalsSnapshot.class), + org.mockito.ArgumentMatchers.eq("00000000-0000-0000-0000-000000000001"), + org.mockito.ArgumentMatchers.eq(voteId)); + } + + @Test + void routesOnlyTheOriginatingProxyAcknowledgement() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); + org.mockito.Mockito.when(handler.getMultiProxyServerName()).thenReturn("Primary"); + UUID voteId = UUID.randomUUID(); + Method handleEnvelope = MultiProxyHandler.class.getDeclaredMethod("handleEnvelope", JsonEnvelope.class); + handleEnvelope.setAccessible(true); + + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyVoteAck(voteId, "Primary", "Replica")); + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyVoteAck(voteId, "Other", "Replica")); + + verify(handler).onMultiProxyVoteAcknowledged(voteId, "Replica"); + } + + @Test + void fencesOnlyPeersThatAdvertiseDurableAcknowledgements() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, + org.mockito.Mockito.withSettings().useConstructor().defaultAnswer(org.mockito.Mockito.CALLS_REAL_METHODS)); + org.mockito.Mockito.when(handler.getMultiProxyMethod()).thenReturn(MultiProxyMethod.SOCKETS); + org.mockito.Mockito.when(handler.getMultiProxyServers()).thenReturn(List.of("Legacy", "Capable")); + Method handleEnvelope = MultiProxyHandler.class.getDeclaredMethod("handleEnvelope", JsonEnvelope.class); + handleEnvelope.setAccessible(true); + + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyCapabilities("Capable", 1)); + + assertEquals(java.util.Set.of("capable"), handler.getMultiProxyVoteRecipients()); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 14f36d535..99f25cd46 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -103,6 +103,36 @@ public void concurrentOnlineDeliveriesReserveVoteIdAtomically() throws Exception org.mockito.ArgumentMatchers.anyInt(), org.mockito.ArgumentMatchers.any(OfflineBungeeVote.class)); } + @Test + public void multiProxyCompletionSurvivesHandlerRestart() { + UUID voteId = UUID.randomUUID(); + + assertTrue(handler.markMultiProxyVoteCompletedDurably(voteId)); + assertTrue(handler.hasMultiProxyVoteCompletion(voteId)); + + VoteCacheHandler restarted = newHandler(storage); + assertTrue(restarted.hasMultiProxyVoteCompletion(voteId)); + } + + @Test + public void failedCompletionPublicationDoesNotEvictAnExistingFence() throws Exception { + Path completionDirectory = tempDir.resolve("vote-cache.json.completed-multiproxy-votes"); + Files.createDirectories(completionDirectory); + for (int index = 0; index < 4096; index++) { + Files.writeString(completionDirectory.resolve(UUID.randomUUID().toString()), "existing-" + index); + } + UUID blockedVoteId = UUID.randomUUID(); + // A directory at the target path makes the atomic publication fail after + // staging, exercising the failure window without changing file permissions. + Files.createDirectory(completionDirectory.resolve(blockedVoteId.toString())); + + assertFalse(handler.markMultiProxyVoteCompletedDurably(blockedVoteId)); + try (java.util.stream.Stream files = Files.list(completionDirectory)) { + assertEquals(4096L, files.filter(Files::isRegularFile).count()); + } + assertTrue(Files.isDirectory(completionDirectory.resolve(blockedVoteId.toString()))); + } + @Test public void distinctVoteIdsAreNotCollapsed() { handler.addServerVote("server", vote(UUID.randomUUID(), 100L)); @@ -637,6 +667,7 @@ public void timedVoteHttpBroadcastDeliveryIdLoadsFromJsonCache() { stubString(timedNode, "VoteId", voteId.toString()); stubString(timedNode, "UUID", "uuid"); stubBoolean(timedNode, "ProxyBroadcastHandled", true); + stubBoolean(timedNode, "MultiProxyForwardingHandled", true); stubString(timedNode, "BroadcastTargets", VoteTimeQueue.encodeBroadcastServers(Set.of("Server1"))); stubString(timedNode, "BroadcastForwardedServers", ""); stubString(timedNode, "HttpBroadcastDeliveryIds", new VoteTimeQueue(voteId, "Player", "Service", 100L, @@ -647,6 +678,16 @@ public void timedVoteHttpBroadcastDeliveryIdLoadsFromJsonCache() { handler.load(); assertEquals(deliveryId, handler.getTimeChangeQueue().element().getHttpBroadcastDeliveryId("SERVER1")); + assertTrue(handler.getTimeChangeQueue().element().isMultiProxyForwardingHandled()); + } + + @Test + public void legacyTimedVoteDefaultsMultiProxyForwardingFenceToFalse() { + VoteTimeQueue vote = new VoteTimeQueue(UUID.randomUUID(), "Player", "Service", 100L); + + assertFalse(vote.isMultiProxyForwardingHandled()); + vote.setMultiProxyForwardingHandled(true); + assertTrue(vote.isMultiProxyForwardingHandled()); } @Test diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index b57bc3af5..4ec6a4b79 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -54,6 +54,9 @@ void setUp() { votingPluginProxy.setGlobalDataHandler(globalDataHandler); votingPluginProxy.setMultiProxyHandler(multiProxyHandler); votingPluginProxy.setDataFolder(temporaryDirectory.toFile()); + Mockito.when(multiProxyHandler.sendMultiProxyEnvelopeAccepted(Mockito.any())).thenReturn(true); + Mockito.when(multiProxyHandler.sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any())).thenReturn(true); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Set.of("Replica")); } @@ -419,6 +422,357 @@ void finalShutdownFencesQueuedVoteBeforeClearingLiveRetry() throws Exception { verify(spyProxy).addVoteParty(); } + @Test + void finalShutdownRetainsUnacknowledgedMultiProxyForwardingForRetry() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + VoteTimeQueue queued = new VoteTimeQueue(java.util.UUID.randomUUID(), "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", false, + "00000000-0000-0000-0000-000000000001"); + queue.add(queued); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.retryPendingVotePersistence()).thenReturn(true); + // The forwarding fence must be durable before later completion persistence + // fails, so shutdown can retry the completion without publishing again. + Mockito.when(voteCache.updateTimeVote(queued)).thenReturn(true, false); + Mockito.when(voteCache.removeTimeVote(queued)).thenReturn(false); + Mockito.when(voteCache.markTimeVoteCompletedDurably(queued)).thenReturn(false); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getPrimaryServer()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getBungeeManageTotals()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getMultiProxySupport()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getMultiProxyOneGlobalReward()).thenReturn(false); + Mockito.when(votingPluginProxy.getConfig().getProxyServerName()).thenReturn("Proxy1"); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Set.of("Proxy2")); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + Mockito.doReturn(true).when(spyProxy).checkVoteDelay(Mockito.anyString(), Mockito.anyString(), + Mockito.anyString(), Mockito.any(), Mockito.anyBoolean()); + Mockito.when(proxyMySQL.getExactQuery(Mockito.any())) + .thenReturn(new java.util.ArrayList<>()); + java.lang.reflect.Field decider = VotingPluginProxy.class.getDeclaredField("proxyBroadcastDecider"); + decider.setAccessible(true); + decider.set(spyProxy, new ProxyBroadcastDecider(spyProxy::getConfig, spyProxy::getAllAvailableServers, + server -> true, server -> false)); + + spyProxy.processQueue(); + assertTrue(queued.isProcessed()); + // A fire-and-forget transport invocation is not a receiver completion ACK. + assertFalse(queued.isMultiProxyForwardingHandled()); + spyProxy.onDisable(false); + + verify(multiProxyHandler, Mockito.atLeastOnce()).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any()); + } + + @Test + void reconstructedQueuedVoteRetainsUnacknowledgedMultiProxyDeliveryState() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + VoteTimeQueue queued = new VoteTimeQueue(java.util.UUID.randomUUID(), "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", false, + "00000000-0000-0000-0000-000000000001"); + queue.add(queued); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateTimeVote(queued)).thenReturn(true, false); + Mockito.when(voteCache.removeTimeVote(queued)).thenReturn(false); + Mockito.when(voteCache.markTimeVoteCompletedDurably(queued)).thenReturn(false); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getPrimaryServer()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getBungeeManageTotals()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getMultiProxySupport()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getMultiProxyOneGlobalReward()).thenReturn(false); + Mockito.when(votingPluginProxy.getConfig().getProxyServerName()).thenReturn("Proxy1"); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Set.of("Proxy2")); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + Mockito.doReturn(true).when(spyProxy).checkVoteDelay(Mockito.anyString(), Mockito.anyString(), + Mockito.anyString(), Mockito.any(), Mockito.anyBoolean()); + Mockito.when(proxyMySQL.getExactQuery(Mockito.any())).thenReturn(new java.util.ArrayList<>()); + java.lang.reflect.Field decider = VotingPluginProxy.class.getDeclaredField("proxyBroadcastDecider"); + decider.setAccessible(true); + decider.set(spyProxy, new ProxyBroadcastDecider(spyProxy::getConfig, spyProxy::getAllAvailableServers, + server -> true, server -> false)); + + spyProxy.processQueue(); + assertFalse(queued.isMultiProxyForwardingHandled()); + VoteTimeQueue restored = new VoteTimeQueue(queued.getVoteId(), queued.getName(), queued.getService(), + queued.getTime(), queued.isProxyBroadcastHandled(), queued.getBroadcastTargets(), + queued.getBroadcastForwardedServers(), queued.getTotals(), queued.isProcessed(), + queued.isMultiProxyForwardingHandled(), queued.getUuid(), queued.getHttpBroadcastDeliveryIds()); + restored.setMultiProxyForwardingRequired(queued.isMultiProxyForwardingRequired()); + restored.setRealVote(queued.isRealVote()); + restored.setMultiProxyOrigin(queued.getMultiProxyOrigin()); + restored.setMultiProxyRecipients(queued.getMultiProxyRecipients()); + restored.setMultiProxyAcknowledgedServers(queued.getMultiProxyAcknowledgedServers()); + queue.clear(); + queue.add(restored); + java.lang.reflect.Field retries = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); + retries.setAccessible(true); + ((java.util.Map) retries.get(spyProxy)).clear(); + + spyProxy.processQueue(); + + verify(multiProxyHandler, Mockito.times(2)).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any()); + } + + @Test + void forwardedMultiProxyVoteRetriesWithStableWireIdWithoutRepeatingPartySideEffect() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(false, true); + Mockito.when(voteCache.retainServerVoteForPersistenceRetry(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.markMultiProxyVoteCompletedDurably(Mockito.any())).thenReturn(true); + java.util.concurrent.ScheduledExecutorService scheduler = Mockito + .mock(java.util.concurrent.ScheduledExecutorService.class); + Mockito.when(scheduler.isShutdown()).thenReturn(false); + votingPluginProxy.setSchedulerForTest(scheduler); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + java.lang.reflect.Method receive = VotingPluginProxy.class.getDeclaredMethod("receiveMultiProxyVote", + String.class, String.class, boolean.class, boolean.class, long.class, + com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class, String.class, java.util.UUID.class); + receive.setAccessible(true); + + receive.invoke(spyProxy, "Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId); + org.mockito.ArgumentCaptor scheduled = org.mockito.ArgumentCaptor.forClass(Runnable.class); + verify(scheduler, Mockito.atLeastOnce()).schedule(scheduled.capture(), Mockito.anyLong(), + Mockito.any(java.util.concurrent.TimeUnit.class)); + Runnable voteRetry = scheduled.getAllValues().stream() + .filter(task -> task.getClass().getName().contains("MultiProxyVoteRetry")) + .findFirst().orElseThrow(); + voteRetry.run(); + // A duplicate socket/Redis delivery of the completed envelope is ignored. + receive.invoke(spyProxy, "Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId); + + verify(spyProxy).addVoteParty(); + java.lang.reflect.Field retries = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); + retries.setAccessible(true); + assertTrue(((java.util.Map) retries.get(spyProxy)).isEmpty()); + } + + @Test + void forwardedMultiProxyVoteRetriesCompletionRecordWithoutRepeatingVoteSideEffects() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.markMultiProxyVoteCompletedDurably(Mockito.any())).thenReturn(false, true); + java.util.concurrent.ScheduledExecutorService scheduler = Mockito + .mock(java.util.concurrent.ScheduledExecutorService.class); + Mockito.when(scheduler.isShutdown()).thenReturn(false); + votingPluginProxy.setSchedulerForTest(scheduler); + votingPluginProxy.setAvailableServers(); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + java.lang.reflect.Method receive = VotingPluginProxy.class.getDeclaredMethod("receiveMultiProxyVote", + String.class, String.class, boolean.class, boolean.class, long.class, + com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class, String.class, java.util.UUID.class); + receive.setAccessible(true); + + receive.invoke(spyProxy, "Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId); + org.mockito.ArgumentCaptor scheduled = org.mockito.ArgumentCaptor.forClass(Runnable.class); + verify(scheduler).schedule(scheduled.capture(), Mockito.anyLong(), + Mockito.any(java.util.concurrent.TimeUnit.class)); + scheduled.getValue().run(); + + verify(spyProxy).addVoteParty(); + verify(voteCache, Mockito.times(2)).markMultiProxyVoteCompletedDurably(voteId); + java.lang.reflect.Field retries = VotingPluginProxy.class.getDeclaredField("multiProxyVoteRetries"); + retries.setAccessible(true); + assertTrue(((java.util.Map) retries.get(spyProxy)).isEmpty()); + } + + @Test + void restartedReceiverSkipsDurablyCompletedMultiProxyVote() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.UUID voteId = java.util.UUID.randomUUID(); + Mockito.when(voteCache.hasMultiProxyVoteCompletion(voteId)).thenReturn(true); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + java.lang.reflect.Method receive = VotingPluginProxy.class.getDeclaredMethod("receiveMultiProxyVote", + String.class, String.class, boolean.class, boolean.class, long.class, + com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class, String.class, java.util.UUID.class); + receive.setAccessible(true); + + receive.invoke(spyProxy, "Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId); + + verify(spyProxy, never()).addVoteParty(); + } + + @Test + void overflowForwardedVoteIsSpilledToDurableReplayQueueInsteadOfDropped() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.addTimeVoteToCache(Mockito.any())).thenAnswer(invocation -> { + queue.add(invocation.getArgument(0)); + return true; + }); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + java.lang.reflect.Field retries = VotingPluginProxy.class.getDeclaredField("multiProxyVoteRetries"); + retries.setAccessible(true); + @SuppressWarnings("unchecked") + java.util.Map retryMap = (java.util.Map) retries.get(spyProxy); + for (int i = 0; i < 1024; i++) retryMap.put(java.util.UUID.randomUUID(), null); + java.util.UUID voteId = java.util.UUID.randomUUID(); + java.lang.reflect.Method receive = VotingPluginProxy.class.getDeclaredMethod("receiveMultiProxyVote", + String.class, String.class, boolean.class, boolean.class, long.class, + com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class, String.class, java.util.UUID.class); + receive.setAccessible(true); + + receive.invoke(spyProxy, "Player", "Service", false, true, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId); + + assertEquals(1, queue.size()); + assertEquals(voteId, queue.element().getVoteId()); + assertEquals("", queue.element().getTotals()); + assertFalse(queue.element().isRealVote()); + assertEquals(1024, retryMap.size()); + } + + @Test + void finalShutdownSettlesConsumedForwardedVoteBeforeDroppingRetryState() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(false, true); + Mockito.when(voteCache.retainServerVoteForPersistenceRetry(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.retryPendingVotePersistence()).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.markMultiProxyVoteCompletedDurably(Mockito.any())).thenReturn(true); + votingPluginProxy.setSchedulerForTest(null); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + java.lang.reflect.Method receive = VotingPluginProxy.class.getDeclaredMethod("receiveMultiProxyVote", + String.class, String.class, boolean.class, boolean.class, long.class, + com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class, String.class, java.util.UUID.class); + receive.setAccessible(true); + + receive.invoke(spyProxy, "Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId); + spyProxy.onDisable(false); + + verify(spyProxy).addVoteParty(); + java.lang.reflect.Field retries = VotingPluginProxy.class.getDeclaredField("multiProxyVoteRetries"); + retries.setAccessible(true); + assertTrue(((java.util.Map) retries.get(spyProxy)).isEmpty()); + } + + @Test + void finalShutdownRetainsForwardedFenceWhenCompletionCannotBePersisted() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.addTimeVoteToCache(Mockito.any())).thenAnswer(invocation -> { + queue.add(invocation.getArgument(0)); + return true; + }); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.markMultiProxyVoteCompletedDurably(Mockito.any())).thenReturn(false); + votingPluginProxy.setSchedulerForTest(null); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + java.lang.reflect.Method receive = VotingPluginProxy.class.getDeclaredMethod("receiveMultiProxyVote", + String.class, String.class, boolean.class, boolean.class, long.class, + com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class, String.class, java.util.UUID.class); + receive.setAccessible(true); + + receive.invoke(spyProxy, "Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId); + spyProxy.onDisable(false); + + java.lang.reflect.Field retries = VotingPluginProxy.class.getDeclaredField("multiProxyVoteRetries"); + retries.setAccessible(true); + assertTrue(((java.util.Map) retries.get(spyProxy)).isEmpty()); + assertEquals(1, queue.size()); + assertTrue(queue.element().isProcessed()); + assertTrue(queue.element().isMultiProxyCompletionPending()); + assertEquals(voteId, queue.element().getVoteId()); + java.lang.reflect.Field enabled = VotingPluginProxy.class.getDeclaredField("enabled"); + enabled.setAccessible(true); + assertFalse(enabled.getBoolean(spyProxy)); + verify(multiProxyHandler).close(); + } + + @Test + void persistedForwardedCompletionWaitsForTombstoneWithoutRepeatingVoteEffects() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + VoteTimeQueue completion = new VoteTimeQueue(voteId, "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", true, + "00000000-0000-0000-0000-000000000001"); + completion.setMultiProxyOrigin("Primary"); + completion.setMultiProxyCompletionPending(true); + queue.add(completion); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.hasMultiProxyVoteCompletion(voteId)).thenReturn(false); + Mockito.when(voteCache.markMultiProxyVoteCompletedDurably(voteId)).thenReturn(true); + Mockito.when(voteCache.removeTimeVote(completion)).thenAnswer(invocation -> queue.remove(completion)); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + spyProxy.processQueue(); + + assertTrue(queue.isEmpty()); + verify(spyProxy, never()).addVoteParty(); + verify(multiProxyHandler).acknowledgeMultiProxyVote(voteId, "Primary"); + } + + @Test + void runtimeReplacementRejectsForwardedRetryBeforeLiveVoteStateExists() throws Exception { + votingPluginProxy.setProxyMySQL(null); + Mockito.when(votingPluginProxy.getConfig().getPrimaryServer()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getBungeeManageTotals()).thenReturn(true); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + java.lang.reflect.Method receive = VotingPluginProxy.class.getDeclaredMethod("receiveMultiProxyVote", + String.class, String.class, boolean.class, boolean.class, long.class, + com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class, String.class, java.util.UUID.class); + receive.setAccessible(true); + + receive.invoke(spyProxy, "Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", java.util.UUID.randomUUID()); + + java.lang.reflect.Field liveRetries = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); + liveRetries.setAccessible(true); + assertTrue(((java.util.Map) liveRetries.get(spyProxy)).isEmpty()); + assertThrows(IllegalStateException.class, spyProxy::prepareForRuntimeReplacement); + } + @Test void preparedRuntimeDoesNotStartAReplacementWindowVote() throws Exception { VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); @@ -1221,7 +1575,7 @@ void invalidVoteStopsBeforeAnyPersistentRewardCacheOrForwardingState() { verify(spyProxy, never()).getVoteCacheHandler(); verify(spyProxy, never()).getGlobalMessageProxyHandler(); verify(proxyMySQL, never()).containsKeyQuery(Mockito.anyString()); - verify(multiProxyHandler, never()).sendMultiProxyEnvelope(Mockito.any()); + verify(multiProxyHandler, never()).sendMultiProxyEnvelopeAccepted(Mockito.any()); assertEquals(0, spyProxy.getVotePartyVotes(), username); assertTrue(spyProxy.getWarnings().stream().anyMatch(warning -> warning.contains("Rejected vote")), username); } From 8f75535b4dbc0cca54b7ffd2330959650c05384f Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 11 Sep 2026 21:39:34 -0600 Subject: [PATCH 072/136] Fix proxy retry durability and reload safety --- .../votingplugin/VotingPluginMain.java | 2 +- .../backendproxy/BackendProxyHandler.java | 12 +- .../presence/BackendPresenceManager.java | 36 +++++ .../votingplugin/proxy/VotingPluginProxy.java | 45 +++++- .../proxy/cache/VoteCacheHandler.java | 133 +++++++++++++++++- ...PluginMainBackendProxyPublicationTest.java | 2 +- .../BackendProxyHandlerLifecycleTest.java | 38 +++++ .../tests/VoteCacheHandlerVoteIdTest.java | 50 +++++++ .../tests/VotingPluginProxyTest.java | 102 +++++++++++++- 9 files changed, 406 insertions(+), 14 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 3c067b27d..d4c26cbde 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1310,7 +1310,7 @@ public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long restart.previousPrepared = true; if (restart.disabled) { restart.presenceStoppedForDisablePreparation = true; - restart.previous.preparePresenceForDisable(); + restart.previous.preparePresenceForDisable(validationDeadlineNanos); } BungeeMethod replacementMethod = restart.replacement == null ? null : restart.replacement.getMethod(); if (restart.replacement != null) restart.replacement.beginPreparedHttpHandoff(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 64d464e2c..16e4b78ca 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -226,12 +226,22 @@ public boolean commitPreparedDisable() { /** Publishes the final presence update while fencing other sends before transport preparation. */ public void preparePresenceForDisable() { + preparePresenceForDisableInternal(null); + } + + /** Publishes the final presence update before the caller's validation deadline. */ + public void preparePresenceForDisable(long deadlineNanos) { + preparePresenceForDisableInternal(deadlineNanos); + } + + private void preparePresenceForDisableInternal(Long deadlineNanos) { transportManager.beginPreparedDisable(); if (presenceManager != null && presenceReportingActivated) { // stopForDisable may reject when the transport cannot accept the final // presence update. Mark this inactive first so rollback can start it again. presenceReportingActivated = false; - presenceManager.stopForDisable(); + if (deadlineNanos == null) presenceManager.stopForDisable(); + else presenceManager.stopForDisable(deadlineNanos); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java index 6d479f3d8..01903dac4 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java @@ -3,9 +3,12 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.bukkit.Bukkit; import org.bukkit.entity.Player; @@ -97,6 +100,39 @@ public void stop() { /** Stops presence and propagates rejection when a configuration disable must be transactional. */ public void stopForDisable() { + stopForDisable(System.nanoTime() + TimeUnit.SECONDS.toNanos(5)); + } + + /** Stops presence before the caller's transactional validation deadline. */ + public void stopForDisable(long deadlineNanos) { + // Plugin-message transport ultimately calls Bukkit's sendPluginMessage API, + // which is primary-thread-only. Control validation runs on its worker, so + // marshal the transactional stopped-presence send before returning the + // result to that worker. Unit-test and shutdown contexts without a server + // retain the direct path used by the non-Control lifecycle. + if (plugin != null && plugin.getServer() != null && !plugin.getServer().isPrimaryThread()) { + CompletableFuture scheduled = new CompletableFuture<>(); + try { + plugin.getBukkitScheduler().executeOrScheduleSync(plugin, () -> { + try { + stop(true); + scheduled.complete(null); + } catch (Throwable failure) { + scheduled.completeExceptionally(failure); + } + }); + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) throw new TimeoutException("Backend stopped presence deadline expired"); + scheduled.get(remaining, TimeUnit.NANOSECONDS); + return; + } catch (Exception failure) { + scheduled.cancel(false); + Throwable cause = failure instanceof ExecutionException && failure.getCause() != null + ? failure.getCause() : failure; + if (cause instanceof RuntimeException runtime) throw runtime; + throw new IllegalStateException("Backend stopped presence could not run on the Bukkit thread", cause); + } + } stop(true); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index b701332fc..b9641c950 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Queue; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -947,7 +948,9 @@ public synchronized void checkOnlineVotes(String player, String uuid, String ser if (isIncompleteRewardJournalOwner(cache)) { if (!materializeRewardJournalOwner(uuid, cache)) continue; if (!cache.isProxyBroadcastHandled() || cache.isProxyBroadcastComplete()) { - getVoteCacheHandler().removeOnlineVote(uuid, cache); + if (!getVoteCacheHandler().tryRemoveOnlineVote(uuid, cache)) { + scheduleCachedVoteDeliveryRetry(); + } } continue; } @@ -1024,6 +1027,9 @@ protected synchronized void retryPendingOnlineBroadcasts(String server) { if (isIncompleteRewardJournalOwner(cache) && !materializeRewardJournalOwner(cachedUuid, cache)) { continue; } + if (retryCompletedRewardJournalOwner(cachedUuid, cache)) { + continue; + } if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { continue; } @@ -1084,6 +1090,9 @@ public synchronized void retryPendingOnlineBroadcasts() { if (isIncompleteRewardJournalOwner(cache) && !materializeRewardJournalOwner(cachedUuid, cache)) { continue; } + if (retryCompletedRewardJournalOwner(cachedUuid, cache)) { + continue; + } if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { continue; } @@ -1222,6 +1231,7 @@ private synchronized void retryCachedVoteDeliveryPersistence() { } for (String uuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(uuid))) { + if (retryCompletedRewardJournalOwner(uuid, vote)) continue; if (vote.isDeliveryStateDirty()) { persistOnlineVoteDelivery(uuid, vote); } @@ -3078,6 +3088,12 @@ protected synchronized boolean settleLiveVoteRetriesForFinalShutdown() { VoteCacheHandler cache = getVoteCacheHandler(); boolean retained = true; for (LiveVoteRetryState retry : liveVoteRetries.values()) { + if (retry.queuedVote != null) { + Queue queuedVotes = cache.getTimeChangeQueue(); + if (queuedVotes == null || !queuedVotes.contains(retry.queuedVote)) { + retained &= cache.addTimeVoteToCache(retry.queuedVote); + } + } Set onlineStates = Collections.newSetFromMap(new java.util.IdentityHashMap<>()); if (retry.rewardJournalOwner != null) onlineStates.add(retry.rewardJournalOwner); if (retry.standaloneBroadcastState != null) onlineStates.add(retry.standaloneBroadcastState); @@ -5133,7 +5149,9 @@ && getGlobalDataHandler().isTimeChangedHappened()) { if (!persistOnlineVoteDelivery(uuid, rewardJournalOwner)) return QueuedVoteResult.RETRY; retryState.rewardJournalsDurable = true; if (!rewardJournalOwner.isProxyBroadcastHandled() || rewardJournalOwner.isProxyBroadcastComplete()) { - getVoteCacheHandler().removeOnlineVote(uuid, rewardJournalOwner); + if (!getVoteCacheHandler().tryRemoveOnlineVote(uuid, rewardJournalOwner)) { + scheduleCachedVoteDeliveryRetry(); + } } } for (String s : rewardServers) { @@ -5408,8 +5426,8 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), recipients); outbox.setRealVote(realVote); outbox.setDeliveryStateDirty(true); - if (outbox.getVoteId() == null || !getVoteCacheHandler().addTimeVoteToCache(outbox)) return false; retryState.queuedVote = outbox; + if (outbox.getVoteId() == null || !getVoteCacheHandler().addTimeVoteToCache(outbox)) return false; newlyCreated = true; } if (!outbox.isMultiProxyForwardingRequired()) { @@ -5536,6 +5554,23 @@ private String rewardJournalDeliveryId(OfflineBungeeVote owner, String server) { return encoded; } + private boolean isRewardJournalOwner(OfflineBungeeVote vote) { + if (vote == null) return false; + for (String key : vote.getHttpDeliveryIds().keySet()) { + if (key.startsWith(REWARD_JOURNAL_TARGET_PREFIX)) return true; + } + return false; + } + + /** Retries cleanup for an owner whose target rows were already materialized. */ + private boolean retryCompletedRewardJournalOwner(String uuid, OfflineBungeeVote vote) { + if (uuid == null || vote == null || !vote.isRewardDelivered() || !isRewardJournalOwner(vote)) return false; + if (!getVoteCacheHandler().tryRemoveOnlineVote(uuid, vote)) { + scheduleCachedVoteDeliveryRetry(); + } + return true; + } + private boolean materializeRewardJournalOwner(String uuid, OfflineBungeeVote owner) { for (Map.Entry entry : owner.getHttpDeliveryIds().entrySet()) { if (!entry.getKey().startsWith(REWARD_JOURNAL_TARGET_PREFIX)) continue; @@ -5559,7 +5594,9 @@ private boolean materializeRewardJournalOwner(String uuid, OfflineBungeeVote own owner.setDeliveryStateDirty(true); if (!persistOnlineVoteDelivery(uuid, owner)) return false; if (!owner.isProxyBroadcastHandled() || owner.isProxyBroadcastComplete()) { - getVoteCacheHandler().removeOnlineVote(uuid, owner); + if (!getVoteCacheHandler().tryRemoveOnlineVote(uuid, owner)) { + scheduleCachedVoteDeliveryRetry(); + } } return true; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index d8b6b35f3..0c0746cf6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -744,11 +744,19 @@ public synchronized boolean addTimeVoteToCache(VoteTimeQueue vote) { if (vote == null) { return false; } + if (containsTimeVote(vote)) { + debug1("Not caching duplicate timed vote " + vote.getVoteId()); + return true; + } timeChangeQueue.add(vote); if (useMySQL) { boolean stored = timedVoteCacheTable.insertTimedVote(vote); if (!stored) { - timeChangeQueue.remove(vote); + // Keep the ACK outbox durable even while SQL is unavailable. The JSON + // store is the same emergency journal used by the other cache lanes and + // is loaded again by loadJsonEmergencyVotes() after a restart. + stored = persistTimeVoteToJson(vote); + if (!stored) timeChangeQueue.remove(vote); } return stored; } @@ -774,6 +782,31 @@ public synchronized boolean addTimeVoteToCache(VoteTimeQueue vote) { } } + private boolean containsTimeVote(VoteTimeQueue expected) { + for (VoteTimeQueue candidate : timeChangeQueue) { + if (candidate == null) continue; + if (expected.getVoteId() != null && expected.getVoteId().equals(candidate.getVoteId())) return true; + if (expected.getVoteId() == null && candidate.getVoteId() == null + && expected.getUuid().equals(candidate.getUuid()) + && expected.getService().equals(candidate.getService()) + && expected.getTime() == candidate.getTime()) return true; + } + return false; + } + + /** Persists a timed vote in the JSON emergency journal. */ + private boolean persistTimeVoteToJson(VoteTimeQueue vote) { + if (jsonStorage == null || jsonStorageQuarantined) return false; + try { + int index = nextCacheIndex(jsonStorage.getTimedVoteCache()); + jsonStorage.addTimedVote(index, vote); + return verifyJsonTimeVote(index, vote); + } catch (RuntimeException failure) { + debug1(failure); + return false; + } + } + /** * Persists changed delivery state for a queued rollover vote. * @@ -782,8 +815,16 @@ public synchronized boolean addTimeVoteToCache(VoteTimeQueue vote) { */ public synchronized boolean updateTimeVote(VoteTimeQueue vote) { if (useMySQL) { - return timedVoteCacheTable.updateTimedVote(vote); + if (timedVoteCacheTable.updateTimedVote(vote)) return true; + // A timed vote can have been admitted to the JSON emergency journal when + // its initial SQL insert failed. Keep delivery-state ACKs durable there + // until the SQL row is available again. + return updateTimeVoteJson(vote); } + return updateTimeVoteJson(vote); + } + + private boolean updateTimeVoteJson(VoteTimeQueue vote) { if (jsonStorage == null || jsonStorageQuarantined) return false; Collection keys = jsonStorage.getTimedVoteCache(); @@ -817,16 +858,47 @@ public synchronized boolean updateTimeVote(VoteTimeQueue vote) { */ public synchronized boolean removeTimeVote(VoteTimeQueue vote) { if (useMySQL) { - if (!timedVoteCacheTable.removeVote(vote)) { - return false; + if (timedVoteCacheTable.removeVote(vote)) { + timeChangeQueue.remove(vote); + return true; } - timeChangeQueue.remove(vote); - return true; + // The SQL row may never have existed when this vote was admitted to the + // JSON emergency journal. Remove that durable fallback before dropping the + // in-memory ACK outbox. + return removeEmergencyTimeVoteJson(vote); + } + return removeTimeVoteJson(vote); + } + + private boolean removeEmergencyTimeVoteJson(VoteTimeQueue vote) { + if (jsonStorage == null || jsonStorageQuarantined) return false; + Collection keys = jsonStorage.getTimedVoteCache(); + if (keys == null) return false; + ArrayList remaining = new ArrayList<>(); + boolean found = false; + for (String key : keys) { + DataNode data = jsonStorage.getTimedVoteCache(key); + if (data == null || !data.isObject()) return false; + if (matchesStoredTimeVote(data, vote)) { + found = true; + continue; + } + VoteTimeQueue queued = decodeJsonTimedVote(data); + if (queued == null) return false; + remaining.add(queued); } + if (!found) return false; + return rewriteJsonTimeVotesAndRemove(vote, remaining); + } + private boolean removeTimeVoteJson(VoteTimeQueue vote) { ArrayList remaining = new ArrayList<>(timeChangeQueue); remaining.remove(vote); if (jsonStorage == null || jsonStorageQuarantined) return false; + return rewriteJsonTimeVotesAndRemove(vote, remaining); + } + + private boolean rewriteJsonTimeVotesAndRemove(VoteTimeQueue vote, Collection remaining) { try { jsonStorage.removeTimedVotes(); int index = 0; @@ -1236,6 +1308,17 @@ private void loadJsonEmergencyVotes() { return; } try { + // Timed votes use the JSON store as an emergency outbox when the SQL + // insert failed. Merge them before the ordinary cache lanes so an ACK + // outbox is not lost across a proxy restart. + Collection timedKeys = jsonStorage.getTimedVoteCache(); + if (timedKeys != null) { + for (String key : timedKeys) { + DataNode data = jsonStorage.getTimedVoteCache(key); + VoteTimeQueue vote = decodeJsonTimedVote(data); + if (vote != null && !containsTimeVote(vote)) timeChangeQueue.add(vote); + } + } Collection servers = jsonStorage.getServers(); if (servers != null) { for (String server : servers) { @@ -1271,6 +1354,44 @@ private void loadJsonEmergencyVotes() { } } + private VoteTimeQueue decodeJsonTimedVote(DataNode data) { + if (data == null || !data.isObject()) return null; + String name = data.has("Name") ? data.get("Name").asString() : ""; + String service = data.has("Service") ? data.get("Service").asString() : ""; + long time = data.has("Time") ? data.get("Time").asLong() : 0L; + UUID voteId = readUuid(data, "VoteId"); + String uuid = data.has("UUID") ? data.get("UUID").asString() : ""; + boolean proxyBroadcastHandled = data.has("ProxyBroadcastHandled") + && data.get("ProxyBroadcastHandled").asBoolean(); + String forwardedServers = data.has("BroadcastForwardedServers") + ? data.get("BroadcastForwardedServers").asString() : ""; + String broadcastTargets = data.has("BroadcastTargets") ? data.get("BroadcastTargets").asString() : ""; + String totals = data.has("Totals") ? data.get("Totals").asString() : ""; + boolean processed = data.has("Processed") && data.get("Processed").asBoolean(); + boolean multiProxyForwardingHandled = data.has("MultiProxyForwardingHandled") + && data.get("MultiProxyForwardingHandled").asBoolean(); + String httpBroadcastDeliveryIds = data.has("HttpBroadcastDeliveryIds") + ? data.get("HttpBroadcastDeliveryIds").asString() : ""; + VoteTimeQueue queuedVote = new VoteTimeQueue(voteId, name, service, time, proxyBroadcastHandled, + VoteTimeQueue.decodeBroadcastForwardedServers(broadcastTargets), + VoteTimeQueue.decodeBroadcastForwardedServers(forwardedServers), totals, processed, + multiProxyForwardingHandled, uuid, + VoteTimeQueue.decodeHttpBroadcastDeliveryIds(httpBroadcastDeliveryIds)); + queuedVote.setMultiProxyForwardingRequired(data.has("MultiProxyForwardingRequired") + && data.get("MultiProxyForwardingRequired").asBoolean()); + queuedVote.setRealVote(!data.has("RealVote") || data.get("RealVote").asBoolean()); + queuedVote.setMultiProxyOrigin(data.has("MultiProxyOrigin") + ? data.get("MultiProxyOrigin").asString() : ""); + queuedVote.setMultiProxyCompletionPending(data.has("MultiProxyCompletionPending") + && data.get("MultiProxyCompletionPending").asBoolean()); + queuedVote.setMultiProxyRecipients(VoteTimeQueue.decodeBroadcastForwardedServers( + data.has("MultiProxyRecipients") ? data.get("MultiProxyRecipients").asString() : "")); + queuedVote.setMultiProxyAcknowledgedServers(VoteTimeQueue.decodeBroadcastForwardedServers( + data.has("MultiProxyAcknowledgedServers") + ? data.get("MultiProxyAcknowledgedServers").asString() : "")); + return queuedVote; + } + private OfflineBungeeVote decodeJsonVote(DataNode data) { if (data == null || !data.isObject()) { return null; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index 480982af0..137b0d792 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -348,7 +348,7 @@ void disablingBungeeCoordinationDrainsActiveHttpTransportBeforeClose() throws Ex plugin.completeBackendProxyHandlerRestart(restart); org.mockito.InOrder disable = org.mockito.Mockito.inOrder(previous); - disable.verify(previous).preparePresenceForDisable(); + disable.verify(previous).preparePresenceForDisable(org.mockito.ArgumentMatchers.anyLong()); disable.verify(previous).prepareForReplacement(org.mockito.ArgumentMatchers.isNull(), org.mockito.ArgumentMatchers.anyLong()); disable.verify(previous).commitPreparedDisable(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 237b53d57..5ca0f1ccf 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -255,6 +255,44 @@ void preparedDisablePropagatesRejectedStoppedPresence() throws Exception { assertFalse(presence.isReporting()); } + @Test + void preparedDisableSchedulesStoppedPresenceOnTheBukkitThread() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + org.bukkit.Server server = mock(org.bukkit.Server.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + when(plugin.getServer()).thenReturn(server); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + when(server.isPrimaryThread()).thenReturn(false); + java.util.concurrent.atomic.AtomicBoolean onScheduledTask = new java.util.concurrent.atomic.AtomicBoolean(); + org.mockito.Mockito.doAnswer(invocation -> { + onScheduledTask.set(true); + try { + Runnable task = invocation.getArgument(1); + task.run(); + } finally { + onScheduledTask.set(false); + } + return null; + }).when(scheduler).executeOrScheduleSync(eq(plugin), any(Runnable.class)); + org.mockito.Mockito.doAnswer(invocation -> { + assertTrue(onScheduledTask.get(), "stopped presence must be sent by the scheduled Bukkit task"); + return null; + }).when(messages).sendMessage(any()); + + BackendPresenceManager presence = new BackendPresenceManager(plugin, BungeeMethod.PLUGINMESSAGING, messages); + setField(presence, "reporting", true); + setField(presence, "server", "backend-1"); + setField(presence, "incarnationId", java.util.UUID.randomUUID()); + setField(presence, "startedAt", 1L); + + presence.stopForDisable(); + + verify(scheduler).executeOrScheduleSync(eq(plugin), any(Runnable.class)); + verify(messages).sendMessage(any()); + assertFalse(presence.isReporting()); + } + @Test void failedPreparedDisableCanRestartPresenceDuringRollback() throws Exception { BackendProxyHandler handler = new BackendProxyHandler(null); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 99f25cd46..e0ba97d80 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -31,9 +31,12 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import com.google.gson.JsonObject; + import com.bencodez.votingplugin.proxy.OfflineBungeeVote; import com.bencodez.votingplugin.proxy.cache.DataNode; import com.bencodez.votingplugin.proxy.cache.IVoteCache; +import com.bencodez.votingplugin.proxy.cache.GsonDataNode; import com.bencodez.votingplugin.proxy.cache.VoteCacheHandler; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; @@ -421,6 +424,30 @@ public void timedVoteDeliveryStateIsUpdatedByVoteId() { verify(storage).save(); } + @Test + public void timedVoteAcknowledgementOutboxSurvivesHandlerRestart() { + UUID voteId = UUID.randomUUID(); + VoteTimeQueue queued = new VoteTimeQueue(voteId, "Player", "Service", 100L, true, + Set.of("backend-1"), Set.of("backend-1"), "totals", false, false, "player-uuid", + Map.of("backend-1", "00000000-0000-0000-0000-000000000181")); + queued.requireMultiProxyAcknowledgements("origin-proxy", Set.of("backend-1")); + queued.acknowledgeMultiProxyRecipient("backend-1"); + handler.addTimeVoteToCache(queued); + + when(storage.getTimedVoteCache()).thenReturn(List.of("0")); + when(storage.getTimedVoteCache("0")).thenReturn(timedVoteNode(queued)); + VoteCacheHandler restarted = newHandler(storage); + restarted.load(); + + VoteTimeQueue recovered = restarted.getTimeChangeQueue().element(); + assertEquals(voteId, recovered.getVoteId()); + assertTrue(recovered.isMultiProxyForwardingRequired()); + assertEquals(Set.of("backend-1"), recovered.getMultiProxyRecipients()); + assertEquals(Set.of("backend-1"), recovered.getMultiProxyAcknowledgedServers()); + assertEquals("00000000-0000-0000-0000-000000000181", + recovered.getHttpBroadcastDeliveryId("backend-1")); + } + @Test public void timedVoteIsRemovedFromMemoryOnlyAfterJsonSaveSucceeds() throws Exception { VoteTimeQueue queued = new VoteTimeQueue(UUID.randomUUID(), "Player", "Service", 100L); @@ -824,6 +851,29 @@ private static OfflineBungeeVote vote(UUID voteId, long time) { return new OfflineBungeeVote(voteId, "Player", "player-uuid", "Service", time, true, "totals"); } + private static DataNode timedVoteNode(VoteTimeQueue vote) { + JsonObject data = new JsonObject(); + data.addProperty("Name", vote.getName()); + data.addProperty("Service", vote.getService()); + data.addProperty("Time", vote.getTime()); + data.addProperty("VoteId", vote.getVoteId().toString()); + data.addProperty("UUID", vote.getUuid()); + data.addProperty("ProxyBroadcastHandled", vote.isProxyBroadcastHandled()); + data.addProperty("Totals", vote.getTotals()); + data.addProperty("Processed", vote.isProcessed()); + data.addProperty("MultiProxyForwardingHandled", vote.isMultiProxyForwardingHandled()); + data.addProperty("MultiProxyForwardingRequired", vote.isMultiProxyForwardingRequired()); + data.addProperty("RealVote", vote.isRealVote()); + data.addProperty("MultiProxyOrigin", vote.getMultiProxyOrigin()); + data.addProperty("MultiProxyCompletionPending", vote.isMultiProxyCompletionPending()); + data.addProperty("MultiProxyRecipients", vote.encodeMultiProxyRecipients()); + data.addProperty("MultiProxyAcknowledgedServers", vote.encodeMultiProxyAcknowledgedServers()); + data.addProperty("BroadcastTargets", vote.encodeBroadcastTargets()); + data.addProperty("BroadcastForwardedServers", vote.encodeBroadcastForwardedServers()); + data.addProperty("HttpBroadcastDeliveryIds", vote.encodeHttpBroadcastDeliveryIds()); + return new GsonDataNode(data); + } + private static VoteCacheHandler newHandler(IVoteCache storage) { return newHandler(storage, true); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 4ec6a4b79..1de031d89 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -382,6 +382,81 @@ void finalShutdownMovesLiveVoteRetryIntoDurableOutbox() throws Exception { assertTrue(((java.util.Map) retries.get(spyProxy)).isEmpty()); } + @Test + void finalShutdownPersistsAckOutboxWhoseInitialInsertFailed() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.tryRemoveOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.retryPendingVotePersistence()).thenReturn(true); + Mockito.when(voteCache.addTimeVoteToCache(Mockito.any())).thenAnswer(new org.mockito.stubbing.Answer() { + private int attempts; + + @Override + public Boolean answer(org.mockito.invocation.InvocationOnMock invocation) { + VoteTimeQueue outbox = invocation.getArgument(0); + if (++attempts == 1) return false; + queue.add(outbox); + return true; + } + }); + Mockito.when(votingPluginProxy.getConfig().getMultiProxySupport()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getPrimaryServer()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getBungeeManageTotals()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getMultiProxyOneGlobalReward()).thenReturn(false); + Mockito.when(votingPluginProxy.getConfig().getProxyServerName()).thenReturn("Proxy1"); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Set.of("Proxy2")); + Mockito.when(multiProxyHandler.getConfiguredMultiProxyVoteRecipients()) + .thenReturn(new java.util.LinkedHashSet<>(java.util.Set.of("Proxy2"))); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Mockito.doNothing().when(spyProxy).addVoteParty(); + Mockito.doReturn(true).when(spyProxy).checkVoteDelay(Mockito.anyString(), Mockito.anyString(), + Mockito.anyString(), Mockito.any(), Mockito.anyBoolean()); + Mockito.when(proxyMySQL.getExactQuery(Mockito.any())).thenReturn(new java.util.ArrayList<>()); + spyProxy.setAvailableServers(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + + Class retryType = Class.forName("com.bencodez.votingplugin.proxy.VotingPluginProxy$LiveVoteRetryState"); + java.lang.reflect.Constructor constructor = retryType.getDeclaredConstructor(); + constructor.setAccessible(true); + Object retry = constructor.newInstance(); + for (java.util.Map.Entry field : java.util.Map.of( + "player", "Player", "service", "Service", "uuid", + "00000000-0000-0000-0000-000000000001", "time", 100L, "realVote", true, + "requestIdentity", "player\u0000service").entrySet()) { + java.lang.reflect.Field target = retryType.getDeclaredField(field.getKey()); + target.setAccessible(true); + target.set(retry, field.getValue()); + } + java.lang.reflect.Field retriesField = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); + retriesField.setAccessible(true); + @SuppressWarnings("unchecked") + java.util.Map retries = + (java.util.Map) retriesField.get(spyProxy); + retries.put(voteId, retry); + java.lang.reflect.Method begin = VotingPluginProxy.class.getDeclaredMethod("beginMultiProxyForwarding", + retryType, VoteTimeQueue.class, String.class, String.class, String.class, long.class, + boolean.class, com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class); + begin.setAccessible(true); + assertFalse((Boolean) begin.invoke(spyProxy, retry, null, "Player", + "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); + java.lang.reflect.Method settle = VotingPluginProxy.class + .getDeclaredMethod("settleLiveVoteRetriesForFinalShutdown"); + settle.setAccessible(true); + assertFalse((Boolean) settle.invoke(spyProxy)); + + org.mockito.ArgumentCaptor outboxes = org.mockito.ArgumentCaptor.forClass(VoteTimeQueue.class); + verify(voteCache, Mockito.atLeast(2)).addTimeVoteToCache(outboxes.capture()); + assertTrue(outboxes.getAllValues().stream().allMatch(outbox -> voteId.equals(outbox.getVoteId()))); + assertEquals(1, queue.size()); + assertEquals(voteId, queue.element().getVoteId()); + assertEquals(java.util.Set.of("proxy2"), queue.element().getMultiProxyRecipients()); + } + @Test void finalShutdownFencesQueuedVoteBeforeClearingLiveRetry() throws Exception { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); @@ -573,6 +648,7 @@ void forwardedMultiProxyVoteRetriesCompletionRecordWithoutRepeatingVoteSideEffec Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.tryRemoveOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); Mockito.when(voteCache.markMultiProxyVoteCompletedDurably(Mockito.any())).thenReturn(false, true); java.util.concurrent.ScheduledExecutorService scheduler = Mockito .mock(java.util.concurrent.ScheduledExecutorService.class); @@ -808,6 +884,7 @@ void durableRewardJournalOwnerRecreatesMissingServerOutbox() { .thenReturn(new java.util.ArrayList<>(java.util.List.of(owner))); Mockito.when(voteCache.addServerVoteDurably(Mockito.eq("Server1"), Mockito.any())).thenReturn(true); Mockito.when(voteCache.updateOnlineVote("player-uuid", owner)).thenReturn(true); + Mockito.when(voteCache.tryRemoveOnlineVote("player-uuid", owner)).thenReturn(true); VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); @@ -817,7 +894,7 @@ void durableRewardJournalOwnerRecreatesMissingServerOutbox() { verify(voteCache).addServerVoteDurably(Mockito.eq("Server1"), Mockito.argThat(vote -> "00000000-0000-0000-0000-000000000199".equals(vote.getHttpDeliveryId("Server1")))); verify(voteCache).updateOnlineVote("player-uuid", owner); - verify(voteCache).removeOnlineVote("player-uuid", owner); + verify(voteCache, Mockito.atLeastOnce()).tryRemoveOnlineVote("player-uuid", owner); } @Test @@ -2314,6 +2391,29 @@ void periodicRetryDeliversBrokerBackedOnlineCacheWithoutVoterLogin() { verify(voteCache).updateOnlineVote("voter-uuid", vote); } + @Test + void periodicRetryRemovesCompletedRewardJournalOwnerAfterTransientDeleteFailure() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote owner = new OfflineBungeeVote(java.util.UUID.randomUUID(), "OfflineVoter", "voter-uuid", + "Service", 100L, true, "totals", false, false, java.util.Collections.emptySet(), + java.util.Collections.emptySet(), true, + java.util.Map.of("__vp_reward_target__:Server1", + "00000000-0000-0000-0000-000000000172"), java.util.Collections.emptyMap()); + Mockito.when(voteCache.getOnlineVoteUUIDs()).thenReturn(java.util.Set.of("voter-uuid")); + Mockito.when(voteCache.getOnlineVotes("voter-uuid")) + .thenReturn(new java.util.ArrayList<>(java.util.List.of(owner))); + Mockito.when(voteCache.tryRemoveOnlineVote("voter-uuid", owner)).thenReturn(false, true); + votingPluginProxy.setSchedulerForTest(null); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.retryPendingOnlineBroadcasts(); + spyProxy.retryPendingOnlineBroadcasts(); + + verify(voteCache, Mockito.times(2)).tryRemoveOnlineVote("voter-uuid", owner); + verify(voteCache, never()).updateOnlineVote("voter-uuid", owner); + } + @Test void rejectedPeriodicHttpBroadcastKeepsItsDeterministicIdentity() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); From ae66bc795771ac40584cccdb21b40bfda63790e2 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 11 Sep 2026 21:55:12 -0600 Subject: [PATCH 073/136] Fix multi-proxy completion races --- .../votingplugin/proxy/VotingPluginProxy.java | 28 ++++- .../proxy/cache/VoteCacheHandler.java | 23 +++- .../proxy/multiproxy/MultiProxyHandler.java | 11 +- .../MultiProxyHandlerLifecycleTest.java | 16 +++ .../tests/VotingPluginProxyTest.java | 105 ++++++++++++++++++ 5 files changed, 177 insertions(+), 6 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index b9641c950..fbabf7fd9 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -3343,7 +3343,12 @@ public synchronized void processQueue() { scheduleTimeVoteRetry(); return; } - if (vote.isProcessed()) liveVoteRetries.remove(vote.getVoteId()); + // A direct listener retry can still be queued after its ACK outbox completes. + // Keep that in-memory fence until the listener consumes it; queued-vote + // processing removes its own fence in vote() when no listener retry exists. + if (vote.isProcessed() && !vote.isMultiProxyForwardingRequired()) { + liveVoteRetries.remove(vote.getVoteId()); + } if (!vote.isProcessed()) { String[] forwardedTotals = decodeForwardedQueueTotals(vote.getTotals()); boolean queuedRealVote = vote.isRealVote(); @@ -5362,7 +5367,27 @@ && getConfig().getPrimaryServer() } if (queuedVote != null) { queuedVote.setProcessed(true); + if (!queuedVote.getMultiProxyOrigin().isBlank()) { + // Persist the receiver's completion phase with the processed fence. A + // failed tombstone write must resume completion, never skip to deletion. + queuedVote.setMultiProxyCompletionPending(true); + } if (!persistTimeVoteDelivery(queuedVote)) { + if (!queuedVote.getMultiProxyOrigin().isBlank()) { + // The receiver may delete its queue row only after a durable completion + // fence exists. That tombstone also makes a sender retry idempotent. + if (getVoteCacheHandler().hasMultiProxyVoteCompletion(voteId) + || getVoteCacheHandler().markMultiProxyVoteCompletedDurably(voteId)) { + acknowledgeCompletedMultiProxyVote(voteId, queuedVote.getMultiProxyOrigin()); + if (getVoteCacheHandler().removeTimeVote(queuedVote)) { + liveVoteRetries.remove(voteId); + return QueuedVoteResult.SUCCESS; + } + } + warn("Unable to persist completed forwarded vote " + queuedVote.getVoteId() + + "; retaining its receiver completion fence for retry"); + return QueuedVoteResult.RETRY; + } // Deleting the completed source row is itself a durable completion // record. Prefer that fallback when updating the marker is unavailable. if (getVoteCacheHandler().removeTimeVote(queuedVote)) { @@ -5565,6 +5590,7 @@ private boolean isRewardJournalOwner(OfflineBungeeVote vote) { /** Retries cleanup for an owner whose target rows were already materialized. */ private boolean retryCompletedRewardJournalOwner(String uuid, OfflineBungeeVote vote) { if (uuid == null || vote == null || !vote.isRewardDelivered() || !isRewardJournalOwner(vote)) return false; + if (vote.isProxyBroadcastHandled() && !vote.isProxyBroadcastComplete()) return false; if (!getVoteCacheHandler().tryRemoveOnlineVote(uuid, vote)) { scheduleCachedVoteDeliveryRetry(); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index 0c0746cf6..022e76bf7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -815,7 +815,9 @@ private boolean persistTimeVoteToJson(VoteTimeQueue vote) { */ public synchronized boolean updateTimeVote(VoteTimeQueue vote) { if (useMySQL) { - if (timedVoteCacheTable.updateTimedVote(vote)) return true; + if (timedVoteCacheTable.updateTimedVote(vote)) { + return !hasJsonTimeVote(vote) || updateTimeVoteJson(vote); + } // A timed vote can have been admitted to the JSON emergency journal when // its initial SQL insert failed. Keep delivery-state ACKs durable there // until the SQL row is available again. @@ -859,6 +861,7 @@ private boolean updateTimeVoteJson(VoteTimeQueue vote) { public synchronized boolean removeTimeVote(VoteTimeQueue vote) { if (useMySQL) { if (timedVoteCacheTable.removeVote(vote)) { + if (hasJsonTimeVote(vote) && !removeEmergencyTimeVoteJson(vote)) return false; timeChangeQueue.remove(vote); return true; } @@ -870,6 +873,24 @@ public synchronized boolean removeTimeVote(VoteTimeQueue vote) { return removeTimeVoteJson(vote); } + private boolean hasJsonTimeVote(VoteTimeQueue vote) { + if (jsonStorage == null) return false; + try { + Collection keys = jsonStorage.getTimedVoteCache(); + if (keys == null) return false; + for (String key : keys) { + DataNode data = jsonStorage.getTimedVoteCache(key); + if (data != null && data.isObject() && matchesStoredTimeVote(data, vote)) return true; + } + return false; + } catch (RuntimeException failure) { + debug1(failure); + // An unreadable emergency journal may contain the same vote. Fail closed + // instead of removing the only in-memory reference and replaying it later. + return true; + } + } + private boolean removeEmergencyTimeVoteJson(VoteTimeQueue vote) { if (jsonStorage == null || jsonStorageQuarantined) return false; Collection keys = jsonStorage.getTimedVoteCache(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java index a69c7900c..d72148463 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java @@ -4,6 +4,7 @@ import java.io.File; import java.util.Collection; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; @@ -453,9 +454,11 @@ public synchronized boolean sendMultiProxyEnvelopeAccepted(JsonEnvelope envelope public synchronized boolean sendMultiProxyEnvelopeAccepted(JsonEnvelope envelope, Collection recipients) { if (envelope == null) return false; if (recipients == null || recipients.isEmpty()) return false; - Set requested = new LinkedHashSet<>(); + Map requested = new LinkedHashMap<>(); for (String recipient : recipients) { - if (recipient != null && !recipient.isBlank()) requested.add(recipient.toLowerCase(Locale.ROOT)); + if (recipient != null && !recipient.isBlank()) { + requested.putIfAbsent(recipient.toLowerCase(Locale.ROOT), recipient); + } } if (requested.isEmpty()) return false; if (getMultiProxyMethod().equals(MultiProxyMethod.SOCKETS)) { @@ -463,7 +466,7 @@ public synchronized boolean sendMultiProxyEnvelopeAccepted(JsonEnvelope envelope boolean accepted = true; int destinations = 0; for (Map.Entry entry : multiproxyClientHandles.entrySet()) { - if (entry.getKey() == null || !requested.contains(entry.getKey().toLowerCase(Locale.ROOT))) continue; + if (entry.getKey() == null || !requested.containsKey(entry.getKey().toLowerCase(Locale.ROOT))) continue; ClientHandler h = entry.getValue(); if (h == null) { accepted = false; @@ -481,7 +484,7 @@ public synchronized boolean sendMultiProxyEnvelopeAccepted(JsonEnvelope envelope if (multiProxyRedis == null) return false; boolean accepted = true; int destinations = 0; - for (String server : requested) { + for (String server : requested.values()) { destinations++; try { multiProxyRedis.publishEnvelope("VotingPluginProxy_" + server, envelope); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java index 9febf5c1e..b308ac8c4 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java @@ -62,6 +62,22 @@ void acceptsAQueuedSocketSendWhenTheConfiguredDestinationReturnsNormally() throw verify(healthy).sendEnvelope(org.mockito.ArgumentMatchers.any()); } + @Test + void redisSubsetSendPreservesConfiguredChannelCasing() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); + org.mockito.Mockito.when(handler.getMultiProxyMethod()).thenReturn(MultiProxyMethod.REDIS); + com.bencodez.simpleapi.servercomm.redis.RedisHandler redis = + mock(com.bencodez.simpleapi.servercomm.redis.RedisHandler.class); + java.lang.reflect.Field connection = MultiProxyHandler.class.getDeclaredField("multiProxyRedis"); + connection.setAccessible(true); + connection.set(handler, redis); + JsonEnvelope envelope = JsonEnvelope.builder("vote").build(); + + assertTrue(handler.sendMultiProxyEnvelopeAccepted(envelope, List.of("Proxy2"))); + + verify(redis).publishEnvelope("VotingPluginProxy_Proxy2", envelope); + } + @Test void stopsEveryReplacedSocketClientEvenWhenOneStopFails() { ClientHandler failing = mock(ClientHandler.class); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 1de031d89..49d64e613 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -457,6 +457,38 @@ public Boolean answer(org.mockito.invocation.InvocationOnMock invocation) { assertEquals(java.util.Set.of("proxy2"), queue.element().getMultiProxyRecipients()); } + @Test + void completedAckOutboxKeepsFenceForAlreadyScheduledListenerRetry() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.UUID voteId = java.util.UUID.randomUUID(); + VoteTimeQueue outbox = new VoteTimeQueue(voteId, "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", true, + "00000000-0000-0000-0000-000000000001"); + outbox.setMultiProxyForwardingRequired(true); + outbox.setMultiProxyForwardingHandled(true); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + queue.add(outbox); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.removeTimeVote(outbox)).thenAnswer(invocation -> queue.remove(outbox)); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Class retryType = Class.forName("com.bencodez.votingplugin.proxy.VotingPluginProxy$LiveVoteRetryState"); + java.lang.reflect.Constructor constructor = retryType.getDeclaredConstructor(); + constructor.setAccessible(true); + Object retry = constructor.newInstance(); + java.lang.reflect.Field retriesField = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); + retriesField.setAccessible(true); + @SuppressWarnings("unchecked") + java.util.Map retries = + (java.util.Map) retriesField.get(spyProxy); + retries.put(voteId, retry); + + spyProxy.processQueue(); + + assertTrue(queue.isEmpty()); + assertTrue(retries.containsKey(voteId)); + } + @Test void finalShutdownFencesQueuedVoteBeforeClearingLiveRetry() throws Exception { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); @@ -827,6 +859,57 @@ void persistedForwardedCompletionWaitsForTombstoneWithoutRepeatingVoteEffects() verify(multiProxyHandler).acknowledgeMultiProxyVote(voteId, "Primary"); } + @Test + void failedReceiverCompletionWriteResumesWithoutRepeatingVoteEffects() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + VoteTimeQueue received = new VoteTimeQueue(voteId, "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "", false, + "00000000-0000-0000-0000-000000000001"); + received.setMultiProxyOrigin("Primary"); + queue.add(received); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.updateTimeVote(received)).thenReturn(true); + Mockito.when(voteCache.addOnlineVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.tryRemoveOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); + Mockito.when(voteCache.hasMultiProxyVoteCompletion(voteId)).thenReturn(false); + Mockito.when(voteCache.markMultiProxyVoteCompletedDurably(voteId)).thenReturn(false, true); + Mockito.when(voteCache.removeTimeVote(received)).thenAnswer(invocation -> queue.remove(received)); + Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getPrimaryServer()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getBungeeManageTotals()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getOnlineMode()).thenReturn(true); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + java.lang.reflect.Field decider = VotingPluginProxy.class.getDeclaredField("proxyBroadcastDecider"); + decider.setAccessible(true); + decider.set(spyProxy, new ProxyBroadcastDecider(spyProxy::getConfig, spyProxy::getAllAvailableServers, + server -> true, server -> false)); + Mockito.doNothing().when(spyProxy).addVoteParty(); + Mockito.doReturn(true).when(spyProxy).checkVoteDelay(Mockito.anyString(), Mockito.anyString(), + Mockito.anyString(), Mockito.any(), Mockito.anyBoolean()); + Mockito.when(proxyMySQL.containsKeyQuery(Mockito.anyString())).thenReturn(true); + Mockito.when(proxyMySQL.getExactQuery(Mockito.any())).thenReturn(new java.util.ArrayList<>()); + spyProxy.setAvailableServers(); + + spyProxy.processQueue(); + + assertTrue(received.isProcessed()); + assertTrue(received.isMultiProxyCompletionPending()); + assertEquals(1, queue.size()); + verify(spyProxy).addVoteParty(); + + spyProxy.processQueue(); + + assertTrue(queue.isEmpty()); + verify(spyProxy).addVoteParty(); + verify(multiProxyHandler).acknowledgeMultiProxyVote(voteId, "Primary"); + } + @Test void runtimeReplacementRejectsForwardedRetryBeforeLiveVoteStateExists() throws Exception { votingPluginProxy.setProxyMySQL(null); @@ -2414,6 +2497,28 @@ void periodicRetryRemovesCompletedRewardJournalOwnerAfterTransientDeleteFailure( verify(voteCache, never()).updateOnlineVote("voter-uuid", owner); } + @Test + void completedRewardJournalOwnerRetainsPendingStandaloneBroadcast() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote owner = new OfflineBungeeVote(java.util.UUID.randomUUID(), "OfflineVoter", "voter-uuid", + "Service", 100L, true, "totals", false, true, java.util.Set.of("Server1"), + java.util.Collections.emptySet(), true, + java.util.Map.of("__vp_reward_target__:Server1", + "00000000-0000-0000-0000-000000000173"), java.util.Collections.emptyMap()); + Mockito.when(voteCache.getOnlineVoteUUIDs()).thenReturn(java.util.Set.of("voter-uuid")); + Mockito.when(voteCache.getOnlineVotes("voter-uuid")) + .thenReturn(new java.util.ArrayList<>(java.util.List.of(owner))); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + spyProxy.retryPendingOnlineBroadcasts(); + + verify(voteCache, never()).tryRemoveOnlineVote("voter-uuid", owner); + assertTrue(owner.needsBroadcastOn("Server1")); + } + @Test void rejectedPeriodicHttpBroadcastKeepsItsDeterministicIdentity() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); From ecd70f386d8c93430d638e7fd50cc7056e86e65c Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 11 Sep 2026 22:23:22 -0600 Subject: [PATCH 074/136] Preserve durable multi-proxy retry fences --- .../votingplugin/proxy/VotingPluginProxy.java | 17 ++++++++ .../proxy/cache/VoteCacheHandler.java | 42 ++----------------- .../tests/VoteCacheHandlerVoteIdTest.java | 20 +++++++++ .../tests/VotingPluginProxyTest.java | 13 +++++- 4 files changed, 51 insertions(+), 41 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index fbabf7fd9..453451b19 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -5454,6 +5454,23 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim retryState.queuedVote = outbox; if (outbox.getVoteId() == null || !getVoteCacheHandler().addTimeVoteToCache(outbox)) return false; newlyCreated = true; + } else { + boolean alreadyQueued = false; + for (VoteTimeQueue candidate : getVoteCacheHandler().getTimeChangeQueue()) { + if (candidate != null && outbox.getVoteId().equals(candidate.getVoteId())) { + alreadyQueued = true; + break; + } + } + if (!alreadyQueued && !getVoteCacheHandler().addTimeVoteToCache(outbox)) { + // A previous durable admission may have failed after the retry state kept + // this object in memory. Re-admit it before any publish; + // addTimeVoteToCache is idempotent for an already-durable queue entry. + return false; + } + // A recovered admission is the first safe chance to publish to legacy peers. + // Already-queued retries must not repeat that fire-and-forget copy. + newlyCreated = !alreadyQueued; } if (!outbox.isMultiProxyForwardingRequired()) { outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), recipients); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index 022e76bf7..65e1226c7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -8,7 +8,6 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Collection; -import java.util.Comparator; import java.util.LinkedHashSet; import java.util.Map; import java.util.Map.Entry; @@ -17,7 +16,6 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.stream.Stream; import com.bencodez.simpleapi.sql.mysql.MySQL; import com.bencodez.simpleapi.sql.mysql.config.MysqlConfig; @@ -32,7 +30,6 @@ */ public abstract class VoteCacheHandler { private static final int MAX_PENDING_PERSISTENCE_VOTES = 1024; - private static final int MAX_MULTI_PROXY_COMPLETIONS = 4096; /** * Queue of timed votes for time change processing. @@ -1021,12 +1018,9 @@ public synchronized boolean markMultiProxyVoteCompletedDurably(UUID voteId) { // The record is visible; the read-back below decides whether it is usable. } if (!hasMultiProxyVoteCompletion(voteId)) return false; - // Publish the new fence before pruning old fences. A pruning failure may - // leave the bounded journal temporarily one entry over capacity, but it - // must never remove an existing fence and then lose the new one. - if (!pruneMultiProxyVoteCompletions(target.getParent())) { - debug1("Unable to prune completed multi-proxy vote fences after publishing " + voteId); - } + // Sender outboxes have no expiry, so their matching receiver fences must + // remain for the same lifetime. Pruning by count can turn a late retry into + // a second reward after enough newer votes complete. return true; } catch (IOException | RuntimeException failure) { debug1(failure); @@ -1050,36 +1044,6 @@ public synchronized boolean hasMultiProxyVoteCompletion(UUID voteId) { } } - private boolean pruneMultiProxyVoteCompletions(Path directory) { - ArrayList records = new ArrayList<>(); - try (Stream files = Files.list(directory)) { - files.filter(Files::isRegularFile).forEach(records::add); - } catch (IOException | RuntimeException failure) { - debug1(failure); - return false; - } - records.sort(Comparator.comparingLong(this::multiProxyCompletionModifiedTime)); - while (records.size() > MAX_MULTI_PROXY_COMPLETIONS) { - Path oldest = records.remove(0); - try { - DurableFiles.deleteIfExists(oldest); - } catch (IOException | RuntimeException failure) { - debug1(failure); - return false; - } - } - return true; - } - - private long multiProxyCompletionModifiedTime(Path record) { - try { - return Files.getLastModifiedTime(record).toMillis(); - } catch (IOException | RuntimeException failure) { - debug1(failure); - return Long.MIN_VALUE; - } - } - private Path timeVoteCompletionPath(VoteTimeQueue vote) { if (vote == null || jsonStorage == null || jsonStorage.getStoragePath() == null) return null; Path storage = jsonStorage.getStoragePath().toAbsolutePath().normalize(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index e0ba97d80..7ecd21070 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -136,6 +136,26 @@ public void failedCompletionPublicationDoesNotEvictAnExistingFence() throws Exce assertTrue(Files.isDirectory(completionDirectory.resolve(blockedVoteId.toString()))); } + @Test + public void completionFencesRemainWhileSenderOutboxesCanRetryIndefinitely() throws Exception { + Path completionDirectory = tempDir.resolve("vote-cache.json.completed-multiproxy-votes"); + Files.createDirectories(completionDirectory); + UUID oldestVoteId = UUID.randomUUID(); + Files.writeString(completionDirectory.resolve(oldestVoteId.toString()), oldestVoteId.toString()); + for (int index = 1; index < 4096; index++) { + UUID voteId = UUID.randomUUID(); + Files.writeString(completionDirectory.resolve(voteId.toString()), voteId.toString()); + } + UUID newestVoteId = UUID.randomUUID(); + + assertTrue(handler.markMultiProxyVoteCompletedDurably(newestVoteId)); + assertTrue(handler.hasMultiProxyVoteCompletion(oldestVoteId)); + assertTrue(handler.hasMultiProxyVoteCompletion(newestVoteId)); + try (java.util.stream.Stream files = Files.list(completionDirectory)) { + assertEquals(4097L, files.filter(Files::isRegularFile).count()); + } + } + @Test public void distinctVoteIdsAreNotCollapsed() { handler.addServerVote("server", vote(UUID.randomUUID(), 100L)); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 49d64e613..1157096b7 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -383,7 +383,7 @@ void finalShutdownMovesLiveVoteRetryIntoDurableOutbox() throws Exception { } @Test - void finalShutdownPersistsAckOutboxWhoseInitialInsertFailed() throws Exception { + void runtimeRetryPersistsAckOutboxWhoseInitialInsertFailedBeforePublishing() throws Exception { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); @@ -410,7 +410,7 @@ public Boolean answer(org.mockito.invocation.InvocationOnMock invocation) { Mockito.when(votingPluginProxy.getConfig().getProxyServerName()).thenReturn("Proxy1"); Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Set.of("Proxy2")); Mockito.when(multiProxyHandler.getConfiguredMultiProxyVoteRecipients()) - .thenReturn(new java.util.LinkedHashSet<>(java.util.Set.of("Proxy2"))); + .thenReturn(new java.util.LinkedHashSet<>(java.util.Set.of("Proxy2", "ProxyLegacy"))); VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); Mockito.doNothing().when(spyProxy).addVoteParty(); @@ -444,6 +444,15 @@ public Boolean answer(org.mockito.invocation.InvocationOnMock invocation) { begin.setAccessible(true); assertFalse((Boolean) begin.invoke(spyProxy, retry, null, "Player", "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); + verify(multiProxyHandler, never()).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any()); + assertFalse((Boolean) begin.invoke(spyProxy, retry, null, "Player", + "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); + @SuppressWarnings({ "rawtypes", "unchecked" }) + org.mockito.ArgumentCaptor> recipients = + (org.mockito.ArgumentCaptor) org.mockito.ArgumentCaptor.forClass(java.util.Collection.class); + verify(multiProxyHandler, Mockito.times(2)).sendMultiProxyEnvelopeAccepted(Mockito.any(), recipients.capture()); + assertEquals(java.util.Set.of(java.util.Set.of("proxy2"), java.util.Set.of("ProxyLegacy")), + new java.util.HashSet<>(recipients.getAllValues())); java.lang.reflect.Method settle = VotingPluginProxy.class .getDeclaredMethod("settleLiveVoteRetriesForFinalShutdown"); settle.setAccessible(true); From f5e6b06f77b8e14ca69de125e2c543c04405d8bf Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 11 Sep 2026 22:37:11 -0600 Subject: [PATCH 075/136] Migrate legacy timed votes before reliable forwarding --- .../votingplugin/proxy/VotingPluginProxy.java | 16 ++++++++-- .../proxy/cache/ProxyTimedVoteCacheTable.java | 32 +++++++++++++++++++ .../proxy/cache/VoteCacheHandler.java | 17 ++++++++++ .../proxy/multiproxy/MultiProxyHandler.java | 5 +++ .../MultiProxyHandlerLifecycleTest.java | 17 ++++++++++ .../tests/VoteCacheHandlerVoteIdTest.java | 23 +++++++++++++ 6 files changed, 108 insertions(+), 2 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 453451b19..baf513116 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -4806,7 +4806,13 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea UUID voteId = queuedVote == null ? null : queuedVote.getVoteId(); if (voteId == null) { - voteId = requestedVoteId == null ? UUID.randomUUID() : requestedVoteId; + voteId = requestedVoteId == null + ? queuedVote == null ? UUID.randomUUID() : legacyTimedVoteId(queuedVote) + : requestedVoteId; + if (queuedVote != null && !getVoteCacheHandler().assignLegacyTimeVoteId(queuedVote, voteId)) { + warn("Unable to assign a stable ID to a legacy timed vote; retaining it for retry"); + return QueuedVoteResult.RETRY; + } } LiveVoteRetryState retryState = liveVoteRetries.get(voteId); if (retryState != null && !requestIdentity.equals(retryState.requestIdentity)) { @@ -5413,6 +5419,12 @@ && getConfig().getPrimaryServer() } } + private UUID legacyTimedVoteId(VoteTimeQueue vote) { + String identity = "legacy-timed-vote\u0000" + vote.getUuid() + "\u0000" + vote.getName() + "\u0000" + + vote.getService() + "\u0000" + vote.getTime(); + return UUID.nameUUIDFromBytes(identity.getBytes(StandardCharsets.UTF_8)); + } + /** * Creates/persists the sender outbox before the first publish. Redis and socket * publish APIs are fire-and-forget, so their return value is never a delivery @@ -5457,7 +5469,7 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim } else { boolean alreadyQueued = false; for (VoteTimeQueue candidate : getVoteCacheHandler().getTimeChangeQueue()) { - if (candidate != null && outbox.getVoteId().equals(candidate.getVoteId())) { + if (candidate != null && java.util.Objects.equals(outbox.getVoteId(), candidate.getVoteId())) { alreadyQueued = true; break; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java index 81576b24e..f6b5f6224 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java @@ -345,6 +345,38 @@ public boolean updateTimedVote(VoteTimeQueue vote) { } } + /** Assigns a stable ID to a pre-ID timed row before reliable forwarding. */ + public boolean assignLegacyTimedVoteId(VoteTimeQueue vote, UUID voteId) { + if (vote == null || voteId == null) return false; + String sql = "UPDATE " + qi(getTableName()) + " SET " + qi("voteId") + " = ? WHERE " + + qi("playerName") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ? AND (" + + qi("voteId") + " IS NULL OR " + qi("voteId") + " = '' OR " + qi("voteId") + " = ?);"; + try (Connection conn = mysql.getConnectionManager().getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, voteId.toString()); + ps.setString(2, vote.getName()); + ps.setString(3, vote.getService()); + ps.setLong(4, vote.getTime()); + ps.setString(5, voteId.toString()); + if (ps.executeUpdate() > 0) return true; + String verifySql = "SELECT " + qi("id") + " FROM " + qi(getTableName()) + " WHERE " + + qi("playerName") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ? AND " + + qi("voteId") + " = ?;"; + try (PreparedStatement verify = conn.prepareStatement(verifySql)) { + verify.setString(1, vote.getName()); + verify.setString(2, vote.getService()); + verify.setLong(3, vote.getTime()); + verify.setString(4, voteId.toString()); + try (ResultSet rows = verify.executeQuery()) { + return rows.next(); + } + } + } catch (SQLException failure) { + debug(failure); + return false; + } + } + // --- GET --- /** * Gets all timed votes. diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index 65e1226c7..dc31d25f9 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -823,6 +823,23 @@ public synchronized boolean updateTimeVote(VoteTimeQueue vote) { return updateTimeVoteJson(vote); } + /** + * Durably upgrades a legacy timed row with the stable ID required by reliable + * multi-proxy delivery. The in-memory object is restored on failure so callers + * cannot mistake an unpersisted identifier for a durable one. + */ + public synchronized boolean assignLegacyTimeVoteId(VoteTimeQueue vote, UUID voteId) { + if (vote == null || voteId == null) return false; + if (vote.getVoteId() != null) return voteId.equals(vote.getVoteId()); + vote.setVoteId(voteId); + boolean jsonPresent = hasJsonTimeVote(vote); + boolean jsonStored = !jsonPresent || updateTimeVoteJson(vote); + boolean primaryStored = useMySQL ? timedVoteCacheTable.assignLegacyTimedVoteId(vote, voteId) : jsonStored; + if (primaryStored && jsonStored) return true; + vote.setVoteId(null); + return false; + } + private boolean updateTimeVoteJson(VoteTimeQueue vote) { if (jsonStorage == null || jsonStorageQuarantined) return false; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java index d72148463..5ffcdfc0e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java @@ -600,6 +600,11 @@ private void handleEnvelope(JsonEnvelope envelope) { final String totals = f.getOrDefault(VotingPluginWire.K_TOTALS, ""); final boolean realVote = Boolean.parseBoolean(f.getOrDefault(VotingPluginWire.K_REAL_VOTE, "false")); final String origin = f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_ORIGIN, ""); + if (!origin.isBlank() && wireVote.voteId == null) { + // Reliable envelopes must never enter receiver processing without the + // stable identity used by its durable completion fence. + return; + } if (!player.isEmpty() && !uuid.isEmpty() && !service.isEmpty()) { triggerVote(player, service, realVote, true, 0L, VoteTotalsSnapshot.parseStorage(totals), uuid, diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java index b308ac8c4..3df63577b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java @@ -113,6 +113,23 @@ void forwardsStableWireVoteIdToDurableTrigger() throws Exception { org.mockito.ArgumentMatchers.eq(voteId)); } + @Test + void rejectsReliableVoteEnvelopeWithoutStableId() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); + Method handleEnvelope = MultiProxyHandler.class.getDeclaredMethod("handleEnvelope", JsonEnvelope.class); + handleEnvelope.setAccessible(true); + + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyVote("Player", + "00000000-0000-0000-0000-000000000001", "Service", 100L, false, true, "totals", null, + false, false, 1, 1, "Primary")); + + verify(handler, org.mockito.Mockito.never()).triggerVote(org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.anyBoolean(), + org.mockito.ArgumentMatchers.anyBoolean(), org.mockito.ArgumentMatchers.anyLong(), + org.mockito.ArgumentMatchers.any(VoteTotalsSnapshot.class), org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.any(), org.mockito.ArgumentMatchers.anyString()); + } + @Test void routesOnlyTheOriginatingProxyAcknowledgement() throws Exception { MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 7ecd21070..227c80724 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -24,6 +24,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; import java.nio.file.Files; import java.nio.file.Path; @@ -427,6 +428,28 @@ public void timedVoteIsPersistedImmediatelyWithItsId() { assertEquals(voteId, handler.getTimeChangeQueue().element().getVoteId()); } + @Test + public void legacyTimedVoteIdIsDurableBeforeReliableForwarding() { + VoteTimeQueue queued = new VoteTimeQueue("Player", "Service", 100L); + JsonObject legacyJson = new JsonObject(); + legacyJson.addProperty("Name", "Player"); + legacyJson.addProperty("Service", "Service"); + legacyJson.addProperty("Time", 100L); + AtomicReference stored = new AtomicReference<>(new GsonDataNode(legacyJson)); + when(storage.getTimedVoteCache()).thenReturn(List.of("2")); + when(storage.getTimedVoteCache("2")).thenAnswer(ignored -> stored.get()); + org.mockito.Mockito.doAnswer(invocation -> { + stored.set(timedVoteNode(invocation.getArgument(1))); + return null; + }).when(storage).addTimedVote(eq(2), org.mockito.ArgumentMatchers.any(VoteTimeQueue.class)); + UUID voteId = UUID.randomUUID(); + + assertTrue(handler.assignLegacyTimeVoteId(queued, voteId)); + assertEquals(voteId, queued.getVoteId()); + assertEquals(voteId.toString(), stored.get().get("VoteId").asString()); + verify(storage).save(); + } + @Test public void timedVoteDeliveryStateIsUpdatedByVoteId() { UUID voteId = UUID.randomUUID(); From 4122d62b162bfba95d9e5f0d29a8695d395ab834 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 11 Sep 2026 22:42:57 -0600 Subject: [PATCH 076/136] Retire completed multi-proxy fences safely --- .../votingplugin/proxy/VotingPluginProxy.java | 79 ++++++++++++++++--- .../votingplugin/proxy/VotingPluginWire.java | 14 ++++ .../proxy/cache/VoteCacheHandler.java | 13 +++ .../proxy/multiproxy/MultiProxyHandler.java | 45 +++++++++++ .../votingplugin/timequeue/VoteTimeQueue.java | 27 +++++++ .../MultiProxyHandlerLifecycleTest.java | 31 ++++++++ .../tests/VoteCacheHandlerVoteIdTest.java | 13 ++- .../tests/VotingPluginProxyTest.java | 58 ++++++++++++++ 8 files changed, 266 insertions(+), 14 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index baf513116..c657a76c8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -2416,6 +2416,16 @@ public void triggerVote(String player, String service, boolean realVote, boolean public void onMultiProxyVoteAcknowledged(UUID voteId, String recipient) { handleMultiProxyVoteAcknowledgement(voteId, recipient); } + + @Override + public void onMultiProxyVoteRetirementAcknowledged(UUID voteId, String recipient) { + handleMultiProxyVoteRetirementAcknowledgement(voteId, recipient); + } + + @Override + public void onMultiProxyVoteRetirementRequested(UUID voteId, String origin) { + handleMultiProxyVoteRetirementRequest(voteId, origin); + } }; multiProxyHandler.loadMultiProxySupport(); } @@ -2569,6 +2579,22 @@ private void acknowledgeCompletedMultiProxyVote(UUID voteId, String origin) { multiProxyHandler.acknowledgeMultiProxyVote(voteId, origin); } + private synchronized void handleMultiProxyVoteRetirementRequest(UUID voteId, String origin) { + if (voteId == null || origin == null || origin.isBlank() || multiProxyHandler == null) return; + for (VoteTimeQueue queued : getVoteCacheHandler().getTimeChangeQueue()) { + if (!voteId.equals(queued.getVoteId())) continue; + // A completion tombstone can be the only durable proof while deletion of + // the receiver's processed queue row is retrying. Retire that row first, + // otherwise deleting the tombstone would let the local queue replay it. + if (!queued.isProcessed() || !origin.equalsIgnoreCase(queued.getMultiProxyOrigin()) + || !getVoteCacheHandler().removeTimeVote(queued)) return; + break; + } + if (!getVoteCacheHandler().removeMultiProxyVoteCompletion(voteId)) return; + removeCompletedMultiProxyVote(voteId); + multiProxyHandler.acknowledgeMultiProxyVoteRetirement(voteId, origin); + } + private void scheduleMultiProxyVoteRetry(MultiProxyVoteRetry retry, boolean allowSchedule, String reason) { retry.attempts++; ScheduledExecutorService scheduler = getScheduler(); @@ -5492,7 +5518,7 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim if (!persistTimeVoteDelivery(outbox)) return false; } if (outbox.hasCompletedMultiProxyAcknowledgements()) { - return markMultiProxyForwardingHandled(retryState, outbox); + return finishMultiProxyRetirement(retryState, outbox); } // A successful publish only means the transport accepted the invocation. // Keep the durable row and wait for an acknowledgement before continuing. @@ -5518,9 +5544,7 @@ private UUID findLiveVoteId(LiveVoteRetryState retryState) { /** Re-sends an unacknowledged outbox using the exact persisted stable ID. */ private boolean retryDurableMultiProxyOutbox(VoteTimeQueue outbox) { if (outbox.hasCompletedMultiProxyAcknowledgements()) { - outbox.setMultiProxyForwardingHandled(true); - outbox.setDeliveryStateDirty(true); - return persistTimeVoteDelivery(outbox); + return finishMultiProxyRetirement(null, outbox); } sendDurableMultiProxyOutbox(outbox); return false; @@ -5528,10 +5552,13 @@ private boolean retryDurableMultiProxyOutbox(VoteTimeQueue outbox) { private boolean sendDurableMultiProxyOutbox(VoteTimeQueue outbox) { if (multiProxyHandler == null || outbox.getVoteId() == null || outbox.getMultiProxyOrigin().isBlank()) return false; + Set pending = new LinkedHashSet<>(outbox.getMultiProxyRecipients()); + pending.removeAll(outbox.getMultiProxyAcknowledgedServers()); + if (pending.isEmpty()) return true; return multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(outbox.getName(), outbox.getUuid(), outbox.getService(), outbox.getTime(), false, outbox.isRealVote(), outbox.getTotals(), outbox.getVoteId(), false, false, 1, 1, outbox.getMultiProxyOrigin()), - outbox.getMultiProxyRecipients()); + pending); } /** Handles an ACK only after confirming it belongs to a configured recipient. */ @@ -5553,21 +5580,47 @@ private synchronized void handleMultiProxyVoteAcknowledgement(UUID voteId, Strin scheduleTimeVoteRetry(); return; } - if (outbox.hasCompletedMultiProxyAcknowledgements()) { - outbox.setMultiProxyForwardingHandled(true); - outbox.setDeliveryStateDirty(true); - if (!persistTimeVoteDelivery(outbox)) { - scheduleTimeVoteRetry(); - return; + scheduleTimeVoteRetry(); + } + + private synchronized void handleMultiProxyVoteRetirementAcknowledgement(UUID voteId, String recipient) { + if (voteId == null || recipient == null || recipient.isBlank()) return; + LiveVoteRetryState state = liveVoteRetries.get(voteId); + VoteTimeQueue outbox = state == null ? null : state.queuedVote; + if (outbox == null) { + for (VoteTimeQueue candidate : getVoteCacheHandler().getTimeChangeQueue()) { + if (voteId.equals(candidate.getVoteId()) && candidate.isMultiProxyForwardingRequired()) { + outbox = candidate; + break; + } } - if (state != null) state.multiProxyForwardingHandled = true; + } + if (outbox == null || !outbox.acknowledgeMultiProxyRetirement(recipient)) return; + outbox.setDeliveryStateDirty(true); + if (!persistTimeVoteDelivery(outbox)) { + scheduleTimeVoteRetry(); + return; } scheduleTimeVoteRetry(); } + private boolean finishMultiProxyRetirement(LiveVoteRetryState retryState, VoteTimeQueue outbox) { + if (!outbox.hasCompletedMultiProxyRetirements()) { + if (multiProxyHandler != null) { + for (String recipient : outbox.getPendingMultiProxyRetirements()) { + multiProxyHandler.requestMultiProxyVoteRetirement(outbox.getVoteId(), + outbox.getMultiProxyOrigin(), recipient); + } + } + scheduleTimeVoteRetry(); + return false; + } + return markMultiProxyForwardingHandled(retryState, outbox); + } + /** Persists a queued vote's multi-proxy side-effect fence before later retryable work. */ private boolean markMultiProxyForwardingHandled(LiveVoteRetryState retryState, VoteTimeQueue queuedVote) { - retryState.multiProxyForwardingHandled = true; + if (retryState != null) retryState.multiProxyForwardingHandled = true; if (queuedVote == null) return true; queuedVote.setMultiProxyForwardingHandled(true); queuedVote.setDeliveryStateDirty(true); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java index 7d86fc5c7..598edac8a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java @@ -70,6 +70,8 @@ private VotingPluginWire() { public static final String SUB_CLEAR_VOTE_PRIMARY = "ClearVotePrimary"; /** Additive acknowledgement for the reliable multi-proxy vote envelope. */ public static final String SUB_MULTI_PROXY_VOTE_ACK = "MultiProxyVoteAck"; + public static final String SUB_MULTI_PROXY_VOTE_RETIRE = "MultiProxyVoteRetire"; + public static final String SUB_MULTI_PROXY_VOTE_RETIRE_ACK = "MultiProxyVoteRetireAck"; /** Additive capability advertisement for durable multi-proxy acknowledgements. */ public static final String SUB_MULTI_PROXY_CAPABILITIES = "MultiProxyCapabilities"; @@ -406,6 +408,18 @@ public static JsonEnvelope multiProxyVoteAck(UUID voteId, String origin, String .put(K_MULTI_PROXY_ORIGIN, safe(origin)).put(K_MULTI_PROXY_RECIPIENT, safe(recipient)).build(); } + /** Requests deletion of a receiver fence after every vote ACK is durable. */ + public static JsonEnvelope multiProxyVoteRetire(UUID voteId, String origin, String recipient) { + return base(SUB_MULTI_PROXY_VOTE_RETIRE).put(K_VOTE_ID, voteId == null ? "" : voteId.toString()) + .put(K_MULTI_PROXY_ORIGIN, safe(origin)).put(K_MULTI_PROXY_RECIPIENT, safe(recipient)).build(); + } + + /** Confirms idempotent receiver-fence retirement to the originating proxy. */ + public static JsonEnvelope multiProxyVoteRetireAck(UUID voteId, String origin, String recipient) { + return base(SUB_MULTI_PROXY_VOTE_RETIRE_ACK).put(K_VOTE_ID, voteId == null ? "" : voteId.toString()) + .put(K_MULTI_PROXY_ORIGIN, safe(origin)).put(K_MULTI_PROXY_RECIPIENT, safe(recipient)).build(); + } + /** Advertises support for the additive durable multi-proxy acknowledgement protocol. */ public static JsonEnvelope multiProxyCapabilities(String recipient, int acknowledgementVersion) { return multiProxyCapabilities(recipient, acknowledgementVersion, false); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index dc31d25f9..a40e71aa0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -1061,6 +1061,19 @@ public synchronized boolean hasMultiProxyVoteCompletion(UUID voteId) { } } + /** Idempotently removes a completion fence after its origin retired the outbox. */ + public synchronized boolean removeMultiProxyVoteCompletion(UUID voteId) { + Path target = multiProxyVoteCompletionPath(voteId); + if (target == null) return false; + try { + DurableFiles.deleteIfExists(target); + return !Files.exists(target); + } catch (IOException | RuntimeException failure) { + debug1(failure); + return false; + } + } + private Path timeVoteCompletionPath(VoteTimeQueue vote) { if (vote == null || jsonStorage == null || jsonStorage.getStoragePath() == null) return null; Path storage = jsonStorage.getStoragePath().toAbsolutePath().normalize(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java index 5ffcdfc0e..7a5a5745c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java @@ -281,6 +281,16 @@ public void onMultiProxyVoteAcknowledged(UUID voteId, String recipient) { // Optional for legacy implementations. } + /** Called on the origin when a receiver confirms fence retirement. */ + public void onMultiProxyVoteRetirementAcknowledged(UUID voteId, String recipient) { + // Optional for legacy implementations. + } + + /** Called on a receiver after a targeted retirement request is authenticated. */ + public void onMultiProxyVoteRetirementRequested(UUID voteId, String origin) { + // Optional for legacy implementations. + } + /** * Publishes an acknowledgement after receiver completion is durable. The * broadcast route keeps sockets and Redis compatible; recipients filter it by @@ -291,6 +301,18 @@ public void acknowledgeMultiProxyVote(UUID voteId, String origin) { sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVoteAck(voteId, origin, getMultiProxyServerName())); } + public void requestMultiProxyVoteRetirement(UUID voteId, String origin, String recipient) { + if (voteId == null || origin == null || origin.isBlank() || recipient == null || recipient.isBlank()) return; + sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVoteRetire(voteId, origin, recipient), + java.util.Set.of(recipient)); + } + + public void acknowledgeMultiProxyVoteRetirement(UUID voteId, String origin) { + if (voteId == null || origin == null || origin.isBlank()) return; + sendMultiProxyEnvelopeAccepted( + VotingPluginWire.multiProxyVoteRetireAck(voteId, origin, getMultiProxyServerName())); + } + /** Returns every configured remote proxy recipient, independent of version. */ public synchronized Set getConfiguredMultiProxyVoteRecipients() { Collection source = getMultiProxyMethod().equals(MultiProxyMethod.SOCKETS) @@ -569,6 +591,29 @@ private void handleEnvelope(JsonEnvelope envelope) { return; } + if (sub.equalsIgnoreCase(VotingPluginWire.SUB_MULTI_PROXY_VOTE_RETIRE)) { + String origin = f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_ORIGIN, ""); + String recipient = f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_RECIPIENT, ""); + if (!recipient.equalsIgnoreCase(getMultiProxyServerName()) || origin.isBlank() + || !getConfiguredMultiProxyVoteRecipients().contains(origin.toLowerCase(Locale.ROOT))) return; + try { + onMultiProxyVoteRetirementRequested( + UUID.fromString(f.getOrDefault(VotingPluginWire.K_VOTE_ID, "")), origin); + } catch (IllegalArgumentException ignored) { } + return; + } + + if (sub.equalsIgnoreCase(VotingPluginWire.SUB_MULTI_PROXY_VOTE_RETIRE_ACK)) { + String origin = f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_ORIGIN, ""); + String recipient = f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_RECIPIENT, ""); + if (!origin.equalsIgnoreCase(getMultiProxyServerName())) return; + try { + onMultiProxyVoteRetirementAcknowledged( + UUID.fromString(f.getOrDefault(VotingPluginWire.K_VOTE_ID, "")), recipient); + } catch (IllegalArgumentException ignored) { } + return; + } + if (sub.equalsIgnoreCase(VotingPluginWire.SUB_MULTI_PROXY_CAPABILITIES)) { String recipient = f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_RECIPIENT, ""); boolean reply = Boolean.parseBoolean(f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_CAPABILITY_REPLY, "false")); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java index 02a507321..84e5a3307 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java @@ -17,6 +17,7 @@ * Represents a vote delayed while a proxy time change is active. */ public class VoteTimeQueue { + private static final String MULTI_PROXY_RETIRED_PREFIX = "retired:"; @Getter @Setter private String name; @@ -228,6 +229,32 @@ public boolean hasCompletedMultiProxyAcknowledgements() { && multiProxyAcknowledgedServers.containsAll(multiProxyRecipients); } + /** Records that a receiver durably removed its completion fence. */ + public boolean acknowledgeMultiProxyRetirement(String recipient) { + if (recipient == null || recipient.isBlank()) return false; + String normalized = recipient.toLowerCase(Locale.ROOT); + if (!multiProxyRecipients.contains(normalized)) return false; + return multiProxyAcknowledgedServers.add(MULTI_PROXY_RETIRED_PREFIX + normalized); + } + + /** Returns whether every receiver acknowledged completion-fence retirement. */ + public boolean hasCompletedMultiProxyRetirements() { + if (!hasCompletedMultiProxyAcknowledgements()) return false; + for (String recipient : multiProxyRecipients) { + if (!multiProxyAcknowledgedServers.contains(MULTI_PROXY_RETIRED_PREFIX + recipient)) return false; + } + return true; + } + + /** Returns receivers that still need an idempotent retirement request. */ + public Set getPendingMultiProxyRetirements() { + Set pending = new LinkedHashSet<>(); + for (String recipient : multiProxyRecipients) { + if (!multiProxyAcknowledgedServers.contains(MULTI_PROXY_RETIRED_PREFIX + recipient)) pending.add(recipient); + } + return pending; + } + public String encodeMultiProxyRecipients() { return encodeBroadcastServers(multiProxyRecipients); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java index 3df63577b..3089b134a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java @@ -144,6 +144,37 @@ void routesOnlyTheOriginatingProxyAcknowledgement() throws Exception { verify(handler).onMultiProxyVoteAcknowledged(voteId, "Replica"); } + @Test + void routesOnlyAuthenticatedTargetedRetirementMessages() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); + org.mockito.Mockito.when(handler.getMultiProxyServerName()).thenReturn("Replica"); + org.mockito.Mockito.doReturn(new java.util.LinkedHashSet<>(List.of("primary"))) + .when(handler).getConfiguredMultiProxyVoteRecipients(); + UUID voteId = UUID.randomUUID(); + Method handleEnvelope = MultiProxyHandler.class.getDeclaredMethod("handleEnvelope", JsonEnvelope.class); + handleEnvelope.setAccessible(true); + + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyVoteRetire(voteId, "Primary", "Replica")); + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyVoteRetire(voteId, "Other", "Replica")); + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyVoteRetire(voteId, "Primary", "Other")); + + verify(handler).onMultiProxyVoteRetirementRequested(voteId, "Primary"); + } + + @Test + void routesOnlyTheOriginatingProxyRetirementAcknowledgement() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); + org.mockito.Mockito.when(handler.getMultiProxyServerName()).thenReturn("Primary"); + UUID voteId = UUID.randomUUID(); + Method handleEnvelope = MultiProxyHandler.class.getDeclaredMethod("handleEnvelope", JsonEnvelope.class); + handleEnvelope.setAccessible(true); + + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyVoteRetireAck(voteId, "Primary", "Replica")); + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyVoteRetireAck(voteId, "Other", "Replica")); + + verify(handler).onMultiProxyVoteRetirementAcknowledged(voteId, "Replica"); + } + @Test void fencesOnlyPeersThatAdvertiseDurableAcknowledgements() throws Exception { MultiProxyHandler handler = mock(MultiProxyHandler.class, diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 227c80724..661e07ca3 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -118,6 +118,16 @@ public void multiProxyCompletionSurvivesHandlerRestart() { assertTrue(restarted.hasMultiProxyVoteCompletion(voteId)); } + @Test + public void multiProxyCompletionRetirementIsDurableAndIdempotent() { + UUID voteId = UUID.randomUUID(); + assertTrue(handler.markMultiProxyVoteCompletedDurably(voteId)); + + assertTrue(handler.removeMultiProxyVoteCompletion(voteId)); + assertFalse(handler.hasMultiProxyVoteCompletion(voteId)); + assertTrue(handler.removeMultiProxyVoteCompletion(voteId)); + } + @Test public void failedCompletionPublicationDoesNotEvictAnExistingFence() throws Exception { Path completionDirectory = tempDir.resolve("vote-cache.json.completed-multiproxy-votes"); @@ -475,6 +485,7 @@ public void timedVoteAcknowledgementOutboxSurvivesHandlerRestart() { Map.of("backend-1", "00000000-0000-0000-0000-000000000181")); queued.requireMultiProxyAcknowledgements("origin-proxy", Set.of("backend-1")); queued.acknowledgeMultiProxyRecipient("backend-1"); + queued.acknowledgeMultiProxyRetirement("backend-1"); handler.addTimeVoteToCache(queued); when(storage.getTimedVoteCache()).thenReturn(List.of("0")); @@ -486,7 +497,7 @@ public void timedVoteAcknowledgementOutboxSurvivesHandlerRestart() { assertEquals(voteId, recovered.getVoteId()); assertTrue(recovered.isMultiProxyForwardingRequired()); assertEquals(Set.of("backend-1"), recovered.getMultiProxyRecipients()); - assertEquals(Set.of("backend-1"), recovered.getMultiProxyAcknowledgedServers()); + assertTrue(recovered.hasCompletedMultiProxyRetirements()); assertEquals("00000000-0000-0000-0000-000000000181", recovered.getHttpBroadcastDeliveryId("backend-1")); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 1157096b7..1f852dfee 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -498,6 +498,64 @@ void completedAckOutboxKeepsFenceForAlreadyScheduledListenerRetry() throws Excep assertTrue(retries.containsKey(voteId)); } + @Test + void completedAckOutboxWaitsForDurableFenceRetirementBeforeFinishing() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.UUID voteId = java.util.UUID.randomUUID(); + VoteTimeQueue outbox = new VoteTimeQueue(voteId, "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", true, + "00000000-0000-0000-0000-000000000001"); + outbox.requireMultiProxyAcknowledgements("Proxy1", java.util.Set.of("Proxy2")); + assertTrue(outbox.acknowledgeMultiProxyRecipient("Proxy2")); + Mockito.when(voteCache.updateTimeVote(outbox)).thenReturn(true); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + Class retryType = Class.forName("com.bencodez.votingplugin.proxy.VotingPluginProxy$LiveVoteRetryState"); + java.lang.reflect.Constructor retryConstructor = retryType.getDeclaredConstructor(); + retryConstructor.setAccessible(true); + Object retry = retryConstructor.newInstance(); + java.lang.reflect.Method finish = VotingPluginProxy.class.getDeclaredMethod("finishMultiProxyRetirement", + retryType, VoteTimeQueue.class); + finish.setAccessible(true); + + assertFalse((Boolean) finish.invoke(spyProxy, retry, outbox)); + assertFalse(outbox.isMultiProxyForwardingHandled()); + verify(multiProxyHandler).requestMultiProxyVoteRetirement(voteId, "Proxy1", "proxy2"); + + assertTrue(outbox.acknowledgeMultiProxyRetirement("Proxy2")); + assertTrue((Boolean) finish.invoke(spyProxy, retry, outbox)); + assertTrue(outbox.isMultiProxyForwardingHandled()); + verify(voteCache).updateTimeVote(outbox); + } + + @Test + void receiverKeepsCompletionFenceUntilItsProcessedQueueRowIsRemoved() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.UUID voteId = java.util.UUID.randomUUID(); + VoteTimeQueue received = new VoteTimeQueue(voteId, "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", true, + "00000000-0000-0000-0000-000000000001"); + received.setMultiProxyOrigin("Primary"); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + queue.add(received); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.removeTimeVote(received)).thenReturn(false, true); + Mockito.when(voteCache.removeMultiProxyVoteCompletion(voteId)).thenReturn(true); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + java.lang.reflect.Method retire = VotingPluginProxy.class.getDeclaredMethod( + "handleMultiProxyVoteRetirementRequest", java.util.UUID.class, String.class); + retire.setAccessible(true); + + retire.invoke(spyProxy, voteId, "Primary"); + verify(voteCache, never()).removeMultiProxyVoteCompletion(voteId); + verify(multiProxyHandler, never()).acknowledgeMultiProxyVoteRetirement(voteId, "Primary"); + + retire.invoke(spyProxy, voteId, "Primary"); + verify(voteCache).removeMultiProxyVoteCompletion(voteId); + verify(multiProxyHandler).acknowledgeMultiProxyVoteRetirement(voteId, "Primary"); + } + @Test void finalShutdownFencesQueuedVoteBeforeClearingLiveRetry() throws Exception { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); From dbcdad9d1d2667b72bedea924139205f6f9fbc54 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 08:10:13 -0600 Subject: [PATCH 077/136] Preserve distinct legacy timed vote rows --- .../votingplugin/proxy/VotingPluginProxy.java | 4 +- .../proxy/cache/ProxyTimedVoteCacheTable.java | 33 ++--- .../proxy/cache/VoteCacheHandler.java | 115 ++++++++++-------- .../votingplugin/timequeue/VoteTimeQueue.java | 27 ++++ .../tests/VoteCacheHandlerVoteIdTest.java | 51 ++++++++ 5 files changed, 159 insertions(+), 71 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index c657a76c8..6806d01f2 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -5446,9 +5446,7 @@ && getConfig().getPrimaryServer() } private UUID legacyTimedVoteId(VoteTimeQueue vote) { - String identity = "legacy-timed-vote\u0000" + vote.getUuid() + "\u0000" + vote.getName() + "\u0000" - + vote.getService() + "\u0000" + vote.getTime(); - return UUID.nameUUIDFromBytes(identity.getBytes(StandardCharsets.UTF_8)); + return vote.legacyTimedVoteId(); } /** diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java index f6b5f6224..5e245099a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java @@ -288,6 +288,7 @@ public boolean insertTimedVote(UUID voteId, String uuid, String playerName, Stri * @return true when the durable state update completed */ public boolean updateTimedVote(VoteTimeQueue vote) { + boolean hasRowId = vote.getTimedVoteCacheRowId() > 0; boolean hasVoteId = vote.getVoteId() != null; String sql = "UPDATE " + qi(getTableName()) + " SET " + qi("proxyBroadcastHandled") + " = ?, " + qi("broadcastTargets") + " = ?, " + qi("broadcastForwardedServers") + " = ?, " + qi("totals") @@ -297,7 +298,7 @@ public boolean updateTimedVote(VoteTimeQueue vote) { + qi("multiProxyRecipients") + " = ?, " + qi("multiProxyAcknowledgedServers") + " = ?, " + qi("uuid") + " = ?, " + qi("httpBroadcastDeliveryIds") + " = ? WHERE " - + (hasVoteId ? qi("voteId") + " = ?;" + + (hasRowId ? qi("id") + " = ?;" : hasVoteId ? qi("voteId") + " = ?;" : qi("playerName") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ?;"); try (Connection conn = mysql.getConnectionManager().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { @@ -330,7 +331,9 @@ public boolean updateTimedVote(VoteTimeQueue vote) { ps.setString(12, vote.encodeMultiProxyAcknowledgedServers()); ps.setString(13, vote.getUuid()); ps.setString(14, vote.encodeHttpBroadcastDeliveryIds()); - if (hasVoteId) { + if (hasRowId) { + ps.setInt(15, vote.getTimedVoteCacheRowId()); + } else if (hasVoteId) { ps.setString(15, vote.getVoteId().toString()); } else { ps.setString(15, vote.getName()); @@ -347,26 +350,21 @@ public boolean updateTimedVote(VoteTimeQueue vote) { /** Assigns a stable ID to a pre-ID timed row before reliable forwarding. */ public boolean assignLegacyTimedVoteId(VoteTimeQueue vote, UUID voteId) { - if (vote == null || voteId == null) return false; + if (vote == null || voteId == null || vote.getTimedVoteCacheRowId() <= 0) return false; String sql = "UPDATE " + qi(getTableName()) + " SET " + qi("voteId") + " = ? WHERE " - + qi("playerName") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ? AND (" + + qi("id") + " = ? AND (" + qi("voteId") + " IS NULL OR " + qi("voteId") + " = '' OR " + qi("voteId") + " = ?);"; try (Connection conn = mysql.getConnectionManager().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { ps.setString(1, voteId.toString()); - ps.setString(2, vote.getName()); - ps.setString(3, vote.getService()); - ps.setLong(4, vote.getTime()); - ps.setString(5, voteId.toString()); + ps.setInt(2, vote.getTimedVoteCacheRowId()); + ps.setString(3, voteId.toString()); if (ps.executeUpdate() > 0) return true; String verifySql = "SELECT " + qi("id") + " FROM " + qi(getTableName()) + " WHERE " - + qi("playerName") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ? AND " - + qi("voteId") + " = ?;"; + + qi("id") + " = ? AND " + qi("voteId") + " = ?;"; try (PreparedStatement verify = conn.prepareStatement(verifySql)) { - verify.setString(1, vote.getName()); - verify.setString(2, vote.getService()); - verify.setLong(3, vote.getTime()); - verify.setString(4, voteId.toString()); + verify.setInt(1, vote.getTimedVoteCacheRowId()); + verify.setString(2, voteId.toString()); try (ResultSet rows = verify.executeQuery()) { return rows.next(); } @@ -419,13 +417,16 @@ public void removeVoteById(int id) { * @return true when the durable delete completed */ public boolean removeVote(VoteTimeQueue vote) { + boolean hasRowId = vote.getTimedVoteCacheRowId() > 0; boolean hasVoteId = vote.getVoteId() != null; String sql = "DELETE FROM " + qi(getTableName()) + " WHERE " - + (hasVoteId ? qi("voteId") + " = ?;" + + (hasRowId ? qi("id") + " = ?;" : hasVoteId ? qi("voteId") + " = ?;" : qi("playerName") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ?;"); try (Connection conn = mysql.getConnectionManager().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { - if (hasVoteId) { + if (hasRowId) { + ps.setInt(1, vote.getTimedVoteCacheRowId()); + } else if (hasVoteId) { ps.setString(1, vote.getVoteId().toString()); } else { ps.setString(1, vote.getName()); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index a40e71aa0..4bc7b4fac 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -11,6 +11,7 @@ import java.util.LinkedHashSet; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Queue; import java.util.Set; import java.util.UUID; @@ -712,11 +713,17 @@ private boolean sameVoteIdentity(OfflineBungeeVote first, OfflineBungeeVote seco && first.getTime() == second.getTime(); } - private boolean matchesStoredTimeVote(DataNode data, VoteTimeQueue vote) { + private boolean matchesStoredTimeVote(String key, DataNode data, VoteTimeQueue vote) { + if (vote.getTimedVoteCacheJsonKey() != null) { + return vote.getTimedVoteCacheJsonKey().equals(key); + } String storedVoteId = readVoteId(data); if (vote.getVoteId() != null && storedVoteId != null && !storedVoteId.isEmpty()) { return vote.getVoteId().toString().equals(storedVoteId); } + // A SQL row is not an emergency JSON entry merely because its legacy + // fields happen to match. Without a JSON entry key, leave it alone. + if (vote.getTimedVoteCacheRowId() > 0) return false; return data.has("Name") && data.has("Service") && data.has("Time") && vote.getName().equals(data.get("Name").asString()) && vote.getService().equals(data.get("Service").asString()) @@ -768,6 +775,7 @@ public synchronized boolean addTimeVoteToCache(VoteTimeQueue vote) { while (keys != null && keys.contains(String.valueOf(index))) { index++; } + vote.setTimedVoteCacheJsonKey(String.valueOf(index)); jsonStorage.addTimedVote(index, vote); if (verifyJsonTimeVote(index, vote)) return true; timeChangeQueue.remove(vote); @@ -791,11 +799,27 @@ private boolean containsTimeVote(VoteTimeQueue expected) { return false; } + private VoteTimeQueue findSqlEmergencyTwin(VoteTimeQueue emergency) { + for (VoteTimeQueue candidate : timeChangeQueue) { + if (candidate == null || candidate.getTimedVoteCacheRowId() <= 0 + || candidate.getTimedVoteCacheJsonKey() != null) continue; + if (candidate.getVoteId() != null || emergency.getVoteId() != null) { + if (candidate.getVoteId() != null && candidate.getVoteId().equals(emergency.getVoteId())) return candidate; + continue; + } + if (Objects.equals(candidate.getUuid(), emergency.getUuid()) + && Objects.equals(candidate.getService(), emergency.getService()) + && candidate.getTime() == emergency.getTime()) return candidate; + } + return null; + } + /** Persists a timed vote in the JSON emergency journal. */ private boolean persistTimeVoteToJson(VoteTimeQueue vote) { if (jsonStorage == null || jsonStorageQuarantined) return false; try { int index = nextCacheIndex(jsonStorage.getTimedVoteCache()); + vote.setTimedVoteCacheJsonKey(String.valueOf(index)); jsonStorage.addTimedVote(index, vote); return verifyJsonTimeVote(index, vote); } catch (RuntimeException failure) { @@ -831,10 +855,12 @@ public synchronized boolean updateTimeVote(VoteTimeQueue vote) { public synchronized boolean assignLegacyTimeVoteId(VoteTimeQueue vote, UUID voteId) { if (vote == null || voteId == null) return false; if (vote.getVoteId() != null) return voteId.equals(vote.getVoteId()); + if (vote.getTimedVoteCacheRowId() <= 0 && vote.getTimedVoteCacheJsonKey() == null) return false; vote.setVoteId(voteId); - boolean jsonPresent = hasJsonTimeVote(vote); + boolean jsonPresent = vote.getTimedVoteCacheJsonKey() != null || hasJsonTimeVote(vote); boolean jsonStored = !jsonPresent || updateTimeVoteJson(vote); - boolean primaryStored = useMySQL ? timedVoteCacheTable.assignLegacyTimedVoteId(vote, voteId) : jsonStored; + boolean primaryStored = !useMySQL || vote.getTimedVoteCacheRowId() <= 0 + || timedVoteCacheTable.assignLegacyTimedVoteId(vote, voteId); if (primaryStored && jsonStored) return true; vote.setVoteId(null); return false; @@ -849,9 +875,10 @@ private boolean updateTimeVoteJson(VoteTimeQueue vote) { } for (String key : keys) { DataNode data = jsonStorage.getTimedVoteCache(key); - if (data != null && data.isObject() && matchesStoredTimeVote(data, vote)) { + if (data != null && data.isObject() && matchesStoredTimeVote(key, data, vote)) { try { int index = Integer.parseInt(key); + vote.setTimedVoteCacheJsonKey(key); jsonStorage.addTimedVote(index, vote); return verifyJsonTimeVote(index, vote); } catch (NumberFormatException e) { @@ -894,7 +921,7 @@ private boolean hasJsonTimeVote(VoteTimeQueue vote) { if (keys == null) return false; for (String key : keys) { DataNode data = jsonStorage.getTimedVoteCache(key); - if (data != null && data.isObject() && matchesStoredTimeVote(data, vote)) return true; + if (data != null && data.isObject() && matchesStoredTimeVote(key, data, vote)) return true; } return false; } catch (RuntimeException failure) { @@ -914,11 +941,11 @@ private boolean removeEmergencyTimeVoteJson(VoteTimeQueue vote) { for (String key : keys) { DataNode data = jsonStorage.getTimedVoteCache(key); if (data == null || !data.isObject()) return false; - if (matchesStoredTimeVote(data, vote)) { + if (matchesStoredTimeVote(key, data, vote)) { found = true; continue; } - VoteTimeQueue queued = decodeJsonTimedVote(data); + VoteTimeQueue queued = decodeJsonTimedVote(key, data); if (queued == null) return false; remaining.add(queued); } @@ -937,15 +964,25 @@ private boolean rewriteJsonTimeVotesAndRemove(VoteTimeQueue vote, Collection usedKeys = new LinkedHashSet<>(); for (VoteTimeQueue queued : remaining) { - jsonStorage.addTimedVote(index++, queued); + String key = queued.getTimedVoteCacheJsonKey(); + if (key == null || !key.matches("\\d+") || !usedKeys.add(key)) { + while (usedKeys.contains(String.valueOf(index))) index++; + key = String.valueOf(index); + usedKeys.add(key); + } + int entryIndex = Integer.parseInt(key); + queued.setTimedVoteCacheJsonKey(key); + jsonStorage.addTimedVote(entryIndex, queued); + index = Math.max(index, entryIndex + 1); } boolean removedDurably = VoteCacheDurability.saveAndVerifyRemoval(jsonStorage, () -> { Collection keys = jsonStorage.getTimedVoteCache(); if (keys == null) return true; for (String key : keys) { DataNode data = jsonStorage.getTimedVoteCache(key); - if (data != null && data.isObject() && matchesStoredTimeVote(data, vote)) return false; + if (data != null && data.isObject() && matchesStoredTimeVote(key, data, vote)) return false; } return true; }); @@ -1149,6 +1186,7 @@ public void load() { VoteTimeQueue.decodeBroadcastForwardedServers(timedVoteRow.getMultiProxyRecipients())); voteTimeQueue.setMultiProxyAcknowledgedServers( VoteTimeQueue.decodeBroadcastForwardedServers(timedVoteRow.getMultiProxyAcknowledgedServers())); + voteTimeQueue.setTimedVoteCacheRowId(timedVoteRow.getId()); timedVotes.add(voteTimeQueue); }); timeChangeQueue.addAll(timedVotes); @@ -1159,46 +1197,8 @@ public void load() { for (String key : jsonStorage.getTimedVoteCache()) { DataNode data = jsonStorage.getTimedVoteCache(key); - if (data != null && data.isObject()) { - String name = data.has("Name") ? data.get("Name").asString() : ""; - String service = data.has("Service") ? data.get("Service").asString() : ""; - long time = data.has("Time") ? data.get("Time").asLong() : 0L; - UUID voteId = readUuid(data, "VoteId"); - String uuid = data.has("UUID") ? data.get("UUID").asString() : ""; - boolean proxyBroadcastHandled = data.has("ProxyBroadcastHandled") - && data.get("ProxyBroadcastHandled").asBoolean(); - String forwardedServers = data.has("BroadcastForwardedServers") - ? data.get("BroadcastForwardedServers").asString() - : ""; - String broadcastTargets = data.has("BroadcastTargets") - ? data.get("BroadcastTargets").asString() - : ""; - String totals = data.has("Totals") ? data.get("Totals").asString() : ""; - boolean processed = data.has("Processed") && data.get("Processed").asBoolean(); - boolean multiProxyForwardingHandled = data.has("MultiProxyForwardingHandled") - && data.get("MultiProxyForwardingHandled").asBoolean(); - String httpBroadcastDeliveryIds = data.has("HttpBroadcastDeliveryIds") - ? data.get("HttpBroadcastDeliveryIds").asString() : ""; - - VoteTimeQueue queuedVote = new VoteTimeQueue(voteId, name, service, time, proxyBroadcastHandled, - VoteTimeQueue.decodeBroadcastForwardedServers(broadcastTargets), - VoteTimeQueue.decodeBroadcastForwardedServers(forwardedServers), totals, processed, - multiProxyForwardingHandled, uuid, - VoteTimeQueue.decodeHttpBroadcastDeliveryIds(httpBroadcastDeliveryIds)); - queuedVote.setMultiProxyForwardingRequired(data.has("MultiProxyForwardingRequired") - && data.get("MultiProxyForwardingRequired").asBoolean()); - queuedVote.setRealVote(!data.has("RealVote") || data.get("RealVote").asBoolean()); - queuedVote.setMultiProxyOrigin(data.has("MultiProxyOrigin") - ? data.get("MultiProxyOrigin").asString() : ""); - queuedVote.setMultiProxyCompletionPending(data.has("MultiProxyCompletionPending") - && data.get("MultiProxyCompletionPending").asBoolean()); - queuedVote.setMultiProxyRecipients(VoteTimeQueue.decodeBroadcastForwardedServers( - data.has("MultiProxyRecipients") ? data.get("MultiProxyRecipients").asString() : "")); - queuedVote.setMultiProxyAcknowledgedServers(VoteTimeQueue.decodeBroadcastForwardedServers( - data.has("MultiProxyAcknowledgedServers") - ? data.get("MultiProxyAcknowledgedServers").asString() : "")); - getTimeChangeQueue().add(queuedVote); - } + VoteTimeQueue queuedVote = decodeJsonTimedVote(key, data); + if (queuedVote != null) getTimeChangeQueue().add(queuedVote); } } catch (Exception e) { @@ -1330,8 +1330,18 @@ private void loadJsonEmergencyVotes() { if (timedKeys != null) { for (String key : timedKeys) { DataNode data = jsonStorage.getTimedVoteCache(key); - VoteTimeQueue vote = decodeJsonTimedVote(data); - if (vote != null && !containsTimeVote(vote)) timeChangeQueue.add(vote); + VoteTimeQueue vote = decodeJsonTimedVote(key, data); + if (vote == null) continue; + VoteTimeQueue sqlTwin = findSqlEmergencyTwin(vote); + if (sqlTwin != null) { + // A failed/uncertain SQL insert can leave the same durable vote in + // both stores. Bind this exact emergency entry to one SQL row so + // later state updates and deletion clean up both copies without + // collapsing separate identical rows. + sqlTwin.setTimedVoteCacheJsonKey(key); + } else if (!containsTimeVote(vote)) { + timeChangeQueue.add(vote); + } } } Collection servers = jsonStorage.getServers(); @@ -1369,7 +1379,7 @@ private void loadJsonEmergencyVotes() { } } - private VoteTimeQueue decodeJsonTimedVote(DataNode data) { + private VoteTimeQueue decodeJsonTimedVote(String key, DataNode data) { if (data == null || !data.isObject()) return null; String name = data.has("Name") ? data.get("Name").asString() : ""; String service = data.has("Service") ? data.get("Service").asString() : ""; @@ -1404,6 +1414,7 @@ private VoteTimeQueue decodeJsonTimedVote(DataNode data) { queuedVote.setMultiProxyAcknowledgedServers(VoteTimeQueue.decodeBroadcastForwardedServers( data.has("MultiProxyAcknowledgedServers") ? data.get("MultiProxyAcknowledgedServers").asString() : "")); + queuedVote.setTimedVoteCacheJsonKey(key); return queuedVote; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java index 84e5a3307..fbc8a08b9 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java @@ -30,6 +30,14 @@ public class VoteTimeQueue { @Getter @Setter private UUID voteId; + /** SQL primary key for a legacy timed-cache row, or {@code -1} when not SQL-backed. */ + @Getter + @Setter + private int timedVoteCacheRowId = -1; + /** Stable key of a JSON timed-cache entry, or {@code null} when not JSON-backed. */ + @Getter + @Setter + private String timedVoteCacheJsonKey; @Getter @Setter private String uuid; @@ -117,6 +125,25 @@ public VoteTimeQueue(UUID voteId, String name, String service, long time, boolea false, ""); } + /** + * Returns the durable identity of a legacy timed-cache entry. It distinguishes + * rows that have the same historical vote fields before they receive a vote ID. + */ + public String getTimedVoteCachePersistenceIdentity() { + if (timedVoteCacheRowId > 0) return "sql:" + timedVoteCacheRowId; + if (timedVoteCacheJsonKey != null && !timedVoteCacheJsonKey.isEmpty()) { + return "json:" + timedVoteCacheJsonKey; + } + return ""; + } + + /** Creates the deterministic ID used when a legacy timed-cache entry has none. */ + public UUID legacyTimedVoteId() { + String identity = "legacy-timed-vote\u0000" + uuid + "\u0000" + name + "\u0000" + service + "\u0000" + + time + "\u0000" + getTimedVoteCachePersistenceIdentity(); + return UUID.nameUUIDFromBytes(identity.getBytes(StandardCharsets.UTF_8)); + } + /** * Creates a queued vote with the original proxy broadcast routing state. * diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 661e07ca3..2da473446 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.same; @@ -35,6 +36,8 @@ import com.google.gson.JsonObject; import com.bencodez.votingplugin.proxy.OfflineBungeeVote; +import com.bencodez.votingplugin.proxy.bungee.BungeeJsonVoteCache; +import com.bencodez.votingplugin.proxy.bungee.VotingPluginBungee; import com.bencodez.votingplugin.proxy.cache.DataNode; import com.bencodez.votingplugin.proxy.cache.IVoteCache; import com.bencodez.votingplugin.proxy.cache.GsonDataNode; @@ -441,6 +444,7 @@ public void timedVoteIsPersistedImmediatelyWithItsId() { @Test public void legacyTimedVoteIdIsDurableBeforeReliableForwarding() { VoteTimeQueue queued = new VoteTimeQueue("Player", "Service", 100L); + queued.setTimedVoteCacheJsonKey("2"); JsonObject legacyJson = new JsonObject(); legacyJson.addProperty("Name", "Player"); legacyJson.addProperty("Service", "Service"); @@ -460,6 +464,47 @@ public void legacyTimedVoteIdIsDurableBeforeReliableForwarding() { verify(storage).save(); } + @Test + public void identicalLegacyTimedJsonRowsKeepDistinctIdsAndDeleteIndividually() throws Exception { + VotingPluginBungee plugin = mock(VotingPluginBungee.class); + when(plugin.getDataFolder()).thenReturn(tempDir.toFile()); + Files.writeString(tempDir.resolve("votecache.json"), """ + {"TimedVoteCache":{"0":{"Name":"Player","Service":"Service","Time":100,"UUID":"player-uuid"}, + "1":{"Name":"Player","Service":"Service","Time":100,"UUID":"player-uuid"}}} + """); + BungeeJsonVoteCache durableStorage = new BungeeJsonVoteCache(plugin); + + VoteCacheHandler durableHandler = newVerifyingHandler(durableStorage); + durableHandler.load(); + VoteTimeQueue first = durableHandler.getTimeChangeQueue().stream() + .filter(vote -> "0".equals(vote.getTimedVoteCacheJsonKey())).findFirst().orElseThrow(); + VoteTimeQueue second = durableHandler.getTimeChangeQueue().stream() + .filter(vote -> "1".equals(vote.getTimedVoteCacheJsonKey())).findFirst().orElseThrow(); + UUID firstId = first.legacyTimedVoteId(); + UUID secondId = second.legacyTimedVoteId(); + + assertNotEquals(firstId, secondId); + assertEquals(firstId, first.legacyTimedVoteId()); + assertTrue(durableHandler.assignLegacyTimeVoteId(first, firstId)); + assertTrue(durableHandler.assignLegacyTimeVoteId(second, secondId)); + assertTrue(durableHandler.removeTimeVote(first)); + + assertFalse(durableStorage.getTimedVoteCache().contains("0")); + assertTrue(durableStorage.getTimedVoteCache().contains("1")); + assertEquals(secondId.toString(), durableStorage.getTimedVoteCache("1").get("VoteId").asString()); + } + + @Test + public void identicalLegacyTimedSqlRowsUsePrimaryKeysForDistinctStableIds() { + VoteTimeQueue first = legacyTimedVote(); + VoteTimeQueue second = legacyTimedVote(); + first.setTimedVoteCacheRowId(41); + second.setTimedVoteCacheRowId(42); + + assertNotEquals(first.legacyTimedVoteId(), second.legacyTimedVoteId()); + assertEquals(first.legacyTimedVoteId(), first.legacyTimedVoteId()); + } + @Test public void timedVoteDeliveryStateIsUpdatedByVoteId() { UUID voteId = UUID.randomUUID(); @@ -905,6 +950,12 @@ private static OfflineBungeeVote vote(UUID voteId, long time) { return new OfflineBungeeVote(voteId, "Player", "player-uuid", "Service", time, true, "totals"); } + private static VoteTimeQueue legacyTimedVote() { + VoteTimeQueue vote = new VoteTimeQueue("Player", "Service", 100L); + vote.setUuid("player-uuid"); + return vote; + } + private static DataNode timedVoteNode(VoteTimeQueue vote) { JsonObject data = new JsonObject(); data.addProperty("Name", vote.getName()); From 58df2c873899eac61e08c2612edd294b4616d417 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 08:28:09 -0600 Subject: [PATCH 078/136] Preserve legacy cache row identities --- .../votingplugin/proxy/OfflineBungeeVote.java | 16 ++ .../votingplugin/proxy/VotingPluginProxy.java | 19 +- .../proxy/bungee/BungeeJsonVoteCache.java | 15 +- .../cache/ProxyOnlineVoteCacheTable.java | 39 +++- .../proxy/cache/ProxyVoteCacheTable.java | 40 +++- .../proxy/cache/VoteCacheHandler.java | 177 +++++++++++++++--- .../proxy/velocity/VelocityJsonVoteCache.java | 9 + .../tests/VoteCacheHandlerVoteIdTest.java | 50 +++++ 8 files changed, 322 insertions(+), 43 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java index 3923ef710..cb662eba2 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/OfflineBungeeVote.java @@ -51,6 +51,22 @@ public class OfflineBungeeVote { @Getter @Setter private boolean deliveryStateDirty; + /** SQL primary key for this vote's server-keyed cache row, when applicable. */ + @Getter + @Setter + private int serverVoteCacheRowId; + /** JSON entry key for this vote's server-keyed emergency cache row, when applicable. */ + @Getter + @Setter + private String serverVoteCacheJsonKey; + /** SQL primary key for this vote's voter-keyed cache row, when applicable. */ + @Getter + @Setter + private int onlineVoteCacheRowId; + /** JSON entry key for this vote's voter-keyed emergency cache row, when applicable. */ + @Getter + @Setter + private String onlineVoteCacheJsonKey; /** Stable HTTP delivery IDs that must be reused for each target server. */ private final Map httpDeliveryIds; /** Stable HTTP standalone-broadcast IDs, separate from reward delivery IDs. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 6806d01f2..6fa9b8e78 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -3716,14 +3716,29 @@ protected boolean sendHttpEnvelopeWithRecovery(String server, JsonEnvelope envel private String stableCachedHttpDeliveryId(String purpose, String server, JsonEnvelope envelope, OfflineBungeeVote cachedVote) { if (cachedVote == null || server == null || envelope == null) return null; - String voteIdentity = cachedVote.getVoteId() == null - ? cachedVote.getUuid() + "\u0000" + cachedVote.getService() + "\u0000" + cachedVote.getTime() + String voteIdentity = cachedVote.getVoteId() == null ? stableCachedVoteRowIdentity(cachedVote) : cachedVote.getVoteId().toString(); String key = "VotingPlugin:http-cache:v1\u0000" + purpose + "\u0000" + server.toLowerCase(Locale.ROOT) + "\u0000" + envelope.getSubChannel() + "\u0000" + voteIdentity; return UUID.nameUUIDFromBytes(key.getBytes(StandardCharsets.UTF_8)).toString(); } + /** + * Distinguishes pre-vote-ID cache rows that can otherwise share every visible + * vote field. SQL primary keys and JSON entry keys are durable before this + * fallback is used; the tuple remains only for callers holding an unbound + * legacy object during a mixed-version transition. + */ + private String stableCachedVoteRowIdentity(OfflineBungeeVote vote) { + if (vote.getServerVoteCacheRowId() > 0) return "server-sql:" + vote.getServerVoteCacheRowId(); + if (vote.getOnlineVoteCacheRowId() > 0) return "online-sql:" + vote.getOnlineVoteCacheRowId(); + if (vote.getServerVoteCacheJsonKey() != null) return "server-json:" + vote.getServerVoteCacheJsonKey(); + if (vote.getOnlineVoteCacheJsonKey() != null) { + return "online-json:" + vote.getUuid() + ":" + vote.getOnlineVoteCacheJsonKey(); + } + return "legacy:" + vote.getUuid() + "\u0000" + vote.getService() + "\u0000" + vote.getTime(); + } + protected synchronized boolean sendHttpEnvelope(String server, String deliveryId, JsonEnvelope envelope) { HttpProxyTransportServer transport = httpTransportServer; return transport != null && transport.send(server, deliveryId, envelope); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java index 65a8064b5..943c10cd6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java @@ -301,9 +301,13 @@ public void removeServerVote(String server, String uuid) { } } - @Override - public void removeVote(String server, OfflineBungeeVote vote) { - Collection votes = getServerVotes(server); + @Override + public void removeVote(String server, OfflineBungeeVote vote) { + if (vote.getServerVoteCacheJsonKey() != null) { + setString("VoteCache." + server + "." + vote.getServerVoteCacheJsonKey(), null); + return; + } + Collection votes = getServerVotes(server); if (votes == null) { return; } @@ -348,6 +352,11 @@ public void removeOnlineVote(OfflineBungeeVote vote) { continue; } for (String num : onlineVotes) { + if (vote.getOnlineVoteCacheJsonKey() != null + && vote.getOnlineVoteCacheJsonKey().equals(num)) { + setString("OnlineCache." + player + "." + num, null); + return; + } GsonDataNode node = getOnlineVotes(player, num); if (node == null) { continue; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java index fcdd4dea1..777483d51 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyOnlineVoteCacheTable.java @@ -162,12 +162,13 @@ public boolean updateProxyBroadcastState(OfflineBungeeVote vote) { return false; } + boolean hasRowId = vote.getOnlineVoteCacheRowId() > 0; boolean hasVoteId = vote.getVoteId() != null; String sql = "UPDATE " + qi(getTableName()) + " SET " + qi("broadcastForwarded") + " = ?, " + qi("proxyBroadcastHandled") + " = ?, " + qi("broadcastTargets") + " = ?, " + qi("broadcastForwardedServers") + " = ?, " + qi("rewardDelivered") + " = ?, " + qi("httpDeliveryIds") + " = ?, " + qi("httpBroadcastDeliveryIds") + " = ? WHERE " - + (hasVoteId ? qi("voteid") + " = ?;" + + (hasRowId ? qi("id") + " = ?;" : hasVoteId ? qi("voteid") + " = ?;" : qi("uuid") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ?;"); try (Connection conn = mysql.getConnectionManager().getConnection(); @@ -185,7 +186,9 @@ public boolean updateProxyBroadcastState(OfflineBungeeVote vote) { ps.setString(4, vote.encodeBroadcastForwardedServers()); ps.setString(6, vote.encodeHttpDeliveryIds()); ps.setString(7, vote.encodeHttpBroadcastDeliveryIds()); - if (hasVoteId) { + if (hasRowId) { + ps.setInt(8, vote.getOnlineVoteCacheRowId()); + } else if (hasVoteId) { ps.setString(8, vote.getVoteId().toString()); } else { if (getDbType() == DbType.POSTGRESQL) { @@ -336,6 +339,22 @@ public boolean tryInsertVote(UUID voteId, String uuid, String playerName, String String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, boolean rewardDelivered, String httpDeliveryIds, String httpBroadcastDeliveryIds) { + return tryInsertVoteAndGetId(voteId, uuid, playerName, service, time, real, text, broadcastForwarded, + proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, rewardDelivered, httpDeliveryIds, + httpBroadcastDeliveryIds) > 0; + } + + /** + * Inserts a vote and returns its durable primary key. Legacy rows can share + * their visible vote fields, so this identity is required for exact updates. + * + * @return the generated primary key, or {@code -1} when insertion or key + * retrieval failed + */ + public int tryInsertVoteAndGetId(UUID voteId, String uuid, String playerName, String service, long time, + boolean real, String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, + String broadcastTargets, String broadcastForwardedServers, boolean rewardDelivered, + String httpDeliveryIds, String httpBroadcastDeliveryIds) { String sql = "INSERT INTO " + qi(getTableName()) + " (" + qi("uuid") + ", " + qi("voteid") + ", " + qi("playerName") + ", " + qi("service") + ", " + qi("time") + ", " + qi("realVote") + ", " @@ -345,7 +364,7 @@ public boolean tryInsertVote(UUID voteId, String uuid, String playerName, String + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; try (Connection conn = mysql.getConnectionManager().getConnection(); - PreparedStatement ps = conn.prepareStatement(sql)) { + PreparedStatement ps = conn.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS)) { if (getDbType() == DbType.POSTGRESQL) { ps.setObject(1, UUID.fromString(uuid)); @@ -379,10 +398,13 @@ public boolean tryInsertVote(UUID voteId, String uuid, String playerName, String ps.setString(13, httpDeliveryIds); ps.setString(14, httpBroadcastDeliveryIds); - return ps.executeUpdate() > 0; + if (ps.executeUpdate() != 1) return -1; + try (ResultSet keys = ps.getGeneratedKeys()) { + return keys.next() ? keys.getInt(1) : -1; + } } catch (SQLException | IllegalArgumentException e) { debug(e); - return false; + return -1; } } @@ -434,14 +456,17 @@ public void removeVote(OfflineBungeeVote vote) { /** Removes one stable vote identity and reports whether the statement completed. */ public boolean tryRemoveVote(OfflineBungeeVote vote) { + boolean hasRowId = vote.getOnlineVoteCacheRowId() > 0; boolean byVoteId = vote.getVoteId() != null; String sql = "DELETE FROM " + qi(getTableName()) + " WHERE " - + (byVoteId ? qi("voteid") + " = ?" : qi("uuid") + " = ? AND " + qi("service") + + (hasRowId ? qi("id") + " = ?" : byVoteId ? qi("voteid") + " = ?" : qi("uuid") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ?") + ";"; try (Connection conn = mysql.getConnectionManager().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { - if (byVoteId) { + if (hasRowId) { + ps.setInt(1, vote.getOnlineVoteCacheRowId()); + } else if (byVoteId) { ps.setString(1, vote.getVoteId().toString()); } else { if (getDbType() == DbType.POSTGRESQL) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java index 1af57ebd7..66d2c3e17 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyVoteCacheTable.java @@ -127,12 +127,13 @@ public boolean updateProxyBroadcastState(OfflineBungeeVote vote, String server) return false; } + boolean hasRowId = vote.getServerVoteCacheRowId() > 0; boolean hasVoteId = vote.getVoteId() != null; String sql = "UPDATE " + qi(getTableName()) + " SET " + qi("broadcastForwarded") + " = ?, " + qi("proxyBroadcastHandled") + " = ?, " + qi("broadcastTargets") + " = ?, " + qi("broadcastForwardedServers") + " = ?, " + qi("rewardDelivered") + " = ?, " + qi("httpDeliveryIds") + " = ?, " + qi("httpBroadcastDeliveryIds") + " = ? WHERE " - + (hasVoteId ? qi("voteid") + " = ? AND " + qi("server") + " = ?;" + + (hasRowId ? qi("id") + " = ?;" : hasVoteId ? qi("voteid") + " = ? AND " + qi("server") + " = ?;" : qi("uuid") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ? AND " + qi("server") + " = ?;"); @@ -151,7 +152,9 @@ public boolean updateProxyBroadcastState(OfflineBungeeVote vote, String server) ps.setString(4, vote.encodeBroadcastForwardedServers()); ps.setString(6, vote.encodeHttpDeliveryIds()); ps.setString(7, vote.encodeHttpBroadcastDeliveryIds()); - if (hasVoteId) { + if (hasRowId) { + ps.setInt(8, vote.getServerVoteCacheRowId()); + } else if (hasVoteId) { ps.setString(8, vote.getVoteId().toString()); ps.setString(9, server); } else { @@ -354,6 +357,22 @@ public boolean tryInsertVote(UUID voteId, String uuid, String playerName, String String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, boolean rewardDelivered, String httpDeliveryIds, String httpBroadcastDeliveryIds, String server) { + return tryInsertVoteAndGetId(voteId, uuid, playerName, service, time, real, text, broadcastForwarded, + proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, rewardDelivered, httpDeliveryIds, + httpBroadcastDeliveryIds, server) > 0; + } + + /** + * Inserts a vote and returns its durable primary key. Delivery-state updates + * must use this row identity because legacy cache rows need not have a vote ID. + * + * @return the generated primary key, or {@code -1} when insertion or key + * retrieval failed + */ + public int tryInsertVoteAndGetId(UUID voteId, String uuid, String playerName, String service, long time, + boolean real, String text, boolean broadcastForwarded, boolean proxyBroadcastHandled, + String broadcastTargets, String broadcastForwardedServers, boolean rewardDelivered, + String httpDeliveryIds, String httpBroadcastDeliveryIds, String server) { String sql = "INSERT INTO " + qi(getTableName()) + " (" + qi("uuid") + ", " + qi("voteid") + ", " + qi("playerName") + ", " + qi("service") + ", " + qi("time") + ", " + qi("realVote") + ", " @@ -363,7 +382,7 @@ public boolean tryInsertVote(UUID voteId, String uuid, String playerName, String + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; try (Connection conn = mysql.getConnectionManager().getConnection(); - PreparedStatement ps = conn.prepareStatement(sql)) { + PreparedStatement ps = conn.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS)) { if (getDbType() == DbType.POSTGRESQL) { ps.setObject(1, UUID.fromString(uuid)); @@ -398,10 +417,13 @@ public boolean tryInsertVote(UUID voteId, String uuid, String playerName, String ps.setString(14, httpBroadcastDeliveryIds); ps.setString(15, server); - return ps.executeUpdate() > 0; + if (ps.executeUpdate() != 1) return -1; + try (ResultSet keys = ps.getGeneratedKeys()) { + return keys.next() ? keys.getInt(1) : -1; + } } catch (SQLException | IllegalArgumentException e) { debug(e); - return false; + return -1; } } @@ -534,15 +556,19 @@ public void removeVote(OfflineBungeeVote vote, String server) { /** Removes one stable vote identity and reports whether the statement completed. */ public boolean tryRemoveVote(OfflineBungeeVote vote, String server) { + boolean hasRowId = vote.getServerVoteCacheRowId() > 0; boolean byVoteId = vote.getVoteId() != null; String sql = "DELETE FROM " + qi(getTableName()) + " WHERE " - + (byVoteId ? qi("voteid") + " = ?" : qi("uuid") + " = ? AND " + qi("service") + + (hasRowId ? qi("id") + " = ?" : byVoteId ? qi("voteid") + " = ?" : qi("uuid") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ?") + " AND " + qi("server") + " = ?;"; try (Connection conn = mysql.getConnectionManager().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { - if (byVoteId) { + if (hasRowId) { + ps.setInt(1, vote.getServerVoteCacheRowId()); + ps.setString(2, server); + } else if (byVoteId) { ps.setString(1, vote.getVoteId().toString()); ps.setString(2, server); } else { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index 4bc7b4fac..eced8926b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -131,12 +131,15 @@ public synchronized boolean addServerVoteDurably(String server, OfflineBungeeVot boolean stored; if (useMySQL) { - stored = voteCacheTable.tryInsertVote(vote.getVoteId(), vote.getUuid(), vote.getPlayerName(), vote.getService(), + int rowId = voteCacheTable.tryInsertVoteAndGetId(vote.getVoteId(), vote.getUuid(), vote.getPlayerName(), vote.getService(), vote.getTime(), vote.isRealVote(), vote.getText(), vote.isBroadcastForwarded(), vote.isProxyBroadcastHandled(), vote.encodeBroadcastTargets(), vote.encodeBroadcastForwardedServers(), vote.isRewardDelivered(), vote.encodeHttpDeliveryIds(), vote.encodeHttpBroadcastDeliveryIds(), server); - if (!stored) { + stored = rowId > 0; + if (stored) { + vote.setServerVoteCacheRowId(rowId); + } else { stored = persistServerVoteToJson(server, vote); } } else { @@ -171,7 +174,9 @@ public synchronized boolean retainServerVoteForPersistenceRetry(String server, O public synchronized boolean updateServerVote(String server, OfflineBungeeVote vote) { if (useMySQL) { if (voteCacheTable.updateProxyBroadcastState(vote, server)) { - return true; + // SQL rows can have an emergency JSON twin after a mixed-version or + // recovery path. Keep that exact twin current before publishing again. + return !hasJsonServerVote(vote, server) || updateServerVoteJson(server, vote); } return updateServerVoteJson(server, vote); } @@ -187,9 +192,10 @@ public synchronized boolean updateServerVote(String server, OfflineBungeeVote vo || !data.has("Time")) { continue; } - if (matchesStoredVote(data, vote)) { + if (matchesStoredServerVote(key, data, vote)) { try { int index = Integer.parseInt(key); + vote.setServerVoteCacheJsonKey(key); jsonStorage.addVote(server, index, vote); return verifyJsonServerVote(server, index, vote); } catch (RuntimeException e) { @@ -290,12 +296,15 @@ public synchronized boolean addOnlineVoteDurably(String uuid, OfflineBungeeVote boolean stored; if (useMySQL) { - stored = onlineVoteCacheTable.tryInsertVote(vote.getVoteId(), vote.getUuid(), vote.getPlayerName(), vote.getService(), + int rowId = onlineVoteCacheTable.tryInsertVoteAndGetId(vote.getVoteId(), vote.getUuid(), vote.getPlayerName(), vote.getService(), vote.getTime(), vote.isRealVote(), vote.getText(), vote.isBroadcastForwarded(), vote.isProxyBroadcastHandled(), vote.encodeBroadcastTargets(), vote.encodeBroadcastForwardedServers(), vote.isRewardDelivered(), vote.encodeHttpDeliveryIds(), vote.encodeHttpBroadcastDeliveryIds()); - if (!stored) { + stored = rowId > 0; + if (stored) { + vote.setOnlineVoteCacheRowId(rowId); + } else { stored = persistOnlineVoteToJson(uuid, vote); } } else { @@ -364,7 +373,7 @@ private boolean containsVoteId(Collection votes, UUID voteId) public synchronized boolean updateOnlineVote(String uuid, OfflineBungeeVote vote) { if (useMySQL) { if (onlineVoteCacheTable.updateProxyBroadcastState(vote)) { - return true; + return !hasJsonOnlineVote(vote, uuid) || updateOnlineVoteJson(uuid, vote); } return updateOnlineVoteJson(uuid, vote); } @@ -380,9 +389,10 @@ public synchronized boolean updateOnlineVote(String uuid, OfflineBungeeVote vote || !data.has("Time")) { continue; } - if (matchesStoredVote(data, vote)) { + if (matchesStoredOnlineVote(key, data, vote)) { try { int index = Integer.parseInt(key); + vote.setOnlineVoteCacheJsonKey(key); jsonStorage.addVoteOnline(uuid, index, vote); return verifyJsonOnlineVote(uuid, index, vote); } catch (RuntimeException e) { @@ -459,12 +469,32 @@ protected boolean removeJsonOnlineVoteDurably(String uuid, OfflineBungeeVote rem } } + /** Returns whether this vote has an exact voter-cache JSON emergency twin. */ + private boolean hasJsonOnlineVote(OfflineBungeeVote vote, String uuid) { + if (jsonStorage == null) return false; + if (jsonStorageQuarantined) return true; + try { + Collection keys = jsonStorage.getOnlineVotes(uuid); + if (keys == null) return false; + for (String key : keys) { + DataNode data = jsonStorage.getOnlineVotes(uuid, key); + if (data != null && data.isObject() && matchesStoredOnlineVote(key, data, vote)) return true; + } + return false; + } catch (RuntimeException failure) { + debug1(failure); + // A JSON journal we cannot inspect may be the only stale twin after a + // restart. Do not claim a delivery-state update is durable. + return true; + } + } + private boolean containsStoredOnlineVote(String uuid, OfflineBungeeVote expected) { Collection keys = jsonStorage.getOnlineVotes(uuid); if (keys == null) return false; for (String key : keys) { DataNode data = jsonStorage.getOnlineVotes(uuid, key); - if (data != null && matchesStoredVote(data, expected)) return true; + if (data != null && matchesStoredOnlineVote(key, data, expected)) return true; } return false; } @@ -539,6 +569,37 @@ private boolean containsOnlineVote(String uuid, OfflineBungeeVote candidate) { return false; } + private OfflineBungeeVote findServerVoteSqlTwin(String server, OfflineBungeeVote emergency) { + for (OfflineBungeeVote candidate : getVotes(server)) { + if (candidate.getServerVoteCacheRowId() <= 0 || candidate.getServerVoteCacheJsonKey() != null) continue; + if (candidate.getVoteId() != null || emergency.getVoteId() != null) { + if (candidate.getVoteId() != null && candidate.getVoteId().equals(emergency.getVoteId())) return candidate; + continue; + } + if (matchesLegacyVote(candidate, emergency)) return candidate; + } + return null; + } + + private OfflineBungeeVote findOnlineVoteSqlTwin(String uuid, OfflineBungeeVote emergency) { + for (OfflineBungeeVote candidate : getOnlineVotes(uuid)) { + if (candidate.getOnlineVoteCacheRowId() <= 0 || candidate.getOnlineVoteCacheJsonKey() != null) continue; + if (candidate.getVoteId() != null || emergency.getVoteId() != null) { + if (candidate.getVoteId() != null && candidate.getVoteId().equals(emergency.getVoteId())) return candidate; + continue; + } + if (matchesLegacyVote(candidate, emergency)) return candidate; + } + return null; + } + + private boolean matchesLegacyVote(OfflineBungeeVote first, OfflineBungeeVote second) { + return Objects.equals(first.getUuid(), second.getUuid()) + && Objects.equals(first.getPlayerName(), second.getPlayerName()) + && Objects.equals(first.getService(), second.getService()) + && first.getTime() == second.getTime(); + } + /** * Reads a vote identifier using the current key and the legacy key. * @@ -587,6 +648,28 @@ private boolean matchesStoredVote(DataNode data, OfflineBungeeVote vote) { && vote.getTime() == data.get("Time").asLong(); } + private boolean matchesStoredServerVote(String key, DataNode data, OfflineBungeeVote vote) { + if (vote.getServerVoteCacheJsonKey() != null) return vote.getServerVoteCacheJsonKey().equals(key); + String storedVoteId = readVoteId(data); + if (vote.getVoteId() != null && !storedVoteId.isEmpty()) { + return vote.getVoteId().toString().equals(storedVoteId); + } + // A SQL row is not a JSON emergency twin merely because a legacy tuple + // matches. Pre-ID rows are not unique by those fields. + if (vote.getServerVoteCacheRowId() > 0) return false; + return matchesStoredVote(data, vote); + } + + private boolean matchesStoredOnlineVote(String key, DataNode data, OfflineBungeeVote vote) { + if (vote.getOnlineVoteCacheJsonKey() != null) return vote.getOnlineVoteCacheJsonKey().equals(key); + String storedVoteId = readVoteId(data); + if (vote.getVoteId() != null && !storedVoteId.isEmpty()) { + return vote.getVoteId().toString().equals(storedVoteId); + } + if (vote.getOnlineVoteCacheRowId() > 0) return false; + return matchesStoredVote(data, vote); + } + /** Persists a server vote in the JSON emergency journal. */ private boolean persistServerVoteToJson(String server, OfflineBungeeVote vote) { if (jsonStorage == null || jsonStorageQuarantined) { @@ -595,6 +678,7 @@ private boolean persistServerVoteToJson(String server, OfflineBungeeVote vote) { try { Collection keys = jsonStorage.getServerVotes(server); int index = nextCacheIndex(keys); + vote.setServerVoteCacheJsonKey(String.valueOf(index)); jsonStorage.addVote(server, index, vote); return verifyJsonServerVote(server, index, vote); } catch (RuntimeException e) { @@ -611,6 +695,7 @@ private boolean persistOnlineVoteToJson(String uuid, OfflineBungeeVote vote) { try { Collection keys = jsonStorage.getOnlineVotes(uuid); int index = nextCacheIndex(keys); + vote.setOnlineVoteCacheJsonKey(String.valueOf(index)); jsonStorage.addVoteOnline(uuid, index, vote); return verifyJsonOnlineVote(uuid, index, vote); } catch (RuntimeException e) { @@ -667,9 +752,10 @@ private boolean updateServerVoteJson(String server, OfflineBungeeVote vote) { } for (String key : keys) { DataNode data = jsonStorage.getServerVotes(server, key); - if (data != null && data.isObject() && matchesStoredVote(data, vote)) { + if (data != null && data.isObject() && matchesStoredServerVote(key, data, vote)) { try { int index = Integer.parseInt(key); + vote.setServerVoteCacheJsonKey(key); jsonStorage.addVote(server, index, vote); return verifyJsonServerVote(server, index, vote); } catch (RuntimeException e) { @@ -691,9 +777,10 @@ private boolean updateOnlineVoteJson(String uuid, OfflineBungeeVote vote) { } for (String key : keys) { DataNode data = jsonStorage.getOnlineVotes(uuid, key); - if (data != null && data.isObject() && matchesStoredVote(data, vote)) { + if (data != null && data.isObject() && matchesStoredOnlineVote(key, data, vote)) { try { int index = Integer.parseInt(key); + vote.setOnlineVoteCacheJsonKey(key); jsonStorage.addVoteOnline(uuid, index, vote); return verifyJsonOnlineVote(uuid, index, vote); } catch (RuntimeException e) { @@ -706,8 +793,22 @@ private boolean updateOnlineVoteJson(String uuid, OfflineBungeeVote vote) { } private boolean sameVoteIdentity(OfflineBungeeVote first, OfflineBungeeVote second) { - if (first.getVoteId() != null && second.getVoteId() != null) { - return first.getVoteId().equals(second.getVoteId()); + if (first.getVoteId() != null || second.getVoteId() != null) { + return first.getVoteId() != null && first.getVoteId().equals(second.getVoteId()); + } + if (first.getServerVoteCacheRowId() > 0 || second.getServerVoteCacheRowId() > 0) { + return first.getServerVoteCacheRowId() > 0 + && first.getServerVoteCacheRowId() == second.getServerVoteCacheRowId(); + } + if (first.getOnlineVoteCacheRowId() > 0 || second.getOnlineVoteCacheRowId() > 0) { + return first.getOnlineVoteCacheRowId() > 0 + && first.getOnlineVoteCacheRowId() == second.getOnlineVoteCacheRowId(); + } + if (first.getServerVoteCacheJsonKey() != null || second.getServerVoteCacheJsonKey() != null) { + return Objects.equals(first.getServerVoteCacheJsonKey(), second.getServerVoteCacheJsonKey()); + } + if (first.getOnlineVoteCacheJsonKey() != null || second.getOnlineVoteCacheJsonKey() != null) { + return Objects.equals(first.getOnlineVoteCacheJsonKey(), second.getOnlineVoteCacheJsonKey()); } return first.getUuid().equals(second.getUuid()) && first.getService().equals(second.getService()) && first.getTime() == second.getTime(); @@ -1148,6 +1249,7 @@ public void load() { VoteTimeQueue.decodeBroadcastForwardedServers(voteRow.getBroadcastForwardedServers()), voteRow.isRewardDelivered(), OfflineBungeeVote.decodeHttpDeliveryIds(voteRow.getHttpDeliveryIds()), OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(voteRow.getHttpBroadcastDeliveryIds())); + vote.setServerVoteCacheRowId(voteRow.getId()); String server = voteRow.getServer(); cachedVotes.putIfAbsent(server, new ArrayList<>()); cachedVotes.get(server).add(vote); @@ -1162,6 +1264,7 @@ public void load() { VoteTimeQueue.decodeBroadcastForwardedServers(voteRow.getBroadcastForwardedServers()), voteRow.isRewardDelivered(), OfflineBungeeVote.decodeHttpDeliveryIds(voteRow.getHttpDeliveryIds()), OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(voteRow.getHttpBroadcastDeliveryIds())); + vote.setOnlineVoteCacheRowId(voteRow.getId()); String player = vote.getUuid(); cachedOnlineVotes.putIfAbsent(player, new ArrayList<>()); cachedOnlineVotes.get(player).add(vote); @@ -1237,12 +1340,14 @@ public void load() { boolean rewardDelivered = data.has("RewardDelivered") && data.get("RewardDelivered").asBoolean(); - votes.add(new OfflineBungeeVote(voteId, name, uuid, service, time, real, text, + OfflineBungeeVote vote = new OfflineBungeeVote(voteId, name, uuid, service, time, real, text, broadcastForwarded, proxyBroadcastHandled, VoteTimeQueue.decodeBroadcastForwardedServers(broadcastTargets), VoteTimeQueue.decodeBroadcastForwardedServers(broadcastForwardedServers), rewardDelivered, OfflineBungeeVote.decodeHttpDeliveryIds(httpDeliveryIds), - OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(httpBroadcastDeliveryIds))); + OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(httpBroadcastDeliveryIds)); + vote.setServerVoteCacheJsonKey(num); + votes.add(vote); } } cachedVotes.put(server, votes); @@ -1283,12 +1388,14 @@ public void load() { boolean rewardDelivered = data.has("RewardDelivered") && data.get("RewardDelivered").asBoolean(); - votes.add(new OfflineBungeeVote(voteId, name, uuid, service, time, real, text, + OfflineBungeeVote vote = new OfflineBungeeVote(voteId, name, uuid, service, time, real, text, broadcastForwarded, proxyBroadcastHandled, VoteTimeQueue.decodeBroadcastForwardedServers(broadcastTargets), VoteTimeQueue.decodeBroadcastForwardedServers(broadcastForwardedServers), rewardDelivered, OfflineBungeeVote.decodeHttpDeliveryIds(httpDeliveryIds), - OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(httpBroadcastDeliveryIds))); + OfflineBungeeVote.decodeHttpBroadcastDeliveryIds(httpBroadcastDeliveryIds)); + vote.setOnlineVoteCacheJsonKey(num); + votes.add(vote); } } cachedOnlineVotes.put(player, votes); @@ -1353,9 +1460,11 @@ private void loadJsonEmergencyVotes() { } for (String key : keys) { OfflineBungeeVote vote = decodeJsonVote(jsonStorage.getServerVotes(server, key)); - if (vote != null && !containsServerVote(server, vote)) { - cachedVotes.computeIfAbsent(server, ignored -> new ArrayList<>()).add(vote); - } + if (vote == null) continue; + vote.setServerVoteCacheJsonKey(key); + OfflineBungeeVote sqlTwin = findServerVoteSqlTwin(server, vote); + if (sqlTwin != null) sqlTwin.setServerVoteCacheJsonKey(key); + else cachedVotes.computeIfAbsent(server, ignored -> new ArrayList<>()).add(vote); } } } @@ -1368,9 +1477,11 @@ private void loadJsonEmergencyVotes() { } for (String key : keys) { OfflineBungeeVote vote = decodeJsonVote(jsonStorage.getOnlineVotes(player, key)); - if (vote != null && !containsOnlineVote(player, vote)) { - cachedOnlineVotes.computeIfAbsent(player, ignored -> new ArrayList<>()).add(vote); - } + if (vote == null) continue; + vote.setOnlineVoteCacheJsonKey(key); + OfflineBungeeVote sqlTwin = findOnlineVoteSqlTwin(player, vote); + if (sqlTwin != null) sqlTwin.setOnlineVoteCacheJsonKey(key); + else cachedOnlineVotes.computeIfAbsent(player, ignored -> new ArrayList<>()).add(vote); } } } @@ -1686,12 +1797,30 @@ protected boolean removeJsonServerVoteDurably(String server, OfflineBungeeVote v } } + /** Returns whether this vote has an exact server-cache JSON emergency twin. */ + private boolean hasJsonServerVote(OfflineBungeeVote vote, String server) { + if (jsonStorage == null) return false; + if (jsonStorageQuarantined) return true; + try { + Collection keys = jsonStorage.getServerVotes(server); + if (keys == null) return false; + for (String key : keys) { + DataNode data = jsonStorage.getServerVotes(server, key); + if (data != null && data.isObject() && matchesStoredServerVote(key, data, vote)) return true; + } + return false; + } catch (RuntimeException failure) { + debug1(failure); + return true; + } + } + private boolean containsStoredServerVote(String server, OfflineBungeeVote expected) { Collection keys = jsonStorage.getServerVotes(server); if (keys == null) return false; for (String key : keys) { DataNode data = jsonStorage.getServerVotes(server, key); - if (data != null && matchesStoredVote(data, expected)) return true; + if (data != null && matchesStoredServerVote(key, data, expected)) return true; } return false; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java index ab5a13194..c9ac77b06 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java @@ -311,6 +311,10 @@ public void removeServerVote(String server, String uuid) { @Override public void removeVote(String server, OfflineBungeeVote vote) { + if (vote.getServerVoteCacheJsonKey() != null) { + remove("VoteCache", server, vote.getServerVoteCacheJsonKey()); + return; + } Collection votes = getServerVotes(server); if (votes == null) { return; @@ -357,6 +361,11 @@ public void removeOnlineVote(OfflineBungeeVote vote) { continue; } for (String num : onlineVotes) { + if (vote.getOnlineVoteCacheJsonKey() != null + && vote.getOnlineVoteCacheJsonKey().equals(num)) { + remove("OnlineCache", player, num); + return; + } ConfigDataNode node = getOnlineVotes(player, num); if (node == null) { continue; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 2da473446..a0f4962b9 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -494,6 +494,56 @@ public void identicalLegacyTimedJsonRowsKeepDistinctIdsAndDeleteIndividually() t assertEquals(secondId.toString(), durableStorage.getTimedVoteCache("1").get("VoteId").asString()); } + @Test + public void identicalLegacyServerJsonRowsKeepDistinctEntryIdentitiesAndDeleteIndividually() throws Exception { + VotingPluginBungee plugin = mock(VotingPluginBungee.class); + when(plugin.getDataFolder()).thenReturn(tempDir.toFile()); + Files.writeString(tempDir.resolve("votecache.json"), """ + {"VoteCache":{"server":{"0":{"Name":"Player","Service":"Service","UUID":"player-uuid","Time":100}, + "1":{"Name":"Player","Service":"Service","UUID":"player-uuid","Time":100}}}} + """); + BungeeJsonVoteCache durableStorage = new BungeeJsonVoteCache(plugin); + VoteCacheHandler durableHandler = newVerifyingHandler(durableStorage); + + durableHandler.load(); + OfflineBungeeVote first = durableHandler.getVotes("server").stream() + .filter(vote -> "0".equals(vote.getServerVoteCacheJsonKey())).findFirst().orElseThrow(); + OfflineBungeeVote second = durableHandler.getVotes("server").stream() + .filter(vote -> "1".equals(vote.getServerVoteCacheJsonKey())).findFirst().orElseThrow(); + + assertEquals(2, durableHandler.getVotes("server").size()); + durableHandler.removeServerVotes("server", new ArrayList<>(List.of(first))); + + assertFalse(durableStorage.getServerVotes("server").contains("0")); + assertTrue(durableStorage.getServerVotes("server").contains("1")); + assertEquals("1", second.getServerVoteCacheJsonKey()); + } + + @Test + public void identicalLegacyOnlineJsonRowsKeepDistinctEntryIdentitiesAndDeleteIndividually() throws Exception { + VotingPluginBungee plugin = mock(VotingPluginBungee.class); + when(plugin.getDataFolder()).thenReturn(tempDir.toFile()); + Files.writeString(tempDir.resolve("votecache.json"), """ + {"OnlineCache":{"player-uuid":{"0":{"Name":"Player","Service":"Service","UUID":"player-uuid","Time":100}, + "1":{"Name":"Player","Service":"Service","UUID":"player-uuid","Time":100}}}} + """); + BungeeJsonVoteCache durableStorage = new BungeeJsonVoteCache(plugin); + VoteCacheHandler durableHandler = newVerifyingHandler(durableStorage); + + durableHandler.load(); + OfflineBungeeVote first = durableHandler.getOnlineVotes("player-uuid").stream() + .filter(vote -> "0".equals(vote.getOnlineVoteCacheJsonKey())).findFirst().orElseThrow(); + OfflineBungeeVote second = durableHandler.getOnlineVotes("player-uuid").stream() + .filter(vote -> "1".equals(vote.getOnlineVoteCacheJsonKey())).findFirst().orElseThrow(); + + assertEquals(2, durableHandler.getOnlineVotes("player-uuid").size()); + assertTrue(durableHandler.tryRemoveOnlineVote("player-uuid", first)); + + assertFalse(durableStorage.getOnlineVotes("player-uuid").contains("0")); + assertTrue(durableStorage.getOnlineVotes("player-uuid").contains("1")); + assertEquals("1", second.getOnlineVoteCacheJsonKey()); + } + @Test public void identicalLegacyTimedSqlRowsUsePrimaryKeysForDistinctStableIds() { VoteTimeQueue first = legacyTimedVote(); From 325eb3bf6b2933673a0b9b5f91c61e4888e37bf2 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 08:35:08 -0600 Subject: [PATCH 079/136] Keep distinct legacy cross-store rows --- .../proxy/cache/VoteCacheHandler.java | 34 +++++-------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index eced8926b..348f607d2 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -570,36 +570,23 @@ private boolean containsOnlineVote(String uuid, OfflineBungeeVote candidate) { } private OfflineBungeeVote findServerVoteSqlTwin(String server, OfflineBungeeVote emergency) { + if (emergency.getVoteId() == null) return null; for (OfflineBungeeVote candidate : getVotes(server)) { - if (candidate.getServerVoteCacheRowId() <= 0 || candidate.getServerVoteCacheJsonKey() != null) continue; - if (candidate.getVoteId() != null || emergency.getVoteId() != null) { - if (candidate.getVoteId() != null && candidate.getVoteId().equals(emergency.getVoteId())) return candidate; - continue; - } - if (matchesLegacyVote(candidate, emergency)) return candidate; + if (candidate.getServerVoteCacheRowId() > 0 && candidate.getServerVoteCacheJsonKey() == null + && emergency.getVoteId().equals(candidate.getVoteId())) return candidate; } return null; } private OfflineBungeeVote findOnlineVoteSqlTwin(String uuid, OfflineBungeeVote emergency) { + if (emergency.getVoteId() == null) return null; for (OfflineBungeeVote candidate : getOnlineVotes(uuid)) { - if (candidate.getOnlineVoteCacheRowId() <= 0 || candidate.getOnlineVoteCacheJsonKey() != null) continue; - if (candidate.getVoteId() != null || emergency.getVoteId() != null) { - if (candidate.getVoteId() != null && candidate.getVoteId().equals(emergency.getVoteId())) return candidate; - continue; - } - if (matchesLegacyVote(candidate, emergency)) return candidate; + if (candidate.getOnlineVoteCacheRowId() > 0 && candidate.getOnlineVoteCacheJsonKey() == null + && emergency.getVoteId().equals(candidate.getVoteId())) return candidate; } return null; } - private boolean matchesLegacyVote(OfflineBungeeVote first, OfflineBungeeVote second) { - return Objects.equals(first.getUuid(), second.getUuid()) - && Objects.equals(first.getPlayerName(), second.getPlayerName()) - && Objects.equals(first.getService(), second.getService()) - && first.getTime() == second.getTime(); - } - /** * Reads a vote identifier using the current key and the legacy key. * @@ -901,16 +888,11 @@ private boolean containsTimeVote(VoteTimeQueue expected) { } private VoteTimeQueue findSqlEmergencyTwin(VoteTimeQueue emergency) { + if (emergency.getVoteId() == null) return null; for (VoteTimeQueue candidate : timeChangeQueue) { if (candidate == null || candidate.getTimedVoteCacheRowId() <= 0 || candidate.getTimedVoteCacheJsonKey() != null) continue; - if (candidate.getVoteId() != null || emergency.getVoteId() != null) { - if (candidate.getVoteId() != null && candidate.getVoteId().equals(emergency.getVoteId())) return candidate; - continue; - } - if (Objects.equals(candidate.getUuid(), emergency.getUuid()) - && Objects.equals(candidate.getService(), emergency.getService()) - && candidate.getTime() == emergency.getTime()) return candidate; + if (emergency.getVoteId().equals(candidate.getVoteId())) return candidate; } return null; } From dc0a0ccdc401bb00f423bcfc6d7220a3129e1339 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 08:41:39 -0600 Subject: [PATCH 080/136] Scope online cache keys by player --- .../proxy/bungee/BungeeJsonVoteCache.java | 15 +++++++-------- .../proxy/velocity/VelocityJsonVoteCache.java | 9 ++++----- .../tests/VoteCacheHandlerVoteIdTest.java | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java index 943c10cd6..413e0cb7c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java @@ -340,9 +340,13 @@ public void removeVote(String server, OfflineBungeeVote vote) { } } - @Override - public void removeOnlineVote(OfflineBungeeVote vote) { - Collection players = getPlayers(); + @Override + public void removeOnlineVote(OfflineBungeeVote vote) { + if (vote.getOnlineVoteCacheJsonKey() != null && vote.getUuid() != null) { + setString("OnlineCache." + vote.getUuid() + "." + vote.getOnlineVoteCacheJsonKey(), null); + return; + } + Collection players = getPlayers(); if (players == null) { return; } @@ -352,11 +356,6 @@ public void removeOnlineVote(OfflineBungeeVote vote) { continue; } for (String num : onlineVotes) { - if (vote.getOnlineVoteCacheJsonKey() != null - && vote.getOnlineVoteCacheJsonKey().equals(num)) { - setString("OnlineCache." + player + "." + num, null); - return; - } GsonDataNode node = getOnlineVotes(player, num); if (node == null) { continue; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java index c9ac77b06..16e176ef1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java @@ -351,6 +351,10 @@ public void removeVote(String server, OfflineBungeeVote vote) { @Override public void removeOnlineVote(OfflineBungeeVote vote) { + if (vote.getOnlineVoteCacheJsonKey() != null && vote.getUuid() != null) { + remove("OnlineCache", vote.getUuid(), vote.getOnlineVoteCacheJsonKey()); + return; + } Collection players = getPlayers(); if (players == null) { return; @@ -361,11 +365,6 @@ public void removeOnlineVote(OfflineBungeeVote vote) { continue; } for (String num : onlineVotes) { - if (vote.getOnlineVoteCacheJsonKey() != null - && vote.getOnlineVoteCacheJsonKey().equals(num)) { - remove("OnlineCache", player, num); - return; - } ConfigDataNode node = getOnlineVotes(player, num); if (node == null) { continue; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index a0f4962b9..26330558f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -544,6 +544,25 @@ public void identicalLegacyOnlineJsonRowsKeepDistinctEntryIdentitiesAndDeleteInd assertEquals("1", second.getOnlineVoteCacheJsonKey()); } + @Test + public void onlineJsonEntryKeysAreScopedToTheirPlayer() throws Exception { + VotingPluginBungee plugin = mock(VotingPluginBungee.class); + when(plugin.getDataFolder()).thenReturn(tempDir.toFile()); + Files.writeString(tempDir.resolve("votecache.json"), """ + {"OnlineCache":{"player-a":{"0":{"Name":"A","Service":"Service","UUID":"player-a","Time":100}}, + "player-b":{"0":{"Name":"B","Service":"Service","UUID":"player-b","Time":100}}}} + """); + BungeeJsonVoteCache durableStorage = new BungeeJsonVoteCache(plugin); + VoteCacheHandler durableHandler = newVerifyingHandler(durableStorage); + durableHandler.load(); + OfflineBungeeVote playerB = durableHandler.getOnlineVotes("player-b").get(0); + + assertTrue(durableHandler.tryRemoveOnlineVote("player-b", playerB)); + + assertTrue(durableStorage.getOnlineVotes("player-a").contains("0")); + assertFalse(durableStorage.getOnlineVotes("player-b").contains("0")); + } + @Test public void identicalLegacyTimedSqlRowsUsePrimaryKeysForDistinctStableIds() { VoteTimeQueue first = legacyTimedVote(); From b7ab91fc8b7d5c43852d13134846c97ac4ee6cb2 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 08:51:16 -0600 Subject: [PATCH 081/136] Preserve distinct legacy timed cache rows --- .../proxy/cache/VoteCacheHandler.java | 7 ++--- .../tests/VoteCacheHandlerVoteIdTest.java | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index 348f607d2..eeb508cea 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -1404,8 +1404,9 @@ public void load() { /** * Loads entries written to the JSON emergency journal while MySQL was - * unavailable. Entries already present in SQL are de-duplicated by vote ID (or - * the legacy UUID/service/time identity). + * unavailable. Entries already present in SQL are de-duplicated only by vote + * ID. Legacy entries without an ID remain distinct because equal fields cannot + * prove that two durable rows represent the same vote. */ private void loadJsonEmergencyVotes() { if (jsonStorage == null) { @@ -1428,7 +1429,7 @@ private void loadJsonEmergencyVotes() { // later state updates and deletion clean up both copies without // collapsing separate identical rows. sqlTwin.setTimedVoteCacheJsonKey(key); - } else if (!containsTimeVote(vote)) { + } else { timeChangeQueue.add(vote); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 26330558f..f039e7b36 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -494,6 +494,33 @@ public void identicalLegacyTimedJsonRowsKeepDistinctIdsAndDeleteIndividually() t assertEquals(secondId.toString(), durableStorage.getTimedVoteCache("1").get("VoteId").asString()); } + @Test + public void legacySqlAndJsonTimedRowsRemainDistinctWithoutVoteIds() throws Exception { + DataNode jsonVote = mock(DataNode.class); + when(storage.getTimedVoteCache()).thenReturn(List.of("2")); + when(storage.getTimedVoteCache("2")).thenReturn(jsonVote); + when(storage.getServers()).thenReturn(Collections.emptyList()); + when(storage.getPlayers()).thenReturn(Collections.emptyList()); + when(jsonVote.isObject()).thenReturn(true); + stubString(jsonVote, "Name", "Player"); + stubString(jsonVote, "Service", "Service"); + stubLong(jsonVote, "Time", 100L); + + VoteTimeQueue sqlVote = legacyTimedVote(); + sqlVote.setTimedVoteCacheRowId(41); + handler.getTimeChangeQueue().add(sqlVote); + var loadEmergencyVotes = VoteCacheHandler.class.getDeclaredMethod("loadJsonEmergencyVotes"); + loadEmergencyVotes.setAccessible(true); + + loadEmergencyVotes.invoke(handler); + + assertEquals(2, handler.getTimeChangeQueue().size()); + VoteTimeQueue jsonLoaded = handler.getTimeChangeQueue().stream() + .filter(vote -> "2".equals(vote.getTimedVoteCacheJsonKey())).findFirst().orElseThrow(); + assertNull(jsonLoaded.getVoteId()); + assertEquals(-1, jsonLoaded.getTimedVoteCacheRowId()); + } + @Test public void identicalLegacyServerJsonRowsKeepDistinctEntryIdentitiesAndDeleteIndividually() throws Exception { VotingPluginBungee plugin = mock(VotingPluginBungee.class); From ff3665842e76583573730542feb1eb4b0d7ed79d Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 09:10:49 -0600 Subject: [PATCH 082/136] Keep unpaired legacy JSON cache rows --- .../proxy/cache/VoteCacheHandler.java | 16 ++++++- .../tests/VoteCacheHandlerVoteIdTest.java | 48 +++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index eeb508cea..ad68de174 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -447,8 +447,14 @@ public synchronized boolean tryRemoveOnlineVote(String uuid, OfflineBungeeVote r } boolean mysqlRemoved = !useMySQL || onlineVoteCacheTable.tryRemoveVote(removedVote); boolean jsonRemoved = jsonStorage == null; - if (jsonStorage != null && !jsonStorageQuarantined) { + if (jsonStorage != null && !jsonStorageQuarantined + && (!useMySQL || removedVote.getOnlineVoteCacheRowId() <= 0 + || removedVote.getOnlineVoteCacheJsonKey() != null)) { jsonRemoved = removeJsonOnlineVoteDurably(uuid, removedVote); + } else if (jsonStorage != null && !jsonStorageQuarantined) { + // An unpaired SQL row has no exact JSON identity. Equal legacy fields do + // not prove that a JSON row is its emergency twin. + jsonRemoved = true; } boolean removedDurably = mysqlRemoved && jsonRemoved; if (!removedDurably) return false; @@ -1755,8 +1761,14 @@ public synchronized void removeServerVotes(String server, ArrayList(List.of(sqlVote))); + ProxyVoteCacheTable sql = mock(ProxyVoteCacheTable.class); + when(sql.tryRemoveVote(same(sqlVote), eq("server"))).thenReturn(true); + setPrivateField(handler, "useMySQL", true); + setPrivateField(handler, "voteCacheTable", sql); + + handler.removeServerVotes("server", new ArrayList<>(List.of(sqlVote))); + + verify(storage, never()).removeVote(eq("server"), same(sqlVote)); + assertTrue(handler.getVotes("server").isEmpty()); + } + + @Test + public void unpairedLegacyOnlineSqlRowDoesNotDeleteJsonRows() throws Exception { + OfflineBungeeVote sqlVote = vote(null, 100L); + sqlVote.setOnlineVoteCacheRowId(42); + cachedVotes(handler, "cachedOnlineVotes").put("player-uuid", new ArrayList<>(List.of(sqlVote))); + ProxyOnlineVoteCacheTable sql = mock(ProxyOnlineVoteCacheTable.class); + when(sql.tryRemoveVote(same(sqlVote))).thenReturn(true); + setPrivateField(handler, "useMySQL", true); + setPrivateField(handler, "onlineVoteCacheTable", sql); + + assertTrue(handler.tryRemoveOnlineVote("player-uuid", sqlVote)); + + verify(storage, never()).removeOnlineVote(same(sqlVote)); + assertTrue(handler.getOnlineVotes("player-uuid").isEmpty()); + } + @Test public void identicalLegacyTimedSqlRowsUsePrimaryKeysForDistinctStableIds() { VoteTimeQueue first = legacyTimedVote(); @@ -1135,6 +1169,20 @@ public void debug1(String msg) { }; } + @SuppressWarnings("unchecked") + private static Map> cachedVotes(VoteCacheHandler target, String fieldName) + throws Exception { + var field = VoteCacheHandler.class.getDeclaredField(fieldName); + field.setAccessible(true); + return (Map>) field.get(target); + } + + private static void setPrivateField(VoteCacheHandler target, String fieldName, Object value) throws Exception { + var field = VoteCacheHandler.class.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } + private static VoteCacheHandler newVerifyingHandler(IVoteCache storage) { return new VoteCacheHandler(null, false, false, null, false, storage) { @Override public void logInfo1(String msg) { } From c3924c55905de98d3eecaddc5c42825a29089ae0 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 09:27:03 -0600 Subject: [PATCH 083/136] Reuse retained HTTP listener on method revert --- .../votingplugin/proxy/VotingPluginProxy.java | 8 ++++++++ .../ProxyMethodConfigurationService.java | 18 +++++++++++------- .../ProxyMethodConfigurationServiceTest.java | 19 +++++++++++++++++++ 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 6fa9b8e78..0d7a1683f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -3823,6 +3823,14 @@ public synchronized void prepareHttpTransportChange(VotingPluginProxyConfig cand PREPARED_HTTP_TRANSPORTS.put(httpTransportPreparationKey(), prepared); } + /** Returns whether the live listener already implements this HTTP snapshot. */ + public synchronized boolean hasMatchingLiveHttpTransport(VotingPluginProxyConfig candidate) { + return candidate != null && httpTransportServer != null + && java.util.Objects.equals(liveHttpHost, candidate.getHttpHost()) + && liveHttpPort == candidate.getHttpPort() + && java.util.Objects.equals(liveHttpPublicEndpoint, candidate.getHttpPublicEndpoint()); + } + /** Cancels a candidate listener when configuration publication fails. */ public synchronized void cancelPreparedHttpTransportChange() { PreparedHttpTransport prepared = PREPARED_HTTP_TRANSPORTS.remove(httpTransportPreparationKey()); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java index 0975bcb1e..6a4bb1cae 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java @@ -86,28 +86,32 @@ private void validate(ProxyMethodConfiguration proposal, VotingPluginProxyConfig public void apply(ProxyMethodConfiguration proposal, String expectedRevision) throws IOException { if (expectedRevision == null) throw new StaleRevisionException(); - boolean prepareHttp = proposal.method() == BungeeMethod.HTTP && read().method() != BungeeMethod.HTTP; + boolean[] preparedHttp = { false }; try { proxy.getConfig().persistControlProxyMethod(proposal.method().name(), expectedRevision, latest -> { validate(proposal, latest); - if (prepareHttp) proxy.prepareHttpTransportChange(latest); + if (proposal.method() == BungeeMethod.HTTP + && !proxy.hasMatchingLiveHttpTransport(latest)) { + proxy.prepareHttpTransportChange(latest); + preparedHttp[0] = true; + } }); proxy.getConfig().verifyControlProxyRoutingInstalled(); } catch (VotingPluginProxyConfig.StaleControlRevisionException e) { - if (prepareHttp) proxy.cancelPreparedHttpTransportChange(); + if (preparedHttp[0]) proxy.cancelPreparedHttpTransportChange(); throw new StaleRevisionException(); } catch (IllegalArgumentException validation) { - if (prepareHttp) proxy.cancelPreparedHttpTransportChange(); + if (preparedHttp[0]) proxy.cancelPreparedHttpTransportChange(); throw validation; } catch (DurableFiles.PublishedException published) { - if (prepareHttp) proxy.cancelPreparedHttpTransportChange(); + if (preparedHttp[0]) proxy.cancelPreparedHttpTransportChange(); throw rollbackAfterFailure(published); } catch (IOException failure) { - if (prepareHttp) proxy.cancelPreparedHttpTransportChange(); + if (preparedHttp[0]) proxy.cancelPreparedHttpTransportChange(); throw failure; } catch (RuntimeException failure) { - if (prepareHttp) proxy.cancelPreparedHttpTransportChange(); + if (preparedHttp[0]) proxy.cancelPreparedHttpTransportChange(); throw rollbackAfterFailure(failure); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java index bb34e5700..aab0dbb5a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java @@ -6,6 +6,7 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -163,6 +164,24 @@ void preparesHttpListenerFromFreshSnapshotBeforePublishingMethod() throws Except verify(config).verifyControlProxyRoutingInstalled(); } + @Test + void reusesRetainedMatchingHttpListenerWhenRevertingDeferredMethod() throws Exception { + when(config.getBungeeMethod()).thenReturn("REDIS"); + VotingPluginProxyConfig fresh = validHttpConfig(); + when(proxy.hasMatchingLiveHttpTransport(fresh)).thenReturn(true); + doAnswer(invocation -> { + VotingPluginProxyConfig.ControlProxyMethodValidator validator = invocation.getArgument(2); + validator.validate(fresh); + return null; + }).when(config).persistControlProxyMethod(org.mockito.ArgumentMatchers.eq("HTTP"), + org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); + + service.apply(new ProxyMethodConfiguration(BungeeMethod.HTTP), service.read().revision()); + + verify(proxy, never()).prepareHttpTransportChange(fresh); + verify(config).verifyControlProxyRoutingInstalled(); + } + @Test void closesPreparedHttpListenerWhenPublicationVerificationFails() throws Exception { when(config.getBungeeMethod()).thenReturn("PLUGINMESSAGING"); From 98e8c797f9ace765f6359bf4ab9027fbf2b075b1 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 11:02:53 -0600 Subject: [PATCH 084/136] Retain HTTP listener settings for queue recovery --- .../votingplugin/proxy/VotingPluginProxy.java | 131 ++++++++++++++++-- .../tests/VotingPluginProxyTest.java | 44 +++++- 2 files changed, 160 insertions(+), 15 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 0d7a1683f..d17357c6a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -11,8 +11,12 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.Base64; import java.sql.SQLException; import java.time.Duration; @@ -200,6 +204,7 @@ public void run() { // poll armed while a replacement is deferred so state cleared after an ack (or // from the durable cache) cannot strand the old HTTP runtime indefinitely. private static final long HTTP_TRANSPORT_RECONCILIATION_POLL_MILLIS = 1_000L; + private static final String HTTP_RETAINED_LISTENER_MAGIC = "VotingPlugin HTTP retained listener v1"; private static final Map PREPARED_HTTP_TRANSPORTS = new ConcurrentHashMap<>(); @Getter @@ -224,6 +229,19 @@ public void run() { private String liveHttpHost; private String liveHttpPublicEndpoint; private int liveHttpPort; + private HttpListenerSettings retainedHttpStartupSettings; + + private static final class HttpListenerSettings { + private final String host; + private final int port; + private final String publicEndpoint; + + private HttpListenerSettings(String host, int port, String publicEndpoint) { + this.host = host; + this.port = port; + this.publicEndpoint = publicEndpoint; + } + } private static final class PreparedHttpTransport { private final HttpProxyTransportServer server; @@ -3470,12 +3488,14 @@ private void reloadRuntime(boolean restartControlServices) { } private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod configuredMethod) { - if (configuredMethod == BungeeMethod.HTTP && !hasChangedLiveHttpConfiguration()) { + if (configuredMethod == BungeeMethod.HTTP && httpTransportServer != null + && !hasChangedLiveHttpConfiguration()) { deferredHttpTransportReconciliation = false; return configuredMethod; } HttpProxyTransportServer transport = httpTransportServer; if (transport != null && httpTransportHasPendingDeliveries(transport)) { + persistLiveHttpListenerSettings(); deferredHttpTransportReconciliation = true; logSevere("Retaining HTTP transport until durable deliveries are acknowledged"); return BungeeMethod.HTTP; @@ -3484,6 +3504,7 @@ private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod co try { if (httpQueueHasPersistedDeliveries( getDataFolderPlugin().toPath().resolve("http").resolve("outgoing-v1"))) { + retainedHttpStartupSettings = loadRetainedHttpListenerSettingsUnchecked(); deferredHttpTransportReconciliation = true; logSevere("Retaining HTTP transport until persisted deliveries are acknowledged"); return BungeeMethod.HTTP; @@ -3491,12 +3512,15 @@ private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod co } catch (IOException unreadableQueue) { // An unreadable durable queue is not proof that it is empty. Reopen HTTP so // its normal bounded loader can validate or recover the state. + retainedHttpStartupSettings = loadRetainedHttpListenerSettingsUnchecked(); deferredHttpTransportReconciliation = true; logSevere("Retaining HTTP transport because its persisted delivery queue could not be inspected"); return BungeeMethod.HTTP; } } if (hasPendingCachedHttpDeliveries()) { + if (transport != null) persistLiveHttpListenerSettings(); + else retainedHttpStartupSettings = loadRetainedHttpListenerSettingsUnchecked(); deferredHttpTransportReconciliation = true; logSevere("Retaining HTTP transport until cached deliveries are acknowledged"); return BungeeMethod.HTTP; @@ -3506,6 +3530,8 @@ private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod co for (String server : servers) { Collection rewards = getVoteCachePendingVotePartyRewardIds(server); if (rewards != null && !rewards.isEmpty()) { + if (transport != null) persistLiveHttpListenerSettings(); + else retainedHttpStartupSettings = loadRetainedHttpListenerSettingsUnchecked(); deferredHttpTransportReconciliation = true; logSevere("Retaining HTTP transport until pending vote-party rewards are acknowledged"); return BungeeMethod.HTTP; @@ -3513,9 +3539,88 @@ private synchronized BungeeMethod retainHttpForPendingDeliveries(BungeeMethod co } } deferredHttpTransportReconciliation = false; + retainedHttpStartupSettings = null; + if (configuredMethod != BungeeMethod.HTTP || hasChangedLiveHttpConfiguration()) { + clearRetainedHttpListenerSettings(); + } return configuredMethod; } + private Path retainedHttpListenerSettingsPath() { + return getDataFolderPlugin().toPath().resolve("http").resolve("retained-listener-v1.bin"); + } + + private void persistLiveHttpListenerSettings() { + if (liveHttpHost == null || liveHttpPublicEndpoint == null || liveHttpPort <= 0) return; + try { + persistRetainedHttpListenerSettings( + new HttpListenerSettings(liveHttpHost, liveHttpPort, liveHttpPublicEndpoint)); + } catch (IOException failure) { + throw new IllegalStateException("HTTP listener settings could not be retained for queued deliveries", failure); + } + } + + private void persistRetainedHttpListenerSettings(HttpListenerSettings settings) throws IOException { + Path target = retainedHttpListenerSettingsPath(); + Files.createDirectories(target.getParent()); + ByteArrayOutputStream encoded = new ByteArrayOutputStream(); + try (DataOutputStream output = new DataOutputStream(encoded)) { + output.writeUTF(HTTP_RETAINED_LISTENER_MAGIC); + output.writeUTF(settings.host); + output.writeInt(settings.port); + output.writeUTF(settings.publicEndpoint); + } + Path temporary = target.resolveSibling(target.getFileName() + "." + UUID.randomUUID() + ".tmp"); + try { + Files.write(temporary, encoded.toByteArray(), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); + try { + Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException unsupported) { + Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING); + } + } finally { + Files.deleteIfExists(temporary); + } + } + + private HttpListenerSettings loadRetainedHttpListenerSettingsUnchecked() { + try { + return loadRetainedHttpListenerSettings(); + } catch (IOException failure) { + throw new IllegalStateException("Retained HTTP listener settings could not be read", failure); + } + } + + private HttpListenerSettings loadRetainedHttpListenerSettings() throws IOException { + Path source = retainedHttpListenerSettingsPath(); + if (!Files.isRegularFile(source)) return null; + try (DataInputStream input = new DataInputStream(Files.newInputStream(source))) { + if (!HTTP_RETAINED_LISTENER_MAGIC.equals(input.readUTF())) { + throw new IOException("Unsupported retained HTTP listener settings"); + } + String host = input.readUTF(); + int port = input.readInt(); + String publicEndpoint = input.readUTF(); + if (input.read() != -1 || host.isBlank() || port < 1 || port > 65535) { + throw new IOException("Invalid retained HTTP listener settings"); + } + try { + validatedHttpEndpoint(publicEndpoint); + } catch (IllegalArgumentException invalid) { + throw new IOException("Invalid retained HTTP public endpoint", invalid); + } + return new HttpListenerSettings(host, port, publicEndpoint); + } + } + + private void clearRetainedHttpListenerSettings() { + try { + Files.deleteIfExists(retainedHttpListenerSettingsPath()); + } catch (IOException failure) { + logSevere("Unable to remove obsolete retained HTTP listener settings: " + failure.getMessage()); + } + } + private boolean hasChangedLiveHttpConfiguration() { return httpTransportServer != null && (!java.util.Objects.equals(liveHttpHost, getConfig().getHttpHost()) || liveHttpPort != getConfig().getHttpPort() @@ -3746,8 +3851,14 @@ protected synchronized boolean sendHttpEnvelope(String server, String deliveryId private void startHttpTransport() { try { + HttpListenerSettings startup = deferredHttpTransportReconciliation + ? retainedHttpStartupSettings : null; + if (startup == null) { + startup = new HttpListenerSettings(getConfig().getHttpHost(), getConfig().getHttpPort(), + getConfig().getHttpPublicEndpoint()); + } PreparedHttpTransport prepared = PREPARED_HTTP_TRANSPORTS.remove(httpTransportPreparationKey()); - if (prepared != null) { + if (prepared != null && retainedHttpStartupSettings == null) { if (!prepared.matches(getConfig())) { prepared.close(); throw new IllegalStateException("Prepared HTTP transport does not match the installed configuration"); @@ -3755,21 +3866,25 @@ private void startHttpTransport() { httpTransportServer = prepared.server; httpEnrollmentAuthority = prepared.authority; prepared.owner.set(this); + startup = new HttpListenerSettings(prepared.host, prepared.port, prepared.publicEndpoint); } else { - URI endpoint = validatedHttpEndpoint(getConfig().getHttpPublicEndpoint()); + if (prepared != null) prepared.close(); + URI endpoint = validatedHttpEndpoint(startup.publicEndpoint); File directory = new File(getDataFolderPlugin(), "http"); HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.toPath(), endpoint.getHost()); httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); httpTransportServer = new HttpProxyTransportServer( - new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, + new InetSocketAddress(startup.host, startup.port), identity, httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope, this::acknowledgeHttpDelivery); httpTransportServer.start(); } - liveHttpHost = getConfig().getHttpHost(); - liveHttpPort = getConfig().getHttpPort(); - liveHttpPublicEndpoint = getConfig().getHttpPublicEndpoint(); - logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" + persistRetainedHttpListenerSettings(startup); + liveHttpHost = startup.host; + liveHttpPort = startup.port; + liveHttpPublicEndpoint = startup.publicEndpoint; + retainedHttpStartupSettings = null; + logInfo("HTTP transport listening securely on " + liveHttpHost + ":" + httpTransportServer.port() + "; use /votingpluginproxy httpcode for each backend"); } catch (Exception failure) { closeHttpTransport(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 1f852dfee..4e69ef12b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -1715,18 +1715,36 @@ void concurrentManualReloadInvalidatesQueuedDeferredHttpReload() throws Exceptio } @Test - void startupRetainsHttpForPersistedQueueWithoutLiveServer() throws Exception { + void startupRetainsHttpForPersistedQueueWithItsLastListenerSettings() throws Exception { java.nio.file.Path backendQueue = temporaryDirectory.resolve("http/outgoing-v1/lobby-1"); java.nio.file.Files.createDirectories(backendQueue); java.nio.file.Files.writeString(backendQueue.resolve(".pending-delivery.json"), "pending"); - VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); - Mockito.doReturn(temporaryDirectory.toFile()).when(spyProxy).getDataFolderPlugin(); - spyProxy.setMethod(BungeeMethod.HTTP); - Mockito.when(spyProxy.getConfig().getBungeeMethod()).thenReturn("REDIS"); + setProxyField(votingPluginProxy, "httpTransportServer", Mockito.mock(HttpProxyTransportServer.class)); + setProxyField(votingPluginProxy, "liveHttpHost", "127.0.0.1"); + setProxyField(votingPluginProxy, "liveHttpPort", 8080); + setProxyField(votingPluginProxy, "liveHttpPublicEndpoint", "https://old.example.test:8080"); + votingPluginProxy.setPendingHttpTransportDeliveries(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + Mockito.when(votingPluginProxy.getConfig().getBungeeMethod()).thenReturn("REDIS"); + votingPluginProxy.reloadFromControl(); + assertTrue(java.nio.file.Files.isRegularFile( + temporaryDirectory.resolve("http/retained-listener-v1.bin"))); - spyProxy.reloadFromControl(); + VotingPluginProxyTestImpl restarted = new VotingPluginProxyTestImpl(); + restarted.setDataFolder(temporaryDirectory.toFile()); + restarted.setMethod(BungeeMethod.HTTP); + Mockito.when(restarted.getConfig().getBungeeMethod()).thenReturn("REDIS"); + Mockito.when(restarted.getConfig().getHttpHost()).thenReturn(""); + Mockito.when(restarted.getConfig().getHttpPort()).thenReturn(0); + Mockito.when(restarted.getConfig().getHttpPublicEndpoint()).thenReturn(""); - assertEquals(BungeeMethod.HTTP, spyProxy.getMethod()); + restarted.reloadFromControl(); + + assertEquals(BungeeMethod.HTTP, restarted.getMethod()); + Object retained = getProxyField(restarted, "retainedHttpStartupSettings"); + assertEquals("127.0.0.1", getField(retained, "host")); + assertEquals(8080, getField(retained, "port")); + assertEquals("https://old.example.test:8080", getField(retained, "publicEndpoint")); } @Test @@ -2832,4 +2850,16 @@ private static void setProxyField(VotingPluginProxy target, String name, Object field.setAccessible(true); field.set(target, value); } + + private static Object getProxyField(VotingPluginProxy target, String name) throws Exception { + java.lang.reflect.Field field = VotingPluginProxy.class.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } + + private static Object getField(Object target, String name) throws Exception { + java.lang.reflect.Field field = target.getClass().getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } } From fa9f1a582f856cb74c3a73aaccece7abfa5f57d5 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 11:25:12 -0600 Subject: [PATCH 085/136] Keep cache expiry and transport shutdown scoped --- .../presence/BackendPresenceManager.java | 3 ++- .../proxy/cache/VoteCacheHandler.java | 16 +++++++------ .../BackendProxyHandlerLifecycleTest.java | 23 +++++++++++++++++++ .../tests/VoteCacheHandlerVoteIdTest.java | 22 ++++++++++++++++++ 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java index 01903dac4..60331ee07 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java @@ -110,7 +110,8 @@ public void stopForDisable(long deadlineNanos) { // marshal the transactional stopped-presence send before returning the // result to that worker. Unit-test and shutdown contexts without a server // retain the direct path used by the non-Control lifecycle. - if (plugin != null && plugin.getServer() != null && !plugin.getServer().isPrimaryThread()) { + if (method == BungeeMethod.PLUGINMESSAGING && plugin != null && plugin.getServer() != null + && !plugin.getServer().isPrimaryThread()) { CompletableFuture scheduled = new CompletableFuture<>(); try { plugin.getBukkitScheduler().executeOrScheduleSync(plugin, () -> { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index ad68de174..a01d4f671 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -8,6 +8,7 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Collection; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.Map; import java.util.Map.Entry; @@ -540,20 +541,21 @@ public void checkVoteCacheTime(int voteCacheTime) { } removeOnlineVotes(expiredOnlineVotes); - // Collect expired server votes - ArrayList expiredServerVotes = new ArrayList<>(); + // Keep JSON entry keys scoped to their owning server. Numeric keys are reused + // independently below every VoteCache. node, so a combined removal + // list can otherwise delete an unrelated row with the same key. + Map> expiredServerVotes = new LinkedHashMap<>(); for (Entry> entry : cachedVotes.entrySet()) { - ArrayList votes = entry.getValue(); - for (OfflineBungeeVote vote : votes) { + for (OfflineBungeeVote vote : entry.getValue()) { if (vote.getTime() + (voteCacheTime * 24 * 60 * 60 * 1000L) < cTime) { debug1("Removing vote from cache: " + vote.toString()); - expiredServerVotes.add(vote); + expiredServerVotes.computeIfAbsent(entry.getKey(), ignored -> new ArrayList<>()).add(vote); } } } - for (String server : cachedVotes.keySet()) { - removeServerVotes(server, expiredServerVotes); + for (Entry> entry : expiredServerVotes.entrySet()) { + removeServerVotes(entry.getKey(), entry.getValue()); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 5ca0f1ccf..fdb6f843c 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -293,6 +293,29 @@ void preparedDisableSchedulesStoppedPresenceOnTheBukkitThread() throws Exception assertFalse(presence.isReporting()); } + @Test + void preparedDisableKeepsHttpStoppedPresenceOffTheBukkitThread() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + org.bukkit.Server server = mock(org.bukkit.Server.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + when(plugin.getServer()).thenReturn(server); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + when(server.isPrimaryThread()).thenReturn(false); + + BackendPresenceManager presence = new BackendPresenceManager(plugin, BungeeMethod.HTTP, messages); + setField(presence, "reporting", true); + setField(presence, "server", "backend-1"); + setField(presence, "incarnationId", java.util.UUID.randomUUID()); + setField(presence, "startedAt", 1L); + + presence.stopForDisable(); + + verify(scheduler, never()).executeOrScheduleSync(eq(plugin), any(Runnable.class)); + verify(messages).sendMessage(any()); + assertFalse(presence.isReporting()); + } + @Test void failedPreparedDisableCanRestartPresenceDuringRollback() throws Exception { BackendProxyHandler handler = new BackendProxyHandler(null); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index d517336e3..7bd36c07f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -548,6 +548,28 @@ public void identicalLegacyServerJsonRowsKeepDistinctEntryIdentitiesAndDeleteInd assertEquals("1", second.getServerVoteCacheJsonKey()); } + @Test + public void expirationDoesNotReuseAJsonEntryKeyAcrossServers() throws Exception { + VotingPluginBungee plugin = mock(VotingPluginBungee.class); + when(plugin.getDataFolder()).thenReturn(tempDir.toFile()); + long expired = System.currentTimeMillis() - java.util.concurrent.TimeUnit.DAYS.toMillis(2); + long current = System.currentTimeMillis(); + Files.writeString(tempDir.resolve("votecache.json"), """ + {"VoteCache":{"server-a":{"0":{"Name":"A","Service":"Service","UUID":"player-a","Time":%d}}, + "server-b":{"0":{"Name":"B","Service":"Service","UUID":"player-b","Time":%d}}}} + """.formatted(expired, current)); + BungeeJsonVoteCache durableStorage = new BungeeJsonVoteCache(plugin); + VoteCacheHandler durableHandler = newVerifyingHandler(durableStorage); + durableHandler.load(); + + durableHandler.checkVoteCacheTime(1); + + assertTrue(durableStorage.getServerVotes("server-a") == null + || durableStorage.getServerVotes("server-a").isEmpty()); + assertTrue(durableStorage.getServerVotes("server-b").contains("0")); + assertEquals(1, durableHandler.getVotes("server-b").size()); + } + @Test public void identicalLegacyOnlineJsonRowsKeepDistinctEntryIdentitiesAndDeleteIndividually() throws Exception { VotingPluginBungee plugin = mock(VotingPluginBungee.class); From 8086691cb623721b29908a7d677f282d804a91a5 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:40:52 -0600 Subject: [PATCH 086/136] Preserve HTTP handoff queue during rollback --- .../transport/HttpBackendProxyTransport.java | 16 ++++++++++++---- .../HttpBackendProxyTransportTest.java | 18 ++++++++++++------ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 1c26a697c..485a19c2f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -123,7 +123,9 @@ HttpBackendProxyTransport recreatePrepared() { // Startup and handoff queues are one FIFO from the caller's perspective. // The handoff queue can still contain messages accepted by the previous // replacement, so rollback must carry it into the restored transport too. - restored.startupQueue.addAll(takeQueuedMessages()); + // Keep the combined queue in the asynchronously drained handoff lane: it can + // be larger than the connector's bounded startup queue. + restored.handoffQueue.addAll(takeQueuedMessages()); } restored.start(configuredDirectory, configuredServerId, configuredConnectionCode, configuredMessageHandler, configuredCredentialGeneration, configuredCredentialGeneration == null && retryInitialization, @@ -321,6 +323,7 @@ private void initialize(Path directory, String serverId, String configuredCode, } } if (discard) replacement.close(); + else startHandoffDrainIfNeeded(); } catch (Exception failure) { startupFailure = new IllegalStateException("Secure HTTP backend enrollment or connection failed", failure); plugin.getLogger().severe("Secure HTTP backend transport is unavailable; check the connection code and proxy endpoint"); @@ -568,7 +571,6 @@ void reservePreparedHandoffCapacity(int messages) { } void acceptHandoffMessages(java.util.List messages) { - Thread drain; synchronized (lifecycle) { if (closed) throw new IllegalStateException("HTTP replacement transport is closed"); if (messages.size() > MAX_HANDOFF_QUEUE - handoffQueue.size()) @@ -579,8 +581,14 @@ void acceptHandoffMessages(java.util.List messages) { handoffQueue.addAll(newer); awaitingPreparedHandoff = false; preparedHandoffReservation = 0; - if (handoffQueue.isEmpty()) return; - if (handoffWorker != null) return; + } + startHandoffDrainIfNeeded(); + } + + private void startHandoffDrainIfNeeded() { + Thread drain; + synchronized (lifecycle) { + if (closed || handoffQueue.isEmpty() || handoffWorker != null || connector == null) return; drain = new Thread(this::drainHandoffMessages, "VotingPlugin-HTTP-Backend-Handoff"); drain.setDaemon(true); handoffWorker = drain; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 4da906bd2..24c88bfd3 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -697,7 +697,7 @@ void preparedDrainPreservesStartupAndExistingHandoffMessagesInFifoOrder() throws @Test @SuppressWarnings("unchecked") - void rollbackRecreateCarriesExistingHandoffMessagesAfterStartupMessages() throws Exception { + void rollbackRecreateKeepsAnOversizedCombinedQueueInTheDrainedHandoffLane() throws Exception { VotingPluginMain plugin = mock(VotingPluginMain.class); when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); HttpBackendProxyTransport transport = new HttpBackendProxyTransport(plugin); @@ -707,10 +707,15 @@ void rollbackRecreateCarriesExistingHandoffMessagesAfterStartupMessages() throws setField(transport, "configuredConnectionCode", ""); setField(transport, "configuredMessageHandler", mock(GlobalMessageHandler.class)); setField(transport, "restoreUnenrolledState", true); + List queued = new ArrayList<>(); JsonEnvelope startup = JsonEnvelope.builder("startup").build(); - JsonEnvelope handoff = JsonEnvelope.builder("handoff").build(); + queued.add(startup); ((java.util.ArrayDeque) field(transport, "startupQueue")).add(startup); - ((java.util.ArrayDeque) field(transport, "handoffQueue")).add(handoff); + for (int index = 0; index < 1024; index++) { + JsonEnvelope handoff = JsonEnvelope.builder("handoff-" + index).build(); + queued.add(handoff); + ((java.util.ArrayDeque) field(transport, "handoffQueue")).add(handoff); + } java.lang.reflect.Field ownersField = HttpBackendProxyTransport.class.getDeclaredField("DIRECTORY_OWNERS"); ownersField.setAccessible(true); @@ -722,9 +727,10 @@ void rollbackRecreateCarriesExistingHandoffMessagesAfterStartupMessages() throws Thread setup = null; try { restored = transport.recreatePrepared(); - assertEquals(List.of(startup, handoff), - new java.util.ArrayList<>((java.util.ArrayDeque) field(restored, "startupQueue"))); - assertTrue(((java.util.ArrayDeque) field(restored, "handoffQueue")).isEmpty()); + assertTrue(((java.util.ArrayDeque) field(restored, "startupQueue")).isEmpty(), + "rollback must not synchronously overfill the connector's 1024-entry queue"); + assertEquals(queued, + new java.util.ArrayList<>((java.util.ArrayDeque) field(restored, "handoffQueue"))); assertTrue(((java.util.ArrayDeque) field(transport, "startupQueue")).isEmpty()); assertTrue(((java.util.ArrayDeque) field(transport, "handoffQueue")).isEmpty()); setup = (Thread) field(restored, "worker"); From 19ed58be4e27523fe5501efc5636bf547b08ffab Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:03:20 -0600 Subject: [PATCH 087/136] Keep staged Redis cleanup off the server thread --- VotingPlugin/pom.xml | 6 ++++ .../votingplugin/VotingPluginMain.java | 9 ++++-- ...PluginMainBackendProxyPublicationTest.java | 29 +++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/VotingPlugin/pom.xml b/VotingPlugin/pom.xml index a84f2902b..e175d7540 100644 --- a/VotingPlugin/pom.xml +++ b/VotingPlugin/pom.xml @@ -201,6 +201,12 @@ false false + + org.bouncycastle:* + + META-INF/versions/25/** + + *:* diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index d4c26cbde..29cdfb2d0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1415,7 +1415,8 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { throw handoffFailure; } } - if (restart.redisHandoffCompleted) closeStagedRedisReplacementAsync(restart.replacement); + if (requiresAsyncStagedReplacementClose(restart)) + closeStagedRedisReplacementAsync(restart.replacement); else try { restart.replacement.close(); } catch (RuntimeException closeFailure) { @@ -1509,7 +1510,7 @@ public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart res } if (restart.replacement != null) { restart.replacement.abortStagedInboundTo(restart.previous); - if (restart.redisHandoffCompleted) closeStagedRedisReplacementAsync(restart.replacement); + if (requiresAsyncStagedReplacementClose(restart)) closeStagedRedisReplacementAsync(restart.replacement); else restart.replacement.close(); } if (backendProxyHandler == restart.previous && restart.previous != null @@ -1524,6 +1525,10 @@ public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart res restart.finished = true; } + private boolean requiresAsyncStagedReplacementClose(BackendProxyRestart restart) { + return restart.redisHandoffCompleted || restart.replacement.getMethod() == BungeeMethod.REDIS; + } + /** A staged Redis listener can spend its bounded join timeout in close(); abort runs on Bukkit. */ private void closeStagedRedisReplacementAsync(BackendProxyHandler replacement) { Thread cleanup = new Thread(() -> { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index 137b0d792..c0e3dae61 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -284,6 +284,35 @@ void stagedRedisAbortFencesOnBukkitButClosesTheReplacementOffThread() throws Exc verify(replacement, org.mockito.Mockito.timeout(1_000)).close(); } + @Test + void stagedRedisAbortBeforeHandoffClosesTheReplacementOffThread() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + when(replacement.getMethod()).thenReturn(BungeeMethod.REDIS); + setBackendProxyHandler(plugin, previous); + VotingPluginMain.BackendProxyRestart restart = restart(previous, replacement); + CountDownLatch closeStarted = new CountDownLatch(1); + CountDownLatch releaseClose = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference closeThread = new java.util.concurrent.atomic.AtomicReference<>(); + doAnswer(invocation -> { + closeThread.set(Thread.currentThread().getName()); + closeStarted.countDown(); + releaseClose.await(1, TimeUnit.SECONDS); + return null; + }).when(replacement).close(); + + try { + org.junit.jupiter.api.Assertions.assertTimeoutPreemptively(java.time.Duration.ofMillis(250), + () -> plugin.abortBackendProxyHandlerRestart(restart)); + assertTrue(closeStarted.await(1, TimeUnit.SECONDS)); + assertTrue(closeThread.get().startsWith("VotingPlugin-Staged-Redis-Rollback")); + } finally { + releaseClose.countDown(); + } + verify(replacement, org.mockito.Mockito.timeout(1_000)).close(); + } + @Test void restoresPreviousHandlerWhenPreparedDeliveryAdmissionFails() throws Exception { VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); From 48185a74a6f62a70515e1d30b9b64ecf39bb91e9 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:21:47 -0600 Subject: [PATCH 088/136] Harden transport retirement and durable proxy state --- .../votingplugin/VotingPluginMain.java | 14 +++------ .../votingplugin/proxy/VotingPluginProxy.java | 8 ++--- .../proxy/cache/VoteCacheHandler.java | 1 + ...PluginMainBackendProxyPublicationTest.java | 31 ++++++++++++++++++- .../tests/VoteCacheHandlerVoteIdTest.java | 23 ++++++++++++++ .../tests/VotingPluginProxyTest.java | 24 ++++++++++++++ 6 files changed, 87 insertions(+), 14 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 29cdfb2d0..6536d1d96 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1446,17 +1446,13 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { } } - /** Retires Redis predecessors asynchronously so listener joins never block Bukkit publication. */ + /** Retires network-backed predecessors without blocking the Bukkit publication callback. */ void closePublishedPreviousBackendProxyHandler(BackendProxyHandler previous, String phase) { if (previous == null) return; - if (previous.requiresRedisRetirement()) { - Thread cleanup = new Thread(() -> closePublishedPreviousBackendProxyHandlerNow(previous, phase), - "VotingPlugin-Retired-Redis-Backend"); - cleanup.setDaemon(true); - cleanup.start(); - return; - } - closePublishedPreviousBackendProxyHandlerNow(previous, phase); + Thread cleanup = new Thread(() -> closePublishedPreviousBackendProxyHandlerNow(previous, phase), + "VotingPlugin-Retired-Backend"); + cleanup.setDaemon(true); + cleanup.start(); } private void closePublishedPreviousBackendProxyHandlerNow(BackendProxyHandler previous, String phase) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index d17357c6a..95143e73f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -3376,6 +3376,10 @@ public synchronized void processQueue() { return; } } + if (vote.isProcessed() && vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { + scheduleTimeVoteRetry(); + return; + } if (vote.isProcessed() && vote.isMultiProxyForwardingRequired() && !vote.isMultiProxyForwardingHandled()) { if (!retryDurableMultiProxyOutbox(vote)) { @@ -3383,10 +3387,6 @@ public synchronized void processQueue() { return; } } - if (vote.isProcessed() && vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { - scheduleTimeVoteRetry(); - return; - } // A direct listener retry can still be queued after its ACK outbox completes. // Keep that in-memory fence until the listener consumes it; queued-vote // processing removes its own fence in vote() when no listener retry exists. diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index a01d4f671..9ee688d4f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -930,6 +930,7 @@ public synchronized boolean updateTimeVote(VoteTimeQueue vote) { if (timedVoteCacheTable.updateTimedVote(vote)) { return !hasJsonTimeVote(vote) || updateTimeVoteJson(vote); } + if (vote.getTimedVoteCacheRowId() > 0) return false; // A timed vote can have been admitted to the JSON emergency journal when // its initial SQL insert failed. Keep delivery-state ACKs durable there // until the SQL row is available again. diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index c0e3dae61..9c7c9e1fc 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -150,7 +150,33 @@ void preparesRedisBeforePublishingAndRetiresItOffTheBukkitThread() throws Except plugin.closePublishedPreviousBackendProxyHandler(previous, "after publication"); assertTrue(closed.await(1, TimeUnit.SECONDS)); - assertTrue(closeThread.get().startsWith("VotingPlugin-Retired-Redis-Backend")); + assertTrue(closeThread.get().startsWith("VotingPlugin-Retired-Backend")); + } + + @Test + void retiresNonRedisNetworkTransportOffTheBukkitThread() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + when(previous.getMethod()).thenReturn(BungeeMethod.MQTT); + CountDownLatch closeStarted = new CountDownLatch(1); + CountDownLatch releaseClose = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference closeThread = new java.util.concurrent.atomic.AtomicReference<>(); + doAnswer(invocation -> { + closeThread.set(Thread.currentThread().getName()); + closeStarted.countDown(); + releaseClose.await(1, TimeUnit.SECONDS); + return null; + }).when(previous).close(); + + try { + org.junit.jupiter.api.Assertions.assertTimeoutPreemptively(java.time.Duration.ofMillis(250), + () -> plugin.closePublishedPreviousBackendProxyHandler(previous, "after publication")); + assertTrue(closeStarted.await(1, TimeUnit.SECONDS)); + assertTrue(closeThread.get().startsWith("VotingPlugin-Retired-Backend")); + } finally { + releaseClose.countDown(); + } + verify(previous, org.mockito.Mockito.timeout(1_000)).close(); } @Test @@ -351,6 +377,7 @@ void predecessorCleanupFailureDoesNotRollBackPublishedReplacement() throws Excep assertSame(replacement, plugin.getBackendProxyHandler()); assertFalse(plugin.requestBackendProxyHandlerRestartAbandonment(restart), "published runtime state must not be treated as rollbackable"); + verify(previous, org.mockito.Mockito.timeout(1_000)).close(); org.mockito.InOrder publication = org.mockito.Mockito.inOrder(previous, replacement); publication.verify(previous).completeHttpHandoff(replacement); publication.verify(replacement).activateInboundMessages(); @@ -376,6 +403,7 @@ void disablingBungeeCoordinationDrainsActiveHttpTransportBeforeClose() throws Ex plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + 1_000_000_000L); plugin.completeBackendProxyHandlerRestart(restart); + verify(previous, org.mockito.Mockito.timeout(1_000)).close(); org.mockito.InOrder disable = org.mockito.Mockito.inOrder(previous); disable.verify(previous).preparePresenceForDisable(org.mockito.ArgumentMatchers.anyLong()); disable.verify(previous).prepareForReplacement(org.mockito.ArgumentMatchers.isNull(), @@ -423,6 +451,7 @@ void disablingUnpreparedTransportStopsPresenceBeforeRejectingFurtherSends() thro plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + 1_000_000_000L); plugin.completeBackendProxyHandlerRestart(restart); + verify(previous, org.mockito.Mockito.timeout(1_000)).close(); org.mockito.InOrder disable = org.mockito.Mockito.inOrder(previous); disable.verify(previous).preparePresenceForDisable(); disable.verify(previous).commitPreparedDisable(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 7bd36c07f..c4068c609 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -42,6 +42,7 @@ import com.bencodez.votingplugin.proxy.cache.IVoteCache; import com.bencodez.votingplugin.proxy.cache.GsonDataNode; import com.bencodez.votingplugin.proxy.cache.ProxyOnlineVoteCacheTable; +import com.bencodez.votingplugin.proxy.cache.ProxyTimedVoteCacheTable; import com.bencodez.votingplugin.proxy.cache.ProxyVoteCacheTable; import com.bencodez.votingplugin.proxy.cache.VoteCacheHandler; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; @@ -674,6 +675,28 @@ public void timedVoteDeliveryStateIsUpdatedByVoteId() { verify(storage).save(); } + @Test + public void timedVoteSqlTwinRejectsJsonOnlyDeliveryUpdate() throws Exception { + UUID voteId = UUID.randomUUID(); + VoteTimeQueue queued = new VoteTimeQueue(voteId, "Player", "Service", 100L); + queued.setTimedVoteCacheRowId(42); + queued.setTimedVoteCacheJsonKey("2"); + DataNode stored = mock(DataNode.class); + when(stored.isObject()).thenReturn(true); + stubString(stored, "VoteId", voteId.toString()); + when(storage.getTimedVoteCache()).thenReturn(List.of("2")); + when(storage.getTimedVoteCache("2")).thenReturn(stored); + ProxyTimedVoteCacheTable sql = mock(ProxyTimedVoteCacheTable.class); + when(sql.updateTimedVote(queued)).thenReturn(false); + setPrivateField(handler, "useMySQL", true); + setPrivateField(handler, "timedVoteCacheTable", sql); + + assertFalse(handler.updateTimeVote(queued)); + + verify(sql).updateTimedVote(queued); + verify(storage, never()).addTimedVote(org.mockito.ArgumentMatchers.anyInt(), same(queued)); + } + @Test public void timedVoteAcknowledgementOutboxSurvivesHandlerRestart() { UUID voteId = UUID.randomUUID(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 4e69ef12b..9707fbb89 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -528,6 +528,30 @@ void completedAckOutboxWaitsForDurableFenceRetirementBeforeFinishing() throws Ex verify(voteCache).updateTimeVote(outbox); } + @Test + void completedAckOutboxPersistsAckProgressBeforeRequestingRetirement() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.UUID voteId = java.util.UUID.randomUUID(); + VoteTimeQueue outbox = new VoteTimeQueue(voteId, "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", true, + "00000000-0000-0000-0000-000000000001"); + outbox.requireMultiProxyAcknowledgements("Proxy1", java.util.Set.of("Proxy2")); + assertTrue(outbox.acknowledgeMultiProxyRecipient("Proxy2")); + outbox.setDeliveryStateDirty(true); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + queue.add(outbox); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.updateTimeVote(outbox)).thenReturn(false); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + spyProxy.processQueue(); + + verify(voteCache).updateTimeVote(outbox); + verify(multiProxyHandler, never()).requestMultiProxyVoteRetirement(Mockito.any(), Mockito.anyString(), + Mockito.anyString()); + } + @Test void receiverKeepsCompletionFenceUntilItsProcessedQueueRowIsRemoved() throws Exception { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); From de0d3f742ec1600b20ed22e90cc4a70b8491cfd0 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:39:34 -0600 Subject: [PATCH 089/136] Keep vote cache twins consistent --- .../proxy/cache/VoteCacheHandler.java | 6 +++ .../tests/VoteCacheHandlerVoteIdTest.java | 40 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index 9ee688d4f..6689f995b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -179,6 +179,9 @@ public synchronized boolean updateServerVote(String server, OfflineBungeeVote vo // recovery path. Keep that exact twin current before publishing again. return !hasJsonServerVote(vote, server) || updateServerVoteJson(server, vote); } + // Do not advance only an emergency JSON twin once this vote has a known + // SQL row. A later SQL retry could otherwise replay stale delivery state. + if (vote.getServerVoteCacheRowId() > 0) return false; return updateServerVoteJson(server, vote); } if (jsonStorage == null || jsonStorageQuarantined) return false; @@ -376,6 +379,9 @@ public synchronized boolean updateOnlineVote(String uuid, OfflineBungeeVote vote if (onlineVoteCacheTable.updateProxyBroadcastState(vote)) { return !hasJsonOnlineVote(vote, uuid) || updateOnlineVoteJson(uuid, vote); } + // Do not advance only an emergency JSON twin once this vote has a known + // SQL row. A later SQL retry could otherwise replay stale delivery state. + if (vote.getOnlineVoteCacheRowId() > 0) return false; return updateOnlineVoteJson(uuid, vote); } if (jsonStorage == null || jsonStorageQuarantined) return false; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index c4068c609..7fd9b866f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -697,6 +697,46 @@ public void timedVoteSqlTwinRejectsJsonOnlyDeliveryUpdate() throws Exception { verify(storage, never()).addTimedVote(org.mockito.ArgumentMatchers.anyInt(), same(queued)); } + @Test + public void serverVoteSqlTwinRejectsJsonOnlyDeliveryUpdate() throws Exception { + OfflineBungeeVote vote = vote(UUID.randomUUID(), 100L); + vote.setServerVoteCacheRowId(41); + vote.setServerVoteCacheJsonKey("4"); + DataNode stored = mock(DataNode.class); + when(stored.isObject()).thenReturn(true); + when(storage.getServerVotes("server")).thenReturn(List.of("4")); + when(storage.getServerVotes("server", "4")).thenReturn(stored); + ProxyVoteCacheTable sql = mock(ProxyVoteCacheTable.class); + when(sql.updateProxyBroadcastState(vote, "server")).thenReturn(false); + setPrivateField(handler, "useMySQL", true); + setPrivateField(handler, "voteCacheTable", sql); + + assertFalse(handler.updateServerVote("server", vote)); + + verify(sql).updateProxyBroadcastState(vote, "server"); + verify(storage, never()).addVote(eq("server"), org.mockito.ArgumentMatchers.anyInt(), same(vote)); + } + + @Test + public void onlineVoteSqlTwinRejectsJsonOnlyDeliveryUpdate() throws Exception { + OfflineBungeeVote vote = vote(UUID.randomUUID(), 100L); + vote.setOnlineVoteCacheRowId(42); + vote.setOnlineVoteCacheJsonKey("3"); + DataNode stored = mock(DataNode.class); + when(stored.isObject()).thenReturn(true); + when(storage.getOnlineVotes("player-uuid")).thenReturn(List.of("3")); + when(storage.getOnlineVotes("player-uuid", "3")).thenReturn(stored); + ProxyOnlineVoteCacheTable sql = mock(ProxyOnlineVoteCacheTable.class); + when(sql.updateProxyBroadcastState(vote)).thenReturn(false); + setPrivateField(handler, "useMySQL", true); + setPrivateField(handler, "onlineVoteCacheTable", sql); + + assertFalse(handler.updateOnlineVote("player-uuid", vote)); + + verify(sql).updateProxyBroadcastState(vote); + verify(storage, never()).addVoteOnline(eq("player-uuid"), org.mockito.ArgumentMatchers.anyInt(), same(vote)); + } + @Test public void timedVoteAcknowledgementOutboxSurvivesHandlerRestart() { UUID voteId = UUID.randomUUID(); From b5d049041dfae08cbbdca570d06c670e2edb1f2f Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:50:49 -0600 Subject: [PATCH 090/136] Prepare transport disable off the server thread --- .../votingplugin/VotingPluginMain.java | 18 ++++++++++-------- ...gPluginMainBackendProxyPublicationTest.java | 8 ++++++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 6536d1d96..909901864 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1303,15 +1303,19 @@ public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long validationDeadlineNanos) { if (restart == null) throw new IllegalArgumentException("Backend proxy restart is required"); + // Disabling always stops a live predecessor's presence transport. That stop + // can perform network/backend work for MQTT, MySQL, and socket transports + // even when they have no HTTP/plugin-message handoff to prepare. Keep it on + // the Control worker rather than falling through to Bukkit publication. + if (restart.disabled && restart.previous != null && !restart.presenceStoppedForDisablePreparation) { + restart.presenceStoppedForDisablePreparation = true; + restart.previous.preparePresenceForDisable(validationDeadlineNanos); + } if (restart.previousRequiresPreparation && !restart.previousPrepared) { // Preparation can close a retrying enrollment transport before a bounded // worker join fails. Mark the restart first so abort/await still owns the // restoration path after a partially completed preparation. restart.previousPrepared = true; - if (restart.disabled) { - restart.presenceStoppedForDisablePreparation = true; - restart.previous.preparePresenceForDisable(validationDeadlineNanos); - } BungeeMethod replacementMethod = restart.replacement == null ? null : restart.replacement.getMethod(); if (restart.replacement != null) restart.replacement.beginPreparedHttpHandoff(); if (!restart.previous.prepareForReplacement(replacementMethod, validationDeadlineNanos)) @@ -1370,10 +1374,8 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { } if (backendProxyHandler != restart.previous) throw new IllegalStateException("Backend proxy handler changed during restart"); if (restart.disabled) { - if (restart.previous != null && !restart.presenceStoppedForDisablePreparation) { - restart.presenceStoppedForDisablePreparation = true; - restart.previous.preparePresenceForDisable(); - } + if (restart.previous != null && !restart.presenceStoppedForDisablePreparation) + throw new IllegalStateException("Backend proxy disable must be prepared before Bukkit publication"); if (restart.previous != null && !restart.previous.commitPreparedDisable()) throw new IllegalStateException("Backend proxy transport accepted a delivery while disabling"); backendProxyHandler = null; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index 9c7c9e1fc..62b65ed50 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -438,12 +438,13 @@ void disablingKeepsPreviousHandlerWhenPreparationAcceptedAnotherDelivery() throw } @Test - void disablingUnpreparedTransportStopsPresenceBeforeRejectingFurtherSends() throws Exception { + void disablingMqttTransportPreparesPresenceOffTheBukkitPublicationPath() throws Exception { VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); BungeeSettings settings = mock(BungeeSettings.class); when(settings.isUseBungeecoord()).thenReturn(false); setField(plugin, "bungeeSettings", settings); BackendProxyHandler previous = mock(BackendProxyHandler.class); + when(previous.getMethod()).thenReturn(BungeeMethod.MQTT); when(previous.commitPreparedDisable()).thenReturn(true); setBackendProxyHandler(plugin, previous); VotingPluginMain.BackendProxyRestart restart = plugin.prepareBackendProxyHandlerRestart(); @@ -453,7 +454,10 @@ void disablingUnpreparedTransportStopsPresenceBeforeRejectingFurtherSends() thro verify(previous, org.mockito.Mockito.timeout(1_000)).close(); org.mockito.InOrder disable = org.mockito.Mockito.inOrder(previous); - disable.verify(previous).preparePresenceForDisable(); + disable.verify(previous).preparePresenceForDisable(org.mockito.ArgumentMatchers.anyLong()); + verify(previous, never()).preparePresenceForDisable(); + verify(previous, never()).prepareForReplacement(org.mockito.ArgumentMatchers.isNull(), + org.mockito.ArgumentMatchers.anyLong()); disable.verify(previous).commitPreparedDisable(); disable.verify(previous).close(); assertNull(plugin.getBackendProxyHandler()); From 17e16d01bac685881f2f4eea5fc615037e2638b5 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 20:53:49 -0600 Subject: [PATCH 091/136] Persist retained HTTP listener settings durably --- .../votingplugin/proxy/VotingPluginProxy.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 95143e73f..fe42d11eb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -11,11 +11,9 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; -import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.Base64; import java.sql.SQLException; @@ -94,6 +92,7 @@ import com.bencodez.votingplugin.proxy.presence.PlayerPresence; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; import com.bencodez.votingplugin.topvoter.TopVoter; +import com.bencodez.votingplugin.util.DurableFiles; import com.bencodez.votingplugin.util.MinecraftUsernameValidator; import com.bencodez.votingplugin.util.ServiceSiteValidator; import com.bencodez.votingplugin.votelog.VoteLogMysqlTable; @@ -3573,11 +3572,7 @@ private void persistRetainedHttpListenerSettings(HttpListenerSettings settings) Path temporary = target.resolveSibling(target.getFileName() + "." + UUID.randomUUID() + ".tmp"); try { Files.write(temporary, encoded.toByteArray(), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); - try { - Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - } catch (AtomicMoveNotSupportedException unsupported) { - Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING); - } + DurableFiles.publishStagedFile(temporary, target); } finally { Files.deleteIfExists(temporary); } @@ -3615,7 +3610,7 @@ private HttpListenerSettings loadRetainedHttpListenerSettings() throws IOExcepti private void clearRetainedHttpListenerSettings() { try { - Files.deleteIfExists(retainedHttpListenerSettingsPath()); + DurableFiles.deleteIfExists(retainedHttpListenerSettingsPath()); } catch (IOException failure) { logSevere("Unable to remove obsolete retained HTTP listener settings: " + failure.getMessage()); } From a29828bedce410fc516efc2f6a84d3d5f413ed5c Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 21:12:44 -0600 Subject: [PATCH 092/136] Retry rejected live HTTP broadcasts --- .../votingplugin/proxy/VotingPluginProxy.java | 26 +++++++++++++++-- .../tests/VotingPluginProxyTest.java | 28 +++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index fe42d11eb..ed86b7dae 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -3823,6 +3823,13 @@ private String stableCachedHttpDeliveryId(String purpose, String server, JsonEnv return UUID.nameUUIDFromBytes(key.getBytes(StandardCharsets.UTF_8)).toString(); } + /** Keeps a live vote's auxiliary broadcast idempotent across listener retries. */ + private String stableLiveHttpBroadcastDeliveryId(UUID voteId, String server) { + String key = "VotingPlugin:http-live-broadcast:v1\u0000" + voteId + "\u0000" + + server.toLowerCase(Locale.ROOT); + return UUID.nameUUIDFromBytes(key.getBytes(StandardCharsets.UTF_8)).toString(); + } + /** * Distinguishes pre-vote-ID cache rows that can otherwise share every visible * vote field. SQL primary keys and JSON entry keys are durable before this @@ -5433,10 +5440,23 @@ && getGlobalDataHandler().isTimeChangedHappened()) { if (getConfig().getBlockedServers().contains(targetServer)) { continue; } + if (broadcastForwardedServers.contains(targetServer)) { + continue; + } - globalMessageProxyHandler.sendMessage(targetServer, bDelay, - VotingPluginWire.voteBroadcast(uuid, player, service, time, - text == null ? "" : text.toString(), true)); + JsonEnvelope broadcast = VotingPluginWire.voteBroadcast(uuid, player, service, time, + text == null ? "" : text.toString(), true); + if (method == BungeeMethod.HTTP) { + String deliveryId = stableLiveHttpBroadcastDeliveryId(voteId, targetServer); + if (!sendStableHttpEnvelope(targetServer, deliveryId, broadcast)) { + retryState.broadcastForwardedServers.addAll(broadcastForwardedServers); + return QueuedVoteResult.RETRY; + } + broadcastForwardedServers.add(targetServer); + retryState.broadcastForwardedServers.add(targetServer); + } else { + globalMessageProxyHandler.sendMessage(targetServer, bDelay, broadcast); + } bDelay++; } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 9707fbb89..c0184b487 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -124,6 +124,34 @@ void rejectedRewardBearingAuxiliarySendRetainsItsStableDeliveryIdForRetry() { votingPluginProxy.getAttemptedVotePartyDeliveryIds()); } + @Test + void rejectedOnlineHttpBroadcastRetriesWithTheSameDeliveryId() throws Exception { + Mockito.when(votingPluginProxy.getConfig().getProxyBroadcastEnabled()).thenReturn(true); + votingPluginProxy.setStandaloneBroadcastForwarding(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + votingPluginProxy.setStableHttpDeliveryResults(true, false, true); + votingPluginProxy.setGlobalMessageProxyHandlerForTest( + Mockito.mock(com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler.class)); + java.lang.reflect.Field decider = VotingPluginProxy.class.getDeclaredField("proxyBroadcastDecider"); + decider.setAccessible(true); + decider.set(votingPluginProxy, new ProxyBroadcastDecider(votingPluginProxy::getConfig, + votingPluginProxy::getAllAvailableServers, server -> true, server -> false)); + java.util.UUID voteId = java.util.UUID.randomUUID(); + + assertThrows(VotingPluginProxy.VoteRetryException.class, + () -> votingPluginProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId)); + votingPluginProxy.vote("Player", "Service", true, false, 100L, null, + "00000000-0000-0000-0000-000000000001", voteId); + + assertEquals(3, votingPluginProxy.getAttemptedVotePartyDeliveryIds().size()); + assertFalse(votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(0) + .equals(votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(1))); + assertEquals(votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(1), + votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(2)); + } + @Test void cachedHttpVoteReusesItsDeterministicIdAfterRestart() { votingPluginProxy.setMethod(BungeeMethod.HTTP); From 12d328c9fcde7d70c722a48e00710909de63a3ab Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 21:27:29 -0600 Subject: [PATCH 093/136] Keep pending proxy ACKs from blocking votes --- .../votingplugin/proxy/VotingPluginProxy.java | 9 ++++--- .../tests/VotingPluginProxyTest.java | 25 +++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index ed86b7dae..dbcd69aa8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -3347,8 +3347,11 @@ private UUID parseUUIDFromString(String uuidAsString) { } public synchronized void processQueue() { - while (getVoteCacheHandler().getTimeChangeQueue().size() > 0) { - VoteTimeQueue vote = getVoteCacheHandler().getTimeChangeQueue().element(); + java.util.Queue timeChangeQueue = getVoteCacheHandler().getTimeChangeQueue(); + // Work from a bounded snapshot so an already-processed ACK outbox can remain + // durable without monopolizing the head of the rollover queue. + for (VoteTimeQueue vote : new ArrayList<>(timeChangeQueue)) { + if (!timeChangeQueue.contains(vote)) continue; if (vote.isMultiProxyCompletionPending()) { if (!getVoteCacheHandler().hasMultiProxyVoteCompletion(vote.getVoteId()) && !getVoteCacheHandler().markMultiProxyVoteCompletedDurably(vote.getVoteId())) { @@ -3383,7 +3386,7 @@ public synchronized void processQueue() { && !vote.isMultiProxyForwardingHandled()) { if (!retryDurableMultiProxyOutbox(vote)) { scheduleTimeVoteRetry(); - return; + continue; } } // A direct listener retry can still be queued after its ACK outbox completes. diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index c0184b487..0d183d548 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -526,6 +526,31 @@ void completedAckOutboxKeepsFenceForAlreadyScheduledListenerRetry() throws Excep assertTrue(retries.containsKey(voteId)); } + @Test + void pendingAckOutboxDoesNotBlockLaterCompletedQueueRows() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + VoteTimeQueue pendingAck = new VoteTimeQueue(java.util.UUID.randomUUID(), "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", true, + "00000000-0000-0000-0000-000000000001"); + pendingAck.requireMultiProxyAcknowledgements("Proxy1", java.util.Set.of("Proxy2")); + VoteTimeQueue completed = new VoteTimeQueue(java.util.UUID.randomUUID(), "Other", "Service", 101L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", true, + "00000000-0000-0000-0000-000000000002"); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + queue.add(pendingAck); + queue.add(completed); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.removeTimeVote(completed)).thenAnswer(invocation -> queue.remove(completed)); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + spyProxy.processQueue(); + + assertEquals(java.util.List.of(pendingAck), new java.util.ArrayList<>(queue)); + verify(multiProxyHandler).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.eq(java.util.Set.of("proxy2"))); + verify(voteCache).removeTimeVote(completed); + } + @Test void completedAckOutboxWaitsForDurableFenceRetirementBeforeFinishing() throws Exception { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); From a4f44eaf82ab33fcbe0003ae9d2040082e7fb012 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 21:38:39 -0600 Subject: [PATCH 094/136] Retain timed SQL rows when deletion fails --- .../proxy/cache/VoteCacheHandler.java | 1 + .../tests/VoteCacheHandlerVoteIdTest.java | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index 6689f995b..bc4f1e0b5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -1004,6 +1004,7 @@ public synchronized boolean removeTimeVote(VoteTimeQueue vote) { timeChangeQueue.remove(vote); return true; } + if (vote.getTimedVoteCacheRowId() > 0) return false; // The SQL row may never have existed when this vote was admitted to the // JSON emergency journal. Remove that durable fallback before dropping the // in-memory ACK outbox. diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 7fd9b866f..7e1869e83 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -697,6 +697,30 @@ public void timedVoteSqlTwinRejectsJsonOnlyDeliveryUpdate() throws Exception { verify(storage, never()).addTimedVote(org.mockito.ArgumentMatchers.anyInt(), same(queued)); } + @Test + public void timedVoteSqlTwinRejectsJsonOnlyRemoval() throws Exception { + UUID voteId = UUID.randomUUID(); + VoteTimeQueue queued = new VoteTimeQueue(voteId, "Player", "Service", 100L); + queued.setTimedVoteCacheRowId(42); + queued.setTimedVoteCacheJsonKey("2"); + handler.getTimeChangeQueue().add(queued); + DataNode stored = mock(DataNode.class); + when(stored.isObject()).thenReturn(true); + stubString(stored, "VoteId", voteId.toString()); + when(storage.getTimedVoteCache()).thenReturn(List.of("2")); + when(storage.getTimedVoteCache("2")).thenReturn(stored); + ProxyTimedVoteCacheTable sql = mock(ProxyTimedVoteCacheTable.class); + when(sql.removeVote(queued)).thenReturn(false); + setPrivateField(handler, "useMySQL", true); + setPrivateField(handler, "timedVoteCacheTable", sql); + + assertFalse(handler.removeTimeVote(queued)); + + assertTrue(handler.getTimeChangeQueue().contains(queued)); + verify(sql).removeVote(queued); + verify(storage, never()).removeTimedVotes(); + } + @Test public void serverVoteSqlTwinRejectsJsonOnlyDeliveryUpdate() throws Exception { OfflineBungeeVote vote = vote(UUID.randomUUID(), 100L); From 91108dda95b06651ccb6af6442811303cbc1c20c Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 21:51:25 -0600 Subject: [PATCH 095/136] Harden mixed-version proxy delivery --- .../backendproxy/BackendProxyHandler.java | 8 ++++ .../proxy/multiproxy/MultiProxyHandler.java | 25 +++++++--- .../BackendProxyHandlerLifecycleTest.java | 17 +++++++ .../MultiProxyHandlerLifecycleTest.java | 48 ++++++++++++++++++- 4 files changed, 90 insertions(+), 8 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 16e4b78ca..e4afbc28f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -25,6 +25,7 @@ import com.bencodez.votingplugin.backendproxy.transport.BackendProxyTransportManager; import com.bencodez.votingplugin.backendproxy.voteparty.BackendVotePartySync; import com.bencodez.votingplugin.proxy.BungeeMethod; +import com.bencodez.votingplugin.proxy.VotingPluginWire; import lombok.Getter; @@ -178,6 +179,13 @@ void dispatchIncomingAfterPublication(JsonEnvelope envelope, Runnable localDispa if (rollbackHandler != null) rollbackHandler.onMessage(envelope); return; } + // Global-data checks perform synchronous SQL and already run on their own + // timer worker. Explicitly dispatch an inbound wake-up asynchronously too: + // plugin messaging can invoke this method on Bukkit's primary thread. + if (VotingPluginWire.SUB_BUNGEE_TIME_CHANGE.equals(envelope.getSubChannel())) { + plugin.getBukkitScheduler().runTaskAsynchronously(plugin, localDispatch); + return; + } plugin.getBukkitScheduler().executeOrScheduleSync(plugin, localDispatch); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java index 7a5a5745c..bbcd6cd37 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java @@ -30,8 +30,9 @@ public abstract class MultiProxyHandler { private HashMap multiproxyClientHandles; private SocketHandler multiproxySocketHandler; - /** Peers that have explicitly advertised the additive durable-ACK protocol. */ - private final Set acknowledgedVoteCapabilityPeers = new LinkedHashSet<>(); + /** A renewable lease prevents a restarted/rolled-back peer staying ACK-capable forever. */ + static final long VOTE_CAPABILITY_LEASE_MILLIS = 5 * 60 * 1000L; + private final Map acknowledgedVoteCapabilityPeers = new HashMap<>(); @Getter private RedisHandler multiProxyRedis; @@ -42,6 +43,10 @@ public abstract class MultiProxyHandler { public MultiProxyHandler() { } + long capabilityNowMillis() { + return System.currentTimeMillis(); + } + /** * Adds a non-voted player to the cache. * @@ -327,8 +332,10 @@ public synchronized Set getConfiguredMultiProxyVoteRecipients() { /** Returns only configured peers that explicitly support durable acknowledgements. */ public synchronized Set getMultiProxyVoteRecipients() { + long now = capabilityNowMillis(); + acknowledgedVoteCapabilityPeers.entrySet().removeIf(entry -> entry.getValue() <= now); Set recipients = getConfiguredMultiProxyVoteRecipients(); - recipients.retainAll(acknowledgedVoteCapabilityPeers); + recipients.retainAll(acknowledgedVoteCapabilityPeers.keySet()); return recipients; } @@ -341,6 +348,7 @@ public synchronized void announceMultiProxyVoteCapability() { * Loads multi-proxy support. */ public synchronized void loadMultiProxySupport() { + acknowledgedVoteCapabilityPeers.clear(); if (!getMultiProxySupportEnabled()) { return; } @@ -623,7 +631,8 @@ private void handleEnvelope(JsonEnvelope envelope) { String normalized = recipient.toLowerCase(Locale.ROOT); synchronized (this) { if (version >= 1 && getConfiguredMultiProxyVoteRecipients().contains(normalized)) { - acknowledgedVoteCapabilityPeers.add(normalized); + acknowledgedVoteCapabilityPeers.put(normalized, + capabilityNowMillis() + VOTE_CAPABILITY_LEASE_MILLIS); replyRequired = !reply; } } @@ -652,8 +661,12 @@ private void handleEnvelope(JsonEnvelope envelope) { } if (!player.isEmpty() && !uuid.isEmpty() && !service.isEmpty()) { - triggerVote(player, service, realVote, true, 0L, VoteTotalsSnapshot.parseStorage(totals), uuid, - wireVote.voteId, origin); + if (origin.isBlank()) { + triggerVote(player, service, realVote, true, 0L, VoteTotalsSnapshot.parseStorage(totals), uuid); + } else { + triggerVote(player, service, realVote, true, 0L, VoteTotalsSnapshot.parseStorage(totals), uuid, + wireVote.voteId, origin); + } } return; } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index fdb6f843c..b99c3a863 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -43,6 +43,7 @@ import com.bencodez.votingplugin.backendproxy.cache.ProcessedVoteCache; import com.bencodez.votingplugin.backendproxy.presence.BackendPresenceManager; import com.bencodez.votingplugin.config.BungeeSettings; +import com.bencodez.votingplugin.proxy.VotingPluginWire; import com.bencodez.votingplugin.backendproxy.transport.MqttBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.MysqlBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.PluginMessagingBackendProxyTransport; @@ -54,6 +55,22 @@ import com.bencodez.votingplugin.proxy.BungeeMethod; class BackendProxyHandlerLifecycleTest { + @Test + void globalDataWakeupMovesOffTheCallingAndPrimaryThreads() { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + BackendProxyHandler handler = new BackendProxyHandler(plugin); + handler.activateInboundMessages(); + Runnable localDispatch = mock(Runnable.class); + + handler.dispatchIncomingAfterPublication( + JsonEnvelope.builder(VotingPluginWire.SUB_BUNGEE_TIME_CHANGE).build(), localDispatch); + + verify(scheduler).runTaskAsynchronously(plugin, localDispatch); + verify(localDispatch, never()).run(); + } + @Test void stagedInboundWaitsForPublicationAndUsesTheBukkitScheduler() throws Exception { com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java index 3089b134a..bcb7f2eb5 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java @@ -94,7 +94,7 @@ void stopsEveryReplacedSocketClientEvenWhenOneStopFails() { } @Test - void forwardsStableWireVoteIdToDurableTrigger() throws Exception { + void routesStableWireVoteWithoutOriginThroughLegacyTrigger() throws Exception { MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); UUID voteId = UUID.randomUUID(); Method handleEnvelope = MultiProxyHandler.class.getDeclaredMethod("handleEnvelope", @@ -105,12 +105,35 @@ void forwardsStableWireVoteIdToDurableTrigger() throws Exception { "00000000-0000-0000-0000-000000000001", "Service", 100L, false, true, "totals", voteId, false, false, 1, 1)); + verify(handler).triggerVote(org.mockito.ArgumentMatchers.eq("Player"), + org.mockito.ArgumentMatchers.eq("Service"), org.mockito.ArgumentMatchers.eq(true), + org.mockito.ArgumentMatchers.eq(true), org.mockito.ArgumentMatchers.eq(0L), + org.mockito.ArgumentMatchers.any(VoteTotalsSnapshot.class), + org.mockito.ArgumentMatchers.eq("00000000-0000-0000-0000-000000000001")); + verify(handler, org.mockito.Mockito.never()).triggerVote(org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.anyBoolean(), + org.mockito.ArgumentMatchers.anyBoolean(), org.mockito.ArgumentMatchers.anyLong(), + org.mockito.ArgumentMatchers.any(VoteTotalsSnapshot.class), org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.any(), org.mockito.ArgumentMatchers.anyString()); + } + + @Test + void forwardsOriginAndStableWireVoteIdToDurableTrigger() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); + UUID voteId = UUID.randomUUID(); + Method handleEnvelope = MultiProxyHandler.class.getDeclaredMethod("handleEnvelope", JsonEnvelope.class); + handleEnvelope.setAccessible(true); + + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyVote("Player", + "00000000-0000-0000-0000-000000000001", "Service", 100L, false, true, "totals", voteId, + false, false, 1, 1, "Primary")); + verify(handler).triggerVote(org.mockito.ArgumentMatchers.eq("Player"), org.mockito.ArgumentMatchers.eq("Service"), org.mockito.ArgumentMatchers.eq(true), org.mockito.ArgumentMatchers.eq(true), org.mockito.ArgumentMatchers.eq(0L), org.mockito.ArgumentMatchers.any(VoteTotalsSnapshot.class), org.mockito.ArgumentMatchers.eq("00000000-0000-0000-0000-000000000001"), - org.mockito.ArgumentMatchers.eq(voteId)); + org.mockito.ArgumentMatchers.eq(voteId), org.mockito.ArgumentMatchers.eq("Primary")); } @Test @@ -188,4 +211,25 @@ void fencesOnlyPeersThatAdvertiseDurableAcknowledgements() throws Exception { assertEquals(java.util.Set.of("capable"), handler.getMultiProxyVoteRecipients()); } + + @Test + void durableAcknowledgementCapabilityExpiresAndCanBeRenewed() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, + org.mockito.Mockito.withSettings().useConstructor().defaultAnswer(org.mockito.Mockito.CALLS_REAL_METHODS)); + org.mockito.Mockito.when(handler.getMultiProxyMethod()).thenReturn(MultiProxyMethod.SOCKETS); + org.mockito.Mockito.when(handler.getMultiProxyServers()).thenReturn(List.of("Capable")); + org.mockito.Mockito.when(handler.capabilityNowMillis()).thenReturn(1_000L, + 1_000L + MultiProxyHandler.VOTE_CAPABILITY_LEASE_MILLIS - 1, + 1_000L + MultiProxyHandler.VOTE_CAPABILITY_LEASE_MILLIS, + 2_000L + MultiProxyHandler.VOTE_CAPABILITY_LEASE_MILLIS); + Method handleEnvelope = MultiProxyHandler.class.getDeclaredMethod("handleEnvelope", JsonEnvelope.class); + handleEnvelope.setAccessible(true); + + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyCapabilities("Capable", 1, true)); + assertEquals(java.util.Set.of("capable"), handler.getMultiProxyVoteRecipients()); + assertTrue(handler.getMultiProxyVoteRecipients().isEmpty()); + + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyCapabilities("Capable", 1, true)); + assertEquals(java.util.Set.of("capable"), handler.getMultiProxyVoteRecipients()); + } } From 76a65de3dec532578e79ade9e494b855f1c49b91 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 22:07:04 -0600 Subject: [PATCH 096/136] Preserve mixed-proxy vote delivery --- .../votingplugin/proxy/VotingPluginProxy.java | 5 ++- .../proxy/cache/VoteCacheHandler.java | 13 ++++++- .../tests/VoteCacheHandlerVoteIdTest.java | 34 ++++++++++++++++--- .../tests/VotingPluginProxyTest.java | 11 +++++- 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index dbcd69aa8..fed808899 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -5634,6 +5634,7 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim } VoteTimeQueue outbox = queuedVote; boolean newlyCreated = false; + boolean sendLegacyCopy = false; if (outbox == null) { outbox = new VoteTimeQueue(null, player, service, time, false, Collections.emptySet(), Collections.emptySet(), totals == null ? "" : totals.toString(), true, uuid); @@ -5646,6 +5647,7 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim retryState.queuedVote = outbox; if (outbox.getVoteId() == null || !getVoteCacheHandler().addTimeVoteToCache(outbox)) return false; newlyCreated = true; + sendLegacyCopy = true; } else { boolean alreadyQueued = false; for (VoteTimeQueue candidate : getVoteCacheHandler().getTimeChangeQueue()) { @@ -5663,6 +5665,7 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim // A recovered admission is the first safe chance to publish to legacy peers. // Already-queued retries must not repeat that fire-and-forget copy. newlyCreated = !alreadyQueued; + sendLegacyCopy = newlyCreated || !outbox.isMultiProxyForwardingRequired(); } if (!outbox.isMultiProxyForwardingRequired()) { outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), recipients); @@ -5676,7 +5679,7 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim } // A successful publish only means the transport accepted the invocation. // Keep the durable row and wait for an acknowledgement before continuing. - if (newlyCreated && !legacyRecipients.isEmpty()) { + if (sendLegacyCopy && !legacyRecipients.isEmpty()) { // Never retry this legacy copy as part of the ACK outbox: a legacy peer has // no receiver dedupe/ACK contract, while capable peers stay fully durable. multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(player, uuid, service, time, diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index bc4f1e0b5..dee493e5c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -432,7 +432,18 @@ public synchronized void clearOnlineVoteRewards(String uuid) { if (vote.isDeliveryStateDirty() && updateOnlineVote(uuid, vote)) vote.setDeliveryStateDirty(false); continue; } - removeOnlineVote(uuid, vote); + // Another proxy has already delivered this reward. Persist that monotonic + // ineligible state before attempting deletion, so a transient delete + // failure cannot leave the retained row eligible for local delivery. + if (!vote.isRewardDelivered()) { + vote.setRewardDelivered(true); + vote.setDeliveryStateDirty(true); + } + if (vote.isDeliveryStateDirty()) { + if (!updateOnlineVote(uuid, vote)) continue; + vote.setDeliveryStateDirty(false); + } + tryRemoveOnlineVote(uuid, vote); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 7e1869e83..88e99ea6c 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -8,11 +8,13 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.spy; import java.util.Collections; import java.util.ArrayList; @@ -938,20 +940,44 @@ public void onlineBroadcastStateUpdateUsesVoteIdWhenTimestampsCollide() { @Test public void globalRewardClearRetainsOnlyIncompleteForwardBroadcasts() { + VoteCacheHandler guarded = spy(handler); OfflineBungeeVote pending = new OfflineBungeeVote(UUID.randomUUID(), "Player", "player-uuid", "Service", 100L, true, "totals", false, true, Set.of("Lobby", "Survival"), Set.of("Lobby"), false); OfflineBungeeVote complete = new OfflineBungeeVote(UUID.randomUUID(), "Player", "player-uuid", "Service", 101L, true, "totals", true, true, Set.of("Lobby"), Set.of("Lobby"), false); - handler.addOnlineVote("player-uuid", pending); - handler.addOnlineVote("player-uuid", complete); + guarded.addOnlineVote("player-uuid", pending); + guarded.addOnlineVote("player-uuid", complete); + doReturn(true).when(guarded).updateOnlineVote(eq("player-uuid"), org.mockito.ArgumentMatchers.any()); - handler.clearOnlineVoteRewards("player-uuid"); + guarded.clearOnlineVoteRewards("player-uuid"); - assertEquals(List.of(pending), handler.getOnlineVotes("player-uuid")); + assertEquals(List.of(pending), guarded.getOnlineVotes("player-uuid")); assertTrue(pending.isRewardDelivered()); assertTrue(pending.needsBroadcastOn("Survival")); } + @Test + public void failedGlobalRewardClearCannotLeaveRetainedVoteEligible() { + VoteCacheHandler guarded = spy(handler); + OfflineBungeeVote vote = vote(UUID.randomUUID(), 100L); + guarded.addOnlineVote("player-uuid", vote); + doReturn(false, true).when(guarded).updateOnlineVote("player-uuid", vote); + doReturn(false).when(guarded).tryRemoveOnlineVote("player-uuid", vote); + + guarded.clearOnlineVoteRewards("player-uuid"); + + assertTrue(vote.isRewardDelivered()); + assertTrue(vote.isDeliveryStateDirty()); + assertEquals(List.of(vote), guarded.getOnlineVotes("player-uuid")); + verify(guarded, never()).tryRemoveOnlineVote("player-uuid", vote); + + guarded.clearOnlineVoteRewards("player-uuid"); + + assertFalse(vote.isDeliveryStateDirty()); + assertEquals(List.of(vote), guarded.getOnlineVotes("player-uuid")); + verify(guarded).tryRemoveOnlineVote("player-uuid", vote); + } + @Test public void completedOnlineVoteRemovalKeepsCollidingVoteId() { UUID removedId = UUID.randomUUID(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 0d183d548..30b3b6f4b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -743,6 +743,8 @@ void reconstructedQueuedVoteRetainsUnacknowledgedMultiProxyDeliveryState() throw Mockito.when(votingPluginProxy.getConfig().getMultiProxyOneGlobalReward()).thenReturn(false); Mockito.when(votingPluginProxy.getConfig().getProxyServerName()).thenReturn("Proxy1"); Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Set.of("Proxy2")); + Mockito.when(multiProxyHandler.getConfiguredMultiProxyVoteRecipients()) + .thenReturn(new java.util.LinkedHashSet<>(java.util.Set.of("Proxy2", "ProxyLegacy"))); VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); Mockito.doNothing().when(spyProxy).addVoteParty(); @@ -773,7 +775,14 @@ void reconstructedQueuedVoteRetainsUnacknowledgedMultiProxyDeliveryState() throw spyProxy.processQueue(); - verify(multiProxyHandler, Mockito.times(2)).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any()); + @SuppressWarnings({ "rawtypes", "unchecked" }) + org.mockito.ArgumentCaptor> recipients = + (org.mockito.ArgumentCaptor) org.mockito.ArgumentCaptor.forClass(java.util.Collection.class); + verify(multiProxyHandler, Mockito.times(3)).sendMultiProxyEnvelopeAccepted(Mockito.any(), recipients.capture()); + assertEquals(1, recipients.getAllValues().stream() + .filter(value -> new java.util.HashSet<>(value).equals(java.util.Set.of("ProxyLegacy"))).count()); + assertEquals(2, recipients.getAllValues().stream() + .filter(value -> new java.util.HashSet<>(value).equals(java.util.Set.of("proxy2"))).count()); } @Test From 20081a6d29672716fc2380bc05bde7b332a516e6 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 22:24:39 -0600 Subject: [PATCH 097/136] Preserve legacy multi-proxy delivery across restarts --- .../votingplugin/proxy/VotingPluginProxy.java | 30 ++++++++----- .../proxy/bungee/BungeeJsonVoteCache.java | 2 + .../proxy/cache/ProxyTimedVoteCacheTable.java | 44 ++++++++++++------- .../proxy/cache/VoteCacheHandler.java | 6 +++ .../proxy/velocity/VelocityJsonVoteCache.java | 2 + .../votingplugin/timequeue/VoteTimeQueue.java | 17 +++++++ .../tests/VoteCacheHandlerVoteIdTest.java | 3 ++ .../tests/VotingPluginProxyTest.java | 12 +++-- 8 files changed, 86 insertions(+), 30 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index fed808899..1dd43343b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -5633,8 +5633,6 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim return true; } VoteTimeQueue outbox = queuedVote; - boolean newlyCreated = false; - boolean sendLegacyCopy = false; if (outbox == null) { outbox = new VoteTimeQueue(null, player, service, time, false, Collections.emptySet(), Collections.emptySet(), totals == null ? "" : totals.toString(), true, uuid); @@ -5642,12 +5640,11 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim // by locating its identity rather than creating a new duplicate record. outbox.setVoteId(findLiveVoteId(retryState)); outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), recipients); + outbox.setMultiProxyLegacyPendingRecipients(legacyRecipients); outbox.setRealVote(realVote); outbox.setDeliveryStateDirty(true); retryState.queuedVote = outbox; if (outbox.getVoteId() == null || !getVoteCacheHandler().addTimeVoteToCache(outbox)) return false; - newlyCreated = true; - sendLegacyCopy = true; } else { boolean alreadyQueued = false; for (VoteTimeQueue candidate : getVoteCacheHandler().getTimeChangeQueue()) { @@ -5662,13 +5659,10 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim // addTimeVoteToCache is idempotent for an already-durable queue entry. return false; } - // A recovered admission is the first safe chance to publish to legacy peers. - // Already-queued retries must not repeat that fire-and-forget copy. - newlyCreated = !alreadyQueued; - sendLegacyCopy = newlyCreated || !outbox.isMultiProxyForwardingRequired(); } if (!outbox.isMultiProxyForwardingRequired()) { outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), recipients); + outbox.setMultiProxyLegacyPendingRecipients(legacyRecipients); outbox.setRealVote(realVote); outbox.setProcessed(true); outbox.setDeliveryStateDirty(true); @@ -5679,12 +5673,17 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim } // A successful publish only means the transport accepted the invocation. // Keep the durable row and wait for an acknowledgement before continuing. - if (sendLegacyCopy && !legacyRecipients.isEmpty()) { + if (!outbox.getMultiProxyLegacyPendingRecipients().isEmpty()) { // Never retry this legacy copy as part of the ACK outbox: a legacy peer has // no receiver dedupe/ACK contract, while capable peers stay fully durable. - multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(player, uuid, service, time, + // Persist the exact pending set until the transport accepts the copy. + if (!multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(player, uuid, service, time, false, realVote, totals == null ? "" : totals.toString(), outbox.getVoteId(), false, false, 1, 1, - outbox.getMultiProxyOrigin()), legacyRecipients); + outbox.getMultiProxyOrigin()), + new LinkedHashSet<>(outbox.getMultiProxyLegacyPendingRecipients()))) return false; + outbox.setMultiProxyLegacyPendingRecipients(Collections.emptySet()); + outbox.setDeliveryStateDirty(true); + if (!persistTimeVoteDelivery(outbox)) return false; } if (!sendDurableMultiProxyOutbox(outbox)) return false; scheduleTimeVoteRetry(); @@ -5703,6 +5702,15 @@ private boolean retryDurableMultiProxyOutbox(VoteTimeQueue outbox) { if (outbox.hasCompletedMultiProxyAcknowledgements()) { return finishMultiProxyRetirement(null, outbox); } + if (!outbox.getMultiProxyLegacyPendingRecipients().isEmpty()) { + if (!multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(outbox.getName(), + outbox.getUuid(), outbox.getService(), outbox.getTime(), false, outbox.isRealVote(), + outbox.getTotals(), outbox.getVoteId(), false, false, 1, 1, outbox.getMultiProxyOrigin()), + new LinkedHashSet<>(outbox.getMultiProxyLegacyPendingRecipients()))) return false; + outbox.setMultiProxyLegacyPendingRecipients(Collections.emptySet()); + outbox.setDeliveryStateDirty(true); + if (!persistTimeVoteDelivery(outbox)) return false; + } sendDurableMultiProxyOutbox(outbox); return false; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java index 413e0cb7c..0185f5135 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java @@ -62,6 +62,8 @@ public void addTimedVote(int num, VoteTimeQueue voteTimedQueue) { setBoolean(path + ".MultiProxyCompletionPending", voteTimedQueue.isMultiProxyCompletionPending()); setString(path + ".MultiProxyRecipients", voteTimedQueue.encodeMultiProxyRecipients()); setString(path + ".MultiProxyAcknowledgedServers", voteTimedQueue.encodeMultiProxyAcknowledgedServers()); + setString(path + ".MultiProxyLegacyPendingRecipients", + voteTimedQueue.encodeMultiProxyLegacyPendingRecipients()); setString(path + ".BroadcastTargets", voteTimedQueue.encodeBroadcastTargets()); setString(path + ".BroadcastForwardedServers", voteTimedQueue.encodeBroadcastForwardedServers()); setString(path + ".HttpBroadcastDeliveryIds", voteTimedQueue.encodeHttpBroadcastDeliveryIds()); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java index 5e245099a..774515b5e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java @@ -48,7 +48,8 @@ public String buildCreateTableSql(DbType dbType) { + qi("multiProxyAcknowledgedServers") + " TEXT, " + qi("broadcastTargets") + " TEXT, " + qi("broadcastForwardedServers") + " TEXT, " - + qi("httpBroadcastDeliveryIds") + " TEXT" + + qi("httpBroadcastDeliveryIds") + " TEXT, " + + qi("multiProxyLegacyPendingRecipients") + " TEXT" + ");"; } @@ -72,6 +73,7 @@ public String buildCreateTableSql(DbType dbType) { + qi("broadcastTargets") + " TEXT," + qi("broadcastForwardedServers") + " TEXT," + qi("httpBroadcastDeliveryIds") + " TEXT," + + qi("multiProxyLegacyPendingRecipients") + " TEXT," + "INDEX idx_time (" + qi("time") + ")" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"; } @@ -143,6 +145,7 @@ private void ensureProxyBroadcastColumns() { ensureColumn("multiProxyRecipients", "TEXT"); ensureColumn("multiProxyAcknowledgedServers", "TEXT"); ensureColumn("httpBroadcastDeliveryIds", "TEXT"); + ensureColumn("multiProxyLegacyPendingRecipients", "TEXT"); } private void ensureColumn(String column, String type) { @@ -192,15 +195,17 @@ private boolean insertTimedVote(UUID voteId, String uuid, String playerName, Str boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, boolean processed, boolean multiProxyForwardingHandled, boolean multiProxyForwardingRequired, boolean realVote, String multiProxyOrigin, boolean multiProxyCompletionPending, String multiProxyRecipients, - String multiProxyAcknowledgedServers, String httpBroadcastDeliveryIds) { + String multiProxyAcknowledgedServers, String httpBroadcastDeliveryIds, + String multiProxyLegacyPendingRecipients) { String sql = "INSERT INTO " + qi(getTableName()) + " (" + qi("playerName") + ", " + qi("service") + ", " + qi("time") + ", " + qi("voteId") + ", " + qi("uuid") + ", " + qi("proxyBroadcastHandled") + ", " + qi("broadcastTargets") + ", " + qi("broadcastForwardedServers") + ", " + qi("totals") + ", " + qi("processed") + ", " + qi("multiProxyForwardingHandled") + ", " + qi("multiProxyForwardingRequired") + ", " + qi("realVote") + ", " + qi("multiProxyOrigin") + ", " + qi("multiProxyCompletionPending") + ", " + qi("multiProxyRecipients") + ", " - + qi("multiProxyAcknowledgedServers") + ", " + qi("httpBroadcastDeliveryIds") - + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; + + qi("multiProxyAcknowledgedServers") + ", " + qi("httpBroadcastDeliveryIds") + ", " + + qi("multiProxyLegacyPendingRecipients") + + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; try (Connection conn = mysql.getConnectionManager().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { ps.setString(1, playerName); @@ -236,6 +241,7 @@ private boolean insertTimedVote(UUID voteId, String uuid, String playerName, Str ps.setString(16, multiProxyRecipients); ps.setString(17, multiProxyAcknowledgedServers); ps.setString(18, httpBroadcastDeliveryIds); + ps.setString(19, multiProxyLegacyPendingRecipients); ps.executeUpdate(); return true; } catch (SQLException e) { @@ -253,7 +259,7 @@ public boolean insertTimedVote(VoteTimeQueue vote) { vote.isMultiProxyForwardingRequired(), vote.isRealVote(), vote.getMultiProxyOrigin(), vote.isMultiProxyCompletionPending(), vote.encodeMultiProxyRecipients(), vote.encodeMultiProxyAcknowledgedServers(), - vote.encodeHttpBroadcastDeliveryIds()); + vote.encodeHttpBroadcastDeliveryIds(), vote.encodeMultiProxyLegacyPendingRecipients()); } /** Backward-compatible insert overload. */ @@ -262,7 +268,7 @@ public boolean insertTimedVote(UUID voteId, String uuid, String playerName, Stri boolean processed, boolean multiProxyForwardingHandled, String httpBroadcastDeliveryIds) { return insertTimedVote(voteId, uuid, playerName, service, time, proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, totals, processed, multiProxyForwardingHandled, false, true, "", false, "", "", - httpBroadcastDeliveryIds); + httpBroadcastDeliveryIds, ""); } /** Backward-compatible insert overload without HTTP delivery state. */ @@ -297,7 +303,7 @@ public boolean updateTimedVote(VoteTimeQueue vote) { + qi("multiProxyOrigin") + " = ?, " + qi("multiProxyCompletionPending") + " = ?, " + qi("multiProxyRecipients") + " = ?, " + qi("multiProxyAcknowledgedServers") + " = ?, " + qi("uuid") + " = ?, " - + qi("httpBroadcastDeliveryIds") + " = ? WHERE " + + qi("httpBroadcastDeliveryIds") + " = ?, " + qi("multiProxyLegacyPendingRecipients") + " = ? WHERE " + (hasRowId ? qi("id") + " = ?;" : hasVoteId ? qi("voteId") + " = ?;" : qi("playerName") + " = ? AND " + qi("service") + " = ? AND " + qi("time") + " = ?;"); try (Connection conn = mysql.getConnectionManager().getConnection(); @@ -331,14 +337,15 @@ public boolean updateTimedVote(VoteTimeQueue vote) { ps.setString(12, vote.encodeMultiProxyAcknowledgedServers()); ps.setString(13, vote.getUuid()); ps.setString(14, vote.encodeHttpBroadcastDeliveryIds()); + ps.setString(15, vote.encodeMultiProxyLegacyPendingRecipients()); if (hasRowId) { - ps.setInt(15, vote.getTimedVoteCacheRowId()); + ps.setInt(16, vote.getTimedVoteCacheRowId()); } else if (hasVoteId) { - ps.setString(15, vote.getVoteId().toString()); + ps.setString(16, vote.getVoteId().toString()); } else { - ps.setString(15, vote.getName()); - ps.setString(16, vote.getService()); - ps.setLong(17, vote.getTime()); + ps.setString(16, vote.getName()); + ps.setString(17, vote.getService()); + ps.setLong(18, vote.getTime()); } ps.executeUpdate(); return true; @@ -503,7 +510,8 @@ private List selectVotes(String sql, Object[] params) { rs.getBoolean("multiProxyCompletionPending"), rs.getString("multiProxyRecipients"), rs.getString("multiProxyAcknowledgedServers"), - rs.getString("httpBroadcastDeliveryIds") + rs.getString("httpBroadcastDeliveryIds"), + rs.getString("multiProxyLegacyPendingRecipients") )); } } @@ -547,6 +555,7 @@ public static class TimedVoteRow { private final String multiProxyRecipients; private final String multiProxyAcknowledgedServers; private final String httpBroadcastDeliveryIds; + private final String multiProxyLegacyPendingRecipients; /** * Constructor for TimedVoteRow. @@ -565,7 +574,8 @@ public TimedVoteRow(int id, String playerName, String service, long time, UUID v boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, boolean processed, boolean multiProxyForwardingHandled, boolean multiProxyForwardingRequired, boolean realVote, String multiProxyOrigin, boolean multiProxyCompletionPending, String multiProxyRecipients, - String multiProxyAcknowledgedServers, String httpBroadcastDeliveryIds) { + String multiProxyAcknowledgedServers, String httpBroadcastDeliveryIds, + String multiProxyLegacyPendingRecipients) { this.id = id; this.playerName = playerName; this.service = service; @@ -585,6 +595,7 @@ public TimedVoteRow(int id, String playerName, String service, long time, UUID v this.multiProxyRecipients = multiProxyRecipients; this.multiProxyAcknowledgedServers = multiProxyAcknowledgedServers; this.httpBroadcastDeliveryIds = httpBroadcastDeliveryIds; + this.multiProxyLegacyPendingRecipients = multiProxyLegacyPendingRecipients; } /** Backward-compatible row constructor without HTTP delivery state. */ @@ -592,7 +603,7 @@ public TimedVoteRow(int id, String playerName, String service, long time, UUID v boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, boolean processed) { this(id, playerName, service, time, voteId, uuid, proxyBroadcastHandled, broadcastTargets, - broadcastForwardedServers, totals, processed, false, false, true, "", false, "", "", null); + broadcastForwardedServers, totals, processed, false, false, true, "", false, "", "", null, null); } /** Backward-compatible row constructor without the durable multi-proxy forwarding fence. */ @@ -601,7 +612,7 @@ public TimedVoteRow(int id, String playerName, String service, long time, UUID v boolean processed, String httpBroadcastDeliveryIds) { this(id, playerName, service, time, voteId, uuid, proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, totals, processed, false, false, true, "", false, "", "", - httpBroadcastDeliveryIds); + httpBroadcastDeliveryIds, null); } /** @@ -686,6 +697,7 @@ public boolean isMultiProxyForwardingHandled() { public String getMultiProxyOrigin() { return multiProxyOrigin; } public boolean isMultiProxyCompletionPending() { return multiProxyCompletionPending; } public String getMultiProxyRecipients() { return multiProxyRecipients; } + public String getMultiProxyLegacyPendingRecipients() { return multiProxyLegacyPendingRecipients; } public String getMultiProxyAcknowledgedServers() { return multiProxyAcknowledgedServers; } public String getHttpBroadcastDeliveryIds() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index dee493e5c..f19161673 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -1298,6 +1298,9 @@ public void load() { VoteTimeQueue.decodeBroadcastForwardedServers(timedVoteRow.getMultiProxyRecipients())); voteTimeQueue.setMultiProxyAcknowledgedServers( VoteTimeQueue.decodeBroadcastForwardedServers(timedVoteRow.getMultiProxyAcknowledgedServers())); + voteTimeQueue.setMultiProxyLegacyPendingRecipients( + VoteTimeQueue.decodeBroadcastForwardedServers( + timedVoteRow.getMultiProxyLegacyPendingRecipients())); voteTimeQueue.setTimedVoteCacheRowId(timedVoteRow.getId()); timedVotes.add(voteTimeQueue); }); @@ -1535,6 +1538,9 @@ private VoteTimeQueue decodeJsonTimedVote(String key, DataNode data) { queuedVote.setMultiProxyAcknowledgedServers(VoteTimeQueue.decodeBroadcastForwardedServers( data.has("MultiProxyAcknowledgedServers") ? data.get("MultiProxyAcknowledgedServers").asString() : "")); + queuedVote.setMultiProxyLegacyPendingRecipients(VoteTimeQueue.decodeBroadcastForwardedServers( + data.has("MultiProxyLegacyPendingRecipients") + ? data.get("MultiProxyLegacyPendingRecipients").asString() : "")); queuedVote.setTimedVoteCacheJsonKey(key); return queuedVote; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java index 16e176ef1..34ee45de0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java @@ -78,6 +78,8 @@ public void addTimedVote(int num, VoteTimeQueue voteTimedQueue) { "MultiProxyRecipients"); setPath(voteTimedQueue.encodeMultiProxyAcknowledgedServers(), "TimedVoteCache", String.valueOf(num), "MultiProxyAcknowledgedServers"); + setPath(voteTimedQueue.encodeMultiProxyLegacyPendingRecipients(), "TimedVoteCache", String.valueOf(num), + "MultiProxyLegacyPendingRecipients"); setPath(voteTimedQueue.encodeBroadcastTargets(), "TimedVoteCache", String.valueOf(num), "BroadcastTargets"); setPath(voteTimedQueue.encodeBroadcastForwardedServers(), "TimedVoteCache", String.valueOf(num), "BroadcastForwardedServers"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java index fbc8a08b9..c3cafd925 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java @@ -83,6 +83,9 @@ public class VoteTimeQueue { /** Recipients whose durable completion acknowledgement was received. */ @Getter private Set multiProxyAcknowledgedServers; + /** Legacy peers whose one-way copy has not yet been accepted by the transport. */ + @Getter + private Set multiProxyLegacyPendingRecipients; /** Stable HTTP standalone-broadcast delivery IDs by target server. */ private final Map httpBroadcastDeliveryIds; @@ -222,6 +225,7 @@ public VoteTimeQueue(UUID voteId, String name, String service, long time, boolea } this.multiProxyRecipients = new LinkedHashSet<>(); this.multiProxyAcknowledgedServers = new LinkedHashSet<>(); + this.multiProxyLegacyPendingRecipients = new LinkedHashSet<>(); this.httpBroadcastDeliveryIds = new LinkedHashMap<>(); if (httpBroadcastDeliveryIds != null) { httpBroadcastDeliveryIds.forEach(this::setHttpBroadcastDeliveryId); @@ -286,6 +290,10 @@ public String encodeMultiProxyRecipients() { return encodeBroadcastServers(multiProxyRecipients); } + public String encodeMultiProxyLegacyPendingRecipients() { + return encodeBroadcastServers(multiProxyLegacyPendingRecipients); + } + public String encodeMultiProxyAcknowledgedServers() { return encodeBroadcastServers(multiProxyAcknowledgedServers); } @@ -301,6 +309,15 @@ public void setMultiProxyRecipients(Set recipients) { } } + public void setMultiProxyLegacyPendingRecipients(Set recipients) { + multiProxyLegacyPendingRecipients.clear(); + if (recipients != null) { + for (String recipient : recipients) { + if (recipient != null && !recipient.isBlank()) multiProxyLegacyPendingRecipients.add(recipient); + } + } + } + public void setMultiProxyAcknowledgedServers(Set recipients) { multiProxyAcknowledgedServers.clear(); if (recipients != null) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 88e99ea6c..0037ebb45 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -770,6 +770,7 @@ public void timedVoteAcknowledgementOutboxSurvivesHandlerRestart() { Set.of("backend-1"), Set.of("backend-1"), "totals", false, false, "player-uuid", Map.of("backend-1", "00000000-0000-0000-0000-000000000181")); queued.requireMultiProxyAcknowledgements("origin-proxy", Set.of("backend-1")); + queued.setMultiProxyLegacyPendingRecipients(Set.of("legacy-proxy")); queued.acknowledgeMultiProxyRecipient("backend-1"); queued.acknowledgeMultiProxyRetirement("backend-1"); handler.addTimeVoteToCache(queued); @@ -783,6 +784,7 @@ public void timedVoteAcknowledgementOutboxSurvivesHandlerRestart() { assertEquals(voteId, recovered.getVoteId()); assertTrue(recovered.isMultiProxyForwardingRequired()); assertEquals(Set.of("backend-1"), recovered.getMultiProxyRecipients()); + assertEquals(Set.of("legacy-proxy"), recovered.getMultiProxyLegacyPendingRecipients()); assertTrue(recovered.hasCompletedMultiProxyRetirements()); assertEquals("00000000-0000-0000-0000-000000000181", recovered.getHttpBroadcastDeliveryId("backend-1")); @@ -1238,6 +1240,7 @@ private static DataNode timedVoteNode(VoteTimeQueue vote) { data.addProperty("MultiProxyCompletionPending", vote.isMultiProxyCompletionPending()); data.addProperty("MultiProxyRecipients", vote.encodeMultiProxyRecipients()); data.addProperty("MultiProxyAcknowledgedServers", vote.encodeMultiProxyAcknowledgedServers()); + data.addProperty("MultiProxyLegacyPendingRecipients", vote.encodeMultiProxyLegacyPendingRecipients()); data.addProperty("BroadcastTargets", vote.encodeBroadcastTargets()); data.addProperty("BroadcastForwardedServers", vote.encodeBroadcastForwardedServers()); data.addProperty("HttpBroadcastDeliveryIds", vote.encodeHttpBroadcastDeliveryIds()); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 30b3b6f4b..5d1b2ce59 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -419,6 +419,8 @@ void runtimeRetryPersistsAckOutboxWhoseInitialInsertFailedBeforePublishing() thr Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); Mockito.when(voteCache.tryRemoveOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); Mockito.when(voteCache.retryPendingVotePersistence()).thenReturn(true); + Mockito.when(voteCache.updateTimeVote(Mockito.any())).thenReturn(true); + Mockito.when(multiProxyHandler.sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any())).thenReturn(true); Mockito.when(voteCache.addTimeVoteToCache(Mockito.any())).thenAnswer(new org.mockito.stubbing.Answer() { private int attempts; @@ -733,7 +735,8 @@ void reconstructedQueuedVoteRetainsUnacknowledgedMultiProxyDeliveryState() throw Mockito.when(voteCache.addServerVoteDurably(Mockito.anyString(), Mockito.any())).thenReturn(true); Mockito.when(voteCache.updateOnlineVote(Mockito.anyString(), Mockito.any())).thenReturn(true); Mockito.when(voteCache.updateServerVote(Mockito.anyString(), Mockito.any())).thenReturn(true); - Mockito.when(voteCache.updateTimeVote(queued)).thenReturn(true, false); + Mockito.when(voteCache.updateTimeVote(Mockito.any())).thenReturn(true, false, true); + Mockito.when(multiProxyHandler.sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any())).thenReturn(true); Mockito.when(voteCache.removeTimeVote(queued)).thenReturn(false); Mockito.when(voteCache.markTimeVoteCompletedDurably(queued)).thenReturn(false); Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); @@ -767,6 +770,9 @@ void reconstructedQueuedVoteRetainsUnacknowledgedMultiProxyDeliveryState() throw restored.setMultiProxyOrigin(queued.getMultiProxyOrigin()); restored.setMultiProxyRecipients(queued.getMultiProxyRecipients()); restored.setMultiProxyAcknowledgedServers(queued.getMultiProxyAcknowledgedServers()); + // The failed post-publication update means durable storage still contains the + // pending legacy recipient captured before the first publish. + restored.setMultiProxyLegacyPendingRecipients(java.util.Set.of("ProxyLegacy")); queue.clear(); queue.add(restored); java.lang.reflect.Field retries = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); @@ -779,9 +785,9 @@ void reconstructedQueuedVoteRetainsUnacknowledgedMultiProxyDeliveryState() throw org.mockito.ArgumentCaptor> recipients = (org.mockito.ArgumentCaptor) org.mockito.ArgumentCaptor.forClass(java.util.Collection.class); verify(multiProxyHandler, Mockito.times(3)).sendMultiProxyEnvelopeAccepted(Mockito.any(), recipients.capture()); - assertEquals(1, recipients.getAllValues().stream() - .filter(value -> new java.util.HashSet<>(value).equals(java.util.Set.of("ProxyLegacy"))).count()); assertEquals(2, recipients.getAllValues().stream() + .filter(value -> new java.util.HashSet<>(value).equals(java.util.Set.of("ProxyLegacy"))).count()); + assertEquals(1, recipients.getAllValues().stream() .filter(value -> new java.util.HashSet<>(value).equals(java.util.Set.of("proxy2"))).count()); } From 74ead431294d819015536dcc465933c73190d7ce Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 22:48:45 -0600 Subject: [PATCH 098/136] Make exclusive transport replacement rollback safe --- .../votingplugin/VotingPluginMain.java | 78 +++++++++-- .../backendproxy/BackendProxyHandler.java | 13 ++ .../BackendProxyTransportManager.java | 12 +- .../transport/MqttBackendProxyTransport.java | 63 ++++++++- .../SocketBackendProxyTransport.java | 117 ++++++++++++---- .../control/BackendControlConnector.java | 36 +++-- ...PluginMainBackendProxyPublicationTest.java | 120 +++++++++++++++++ .../BackendProxyHandlerLifecycleTest.java | 126 ++++++++++++++++-- 8 files changed, 498 insertions(+), 67 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 909901864..a7c98de5c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1255,6 +1255,9 @@ public static final class BackendProxyRestart { private boolean redisHandoffInProgress; private boolean finished; private boolean abandonmentRequested; + // Same-method socket and MQTT replacements must not start a second runtime + // endpoint before worker-side preparation retires the predecessor. + private boolean replacementLoadDeferred; private volatile boolean published; private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler replacement, boolean disabled, @@ -1264,6 +1267,14 @@ private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler re this.disabled = disabled; this.previousRequiresPreparation = previousRequiresPreparation; } + + /** Only exclusive same-method transports may be restored by the Control worker. */ + public boolean requiresWorkerRollback() { + return previousPrepared && previous != null && replacement != null + && ((previous.getMethod() == BungeeMethod.SOCKETS && replacement.getMethod() == BungeeMethod.SOCKETS) + || (previous.getMethod() == BungeeMethod.MQTT + && replacement.getMethod() == BungeeMethod.MQTT)); + } } public static final class BackendProxyRestartPreparationException extends RuntimeException { @@ -1286,19 +1297,29 @@ public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { return new BackendProxyRestart(previous, null, true, previousRequiresPreparation); } BungeeMethod replacementMethod = BungeeMethod.getByName(bungeeSettings.getBungeeMethod()); + boolean sameSocketReplacement = previous != null && previous.getMethod() == BungeeMethod.SOCKETS + && replacementMethod == BungeeMethod.SOCKETS; + boolean sameMqttReplacement = previous != null && previous.getMethod() == BungeeMethod.MQTT + && replacementMethod == BungeeMethod.MQTT; + boolean deferredReplacementLoad = sameSocketReplacement || sameMqttReplacement; // Every transition away from an active HTTP transport must first drain its // durable outgoing queue. Restricting preparation to HTTP-to-HTTP swaps can // strand accepted deliveries when another transport is published. boolean previousRequiresPreparation = previous != null - && (previous.requiresPreparationForReplacement() || previous.requiresRedisRetirement()); + && (deferredReplacementLoad || previous.requiresPreparationForReplacement() + || previous.requiresRedisRetirement()); BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); - try { - replacement.loadForReplacement(); - } catch (RuntimeException failure) { - replacement.close(); - throw failure; + if (!deferredReplacementLoad) { + try { + replacement.loadForReplacement(); + } catch (RuntimeException failure) { + replacement.close(); + throw failure; + } } - return new BackendProxyRestart(previous, replacement, false, previousRequiresPreparation); + BackendProxyRestart restart = new BackendProxyRestart(previous, replacement, false, previousRequiresPreparation); + restart.replacementLoadDeferred = deferredReplacementLoad; + return restart; } public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long validationDeadlineNanos) { @@ -1323,6 +1344,10 @@ public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long if (replacementMethod == BungeeMethod.HTTP) restart.previous.reservePreparedHttpHandoff(restart.replacement); } + if (restart.replacement != null && restart.replacementLoadDeferred) { + restart.replacement.loadForReplacement(); + restart.replacementLoadDeferred = false; + } if (restart.replacement != null) restart.replacement.validateTransport(validationDeadlineNanos); // Same-Redis handoff fences callbacks and joins the retiring listener. Those // waits are bounded, but they must never run on the Bukkit publication task. @@ -1402,6 +1427,12 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { } catch (RuntimeException handoffFailure) { backendProxyHandler = restart.previous; restart.replacement.abortStagedInboundTo(restart.previous); + if (restart.requiresWorkerRollback()) { + // The Control worker owns exclusive socket/MQTT teardown and + // predecessor reconnection. Publication has only restored the + // Bukkit-visible handler and fenced staged inbound callbacks. + throw handoffFailure; + } // A validated Redis promotion may already own accepted, deduplicated // replay envelopes even though inbound publication has not opened. Move // those envelopes back before closing the staged replacement, whose @@ -1501,26 +1532,47 @@ public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart res return; } } + RuntimeException cleanupFailure = null; if (restart.redisHandoffCompleted && restart.previous != null && restart.replacement != null) { // Preserve the promoted replacement's pre-publication replay queue before // its close fences and clears it, then retire that staged listener. - restart.previous.restoreAfterFailedReplacement(restart.replacement); + try { + restart.previous.restoreAfterFailedReplacement(restart.replacement); + } catch (RuntimeException failure) { + cleanupFailure = failure; + } } if (restart.replacement != null) { - restart.replacement.abortStagedInboundTo(restart.previous); - if (requiresAsyncStagedReplacementClose(restart)) closeStagedRedisReplacementAsync(restart.replacement); - else restart.replacement.close(); + try { + restart.replacement.abortStagedInboundTo(restart.previous); + if (requiresAsyncStagedReplacementClose(restart)) closeStagedRedisReplacementAsync(restart.replacement); + else restart.replacement.close(); + } catch (RuntimeException failure) { + if (cleanupFailure == null) cleanupFailure = failure; + else cleanupFailure.addSuppressed(failure); + } } if (backendProxyHandler == restart.previous && restart.previous != null && (restart.previousPrepared || restart.redisHandoffCompleted || restart.previous.getMethod() == BungeeMethod.PLUGINMESSAGING)) { - if (!restart.redisHandoffCompleted) restart.previous.restoreAfterFailedReplacement(); + if (!restart.redisHandoffCompleted) try { + restart.previous.restoreAfterFailedReplacement(); + } catch (RuntimeException failure) { + if (cleanupFailure == null) cleanupFailure = failure; + else cleanupFailure.addSuppressed(failure); + } } if (backendProxyHandler == restart.previous && restart.previous != null && restart.presenceStoppedForDisablePreparation) { - restart.previous.restorePresenceAfterFailedDisablePreparation(); + try { + restart.previous.restorePresenceAfterFailedDisablePreparation(); + } catch (RuntimeException failure) { + if (cleanupFailure == null) cleanupFailure = failure; + else cleanupFailure.addSuppressed(failure); + } } restart.finished = true; + if (cleanupFailure != null) throw cleanupFailure; } private boolean requiresAsyncStagedReplacementClose(BackendProxyRestart restart) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index e4afbc28f..8847eeebb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -202,6 +202,19 @@ public boolean prepareForReplacement(BungeeMethod replacementMethod) { } public boolean prepareForReplacement(BungeeMethod replacementMethod, long deadlineNanos) { + // SocketHandler swallows a listener bind failure. A same-method replacement + // must therefore retire its listener before the staged handler is started; + // rollback recreates this prepared transport if validation later fails. + if (method == BungeeMethod.SOCKETS && replacementMethod == BungeeMethod.SOCKETS) { + transportManager.prepareForReplacement(); + return true; + } + if (method == BungeeMethod.MQTT && replacementMethod == BungeeMethod.MQTT) { + // A duplicate MQTT ClientID disconnects the live broker session, so stage + // only after retiring the predecessor and restore it on validation rollback. + transportManager.prepareForReplacement(); + return true; + } if (method == BungeeMethod.HTTP) { transportManager.prepareForReplacement(); return true; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 3571fbd25..924380d34 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -218,9 +218,11 @@ public void prepareForReplacement() { if (preparedTransport != candidate) throw failure; HttpBackendProxyTransport http = candidate instanceof HttpBackendProxyTransport prepared ? prepared : null; - if (http != null && !http.isClosedForReplacement()) { + MqttBackendProxyTransport mqtt = candidate instanceof MqttBackendProxyTransport prepared + ? prepared : null; + if ((http != null && !http.isClosedForReplacement()) || (mqtt != null && mqtt.isConnected())) { // A failed flush deliberately restarts the existing connector. Reinstall - // that live instance instead of creating a second directory owner. + // that live instance instead of creating a second directory owner/client. transport = candidate; preparedTransport = null; while (!preparedSends.isEmpty() && transport.send(preparedSends.peekFirst())) @@ -528,6 +530,12 @@ public synchronized void restorePreparedTransport() { if (transport != null) return; if (preparedTransport instanceof HttpBackendProxyTransport http) { transport = http.recreatePrepared(); + } else if (preparedTransport instanceof SocketBackendProxyTransport socket) { + socket.restoreAfterFailedReplacement(); + transport = socket; + } else if (preparedTransport instanceof MqttBackendProxyTransport mqtt) { + mqtt.restoreAfterFailedReplacement(); + transport = mqtt; } else { throw new IllegalStateException("Prepared backend proxy transport cannot be restored"); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java index 5250c002c..85e2d1253 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/MqttBackendProxyTransport.java @@ -15,7 +15,13 @@ public class MqttBackendProxyTransport implements BackendProxyTransport { private final VotingPluginMain plugin; @Getter private MqttHandler mqttHandler; + private GlobalMessageHandler messageHandler; private String publishTopic; + private String subscriptionTopic; + private String clientId; + private String brokerUrl; + private String username; + private String password; public MqttBackendProxyTransport(VotingPluginMain plugin) { this.plugin = plugin; @@ -24,15 +30,18 @@ public MqttBackendProxyTransport(VotingPluginMain plugin) { @Override public void start(GlobalMessageHandler messageHandler) { try { + this.messageHandler = messageHandler; publishTopic = plugin.getBungeeSettings().getMqttPrefix() + "votingplugin/servers/proxy"; - String id = plugin.getBungeeSettings().getMqttClientID(); - if (id.isEmpty()) { - id = plugin.getOptions().getServer(); + subscriptionTopic = plugin.getBungeeSettings().getMqttPrefix() + "votingplugin/servers/" + + plugin.getOptions().getServer(); + clientId = plugin.getBungeeSettings().getMqttClientID(); + if (clientId.isEmpty()) { + clientId = plugin.getOptions().getServer(); } - mqttHandler = new MqttHandler(new MqttServerComm(id, plugin.getBungeeSettings().getMqttBrokerURL(), - plugin.getBungeeSettings().getMqttUsername(), plugin.getBungeeSettings().getMqttPassword()), 2); - mqttHandler.subscribeEnvelopes(plugin.getBungeeSettings().getMqttPrefix() + "votingplugin/servers/" - + plugin.getOptions().getServer(), (topic, envelope) -> messageHandler.onMessage(envelope)); + brokerUrl = plugin.getBungeeSettings().getMqttBrokerURL(); + username = plugin.getBungeeSettings().getMqttUsername(); + password = plugin.getBungeeSettings().getMqttPassword(); + startCapturedConnection(); } catch (MqttException e) { throw new IllegalStateException("MQTT backend proxy transport initialization failed", e); } catch (Exception e) { @@ -40,11 +49,39 @@ public void start(GlobalMessageHandler messageHandler) { } } + protected MqttHandler createMqttHandler(MqttServerComm server) throws MqttException { + return new MqttHandler(server, 2); + } + + protected MqttServerComm createMqttServerComm() throws MqttException { + return new MqttServerComm(clientId, brokerUrl, username, password); + } + + private void startCapturedConnection() throws Exception { + MqttHandler candidate = createMqttHandler(createMqttServerComm()); + try { + candidate.subscribeEnvelopes(subscriptionTopic, (topic, envelope) -> messageHandler.onMessage(envelope)); + mqttHandler = candidate; + } catch (Exception subscriptionFailure) { + try { + candidate.disconnect(); + } catch (Exception disconnectFailure) { + subscriptionFailure.addSuppressed(disconnectFailure); + } + throw subscriptionFailure; + } + } + @Override public void validate() { if (mqttHandler == null) throw new IllegalStateException("MQTT backend proxy transport initialization failed"); } + /** Returns whether a failed disconnect left the existing broker session usable. */ + public boolean isConnected() { + return mqttHandler != null && mqttHandler.isConnected(); + } + @Override public boolean send(JsonEnvelope envelope) { if (mqttHandler == null) { @@ -86,4 +123,16 @@ public void close() { } } } + + /** Reconnects a prepared predecessor with its original identity after rollback. */ + public void restoreAfterFailedReplacement() { + if (messageHandler == null || clientId == null || brokerUrl == null || subscriptionTopic == null) { + throw new IllegalStateException("MQTT backend proxy transport cannot be restored before startup"); + } + try { + startCapturedConnection(); + } catch (Exception e) { + throw new IllegalStateException("MQTT backend proxy transport restoration failed", e); + } + } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java index f24abad98..0fe6c8fcc 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java @@ -1,6 +1,9 @@ package com.bencodez.votingplugin.backendproxy.transport; import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.ServerSocket; import com.bencodez.simpleapi.encryption.EncryptionHandler; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; @@ -8,6 +11,7 @@ import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; import com.bencodez.simpleapi.servercomm.sockets.SocketReceiver; +import com.bencodez.simpleapi.servercomm.sockets.SocketServer; import com.bencodez.votingplugin.VotingPluginMain; import lombok.Getter; @@ -19,6 +23,13 @@ public class SocketBackendProxyTransport implements BackendProxyTransport { private ClientHandler clientHandler; @Getter private SocketHandler socketHandler; + private GlobalMessageHandler messageHandler; + private EncryptionHandler encryptionHandler; + private String bungeeHost; + private int bungeePort; + private String spigotHost; + private int spigotPort; + private boolean debug; public SocketBackendProxyTransport(VotingPluginMain plugin) { this.plugin = plugin; @@ -26,25 +37,53 @@ public SocketBackendProxyTransport(VotingPluginMain plugin) { @Override public void start(GlobalMessageHandler messageHandler) { - EncryptionHandler encryptionHandler = new EncryptionHandler(plugin.getName(), + this.messageHandler = messageHandler; + encryptionHandler = new EncryptionHandler(plugin.getName(), new File(plugin.getDataFolder(), "secretkey.key")); - clientHandler = new ClientHandler(plugin.getBungeeSettings().getBungeeServerHost(), - plugin.getBungeeSettings().getBungeeServerPort(), encryptionHandler, - plugin.getBungeeSettings().isBungeeDebug()); - socketHandler = new SocketHandler("vp-socket", plugin.getBungeeSettings().getSpigotServerHost(), - plugin.getBungeeSettings().getSpigotServerPort(), encryptionHandler, - plugin.getBungeeSettings().isBungeeDebug()) { - @Override - public void log(String str) { - plugin.getLogger().info(str); - } - }; - socketHandler.add(new SocketReceiver() { - @Override - public void onReceiveEnvelope(JsonEnvelope envelope) { - messageHandler.onMessage(envelope); - } - }); + bungeeHost = plugin.getBungeeSettings().getBungeeServerHost(); + bungeePort = plugin.getBungeeSettings().getBungeeServerPort(); + spigotHost = plugin.getBungeeSettings().getSpigotServerHost(); + spigotPort = plugin.getBungeeSettings().getSpigotServerPort(); + debug = plugin.getBungeeSettings().isBungeeDebug(); + startConnections(); + } + + private void startConnections() { + clientHandler = new ClientHandler(bungeeHost, bungeePort, encryptionHandler, debug); + try { + verifyListenerPortAvailable(); + socketHandler = new SocketHandler("vp-socket", spigotHost, spigotPort, encryptionHandler, debug) { + @Override + public void log(String str) { + plugin.getLogger().info(str); + } + }; + socketHandler.add(new SocketReceiver() { + @Override + public void onReceiveEnvelope(JsonEnvelope envelope) { + messageHandler.onMessage(envelope); + } + }); + } catch (RuntimeException failure) { + clientHandler.stopConnection(); + clientHandler = null; + throw failure; + } + } + + /** + * SocketHandler logs and closes itself when its constructor cannot bind. Probe + * first so replacement validation cannot mistake that swallowed failure for a + * live listener. + */ + private void verifyListenerPortAvailable() { + try (ServerSocket probe = new ServerSocket()) { + probe.setReuseAddress(false); + probe.bind(new InetSocketAddress(spigotHost, spigotPort)); + } catch (IOException unavailable) { + throw new IllegalStateException("Socket backend proxy listener is unavailable at " + spigotHost + ":" + + spigotPort, unavailable); + } } @Override @@ -65,13 +104,43 @@ public void validate() { @Override public void close() { - if (socketHandler != null) { - socketHandler.closeConnection(); - socketHandler = null; + try { + closeSocketListener(); + } finally { + if (clientHandler != null) { + clientHandler.stopConnection(); + clientHandler = null; + } } - if (clientHandler != null) { - clientHandler.stopConnection(); - clientHandler = null; + } + + private void closeSocketListener() { + SocketHandler closing = socketHandler; + socketHandler = null; + if (closing == null) return; + SocketServer server = closing.getServer(); + closing.closeConnection(); + if (server == null) return; + try { + server.join(1000L); + if (server.isAlive()) { + throw new IllegalStateException("Socket backend proxy listener did not stop before replacement"); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while retiring the socket backend proxy listener", interrupted); + } finally { + // SocketServer's accept-error recovery can race close() and re-bind once + // before it observes its stopped flag. Always close the final socket, even + // after a timeout or interruption leaves the worker incomplete. + server.close(); + } + } + + public void restoreAfterFailedReplacement() { + if (messageHandler == null || encryptionHandler == null) { + throw new IllegalStateException("Socket backend proxy transport cannot be restored before startup"); } + startConnections(); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 48738ef52..94d825562 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -149,9 +149,15 @@ private void reloadConfiguration(String fileName, boolean proxyMethodOnly) throw try { VotingPluginMain.BackendProxyRestart prepared = preparedRestart.get(); if (preparationAbandoned.get()) { - try { - if (prepared != null) plugin.abortBackendProxyHandlerRestart(prepared); - } finally { rollbackAbortSettled.countDown(); } + // The Control worker performs transport restoration after this + // Bukkit callback has marked the staged handler unavailable. + if (prepared != null && prepared.requiresWorkerRollback()) { + plugin.requestBackendProxyHandlerRestartAbandonment(prepared); + } else if (prepared != null) { + try { + plugin.abortBackendProxyHandlerRestart(prepared); + } finally { rollbackAbortSettled.countDown(); } + } } } finally { preparationState.set(2); @@ -212,14 +218,24 @@ private void reloadConfiguration(String fileName, boolean proxyMethodOnly) throw } VotingPluginMain.BackendProxyRestart prepared = restart; try { - Future abort = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { - try { plugin.abortBackendProxyHandlerRestart(prepared); } - finally { rollbackAbortSettled.countDown(); } - return null; - }); - abort.get(5, TimeUnit.SECONDS); + if (prepared.requiresWorkerRollback()) { + // This worker owns the exclusive socket/MQTT teardown and restoration. + // Bukkit state was already fenced by requestBackendProxyHandlerRestartAbandonment. + plugin.abortBackendProxyHandlerRestart(prepared); + rollbackAbortSettled.countDown(); + } else { + Future abort = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { + try { plugin.abortBackendProxyHandlerRestart(prepared); } + finally { rollbackAbortSettled.countDown(); } + return null; + }); + abort.get(5, TimeUnit.SECONDS); + } finishPendingBackendProxyRollback(); - } catch (Exception cleanupFailure) { failure.addSuppressed(cleanupFailure); } + } catch (Exception cleanupFailure) { + rollbackAbortSettled.countDown(); + failure.addSuppressed(cleanupFailure); + } } if (pendingRollback != null && restart == null) { rollbackAbortSettled.countDown(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index 62b65ed50..be99106a3 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -19,6 +19,7 @@ import java.lang.reflect.Field; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.bukkit.configuration.file.FileConfiguration; import org.junit.jupiter.api.Test; @@ -153,6 +154,48 @@ void preparesRedisBeforePublishingAndRetiresItOffTheBukkitThread() throws Except assertTrue(closeThread.get().startsWith("VotingPlugin-Retired-Backend")); } + @Test + void loadsDeferredSocketReplacementAfterPredecessorPreparation() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + when(replacement.getMethod()).thenReturn(BungeeMethod.SOCKETS); + when(previous.prepareForReplacement(org.mockito.ArgumentMatchers.eq(BungeeMethod.SOCKETS), + org.mockito.ArgumentMatchers.anyLong())).thenReturn(true); + VotingPluginMain.BackendProxyRestart restart = restart(previous, replacement, true); + setRestartField(restart, "replacementLoadDeferred", true); + + plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + TimeUnit.SECONDS.toNanos(1)); + + org.mockito.InOrder order = org.mockito.Mockito.inOrder(previous, replacement); + order.verify(previous).prepareForReplacement(org.mockito.ArgumentMatchers.eq(BungeeMethod.SOCKETS), + org.mockito.ArgumentMatchers.anyLong()); + order.verify(replacement).loadForReplacement(); + order.verify(replacement).validateTransport(org.mockito.ArgumentMatchers.anyLong()); + } + + @Test + void loadsDeferredMqttReplacementAfterPredecessorPreparation() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + when(replacement.getMethod()).thenReturn(BungeeMethod.MQTT); + when(previous.prepareForReplacement(org.mockito.ArgumentMatchers.eq(BungeeMethod.MQTT), + org.mockito.ArgumentMatchers.anyLong())).thenReturn(true); + VotingPluginMain.BackendProxyRestart restart = restart(previous, replacement, true); + setRestartField(restart, "replacementLoadDeferred", true); + + plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + TimeUnit.SECONDS.toNanos(1)); + + org.mockito.InOrder order = org.mockito.Mockito.inOrder(previous, replacement); + order.verify(previous).prepareForReplacement(org.mockito.ArgumentMatchers.eq(BungeeMethod.MQTT), + org.mockito.ArgumentMatchers.anyLong()); + order.verify(replacement).loadForReplacement(); + order.verify(replacement).validateTransport(org.mockito.ArgumentMatchers.anyLong()); + } + @Test void retiresNonRedisNetworkTransportOffTheBukkitThread() throws Exception { VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); @@ -214,6 +257,76 @@ void restoresWorkerRetiredRedisListenerWhenPublicationIsAborted() throws Excepti verify(previous).restoreAfterFailedReplacement(replacement); } + @Test + void controlWorkerRollbackRestoresPreparedPredecessorAfterStagedCloseFailure() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + when(previous.getMethod()).thenReturn(BungeeMethod.MQTT); + when(replacement.getMethod()).thenReturn(BungeeMethod.MQTT); + doThrow(new IllegalStateException("candidate close failed")).when(replacement).close(); + AtomicReference restoreThread = new AtomicReference<>(); + doAnswer(ignored -> { + restoreThread.set(Thread.currentThread().getName()); + return null; + }).when(previous).restoreAfterFailedReplacement(); + VotingPluginMain.BackendProxyRestart restart = restart(previous, replacement, true); + setRestartField(restart, "previousPrepared", true); + assertTrue(restart.requiresWorkerRollback()); + AtomicReference failure = new AtomicReference<>(); + Thread controlWorker = new Thread(() -> { + try { + plugin.abortBackendProxyHandlerRestart(restart); + } catch (RuntimeException expected) { + failure.set(expected); + } + }, "votingplugin-control-backend"); + + controlWorker.start(); + controlWorker.join(TimeUnit.SECONDS.toMillis(1)); + + assertFalse(controlWorker.isAlive()); + assertTrue(failure.get() instanceof IllegalStateException); + verify(replacement).close(); + verify(previous).restoreAfterFailedReplacement(); + assertTrue(restoreThread.get().startsWith("votingplugin-control-backend")); + } + + @Test + void exclusiveHandoffFailureDefersRollbackNetworkWorkToControlWorker() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BackendProxyHandler replacement = mock(BackendProxyHandler.class); + setBackendProxyHandler(plugin, previous); + when(previous.getMethod()).thenReturn(BungeeMethod.SOCKETS); + when(replacement.getMethod()).thenReturn(BungeeMethod.SOCKETS); + doThrow(new IllegalStateException("handoff admission failed")) + .when(previous).completeHttpHandoff(replacement); + VotingPluginMain.BackendProxyRestart restart = restart(previous, replacement, true); + setRestartField(restart, "previousPrepared", true); + + assertThrows(IllegalStateException.class, () -> plugin.completeBackendProxyHandlerRestart(restart)); + + verify(replacement).abortStagedInboundTo(previous); + verify(replacement, never()).close(); + verify(previous, never()).restoreAfterFailedReplacement(); + AtomicReference restoreThread = new AtomicReference<>(); + doAnswer(ignored -> { + restoreThread.set(Thread.currentThread().getName()); + return null; + }).when(previous).restoreAfterFailedReplacement(); + Thread controlWorker = new Thread(() -> plugin.abortBackendProxyHandlerRestart(restart), + "votingplugin-control-backend"); + controlWorker.start(); + controlWorker.join(TimeUnit.SECONDS.toMillis(1)); + + assertFalse(controlWorker.isAlive()); + verify(replacement).close(); + verify(previous).restoreAfterFailedReplacement(); + assertTrue(restoreThread.get().startsWith("votingplugin-control-backend")); + } + @Test void rollbackRestoresPromotedRedisReplayBeforeClosingReplacement() throws Exception { VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); @@ -485,4 +598,11 @@ private void setField(VotingPluginMain plugin, String name, Object value) throws field.setAccessible(true); field.set(plugin, value); } + + private void setRestartField(VotingPluginMain.BackendProxyRestart restart, String name, Object value) + throws Exception { + Field field = VotingPluginMain.BackendProxyRestart.class.getDeclaredField(name); + field.setAccessible(true); + field.set(restart, value); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index b99c3a863..265d3a04b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -1,6 +1,7 @@ package com.bencodez.votingplugin.backendproxy; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -22,6 +23,8 @@ import static org.mockito.ArgumentMatchers.eq; import java.lang.reflect.Field; +import java.net.ServerSocket; +import java.nio.file.Path; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; @@ -30,6 +33,8 @@ import java.util.logging.Logger; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.eclipse.paho.client.mqttv3.MqttException; import com.bencodez.simpleapi.scheduler.BukkitScheduler; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; @@ -37,6 +42,7 @@ import com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessageHandler; import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; +import com.bencodez.simpleapi.servercomm.mqtt.MqttServerComm; import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; import com.bencodez.simpleapi.servercomm.redis.RedisHandler; import com.bencodez.votingplugin.backendproxy.global.BackendGlobalDataSync; @@ -429,6 +435,43 @@ void releasesSocketListenerBeforeSamePortReplacement() throws Exception { assertNull(handler.getSocketHandler()); } + @Test + void rejectsSamePortSocketReplacementUntilThePreviousListenerIsPrepared(@TempDir Path dataFolder) + throws Exception { + int port; + try (ServerSocket allocation = new ServerSocket(0)) { + port = allocation.getLocalPort(); + } + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BungeeSettings settings = mock(BungeeSettings.class); + when(plugin.getBungeeSettings()).thenReturn(settings); + when(plugin.getName()).thenReturn("socket-lifecycle-test"); + when(plugin.getDataFolder()).thenReturn(dataFolder.toFile()); + when(plugin.getLogger()).thenReturn(Logger.getLogger("socket-lifecycle-test")); + when(settings.getBungeeServerHost()).thenReturn("127.0.0.1"); + when(settings.getBungeeServerPort()).thenReturn(1); + when(settings.getSpigotServerHost()).thenReturn("127.0.0.1"); + when(settings.getSpigotServerPort()).thenReturn(port); + + SocketBackendProxyTransport previous = new SocketBackendProxyTransport(plugin); + SocketBackendProxyTransport replacement = new SocketBackendProxyTransport(plugin); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + try { + previous.start(messages); + assertThrows(IllegalStateException.class, () -> replacement.start(messages)); + assertNull(replacement.getClientHandler()); + + previous.prepareForReplacement(); + assertDoesNotThrow(() -> replacement.start(messages)); + replacement.close(); + assertDoesNotThrow(previous::restoreAfterFailedReplacement); + assertNotNull(previous.getSocketHandler()); + } finally { + replacement.close(); + previous.close(); + } + } + @Test void keepsRedisSubscriberUntilReplacementIsReady() throws Exception { BackendProxyHandler handler = new BackendProxyHandler(null); @@ -468,31 +511,83 @@ void releasesMqttSubscriberBeforeSameMethodReplacement() throws Exception { } @Test - void failedSameSocketReplacementLeavesExistingTransportAvailable() throws Exception { + void preparesSameSocketReplacementByRetiringTheExistingListener() throws Exception { BackendProxyHandler handler = handlerWithTransport(BungeeMethod.SOCKETS); SocketHandler socket = mock(SocketHandler.class); SocketBackendProxyTransport transport = new SocketBackendProxyTransport(null); setField(transport, "socketHandler", socket); setTransport(handler, transport); - assertFalse(handler.prepareForReplacement(BungeeMethod.SOCKETS)); + assertTrue(handler.prepareForReplacement(BungeeMethod.SOCKETS)); - assertSame(socket, handler.getSocketHandler()); - verifyNoInteractions(socket); + assertNull(handler.getSocketHandler()); + verify(socket).closeConnection(); } @Test - void failedSameMqttReplacementLeavesExistingTransportAvailable() throws Exception { + void preparesSameMqttReplacementByRetiringTheExistingClient() throws Exception { BackendProxyHandler handler = handlerWithTransport(BungeeMethod.MQTT); MqttHandler mqtt = mock(MqttHandler.class); MqttBackendProxyTransport transport = new MqttBackendProxyTransport(null); setField(transport, "mqttHandler", mqtt); setTransport(handler, transport); - assertFalse(handler.prepareForReplacement(BungeeMethod.MQTT)); + assertTrue(handler.prepareForReplacement(BungeeMethod.MQTT)); - assertSame(mqtt, handler.getMqttHandler()); - verifyNoInteractions(mqtt); + assertNull(handler.getMqttHandler()); + verify(mqtt).disconnect(); + } + + @Test + void restoresPreparedMqttClientWhenReplacementValidationIsAbandoned() throws Exception { + BackendProxyTransportManager manager = new BackendProxyTransportManager(null); + MqttBackendProxyTransport mqtt = mock(MqttBackendProxyTransport.class); + setField(manager, "preparedTransport", mqtt); + + manager.restoreAfterFailedReplacement(); + + assertSame(mqtt, transport(manager)); + verify(mqtt).restoreAfterFailedReplacement(); + } + + @Test + void failedMqttDisconnectKeepsAStillConnectedPredecessor() throws Exception { + BackendProxyTransportManager manager = new BackendProxyTransportManager(null); + MqttBackendProxyTransport mqtt = mock(MqttBackendProxyTransport.class); + doThrow(new IllegalStateException("disconnect failed")).when(mqtt).prepareForReplacement(); + when(mqtt.isConnected()).thenReturn(true); + setField(manager, "transport", mqtt); + + assertThrows(IllegalStateException.class, manager::prepareForReplacement); + + assertSame(mqtt, transport(manager)); + verify(mqtt, never()).restoreAfterFailedReplacement(); + } + + @Test + void disconnectsPartialMqttClientWhenSubscriptionFails() throws Exception { + MqttHandler partial = mock(MqttHandler.class); + doThrow(new MqttException(0)).when(partial).subscribeEnvelopes(any(), any()); + MqttBackendProxyTransport transport = new MqttBackendProxyTransport(null) { + @Override + protected MqttServerComm createMqttServerComm() { + return mock(MqttServerComm.class); + } + + @Override + protected MqttHandler createMqttHandler(MqttServerComm server) { + return partial; + } + }; + setField(transport, "messageHandler", mock(GlobalMessageHandler.class)); + setField(transport, "clientId", "backend-1"); + setField(transport, "brokerUrl", "tcp://broker.invalid:1883"); + setField(transport, "subscriptionTopic", "votingplugin/servers/backend-1"); + + assertThrows(IllegalStateException.class, transport::restoreAfterFailedReplacement); + + assertNull(transport.getMqttHandler()); + verify(partial).disconnect(); } @Test @@ -982,8 +1077,17 @@ private BackendProxyTransport transport(BackendProxyTransportManager manager) th } private void setField(Object target, String name, Object value) throws Exception { - Field field = target.getClass().getDeclaredField(name); - field.setAccessible(true); - field.set(target, value); + Class type = target.getClass(); + while (type != null) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + return; + } catch (NoSuchFieldException ignored) { + type = type.getSuperclass(); + } + } + throw new NoSuchFieldException(name); } } From 32f7c1b766d1d54184bc88ae474b7ef4e9485d14 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 12 Sep 2026 23:05:03 -0600 Subject: [PATCH 099/136] Harden presence and legacy multi-proxy retries --- .../presence/BackendPresenceManager.java | 45 ++++++++---- .../votingplugin/proxy/VotingPluginProxy.java | 3 +- .../BackendProxyHandlerLifecycleTest.java | 71 +++++++++++++++++++ .../tests/VotingPluginProxyTest.java | 30 ++++++++ 4 files changed, 135 insertions(+), 14 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java index 60331ee07..91c601d81 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java @@ -37,6 +37,8 @@ public class BackendPresenceManager { private final Object lifecycleLock = new Object(); private boolean reporting; + /** Invalidates delayed lifecycle callbacks when a fresh incarnation starts. */ + private long lifecycleGeneration; private String server; private UUID incarnationId; private long startedAt; @@ -60,6 +62,7 @@ public void start() { } String configuredServer = plugin.getBungeeSettings().getServer(); synchronized (lifecycleLock) { + lifecycleGeneration++; long now = System.currentTimeMillis(); incarnationId = UUID.randomUUID(); startedAt = now; @@ -112,11 +115,15 @@ public void stopForDisable(long deadlineNanos) { // retain the direct path used by the non-Control lifecycle. if (method == BungeeMethod.PLUGINMESSAGING && plugin != null && plugin.getServer() != null && !plugin.getServer().isPrimaryThread()) { + long expectedGeneration; + synchronized (lifecycleLock) { + expectedGeneration = lifecycleGeneration; + } CompletableFuture scheduled = new CompletableFuture<>(); try { plugin.getBukkitScheduler().executeOrScheduleSync(plugin, () -> { try { - stop(true); + stop(true, expectedGeneration); scheduled.complete(null); } catch (Throwable failure) { scheduled.completeExceptionally(failure); @@ -138,7 +145,18 @@ public void stopForDisable(long deadlineNanos) { } private void stop(boolean requireStoppedDelivery) { + stop(requireStoppedDelivery, -1L); + } + + /** + * Stops only the generation captured by a delayed callback. A cancelled future + * does not necessarily cancel a task already queued on the Bukkit scheduler. + */ + private void stop(boolean requireStoppedDelivery, long expectedGeneration) { synchronized (lifecycleLock) { + if (expectedGeneration >= 0L && lifecycleGeneration != expectedGeneration) { + return; + } String activeServer = server; UUID activeIncarnationId = incarnationId; long activeStartedAt = startedAt; @@ -149,18 +167,21 @@ private void stop(boolean requireStoppedDelivery) { heartbeatTask.cancel(false); heartbeatTask = null; } - if (wasReporting && activeServer != null && activeIncarnationId != null) { - JsonEnvelope stopped = VotingPluginWire.backendStopped(activeServer, activeIncarnationId, - activeStartedAt, nextTimestamp()); - if (requireStoppedDelivery) globalMessageHandler.sendMessage(stopped); - else send(stopped); + try { + if (wasReporting && activeServer != null && activeIncarnationId != null) { + JsonEnvelope stopped = VotingPluginWire.backendStopped(activeServer, activeIncarnationId, + activeStartedAt, nextTimestamp()); + if (requireStoppedDelivery) globalMessageHandler.sendMessage(stopped); + else send(stopped); + } + } finally { + incarnationId = null; + lastResyncRequestId = null; + lastResyncRequestAtNanos = 0L; + lastSnapshotRequestId = null; + lastSnapshotRequestAtNanos = 0L; + playerSessions.clear(); } - incarnationId = null; - lastResyncRequestId = null; - lastResyncRequestAtNanos = 0L; - lastSnapshotRequestId = null; - lastSnapshotRequestAtNanos = 0L; - playerSessions.clear(); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 1dd43343b..e35545e53 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -5627,10 +5627,9 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim if (recipients.isEmpty()) { // Older peers do not understand acknowledgements. Preserve their historical // fire-and-forget route instead of creating an outbox they can never ACK. - multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(player, uuid, service, time, + return multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(player, uuid, service, time, false, realVote, totals == null ? "" : totals.toString(), findLiveVoteId(retryState), false, false, 1, 1, getConfig().getProxyServerName()), legacyRecipients); - return true; } VoteTimeQueue outbox = queuedVote; if (outbox == null) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 265d3a04b..a7e81124f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -272,10 +272,67 @@ void preparedDisablePropagatesRejectedStoppedPresence() throws Exception { setField(presence, "server", "backend-1"); setField(presence, "incarnationId", java.util.UUID.randomUUID()); setField(presence, "startedAt", 1L); + @SuppressWarnings("unchecked") + java.util.concurrent.ConcurrentHashMap sessions = + (java.util.concurrent.ConcurrentHashMap) getField( + presence, "playerSessions"); + sessions.put("player", com.bencodez.votingplugin.backendproxy.presence.BackendPlayerPresenceSession + .create("Player", "550e8400-e29b-41d4-a716-446655440000")); + setField(presence, "lastResyncRequestId", java.util.UUID.randomUUID()); + setField(presence, "lastResyncRequestAtNanos", 42L); + setField(presence, "lastSnapshotRequestId", java.util.UUID.randomUUID()); + setField(presence, "lastSnapshotRequestAtNanos", 84L); doThrow(new IllegalStateException("outbound queue full")).when(messages).sendMessage(any()); assertThrows(IllegalStateException.class, presence::stopForDisable); assertFalse(presence.isReporting()); + assertNull(getField(presence, "incarnationId")); + assertEquals(0, presence.getTrackedSessionCount()); + assertNull(getField(presence, "lastResyncRequestId")); + assertEquals(0L, getField(presence, "lastResyncRequestAtNanos")); + assertNull(getField(presence, "lastSnapshotRequestId")); + assertEquals(0L, getField(presence, "lastSnapshotRequestAtNanos")); + } + + @Test + void delayedPreparedDisableCannotStopANewerPresenceGeneration() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + org.bukkit.Server server = mock(org.bukkit.Server.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + when(plugin.getServer()).thenReturn(server); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + when(server.isPrimaryThread()).thenReturn(false); + AtomicReference queued = new AtomicReference<>(); + org.mockito.Mockito.doAnswer(invocation -> { + queued.set(invocation.getArgument(1)); + return null; + }).when(scheduler).executeOrScheduleSync(eq(plugin), any(Runnable.class)); + + BackendPresenceManager presence = new BackendPresenceManager(plugin, BungeeMethod.PLUGINMESSAGING, messages); + java.util.UUID firstIncarnation = java.util.UUID.randomUUID(); + setField(presence, "lifecycleGeneration", 1L); + setField(presence, "reporting", true); + setField(presence, "server", "backend-1"); + setField(presence, "incarnationId", firstIncarnation); + setField(presence, "startedAt", 1L); + + assertThrows(IllegalStateException.class, + () -> presence.stopForDisable(System.nanoTime() - 1L)); + assertNotNull(queued.get()); + + java.util.UUID replacementIncarnation = java.util.UUID.randomUUID(); + setField(presence, "lifecycleGeneration", 2L); + setField(presence, "reporting", true); + setField(presence, "server", "backend-1"); + setField(presence, "incarnationId", replacementIncarnation); + setField(presence, "startedAt", 2L); + + queued.get().run(); + + assertTrue(presence.isReporting()); + assertEquals(replacementIncarnation, getField(presence, "incarnationId")); + verifyNoInteractions(messages); } @Test @@ -1090,4 +1147,18 @@ private void setField(Object target, String name, Object value) throws Exception } throw new NoSuchFieldException(name); } + + private Object getField(Object target, String name) throws Exception { + Class type = target.getClass(); + while (type != null) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (NoSuchFieldException ignored) { + type = type.getSuperclass(); + } + } + throw new NoSuchFieldException(name); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 5d1b2ce59..fb8cb0f71 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -496,6 +496,36 @@ public Boolean answer(org.mockito.invocation.InvocationOnMock invocation) { assertEquals(java.util.Set.of("proxy2"), queue.element().getMultiProxyRecipients()); } + @Test + void rejectedLegacyOnlyMultiProxySendKeepsLiveVoteForRetry() throws Exception { + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Collections.emptySet()); + Mockito.when(multiProxyHandler.getConfiguredMultiProxyVoteRecipients()) + .thenReturn(new java.util.LinkedHashSet<>(java.util.Set.of("ProxyLegacy"))); + Mockito.when(multiProxyHandler.sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any())).thenReturn(false); + Mockito.when(votingPluginProxy.getConfig().getProxyServerName()).thenReturn("Proxy1"); + + Class retryType = Class.forName("com.bencodez.votingplugin.proxy.VotingPluginProxy$LiveVoteRetryState"); + java.lang.reflect.Constructor constructor = retryType.getDeclaredConstructor(); + constructor.setAccessible(true); + Object retry = constructor.newInstance(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + java.lang.reflect.Field retriesField = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); + retriesField.setAccessible(true); + @SuppressWarnings("unchecked") + java.util.Map retries = + (java.util.Map) retriesField.get(votingPluginProxy); + retries.put(voteId, retry); + java.lang.reflect.Method begin = VotingPluginProxy.class.getDeclaredMethod("beginMultiProxyForwarding", + retryType, VoteTimeQueue.class, String.class, String.class, String.class, long.class, + boolean.class, com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class); + begin.setAccessible(true); + + assertFalse((Boolean) begin.invoke(votingPluginProxy, retry, null, "Player", + "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); + verify(multiProxyHandler).sendMultiProxyEnvelopeAccepted(Mockito.any(), + Mockito.eq(java.util.Set.of("ProxyLegacy"))); + } + @Test void completedAckOutboxKeepsFenceForAlreadyScheduledListenerRetry() throws Exception { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); From a2ad2e4518b689c828e321b39970a48999dc2cf0 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 13 Sep 2026 09:32:10 -0600 Subject: [PATCH 100/136] Harden backend transport cleanup --- .../BackendProxyTransportManager.java | 8 ++- .../transport/HttpBackendProxyTransport.java | 20 +++++-- .../SocketBackendProxyTransport.java | 31 +++++++++-- .../BackendProxyHandlerLifecycleTest.java | 54 +++++++++++++++++++ .../HttpBackendProxyTransportTest.java | 15 ++++++ 5 files changed, 117 insertions(+), 11 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 924380d34..939ea2047 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -472,7 +472,13 @@ private void drainPluginMessageHandoff(long generation) { } target = transport; } - if (!(target instanceof PluginMessagingBackendProxyTransport)) return; + if (!(target instanceof PluginMessagingBackendProxyTransport)) { + synchronized (this) { + if (handoffGeneration == generation) pluginMessageHandoffScheduled = false; + notifyAll(); + } + return; + } try { if (!target.send(envelope)) { schedulePluginMessageHandoff(generation); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 485a19c2f..919c96f54 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -322,18 +322,28 @@ private void initialize(Path directory, String serverId, String configuredCode, if (inboundActive) invokeConnectorLifecycle(replacement, "activateIncoming"); } } - if (discard) replacement.close(); - else startHandoffDrainIfNeeded(); + if (!discard) startHandoffDrainIfNeeded(); } catch (Exception failure) { startupFailure = new IllegalStateException("Secure HTTP backend enrollment or connection failed", failure); plugin.getLogger().severe("Secure HTTP backend transport is unavailable; check the connection code and proxy endpoint"); } finally { credentialRestoreComplete.countDown(); + finishInitialization(installed, replacement, acquired, owner, startupComplete); + } + } + + static void finishInitialization(boolean installed, HttpBackendTransportConnector replacement, + boolean acquired, Semaphore owner, CountDownLatch completion) { + try { if (!installed) { - if (replacement != null) replacement.close(); - if (acquired) owner.release(); + try { + if (replacement != null) replacement.close(); + } finally { + if (acquired) owner.release(); + } } - startupComplete.countDown(); + } finally { + completion.countDown(); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java index 0fe6c8fcc..da30de584 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/SocketBackendProxyTransport.java @@ -104,13 +104,34 @@ public void validate() { @Override public void close() { + close(false); + } + + @Override + public void prepareForReplacement() { + close(true); + } + + private void close(boolean strict) { + RuntimeException failure = null; try { closeSocketListener(); - } finally { - if (clientHandler != null) { - clientHandler.stopConnection(); - clientHandler = null; - } + } catch (RuntimeException listenerFailure) { + failure = listenerFailure; + } + ClientHandler closingClient = clientHandler; + clientHandler = null; + if (closingClient != null) try { + closingClient.stopConnection(); + } catch (RuntimeException clientFailure) { + if (failure == null) failure = clientFailure; + else failure.addSuppressed(clientFailure); + } + if (failure == null) return; + if (strict) throw failure; + if (plugin != null && plugin.getLogger() != null) { + plugin.getLogger().warning("Socket backend proxy transport did not stop cleanly"); + plugin.debug(failure); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index a7e81124f..718113d53 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -492,6 +492,35 @@ void releasesSocketListenerBeforeSamePortReplacement() throws Exception { assertNull(handler.getSocketHandler()); } + @Test + void ordinarySocketCloseToleratesListenerTimeoutButReplacementPreparationRejectsIt() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + SocketHandler ordinarySocket = mock(SocketHandler.class); + com.bencodez.simpleapi.servercomm.sockets.SocketServer ordinaryServer = + mock(com.bencodez.simpleapi.servercomm.sockets.SocketServer.class); + when(ordinarySocket.getServer()).thenReturn(ordinaryServer); + when(ordinaryServer.isAlive()).thenReturn(true); + SocketBackendProxyTransport ordinary = new SocketBackendProxyTransport(plugin); + setField(ordinary, "socketHandler", ordinarySocket); + + assertDoesNotThrow(ordinary::close); + verify(ordinarySocket).closeConnection(); + verify(ordinaryServer).close(); + + SocketHandler replacementSocket = mock(SocketHandler.class); + com.bencodez.simpleapi.servercomm.sockets.SocketServer replacementServer = + mock(com.bencodez.simpleapi.servercomm.sockets.SocketServer.class); + when(replacementSocket.getServer()).thenReturn(replacementServer); + when(replacementServer.isAlive()).thenReturn(true); + SocketBackendProxyTransport replacement = new SocketBackendProxyTransport(plugin); + setField(replacement, "socketHandler", replacementSocket); + + assertThrows(IllegalStateException.class, replacement::prepareForReplacement); + verify(replacementSocket).closeConnection(); + verify(replacementServer).close(); + } + @Test void rejectsSamePortSocketReplacementUntilThePreviousListenerIsPrepared(@TempDir Path dataFolder) throws Exception { @@ -955,6 +984,31 @@ void pluginMessageHandoffRunsThroughTheBukkitScheduler() throws Exception { verify(nextTransport, org.mockito.Mockito.timeout(1000)).send(afterPreparation); } + @Test + void stalePluginMessageHandoffCallbackClearsItsScheduledFlag() throws Exception { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + com.bencodez.simpleapi.scheduler.BukkitScheduler scheduler = + mock(com.bencodez.simpleapi.scheduler.BukkitScheduler.class); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + java.util.concurrent.atomic.AtomicReference scheduled = new java.util.concurrent.atomic.AtomicReference<>(); + doAnswer(invocation -> { + scheduled.set(invocation.getArgument(1)); + return null; + }).when(scheduler).runTask(eq(plugin), any(Runnable.class)); + BackendProxyTransportManager manager = new BackendProxyTransportManager(plugin); + PluginMessagingBackendProxyTransport pluginMessages = mock(PluginMessagingBackendProxyTransport.class); + setField(manager, "transport", pluginMessages); + + manager.send(com.bencodez.simpleapi.servercomm.codec.JsonEnvelope.builder("queued").build()); + assertNotNull(scheduled.get()); + assertTrue((boolean) getField(manager, "pluginMessageHandoffScheduled")); + setField(manager, "transport", null); + + scheduled.get().run(); + + assertFalse((boolean) getField(manager, "pluginMessageHandoffScheduled")); + } + @Test void releasesMysqlSubscriberBeforeSameMethodReplacement() throws Exception { MysqlBackendProxyTransport handler = new MysqlBackendProxyTransport(null); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 24c88bfd3..35138136b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -132,6 +132,21 @@ void failedStartupTransferRetainsTheUnacceptedEnvelope() { assertEquals(java.util.List.of(pending), new java.util.ArrayList<>(queue)); } + @Test + void failedUninstalledConnectorCloseStillReleasesOwnerAndCompletesStartup() { + HttpBackendTransportConnector connector = mock(HttpBackendTransportConnector.class); + org.mockito.Mockito.doThrow(new IllegalStateException("close failed")).when(connector).close(); + java.util.concurrent.Semaphore owner = new java.util.concurrent.Semaphore(0); + CountDownLatch completion = new CountDownLatch(1); + + assertThrows(IllegalStateException.class, + () -> HttpBackendProxyTransport.finishInitialization(false, connector, true, owner, completion)); + + verify(connector).close(); + assertEquals(1, owner.availablePermits()); + assertEquals(0L, completion.getCount()); + } + @Test @SuppressWarnings("unchecked") void orderlyShutdownFlushesRecoveryQueueBeforeHandoffQueue() throws Exception { From 283c2b2c4f6b6d36c1f6b1b968546f360f847e80 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 13 Sep 2026 09:45:22 -0600 Subject: [PATCH 101/136] Harden proxy compatibility edge cases --- .../votingplugin/proxy/VotingPluginProxy.java | 4 +- .../proxy/bungee/BungeeJsonVoteCache.java | 6 +- .../proxy/bungee/VoteEventBungee.java | 3 +- .../proxy/multiproxy/MultiProxyHandler.java | 25 ++++-- .../proxy/velocity/VelocityJsonVoteCache.java | 6 +- .../proxy/velocity/VoteEventVelocity.java | 3 +- .../MultiProxyHandlerLifecycleTest.java | 14 ++-- .../tests/JsonVoteCacheLegacyRemovalTest.java | 22 ++++++ .../tests/ProxyVoteEventNullServiceTest.java | 78 +++++++++++++++++++ .../tests/VotingPluginProxyTest.java | 5 +- 10 files changed, 145 insertions(+), 21 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/ProxyVoteEventNullServiceTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index e35545e53..072832f4e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -5627,9 +5627,9 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim if (recipients.isEmpty()) { // Older peers do not understand acknowledgements. Preserve their historical // fire-and-forget route instead of creating an outbox they can never ACK. - return multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(player, uuid, service, time, + return multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.vote(player, uuid, service, time, false, realVote, totals == null ? "" : totals.toString(), findLiveVoteId(retryState), false, false, - 1, 1, getConfig().getProxyServerName()), legacyRecipients); + 1, 1), legacyRecipients); } VoteTimeQueue outbox = queuedVote; if (outbox == null) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java index 0185f5135..9173e279e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java @@ -178,7 +178,11 @@ public int getVotePartyCache(String server) { public Collection getPendingVotePartyRewardServers() { Collection encoded = getKeys("VoteParty.PendingRewards"); Collection servers = new ArrayList<>(); - if (encoded != null) for (String key : encoded) servers.add(decodeServerKey(key)); + if (encoded != null) for (String key : encoded) try { + servers.add(decodeServerKey(key)); + } catch (IllegalArgumentException malformedKey) { + // Preserve later valid deliveries when one persisted key is malformed. + } return servers; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VoteEventBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VoteEventBungee.java index fa826804a..f01f01cb1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VoteEventBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VoteEventBungee.java @@ -31,7 +31,8 @@ public VoteEventBungee(VotingPluginBungee plugin) { @EventHandler public void onVote(VotifierEvent event) { Vote vote = event.getVote(); - String serviceSite = vote.getServiceName().isEmpty() ? "Empty" : vote.getServiceName(); + String serviceName = vote.getServiceName(); + String serviceSite = serviceName == null || serviceName.isEmpty() ? "Empty" : serviceName; plugin.getProxy().getScheduler().runAsync(plugin, new RetryingVote(vote.getUsername(), serviceSite)); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java index bbcd6cd37..ddddb2e4f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java @@ -320,12 +320,17 @@ public void acknowledgeMultiProxyVoteRetirement(UUID voteId, String origin) { /** Returns every configured remote proxy recipient, independent of version. */ public synchronized Set getConfiguredMultiProxyVoteRecipients() { - Collection source = getMultiProxyMethod().equals(MultiProxyMethod.SOCKETS) + return new LinkedHashSet<>(configuredMultiProxyRecipientNames().values()); + } + + private synchronized Map configuredMultiProxyRecipientNames() { + Collection source = MultiProxyMethod.SOCKETS.equals(getMultiProxyMethod()) ? getMultiProxyServers() : getProxyServers(); - Set recipients = new LinkedHashSet<>(); + Map recipients = new LinkedHashMap<>(); if (source == null) return recipients; for (String server : source) { - if (server != null && !server.isBlank()) recipients.add(server.toLowerCase(Locale.ROOT)); + if (server != null && !server.isBlank()) + recipients.putIfAbsent(server.toLowerCase(Locale.ROOT), server); } return recipients; } @@ -334,8 +339,10 @@ public synchronized Set getConfiguredMultiProxyVoteRecipients() { public synchronized Set getMultiProxyVoteRecipients() { long now = capabilityNowMillis(); acknowledgedVoteCapabilityPeers.entrySet().removeIf(entry -> entry.getValue() <= now); - Set recipients = getConfiguredMultiProxyVoteRecipients(); - recipients.retainAll(acknowledgedVoteCapabilityPeers.keySet()); + Set recipients = new LinkedHashSet<>(); + for (Map.Entry configured : configuredMultiProxyRecipientNames().entrySet()) { + if (acknowledgedVoteCapabilityPeers.containsKey(configured.getKey())) recipients.add(configured.getValue()); + } return recipients; } @@ -485,9 +492,11 @@ public synchronized boolean sendMultiProxyEnvelopeAccepted(JsonEnvelope envelope if (envelope == null) return false; if (recipients == null || recipients.isEmpty()) return false; Map requested = new LinkedHashMap<>(); + Map configuredNames = configuredMultiProxyRecipientNames(); for (String recipient : recipients) { if (recipient != null && !recipient.isBlank()) { - requested.putIfAbsent(recipient.toLowerCase(Locale.ROOT), recipient); + String normalized = recipient.toLowerCase(Locale.ROOT); + requested.putIfAbsent(normalized, configuredNames.getOrDefault(normalized, recipient)); } } if (requested.isEmpty()) return false; @@ -603,7 +612,7 @@ private void handleEnvelope(JsonEnvelope envelope) { String origin = f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_ORIGIN, ""); String recipient = f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_RECIPIENT, ""); if (!recipient.equalsIgnoreCase(getMultiProxyServerName()) || origin.isBlank() - || !getConfiguredMultiProxyVoteRecipients().contains(origin.toLowerCase(Locale.ROOT))) return; + || !configuredMultiProxyRecipientNames().containsKey(origin.toLowerCase(Locale.ROOT))) return; try { onMultiProxyVoteRetirementRequested( UUID.fromString(f.getOrDefault(VotingPluginWire.K_VOTE_ID, "")), origin); @@ -630,7 +639,7 @@ private void handleEnvelope(JsonEnvelope envelope) { int version = Integer.parseInt(f.getOrDefault(VotingPluginWire.K_MULTI_PROXY_ACK_VERSION, "0")); String normalized = recipient.toLowerCase(Locale.ROOT); synchronized (this) { - if (version >= 1 && getConfiguredMultiProxyVoteRecipients().contains(normalized)) { + if (version >= 1 && configuredMultiProxyRecipientNames().containsKey(normalized)) { acknowledgedVoteCapabilityPeers.put(normalized, capabilityNowMillis() + VOTE_CAPABILITY_LEASE_MILLIS); replyRequired = !reply; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java index 34ee45de0..0fe557194 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java @@ -189,7 +189,11 @@ public int getVotePartyCache(String server) { public Collection getPendingVotePartyRewardServers() { Collection encoded = getKeys(getNode("VoteParty", "PendingRewards")); Collection servers = new ArrayList<>(); - if (encoded != null) for (String key : encoded) servers.add(decodeServerKey(key)); + if (encoded != null) for (String key : encoded) try { + servers.add(decodeServerKey(key)); + } catch (IllegalArgumentException malformedKey) { + // Preserve later valid deliveries when one persisted key is malformed. + } return servers; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VoteEventVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VoteEventVelocity.java index cc3a15e28..364490960 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VoteEventVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VoteEventVelocity.java @@ -30,7 +30,8 @@ public VoteEventVelocity(VotingPluginVelocity plugin) { public void onVotifierEvent(VotifierEvent event) { final String serviceSiteVote = event.getVote().getServiceName(); final String name = event.getVote().getUsername(); - plugin.getTimer().execute(new RetryingVote(name, serviceSiteVote.isEmpty() ? "Empty" : serviceSiteVote)); + plugin.getTimer().execute(new RetryingVote(name, + serviceSiteVote == null || serviceSiteVote.isEmpty() ? "Empty" : serviceSiteVote)); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java index bcb7f2eb5..4d9bfce6b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java @@ -66,6 +66,7 @@ void acceptsAQueuedSocketSendWhenTheConfiguredDestinationReturnsNormally() throw void redisSubsetSendPreservesConfiguredChannelCasing() throws Exception { MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); org.mockito.Mockito.when(handler.getMultiProxyMethod()).thenReturn(MultiProxyMethod.REDIS); + org.mockito.Mockito.when(handler.getProxyServers()).thenReturn(List.of("Proxy2")); com.bencodez.simpleapi.servercomm.redis.RedisHandler redis = mock(com.bencodez.simpleapi.servercomm.redis.RedisHandler.class); java.lang.reflect.Field connection = MultiProxyHandler.class.getDeclaredField("multiProxyRedis"); @@ -73,7 +74,8 @@ void redisSubsetSendPreservesConfiguredChannelCasing() throws Exception { connection.set(handler, redis); JsonEnvelope envelope = JsonEnvelope.builder("vote").build(); - assertTrue(handler.sendMultiProxyEnvelopeAccepted(envelope, List.of("Proxy2"))); + assertEquals(java.util.Set.of("Proxy2"), handler.getConfiguredMultiProxyVoteRecipients()); + assertTrue(handler.sendMultiProxyEnvelopeAccepted(envelope, List.of("proxy2"))); verify(redis).publishEnvelope("VotingPluginProxy_Proxy2", envelope); } @@ -171,8 +173,8 @@ void routesOnlyTheOriginatingProxyAcknowledgement() throws Exception { void routesOnlyAuthenticatedTargetedRetirementMessages() throws Exception { MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); org.mockito.Mockito.when(handler.getMultiProxyServerName()).thenReturn("Replica"); - org.mockito.Mockito.doReturn(new java.util.LinkedHashSet<>(List.of("primary"))) - .when(handler).getConfiguredMultiProxyVoteRecipients(); + org.mockito.Mockito.when(handler.getMultiProxyMethod()).thenReturn(MultiProxyMethod.REDIS); + org.mockito.Mockito.when(handler.getProxyServers()).thenReturn(List.of("Primary")); UUID voteId = UUID.randomUUID(); Method handleEnvelope = MultiProxyHandler.class.getDeclaredMethod("handleEnvelope", JsonEnvelope.class); handleEnvelope.setAccessible(true); @@ -209,7 +211,7 @@ void fencesOnlyPeersThatAdvertiseDurableAcknowledgements() throws Exception { handleEnvelope.invoke(handler, VotingPluginWire.multiProxyCapabilities("Capable", 1)); - assertEquals(java.util.Set.of("capable"), handler.getMultiProxyVoteRecipients()); + assertEquals(java.util.Set.of("Capable"), handler.getMultiProxyVoteRecipients()); } @Test @@ -226,10 +228,10 @@ void durableAcknowledgementCapabilityExpiresAndCanBeRenewed() throws Exception { handleEnvelope.setAccessible(true); handleEnvelope.invoke(handler, VotingPluginWire.multiProxyCapabilities("Capable", 1, true)); - assertEquals(java.util.Set.of("capable"), handler.getMultiProxyVoteRecipients()); + assertEquals(java.util.Set.of("Capable"), handler.getMultiProxyVoteRecipients()); assertTrue(handler.getMultiProxyVoteRecipients().isEmpty()); handleEnvelope.invoke(handler, VotingPluginWire.multiProxyCapabilities("Capable", 1, true)); - assertEquals(java.util.Set.of("capable"), handler.getMultiProxyVoteRecipients()); + assertEquals(java.util.Set.of("Capable"), handler.getMultiProxyVoteRecipients()); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/JsonVoteCacheLegacyRemovalTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/JsonVoteCacheLegacyRemovalTest.java index 5ce869259..a6e0d8f61 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/JsonVoteCacheLegacyRemovalTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/JsonVoteCacheLegacyRemovalTest.java @@ -1,6 +1,7 @@ package com.bencodez.votingplugin.tests; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -51,6 +52,27 @@ void velocityExactRemovalHonorsLegacyVoteId() { assertTrue(cache.contains("VoteCache", "server", "1")); } + @Test + void bungeePendingRewardEnumerationSkipsMalformedServerKeys() { + VotingPluginBungee plugin = mock(VotingPluginBungee.class); + when(plugin.getDataFolder()).thenReturn(temporaryDirectory.toFile()); + BungeeJsonVoteCache cache = new BungeeJsonVoteCache(plugin); + cache.setBoolean("VoteParty.PendingRewards.invalid*.delivery", true); + cache.setPendingVotePartyReward("Lobby", "delivery", true); + + assertEquals(java.util.List.of("lobby"), cache.getPendingVotePartyRewardServers()); + } + + @Test + void velocityPendingRewardEnumerationSkipsMalformedServerKeys() { + VelocityJsonVoteCache cache = new VelocityJsonVoteCache( + temporaryDirectory.resolve("velocity-pending-rewards.json").toFile()); + cache.set(new Object[] { "VoteParty", "PendingRewards", "invalid*", "delivery" }, true); + cache.setPendingVotePartyReward("Lobby", "delivery", true); + + assertEquals(java.util.List.of("lobby"), cache.getPendingVotePartyRewardServers()); + } + private static OfflineBungeeVote vote(UUID voteId) { return new OfflineBungeeVote(voteId, "Player", "player-uuid", "Service", 100L, true, "totals"); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/ProxyVoteEventNullServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/ProxyVoteEventNullServiceTest.java new file mode 100644 index 000000000..2a5be828d --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/ProxyVoteEventNullServiceTest.java @@ -0,0 +1,78 @@ +package com.bencodez.votingplugin.tests; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.Test; + +import com.bencodez.votingplugin.proxy.bungee.VoteEventBungee; +import com.bencodez.votingplugin.proxy.bungee.VotingPluginBungee; +import com.bencodez.votingplugin.proxy.velocity.VoteEventVelocity; +import com.bencodez.votingplugin.proxy.velocity.VotingPluginVelocity; +import com.vexsoftware.votifier.model.Vote; + +class ProxyVoteEventNullServiceTest { + @Test + void bungeeNullServiceSchedulesVoteWithCompatibilityName() throws Exception { + VotingPluginBungee plugin = mock(VotingPluginBungee.class); + net.md_5.bungee.api.ProxyServer proxy = mock(net.md_5.bungee.api.ProxyServer.class); + net.md_5.bungee.api.scheduler.TaskScheduler scheduler = + mock(net.md_5.bungee.api.scheduler.TaskScheduler.class); + when(plugin.getProxy()).thenReturn(proxy); + when(proxy.getScheduler()).thenReturn(scheduler); + AtomicReference task = new AtomicReference<>(); + doAnswer(invocation -> { + task.set(invocation.getArgument(1)); + return null; + }).when(scheduler).runAsync(eq(plugin), any(Runnable.class)); + Vote vote = mock(Vote.class); + when(vote.getUsername()).thenReturn("Player"); + when(vote.getServiceName()).thenReturn(null); + com.vexsoftware.votifier.bungee.events.VotifierEvent event = + mock(com.vexsoftware.votifier.bungee.events.VotifierEvent.class); + when(event.getVote()).thenReturn(vote); + + new VoteEventBungee(plugin).onVote(event); + + assertNotNull(task.get()); + assertEquals("Empty", field(task.get(), "service")); + } + + @Test + void velocityNullServiceSchedulesVoteWithCompatibilityName() throws Exception { + VotingPluginVelocity plugin = mock(VotingPluginVelocity.class); + ScheduledExecutorService timer = mock(ScheduledExecutorService.class); + when(plugin.getTimer()).thenReturn(timer); + AtomicReference task = new AtomicReference<>(); + doAnswer(invocation -> { + task.set(invocation.getArgument(0)); + return null; + }).when(timer).execute(any(Runnable.class)); + Vote vote = mock(Vote.class); + when(vote.getUsername()).thenReturn("Player"); + when(vote.getServiceName()).thenReturn(null); + com.vexsoftware.votifier.velocity.event.VotifierEvent event = + mock(com.vexsoftware.votifier.velocity.event.VotifierEvent.class); + when(event.getVote()).thenReturn(vote); + + new VoteEventVelocity(plugin).onVotifierEvent(event); + + assertNotNull(task.get()); + assertEquals("Empty", field(task.get(), "service")); + } + + private static Object field(Object target, String name) throws Exception { + Field field = target.getClass().getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index fb8cb0f71..933da58ad 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -522,8 +522,11 @@ void rejectedLegacyOnlyMultiProxySendKeepsLiveVoteForRetry() throws Exception { assertFalse((Boolean) begin.invoke(votingPluginProxy, retry, null, "Player", "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); - verify(multiProxyHandler).sendMultiProxyEnvelopeAccepted(Mockito.any(), + org.mockito.ArgumentCaptor envelope = org.mockito.ArgumentCaptor.forClass(JsonEnvelope.class); + verify(multiProxyHandler).sendMultiProxyEnvelopeAccepted(envelope.capture(), Mockito.eq(java.util.Set.of("ProxyLegacy"))); + assertEquals("", envelope.getValue().getFields() + .getOrDefault(VotingPluginWire.K_MULTI_PROXY_ORIGIN, "")); } @Test From 0ee7950c98d53e6a42b0d63d28894c7d299bd878 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 13 Sep 2026 11:14:38 -0600 Subject: [PATCH 102/136] Harden Control operations and backend scheduling --- .../global/BackendGlobalDataSync.java | 115 +++++-- .../control/BackendConfigurationService.java | 47 ++- .../control/BackendControlConnector.java | 161 ++++++++- .../control/PluginDeploymentService.java | 323 ++++++++++++++++++ .../votingplugin/proxy/VotingPluginProxy.java | 2 +- .../votingplugin/proxy/cache/IVoteCache.java | 75 +++- .../proxy/cache/VoteCacheHandler.java | 33 +- .../proxy/control/ControlConnector.java | 147 +++++++- .../global/BackendGlobalDataSyncTest.java | 158 +++++++++ .../BackendConfigurationServiceTest.java | 30 +- .../BackendControlConnectorProtocolTest.java | 25 +- .../control/PluginDeploymentServiceTest.java | 148 ++++++++ .../tests/VoteCacheHandlerVoteIdTest.java | 90 +++++ .../tests/VotingPluginProxyTest.java | 25 ++ docs/control-agent-contract.md | 26 +- docs/control-connector.md | 9 + 16 files changed, 1348 insertions(+), 66 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSyncTest.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSync.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSync.java index 29c24a7db..daba21034 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSync.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSync.java @@ -4,9 +4,12 @@ import java.time.ZoneOffset; import java.util.HashMap; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import com.bencodez.advancedcore.api.time.TimeType; @@ -29,6 +32,8 @@ public class BackendGlobalDataSync { private final VotingPluginMain plugin; private final Consumer sender; + private final AtomicBoolean forceUpdateInProgress = new AtomicBoolean(false); + private final Set timeChangesInProgress = ConcurrentHashMap.newKeySet(); @Getter private GlobalDataHandler globalDataHandler; @@ -46,26 +51,40 @@ public void checkGlobalData() { } HashMap data = globalDataHandler.getExact(plugin.getBungeeSettings().getServer()); - if (data.containsKey("ForceUpdate") && checkGlobalDataTimeValue(data.get("ForceUpdate"))) { - if (plugin.getStorageType().equals(UserStorage.MYSQL)) { - plugin.getMysql().clearCacheBasic(); + if (data.containsKey("ForceUpdate") && checkGlobalDataTimeValue(data.get("ForceUpdate")) + && forceUpdateInProgress.compareAndSet(false, true)) { + String serverName = plugin.getBungeeSettings().getServer(); + try { + if (plugin.getStorageType().equals(UserStorage.MYSQL)) { + plugin.getMysql().clearCacheBasic(); + } + plugin.getBukkitScheduler().executeOrScheduleSync(plugin, () -> { + try { + plugin.getUserManager().getDataManager().clearCache(); + plugin.setUpdate(true); + plugin.update(); + } catch (RuntimeException failure) { + forceUpdateInProgress.set(false); + plugin.debug(failure); + return; + } + try { + plugin.getBukkitScheduler().runTaskAsynchronously(plugin, + () -> clearForceUpdateFlag(serverName)); + } catch (RuntimeException failure) { + forceUpdateInProgress.set(false); + plugin.debug(failure); + } + }); + } catch (RuntimeException failure) { + forceUpdateInProgress.set(false); + plugin.debug(failure); } - plugin.getUserManager().getDataManager().clearCache(); - plugin.setUpdate(true); - plugin.update(); - globalDataHandler.setBoolean(plugin.getBungeeSettings().getServer(), "ForceUpdate", false); } - boolean forceUpdate = checkGlobalDataTime(TimeType.MONTH, data); - forceUpdate |= checkGlobalDataTime(TimeType.WEEK, data); - forceUpdate |= checkGlobalDataTime(TimeType.DAY, data); - - if (forceUpdate) { - HashMap dataToSet = new HashMap<>(); - dataToSet.put("FinishedProcessing", new DataValueBoolean(true)); - dataToSet.put("Processing", new DataValueBoolean(false)); - globalDataHandler.setData(plugin.getBungeeSettings().getServer(), dataToSet); - } + checkGlobalDataTime(TimeType.MONTH, data); + checkGlobalDataTime(TimeType.WEEK, data); + checkGlobalDataTime(TimeType.DAY, data); } public boolean checkGlobalDataTime(TimeType type, HashMap data) { @@ -80,18 +99,58 @@ public boolean checkGlobalDataTime(TimeType type, HashMap dat globalDataHandler.setBoolean(plugin.getBungeeSettings().getServer(), type.toString(), false); return false; } + if (!timeChangesInProgress.add(type)) { + return false; + } - globalDataHandler.setBoolean(plugin.getBungeeSettings().getServer(), "Processing", true); + String serverName = plugin.getBungeeSettings().getServer(); + globalDataHandler.setBoolean(serverName, "Processing", true); plugin.debug("Detected time change from bungee: " + type.toString()); - plugin.getTimeChecker().forceChanged(type, false, true, true); - globalDataHandler.setBoolean(plugin.getBungeeSettings().getServer(), type.toString(), false); - - JsonEnvelope.Builder builder = JsonEnvelope.builder("TimeChangeFinished").schema(VotingPluginWire.SCHEMA_VERSION); - builder.put("server", plugin.getBungeeSettings().getServer()); - sender.accept(builder.build()); + try { + plugin.getBukkitScheduler().executeOrScheduleSync(plugin, () -> { + try { + plugin.getTimeChecker().forceChanged(type, false, true, true); + } catch (RuntimeException failure) { + timeChangesInProgress.remove(type); + plugin.debug(failure); + return; + } + try { + plugin.getBukkitScheduler().runTaskAsynchronously(plugin, + () -> finishTimeChange(type, serverName)); + } catch (RuntimeException failure) { + timeChangesInProgress.remove(type); + plugin.debug(failure); + } + }); + } catch (RuntimeException failure) { + timeChangesInProgress.remove(type); + plugin.debug(failure); + return false; + } return true; } + private void finishTimeChange(TimeType type, String serverName) { + boolean completed = false; + try { + globalDataHandler.setBoolean(serverName, type.toString(), false); + JsonEnvelope.Builder builder = JsonEnvelope.builder("TimeChangeFinished") + .schema(VotingPluginWire.SCHEMA_VERSION); + builder.put("server", serverName); + sender.accept(builder.build()); + completed = true; + } finally { + timeChangesInProgress.remove(type); + if (completed && timeChangesInProgress.isEmpty()) { + HashMap dataToSet = new HashMap<>(); + dataToSet.put("FinishedProcessing", new DataValueBoolean(true)); + dataToSet.put("Processing", new DataValueBoolean(false)); + globalDataHandler.setData(serverName, dataToSet); + } + } + } + public boolean checkGlobalDataTimeValue(DataValue data) { if (data.isBoolean()) { return data.getBoolean(); @@ -99,6 +158,14 @@ public boolean checkGlobalDataTimeValue(DataValue data) { return Boolean.valueOf(data.getString()); } + private void clearForceUpdateFlag(String serverName) { + try { + globalDataHandler.setBoolean(serverName, "ForceUpdate", false); + } finally { + forceUpdateInProgress.set(false); + } + } + public void load() { if (!plugin.getBungeeSettings().isGloblalDataEnabled()) { return; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index a790b8e89..c4e7d61c3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -110,6 +110,7 @@ private ApplyResult apply(String fileName, String proposedContent, String expect Path staging = Files.createTempFile(target.getParent(), ".control-", ".yml"); Path backupStaging = Files.createTempFile(target.getParent(), ".control-backup-", ".yml"); boolean installed = false; + boolean reloadAttempted = false; try { Files.writeString(staging, preview.resolvedContent(), StandardCharsets.UTF_8, StandardOpenOption.TRUNCATE_EXISTING); @@ -126,6 +127,7 @@ private ApplyResult apply(String fileName, String proposedContent, String expect installed = true; throw published; } + reloadAttempted = true; applyAction.run(fileName); String applied = readRaw(target, false); String installedRevision = revision(preview.resolvedContent()); @@ -158,7 +160,7 @@ private ApplyResult apply(String fileName, String proposedContent, String expect failure.addSuppressed(rollbackFailure); } } - throw new ApplyFailureException(rolledBack, failure); + throw new ApplyFailureException(rolledBack, reloadAttempted, failure); } finally { Files.deleteIfExists(staging); Files.deleteIfExists(backupStaging); @@ -394,7 +396,7 @@ private QuickProposal quickProposal(String preset, Map options, case "vote-logging" -> Set.of("enabled", "purgeDays", "useMainMySQL"); case "common-settings" -> Set.of("processRewards", "autoCreateVoteSites", "extraAllSitesCheck", "countFakeVotes", "disableNoServiceSiteMessage", "disableUpdateChecking"); - case "vote-party" -> Set.of("votesRequired", "broadcast", "giveAllPlayers", "onlineOnly", "command"); + case "vote-party" -> Set.of("enabled", "votesRequired", "broadcast", "giveAllPlayers", "onlineOnly", "command"); case "sync-vote-sites" -> Set.of("sourceContent"); default -> throw new IllegalArgumentException("quick setup preset is unsupported"); }); @@ -498,7 +500,7 @@ private QuickProposal quickProposal(String preset, Map options, return new QuickProposal(fileName, yaml.saveToString()); } if ("vote-party".equals(preset)) { - yaml.set("VoteParty.Enabled", true); + yaml.set("VoteParty.Enabled", booleanOption(options, "enabled")); yaml.set("VoteParty.VotesRequired", boundedInteger(option(options, "votesRequired", "[0-9]{1,6}"), 1, 100000)); yaml.set("VoteParty.GiveAllPlayers", booleanOption(options, "giveAllPlayers")); yaml.set("VoteParty.GiveOnlinePlayersOnly", booleanOption(options, "onlineOnly")); @@ -811,17 +813,25 @@ private Path resolve(String fileName) throws IOException { } private static String readRaw(Path path, boolean allowMissing) throws IOException { - if (!Files.exists(path) && allowMissing) return ""; - if (!Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS) - || Files.size(path) > MAX_CONTENT_BYTES) { - throw new IOException("configuration file is missing or too large"); + if (!Files.exists(path, LinkOption.NOFOLLOW_LINKS)) { + if (allowMissing) return ""; + throw new java.nio.file.NoSuchFileException(path.toString()); + } + if (!Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)) { + throw new ConfigurationReadException(ConfigurationReadFailure.UNSAFE, + "configuration file is not a regular file"); + } + if (Files.size(path) > MAX_CONTENT_BYTES) { + throw new ConfigurationReadException(ConfigurationReadFailure.TOO_LARGE, + "configuration file exceeds the managed size limit"); } byte[] bytes = Files.readAllBytes(path); try { return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) .onUnmappableCharacter(CodingErrorAction.REPORT).decode(java.nio.ByteBuffer.wrap(bytes)).toString(); } catch (CharacterCodingException e) { - throw new IOException("configuration file is not valid UTF-8", e); + throw new ConfigurationReadException(ConfigurationReadFailure.INVALID_ENCODING, + "configuration file is not valid UTF-8", e); } } @@ -1135,10 +1145,29 @@ public record QuickState(Map options, String revision) { } @FunctionalInterface public interface ReloadAction { void run() throws Exception; } @FunctionalInterface public interface ApplyAction { void run(String fileName) throws Exception; } @FunctionalInterface interface MoveAction { void move(Path source, Path target) throws IOException; } + public enum ConfigurationReadFailure { UNSAFE, TOO_LARGE, INVALID_ENCODING } + @SuppressWarnings("serial") public static final class ConfigurationReadException extends IOException { + private final ConfigurationReadFailure failure; + private ConfigurationReadException(ConfigurationReadFailure failure, String message) { + super(message); + this.failure = failure; + } + private ConfigurationReadException(ConfigurationReadFailure failure, String message, Throwable cause) { + super(message, cause); + this.failure = failure; + } + public ConfigurationReadFailure failure() { return failure; } + } @SuppressWarnings("serial") public static final class StaleRevisionException extends RuntimeException { } @SuppressWarnings("serial") public static final class ApplyFailureException extends IOException { private final boolean rolledBack; - private ApplyFailureException(boolean rolledBack, Throwable cause) { super(cause); this.rolledBack = rolledBack; } + private final boolean reloadAttempted; + private ApplyFailureException(boolean rolledBack, boolean reloadAttempted, Throwable cause) { + super(cause); + this.rolledBack = rolledBack; + this.reloadAttempted = reloadAttempted; + } public boolean rolledBack() { return rolledBack; } + public boolean reloadAttempted() { return reloadAttempted; } } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 94d825562..c003a8109 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -51,7 +51,8 @@ public final class BackendControlConnector implements AutoCloseable { private static final long INSPECTION_SHUTDOWN_TIMEOUT_SECONDS = 5; private static final Pattern NODE_ID = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,63}"); private static final Set CAPABILITIES = Set.of("config.files.v1", "config.file-comments.v1", - "config.quick-setup.v1", "config.vote-sites-sync.v1", "config.proxy-method.v1", "data.inspect.v1"); + "config.quick-setup.v1", "config.vote-sites-sync.v1", "config.proxy-method.v1", + "config.proxy-method.v2", "data.inspect.v1"); private final VotingPluginMain plugin; private final Path dataDirectory; @@ -64,6 +65,7 @@ public final class BackendControlConnector implements AutoCloseable { private final HttpClient http; private final BackendConfigurationService configurations; private final ControlInspectionService inspections; + private final PluginDeploymentService deployments; private final UUID sessionId = UUID.randomUUID(); private final Map completed = new LinkedHashMap<>(); private final boolean recovering; @@ -78,12 +80,14 @@ public final class BackendControlConnector implements AutoCloseable { private volatile boolean quickSetupsAccepted; private volatile boolean voteSitesSyncAccepted; private volatile boolean inspectionsAccepted; + private volatile boolean deploymentsAccepted; private volatile int inspectionFailures; private volatile long inspectionRetryAtNanos; private volatile int failures; private volatile ScheduledFuture scheduled; private volatile ScheduledFuture operationPolling; private volatile ScheduledFuture inspectionPolling; + private volatile ScheduledFuture deploymentPolling; private volatile Future activeReload; private volatile CompletableFuture activeOperation; @@ -112,6 +116,13 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set .followRedirects(HttpClient.Redirect.NEVER).build(); configurations = new BackendConfigurationService(plugin.getDataFolder().toPath(), this::reloadConfiguration); inspections = new ControlInspectionService(plugin); + PluginDeploymentService prepared = null; + try { + prepared = PluginDeploymentService.backend(plugin.getServer().getUpdateFolderFile().toPath()); + } catch (Exception failure) { + plugin.getLogger().warning("[Control] Plugin deployment staging is unavailable; capability not advertised"); + } + deployments = prepared; } private void reloadConfiguration(String fileName) throws Exception { @@ -354,6 +365,64 @@ public void start() { OPERATION_POLL_MILLIS, OPERATION_POLL_MILLIS, TimeUnit.MILLISECONDS); inspectionPolling = inspectionExecutor.scheduleWithFixedDelay(this::pollInspections, OPERATION_POLL_MILLIS, OPERATION_POLL_MILLIS, TimeUnit.MILLISECONDS); + deploymentPolling = executor.scheduleWithFixedDelay(this::pollDeployments, + OPERATION_POLL_MILLIS, OPERATION_POLL_MILLIS, TimeUnit.MILLISECONDS); + } + + /** Deployment is a separate leased lane; its 64 MiB I/O never runs on Bukkit's primary thread. */ + private void pollDeployments() { + if (closed || deployments == null || !registered || failures != 0 || !deploymentsAccepted + || !running.compareAndSet(false, true)) return; + CompletableFuture operation = new CompletableFuture<>(); + synchronized (operationLifecycle) { + if (closed) { running.set(false); return; } + activeOperation = operation; + } + try { + claimAndDeploy(); + } catch (Exception failure) { + registered = false; + failures = Math.min(30, failures + 1); + if (failures == 1 || failures % 10 == 0) { + plugin.getLogger().warning("[Control] Bukkit deployment polling unavailable; VotingPlugin remains active"); + } + ScheduledFuture heartbeat = scheduled; + if (heartbeat != null) heartbeat.cancel(false); + if (!closed) schedule(Math.min(TimeUnit.MINUTES.toMillis(5), 1000L << Math.min(failures - 1, 8))); + } finally { + operation.complete(null); + synchronized (operationLifecycle) { if (activeOperation == operation) activeOperation = null; } + running.set(false); + } + } + + private void claimAndDeploy() throws Exception { + JsonObject body = new JsonObject(); + body.addProperty("sessionId", sessionId.toString()); + Response response = send("POST", "/api/v1/nodes/" + settings.nodeId() + "/deployments", body); + if (response.status() == 204 || closed) return; + JsonObject claimed = requireObject(response, 200); + PluginDeploymentService.Task task = deploymentTask(claimed); + PluginDeploymentService.Result result = deployments.deploy(task, settings.endpoint(), settings.nodeId(), sessionId, + credential, http, Duration.ofMillis(settings.requestTimeoutMillis()), () -> !closed); + if (closed) return; + JsonObject submitted = new JsonObject(); + submitted.addProperty("sessionId", sessionId.toString()); + submitted.addProperty("success", result.success()); + submitted.addProperty("code", result.code()); + submitted.addProperty("message", boundedResultMessage(result.message())); + submitted.addProperty("attemptId", task.attemptId().toString()); + requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/deployments/" + task.deploymentId() + + "/result", submitted), 200); + } + + private static PluginDeploymentService.Task deploymentTask(JsonObject task) { + try { + return new PluginDeploymentService.Task(UUID.fromString(string(task, "deploymentId")), string(task, "artifactId"), + string(task, "sha256"), Long.parseLong(string(task, "size")), UUID.fromString(string(task, "attemptId"))); + } catch (RuntimeException failure) { + throw new IllegalArgumentException("deployment task is invalid"); + } } /** Polls the separately negotiated read-only lane on the connector worker. */ @@ -471,7 +540,9 @@ private void cycle() { quickSetupsAccepted = negotiatedCapability(node, "config.quick-setup.v1", quickSetupsAccepted); voteSitesSyncAccepted = negotiatedCapability(node, "config.vote-sites-sync.v1", voteSitesSyncAccepted); boolean inspectionsWereAccepted = inspectionsAccepted; - inspectionsAccepted = negotiatedCapability(node, "data.inspect.v1", inspectionsAccepted); + inspectionsAccepted = negotiatedCapability(node, "data.inspect.v1", inspectionsAccepted); + deploymentsAccepted = deployments != null + && negotiatedCapability(node, PluginDeploymentService.CAPABILITY, deploymentsAccepted); if (inspectionsAccepted && !inspectionsWereAccepted) { inspectionFailures = 0; inspectionRetryAtNanos = 0; @@ -522,6 +593,7 @@ private JsonObject register() throws Exception { quickSetupsAccepted = false; voteSitesSyncAccepted = false; inspectionsAccepted = false; + deploymentsAccepted = false; JsonObject body = sessionBody(); body.addProperty("nodeId", settings.nodeId()); body.addProperty("displayName", settings.nodeId()); @@ -532,13 +604,13 @@ private JsonObject register() throws Exception { .map(installed -> installed.getDescription().getName()).filter(name -> name != null && !name.isBlank()) .distinct().sorted(String.CASE_INSENSITIVE_ORDER).limit(128).forEach(detectedPlugins::add); body.add("detectedPlugins", detectedPlugins); - addCapabilities(body); + addCapabilities(body, deployments != null); return requireObject(send("POST", "/api/v1/nodes/register", body), 200, 201); } private JsonObject heartbeat() throws Exception { JsonObject body = sessionBody(); - addCapabilities(body); + addCapabilities(body, deployments != null); Response response = send("PUT", "/api/v1/nodes/" + settings.nodeId() + "/heartbeat", body); if (response.status() == 404) { registered = false; @@ -779,8 +851,9 @@ private TaskResult execute(UUID operationId, JsonObject task) { } catch (BackendConfigurationService.StaleRevisionException e) { return TaskResult.failure("STALE_REVISION", "Configuration changed after preview"); } catch (BackendConfigurationService.ApplyFailureException e) { - logConfigurationFailure("reload", e); - return TaskResult.failure("RELOAD_FAILED", reloadFailureMessage(e), e.rolledBack()); + logConfigurationFailure(e.reloadAttempted() ? "reload" : "write", e); + return TaskResult.failure(e.reloadAttempted() ? "RELOAD_FAILED" : "WRITE_FAILED", + applyFailureMessage(e), e.rolledBack()); } catch (IllegalArgumentException e) { return TaskResult.failure("VALIDATION_ERROR", e.getMessage()); } catch (Exception e) { @@ -789,7 +862,7 @@ private TaskResult execute(UUID operationId, JsonObject task) { } private TaskResult operationFailure(String type, Throwable failure) { - String code = operationFailureCode(type); + String code = operationFailureCode(type, failure); String action = "READ".equals(type) ? "read" : "PREVIEW".equals(type) ? "preview" : "apply"; logConfigurationFailure(action, failure); return TaskResult.failure(code, operationFailureMessage(type, failure)); @@ -799,22 +872,77 @@ private void logConfigurationFailure(String action, Throwable failure) { plugin.getLogger().log(Level.WARNING, "[Control] Configuration " + action + " failed", failure); } - static String operationFailureMessage(String type, Throwable ignored) { - if ("READ".equals(type)) return "Configuration read failed; see the backend log"; - if ("PREVIEW".equals(type)) return "Configuration preview failed; see the backend log"; - return "Configuration apply failed; see the backend log"; + static String operationFailureMessage(String type, Throwable failure) { + if ("READ".equals(type)) { + if (hasCause(failure, java.nio.file.NoSuchFileException.class)) { + return "The managed configuration file does not exist on this backend"; + } + if (hasCause(failure, java.nio.file.AccessDeniedException.class) + || hasCause(failure, SecurityException.class)) { + return "The managed configuration file is not readable by the backend"; + } + BackendConfigurationService.ConfigurationReadException readFailure = cause( + failure, BackendConfigurationService.ConfigurationReadException.class); + if (readFailure != null) return switch (readFailure.failure()) { + case TOO_LARGE -> "The managed configuration file exceeds the 512 KiB limit"; + case INVALID_ENCODING -> "The managed configuration file is not valid UTF-8"; + case UNSAFE -> "The managed configuration path is not a safe regular file"; + }; + return "The backend could not safely read the managed configuration file"; + } + if ("PREVIEW".equals(type)) return "The backend could not prepare a configuration preview"; + return "The backend could not apply the managed configuration"; } - static String reloadFailureMessage(Throwable ignored) { - return "Configuration reload failed; see the backend log"; + static String applyFailureMessage(BackendConfigurationService.ApplyFailureException failure) { + if (!failure.reloadAttempted()) { + return failure.rolledBack() ? "Configuration write failed; the previous file was restored" + : "Configuration write failed before runtime reload"; + } + String reason = hasCause(failure, java.util.concurrent.TimeoutException.class) + || hasCause(failure, java.net.SocketTimeoutException.class) + ? "the configured transport did not become ready before its deadline" + : hasCause(failure, java.net.ConnectException.class) + ? "the configured transport endpoint was unavailable" + : "the backend runtime rejected the new configuration"; + return failure.rolledBack() + ? "Runtime reload failed because " + reason + "; the previous file was restored" + : "Runtime reload failed because " + reason + "; automatic rollback did not complete"; } static String operationFailureCode(String type) { + return operationFailureCode(type, null); + } + + static String operationFailureCode(String type, Throwable failure) { + if ("READ".equals(type) && hasCause(failure, java.nio.file.NoSuchFileException.class)) { + return "CONFIGURATION_MISSING"; + } + if ("READ".equals(type) && (hasCause(failure, java.nio.file.AccessDeniedException.class) + || hasCause(failure, SecurityException.class))) return "CONFIGURATION_UNREADABLE"; + BackendConfigurationService.ConfigurationReadException readFailure = cause( + failure, BackendConfigurationService.ConfigurationReadException.class); + if ("READ".equals(type) && readFailure != null) return switch (readFailure.failure()) { + case TOO_LARGE -> "CONFIGURATION_TOO_LARGE"; + case INVALID_ENCODING -> "CONFIGURATION_INVALID_ENCODING"; + case UNSAFE -> "CONFIGURATION_UNSAFE"; + }; if ("READ".equals(type)) return "READ_FAILED"; if ("PREVIEW".equals(type)) return "PREVIEW_FAILED"; return "APPLY_FAILED"; } + private static boolean hasCause(Throwable failure, Class type) { + return cause(failure, type) != null; + } + + private static T cause(Throwable failure, Class type) { + for (Throwable current = failure; current != null; current = current.getCause()) { + if (type.isInstance(current)) return type.cast(current); + } + return null; + } + private TaskResult executeFile(UUID operationId, String type, JsonObject configuration, JsonObject task) throws IOException { String fileName = string(configuration, "fileName"); @@ -925,9 +1053,12 @@ private JsonObject sessionBody() { return body; } - static void addCapabilities(JsonObject body) { + static void addCapabilities(JsonObject body) { addCapabilities(body, false); } + + static void addCapabilities(JsonObject body, boolean deploymentReady) { JsonArray capabilities = new JsonArray(); CAPABILITIES.stream().sorted().forEach(capabilities::add); + if (deploymentReady) capabilities.add(PluginDeploymentService.CAPABILITY); body.add("capabilities", capabilities); JsonArray required = new JsonArray(); required.add("config.files.v1"); @@ -995,6 +1126,8 @@ public void close() { if (polling != null) polling.cancel(false); ScheduledFuture inspection = inspectionPolling; if (inspection != null) inspection.cancel(false); + ScheduledFuture deployment = deploymentPolling; + if (deployment != null) deployment.cancel(false); inspectionExecutor.shutdownNow(); if (reload != null && Bukkit.isPrimaryThread()) reload.cancel(false); awaitShutdown(executor, operation); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java new file mode 100644 index 000000000..37b449cd7 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java @@ -0,0 +1,323 @@ +package com.bencodez.votingplugin.control; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.charset.CodingErrorAction; +import java.nio.channels.FileChannel; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.time.Duration; +import java.util.HexFormat; +import java.util.Locale; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +/** + * Bounded, pull-only staging for the optional {@code plugin.deploy.v1} capability. + * It deliberately never reloads a plugin or starts a server. The small on-disk + * marker makes a leased task idempotent after a lost result acknowledgement. + */ +public final class PluginDeploymentService { + public static final String CAPABILITY = "plugin.deploy.v1"; + public static final long MAX_ARTIFACT_BYTES = 64L * 1024L * 1024L; + private static final int BUFFER_BYTES = 32 * 1024; + private static final int MAX_JAR_ENTRIES = 10_000; + private static final long MAX_INSPECTED_UNCOMPRESSED_BYTES = 256L * 1024L * 1024L; + private static final int MAX_PLUGIN_YML_BYTES = 64 * 1024; + private static final String MARKER = ".control-deployment"; + + private final Path target; + private final Path root; + private final Path marker; + private final boolean replaceExisting; + private final AtomicBoolean staging = new AtomicBoolean(); + + private PluginDeploymentService(Path target, boolean replaceExisting) throws IOException { + this.target = target.toAbsolutePath().normalize(); + this.replaceExisting = replaceExisting; + Path parent = this.target.getParent(); + if (parent == null) throw new IOException("deployment target has no parent"); + Files.createDirectories(parent); + this.root = parent.toRealPath(LinkOption.NOFOLLOW_LINKS); + if (!Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(root)) { + throw new IOException("deployment staging directory is unsafe"); + } + if (!this.target.startsWith(root) || Files.isSymbolicLink(this.target)) { + throw new IOException("deployment target escapes staging directory"); + } + if (Files.exists(this.target, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(this.target, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("deployment target is not a regular file"); + } + this.marker = root.resolve(this.target.getFileName() + MARKER); + } + + /** Bukkit's update folder preserves the currently loaded plugin JAR. */ + public static PluginDeploymentService backend(Path updateDirectory) throws IOException { + if (updateDirectory == null) throw new IOException("Bukkit update folder is unavailable"); + return new PluginDeploymentService(updateDirectory.resolve("VotingPlugin.jar"), false); + } + + /** Proxies atomically replace their discovered plugin JAR only after a durable backup. */ + public static PluginDeploymentService proxy(Path currentPluginJar) throws IOException { + if (currentPluginJar == null || !currentPluginJar.getFileName().toString().endsWith(".jar")) { + throw new IOException("proxy plugin JAR is unavailable"); + } + if (!Files.isRegularFile(currentPluginJar, LinkOption.NOFOLLOW_LINKS) + || Files.isSymbolicLink(currentPluginJar)) throw new IOException("proxy plugin JAR is unsafe"); + return new PluginDeploymentService(currentPluginJar, true); + } + + public boolean isStaging() { return staging.get(); } + + public Result deploy(Task task, URI endpoint, String nodeId, UUID sessionId, String credential, + HttpClient http, Duration timeout, BooleanSupplier active) { + if (!staging.compareAndSet(false, true)) return Result.failure("DEPLOYMENT_FAILED", "Another deployment is still staging"); + try { + validate(task); + if (alreadyStaged(task)) return Result.restartRequired(); + if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before download"); + URI artifact = endpoint.resolve("/api/v1/nodes/" + nodeId + "/deployments/" + task.deploymentId() + + "/artifact"); + HttpRequest request = HttpRequest.newBuilder(artifact).timeout(timeout) + .header("Authorization", "Bearer " + credential) + .header("X-Node-Session", sessionId.toString()) + .header("X-Deployment-Attempt", task.attemptId().toString()).GET().build(); + HttpResponse response = http.send(request, HttpResponse.BodyHandlers.ofInputStream()); + if (response.statusCode() != 200) { + response.body().close(); + return Result.failure("DOWNLOAD_FAILED", "Artifact download was rejected"); + } + String contentLength = response.headers().firstValue("Content-Length").orElse(null); + if (contentLength != null && (!contentLength.matches("[0-9]{1,9}") + || Long.parseLong(contentLength) != task.size())) { + response.body().close(); + return Result.failure("SIZE_MISMATCH", "Artifact size did not match the deployment task"); + } + try (InputStream body = response.body()) { + return stage(task, body, active); + } catch (IOException failure) { + return Result.failure("STAGING_FAILED", "Artifact could not be verified or staged on this node"); + } + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + return Result.failure("CANCELLED", "Deployment download was interrupted"); + } catch (IOException | RuntimeException failure) { + return Result.failure("DOWNLOAD_FAILED", "Artifact download failed"); + } finally { + staging.set(false); + } + } + + /** Package-visible for deterministic artifact-validation tests. */ + Result stage(Task task, InputStream body, BooleanSupplier active) throws IOException { + validate(task); + if (alreadyStaged(task)) return Result.restartRequired(); + Path temporary = Files.createTempFile(root, target.getFileName().toString() + ".", ".download"); + try { + MessageDigest digest = sha256(); + long written = copyExact(body, temporary, task.size(), digest, active); + force(temporary); + if (written != task.size()) return Result.failure("SIZE_MISMATCH", "Artifact size did not match the deployment task"); + if (!HexFormat.of().formatHex(digest.digest()).equals(task.sha256())) { + return Result.failure("HASH_MISMATCH", "Artifact digest did not match the deployment task"); + } + inspectJar(temporary); + if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging"); + activate(temporary); + writeMarker(task); + return Result.restartRequired(); + } catch (CancelledDeploymentException failure) { + return Result.failure("CANCELLED", "Deployment was cancelled before staging"); + } catch (InvalidArtifactException failure) { + return Result.failure("INVALID_ARTIFACT", "Artifact is not a bounded VotingPlugin JAR"); + } finally { + Files.deleteIfExists(temporary); + } + } + + private long copyExact(InputStream input, Path temporary, long expected, MessageDigest digest, + BooleanSupplier active) throws IOException { + long total = 0; + byte[] buffer = new byte[BUFFER_BYTES]; + try (var output = Files.newOutputStream(temporary, StandardOpenOption.TRUNCATE_EXISTING)) { + for (int read; (read = input.read(buffer)) != -1;) { + if (!active.getAsBoolean()) throw new CancelledDeploymentException(); + total += read; + if (total > expected || total > MAX_ARTIFACT_BYTES) throw new InvalidArtifactException(); + digest.update(buffer, 0, read); + output.write(buffer, 0, read); + } + } + return total; + } + + private void inspectJar(Path artifact) throws IOException { + try (JarFile jar = new JarFile(artifact.toFile(), false)) { + int entries = 0; + long uncompressed = 0; + JarEntry pluginYml = null; + var iterator = jar.entries(); + while (iterator.hasMoreElements()) { + JarEntry entry = iterator.nextElement(); + if (++entries > MAX_JAR_ENTRIES || unsafeEntry(entry.getName())) throw new InvalidArtifactException(); + long size = entry.getSize(); + long compressed = entry.getCompressedSize(); + if (size < 0 || compressed < 0 || compressed > 0 && size > compressed * 200L + || (uncompressed += size) > MAX_INSPECTED_UNCOMPRESSED_BYTES) { + throw new InvalidArtifactException(); + } + if ("plugin.yml".equals(entry.getName())) { + if (pluginYml != null || size > MAX_PLUGIN_YML_BYTES) throw new InvalidArtifactException(); + pluginYml = entry; + } + } + if (pluginYml == null || !isVotingPluginYml(jar, pluginYml)) throw new InvalidArtifactException(); + } catch (java.util.zip.ZipException failure) { + throw new InvalidArtifactException(); + } + } + + private static boolean unsafeEntry(String name) { + if (name == null || name.isEmpty() || name.length() > 512 || name.startsWith("/") + || name.startsWith("\\") || name.contains("\\") || name.indexOf('\0') >= 0) return true; + for (String component : name.split("/", -1)) { + if (component.equals(".") || component.equals("..")) return true; + } + return false; + } + + private static boolean isVotingPluginYml(JarFile jar, JarEntry entry) throws IOException { + byte[] bytes; + try (InputStream input = jar.getInputStream(entry)) { bytes = input.readNBytes(MAX_PLUGIN_YML_BYTES + 1); } + if (bytes.length > MAX_PLUGIN_YML_BYTES) return false; + String yml; + try { + yml = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT).decode(java.nio.ByteBuffer.wrap(bytes)).toString(); + } catch (java.nio.charset.CharacterCodingException failure) { + return false; + } + boolean found = false; + for (String line : yml.split("\\R")) { + if (!line.startsWith("name:")) continue; + if (found) return false; + found = true; + String value = line.substring(5).trim(); + int comment = value.indexOf('#'); + if (comment >= 0) value = value.substring(0, comment).trim(); + if (!"VotingPlugin".equals(unquote(value))) return false; + } + return found; + } + + private static String unquote(String value) { + return value.length() >= 2 && ((value.startsWith("\"") && value.endsWith("\"")) + || (value.startsWith("'") && value.endsWith("'"))) ? value.substring(1, value.length() - 1) : value; + } + + private void activate(Path temporary) throws IOException { + if (replaceExisting) { + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) { + throw new IOException("current proxy plugin JAR is unsafe"); + } + Path backup = root.resolve(target.getFileName() + ".control-backup"); + if (Files.exists(backup, LinkOption.NOFOLLOW_LINKS) + && (!Files.isRegularFile(backup, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(backup))) { + throw new IOException("proxy plugin backup is unsafe"); + } + Path backupTemp = Files.createTempFile(root, target.getFileName().toString() + ".", ".backup"); + try { + Files.copy(target, backupTemp, StandardCopyOption.REPLACE_EXISTING); + force(backupTemp); + move(backupTemp, backup); + forceDirectory(root); + } finally { Files.deleteIfExists(backupTemp); } + } + move(temporary, target); + force(target); + forceDirectory(root); + } + + private void writeMarker(Task task) throws IOException { + Path temporary = Files.createTempFile(root, target.getFileName().toString() + ".", ".marker"); + try { + Files.writeString(temporary, task.deploymentId() + "\n" + task.sha256() + "\n" + task.size() + "\n", + StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); + force(temporary); + move(temporary, marker); + forceDirectory(root); + } finally { Files.deleteIfExists(temporary); } + } + + private boolean alreadyStaged(Task task) throws IOException { + if (!Files.isRegularFile(marker, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(marker) + || Files.size(marker) > 256 || !Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) + || Files.isSymbolicLink(target) || Files.size(target) != task.size()) return false; + String[] fields = Files.readString(marker, StandardCharsets.US_ASCII).split("\\R", -1); + if (fields.length < 3 || !task.deploymentId().toString().equals(fields[0]) + || !task.sha256().equals(fields[1]) || !Long.toString(task.size()).equals(fields[2])) return false; + MessageDigest digest = sha256(); + try (InputStream input = Files.newInputStream(target, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)) { + byte[] bytes = new byte[BUFFER_BYTES]; + for (int read; (read = input.read(bytes)) != -1;) digest.update(bytes, 0, read); + } + if (!task.sha256().equals(HexFormat.of().formatHex(digest.digest()))) return false; + try { + inspectJar(target); + return true; + } catch (InvalidArtifactException invalid) { + return false; + } + } + + private static void move(Path source, Path destination) throws IOException { + try { Files.move(source, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (AtomicMoveNotSupportedException ignored) { Files.move(source, destination, StandardCopyOption.REPLACE_EXISTING); } + } + + private static void force(Path file) throws IOException { + try (FileChannel channel = FileChannel.open(file, StandardOpenOption.WRITE)) { channel.force(true); } + } + + private static void forceDirectory(Path directory) throws IOException { + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { channel.force(true); } + catch (UnsupportedOperationException ignored) { /* Some supported filesystems cannot fsync directories. */ } + } + + private static MessageDigest sha256() { + try { return MessageDigest.getInstance("SHA-256"); } + catch (java.security.NoSuchAlgorithmException failure) { throw new IllegalStateException("SHA-256 is unavailable", failure); } + } + + private static void validate(Task task) { + if (task == null || task.deploymentId() == null || task.attemptId() == null || task.artifactId() == null + || !task.artifactId().matches("[A-Za-z0-9][A-Za-z0-9._-]{0,127}") || task.sha256() == null + || !task.sha256().matches("[0-9a-fA-F]{64}") || task.size() < 1 || task.size() > MAX_ARTIFACT_BYTES) { + throw new IllegalArgumentException("deployment task is invalid"); + } + } + + public record Task(UUID deploymentId, String artifactId, String sha256, long size, UUID attemptId) { + public Task { if (sha256 != null) sha256 = sha256.toLowerCase(Locale.ROOT); } + } + public record Result(boolean success, String code, String message) { + static Result restartRequired() { return new Result(true, "RESTART_REQUIRED", "Plugin update staged; restart is required"); } + static Result failure(String code, String message) { return new Result(false, code, message); } + } + private static final class InvalidArtifactException extends IOException { private static final long serialVersionUID = 1L; } + private static final class CancelledDeploymentException extends IOException { private static final long serialVersionUID = 1L; } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 072832f4e..48f209b85 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -3375,7 +3375,7 @@ public synchronized void processQueue() { retryPendingTimeBroadcasts(); if (vote.hasPendingHttpBroadcastDeliveryIds()) { scheduleTimeVoteRetry(); - return; + continue; } } if (vote.isProcessed() && vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java index aa5da8667..eeb6ec592 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java @@ -1,8 +1,12 @@ package com.bencodez.votingplugin.proxy.cache; import java.util.Collection; -import java.nio.file.Path; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import com.bencodez.votingplugin.util.DurableFiles; @@ -14,6 +18,75 @@ */ public interface IVoteCache { + static final String EMERGENCY_JOURNAL_MARKER_FILE_SUFFIX = ".vote-emergency-journal"; + static final String EMERGENCY_JOURNAL_MARKER_VERSION = "1"; + + /** + * Returns the marker file used for emergency-journal recovery coordination. + * + * @return marker file path, or {@code null} when no file-backed cache exists + */ + default Path getEmergencyJournalMarkerPath() { + Path storagePath = getStoragePath(); + if (storagePath == null) { + return null; + } + return storagePath.resolveSibling(storagePath.getFileName() + EMERGENCY_JOURNAL_MARKER_FILE_SUFFIX); + } + + /** + * Returns whether this JSON cache has a marker that indicates emergency journal + * compatibility for MySQL recovery. Legacy files without this marker are not + * treated as emergency recoverables. + * + * @return true when the marker file exists + */ + default boolean hasEmergencyJournalMarker() { + Path markerPath = getEmergencyJournalMarkerPath(); + if (markerPath == null) { + return false; + } + try { + return Files.isRegularFile(markerPath, LinkOption.NOFOLLOW_LINKS) && !Files.isSymbolicLink(markerPath) + && Files.size(markerPath) <= 8 + && EMERGENCY_JOURNAL_MARKER_VERSION.equals(Files.readString(markerPath, StandardCharsets.UTF_8).trim()); + } catch (IOException ignored) { + return false; + } + } + + /** + * Writes the emergency-journal marker file used to indicate that this cache has + * participated in emergency journal recovery flow. + * + * @throws IOException when marker publication fails + */ + default void markEmergencyJournalUsed() throws IOException { + Path markerPath = getEmergencyJournalMarkerPath(); + if (markerPath == null) { + throw new IOException("Vote cache has no emergency journal marker path"); + } + Path normalizedMarker = markerPath.toAbsolutePath().normalize(); + Path parent = normalizedMarker.getParent(); + if (parent != null && Files.isSymbolicLink(parent)) { + throw new IOException("Emergency journal marker parent is a symbolic link"); + } + if (Files.isSymbolicLink(normalizedMarker)) { + throw new IOException("Emergency journal marker is a symbolic link"); + } + if (parent != null) { + Files.createDirectories(parent); + } + Path staged = Files.createTempFile(parent, normalizedMarker.getFileName().toString(), ".journal"); + try { + Files.writeString(staged, EMERGENCY_JOURNAL_MARKER_VERSION, StandardCharsets.UTF_8, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + DurableFiles.publishStagedFile(staged, normalizedMarker); + } finally { + Files.deleteIfExists(staged); + } + } + /** * Returns the file backing this JSON cache, when one exists. * diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index f19161673..e8183eb53 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -687,6 +687,9 @@ private boolean persistServerVoteToJson(String server, OfflineBungeeVote vote) { if (jsonStorage == null || jsonStorageQuarantined) { return false; } + if (!ensureEmergencyJournalMarker()) { + return false; + } try { Collection keys = jsonStorage.getServerVotes(server); int index = nextCacheIndex(keys); @@ -704,6 +707,9 @@ private boolean persistOnlineVoteToJson(String uuid, OfflineBungeeVote vote) { if (jsonStorage == null || jsonStorageQuarantined) { return false; } + if (!ensureEmergencyJournalMarker()) { + return false; + } try { Collection keys = jsonStorage.getOnlineVotes(uuid); int index = nextCacheIndex(keys); @@ -925,6 +931,9 @@ private VoteTimeQueue findSqlEmergencyTwin(VoteTimeQueue emergency) { /** Persists a timed vote in the JSON emergency journal. */ private boolean persistTimeVoteToJson(VoteTimeQueue vote) { if (jsonStorage == null || jsonStorageQuarantined) return false; + if (!ensureEmergencyJournalMarker()) { + return false; + } try { int index = nextCacheIndex(jsonStorage.getTimedVoteCache()); vote.setTimedVoteCacheJsonKey(String.valueOf(index)); @@ -1439,7 +1448,7 @@ public void load() { * prove that two durable rows represent the same vote. */ private void loadJsonEmergencyVotes() { - if (jsonStorage == null) { + if (jsonStorage == null || !jsonStorage.hasEmergencyJournalMarker()) { return; } try { @@ -1503,6 +1512,28 @@ private void loadJsonEmergencyVotes() { } } + private boolean ensureEmergencyJournalMarker() { + if (jsonStorage == null) { + return false; + } + if (jsonStorage.hasEmergencyJournalMarker()) { + return true; + } + try { + if (useMySQL) { + // During MySQL recovery, only trust post-mark emergency data by discarding + // stale legacy JSON cache rows before the first durable marker write. + jsonStorage.clearData(); + jsonStorage.saveDurably(); + } + jsonStorage.markEmergencyJournalUsed(); + return true; + } catch (IOException | RuntimeException failure) { + debug1(failure); + return false; + } + } + private VoteTimeQueue decodeJsonTimedVote(String key, DataNode data) { if (data == null || !data.isObject()) return null; String name = data.has("Name") ? data.get("Name").asString() : ""; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index aa1f4fa40..22ce1aab0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; @@ -22,6 +23,8 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadLocalRandom; @@ -36,6 +39,7 @@ import com.bencodez.votingplugin.proxy.VotingPluginProxy; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; import com.bencodez.votingplugin.proxy.presence.BackendPresenceStatus; +import com.bencodez.votingplugin.control.PluginDeploymentService; import com.bencodez.votingplugin.proxy.control.ProxyControlResultStore.Route; import com.bencodez.votingplugin.proxy.control.ProxyControlResultStore.StoredResult; import com.bencodez.votingplugin.util.BoundedHttpBodyHandler; @@ -59,6 +63,7 @@ public final class ControlConnector implements AutoCloseable { private static final String COMMUNICATION_TEST_CAPABILITY = "config.transport-test.v1"; private static final String COMMUNICATION_TEST_PRESET = "communication-test"; private static final String PROXY_METHOD_CAPABILITY = "config.proxy-method.v1"; + private static final String PROXY_METHOD_HTTP_CAPABILITY = "config.proxy-method.v2"; private static final String PROXY_FILE_CAPABILITY = "config.proxy-files.v1"; private static final String PROXY_METHOD_PRESET = "proxy-method"; private static final String INTERNAL_OPERATION_TYPE = "_controlOperationType"; @@ -77,6 +82,10 @@ public final class ControlConnector implements AutoCloseable { private final ProxyRoutingConfigurationService configurationService; private final ProxyMethodConfigurationService methodConfigurationService; private final ProxyConfigurationFileService fileConfigurationService; + private final PluginDeploymentService deployments; + private final HttpClient deploymentHttp; + private final String deploymentCredential; + private final ExecutorService deploymentExecutor; private final Function> communicationTest; private final Runnable runtimeReplacement; private final Path dataDirectory; @@ -91,11 +100,13 @@ public final class ControlConnector implements AutoCloseable { private volatile boolean closed; private volatile boolean registered; private volatile boolean configurationAccepted; + private volatile boolean deploymentsAccepted; private volatile Set acceptedCapabilities = Set.of(); private volatile int failures; private volatile long snapshotSequence; private volatile ScheduledFuture scheduled; private volatile ScheduledFuture operationPolling; + private volatile ScheduledFuture deploymentPolling; private volatile CompletableFuture activeRequest; private volatile CompletableFuture activeOperation; private volatile Status status = Status.STARTING; @@ -104,14 +115,14 @@ public ControlConnector(Settings settings, ScheduledExecutorService scheduler, T Supplier> snapshotSource, Consumer logger, UUID sessionId, LongSupplier jitterSource) { this(settings, scheduler, transport, snapshotSource, logger, sessionId, jitterSource, null, - null, null, false, null, null, null, null, null); + null, null, false, null, null, null, null, null, null, null, null); } ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, Supplier> snapshotSource, Consumer logger, UUID sessionId, LongSupplier jitterSource, ProxyRoutingConfigurationService configurationService) { this(settings, scheduler, transport, snapshotSource, logger, sessionId, jitterSource, configurationService, - null, null, false, null, null, null, null, null); + null, null, false, null, null, null, null, null, null, null, null); } ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, @@ -119,10 +130,11 @@ public ControlConnector(Settings settings, ScheduledExecutorService scheduler, T LongSupplier jitterSource, ProxyRoutingConfigurationService configurationService, Map recoveredTasks) { this(settings, scheduler, transport, snapshotSource, logger, sessionId, jitterSource, configurationService, - null, null, false, null, null, null, null, null); + null, null, false, null, null, null, null, null, null, null, null); completedTasks.putAll(recoveredTasks); } + @SuppressWarnings("unused") private ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, Supplier> snapshotSource, Consumer logger, UUID sessionId, LongSupplier jitterSource, ProxyRoutingConfigurationService configurationService, Path dataDirectory, @@ -130,6 +142,19 @@ private ControlConnector(Settings settings, ScheduledExecutorService scheduler, Function> communicationTest, ProxyMethodConfigurationService methodConfigurationService, Runnable runtimeReplacement, ProxyConfigurationFileService fileConfigurationService) { + this(settings, scheduler, transport, snapshotSource, logger, sessionId, jitterSource, configurationService, + dataDirectory, route, recovering, recoveryComplete, communicationTest, methodConfigurationService, + runtimeReplacement, fileConfigurationService, null, null, null); + } + + private ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, + Supplier> snapshotSource, Consumer logger, UUID sessionId, + LongSupplier jitterSource, ProxyRoutingConfigurationService configurationService, Path dataDirectory, + Route route, boolean recovering, Runnable recoveryComplete, + Function> communicationTest, + ProxyMethodConfigurationService methodConfigurationService, Runnable runtimeReplacement, + ProxyConfigurationFileService fileConfigurationService, PluginDeploymentService deployments, + HttpClient deploymentHttp, String deploymentCredential) { this.settings = Objects.requireNonNull(settings, "settings"); this.scheduler = Objects.requireNonNull(scheduler, "scheduler"); this.transport = Objects.requireNonNull(transport, "transport"); @@ -142,6 +167,14 @@ private ControlConnector(Settings settings, ScheduledExecutorService scheduler, this.communicationTest = communicationTest; this.runtimeReplacement = runtimeReplacement; this.fileConfigurationService = fileConfigurationService; + this.deployments = deployments; + this.deploymentHttp = deploymentHttp; + this.deploymentCredential = deploymentCredential; + this.deploymentExecutor = deployments == null ? null : Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "votingplugin-control-proxy-deployment"); + thread.setDaemon(true); + return thread; + }); this.dataDirectory = dataDirectory; this.route = route; this.recovering = recovering; @@ -210,12 +243,15 @@ public static ControlConnector create(VotingPluginProxy proxy) throws IOExceptio backends.sort(Comparator.comparing(ObservedBackend::backendId)); return List.copyOf(backends); }; + PluginDeploymentService deployments = prepareDeployment(proxy); + HttpClient deploymentHttp = deployments == null ? null : HttpClient.newBuilder() + .connectTimeout(Duration.ofMillis(settings.connectTimeoutMillis())).followRedirects(HttpClient.Redirect.NEVER).build(); ControlConnector connector = new ControlConnector(settings, proxy.getScheduler(), transport, snapshot, message -> proxy.log("[Control] " + message), UUID.randomUUID(), () -> ThreadLocalRandom.current().nextLong(), new ProxyRoutingConfigurationService(proxy), dataDirectory, route, recovering, proxy::restartControlServicesAfterRecovery, server -> proxy.testBackendCommunication(server, 5000L), new ProxyMethodConfigurationService(proxy), - () -> proxy.reloadCore(true), new ProxyConfigurationFileService(proxy)); + () -> proxy.reloadCore(true), new ProxyConfigurationFileService(proxy), deployments, deploymentHttp, credential); if (recovered != null) connector.completedTasks.putAll(recovered.results()); return connector; } @@ -228,6 +264,80 @@ public void start() { schedule(0); operationPolling = scheduler.scheduleWithFixedDelay(this::pollOperations, OPERATION_POLL_MILLIS, OPERATION_POLL_MILLIS, TimeUnit.MILLISECONDS); + deploymentPolling = scheduler.scheduleWithFixedDelay(this::pollDeployments, + OPERATION_POLL_MILLIS, OPERATION_POLL_MILLIS, TimeUnit.MILLISECONDS); + } + + /** Keeps large artifact transfer and disk staging off proxy event threads and serializes it with operations. */ + private void pollDeployments() { + synchronized (operationLifecycle) { + if (closed || !registered || status != Status.CONNECTED || !deploymentsAccepted || deployments == null + || !inFlight.compareAndSet(false, true)) return; + } + CompletableFuture done = new CompletableFuture<>(); + activeOperation = done; + CompletableFuture work; + try { + CompletableFuture claim = transport.send(deploymentClaimRequest()); + activeRequest = claim; + work = claim.thenCompose(this::handleDeploymentClaim); + } catch (RuntimeException failure) { + work = CompletableFuture.failedFuture(failure); + } + work.whenComplete((ignored, failure) -> { + activeRequest = null; + if (failure == null) done.complete(null); else { registered = false; done.completeExceptionally(unwrap(failure)); } + if (activeOperation == done) activeOperation = null; + finishCycle(); + if (failure != null && !closed) onFailure(unwrap(failure)); + }); + } + + private Request deploymentClaimRequest() { + JsonObject body = sessionBody(); + return new Request("POST", "/api/v1/nodes/" + settings.nodeId() + "/deployments", body.toString()); + } + + private CompletableFuture handleDeploymentClaim(Response response) { + if (response.statusCode == 204 || closed) return CompletableFuture.completedFuture(null); + if (response.statusCode == 404) { registered = false; throw new RegistryLostException(); } + requireSuccess(response); + PluginDeploymentService.Task task = deploymentTask(parseObject(response.body)); + return CompletableFuture.supplyAsync(() -> deployments.deploy(task, settings.endpoint(), settings.nodeId(), sessionId, + deploymentCredential, deploymentHttp, Duration.ofMillis(settings.requestTimeoutMillis()), () -> !closed), + deploymentExecutor) + .thenCompose(result -> { + if (closed) return CompletableFuture.completedFuture(null); + JsonObject body = new JsonObject(); + body.addProperty("sessionId", sessionId.toString()); + body.addProperty("success", result.success()); + body.addProperty("code", result.code()); + body.addProperty("message", boundedResultMessage(result.message())); + body.addProperty("attemptId", task.attemptId().toString()); + CompletableFuture submitted = transport.send(new Request("POST", "/api/v1/nodes/" + + settings.nodeId() + "/deployments/" + task.deploymentId() + "/result", body.toString())); + activeRequest = submitted; + return submitted.thenAccept(ControlConnector::requireSuccess); + }); + } + + private static PluginDeploymentService.Task deploymentTask(JsonObject task) { + try { + return new PluginDeploymentService.Task(UUID.fromString(requireString(task, "deploymentId")), + requireString(task, "artifactId"), requireString(task, "sha256"), + Long.parseLong(requireString(task, "size")), UUID.fromString(requireString(task, "attemptId"))); + } catch (RuntimeException failure) { throw new MalformedResponseException(); } + } + + private static PluginDeploymentService prepareDeployment(VotingPluginProxy proxy) { + try { + var source = proxy.getClass().getProtectionDomain().getCodeSource(); + if (source == null || !"file".equalsIgnoreCase(source.getLocation().getProtocol())) return null; + return PluginDeploymentService.proxy(Path.of(source.getLocation().toURI())); + } catch (IOException | URISyntaxException | RuntimeException failure) { + proxy.log("[Control] Plugin deployment staging is unavailable; capability not advertised"); + return null; + } } /** Polls only the operation queue; heartbeat and presence retain their configured cadence. */ @@ -434,7 +544,9 @@ private void handlePrimaryResponse(Response response, boolean registration) { } acceptedCapabilities = Set.copyOf(negotiated); configurationAccepted = acceptedCapabilities.stream().anyMatch(Set.of(CONFIGURATION_CAPABILITY, - COMMUNICATION_TEST_CAPABILITY, PROXY_METHOD_CAPABILITY, PROXY_FILE_CAPABILITY)::contains); + COMMUNICATION_TEST_CAPABILITY, PROXY_METHOD_CAPABILITY, PROXY_METHOD_HTTP_CAPABILITY, + PROXY_FILE_CAPABILITY)::contains); + deploymentsAccepted = deployments != null && acceptedCapabilities.contains(PluginDeploymentService.CAPABILITY); } } @@ -811,7 +923,7 @@ private CompletableFuture executeTask(UUID operationId, JsonObject t if (isCommunicationTest(requested)) return acceptedCapabilities.contains(COMMUNICATION_TEST_CAPABILITY) ? executeCommunicationTest(task, requested) : completed(TaskResult.failure("UNSUPPORTED", "Communication testing was not negotiated")); - if (isProxyMethod(requested)) return acceptedCapabilities.contains(PROXY_METHOD_CAPABILITY) + if (isProxyMethod(requested)) return acceptedCapabilities.contains(proxyMethodCapability(requested)) ? executeProxyMethod(operationId, task, requested) : completed(TaskResult.failure("UNSUPPORTED", "Proxy method control was not negotiated")); if (!acceptedCapabilities.contains(CONFIGURATION_CAPABILITY)) { @@ -998,10 +1110,16 @@ private boolean capabilityAccepted(StoredResult result) { private static String requiredCapability(JsonObject configuration) { if (isProxyFile(configuration)) return PROXY_FILE_CAPABILITY; if (isCommunicationTest(configuration)) return COMMUNICATION_TEST_CAPABILITY; - if (isProxyMethod(configuration)) return PROXY_METHOD_CAPABILITY; + if (isProxyMethod(configuration)) return proxyMethodCapability(configuration); return CONFIGURATION_CAPABILITY; } + private static String proxyMethodCapability(JsonObject configuration) { + JsonObject options = configuration == null ? null : configuration.getAsJsonObject("options"); + return options != null && options.has("method") && "HTTP".equals(options.get("method").getAsString()) + ? PROXY_METHOD_HTTP_CAPABILITY : PROXY_METHOD_CAPABILITY; + } + private static CompletableFuture completed(TaskResult result) { return CompletableFuture.completedFuture(result); } @@ -1093,7 +1211,9 @@ private void addCapabilities(JsonObject body) { if (configurationService != null) advertised.add(CONFIGURATION_CAPABILITY); if (communicationTest != null) advertised.add(COMMUNICATION_TEST_CAPABILITY); if (methodConfigurationService != null) advertised.add(PROXY_METHOD_CAPABILITY); + if (methodConfigurationService != null) advertised.add(PROXY_METHOD_HTTP_CAPABILITY); if (fileConfigurationService != null) advertised.add(PROXY_FILE_CAPABILITY); + if (deployments != null) advertised.add(PluginDeploymentService.CAPABILITY); body.add("capabilities", advertised); JsonArray required = new JsonArray(); required.add("presence.snapshot"); @@ -1133,10 +1253,13 @@ public void close() { } ScheduledFuture polling = operationPolling; if (polling != null) polling.cancel(false); + ScheduledFuture deployment = deploymentPolling; + if (deployment != null) deployment.cancel(false); CompletableFuture request = activeRequest; if (request != null) { request.cancel(true); } + if (deploymentExecutor != null) deploymentExecutor.shutdownNow(); CompletableFuture operation = activeOperation; if (operation != null) { try { @@ -1150,6 +1273,16 @@ public void close() { throw new IllegalStateException("Interrupted while waiting for the Control operation", e); } } + if (deploymentExecutor != null) { + try { + if (!deploymentExecutor.awaitTermination(OPERATION_SHUTDOWN_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new IllegalStateException("Control deployment worker did not stop before connector shutdown"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for the Control deployment worker", e); + } + } transport.close(); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSyncTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSyncTest.java new file mode 100644 index 000000000..e02b015f4 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSyncTest.java @@ -0,0 +1,158 @@ +package com.bencodez.votingplugin.backendproxy.global; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.Test; + +import com.bencodez.advancedcore.api.time.TimeChecker; +import com.bencodez.advancedcore.api.time.TimeType; +import com.bencodez.advancedcore.api.user.UserManager; +import com.bencodez.advancedcore.api.user.UserStorage; +import com.bencodez.advancedcore.api.user.usercache.UserDataManager; +import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalDataHandler; +import com.bencodez.simpleapi.scheduler.BukkitScheduler; +import com.bencodez.simpleapi.sql.data.DataValueBoolean; +import com.bencodez.simpleapi.sql.data.DataValueString; +import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.config.BungeeSettings; + +class BackendGlobalDataSyncTest { + + @Test + void bungeeForceUpdateStateMutationRunsOnTheBukkitScheduler() { + VotingPluginMain plugin = mock(VotingPluginMain.class); + BungeeSettings bungeeSettings = mock(BungeeSettings.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + GlobalDataHandler globalDataHandler = mock(GlobalDataHandler.class); + UserDataManager dataManager = mock(UserDataManager.class); + UserManager userManager = mock(UserManager.class); + AtomicReference scheduled = new AtomicReference<>(); + AtomicReference asyncWrite = new AtomicReference<>(); + AtomicInteger schedulerRuns = new AtomicInteger(); + + when(plugin.getBungeeSettings()).thenReturn(bungeeSettings); + when(bungeeSettings.getServer()).thenReturn("lobby"); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + when(plugin.getStorageType()).thenReturn(UserStorage.FLAT); + when(plugin.getUserManager()).thenReturn(userManager); + when(userManager.getDataManager()).thenReturn(dataManager); + + org.mockito.Mockito.doAnswer(invocation -> { + scheduled.set(invocation.getArgument(1)); + schedulerRuns.incrementAndGet(); + return null; + }).when(scheduler).executeOrScheduleSync(eq(plugin), any(Runnable.class)); + org.mockito.Mockito.doAnswer(invocation -> { + asyncWrite.set(invocation.getArgument(1)); + return null; + }).when(scheduler).runTaskAsynchronously(eq(plugin), any(Runnable.class)); + + BackendGlobalDataSync sync = new BackendGlobalDataSync(plugin, ignored -> { + }); + setField(sync, "globalDataHandler", globalDataHandler); + + HashMap data = new HashMap<>(); + data.put("ForceUpdate", new DataValueBoolean(true)); + when(globalDataHandler.getExact("lobby")).thenReturn(data); + + sync.checkGlobalData(); + + verify(plugin, never()).getMysql(); + verify(plugin).getBukkitScheduler(); + verify(globalDataHandler, never()).setBoolean("lobby", "ForceUpdate", false); + verify(plugin, never()).setUpdate(anyBoolean()); + verify(plugin, never()).update(); + verify(dataManager, never()).clearCache(); + assertNotNull(scheduled.get()); + assertTrue(schedulerRuns.get() == 1); + + sync.checkGlobalData(); + assertTrue(schedulerRuns.get() == 1); + verify(plugin, never()).setUpdate(anyBoolean()); + verify(plugin, never()).update(); + verify(dataManager, never()).clearCache(); + + scheduled.get().run(); + verify(plugin).setUpdate(true); + verify(plugin).update(); + verify(dataManager).clearCache(); + assertNotNull(asyncWrite.get()); + asyncWrite.get().run(); + verify(globalDataHandler).setBoolean("lobby", "ForceUpdate", false); + + sync.checkGlobalData(); + assertTrue(schedulerRuns.get() == 2); + } + + @Test + void bungeeTimeChangeForceChangedRunsOnTheBukkitScheduler() { + VotingPluginMain plugin = mock(VotingPluginMain.class); + BungeeSettings bungeeSettings = mock(BungeeSettings.class); + BukkitScheduler scheduler = mock(BukkitScheduler.class); + TimeChecker timeChecker = mock(TimeChecker.class); + GlobalDataHandler globalDataHandler = mock(GlobalDataHandler.class); + AtomicReference scheduled = new AtomicReference<>(); + AtomicReference asyncCompletion = new AtomicReference<>(); + + when(plugin.getBungeeSettings()).thenReturn(bungeeSettings); + when(bungeeSettings.getServer()).thenReturn("lobby"); + when(plugin.getBukkitScheduler()).thenReturn(scheduler); + when(plugin.getTimeChecker()).thenReturn(timeChecker); + + org.mockito.Mockito.doAnswer(invocation -> { + scheduled.set(invocation.getArgument(1)); + return null; + }).when(scheduler).executeOrScheduleSync(eq(plugin), any(Runnable.class)); + org.mockito.Mockito.doAnswer(invocation -> { + asyncCompletion.set(invocation.getArgument(1)); + return null; + }).when(scheduler).runTaskAsynchronously(eq(plugin), any(Runnable.class)); + + BackendGlobalDataSync sync = new BackendGlobalDataSync(plugin, ignored -> { }); + setField(sync, "globalDataHandler", globalDataHandler); + + HashMap data = new HashMap<>(); + data.put("LastUpdated", new DataValueString( + "" + LocalDateTime.now().atZone(ZoneOffset.UTC).toInstant().toEpochMilli())); + data.put(TimeType.DAY.toString(), new DataValueBoolean(true)); + + assertTrue(sync.checkGlobalDataTime(TimeType.DAY, data)); + + verify(plugin).getBukkitScheduler(); + verify(timeChecker, never()).forceChanged(any(), any(Boolean.class), any(Boolean.class), any(Boolean.class)); + assertNotNull(scheduled.get()); + scheduled.get().run(); + verify(timeChecker).forceChanged(TimeType.DAY, false, true, true); + verify(globalDataHandler, never()).setBoolean("lobby", TimeType.DAY.toString(), false); + verify(globalDataHandler, never()).setData(eq("lobby"), any()); + assertNotNull(asyncCompletion.get()); + asyncCompletion.get().run(); + verify(globalDataHandler).setBoolean("lobby", TimeType.DAY.toString(), false); + verify(globalDataHandler).setData(eq("lobby"), any()); + } + + private static void setField(Object target, String fieldName, Object value) { + try { + Field field = BackendGlobalDataSync.class.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } catch (ReflectiveOperationException failure) { + throw new AssertionError(failure); + } + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index 8d707bfea..1dfd27974 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -501,9 +501,10 @@ class BackendConfigurationServiceTest { Files.writeString(directory.resolve("SpecialRewards.yml"), "VoteParty:\n Enabled: false\n"); BackendConfigurationService.QuickPreview party = service.previewQuickSetup("vote-party", Map.of( - "votesRequired", "25", "command", "give %player% diamond 1", "broadcast", "Party!", + "enabled", "false", "votesRequired", "25", "command", "give %player% diamond 1", "broadcast", "Party!", "giveAllPlayers", "false", "onlineOnly", "true")); assertTrue(party.proposal().content().contains("VotesRequired: 25")); + assertTrue(party.proposal().content().contains("Enabled: false")); } @Test void guidedSettingsReadTheInstalledValuesInsteadOfAssumingDefaults() throws Exception { @@ -529,6 +530,30 @@ class BackendConfigurationServiceTest { assertEquals("2", service.readQuickSetup("vote-party", Map.of()).options().get("rewardCommandCount")); } + @Test void votePartyEnabledOnlyChangesWhenExplicitlyEdited() throws Exception { + Path rewards = directory.resolve("SpecialRewards.yml"); + Files.writeString(rewards, "VoteParty:\n Enabled: false\n VotesRequired: 20\n" + + " GiveAllPlayers: false\n GiveOnlinePlayersOnly: true\n Broadcast: Keep\n"); + BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); + + BackendConfigurationService.QuickState disabled = service.readQuickSetup("vote-party", Map.of()); + assertEquals("false", disabled.options().get("enabled")); + BackendConfigurationService.QuickPreview unrelated = service.previewQuickSetup("vote-party", Map.of( + "enabled", disabled.options().get("enabled"), "votesRequired", "25", "command", "", + "broadcast", "Keep", "giveAllPlayers", "false", "onlineOnly", "true")); + assertTrue(unrelated.proposal().content().contains("Enabled: false")); + service.applyQuickSetup("vote-party", Map.of("enabled", "false", "votesRequired", "25", "command", "", + "broadcast", "Keep", "giveAllPlayers", "false", "onlineOnly", "true"), unrelated.revision()); + assertEquals("false", service.readQuickSetup("vote-party", Map.of()).options().get("enabled")); + + BackendConfigurationService.QuickPreview enable = service.previewQuickSetup("vote-party", Map.of( + "enabled", "true", "votesRequired", "25", "command", "", "broadcast", "Keep", + "giveAllPlayers", "false", "onlineOnly", "true")); + service.applyQuickSetup("vote-party", Map.of("enabled", "true", "votesRequired", "25", "command", "", + "broadcast", "Keep", "giveAllPlayers", "false", "onlineOnly", "true"), enable.revision()); + assertEquals("true", service.readQuickSetup("vote-party", Map.of()).options().get("enabled")); + } + @Test void oversizedInstalledGuidedValuesFailInsteadOfWedgingResultSubmission() throws Exception { Files.writeString(directory.resolve("SpecialRewards.yml"), "VoteParty:\n Broadcast: '" + "é".repeat(251) + "'\n"); @@ -607,7 +632,7 @@ class BackendConfigurationServiceTest { assertFalse(reward.proposal().content().contains("New message")); BackendConfigurationService.QuickPreview party = service.previewQuickSetup("vote-party", Map.of( - "votesRequired", "20", "command", "new party", "broadcast", "Party!", + "enabled", "true", "votesRequired", "20", "command", "new party", "broadcast", "Party!", "giveAllPlayers", "false", "onlineOnly", "true")); assertTrue(party.proposal().content().contains("existing party")); assertTrue(party.proposal().content().contains("new party")); @@ -837,6 +862,7 @@ class BackendConfigurationServiceTest { })); assertTrue(failure.rolledBack()); + assertTrue(failure.reloadAttempted()); assertEquals(2, targeted.get()); assertEquals(original, Files.readString(settings)); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java index c0452d01d..3a7570674 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java @@ -95,6 +95,18 @@ class BackendControlConnectorProtocolTest { .anyMatch(value -> "data.inspect.v1".equals(value.getAsString()))); } + @Test void deploymentCapabilityIsOnlyAddedWhenStagingWasPrepared() { + JsonObject unavailable = new JsonObject(); + BackendControlConnector.addCapabilities(unavailable); + assertFalse(unavailable.getAsJsonArray("capabilities").asList().stream() + .anyMatch(value -> PluginDeploymentService.CAPABILITY.equals(value.getAsString()))); + + JsonObject ready = new JsonObject(); + BackendControlConnector.addCapabilities(ready, true); + assertTrue(ready.getAsJsonArray("capabilities").asList().stream() + .anyMatch(value -> PluginDeploymentService.CAPABILITY.equals(value.getAsString()))); + } + @Test void heartbeatRetainsOmittedCapabilitiesAndHonorsExplicitReplacement() { JsonObject omitted = new JsonObject(); assertTrue(BackendControlConnector.negotiatedCapability(omitted, "config.files.v1", true)); @@ -136,16 +148,17 @@ class BackendControlConnectorProtocolTest { IllegalStateException failure = new IllegalStateException( "/srv/private/VoteSites.yml jdbc:mysql://database.internal user=secret"); - assertEquals("Configuration read failed; see the backend log", + assertEquals("The backend could not safely read the managed configuration file", BackendControlConnector.operationFailureMessage("READ", failure)); - assertEquals("Configuration preview failed; see the backend log", + assertEquals("The backend could not prepare a configuration preview", BackendControlConnector.operationFailureMessage("PREVIEW", failure)); - assertEquals("Configuration apply failed; see the backend log", + assertEquals("The backend could not apply the managed configuration", BackendControlConnector.operationFailureMessage("APPLY", failure)); - assertEquals("Configuration reload failed; see the backend log", - BackendControlConnector.reloadFailureMessage(failure)); assertFalse(BackendControlConnector.operationFailureMessage("READ", failure).contains("/srv")); - assertFalse(BackendControlConnector.reloadFailureMessage(failure).contains("secret")); + assertEquals("CONFIGURATION_MISSING", BackendControlConnector.operationFailureCode("READ", + new java.nio.file.NoSuchFileException("/srv/private/Config.yml"))); + assertEquals("CONFIGURATION_UNREADABLE", BackendControlConnector.operationFailureCode("READ", + new java.nio.file.AccessDeniedException("/srv/private/Config.yml"))); } @Test void unexpectedInspectionFailureMessagesNeverExposeTheCause() { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java new file mode 100644 index 000000000..c813891b4 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java @@ -0,0 +1,148 @@ +package com.bencodez.votingplugin.control; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicBoolean; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HexFormat; +import java.util.UUID; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class PluginDeploymentServiceTest { + @TempDir Path directory; + + @Test void backendStagesOnlyAnExactVerifiedVotingPluginJarAndIsIdempotent() throws Exception { + byte[] artifact = jar("name: VotingPlugin\nmain: example.Main\n"); + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + PluginDeploymentService.Task task = task(artifact); + + PluginDeploymentService.Result staged = service.stage(task, new ByteArrayInputStream(artifact), () -> true); + + assertTrue(staged.success()); + assertEquals("RESTART_REQUIRED", staged.code()); + assertEquals(artifact.length, Files.size(directory.resolve("update/VotingPlugin.jar"))); + assertTrue(service.stage(task, new ByteArrayInputStream(new byte[0]), () -> true).success()); + } + + @Test void backendIgnoresMatchingMarkerOnlyWhenTargetIsValidThenRestagesWhenCorrupted() throws Exception { + byte[] artifact = jar("name: VotingPlugin\n"); + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + PluginDeploymentService.Task task = task(artifact); + + service.stage(task, new ByteArrayInputStream(artifact), () -> true); + assertArrayEquals(artifact, Files.readAllBytes(directory.resolve("update/VotingPlugin.jar"))); + + byte[] corrupted = new byte[artifact.length]; + Arrays.fill(corrupted, (byte) 0x42); + Files.write(directory.resolve("update/VotingPlugin.jar"), corrupted); + assertArrayEquals(corrupted, Files.readAllBytes(directory.resolve("update/VotingPlugin.jar"))); + + assertTrue(service.stage(task, new ByteArrayInputStream(artifact), () -> true).success()); + assertArrayEquals(artifact, Files.readAllBytes(directory.resolve("update/VotingPlugin.jar"))); + } + + @Test void invalidPluginIdentityAndDigestNeverReachTheUpdateFolder() throws Exception { + byte[] wrongPlugin = jar("name: NotVotingPlugin\n"); + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + PluginDeploymentService.Result invalid = service.stage(task(wrongPlugin), new ByteArrayInputStream(wrongPlugin), () -> true); + assertFalse(invalid.success()); + assertEquals("INVALID_ARTIFACT", invalid.code()); + assertFalse(Files.exists(directory.resolve("update/VotingPlugin.jar"))); + + byte[] good = jar("name: VotingPlugin\n"); + PluginDeploymentService.Task mismatch = new PluginDeploymentService.Task(UUID.randomUUID(), "vp.jar", + "0".repeat(64), good.length, UUID.randomUUID()); + assertEquals("HASH_MISMATCH", service.stage(mismatch, new ByteArrayInputStream(good), () -> true).code()); + } + + @Test void proxyKeepsTheCurrentJarAsABackupBeforeReplacement() throws Exception { + Path current = directory.resolve("VotingPlugin.jar"); + byte[] original = jar("name: VotingPlugin\nversion: 0\n"); + byte[] first = jar("name: VotingPlugin\nversion: 1\n"); + byte[] second = jar("name: VotingPlugin\nversion: 2\n"); + Files.write(current, original); + PluginDeploymentService service = PluginDeploymentService.proxy(current); + + assertEquals("RESTART_REQUIRED", service.stage(task(first), new ByteArrayInputStream(first), () -> true).code()); + assertEquals("RESTART_REQUIRED", service.stage(task(second), new ByteArrayInputStream(second), () -> true).code()); + assertArrayEquals(first, Files.readAllBytes(directory.resolve("VotingPlugin.jar.control-backup"))); + assertArrayEquals(second, Files.readAllBytes(current)); + } + + @Test void cancellationDuringCopyDoesNotPublish() throws Exception { + byte[] artifact = jar("name: VotingPlugin\n"); + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + PluginDeploymentService.Task task = task(artifact); + + assertEquals("RESTART_REQUIRED", service.stage(task, new ByteArrayInputStream(artifact), () -> true).code()); + String markerBefore = Files.readString(directory.resolve("update/VotingPlugin.jar.control-deployment")); + byte[] targetBefore = Files.readAllBytes(directory.resolve("update/VotingPlugin.jar")); + + AtomicBoolean active = new AtomicBoolean(true); + InputStream body = new CancellingInputStream(new ByteArrayInputStream(artifact), active); + PluginDeploymentService.Task replacement = task(artifact); + assertEquals("CANCELLED", service.stage(replacement, body, active::get).code()); + + assertEquals(markerBefore, Files.readString(directory.resolve("update/VotingPlugin.jar.control-deployment"))); + assertArrayEquals(targetBefore, Files.readAllBytes(directory.resolve("update/VotingPlugin.jar"))); + assertFalse(active.get()); + } + + private static PluginDeploymentService.Task task(byte[] artifact) throws Exception { + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(artifact)); + return new PluginDeploymentService.Task(UUID.randomUUID(), "votingplugin.jar", digest, artifact.length, + UUID.randomUUID()); + } + + private static byte[] jar(String pluginYml) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (JarOutputStream jar = new JarOutputStream(bytes)) { + jar.putNextEntry(new JarEntry("plugin.yml")); + jar.write(pluginYml.getBytes(StandardCharsets.UTF_8)); + jar.closeEntry(); + jar.putNextEntry(new JarEntry("example/Main.class")); + jar.write(new byte[] { 0, 1, 2 }); + jar.closeEntry(); + } + return bytes.toByteArray(); + } + + private static class CancellingInputStream extends InputStream { + private final InputStream delegate; + private final AtomicBoolean active; + private boolean cancelled; + + private CancellingInputStream(InputStream delegate, AtomicBoolean active) { + this.delegate = delegate; + this.active = active; + } + + @Override + public int read(byte[] bytes, int start, int length) throws IOException { + int read = delegate.read(bytes, start, length); + if (!cancelled && read > 0) { + cancelled = true; + active.set(false); + } + return read; + } + + @Override + public int read() throws IOException { return delegate.read(); } + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java index 0037ebb45..409bf0356 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VoteCacheHandlerVoteIdTest.java @@ -405,6 +405,79 @@ public void currentVoteIdKeyLoadsFromJsonCache() { assertEquals(voteId, handler.getVotes("server").get(0).getVoteId()); } + @Test + public void legacyMySqlEmergencyRowsAreIgnoredWithoutJournalMarker() throws Exception { + VotingPluginBungee plugin = mock(VotingPluginBungee.class); + when(plugin.getDataFolder()).thenReturn(tempDir.toFile()); + Path cacheFile = tempDir.resolve("votecache.json"); + Files.writeString(cacheFile, """ + {"VoteCache":{"server":{"0":{"Name":"Player","Service":"Service","UUID":"player-uuid","Time":100}}}, + "OnlineCache":{"player-uuid":{"0":{"Name":"Player","Service":"Service","UUID":"player-uuid","Time":100}}}, + "TimedVoteCache":{"0":{"Name":"Player","Service":"Service","Time":100}}} + """); + BungeeJsonVoteCache durableStorage = new BungeeJsonVoteCache(plugin); + VoteCacheHandler mysqlEnabled = newMysqlEnabledLoadHandler(durableStorage); + + mysqlEnabled.load(); + + assertTrue(mysqlEnabled.getVotes("server").isEmpty()); + assertTrue(mysqlEnabled.getOnlineVotes("player-uuid").isEmpty()); + assertTrue(mysqlEnabled.getTimeChangeQueue().isEmpty()); + } + + @Test + public void markedMySqlEmergencyRowsAreRecoveredFromJournal() throws Exception { + VotingPluginBungee plugin = mock(VotingPluginBungee.class); + when(plugin.getDataFolder()).thenReturn(tempDir.toFile()); + Path cacheFile = tempDir.resolve("votecache.json"); + Files.writeString(cacheFile, """ + {"VoteCache":{"server":{"0":{"Name":"Player","Service":"Service","UUID":"player-uuid","Time":100}}}, + "OnlineCache":{"player-uuid":{"0":{"Name":"Player","Service":"Service","UUID":"player-uuid","Time":100}}}, + "TimedVoteCache":{"0":{"Name":"Player","Service":"Service","Time":100}}} + """); + BungeeJsonVoteCache durableStorage = new BungeeJsonVoteCache(plugin); + durableStorage.markEmergencyJournalUsed(); + VoteCacheHandler mysqlEnabled = newMysqlEnabledLoadHandler(durableStorage); + + mysqlEnabled.load(); + + assertEquals(1, mysqlEnabled.getVotes("server").size()); + assertEquals(1, mysqlEnabled.getOnlineVotes("player-uuid").size()); + assertEquals(1, mysqlEnabled.getTimeChangeQueue().size()); + } + + @Test + public void legacyRowsNeverImportAfterMySqlStartsEmergencyJournal() throws Exception { + VotingPluginBungee plugin = mock(VotingPluginBungee.class); + when(plugin.getDataFolder()).thenReturn(tempDir.toFile()); + Path cacheFile = tempDir.resolve("votecache.json"); + Files.writeString(cacheFile, """ + {"VoteCache":{"server":{"0":{"Name":"LegacyPlayer","Service":"Service","UUID":"legacy-uuid","Time":100}}}, + "OnlineCache":{"legacy-uuid":{"0":{"Name":"LegacyPlayer","Service":"Service","UUID":"legacy-uuid","Time":100}}}, + "TimedVoteCache":{"0":{"Name":"LegacyPlayer","Service":"Service","Time":100}}} + """); + BungeeJsonVoteCache durableStorage = new BungeeJsonVoteCache(plugin); + VoteCacheHandler writer = newMysqlEnabledLoadHandler(durableStorage); + + assertTrue(writer.addServerVoteDurably("server-new", + new OfflineBungeeVote(UUID.randomUUID(), "NewPlayer", "new-uuid", "Service", 200L, true, ""))); + assertTrue(writer.addOnlineVoteDurably("new-uuid", + new OfflineBungeeVote(UUID.randomUUID(), "NewPlayer", "new-uuid", "Service", 200L, true, ""))); + VoteTimeQueue emergencyTimed = new VoteTimeQueue(UUID.randomUUID(), "NewPlayer", "Service", 300L); + emergencyTimed.setUuid("new-uuid"); + assertTrue(writer.addTimeVoteToCache(emergencyTimed)); + + VoteCacheHandler restarted = newMysqlEnabledLoadHandler(durableStorage); + restarted.load(); + + assertEquals(1, restarted.getVotes("server-new").size()); + assertTrue(restarted.getVotes("server").isEmpty()); + assertEquals(1, restarted.getOnlineVotes("new-uuid").size()); + assertTrue(restarted.getOnlineVotes("legacy-uuid").isEmpty()); + assertEquals(1, restarted.getTimeChangeQueue().size()); + assertEquals("new-uuid", restarted.getTimeChangeQueue().element().getUuid()); + } + @Test public void legacyVoteIdKeyLoadsFromJsonCache() { UUID voteId = UUID.randomUUID(); @@ -502,6 +575,7 @@ public void identicalLegacyTimedJsonRowsKeepDistinctIdsAndDeleteIndividually() t @Test public void legacySqlAndJsonTimedRowsRemainDistinctWithoutVoteIds() throws Exception { DataNode jsonVote = mock(DataNode.class); + when(storage.hasEmergencyJournalMarker()).thenReturn(true); when(storage.getTimedVoteCache()).thenReturn(List.of("2")); when(storage.getTimedVoteCache("2")).thenReturn(jsonVote); when(storage.getServers()).thenReturn(Collections.emptyList()); @@ -1307,6 +1381,22 @@ public void debug1(String msg) { }; } + private static VoteCacheHandler newMysqlEnabledLoadHandler(IVoteCache storage) throws Exception { + ProxyVoteCacheTable voteTable = mock(ProxyVoteCacheTable.class); + ProxyOnlineVoteCacheTable onlineTable = mock(ProxyOnlineVoteCacheTable.class); + ProxyTimedVoteCacheTable timedTable = mock(ProxyTimedVoteCacheTable.class); + when(voteTable.getAllVotes()).thenReturn(Collections.emptyList()); + when(onlineTable.getAllVotes()).thenReturn(Collections.emptyList()); + when(timedTable.getAllVotes()).thenReturn(Collections.emptyList()); + + VoteCacheHandler mysqlEnabled = newHandler(storage); + setPrivateField(mysqlEnabled, "useMySQL", true); + setPrivateField(mysqlEnabled, "voteCacheTable", voteTable); + setPrivateField(mysqlEnabled, "onlineVoteCacheTable", onlineTable); + setPrivateField(mysqlEnabled, "timedVoteCacheTable", timedTable); + return mysqlEnabled; + } + @SuppressWarnings("unchecked") private static Map> cachedVotes(VoteCacheHandler target, String fieldName) throws Exception { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 933da58ad..d11910627 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -586,6 +586,31 @@ void pendingAckOutboxDoesNotBlockLaterCompletedQueueRows() { verify(voteCache).removeTimeVote(completed); } + @Test + void retainedHttpRolloverRowWithPendingBroadcastIdsDoesNotStarveLaterQueueRows() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + VoteTimeQueue retained = new VoteTimeQueue(java.util.UUID.randomUUID(), "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", true, "player-uuid", + java.util.Map.of("Server1", "00000000-0000-0000-0000-000000000123")); + VoteTimeQueue completed = new VoteTimeQueue(java.util.UUID.randomUUID(), "Other", "Service", 101L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", true); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + queue.add(retained); + queue.add(completed); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.removeTimeVote(completed)).thenAnswer(invocation -> queue.remove(completed)); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + spyProxy.processQueue(); + + assertEquals(1, queue.size()); + assertEquals(java.util.List.of(retained), new java.util.ArrayList<>(queue)); + assertTrue(retained.hasPendingHttpBroadcastDeliveryIds()); + verify(voteCache, Mockito.never()).removeTimeVote(retained); + verify(voteCache).removeTimeVote(completed); + } + @Test void completedAckOutboxWaitsForDurableFenceRetirementBeforeFinishing() throws Exception { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); diff --git a/docs/control-agent-contract.md b/docs/control-agent-contract.md index 8375e58f1..c1fe8aa18 100644 --- a/docs/control-agent-contract.md +++ b/docs/control-agent-contract.md @@ -1,15 +1,39 @@ # VotingPlugin Control agent contract This is the compact source of truth for an AI agent or Control client implementing the Bukkit integration. The connector -has two separate lanes: +has three separate lanes: - configuration operations use the negotiated `config.files.v1` / `config.quick-setup.v1` contract and may write only managed VotingPlugin YAML after preview and approval; - inspections use the optional `data.inspect.v1` contract and are always read-only. +- plugin deployment uses the optional `plugin.deploy.v1` contract and only stages a verified JAR for an explicit + administrator-selected restart; it never reloads or restarts the process. Do not translate an inspection request into a configuration operation. Do not add raw SQL, arbitrary commands, player enumeration, database browsing, filesystem paths, or generic key/value reads to either contract. +## Verified plugin deployment (`plugin.deploy.v1`) + +The capability is advertised only when the node can identify a safe staging target. Control assigns work through +`POST /api/v1/nodes/{nodeId}/deployments` with the current `sessionId`. A task contains `deploymentId`, `artifactId`, +lowercase `sha256`, byte `size`, and a leased `attemptId`. The node downloads only that assignment from +`GET /api/v1/nodes/{nodeId}/deployments/{deploymentId}/artifact`, authenticating with its enrolled credential and the +exact `X-Node-Session` and `X-Deployment-Attempt` headers. + +Downloads are limited to 64 MiB, do not follow redirects, require the assigned length and SHA-256, and must be a bounded +JAR with exactly one root `plugin.yml` declaring `VotingPlugin`. Staging rejects unsafe ZIP entries, symlinked targets, +oversized expansion, and compression bombs. Bukkit nodes atomically publish `VotingPlugin.jar` to the server update +folder. Proxy nodes first refresh a durable `.control-backup` from the current verified JAR, then atomically replace the +same discovered plugin JAR. Both write a small idempotency marker only after activation; a marker is trusted only after +rehashing and reinspecting the target. + +The node reports the result to `POST /api/v1/nodes/{nodeId}/deployments/{deploymentId}/result` with its `sessionId` and +the leased `attemptId`. Success is `RESTART_REQUIRED`; it means the JAR was staged, not loaded. Failures distinguish +download, size/hash, invalid artifact, staging/write, cancellation, and lost capability. Transfer and disk work run on a +dedicated background worker. Disable/reload marks the worker inactive, interrupts it, and prevents a stale completion +from submitting success. Mixed-version nodes that do not negotiate `plugin.deploy.v1` remain connected but are excluded +from deployment targets. + ## Proxy file contract (`config.proxy-files.v1`) This is a proxy-only capability, advertised by an enrolled BungeeCord or Velocity node. It is separate from diff --git a/docs/control-connector.md b/docs/control-connector.md index 4e797fef5..64e89e002 100644 --- a/docs/control-connector.md +++ b/docs/control-connector.md @@ -202,6 +202,15 @@ restores the previous hosted settings until that result is acknowledged, preserv ## Discovery semantics +When Control and a node both negotiate `plugin.deploy.v1`, the authenticated WebUI may explicitly select and upload a +VotingPlugin JAR for staging on that capable node. The connector downloads only its leased assignment, verifies its +declared size, SHA-256, bounded JAR structure, and root `plugin.yml`, and performs disk work on a dedicated background +worker. Bukkit stages into the configured update folder; BungeeCord and Velocity refresh a local backup and atomically +replace the currently discovered plugin JAR. A successful result is `RESTART_REQUIRED`: Control never automatically +reloads or restarts a server. Older nodes simply omit the capability and remain usable for their negotiated features. +See [the Control agent contract](control-agent-contract.md#verified-plugin-deployment-plugindeployv1) for the exact lease, +download, result, and safety contract. + Both platforms use the same connector implementation and protocol version `1`. Each proxy process creates a new session ID, registers, and sends full replacement backend snapshots with a monotonically increasing sequence. Registration and retries are idempotent. A Control restart is detected by a 404 heartbeat/presence response and causes re-registration. From 3acae789142d712da87e690297e3177629075376 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 13 Sep 2026 11:45:01 -0600 Subject: [PATCH 103/136] Keep global sync tests snapshot-compatible --- .../votingplugin/backendproxy/global/BackendGlobalDataSync.java | 2 +- .../backendproxy/global/BackendGlobalDataSyncTest.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSync.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSync.java index daba21034..febe99c28 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSync.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSync.java @@ -55,7 +55,7 @@ public void checkGlobalData() { && forceUpdateInProgress.compareAndSet(false, true)) { String serverName = plugin.getBungeeSettings().getServer(); try { - if (plugin.getStorageType().equals(UserStorage.MYSQL)) { + if (UserStorage.MYSQL.equals(plugin.getStorageType())) { plugin.getMysql().clearCacheBasic(); } plugin.getBukkitScheduler().executeOrScheduleSync(plugin, () -> { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSyncTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSyncTest.java index e02b015f4..800b14c10 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSyncTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/global/BackendGlobalDataSyncTest.java @@ -22,7 +22,6 @@ import com.bencodez.advancedcore.api.time.TimeChecker; import com.bencodez.advancedcore.api.time.TimeType; import com.bencodez.advancedcore.api.user.UserManager; -import com.bencodez.advancedcore.api.user.UserStorage; import com.bencodez.advancedcore.api.user.usercache.UserDataManager; import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalDataHandler; import com.bencodez.simpleapi.scheduler.BukkitScheduler; @@ -48,7 +47,6 @@ void bungeeForceUpdateStateMutationRunsOnTheBukkitScheduler() { when(plugin.getBungeeSettings()).thenReturn(bungeeSettings); when(bungeeSettings.getServer()).thenReturn("lobby"); when(plugin.getBukkitScheduler()).thenReturn(scheduler); - when(plugin.getStorageType()).thenReturn(UserStorage.FLAT); when(plugin.getUserManager()).thenReturn(userManager); when(userManager.getDataManager()).thenReturn(dataManager); From 60fb2621127cddb1694f5790c107321b247facfa Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 13 Sep 2026 17:18:45 -0600 Subject: [PATCH 104/136] Harden mixed-version control deployment --- .../control/BackendConfigurationService.java | 5 +- .../control/BackendControlConnector.java | 1 + .../control/PluginDeploymentService.java | 113 +++++++++++++++--- .../proxy/control/ControlConnector.java | 1 + .../BackendConfigurationServiceTest.java | 11 ++ .../control/PluginDeploymentServiceTest.java | 86 +++++++++++++ 6 files changed, 199 insertions(+), 18 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index c4e7d61c3..b15fc60c1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -500,7 +500,10 @@ private QuickProposal quickProposal(String preset, Map options, return new QuickProposal(fileName, yaml.saveToString()); } if ("vote-party".equals(preset)) { - yaml.set("VoteParty.Enabled", booleanOption(options, "enabled")); + // config.quick-setup.v1 historically always enabled VoteParty and therefore + // never sent this field. Keep that wire contract valid while newer clients + // can still explicitly choose either state. + yaml.set("VoteParty.Enabled", booleanOption(options, "enabled", true)); yaml.set("VoteParty.VotesRequired", boundedInteger(option(options, "votesRequired", "[0-9]{1,6}"), 1, 100000)); yaml.set("VoteParty.GiveAllPlayers", booleanOption(options, "giveAllPlayers")); yaml.set("VoteParty.GiveOnlinePlayersOnly", booleanOption(options, "onlineOnly")); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index c003a8109..b9e12f29e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -1128,6 +1128,7 @@ public void close() { if (inspection != null) inspection.cancel(false); ScheduledFuture deployment = deploymentPolling; if (deployment != null) deployment.cancel(false); + if (deployments != null) deployments.cancel(); inspectionExecutor.shutdownNow(); if (reload != null && Bukkit.isPrimaryThread()) reload.cancel(false); awaitShutdown(executor, operation); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java index 37b449cd7..a1114c26a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java @@ -21,6 +21,7 @@ import java.util.Locale; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BooleanSupplier; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -44,6 +45,7 @@ public final class PluginDeploymentService { private final Path marker; private final boolean replaceExisting; private final AtomicBoolean staging = new AtomicBoolean(); + private final AtomicReference activeResponse = new AtomicReference<>(); private PluginDeploymentService(Path target, boolean replaceExisting) throws IOException { this.target = target.toAbsolutePath().normalize(); @@ -83,6 +85,14 @@ public static PluginDeploymentService proxy(Path currentPluginJar) throws IOExce public boolean isStaging() { return staging.get(); } + /** Unblocks an in-progress response read during connector shutdown. */ + public void cancel() { + InputStream response = activeResponse.getAndSet(null); + if (response != null) { + try { response.close(); } catch (IOException ignored) { /* Shutdown is already in progress. */ } + } + } + public Result deploy(Task task, URI endpoint, String nodeId, UUID sessionId, String credential, HttpClient http, Duration timeout, BooleanSupplier active) { if (!staging.compareAndSet(false, true)) return Result.failure("DEPLOYMENT_FAILED", "Another deployment is still staging"); @@ -97,20 +107,31 @@ public Result deploy(Task task, URI endpoint, String nodeId, UUID sessionId, Str .header("X-Node-Session", sessionId.toString()) .header("X-Deployment-Attempt", task.attemptId().toString()).GET().build(); HttpResponse response = http.send(request, HttpResponse.BodyHandlers.ofInputStream()); - if (response.statusCode() != 200) { - response.body().close(); - return Result.failure("DOWNLOAD_FAILED", "Artifact download was rejected"); - } - String contentLength = response.headers().firstValue("Content-Length").orElse(null); - if (contentLength != null && (!contentLength.matches("[0-9]{1,9}") - || Long.parseLong(contentLength) != task.size())) { - response.body().close(); - return Result.failure("SIZE_MISMATCH", "Artifact size did not match the deployment task"); - } - try (InputStream body = response.body()) { - return stage(task, body, active); - } catch (IOException failure) { - return Result.failure("STAGING_FAILED", "Artifact could not be verified or staged on this node"); + InputStream body = response.body(); + activeResponse.set(body); + try { + if (response.statusCode() != 200) { + body.close(); + return Result.failure("DOWNLOAD_FAILED", "Artifact download was rejected"); + } + String contentLength = response.headers().firstValue("Content-Length").orElse(null); + if (contentLength != null && (!contentLength.matches("[0-9]{1,9}") + || Long.parseLong(contentLength) != task.size())) { + body.close(); + return Result.failure("SIZE_MISMATCH", "Artifact size did not match the deployment task"); + } + if (!active.getAsBoolean()) { + body.close(); + return Result.failure("CANCELLED", "Deployment was cancelled before download"); + } + try (body) { + return stage(task, body, active); + } catch (IOException failure) { + if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging"); + return Result.failure("STAGING_FAILED", "Artifact could not be verified or staged on this node"); + } + } finally { + activeResponse.compareAndSet(body, null); } } catch (InterruptedException failure) { Thread.currentThread().interrupt(); @@ -126,7 +147,9 @@ public Result deploy(Task task, URI endpoint, String nodeId, UUID sessionId, Str Result stage(Task task, InputStream body, BooleanSupplier active) throws IOException { validate(task); if (alreadyStaged(task)) return Result.restartRequired(); + activeResponse.compareAndSet(null, body); Path temporary = Files.createTempFile(root, target.getFileName().toString() + ".", ".download"); + Activation activation = null; try { MessageDigest digest = sha256(); long written = copyExact(body, temporary, task.size(), digest, active); @@ -137,14 +160,24 @@ Result stage(Task task, InputStream body, BooleanSupplier active) throws IOExcep } inspectJar(temporary); if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging"); - activate(temporary); + activation = new Activation(); + activate(temporary, activation); writeMarker(task); + activation.discard(); return Result.restartRequired(); } catch (CancelledDeploymentException failure) { return Result.failure("CANCELLED", "Deployment was cancelled before staging"); } catch (InvalidArtifactException failure) { return Result.failure("INVALID_ARTIFACT", "Artifact is not a bounded VotingPlugin JAR"); + } catch (IOException failure) { + if (activation != null) { + try { activation.rollback(); } + catch (IOException rollbackFailure) { failure.addSuppressed(rollbackFailure); } + } + if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging"); + throw failure; } finally { + activeResponse.compareAndSet(body, null); Files.deleteIfExists(temporary); } } @@ -154,8 +187,10 @@ private long copyExact(InputStream input, Path temporary, long expected, Message long total = 0; byte[] buffer = new byte[BUFFER_BYTES]; try (var output = Files.newOutputStream(temporary, StandardOpenOption.TRUNCATE_EXISTING)) { - for (int read; (read = input.read(buffer)) != -1;) { + for (;;) { if (!active.getAsBoolean()) throw new CancelledDeploymentException(); + int read = input.read(buffer); + if (read == -1) break; total += read; if (total > expected || total > MAX_ARTIFACT_BYTES) throw new InvalidArtifactException(); digest.update(buffer, 0, read); @@ -229,7 +264,7 @@ private static String unquote(String value) { || (value.startsWith("'") && value.endsWith("'"))) ? value.substring(1, value.length() - 1) : value; } - private void activate(Path temporary) throws IOException { + private void activate(Path temporary, Activation activation) throws IOException { if (replaceExisting) { if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) { throw new IOException("current proxy plugin JAR is unsafe"); @@ -248,10 +283,54 @@ private void activate(Path temporary) throws IOException { } finally { Files.deleteIfExists(backupTemp); } } move(temporary, target); + activation.published = true; force(target); forceDirectory(root); } + /** Holds a private copy until the durable idempotency marker has been published. */ + private final class Activation { + private final Path previous; + private boolean published; + + private Activation() throws IOException { + if (!Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + previous = null; + return; + } + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) { + throw new IOException("deployment target is unsafe"); + } + previous = Files.createTempFile(root, target.getFileName().toString() + ".", ".rollback"); + Files.copy(target, previous, StandardCopyOption.REPLACE_EXISTING); + force(previous); + } + + private void rollback() throws IOException { + if (!published) { + discard(); + return; + } + if (previous == null) { + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) { + throw new IOException("deployed target cannot be safely removed"); + } + Files.delete(target); + } else { + move(previous, target); + force(target); + } + forceDirectory(root); + discard(); + } + + private void discard() { + if (previous == null) return; + try { Files.deleteIfExists(previous); } + catch (IOException ignored) { /* A private stale rollback copy is safer than a false deployment result. */ } + } + } + private void writeMarker(Task task) throws IOException { Path temporary = Files.createTempFile(root, target.getFileName().toString() + ".", ".marker"); try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 22ce1aab0..89e83d5a3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -1255,6 +1255,7 @@ public void close() { if (polling != null) polling.cancel(false); ScheduledFuture deployment = deploymentPolling; if (deployment != null) deployment.cancel(false); + if (deployments != null) deployments.cancel(); CompletableFuture request = activeRequest; if (request != null) { request.cancel(true); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index 1dfd27974..6c7c4c6ae 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -554,6 +554,17 @@ class BackendConfigurationServiceTest { assertEquals("true", service.readQuickSetup("vote-party", Map.of()).options().get("enabled")); } + @Test void legacyVotePartyQuickSetupDefaultsAnOmittedEnabledOptionToTrue() throws Exception { + Files.writeString(directory.resolve("SpecialRewards.yml"), "VoteParty:\n Enabled: false\n"); + BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); + + BackendConfigurationService.QuickPreview preview = service.previewQuickSetup("vote-party", Map.of( + "votesRequired", "25", "command", "", "broadcast", "", "giveAllPlayers", "false", + "onlineOnly", "true")); + + assertTrue(preview.proposal().content().contains("Enabled: true")); + } + @Test void oversizedInstalledGuidedValuesFailInsteadOfWedgingResultSubmission() throws Exception { Files.writeString(directory.resolve("SpecialRewards.yml"), "VoteParty:\n Broadcast: '" + "é".repeat(251) + "'\n"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java index c813891b4..e04f5adfb 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; @@ -10,6 +11,11 @@ import java.io.IOException; import java.io.InputStream; import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -103,6 +109,56 @@ class PluginDeploymentServiceTest { assertFalse(active.get()); } + @Test void markerPublicationFailureRestoresThePreviousArtifact() throws Exception { + Path update = directory.resolve("update"); + Files.createDirectories(update); + byte[] original = jar("name: VotingPlugin\nversion: original\n"); + byte[] candidate = jar("name: VotingPlugin\nversion: candidate\n"); + Path target = update.resolve("VotingPlugin.jar"); + Files.write(target, original); + Path marker = update.resolve("VotingPlugin.jar.control-deployment"); + Files.createDirectory(marker); + Files.writeString(marker.resolve("keep"), "marker publication must fail"); + PluginDeploymentService service = PluginDeploymentService.backend(update); + + assertThrows(IOException.class, () -> service.stage(task(candidate), new ByteArrayInputStream(candidate), () -> true)); + + assertArrayEquals(original, Files.readAllBytes(target)); + } + + @Test void proxyMarkerPublicationFailureRestoresTheCurrentJarAndKeepsItsBackup() throws Exception { + byte[] original = jar("name: VotingPlugin\nversion: original\n"); + byte[] candidate = jar("name: VotingPlugin\nversion: candidate\n"); + Path target = directory.resolve("VotingPlugin.jar"); + Files.write(target, original); + Path marker = directory.resolve("VotingPlugin.jar.control-deployment"); + Files.createDirectory(marker); + Files.writeString(marker.resolve("keep"), "marker publication must fail"); + PluginDeploymentService service = PluginDeploymentService.proxy(target); + + assertThrows(IOException.class, () -> service.stage(task(candidate), new ByteArrayInputStream(candidate), () -> true)); + + assertArrayEquals(original, Files.readAllBytes(target)); + assertArrayEquals(original, Files.readAllBytes(directory.resolve("VotingPlugin.jar.control-backup"))); + } + + @Test void cancellationClosesAStalledArtifactStream() throws Exception { + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + AtomicBoolean active = new AtomicBoolean(true); + BlockingInputStream stalled = new BlockingInputStream(active); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future result = executor.submit(() -> service.stage( + task(new byte[] { 1 }), stalled, active::get)); + assertTrue(stalled.awaitRead()); + service.cancel(); + assertEquals("CANCELLED", result.get(5, TimeUnit.SECONDS).code()); + assertFalse(Files.exists(directory.resolve("update/VotingPlugin.jar"))); + } finally { + executor.shutdownNow(); + } + } + private static PluginDeploymentService.Task task(byte[] artifact) throws Exception { String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(artifact)); return new PluginDeploymentService.Task(UUID.randomUUID(), "votingplugin.jar", digest, artifact.length, @@ -145,4 +201,34 @@ public int read(byte[] bytes, int start, int length) throws IOException { @Override public int read() throws IOException { return delegate.read(); } } + + private static class BlockingInputStream extends InputStream { + private final AtomicBoolean active; + private final CountDownLatch reading = new CountDownLatch(1); + private boolean closed; + + private BlockingInputStream(AtomicBoolean active) { this.active = active; } + + private boolean awaitRead() throws InterruptedException { return reading.await(5, TimeUnit.SECONDS); } + + @Override + public synchronized int read(byte[] bytes, int start, int length) throws IOException { + reading.countDown(); + while (!closed) { + try { wait(); } + catch (InterruptedException failure) { Thread.currentThread().interrupt(); throw new IOException(failure); } + } + throw new IOException("stream closed"); + } + + @Override + public int read() throws IOException { return read(new byte[1], 0, 1); } + + @Override + public synchronized void close() { + closed = true; + active.set(false); + notifyAll(); + } + } } From b1664b2d57e553d3dbf3468c8cee2795269897dd Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 13 Sep 2026 17:56:37 -0600 Subject: [PATCH 105/136] Harden artifact reads and legacy proxy delivery --- .../transport/RedisBackendProxyTransport.java | 19 ++++++++++++++--- .../votingplugin/proxy/VotingPluginProxy.java | 9 ++++---- .../servicesites/ServiceSiteHandler.java | 18 ++++++++++++++++ .../RedisBackendProxyTransportTest.java | 21 +++++++++++++++++++ .../MultiProxyHandlerLifecycleTest.java | 10 +++++++++ 5 files changed, 69 insertions(+), 8 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index e105d6e4c..2e9de9272 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -176,7 +176,11 @@ void dispatchIdentified(JsonEnvelope envelope, String deliveryId) { } void dispatchLegacy(JsonEnvelope envelope) { - String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(envelope); + // A mixed-version publisher may put the reliable-only identity on a payload + // that is delivered through the legacy path. Do not expose that identity to + // handlers or let it alter legacy duplicate accounting. + JsonEnvelope legacyEnvelope = withoutReliableFields(envelope); + String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(legacyEnvelope); int encodedBytes = ProcessedVoteCache.legacyRedisDeliveryBytes(signature); boolean dispatch = false; synchronized (legacyLifecycle) { @@ -191,7 +195,7 @@ void dispatchLegacy(JsonEnvelope envelope) { && handoffDeliveryCount() < MAX_REPLAY_HANDOFF_DELIVERIES && bufferedLegacyDeliveryBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES - encodedBytes) { bufferedLegacyDeliveries.add(new BufferedHandoffDelivery( - nextHandoffSequence++, envelope, signature, false)); + nextHandoffSequence++, legacyEnvelope, signature, false)); bufferedLegacyDeliveryBytes += encodedBytes; } else { if (standbySubscriber) { @@ -214,7 +218,16 @@ && handoffDeliveryCount() < MAX_REPLAY_HANDOFF_DELIVERIES } if (dispatch) dispatchesInFlight++; } - if (dispatch) dispatchTracked(envelope); + if (dispatch) dispatchTracked(legacyEnvelope); + } + + private static JsonEnvelope withoutReliableFields(JsonEnvelope envelope) { + if (!envelope.getFields().containsKey(VotingPluginWire.K_REDIS_DELIVERY_ID)) return envelope; + JsonEnvelope.Builder builder = JsonEnvelope.builder(envelope.getSubChannel()).schema(envelope.getSchema()); + for (java.util.Map.Entry field : envelope.getFields().entrySet()) { + if (!VotingPluginWire.K_REDIS_DELIVERY_ID.equals(field.getKey())) builder.put(field.getKey(), field.getValue()); + } + return builder.build(); } /** Holds a consumed subscriber callback until a same-Redis transfer commits or rolls back. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 48f209b85..ba6f7f3f1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -5676,9 +5676,8 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim // Never retry this legacy copy as part of the ACK outbox: a legacy peer has // no receiver dedupe/ACK contract, while capable peers stay fully durable. // Persist the exact pending set until the transport accepts the copy. - if (!multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(player, uuid, service, time, - false, realVote, totals == null ? "" : totals.toString(), outbox.getVoteId(), false, false, 1, 1, - outbox.getMultiProxyOrigin()), + if (!multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.vote(player, uuid, service, time, + false, realVote, totals == null ? "" : totals.toString(), outbox.getVoteId(), false, false, 1, 1), new LinkedHashSet<>(outbox.getMultiProxyLegacyPendingRecipients()))) return false; outbox.setMultiProxyLegacyPendingRecipients(Collections.emptySet()); outbox.setDeliveryStateDirty(true); @@ -5702,9 +5701,9 @@ private boolean retryDurableMultiProxyOutbox(VoteTimeQueue outbox) { return finishMultiProxyRetirement(null, outbox); } if (!outbox.getMultiProxyLegacyPendingRecipients().isEmpty()) { - if (!multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(outbox.getName(), + if (!multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.vote(outbox.getName(), outbox.getUuid(), outbox.getService(), outbox.getTime(), false, outbox.isRealVote(), - outbox.getTotals(), outbox.getVoteId(), false, false, 1, 1, outbox.getMultiProxyOrigin()), + outbox.getTotals(), outbox.getVoteId(), false, false, 1, 1), new LinkedHashSet<>(outbox.getMultiProxyLegacyPendingRecipients()))) return false; outbox.setMultiProxyLegacyPendingRecipients(Collections.emptySet()); outbox.setDeliveryStateDirty(true); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/servicesites/ServiceSiteHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/servicesites/ServiceSiteHandler.java index 356cddc7c..1f52833c0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/servicesites/ServiceSiteHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/servicesites/ServiceSiteHandler.java @@ -21,6 +21,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -228,6 +229,20 @@ static byte[] readSuccessfulResponse(HttpResponse response, long ti throw new java.io.UncheckedIOException(e); } }); + AtomicBoolean finished = new AtomicBoolean(); + Thread deadline = new Thread(() -> { + try { + TimeUnit.NANOSECONDS.sleep(unit.toNanos(timeout)); + if (finished.compareAndSet(false, true)) input.close(); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } catch (IOException ignored) { + // The reader will report the resulting failure to the caller. + } + }, "VotingPlugin-ServiceSite-ReadDeadline"); + deadline.setDaemon(true); + deadline.setContextClassLoader(ClassLoader.getPlatformClassLoader()); + deadline.start(); try { return read.get(timeout, unit); } catch (TimeoutException e) { @@ -242,6 +257,9 @@ static byte[] readSuccessfulResponse(HttpResponse response, long ti read.cancel(true); Thread.currentThread().interrupt(); throw new IOException("Interrupted while reading service site response", e); + } finally { + finished.set(true); + deadline.interrupt(); } } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java index ffe3d9f06..ff2590734 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java @@ -164,6 +164,27 @@ void standbyReplaysMixedLegacyAndIdentifiedDeliveriesInArrivalOrder() throws Exc order.verify(messages).onMessage(identified); } + @Test + void legacyCopiesDoNotExposeReliableDeliveryIdentity() throws Exception { + ProcessedVoteCache cache = new ProcessedVoteCache(); + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, cache); + Field identity = RedisBackendProxyTransport.class.getDeclaredField("subscriberIdentity"); + identity.setAccessible(true); + cache.registerRedisSubscriber(identity.get(transport)); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + setField(transport, "messageHandler", messages); + String deliveryId = "00000000-0000-0000-0000-000000000099"; + JsonEnvelope mixed = JsonEnvelope.builder(VotingPluginWire.SUB_VOTE_UPDATE) + .put(VotingPluginWire.K_REDIS_DELIVERY_ID, deliveryId).put("payload", "legacy").build(); + + transport.dispatchLegacy(mixed); + + org.mockito.ArgumentCaptor captured = org.mockito.ArgumentCaptor.forClass(JsonEnvelope.class); + verify(messages).onMessage(captured.capture()); + assertFalse(captured.getValue().getFields().containsKey(VotingPluginWire.K_REDIS_DELIVERY_ID)); + assertEquals("legacy", captured.getValue().getFields().get("payload")); + } + @Test void identifiedStandbyOverflowAbortsPublication() throws Exception { RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, mock(ProcessedVoteCache.class)); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java index 4d9bfce6b..144808061 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java @@ -119,6 +119,16 @@ void routesStableWireVoteWithoutOriginThroughLegacyTrigger() throws Exception { org.mockito.ArgumentMatchers.any(), org.mockito.ArgumentMatchers.anyString()); } + @Test + void legacyVoteEnvelopeHasNoReliableOriginMetadata() { + UUID voteId = UUID.randomUUID(); + JsonEnvelope envelope = VotingPluginWire.vote("Player", "00000000-0000-0000-0000-000000000001", + "Service", 100L, false, true, "totals", voteId, false, false, 1, 1); + + assertEquals(voteId.toString(), envelope.getFields().get(VotingPluginWire.K_VOTE_ID)); + assertFalse(envelope.getFields().containsKey(VotingPluginWire.K_MULTI_PROXY_ORIGIN)); + } + @Test void forwardsOriginAndStableWireVoteIdToDurableTrigger() throws Exception { MultiProxyHandler handler = mock(MultiProxyHandler.class, org.mockito.Mockito.CALLS_REAL_METHODS); From 3adf47c4dbd229245ec05ec57abbf54ddde4d57d Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 13 Sep 2026 18:16:10 -0600 Subject: [PATCH 106/136] Bound deployments and same-port HTTP changes --- .../control/PluginDeploymentService.java | 13 ++++++++++++ .../votingplugin/proxy/VotingPluginProxy.java | 7 +++++++ .../ProxyMethodConfigurationService.java | 6 +++++- .../ProxyMethodConfigurationServiceTest.java | 20 +++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java index a1114c26a..4e58ff14c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java @@ -22,6 +22,10 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import java.util.function.BooleanSupplier; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -39,6 +43,11 @@ public final class PluginDeploymentService { private static final long MAX_INSPECTED_UNCOMPRESSED_BYTES = 256L * 1024L * 1024L; private static final int MAX_PLUGIN_YML_BYTES = 64 * 1024; private static final String MARKER = ".control-deployment"; + private static final ScheduledExecutorService BODY_DEADLINE_EXECUTOR = Executors.newSingleThreadScheduledExecutor(r -> { + Thread thread = new Thread(r, "VotingPlugin-control-artifact-deadline"); + thread.setDaemon(true); + return thread; + }); private final Path target; private final Path root; @@ -109,6 +118,9 @@ public Result deploy(Task task, URI endpoint, String nodeId, UUID sessionId, Str HttpResponse response = http.send(request, HttpResponse.BodyHandlers.ofInputStream()); InputStream body = response.body(); activeResponse.set(body); + ScheduledFuture bodyDeadline = BODY_DEADLINE_EXECUTOR.schedule(() -> { + try { body.close(); } catch (IOException ignored) { /* The staging operation reports the timeout. */ } + }, timeout.toMillis(), TimeUnit.MILLISECONDS); try { if (response.statusCode() != 200) { body.close(); @@ -131,6 +143,7 @@ public Result deploy(Task task, URI endpoint, String nodeId, UUID sessionId, Str return Result.failure("STAGING_FAILED", "Artifact could not be verified or staged on this node"); } } finally { + bodyDeadline.cancel(false); activeResponse.compareAndSet(body, null); } } catch (InterruptedException failure) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index ba6f7f3f1..6245a9934 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -3951,6 +3951,13 @@ public synchronized boolean hasMatchingLiveHttpTransport(VotingPluginProxyConfig && java.util.Objects.equals(liveHttpPublicEndpoint, candidate.getHttpPublicEndpoint()); } + /** Returns whether a candidate would collide with the currently bound HTTP listener. */ + public synchronized boolean hasLiveHttpBind(VotingPluginProxyConfig candidate) { + return candidate != null && httpTransportServer != null + && java.util.Objects.equals(liveHttpHost, candidate.getHttpHost()) + && liveHttpPort == candidate.getHttpPort(); + } + /** Cancels a candidate listener when configuration publication fails. */ public synchronized void cancelPreparedHttpTransportChange() { PreparedHttpTransport prepared = PREPARED_HTTP_TRANSPORTS.remove(httpTransportPreparationKey()); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java index 6a4bb1cae..4d04c9f84 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java @@ -91,8 +91,12 @@ public void apply(ProxyMethodConfiguration proposal, String expectedRevision) th proxy.getConfig().persistControlProxyMethod(proposal.method().name(), expectedRevision, latest -> { validate(proposal, latest); - if (proposal.method() == BungeeMethod.HTTP + if (proposal.method() == BungeeMethod.HTTP && !proxy.hasMatchingLiveHttpTransport(latest)) { + if (proxy.hasLiveHttpBind(latest)) { + throw new IllegalArgumentException( + "HTTP.PublicEndpoint changed while the HTTP listener is bound; restart is required before applying it"); + } proxy.prepareHttpTransportChange(latest); preparedHttp[0] = true; } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java index aab0dbb5a..2587c9996 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java @@ -201,6 +201,26 @@ void closesPreparedHttpListenerWhenPublicationVerificationFails() throws Excepti verify(proxy).cancelPreparedHttpTransportChange(); } + @Test + void rejectsPublicEndpointChangeOnAnOccupiedListenerBeforePublishing() throws Exception { + when(config.getBungeeMethod()).thenReturn("HTTP"); + VotingPluginProxyConfig fresh = validHttpConfig(); + when(fresh.getHttpPublicEndpoint()).thenReturn("https://new.example.test:1297"); + when(proxy.hasMatchingLiveHttpTransport(fresh)).thenReturn(false); + when(proxy.hasLiveHttpBind(fresh)).thenReturn(true); + doAnswer(invocation -> { + VotingPluginProxyConfig.ControlProxyMethodValidator validator = invocation.getArgument(2); + validator.validate(fresh); + return null; + }).when(config).persistControlProxyMethod(org.mockito.ArgumentMatchers.eq("HTTP"), + org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); + + assertThrows(IllegalArgumentException.class, + () -> service.apply(new ProxyMethodConfiguration(BungeeMethod.HTTP), service.read().revision())); + verify(proxy, never()).prepareHttpTransportChange(fresh); + verify(config, never()).verifyControlProxyRoutingInstalled(); + } + private VotingPluginProxyConfig validHttpConfig() { VotingPluginProxyConfig fresh = mock(VotingPluginProxyConfig.class); when(fresh.getHttpHost()).thenReturn("127.0.0.1"); From a1fafd47a5dc7cedd8c37d57885f4a2a54edd1d2 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 13 Sep 2026 19:17:47 -0600 Subject: [PATCH 107/136] Keep HTTP delivery on verified capable peers --- .../votingplugin/proxy/VotingPluginProxy.java | 36 ++++++++- .../proxy/multiproxy/MultiProxyHandler.java | 55 ++++++++++++++ .../MultiProxyHandlerLifecycleTest.java | 35 +++++++++ .../tests/VotingPluginProxyTest.java | 74 +++++++++++++++++++ 4 files changed, 196 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 6245a9934..d8cac233c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -5623,15 +5623,18 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim if (multiProxyHandler == null) return false; multiProxyHandler.announceMultiProxyVoteCapability(); Set recipients = multiProxyHandler.getMultiProxyVoteRecipients(); + Set renewingRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal(); Set configuredRecipients = multiProxyHandler.getConfiguredMultiProxyVoteRecipients(); // Keep custom MultiProxyHandler integrations that override only the // established recipient method source-compatible while the base handler // learns capabilities. if (configuredRecipients.isEmpty() && !recipients.isEmpty()) configuredRecipients.addAll(recipients); if (configuredRecipients.isEmpty()) return true; + Set durableRecipients = new LinkedHashSet<>(recipients); + durableRecipients.addAll(renewingRecipients); Set legacyRecipients = new LinkedHashSet<>(configuredRecipients); - legacyRecipients.removeAll(recipients); - if (recipients.isEmpty()) { + legacyRecipients.removeAll(durableRecipients); + if (durableRecipients.isEmpty()) { // Older peers do not understand acknowledgements. Preserve their historical // fire-and-forget route instead of creating an outbox they can never ACK. return multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.vote(player, uuid, service, time, @@ -5645,7 +5648,7 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim // The live retry key is the stable vote ID; copy it from the enclosing state // by locating its identity rather than creating a new duplicate record. outbox.setVoteId(findLiveVoteId(retryState)); - outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), recipients); + outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), durableRecipients); outbox.setMultiProxyLegacyPendingRecipients(legacyRecipients); outbox.setRealVote(realVote); outbox.setDeliveryStateDirty(true); @@ -5667,7 +5670,7 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim } } if (!outbox.isMultiProxyForwardingRequired()) { - outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), recipients); + outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), durableRecipients); outbox.setMultiProxyLegacyPendingRecipients(legacyRecipients); outbox.setRealVote(realVote); outbox.setProcessed(true); @@ -5725,6 +5728,31 @@ private boolean sendDurableMultiProxyOutbox(VoteTimeQueue outbox) { Set pending = new LinkedHashSet<>(outbox.getMultiProxyRecipients()); pending.removeAll(outbox.getMultiProxyAcknowledgedServers()); if (pending.isEmpty()) return true; + // A durable outbox recipient is known to have required acknowledgements when + // the row was created. It must therefore have a *current* capability lease + // before a retry publishes to it. In particular, a proxy restart/reload clears + // in-memory capability knowledge while the outbox survives; treating that + // recipient as legacy would let a rolled-back peer execute every retry without + // ever acknowledging it. + Set leasedRecipients = new LinkedHashSet<>(); + for (String recipient : multiProxyHandler.getMultiProxyVoteRecipients()) { + if (recipient != null) leasedRecipients.add(recipient.toLowerCase(Locale.ROOT)); + } + Set awaitingRenewal = new LinkedHashSet<>(); + for (String recipient : pending) { + if (recipient != null && !leasedRecipients.contains(recipient.toLowerCase(Locale.ROOT))) { + awaitingRenewal.add(recipient.toLowerCase(Locale.ROOT)); + } + } + if (!awaitingRenewal.isEmpty()) { + // Queue retries may be the only activity after an ACK was lost. Renew the + // handshake here so a recovered peer can become eligible without waiting + // for an unrelated new vote, while the handler bounds advertisements. + multiProxyHandler.renewMultiProxyVoteCapabilityIfDue(); + } + pending.removeIf(recipient -> recipient != null + && awaitingRenewal.contains(recipient.toLowerCase(Locale.ROOT))); + if (pending.isEmpty()) return false; return multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(outbox.getName(), outbox.getUuid(), outbox.getService(), outbox.getTime(), false, outbox.isRealVote(), outbox.getTotals(), outbox.getVoteId(), false, false, 1, 1, outbox.getMultiProxyOrigin()), diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java index ddddb2e4f..332afca51 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java @@ -3,6 +3,7 @@ import java.io.File; import java.util.Collection; +import java.util.HashSet; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -32,7 +33,17 @@ public abstract class MultiProxyHandler { private SocketHandler multiproxySocketHandler; /** A renewable lease prevents a restarted/rolled-back peer staying ACK-capable forever. */ static final long VOTE_CAPABILITY_LEASE_MILLIS = 5 * 60 * 1000L; + /** Bound retry-driven capability advertisements while a durable outbox is waiting. */ + static final long VOTE_CAPABILITY_RENEWAL_MIN_INTERVAL_MILLIS = 30 * 1000L; private final Map acknowledgedVoteCapabilityPeers = new HashMap<>(); + /** + * Peers which have previously completed the durable-delivery handshake. Retain + * this separately from the renewable lease: once a known-capable peer drops + * offline, sending it a legacy vote would acknowledge a fire-and-forget publish + * as success and lose the sender's durable completion fence. + */ + private final Set knownVoteCapabilityPeers = new HashSet<>(); + private long lastVoteCapabilityAdvertisementMillis = Long.MIN_VALUE; @Getter private RedisHandler multiProxyRedis; @@ -77,6 +88,8 @@ public synchronized void close() { stopSocketClients(multiproxyClientHandles); multiproxyClientHandles = null; acknowledgedVoteCapabilityPeers.clear(); + knownVoteCapabilityPeers.clear(); + lastVoteCapabilityAdvertisementMillis = Long.MIN_VALUE; } /** @@ -346,9 +359,48 @@ public synchronized Set getMultiProxyVoteRecipients() { return recipients; } + /** + * Returns configured peers which previously advertised durable acknowledgements + * but whose renewable lease has expired. A sender must retain these recipients + * in its durable outbox and wait for the capability handshake to renew instead + * of demoting them to the legacy one-way route. + */ + public synchronized Set getMultiProxyVoteRecipientsAwaitingCapabilityRenewal() { + long now = capabilityNowMillis(); + acknowledgedVoteCapabilityPeers.entrySet().removeIf(entry -> entry.getValue() <= now); + Set recipients = new LinkedHashSet<>(); + for (Map.Entry configured : configuredMultiProxyRecipientNames().entrySet()) { + if (knownVoteCapabilityPeers.contains(configured.getKey()) + && !acknowledgedVoteCapabilityPeers.containsKey(configured.getKey())) { + recipients.add(configured.getValue()); + } + } + return recipients; + } + /** Broadcasts this node's durable-ACK capability to configured peers. */ public synchronized void announceMultiProxyVoteCapability() { + lastVoteCapabilityAdvertisementMillis = capabilityNowMillis(); + sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyCapabilities(getMultiProxyServerName(), 1)); + } + + /** + * Re-advertises durable delivery support for an outbox that is waiting for an + * expired peer lease. This is deliberately rate limited: the queue retries + * much more frequently than the handshake needs to be broadcast. + * + * @return whether an advertisement was attempted + */ + public synchronized boolean renewMultiProxyVoteCapabilityIfDue() { + long now = capabilityNowMillis(); + if (lastVoteCapabilityAdvertisementMillis != Long.MIN_VALUE + && now >= lastVoteCapabilityAdvertisementMillis + && now - lastVoteCapabilityAdvertisementMillis < VOTE_CAPABILITY_RENEWAL_MIN_INTERVAL_MILLIS) { + return false; + } + lastVoteCapabilityAdvertisementMillis = now; sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyCapabilities(getMultiProxyServerName(), 1)); + return true; } /** @@ -356,6 +408,8 @@ public synchronized void announceMultiProxyVoteCapability() { */ public synchronized void loadMultiProxySupport() { acknowledgedVoteCapabilityPeers.clear(); + knownVoteCapabilityPeers.clear(); + lastVoteCapabilityAdvertisementMillis = Long.MIN_VALUE; if (!getMultiProxySupportEnabled()) { return; } @@ -640,6 +694,7 @@ private void handleEnvelope(JsonEnvelope envelope) { String normalized = recipient.toLowerCase(Locale.ROOT); synchronized (this) { if (version >= 1 && configuredMultiProxyRecipientNames().containsKey(normalized)) { + knownVoteCapabilityPeers.add(normalized); acknowledgedVoteCapabilityPeers.put(normalized, capabilityNowMillis() + VOTE_CAPABILITY_LEASE_MILLIS); replyRequired = !reply; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java index 144808061..8fbd1b5ef 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java @@ -244,4 +244,39 @@ void durableAcknowledgementCapabilityExpiresAndCanBeRenewed() throws Exception { handleEnvelope.invoke(handler, VotingPluginWire.multiProxyCapabilities("Capable", 1, true)); assertEquals(java.util.Set.of("Capable"), handler.getMultiProxyVoteRecipients()); } + + @Test + void expiredKnownCapabilityWaitsForRenewalInsteadOfBecomingLegacy() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, + org.mockito.Mockito.withSettings().useConstructor().defaultAnswer(org.mockito.Mockito.CALLS_REAL_METHODS)); + org.mockito.Mockito.when(handler.getMultiProxyMethod()).thenReturn(MultiProxyMethod.SOCKETS); + org.mockito.Mockito.when(handler.getMultiProxyServers()).thenReturn(List.of("Capable")); + org.mockito.Mockito.when(handler.capabilityNowMillis()).thenReturn(1_000L, + 1_000L + MultiProxyHandler.VOTE_CAPABILITY_LEASE_MILLIS, + 1_000L + MultiProxyHandler.VOTE_CAPABILITY_LEASE_MILLIS); + Method handleEnvelope = MultiProxyHandler.class.getDeclaredMethod("handleEnvelope", JsonEnvelope.class); + handleEnvelope.setAccessible(true); + + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyCapabilities("Capable", 1, true)); + assertTrue(handler.getMultiProxyVoteRecipients().isEmpty()); + assertEquals(java.util.Set.of("Capable"), handler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal()); + } + + @Test + void renewalAdvertisementIsRateLimitedAfterAnInitialHandshake() { + MultiProxyHandler handler = mock(MultiProxyHandler.class, + org.mockito.Mockito.withSettings().useConstructor().defaultAnswer(org.mockito.Mockito.CALLS_REAL_METHODS)); + org.mockito.Mockito.when(handler.getMultiProxyServerName()).thenReturn("Primary"); + org.mockito.Mockito.when(handler.capabilityNowMillis()).thenReturn(1_000L, + 1_000L + MultiProxyHandler.VOTE_CAPABILITY_RENEWAL_MIN_INTERVAL_MILLIS - 1, + 1_000L + MultiProxyHandler.VOTE_CAPABILITY_RENEWAL_MIN_INTERVAL_MILLIS); + org.mockito.Mockito.when(handler.sendMultiProxyEnvelopeAccepted(org.mockito.ArgumentMatchers.any())).thenReturn(true); + org.mockito.Mockito.clearInvocations(handler); + + handler.announceMultiProxyVoteCapability(); + assertFalse(handler.renewMultiProxyVoteCapabilityIfDue()); + assertTrue(handler.renewMultiProxyVoteCapabilityIfDue()); + org.mockito.Mockito.verify(handler, org.mockito.Mockito.times(2)) + .sendMultiProxyEnvelopeAccepted(org.mockito.ArgumentMatchers.any()); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index d11910627..b846efb15 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -529,6 +529,79 @@ void rejectedLegacyOnlyMultiProxySendKeepsLiveVoteForRetry() throws Exception { .getOrDefault(VotingPluginWire.K_MULTI_PROXY_ORIGIN, "")); } + @Test + void expiredCapableMultiProxyPeerStaysInDurableOutboxUntilRenewal() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.addTimeVoteToCache(Mockito.any())).thenAnswer(invocation -> { + queue.add(invocation.getArgument(0)); + return true; + }); + Mockito.when(voteCache.updateTimeVote(Mockito.any())).thenReturn(true); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Collections.emptySet()); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal()) + .thenReturn(java.util.Set.of("Capable")); + Mockito.when(multiProxyHandler.getConfiguredMultiProxyVoteRecipients()) + .thenReturn(new java.util.LinkedHashSet<>(java.util.Set.of("Capable"))); + Mockito.when(votingPluginProxy.getConfig().getProxyServerName()).thenReturn("Proxy1"); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + Class retryType = Class.forName("com.bencodez.votingplugin.proxy.VotingPluginProxy$LiveVoteRetryState"); + java.lang.reflect.Constructor constructor = retryType.getDeclaredConstructor(); + constructor.setAccessible(true); + Object retry = constructor.newInstance(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + java.lang.reflect.Field retriesField = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); + retriesField.setAccessible(true); + @SuppressWarnings("unchecked") + java.util.Map retries = + (java.util.Map) retriesField.get(spyProxy); + retries.put(voteId, retry); + java.lang.reflect.Method begin = VotingPluginProxy.class.getDeclaredMethod("beginMultiProxyForwarding", + retryType, VoteTimeQueue.class, String.class, String.class, String.class, long.class, + boolean.class, com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class); + begin.setAccessible(true); + + assertFalse((Boolean) begin.invoke(spyProxy, retry, null, "Player", + "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); + assertEquals(1, queue.size()); + assertEquals(java.util.Set.of("capable"), queue.element().getMultiProxyRecipients()); + assertTrue(queue.element().getMultiProxyLegacyPendingRecipients().isEmpty()); + verify(multiProxyHandler, Mockito.never()).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any()); + } + + @Test + void persistedOutboxWaitsForFreshCapabilityLeaseAfterRestart() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + VoteTimeQueue outbox = new VoteTimeQueue(java.util.UUID.randomUUID(), "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", true, + "00000000-0000-0000-0000-000000000001"); + outbox.requireMultiProxyAcknowledgements("Proxy1", java.util.Set.of("Capable")); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + queue.add(outbox); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + // A restart loses in-memory capability knowledge, but the durable outbox + // still records that Capable requires acknowledgements. Do not publish the + // reliable envelope until a new capability exchange restores its lease. + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()) + .thenReturn(java.util.Collections.emptySet(), java.util.Set.of("Capable")); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + spyProxy.processQueue(); + + verify(multiProxyHandler).renewMultiProxyVoteCapabilityIfDue(); + verify(multiProxyHandler, Mockito.never()).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any()); + assertTrue(queue.contains(outbox)); + + spyProxy.processQueue(); + + verify(multiProxyHandler).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.eq(java.util.Set.of("capable"))); + assertTrue(queue.contains(outbox)); + } + @Test void completedAckOutboxKeepsFenceForAlreadyScheduledListenerRetry() throws Exception { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); @@ -576,6 +649,7 @@ void pendingAckOutboxDoesNotBlockLaterCompletedQueueRows() { queue.add(completed); Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); Mockito.when(voteCache.removeTimeVote(completed)).thenAnswer(invocation -> queue.remove(completed)); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Set.of("Proxy2")); VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); From 2e49dd66699aefe24a572416053c9954e6e864e4 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 13 Sep 2026 20:03:24 -0600 Subject: [PATCH 108/136] Persist multi-proxy capability peers --- .../votingplugin/proxy/VotingPluginProxy.java | 4 + .../multiproxy/MultiProxyCapabilityStore.java | 124 ++++++++++++++++++ .../proxy/multiproxy/MultiProxyHandler.java | 89 ++++++++++++- .../MultiProxyHandlerLifecycleTest.java | 97 ++++++++++++++ .../tests/VotingPluginProxyTest.java | 23 ++++ 5 files changed, 334 insertions(+), 3 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyCapabilityStore.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index d8cac233c..e51febc33 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -5621,6 +5621,10 @@ private UUID legacyTimedVoteId(VoteTimeQueue vote) { private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTimeQueue queuedVote, String player, String uuid, String service, long time, boolean realVote, VoteTotalsSnapshot totals) { if (multiProxyHandler == null) return false; + // A corrupt or unpersistable peer-classification file cannot safely identify + // which targets are legacy. Stop before creating an ACK outbox that a legacy + // peer could never complete; the handler emits the operator recovery message. + if (multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked()) return false; multiProxyHandler.announceMultiProxyVoteCapability(); Set recipients = multiProxyHandler.getMultiProxyVoteRecipients(); Set renewingRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyCapabilityStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyCapabilityStore.java new file mode 100644 index 000000000..385119984 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyCapabilityStore.java @@ -0,0 +1,124 @@ +package com.bencodez.votingplugin.proxy.multiproxy; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Set; + +import com.bencodez.votingplugin.util.DurableFiles; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +/** + * Stores only the stable identities of peers that have completed the + * durable-delivery capability handshake. A lease is deliberately not stored: + * every process must renew a peer's current handshake before publishing a + * durable outbox retry. + */ +final class MultiProxyCapabilityStore { + static final String FILE_NAME = ".multiproxy-capability-peers.json"; + private static final int VERSION = 1; + private static final int MAX_BYTES = 64 * 1024; + private static final int MAX_PEERS = 256; + private static final int MAX_PEER_LENGTH = 256; + + private MultiProxyCapabilityStore() { } + + static State load(Path dataDirectory) throws IOException { + Path target = target(dataDirectory); + if (!Files.exists(target, LinkOption.NOFOLLOW_LINKS)) return State.empty(); + if (Files.isSymbolicLink(target) || !Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) + || Files.size(target) > MAX_BYTES) throw new IOException("Multi-proxy capability state is unsafe"); + byte[] bytes; + try (SeekableByteChannel channel = Files.newByteChannel(target, + Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS))) { + ByteBuffer buffer = ByteBuffer.allocate((int) Files.size(target) + 1); + while (channel.read(buffer) >= 0 && buffer.hasRemaining()) { } + if (!buffer.hasRemaining()) throw new IOException("Multi-proxy capability state is too large"); + buffer.flip(); + bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + } + try { + JsonElement parsed = JsonParser.parseString(new String(bytes, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw invalid(); + JsonObject root = parsed.getAsJsonObject(); + if (!root.has("version") || root.get("version").getAsInt() != VERSION + || !root.has("peers") || !root.get("peers").isJsonArray()) throw invalid(); + JsonArray peers = root.getAsJsonArray("peers"); + if (peers.size() > MAX_PEERS) throw invalid(); + Set restored = new LinkedHashSet<>(); + for (JsonElement peer : peers) { + if (!peer.isJsonPrimitive() || !peer.getAsJsonPrimitive().isString()) throw invalid(); + String name = peer.getAsString(); + if (!validPeer(name) || !restored.add(name)) throw invalid(); + } + return new State(restored); + } catch (RuntimeException malformed) { + throw new IOException("Multi-proxy capability state is malformed", malformed); + } + } + + static void save(Path dataDirectory, Collection peers) throws IOException { + if (dataDirectory == null || peers == null || peers.size() > MAX_PEERS) throw new IOException("Invalid capability state"); + Set copy = new LinkedHashSet<>(); + for (String peer : peers) { + if (!validPeer(peer) || !copy.add(peer)) throw new IOException("Invalid capability peer"); + } + JsonObject root = new JsonObject(); + root.addProperty("version", VERSION); + JsonArray listed = new JsonArray(); + copy.stream().sorted().forEach(listed::add); + root.add("peers", listed); + byte[] serialized = root.toString().getBytes(StandardCharsets.UTF_8); + if (serialized.length > MAX_BYTES) throw new IOException("Multi-proxy capability state is too large"); + + Files.createDirectories(dataDirectory); + Path target = target(dataDirectory); + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS) + && (Files.isSymbolicLink(target) || !Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS))) { + throw new IOException("Multi-proxy capability state is unsafe"); + } + Path staging = Files.createTempFile(dataDirectory, ".multiproxy-capabilities-", ".json"); + try { + Files.write(staging, serialized, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + DurableFiles.publishStagedFile(staging, target); + } finally { + Files.deleteIfExists(staging); + } + } + + private static Path target(Path dataDirectory) { + if (dataDirectory == null) throw new IllegalArgumentException("dataDirectory"); + return dataDirectory.resolve(FILE_NAME); + } + + private static boolean validPeer(String peer) { + return peer != null && !peer.isBlank() && peer.length() <= MAX_PEER_LENGTH + && peer.equals(peer.toLowerCase(Locale.ROOT)); + } + + private static IllegalArgumentException invalid() { + return new IllegalArgumentException("Invalid capability state"); + } + + record State(Set peers) { + State { + peers = Set.copyOf(peers); + } + + static State empty() { + return new State(Set.of()); + } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java index 332afca51..f42a05f2e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java @@ -2,6 +2,8 @@ package com.bencodez.votingplugin.proxy.multiproxy; import java.io.File; +import java.io.IOException; +import java.nio.file.Path; import java.util.Collection; import java.util.HashSet; import java.util.HashMap; @@ -43,6 +45,14 @@ public abstract class MultiProxyHandler { * as success and lose the sender's durable completion fence. */ private final Set knownVoteCapabilityPeers = new HashSet<>(); + /** + * We cannot safely classify configured peers after the durable state is corrupt + * or cannot be updated. Do not create a new mixed-version outbox in that + * state: an old peer can never acknowledge it, while demoting a formerly + * capable peer would lose its sender fence. Recovery is deliberately an + * operator action (repair/remove the named state file, then restart). + */ + private boolean voteCapabilityRecoveryBlocked; private long lastVoteCapabilityAdvertisementMillis = Long.MIN_VALUE; @Getter @@ -89,6 +99,7 @@ public synchronized void close() { multiproxyClientHandles = null; acknowledgedVoteCapabilityPeers.clear(); knownVoteCapabilityPeers.clear(); + voteCapabilityRecoveryBlocked = false; lastVoteCapabilityAdvertisementMillis = Long.MIN_VALUE; } @@ -403,13 +414,84 @@ public synchronized boolean renewMultiProxyVoteCapabilityIfDue() { return true; } + /** + * Whether durable peer classification needs an explicit on-disk recovery. + * Callers must not start a new multi-proxy forwarding attempt while this is + * true, because a legacy peer cannot complete an acknowledgement outbox. + */ + public synchronized boolean isMultiProxyVoteCapabilityRecoveryBlocked() { + return voteCapabilityRecoveryBlocked; + } + + /** Restores durable peer identities without restoring any expired lease. */ + synchronized void restoreVoteCapabilityPeers() { + acknowledgedVoteCapabilityPeers.clear(); + knownVoteCapabilityPeers.clear(); + voteCapabilityRecoveryBlocked = false; + Path dataDirectory = capabilityStateDirectory(); + // Third-party MultiProxyHandler integrations predate durable state and may + // intentionally provide no plugin folder. Preserve their established + // in-memory handshake behavior; built-in proxy handlers always provide one. + if (dataDirectory == null) return; + try { + MultiProxyCapabilityStore.State restored = MultiProxyCapabilityStore.load(dataDirectory); + Set configured = configuredMultiProxyRecipientNames().keySet(); + for (String peer : restored.peers()) { + if (configured.contains(peer)) knownVoteCapabilityPeers.add(peer); + } + } catch (IOException | IllegalArgumentException stateFailure) { + blockVoteCapabilityRecovery(); + } + } + + /** + * Makes a peer eligible for durable publishes only after its durable identity is + * recorded. A failed write never silently converts an acknowledged-capable + * peer to the legacy route during this process. + */ + private boolean acceptVoteCapabilityPeer(String normalized) { + if (voteCapabilityRecoveryBlocked) return false; + boolean newPeer = !knownVoteCapabilityPeers.contains(normalized); + if (!newPeer) return true; + Set updated = new HashSet<>(knownVoteCapabilityPeers); + updated.add(normalized); + Path dataDirectory = capabilityStateDirectory(); + if (dataDirectory == null) { + knownVoteCapabilityPeers.clear(); + knownVoteCapabilityPeers.addAll(updated); + return true; + } + try { + MultiProxyCapabilityStore.save(dataDirectory, updated); + knownVoteCapabilityPeers.clear(); + knownVoteCapabilityPeers.addAll(updated); + return true; + } catch (IOException | IllegalArgumentException stateFailure) { + blockVoteCapabilityRecovery(); + return false; + } + } + + private void blockVoteCapabilityRecovery() { + voteCapabilityRecoveryBlocked = true; + logInfo("Multi-proxy forwarding is blocked: repair or remove " + MultiProxyCapabilityStore.FILE_NAME + + " and restart before forwarding votes"); + } + + private Path capabilityStateDirectory() { + File dataFolder = getPluginDataFolder(); + return dataFolder == null ? null : dataFolder.toPath(); + } + /** * Loads multi-proxy support. */ public synchronized void loadMultiProxySupport() { acknowledgedVoteCapabilityPeers.clear(); knownVoteCapabilityPeers.clear(); + voteCapabilityRecoveryBlocked = false; lastVoteCapabilityAdvertisementMillis = Long.MIN_VALUE; + restoreVoteCapabilityPeers(); if (!getMultiProxySupportEnabled()) { return; } @@ -694,9 +776,10 @@ private void handleEnvelope(JsonEnvelope envelope) { String normalized = recipient.toLowerCase(Locale.ROOT); synchronized (this) { if (version >= 1 && configuredMultiProxyRecipientNames().containsKey(normalized)) { - knownVoteCapabilityPeers.add(normalized); - acknowledgedVoteCapabilityPeers.put(normalized, - capabilityNowMillis() + VOTE_CAPABILITY_LEASE_MILLIS); + if (acceptVoteCapabilityPeer(normalized)) { + acknowledgedVoteCapabilityPeers.put(normalized, + capabilityNowMillis() + VOTE_CAPABILITY_LEASE_MILLIS); + } replyRequired = !reply; } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java index 8fbd1b5ef..4187aae4a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java @@ -6,14 +6,18 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.UUID; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; @@ -262,6 +266,99 @@ void expiredKnownCapabilityWaitsForRenewalInsteadOfBecomingLegacy() throws Excep assertEquals(java.util.Set.of("Capable"), handler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal()); } + @Test + void durableCapabilityIdentitySurvivesRestartAndFencesAnOfflinePeer(@TempDir Path dataDirectory) throws Exception { + MultiProxyHandler beforeRestart = capabilityHandler(dataDirectory, "Capable", "Legacy"); + handleCapability(beforeRestart, "Capable"); + assertEquals(java.util.Set.of("Capable"), beforeRestart.getMultiProxyVoteRecipients()); + + MultiProxyHandler afterRestart = capabilityHandler(dataDirectory, "Capable", "Legacy"); + afterRestart.restoreVoteCapabilityPeers(); + + assertTrue(afterRestart.getMultiProxyVoteRecipients().isEmpty()); + assertEquals(java.util.Set.of("Capable"), + afterRestart.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal()); + assertFalse(afterRestart.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal().contains("Legacy")); + } + + @Test + void malformedCapabilityStateBlocksForwardingUntilTheOperatorRepairsItAndRestarts(@TempDir Path dataDirectory) + throws Exception { + Files.writeString(dataDirectory.resolve(".multiproxy-capability-peers.json"), "not-json"); + MultiProxyHandler handler = capabilityHandler(dataDirectory, "Capable", "Legacy"); + handler.restoreVoteCapabilityPeers(); + + assertTrue(handler.isMultiProxyVoteCapabilityRecoveryBlocked()); + assertTrue(handler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal().isEmpty()); + handleCapability(handler, "Capable"); + assertTrue(handler.getMultiProxyVoteRecipients().isEmpty()); + assertTrue(handler.isMultiProxyVoteCapabilityRecoveryBlocked()); + + // Removing the invalid local state and restarting deterministically returns + // to fresh-install semantics: Legacy remains legacy and a new handshake + // classifies only Capable as an ACK peer. + Files.delete(dataDirectory.resolve(".multiproxy-capability-peers.json")); + MultiProxyHandler recovered = capabilityHandler(dataDirectory, "Capable", "Legacy"); + recovered.restoreVoteCapabilityPeers(); + assertFalse(recovered.isMultiProxyVoteCapabilityRecoveryBlocked()); + assertTrue(recovered.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal().isEmpty()); + handleCapability(recovered, "Capable"); + assertEquals(java.util.Set.of("Capable"), recovered.getMultiProxyVoteRecipients()); + assertFalse(recovered.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal().contains("Legacy")); + } + + @Test + void nonCanonicalCapabilityPeerBlocksForwardingInsteadOfBecomingLegacy(@TempDir Path dataDirectory) + throws Exception { + Files.writeString(dataDirectory.resolve(".multiproxy-capability-peers.json"), + "{\"version\":1,\"peers\":[\"Capable\"]}"); + + MultiProxyHandler handler = capabilityHandler(dataDirectory, "Capable", "Legacy"); + handler.restoreVoteCapabilityPeers(); + + assertTrue(handler.isMultiProxyVoteCapabilityRecoveryBlocked()); + assertTrue(handler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal().isEmpty()); + } + + @Test + void freshInstallKeepsLegacyPeersOnTheHistoricalRoute(@TempDir Path dataDirectory) { + MultiProxyHandler handler = capabilityHandler(dataDirectory, "Legacy"); + handler.restoreVoteCapabilityPeers(); + assertTrue(handler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal().isEmpty()); + } + + @Test + void capabilityStoreRejectsSymlinkedState(@TempDir Path dataDirectory) throws Exception { + Path outside = Files.createTempFile("multiproxy-capability-outside", ".json"); + try { + Files.createSymbolicLink(dataDirectory.resolve(".multiproxy-capability-peers.json"), outside); + assertThrows(java.io.IOException.class, () -> MultiProxyCapabilityStore.load(dataDirectory)); + MultiProxyHandler handler = capabilityHandler(dataDirectory, "Capable"); + handler.restoreVoteCapabilityPeers(); + handleCapability(handler, "Capable"); + assertTrue(handler.getMultiProxyVoteRecipients().isEmpty()); + assertTrue(handler.isMultiProxyVoteCapabilityRecoveryBlocked()); + assertTrue(handler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal().isEmpty()); + } finally { + Files.deleteIfExists(outside); + } + } + + private static MultiProxyHandler capabilityHandler(Path dataDirectory, String... peers) { + MultiProxyHandler handler = mock(MultiProxyHandler.class, + org.mockito.Mockito.withSettings().useConstructor().defaultAnswer(org.mockito.Mockito.CALLS_REAL_METHODS)); + org.mockito.Mockito.when(handler.getMultiProxyMethod()).thenReturn(MultiProxyMethod.SOCKETS); + org.mockito.Mockito.when(handler.getMultiProxyServers()).thenReturn(List.of(peers)); + org.mockito.Mockito.when(handler.getPluginDataFolder()).thenReturn(dataDirectory.toFile()); + return handler; + } + + private static void handleCapability(MultiProxyHandler handler, String peer) throws Exception { + Method handleEnvelope = MultiProxyHandler.class.getDeclaredMethod("handleEnvelope", JsonEnvelope.class); + handleEnvelope.setAccessible(true); + handleEnvelope.invoke(handler, VotingPluginWire.multiProxyCapabilities(peer, 1, true)); + } + @Test void renewalAdvertisementIsRateLimitedAfterAnInitialHandshake() { MultiProxyHandler handler = mock(MultiProxyHandler.class, diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index b846efb15..289f4d700 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -529,6 +529,29 @@ void rejectedLegacyOnlyMultiProxySendKeepsLiveVoteForRetry() throws Exception { .getOrDefault(VotingPluginWire.K_MULTI_PROXY_ORIGIN, "")); } + @Test + void capabilityRecoveryBlockPreventsCreatingAMixedVersionAcknowledgementOutbox() throws Exception { + Mockito.when(multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked()).thenReturn(true); + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + + Class retryType = Class.forName("com.bencodez.votingplugin.proxy.VotingPluginProxy$LiveVoteRetryState"); + java.lang.reflect.Constructor constructor = retryType.getDeclaredConstructor(); + constructor.setAccessible(true); + Object retry = constructor.newInstance(); + java.lang.reflect.Method begin = VotingPluginProxy.class.getDeclaredMethod("beginMultiProxyForwarding", + retryType, VoteTimeQueue.class, String.class, String.class, String.class, long.class, + boolean.class, com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class); + begin.setAccessible(true); + + assertFalse((Boolean) begin.invoke(spyProxy, retry, null, "Player", + "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); + verify(multiProxyHandler, never()).announceMultiProxyVoteCapability(); + verify(multiProxyHandler, never()).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any()); + verify(voteCache, never()).addTimeVoteToCache(Mockito.any()); + } + @Test void expiredCapableMultiProxyPeerStaysInDurableOutboxUntilRenewal() throws Exception { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); From 0f0ed271185b980bcbe2eebce4b6217012645f91 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:17:50 -0600 Subject: [PATCH 109/136] Harden mixed-version proxy capability discovery --- .../votingplugin/proxy/VotingPluginProxy.java | 130 ++++++++++++++++-- .../proxy/bungee/BungeeJsonVoteCache.java | 2 + .../proxy/cache/ProxyTimedVoteCacheTable.java | 69 ++++++---- .../proxy/cache/VoteCacheHandler.java | 4 + .../multiproxy/MultiProxyCapabilityStore.java | 61 +++++++- .../proxy/multiproxy/MultiProxyHandler.java | 111 +++++++++++++-- .../proxy/velocity/VelocityJsonVoteCache.java | 2 + .../votingplugin/timequeue/VoteTimeQueue.java | 9 ++ .../MultiProxyHandlerLifecycleTest.java | 82 +++++++++++ .../tests/VotingPluginProxyTest.java | 128 ++++++++++++++++- 10 files changed, 541 insertions(+), 57 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index e51febc33..2f386e69e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -5625,27 +5625,47 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim // which targets are legacy. Stop before creating an ACK outbox that a legacy // peer could never complete; the handler emits the operator recovery message. if (multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked()) return false; - multiProxyHandler.announceMultiProxyVoteCapability(); + // Capability retries can be much more frequent than a normal vote. Start + // discovery promptly, but rate-limit repeated advertisements while the vote + // waits for a newly configured peer to identify itself. + multiProxyHandler.renewMultiProxyVoteCapabilityIfDue(); Set recipients = multiProxyHandler.getMultiProxyVoteRecipients(); Set renewingRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal(); + Set discoveringRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery(); + // Recording a first-observation deadline can itself fail and transition the + // handler into recovery-blocked state. Recheck before interpreting an empty + // discovery set as permission to use the lossy legacy route. + if (multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked()) return false; Set configuredRecipients = multiProxyHandler.getConfiguredMultiProxyVoteRecipients(); // Keep custom MultiProxyHandler integrations that override only the // established recipient method source-compatible while the base handler // learns capabilities. if (configuredRecipients.isEmpty() && !recipients.isEmpty()) configuredRecipients.addAll(recipients); if (configuredRecipients.isEmpty()) return true; + if (!discoveringRecipients.isEmpty()) { + // Do not publish any part of this vote while a newly configured peer can + // still complete the capability handshake. Capturing only the known peers + // now would make the later legacy fallback hard to fence without a second, + // duplicate-prone outbox. The durable peer deadline bounds this retry even + // across a proxy restart. + admitMultiProxyDiscoveryOutbox(retryState, queuedVote, player, uuid, service, time, realVote, totals); + return false; + } Set durableRecipients = new LinkedHashSet<>(recipients); durableRecipients.addAll(renewingRecipients); Set legacyRecipients = new LinkedHashSet<>(configuredRecipients); legacyRecipients.removeAll(durableRecipients); - if (durableRecipients.isEmpty()) { + VoteTimeQueue outbox = queuedVote; + if (outbox != null && outbox.isMultiProxyCapabilityDiscoveryPending() + && !resolveMultiProxyDiscoveryOutbox(outbox)) return false; + if (durableRecipients.isEmpty() + && (outbox == null || !outbox.isMultiProxyForwardingRequired())) { // Older peers do not understand acknowledgements. Preserve their historical // fire-and-forget route instead of creating an outbox they can never ACK. return multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.vote(player, uuid, service, time, false, realVote, totals == null ? "" : totals.toString(), findLiveVoteId(retryState), false, false, 1, 1), legacyRecipients); } - VoteTimeQueue outbox = queuedVote; if (outbox == null) { outbox = new VoteTimeQueue(null, player, service, time, false, Collections.emptySet(), Collections.emptySet(), totals == null ? "" : totals.toString(), true, uuid); @@ -5673,7 +5693,11 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim return false; } } - if (!outbox.isMultiProxyForwardingRequired()) { + if (outbox.isMultiProxyCapabilityDiscoveryPending()) { + // A crash can leave this row before classification. Resolve and persist the + // recipient split before publishing either route. + if (!resolveMultiProxyDiscoveryOutbox(outbox)) return false; + } else if (!outbox.isMultiProxyForwardingRequired()) { outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), durableRecipients); outbox.setMultiProxyLegacyPendingRecipients(legacyRecipients); outbox.setRealVote(realVote); @@ -5689,19 +5713,89 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim if (!outbox.getMultiProxyLegacyPendingRecipients().isEmpty()) { // Never retry this legacy copy as part of the ACK outbox: a legacy peer has // no receiver dedupe/ACK contract, while capable peers stay fully durable. - // Persist the exact pending set until the transport accepts the copy. - if (!multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.vote(player, uuid, service, time, - false, realVote, totals == null ? "" : totals.toString(), outbox.getVoteId(), false, false, 1, 1), - new LinkedHashSet<>(outbox.getMultiProxyLegacyPendingRecipients()))) return false; + // Clear the replayable set durably before publishing. If the transport + // rejects synchronously, restore it for retry; a process crash retains the + // historical legacy at-most-once behavior instead of duplicating rewards. + Set legacyPending = new LinkedHashSet<>(outbox.getMultiProxyLegacyPendingRecipients()); outbox.setMultiProxyLegacyPendingRecipients(Collections.emptySet()); outbox.setDeliveryStateDirty(true); if (!persistTimeVoteDelivery(outbox)) return false; + if (!multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.vote(player, uuid, service, time, + false, realVote, totals == null ? "" : totals.toString(), outbox.getVoteId(), false, false, 1, 1), + legacyPending)) { + outbox.setMultiProxyLegacyPendingRecipients(legacyPending); + outbox.setDeliveryStateDirty(true); + persistTimeVoteDelivery(outbox); + return false; + } + } + if (outbox.getMultiProxyRecipients().isEmpty()) { + // Discovery resolved every peer to the legacy route. There can be no ACK + // retirement phase, so retire the durable intent after the one-way copy was + // durably marked accepted. + boolean handled = markMultiProxyForwardingHandled(retryState, outbox); + if (handled) scheduleTimeVoteRetry(); + return handled; } if (!sendDurableMultiProxyOutbox(outbox)) return false; scheduleTimeVoteRetry(); return false; } + /** + * Persists a forwarding intent before the first discovery-induced retry. The + * local vote effects have already completed at this point; without this row a + * shutdown between the retry result and the next listener attempt loses the + * only record that the vote still needs forwarding. + */ + private VoteTimeQueue admitMultiProxyDiscoveryOutbox(LiveVoteRetryState retryState, VoteTimeQueue queuedVote, + String player, String uuid, String service, long time, boolean realVote, VoteTotalsSnapshot totals) { + VoteTimeQueue outbox = queuedVote; + if (outbox == null) { + outbox = new VoteTimeQueue(null, player, service, time, false, Collections.emptySet(), + Collections.emptySet(), totals == null ? "" : totals.toString(), true, uuid); + outbox.setVoteId(findLiveVoteId(retryState)); + if (outbox.getVoteId() == null) return null; + retryState.queuedVote = outbox; + } + outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), Collections.emptySet()); + outbox.setMultiProxyCapabilityDiscoveryPending(true); + outbox.setMultiProxyLegacyPendingRecipients(Collections.emptySet()); + outbox.setRealVote(realVote); + outbox.setProcessed(true); + outbox.setDeliveryStateDirty(true); + boolean alreadyQueued = false; + for (VoteTimeQueue candidate : getVoteCacheHandler().getTimeChangeQueue()) { + if (candidate != null && java.util.Objects.equals(outbox.getVoteId(), candidate.getVoteId())) { + alreadyQueued = true; + break; + } + } + if (!alreadyQueued) return getVoteCacheHandler().addTimeVoteToCache(outbox) ? outbox : null; + return persistTimeVoteDelivery(outbox) ? outbox : null; + } + + /** Resolves a restored discovery-pending outbox and persists its recipient split before sending. */ + private boolean resolveMultiProxyDiscoveryOutbox(VoteTimeQueue outbox) { + if (multiProxyHandler == null || multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked()) return false; + multiProxyHandler.renewMultiProxyVoteCapabilityIfDue(); + Set recipients = multiProxyHandler.getMultiProxyVoteRecipients(); + Set renewingRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal(); + Set discoveringRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery(); + if (multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked() || !discoveringRecipients.isEmpty()) return false; + Set configuredRecipients = multiProxyHandler.getConfiguredMultiProxyVoteRecipients(); + if (configuredRecipients.isEmpty() && !recipients.isEmpty()) configuredRecipients.addAll(recipients); + Set durableRecipients = new LinkedHashSet<>(recipients); + durableRecipients.addAll(renewingRecipients); + Set legacyRecipients = new LinkedHashSet<>(configuredRecipients); + legacyRecipients.removeAll(durableRecipients); + outbox.requireMultiProxyAcknowledgements(getConfig().getProxyServerName(), durableRecipients); + outbox.setMultiProxyLegacyPendingRecipients(legacyRecipients); + outbox.setMultiProxyCapabilityDiscoveryPending(false); + outbox.setDeliveryStateDirty(true); + return persistTimeVoteDelivery(outbox); + } + private UUID findLiveVoteId(LiveVoteRetryState retryState) { for (Map.Entry entry : liveVoteRetries.entrySet()) { if (entry.getValue() == retryState) return entry.getKey(); @@ -5711,17 +5805,28 @@ private UUID findLiveVoteId(LiveVoteRetryState retryState) { /** Re-sends an unacknowledged outbox using the exact persisted stable ID. */ private boolean retryDurableMultiProxyOutbox(VoteTimeQueue outbox) { + if (outbox.isMultiProxyCapabilityDiscoveryPending() && !resolveMultiProxyDiscoveryOutbox(outbox)) return false; if (outbox.hasCompletedMultiProxyAcknowledgements()) { return finishMultiProxyRetirement(null, outbox); } if (!outbox.getMultiProxyLegacyPendingRecipients().isEmpty()) { - if (!multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.vote(outbox.getName(), - outbox.getUuid(), outbox.getService(), outbox.getTime(), false, outbox.isRealVote(), - outbox.getTotals(), outbox.getVoteId(), false, false, 1, 1), - new LinkedHashSet<>(outbox.getMultiProxyLegacyPendingRecipients()))) return false; + Set legacyPending = new LinkedHashSet<>(outbox.getMultiProxyLegacyPendingRecipients()); outbox.setMultiProxyLegacyPendingRecipients(Collections.emptySet()); outbox.setDeliveryStateDirty(true); if (!persistTimeVoteDelivery(outbox)) return false; + if (!multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.vote(outbox.getName(), + outbox.getUuid(), outbox.getService(), outbox.getTime(), false, outbox.isRealVote(), + outbox.getTotals(), outbox.getVoteId(), false, false, 1, 1), legacyPending)) { + outbox.setMultiProxyLegacyPendingRecipients(legacyPending); + outbox.setDeliveryStateDirty(true); + persistTimeVoteDelivery(outbox); + return false; + } + } + if (outbox.getMultiProxyRecipients().isEmpty()) { + boolean handled = markMultiProxyForwardingHandled(null, outbox); + if (handled) scheduleTimeVoteRetry(); + return handled; } sendDurableMultiProxyOutbox(outbox); return false; @@ -5825,6 +5930,7 @@ private boolean markMultiProxyForwardingHandled(LiveVoteRetryState retryState, V if (retryState != null) retryState.multiProxyForwardingHandled = true; if (queuedVote == null) return true; queuedVote.setMultiProxyForwardingHandled(true); + queuedVote.setMultiProxyCapabilityDiscoveryPending(false); queuedVote.setDeliveryStateDirty(true); return persistTimeVoteDelivery(queuedVote); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java index 9173e279e..2d369e1d4 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java @@ -57,6 +57,8 @@ public void addTimedVote(int num, VoteTimeQueue voteTimedQueue) { setBoolean(path + ".Processed", voteTimedQueue.isProcessed()); setBoolean(path + ".MultiProxyForwardingHandled", voteTimedQueue.isMultiProxyForwardingHandled()); setBoolean(path + ".MultiProxyForwardingRequired", voteTimedQueue.isMultiProxyForwardingRequired()); + setBoolean(path + ".MultiProxyCapabilityDiscoveryPending", + voteTimedQueue.isMultiProxyCapabilityDiscoveryPending()); setBoolean(path + ".RealVote", voteTimedQueue.isRealVote()); setString(path + ".MultiProxyOrigin", voteTimedQueue.getMultiProxyOrigin()); setBoolean(path + ".MultiProxyCompletionPending", voteTimedQueue.isMultiProxyCompletionPending()); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java index 774515b5e..1782cf756 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/ProxyTimedVoteCacheTable.java @@ -41,6 +41,7 @@ public String buildCreateTableSql(DbType dbType) { + qi("processed") + " BOOLEAN NOT NULL DEFAULT FALSE, " + qi("multiProxyForwardingHandled") + " BOOLEAN NOT NULL DEFAULT FALSE, " + qi("multiProxyForwardingRequired") + " BOOLEAN NOT NULL DEFAULT FALSE, " + + qi("multiProxyCapabilityDiscoveryPending") + " BOOLEAN NOT NULL DEFAULT FALSE, " + qi("realVote") + " BOOLEAN NOT NULL DEFAULT TRUE, " + qi("multiProxyOrigin") + " VARCHAR(100), " + qi("multiProxyCompletionPending") + " BOOLEAN NOT NULL DEFAULT FALSE, " @@ -65,6 +66,7 @@ public String buildCreateTableSql(DbType dbType) { + qi("processed") + " TINYINT(1) NOT NULL DEFAULT 0," + qi("multiProxyForwardingHandled") + " TINYINT(1) NOT NULL DEFAULT 0," + qi("multiProxyForwardingRequired") + " TINYINT(1) NOT NULL DEFAULT 0," + + qi("multiProxyCapabilityDiscoveryPending") + " TINYINT(1) NOT NULL DEFAULT 0," + qi("realVote") + " TINYINT(1) NOT NULL DEFAULT 1," + qi("multiProxyOrigin") + " VARCHAR(100)," + qi("multiProxyCompletionPending") + " TINYINT(1) NOT NULL DEFAULT 0," @@ -135,6 +137,9 @@ private void ensureProxyBroadcastColumns() { ensureColumn("multiProxyForwardingRequired", getDbType() == DbType.POSTGRESQL ? "BOOLEAN NOT NULL DEFAULT FALSE" : "TINYINT(1) NOT NULL DEFAULT 0"); + ensureColumn("multiProxyCapabilityDiscoveryPending", getDbType() == DbType.POSTGRESQL + ? "BOOLEAN NOT NULL DEFAULT FALSE" + : "TINYINT(1) NOT NULL DEFAULT 0"); ensureColumn("realVote", getDbType() == DbType.POSTGRESQL ? "BOOLEAN NOT NULL DEFAULT TRUE" : "TINYINT(1) NOT NULL DEFAULT 1"); @@ -194,6 +199,7 @@ private void ensureIndexes() { private boolean insertTimedVote(UUID voteId, String uuid, String playerName, String service, long time, boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, boolean processed, boolean multiProxyForwardingHandled, boolean multiProxyForwardingRequired, + boolean multiProxyCapabilityDiscoveryPending, boolean realVote, String multiProxyOrigin, boolean multiProxyCompletionPending, String multiProxyRecipients, String multiProxyAcknowledgedServers, String httpBroadcastDeliveryIds, String multiProxyLegacyPendingRecipients) { @@ -202,10 +208,11 @@ private boolean insertTimedVote(UUID voteId, String uuid, String playerName, Str + qi("broadcastTargets") + ", " + qi("broadcastForwardedServers") + ", " + qi("totals") + ", " + qi("processed") + ", " + qi("multiProxyForwardingHandled") + ", " + qi("multiProxyForwardingRequired") + ", " + qi("realVote") + ", " + qi("multiProxyOrigin") + + ", " + qi("multiProxyCapabilityDiscoveryPending") + ", " + qi("multiProxyCompletionPending") + ", " + qi("multiProxyRecipients") + ", " + qi("multiProxyAcknowledgedServers") + ", " + qi("httpBroadcastDeliveryIds") + ", " + qi("multiProxyLegacyPendingRecipients") - + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; + + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; try (Connection conn = mysql.getConnectionManager().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { ps.setString(1, playerName); @@ -236,12 +243,14 @@ private boolean insertTimedVote(UUID voteId, String uuid, String playerName, Str ps.setInt(13, realVote ? 1 : 0); } ps.setString(14, multiProxyOrigin); - if (getDbType() == DbType.POSTGRESQL) ps.setBoolean(15, multiProxyCompletionPending); - else ps.setInt(15, multiProxyCompletionPending ? 1 : 0); - ps.setString(16, multiProxyRecipients); - ps.setString(17, multiProxyAcknowledgedServers); - ps.setString(18, httpBroadcastDeliveryIds); - ps.setString(19, multiProxyLegacyPendingRecipients); + if (getDbType() == DbType.POSTGRESQL) ps.setBoolean(15, multiProxyCapabilityDiscoveryPending); + else ps.setInt(15, multiProxyCapabilityDiscoveryPending ? 1 : 0); + if (getDbType() == DbType.POSTGRESQL) ps.setBoolean(16, multiProxyCompletionPending); + else ps.setInt(16, multiProxyCompletionPending ? 1 : 0); + ps.setString(17, multiProxyRecipients); + ps.setString(18, multiProxyAcknowledgedServers); + ps.setString(19, httpBroadcastDeliveryIds); + ps.setString(20, multiProxyLegacyPendingRecipients); ps.executeUpdate(); return true; } catch (SQLException e) { @@ -256,7 +265,8 @@ public boolean insertTimedVote(VoteTimeQueue vote) { return insertTimedVote(vote.getVoteId(), vote.getUuid(), vote.getName(), vote.getService(), vote.getTime(), vote.isProxyBroadcastHandled(), vote.encodeBroadcastTargets(), vote.encodeBroadcastForwardedServers(), vote.getTotals(), vote.isProcessed(), vote.isMultiProxyForwardingHandled(), - vote.isMultiProxyForwardingRequired(), vote.isRealVote(), vote.getMultiProxyOrigin(), + vote.isMultiProxyForwardingRequired(), vote.isMultiProxyCapabilityDiscoveryPending(), vote.isRealVote(), + vote.getMultiProxyOrigin(), vote.isMultiProxyCompletionPending(), vote.encodeMultiProxyRecipients(), vote.encodeMultiProxyAcknowledgedServers(), vote.encodeHttpBroadcastDeliveryIds(), vote.encodeMultiProxyLegacyPendingRecipients()); @@ -267,7 +277,7 @@ public boolean insertTimedVote(UUID voteId, String uuid, String playerName, Stri boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, boolean processed, boolean multiProxyForwardingHandled, String httpBroadcastDeliveryIds) { return insertTimedVote(voteId, uuid, playerName, service, time, proxyBroadcastHandled, broadcastTargets, - broadcastForwardedServers, totals, processed, multiProxyForwardingHandled, false, true, "", false, "", "", + broadcastForwardedServers, totals, processed, multiProxyForwardingHandled, false, false, true, "", false, "", "", httpBroadcastDeliveryIds, ""); } @@ -300,7 +310,8 @@ public boolean updateTimedVote(VoteTimeQueue vote) { + qi("broadcastTargets") + " = ?, " + qi("broadcastForwardedServers") + " = ?, " + qi("totals") + " = ?, " + qi("processed") + " = ?, " + qi("multiProxyForwardingHandled") + " = ?, " + qi("multiProxyForwardingRequired") + " = ?, " + qi("realVote") + " = ?, " - + qi("multiProxyOrigin") + " = ?, " + qi("multiProxyCompletionPending") + " = ?, " + + qi("multiProxyOrigin") + " = ?, " + qi("multiProxyCapabilityDiscoveryPending") + " = ?, " + + qi("multiProxyCompletionPending") + " = ?, " + qi("multiProxyRecipients") + " = ?, " + qi("multiProxyAcknowledgedServers") + " = ?, " + qi("uuid") + " = ?, " + qi("httpBroadcastDeliveryIds") + " = ?, " + qi("multiProxyLegacyPendingRecipients") + " = ? WHERE " @@ -331,21 +342,23 @@ public boolean updateTimedVote(VoteTimeQueue vote) { ps.setInt(8, vote.isRealVote() ? 1 : 0); } ps.setString(9, vote.getMultiProxyOrigin()); - if (getDbType() == DbType.POSTGRESQL) ps.setBoolean(10, vote.isMultiProxyCompletionPending()); - else ps.setInt(10, vote.isMultiProxyCompletionPending() ? 1 : 0); - ps.setString(11, vote.encodeMultiProxyRecipients()); - ps.setString(12, vote.encodeMultiProxyAcknowledgedServers()); - ps.setString(13, vote.getUuid()); - ps.setString(14, vote.encodeHttpBroadcastDeliveryIds()); - ps.setString(15, vote.encodeMultiProxyLegacyPendingRecipients()); + if (getDbType() == DbType.POSTGRESQL) ps.setBoolean(10, vote.isMultiProxyCapabilityDiscoveryPending()); + else ps.setInt(10, vote.isMultiProxyCapabilityDiscoveryPending() ? 1 : 0); + if (getDbType() == DbType.POSTGRESQL) ps.setBoolean(11, vote.isMultiProxyCompletionPending()); + else ps.setInt(11, vote.isMultiProxyCompletionPending() ? 1 : 0); + ps.setString(12, vote.encodeMultiProxyRecipients()); + ps.setString(13, vote.encodeMultiProxyAcknowledgedServers()); + ps.setString(14, vote.getUuid()); + ps.setString(15, vote.encodeHttpBroadcastDeliveryIds()); + ps.setString(16, vote.encodeMultiProxyLegacyPendingRecipients()); if (hasRowId) { - ps.setInt(16, vote.getTimedVoteCacheRowId()); + ps.setInt(17, vote.getTimedVoteCacheRowId()); } else if (hasVoteId) { - ps.setString(16, vote.getVoteId().toString()); + ps.setString(17, vote.getVoteId().toString()); } else { - ps.setString(16, vote.getName()); - ps.setString(17, vote.getService()); - ps.setLong(18, vote.getTime()); + ps.setString(17, vote.getName()); + ps.setString(18, vote.getService()); + ps.setLong(19, vote.getTime()); } ps.executeUpdate(); return true; @@ -505,6 +518,7 @@ private List selectVotes(String sql, Object[] params) { rs.getBoolean("processed"), rs.getBoolean("multiProxyForwardingHandled"), rs.getBoolean("multiProxyForwardingRequired"), + rs.getBoolean("multiProxyCapabilityDiscoveryPending"), rs.getBoolean("realVote"), rs.getString("multiProxyOrigin"), rs.getBoolean("multiProxyCompletionPending"), @@ -549,6 +563,7 @@ public static class TimedVoteRow { private final boolean processed; private final boolean multiProxyForwardingHandled; private final boolean multiProxyForwardingRequired; + private final boolean multiProxyCapabilityDiscoveryPending; private final boolean realVote; private final String multiProxyOrigin; private final boolean multiProxyCompletionPending; @@ -573,7 +588,8 @@ public static class TimedVoteRow { public TimedVoteRow(int id, String playerName, String service, long time, UUID voteId, String uuid, boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, boolean processed, boolean multiProxyForwardingHandled, boolean multiProxyForwardingRequired, - boolean realVote, String multiProxyOrigin, boolean multiProxyCompletionPending, String multiProxyRecipients, + boolean multiProxyCapabilityDiscoveryPending, boolean realVote, String multiProxyOrigin, + boolean multiProxyCompletionPending, String multiProxyRecipients, String multiProxyAcknowledgedServers, String httpBroadcastDeliveryIds, String multiProxyLegacyPendingRecipients) { this.id = id; @@ -589,6 +605,7 @@ public TimedVoteRow(int id, String playerName, String service, long time, UUID v this.processed = processed; this.multiProxyForwardingHandled = multiProxyForwardingHandled; this.multiProxyForwardingRequired = multiProxyForwardingRequired; + this.multiProxyCapabilityDiscoveryPending = multiProxyCapabilityDiscoveryPending; this.realVote = realVote; this.multiProxyOrigin = multiProxyOrigin; this.multiProxyCompletionPending = multiProxyCompletionPending; @@ -603,7 +620,7 @@ public TimedVoteRow(int id, String playerName, String service, long time, UUID v boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, boolean processed) { this(id, playerName, service, time, voteId, uuid, proxyBroadcastHandled, broadcastTargets, - broadcastForwardedServers, totals, processed, false, false, true, "", false, "", "", null, null); + broadcastForwardedServers, totals, processed, false, false, false, true, "", false, "", "", null, null); } /** Backward-compatible row constructor without the durable multi-proxy forwarding fence. */ @@ -611,7 +628,7 @@ public TimedVoteRow(int id, String playerName, String service, long time, UUID v boolean proxyBroadcastHandled, String broadcastTargets, String broadcastForwardedServers, String totals, boolean processed, String httpBroadcastDeliveryIds) { this(id, playerName, service, time, voteId, uuid, proxyBroadcastHandled, broadcastTargets, - broadcastForwardedServers, totals, processed, false, false, true, "", false, "", "", + broadcastForwardedServers, totals, processed, false, false, false, true, "", false, "", "", httpBroadcastDeliveryIds, null); } @@ -693,6 +710,8 @@ public boolean isMultiProxyForwardingHandled() { } public boolean isMultiProxyForwardingRequired() { return multiProxyForwardingRequired; } + + public boolean isMultiProxyCapabilityDiscoveryPending() { return multiProxyCapabilityDiscoveryPending; } public boolean isRealVote() { return realVote; } public String getMultiProxyOrigin() { return multiProxyOrigin; } public boolean isMultiProxyCompletionPending() { return multiProxyCompletionPending; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java index e8183eb53..20106e200 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VoteCacheHandler.java @@ -1299,6 +1299,8 @@ public void load() { timedVoteRow.isMultiProxyForwardingHandled(), timedVoteRow.getUuid(), VoteTimeQueue.decodeHttpBroadcastDeliveryIds(timedVoteRow.getHttpBroadcastDeliveryIds())); voteTimeQueue.setMultiProxyForwardingRequired(timedVoteRow.isMultiProxyForwardingRequired()); + voteTimeQueue.setMultiProxyCapabilityDiscoveryPending( + timedVoteRow.isMultiProxyCapabilityDiscoveryPending()); voteTimeQueue.setRealVote(timedVoteRow.isRealVote()); voteTimeQueue.setMultiProxyOrigin(timedVoteRow.getMultiProxyOrigin() == null ? "" : timedVoteRow.getMultiProxyOrigin()); @@ -1559,6 +1561,8 @@ private VoteTimeQueue decodeJsonTimedVote(String key, DataNode data) { VoteTimeQueue.decodeHttpBroadcastDeliveryIds(httpBroadcastDeliveryIds)); queuedVote.setMultiProxyForwardingRequired(data.has("MultiProxyForwardingRequired") && data.get("MultiProxyForwardingRequired").asBoolean()); + queuedVote.setMultiProxyCapabilityDiscoveryPending(data.has("MultiProxyCapabilityDiscoveryPending") + && data.get("MultiProxyCapabilityDiscoveryPending").asBoolean()); queuedVote.setRealVote(!data.has("RealVote") || data.get("RealVote").asBoolean()); queuedVote.setMultiProxyOrigin(data.has("MultiProxyOrigin") ? data.get("MultiProxyOrigin").asString() : ""); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyCapabilityStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyCapabilityStore.java index 385119984..70b3808f9 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyCapabilityStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyCapabilityStore.java @@ -9,8 +9,10 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Collection; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.Locale; +import java.util.Map; import java.util.Set; import com.bencodez.votingplugin.util.DurableFiles; @@ -21,13 +23,15 @@ /** * Stores only the stable identities of peers that have completed the - * durable-delivery capability handshake. A lease is deliberately not stored: + * durable-delivery capability handshake and the bounded discovery deadlines + * for peers which have not advertised yet. A lease is deliberately not stored: * every process must renew a peer's current handshake before publishing a * durable outbox retry. */ final class MultiProxyCapabilityStore { static final String FILE_NAME = ".multiproxy-capability-peers.json"; - private static final int VERSION = 1; + private static final int VERSION = 2; + private static final int LEGACY_VERSION = 1; private static final int MAX_BYTES = 64 * 1024; private static final int MAX_PEERS = 256; private static final int MAX_PEER_LENGTH = 256; @@ -53,7 +57,8 @@ static State load(Path dataDirectory) throws IOException { JsonElement parsed = JsonParser.parseString(new String(bytes, StandardCharsets.UTF_8)); if (!parsed.isJsonObject()) throw invalid(); JsonObject root = parsed.getAsJsonObject(); - if (!root.has("version") || root.get("version").getAsInt() != VERSION + if (!root.has("version") || (root.get("version").getAsInt() != VERSION + && root.get("version").getAsInt() != LEGACY_VERSION) || !root.has("peers") || !root.get("peers").isJsonArray()) throw invalid(); JsonArray peers = root.getAsJsonArray("peers"); if (peers.size() > MAX_PEERS) throw invalid(); @@ -63,14 +68,45 @@ static State load(Path dataDirectory) throws IOException { String name = peer.getAsString(); if (!validPeer(name) || !restored.add(name)) throw invalid(); } - return new State(restored); + Map discoveryDeadlines = new LinkedHashMap<>(); + if (root.has("discoveryDeadlines")) { + if (!root.get("discoveryDeadlines").isJsonObject()) throw invalid(); + JsonObject storedDeadlines = root.getAsJsonObject("discoveryDeadlines"); + if (storedDeadlines.size() > MAX_PEERS) throw invalid(); + for (Map.Entry entry : storedDeadlines.entrySet()) { + if (!validPeer(entry.getKey()) || !entry.getValue().isJsonPrimitive() + || !entry.getValue().getAsJsonPrimitive().isNumber()) throw invalid(); + long deadline = entry.getValue().getAsLong(); + if (deadline <= 0L || discoveryDeadlines.put(entry.getKey(), deadline) != null) throw invalid(); + } + } + long lastObservedMillis = 0L; + if (root.has("lastObservedMillis")) { + if (!root.get("lastObservedMillis").isJsonPrimitive() + || !root.get("lastObservedMillis").getAsJsonPrimitive().isNumber()) throw invalid(); + lastObservedMillis = root.get("lastObservedMillis").getAsLong(); + if (lastObservedMillis < 0L) throw invalid(); + } + return new State(restored, discoveryDeadlines, lastObservedMillis); } catch (RuntimeException malformed) { throw new IOException("Multi-proxy capability state is malformed", malformed); } } static void save(Path dataDirectory, Collection peers) throws IOException { - if (dataDirectory == null || peers == null || peers.size() > MAX_PEERS) throw new IOException("Invalid capability state"); + save(dataDirectory, peers, Map.of()); + } + + static void save(Path dataDirectory, Collection peers, Map discoveryDeadlines) + throws IOException { + save(dataDirectory, peers, discoveryDeadlines, 0L); + } + + static void save(Path dataDirectory, Collection peers, Map discoveryDeadlines, + long lastObservedMillis) throws IOException { + if (dataDirectory == null || peers == null || discoveryDeadlines == null || peers.size() > MAX_PEERS + || discoveryDeadlines.size() > MAX_PEERS || lastObservedMillis < 0L) + throw new IOException("Invalid capability state"); Set copy = new LinkedHashSet<>(); for (String peer : peers) { if (!validPeer(peer) || !copy.add(peer)) throw new IOException("Invalid capability peer"); @@ -80,6 +116,16 @@ static void save(Path dataDirectory, Collection peers) throws IOExceptio JsonArray listed = new JsonArray(); copy.stream().sorted().forEach(listed::add); root.add("peers", listed); + JsonObject storedDeadlines = new JsonObject(); + for (String peer : discoveryDeadlines.keySet().stream().sorted().toList()) { + Long deadline = discoveryDeadlines.get(peer); + if (!validPeer(peer) || deadline == null || deadline <= 0L) { + throw new IOException("Invalid capability discovery state"); + } + storedDeadlines.addProperty(peer, deadline); + } + root.add("discoveryDeadlines", storedDeadlines); + root.addProperty("lastObservedMillis", lastObservedMillis); byte[] serialized = root.toString().getBytes(StandardCharsets.UTF_8); if (serialized.length > MAX_BYTES) throw new IOException("Multi-proxy capability state is too large"); @@ -112,13 +158,14 @@ private static IllegalArgumentException invalid() { return new IllegalArgumentException("Invalid capability state"); } - record State(Set peers) { + record State(Set peers, Map discoveryDeadlines, long lastObservedMillis) { State { peers = Set.copyOf(peers); + discoveryDeadlines = Map.copyOf(discoveryDeadlines); } static State empty() { - return new State(Set.of()); + return new State(Set.of(), Map.of(), 0L); } } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java index f42a05f2e..f533c96f9 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java @@ -37,6 +37,12 @@ public abstract class MultiProxyHandler { static final long VOTE_CAPABILITY_LEASE_MILLIS = 5 * 60 * 1000L; /** Bound retry-driven capability advertisements while a durable outbox is waiting. */ static final long VOTE_CAPABILITY_RENEWAL_MIN_INTERVAL_MILLIS = 30 * 1000L; + /** + * A newly configured peer gets this one bounded opportunity to answer the + * capability handshake before it is treated as a legacy one-way peer. The + * deadline is durable so a restart cannot extend the window indefinitely. + */ + static final long VOTE_CAPABILITY_DISCOVERY_WINDOW_MILLIS = 30 * 1000L; private final Map acknowledgedVoteCapabilityPeers = new HashMap<>(); /** * Peers which have previously completed the durable-delivery handshake. Retain @@ -45,6 +51,8 @@ public abstract class MultiProxyHandler { * as success and lose the sender's durable completion fence. */ private final Set knownVoteCapabilityPeers = new HashSet<>(); + /** Durable, per-peer deadlines for the initial capability discovery window. */ + private final Map voteCapabilityDiscoveryDeadlines = new HashMap<>(); /** * We cannot safely classify configured peers after the durable state is corrupt * or cannot be updated. Do not create a new mixed-version outbox in that @@ -54,6 +62,8 @@ public abstract class MultiProxyHandler { */ private boolean voteCapabilityRecoveryBlocked; private long lastVoteCapabilityAdvertisementMillis = Long.MIN_VALUE; + /** Last wall-clock observation durably associated with discovery deadlines. */ + private long lastVoteCapabilityObservationMillis; @Getter private RedisHandler multiProxyRedis; @@ -99,8 +109,10 @@ public synchronized void close() { multiproxyClientHandles = null; acknowledgedVoteCapabilityPeers.clear(); knownVoteCapabilityPeers.clear(); + voteCapabilityDiscoveryDeadlines.clear(); voteCapabilityRecoveryBlocked = false; lastVoteCapabilityAdvertisementMillis = Long.MIN_VALUE; + lastVoteCapabilityObservationMillis = 0L; } /** @@ -389,6 +401,50 @@ public synchronized Set getMultiProxyVoteRecipientsAwaitingCapabilityRen return recipients; } + /** + * Returns never-observed configured peers whose initial capability handshake + * is still within its bounded discovery window. Callers retain the vote for a + * retry while this set is non-empty, rather than publishing a one-way legacy + * copy before a newly upgraded peer can identify itself. + * + *

The deadline is recorded before it is exposed. If recording it fails, we + * block forwarding rather than allowing repeated restarts to make this window + * unbounded or silently demoting a peer to legacy.

+ */ + public synchronized Set getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery() { + if (voteCapabilityRecoveryBlocked) return Set.of(); + long now = capabilityNowMillis(); + Map configured = configuredMultiProxyRecipientNames(); + Map updated = new HashMap<>(voteCapabilityDiscoveryDeadlines); + boolean clockRolledBack = lastVoteCapabilityObservationMillis > 0L && now < lastVoteCapabilityObservationMillis; + // A wall clock moving backwards must never grant a second discovery window. + // Expire the already-recorded opportunity rather than waiting for the clock + // to catch up (which can otherwise retain a vote indefinitely after reboot). + if (clockRolledBack) { + updated.replaceAll((peer, deadline) -> deadline > now ? now : deadline); + } + boolean changed = updated.keySet().removeIf(peer -> !configured.containsKey(peer) + || knownVoteCapabilityPeers.contains(peer)); + changed |= clockRolledBack && !updated.equals(voteCapabilityDiscoveryDeadlines); + for (String peer : configured.keySet()) { + if (knownVoteCapabilityPeers.contains(peer) || updated.containsKey(peer)) continue; + updated.put(peer, discoveryDeadline(now)); + changed = true; + } + long observed = Math.max(0L, now); + boolean observationChanged = !updated.isEmpty() && observed != lastVoteCapabilityObservationMillis; + if ((changed || observationChanged) + && !saveVoteCapabilityState(knownVoteCapabilityPeers, updated, observed)) return Set.of(); + Set recipients = new LinkedHashSet<>(); + for (Map.Entry configuredPeer : configured.entrySet()) { + Long deadline = voteCapabilityDiscoveryDeadlines.get(configuredPeer.getKey()); + if (!knownVoteCapabilityPeers.contains(configuredPeer.getKey()) && deadline != null && deadline > now) { + recipients.add(configuredPeer.getValue()); + } + } + return recipients; + } + /** Broadcasts this node's durable-ACK capability to configured peers. */ public synchronized void announceMultiProxyVoteCapability() { lastVoteCapabilityAdvertisementMillis = capabilityNowMillis(); @@ -427,7 +483,9 @@ public synchronized boolean isMultiProxyVoteCapabilityRecoveryBlocked() { synchronized void restoreVoteCapabilityPeers() { acknowledgedVoteCapabilityPeers.clear(); knownVoteCapabilityPeers.clear(); + voteCapabilityDiscoveryDeadlines.clear(); voteCapabilityRecoveryBlocked = false; + lastVoteCapabilityObservationMillis = 0L; Path dataDirectory = capabilityStateDirectory(); // Third-party MultiProxyHandler integrations predate durable state and may // intentionally provide no plugin folder. Preserve their established @@ -436,9 +494,30 @@ synchronized void restoreVoteCapabilityPeers() { try { MultiProxyCapabilityStore.State restored = MultiProxyCapabilityStore.load(dataDirectory); Set configured = configuredMultiProxyRecipientNames().keySet(); - for (String peer : restored.peers()) { - if (configured.contains(peer)) knownVoteCapabilityPeers.add(peer); + Set restoredPeers = new HashSet<>(); + for (String peer : restored.peers()) if (configured.contains(peer)) restoredPeers.add(peer); + Map restoredDeadlines = new HashMap<>(); + for (Map.Entry entry : restored.discoveryDeadlines().entrySet()) { + if (configured.contains(entry.getKey()) && !restoredPeers.contains(entry.getKey())) { + restoredDeadlines.put(entry.getKey(), entry.getValue()); + } } + long now = capabilityNowMillis(); + boolean clockRolledBack = restored.lastObservedMillis() > 0L && now < restored.lastObservedMillis(); + if (clockRolledBack) { + restoredDeadlines.replaceAll((peer, deadline) -> deadline > now ? now : deadline); + } + long observed = Math.max(0L, clockRolledBack ? now : Math.max(now, restored.lastObservedMillis())); + boolean pruned = !restoredPeers.equals(restored.peers()) + || !restoredDeadlines.equals(restored.discoveryDeadlines()); + boolean observationChanged = !restoredDeadlines.isEmpty() && observed != restored.lastObservedMillis(); + // Persist removal before installing the filtered identities. Otherwise a + // later re-add of the same name could resurrect an ACK-capable identity. + if ((pruned || observationChanged) + && !saveVoteCapabilityState(restoredPeers, restoredDeadlines, observed)) return; + knownVoteCapabilityPeers.addAll(restoredPeers); + voteCapabilityDiscoveryDeadlines.putAll(restoredDeadlines); + lastVoteCapabilityObservationMillis = observed; } catch (IOException | IllegalArgumentException stateFailure) { blockVoteCapabilityRecovery(); } @@ -455,16 +534,28 @@ private boolean acceptVoteCapabilityPeer(String normalized) { if (!newPeer) return true; Set updated = new HashSet<>(knownVoteCapabilityPeers); updated.add(normalized); + Map updatedDeadlines = new HashMap<>(voteCapabilityDiscoveryDeadlines); + updatedDeadlines.remove(normalized); + return saveVoteCapabilityState(updated, updatedDeadlines, lastVoteCapabilityObservationMillis); + } + + private long discoveryDeadline(long now) { + return now > Long.MAX_VALUE - VOTE_CAPABILITY_DISCOVERY_WINDOW_MILLIS ? Long.MAX_VALUE + : now + VOTE_CAPABILITY_DISCOVERY_WINDOW_MILLIS; + } + + /** Saves and installs peer classification atomically from the sender's view. */ + private boolean saveVoteCapabilityState(Collection peers, Map discoveryDeadlines, + long lastObservedMillis) { Path dataDirectory = capabilityStateDirectory(); - if (dataDirectory == null) { - knownVoteCapabilityPeers.clear(); - knownVoteCapabilityPeers.addAll(updated); - return true; - } try { - MultiProxyCapabilityStore.save(dataDirectory, updated); + if (dataDirectory != null) MultiProxyCapabilityStore.save(dataDirectory, peers, discoveryDeadlines, + lastObservedMillis); knownVoteCapabilityPeers.clear(); - knownVoteCapabilityPeers.addAll(updated); + knownVoteCapabilityPeers.addAll(peers); + voteCapabilityDiscoveryDeadlines.clear(); + voteCapabilityDiscoveryDeadlines.putAll(discoveryDeadlines); + lastVoteCapabilityObservationMillis = lastObservedMillis; return true; } catch (IOException | IllegalArgumentException stateFailure) { blockVoteCapabilityRecovery(); @@ -489,8 +580,10 @@ private Path capabilityStateDirectory() { public synchronized void loadMultiProxySupport() { acknowledgedVoteCapabilityPeers.clear(); knownVoteCapabilityPeers.clear(); + voteCapabilityDiscoveryDeadlines.clear(); voteCapabilityRecoveryBlocked = false; lastVoteCapabilityAdvertisementMillis = Long.MIN_VALUE; + lastVoteCapabilityObservationMillis = 0L; restoreVoteCapabilityPeers(); if (!getMultiProxySupportEnabled()) { return; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java index 0fe557194..341153230 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java @@ -70,6 +70,8 @@ public void addTimedVote(int num, VoteTimeQueue voteTimedQueue) { "MultiProxyForwardingHandled"); setPath(voteTimedQueue.isMultiProxyForwardingRequired(), "TimedVoteCache", String.valueOf(num), "MultiProxyForwardingRequired"); + setPath(voteTimedQueue.isMultiProxyCapabilityDiscoveryPending(), "TimedVoteCache", String.valueOf(num), + "MultiProxyCapabilityDiscoveryPending"); setPath(voteTimedQueue.isRealVote(), "TimedVoteCache", String.valueOf(num), "RealVote"); setPath(voteTimedQueue.getMultiProxyOrigin(), "TimedVoteCache", String.valueOf(num), "MultiProxyOrigin"); setPath(voteTimedQueue.isMultiProxyCompletionPending(), "TimedVoteCache", String.valueOf(num), diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java index c3cafd925..3d6f548c1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/VoteTimeQueue.java @@ -58,6 +58,15 @@ public class VoteTimeQueue { @Getter @Setter private boolean multiProxyForwardingRequired; + /** + * The forwarding intent is durable, but peer capability discovery has not + * finished yet. This is deliberately distinct from an ACK outbox with an + * empty recipient set: recovery must classify the configured peers before it + * can choose either the durable or legacy route. + */ + @Getter + @Setter + private boolean multiProxyCapabilityDiscoveryPending; /** Original vote type; legacy rows default to a real vote. */ @Getter @Setter diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java index 4187aae4a..86e908d22 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import java.lang.reflect.Method; import java.nio.file.Files; @@ -327,6 +328,87 @@ void freshInstallKeepsLegacyPeersOnTheHistoricalRoute(@TempDir Path dataDirector assertTrue(handler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal().isEmpty()); } + @Test + void newlyConfiguredPeerWaitsForCapabilityAnnouncementBeforeItIsClassified() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, + org.mockito.Mockito.withSettings().useConstructor().defaultAnswer(org.mockito.Mockito.CALLS_REAL_METHODS)); + org.mockito.Mockito.when(handler.getMultiProxyMethod()).thenReturn(MultiProxyMethod.SOCKETS); + org.mockito.Mockito.when(handler.getMultiProxyServers()).thenReturn(List.of("Candidate")); + org.mockito.Mockito.when(handler.capabilityNowMillis()).thenReturn(1_000L); + + assertEquals(java.util.Set.of("Candidate"), + handler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery()); + handleCapability(handler, "Candidate"); + + assertTrue(handler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery().isEmpty()); + assertEquals(java.util.Set.of("Candidate"), handler.getMultiProxyVoteRecipients()); + } + + @Test + void missedCapabilityAnnouncementFallsBackOnlyAfterItsBoundedPersistedWindow(@TempDir Path dataDirectory) { + MultiProxyHandler beforeRestart = capabilityHandler(dataDirectory, "Legacy"); + org.mockito.Mockito.when(beforeRestart.capabilityNowMillis()).thenReturn(1_000L); + assertEquals(java.util.Set.of("Legacy"), + beforeRestart.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery()); + + MultiProxyHandler afterRestart = capabilityHandler(dataDirectory, "Legacy"); + afterRestart.restoreVoteCapabilityPeers(); + org.mockito.Mockito.when(afterRestart.capabilityNowMillis()).thenReturn( + 1_000L + MultiProxyHandler.VOTE_CAPABILITY_DISCOVERY_WINDOW_MILLIS); + assertTrue(afterRestart.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery().isEmpty()); + // The expired deadline remains a legacy classification rather than beginning + // another discovery window on every retry/restart. + assertTrue(afterRestart.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery().isEmpty()); + } + + @Test + void clockRollbackExpiresDiscoveryInsteadOfExtendingItsPersistedWindow(@TempDir Path dataDirectory) { + MultiProxyHandler handler = capabilityHandler(dataDirectory, "Candidate"); + org.mockito.Mockito.when(handler.capabilityNowMillis()).thenReturn(1_000L, 500L); + + assertEquals(java.util.Set.of("Candidate"), handler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery()); + assertTrue(handler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery().isEmpty()); + + MultiProxyCapabilityStore.State persisted = assertDoesNotThrow(() -> MultiProxyCapabilityStore.load(dataDirectory)); + assertEquals(500L, persisted.discoveryDeadlines().get("candidate")); + assertEquals(500L, persisted.lastObservedMillis()); + } + + @Test + void removedCapabilityPeerIsPrunedDurablyBeforeSameNameIsReadded(@TempDir Path dataDirectory) throws Exception { + MultiProxyHandler original = capabilityHandler(dataDirectory, "Capable"); + handleCapability(original, "Capable"); + + MultiProxyHandler removed = capabilityHandler(dataDirectory, "Legacy"); + removed.restoreVoteCapabilityPeers(); + assertTrue(MultiProxyCapabilityStore.load(dataDirectory).peers().isEmpty()); + + MultiProxyHandler readded = capabilityHandler(dataDirectory, "Capable"); + readded.restoreVoteCapabilityPeers(); + assertTrue(readded.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal().isEmpty()); + assertEquals(java.util.Set.of("Capable"), + readded.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery()); + } + + @Test + void failedConfigurationPruningBlocksForwardingWithoutInstallingStalePeerState(@TempDir Path dataDirectory) + throws Exception { + MultiProxyCapabilityStore.save(dataDirectory, java.util.Set.of("capable"), Map.of(), 0L); + MultiProxyHandler removed = capabilityHandler(dataDirectory, "Legacy"); + try (org.mockito.MockedStatic store = org.mockito.Mockito.mockStatic( + MultiProxyCapabilityStore.class, org.mockito.Mockito.CALLS_REAL_METHODS)) { + store.when(() -> MultiProxyCapabilityStore.save(org.mockito.ArgumentMatchers.eq(dataDirectory), + org.mockito.ArgumentMatchers.anyCollection(), org.mockito.ArgumentMatchers.anyMap(), + org.mockito.ArgumentMatchers.anyLong())) + .thenThrow(new java.io.IOException("read-only capability state")); + removed.restoreVoteCapabilityPeers(); + } + + assertTrue(removed.isMultiProxyVoteCapabilityRecoveryBlocked()); + assertTrue(removed.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal().isEmpty()); + assertEquals(java.util.Set.of("capable"), MultiProxyCapabilityStore.load(dataDirectory).peers()); + } + @Test void capabilityStoreRejectsSymlinkedState(@TempDir Path dataDirectory) throws Exception { Path outside = Files.createTempFile("multiproxy-capability-outside", ".json"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 289f4d700..0e13394ee 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -529,6 +529,95 @@ void rejectedLegacyOnlyMultiProxySendKeepsLiveVoteForRetry() throws Exception { .getOrDefault(VotingPluginWire.K_MULTI_PROXY_ORIGIN, "")); } + @Test + void capabilityDiscoveryDefersVoteUntilExpiryThenSendsOneLegacyCopy() throws Exception { + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Collections.emptySet()); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal()) + .thenReturn(java.util.Collections.emptySet()); + Mockito.when(multiProxyHandler.getConfiguredMultiProxyVoteRecipients()) + .thenReturn(new java.util.LinkedHashSet<>(java.util.Set.of("Candidate"))); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery()) + .thenReturn(java.util.Set.of("Candidate"), java.util.Collections.emptySet()); + Mockito.when(votingPluginProxy.getConfig().getProxyServerName()).thenReturn("Proxy1"); + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.addTimeVoteToCache(Mockito.any())).thenAnswer(invocation -> { + queue.add(invocation.getArgument(0)); + return true; + }); + Mockito.when(voteCache.updateTimeVote(Mockito.any())).thenReturn(true); + Mockito.when(multiProxyHandler.sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any())) + .thenAnswer(invocation -> { + assertTrue(queue.element().getMultiProxyLegacyPendingRecipients().isEmpty(), + "legacy recipients must be durably cleared before the non-deduplicating send"); + return true; + }); + VotingPluginProxyTestImpl proxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(proxy).getVoteCacheHandler(); + + Class retryType = Class.forName("com.bencodez.votingplugin.proxy.VotingPluginProxy$LiveVoteRetryState"); + java.lang.reflect.Constructor constructor = retryType.getDeclaredConstructor(); + constructor.setAccessible(true); + Object retry = constructor.newInstance(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + java.lang.reflect.Field retriesField = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); + retriesField.setAccessible(true); + @SuppressWarnings("unchecked") + java.util.Map retries = + (java.util.Map) retriesField.get(proxy); + retries.put(voteId, retry); + java.lang.reflect.Method begin = VotingPluginProxy.class.getDeclaredMethod("beginMultiProxyForwarding", + retryType, VoteTimeQueue.class, String.class, String.class, String.class, long.class, + boolean.class, com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class); + begin.setAccessible(true); + + assertFalse((Boolean) begin.invoke(proxy, retry, null, "Player", + "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); + verify(multiProxyHandler, never()).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any()); + assertEquals(1, queue.size()); + assertTrue(queue.element().isMultiProxyCapabilityDiscoveryPending()); + assertTrue(queue.element().isProcessed()); + + assertTrue((Boolean) begin.invoke(proxy, retry, queue.element(), "Player", + "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); + verify(multiProxyHandler).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.eq(java.util.Set.of("Candidate"))); + assertFalse(queue.element().isMultiProxyCapabilityDiscoveryPending()); + assertTrue(queue.element().isMultiProxyForwardingHandled(), + "the durable discovery fence must be retired before reporting the legacy fallback complete"); + assertTrue(queue.element().getMultiProxyLegacyPendingRecipients().isEmpty()); + assertTrue((Boolean) begin.invoke(proxy, retry, queue.element(), "Player", + "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); + verify(multiProxyHandler, Mockito.times(1)).sendMultiProxyEnvelopeAccepted(Mockito.any(), + Mockito.eq(java.util.Set.of("Candidate"))); + verify(multiProxyHandler, Mockito.times(4)).renewMultiProxyVoteCapabilityIfDue(); + } + + @Test + void capabilityDiscoveryPersistenceFailureCannotFallThroughToLegacySend() throws Exception { + Mockito.when(multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked()).thenReturn(false, true); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Collections.emptySet()); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal()) + .thenReturn(java.util.Collections.emptySet()); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery()) + .thenReturn(java.util.Collections.emptySet()); + Mockito.when(multiProxyHandler.getConfiguredMultiProxyVoteRecipients()) + .thenReturn(new java.util.LinkedHashSet<>(java.util.Set.of("Candidate"))); + + Class retryType = Class.forName("com.bencodez.votingplugin.proxy.VotingPluginProxy$LiveVoteRetryState"); + java.lang.reflect.Constructor constructor = retryType.getDeclaredConstructor(); + constructor.setAccessible(true); + Object retry = constructor.newInstance(); + java.lang.reflect.Method begin = VotingPluginProxy.class.getDeclaredMethod("beginMultiProxyForwarding", + retryType, VoteTimeQueue.class, String.class, String.class, String.class, long.class, + boolean.class, com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class); + begin.setAccessible(true); + + assertFalse((Boolean) begin.invoke(votingPluginProxy, retry, null, "Player", + "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); + verify(multiProxyHandler, never()).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any()); + } + @Test void capabilityRecoveryBlockPreventsCreatingAMixedVersionAcknowledgementOutbox() throws Exception { Mockito.when(multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked()).thenReturn(true); @@ -625,6 +714,37 @@ void persistedOutboxWaitsForFreshCapabilityLeaseAfterRestart() { assertTrue(queue.contains(outbox)); } + @Test + void restartResumesDurableDiscoveryOutboxBeforePublishingIt() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + VoteTimeQueue outbox = new VoteTimeQueue(java.util.UUID.randomUUID(), "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", true, + "00000000-0000-0000-0000-000000000001"); + outbox.requireMultiProxyAcknowledgements("Proxy1", java.util.Collections.emptySet()); + outbox.setMultiProxyCapabilityDiscoveryPending(true); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + queue.add(outbox); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.updateTimeVote(Mockito.any())).thenReturn(true); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Set.of("Capable")); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal()) + .thenReturn(java.util.Collections.emptySet()); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery()) + .thenReturn(java.util.Collections.emptySet()); + Mockito.when(multiProxyHandler.getConfiguredMultiProxyVoteRecipients()) + .thenReturn(new java.util.LinkedHashSet<>(java.util.Set.of("Capable"))); + Mockito.when(votingPluginProxy.getConfig().getProxyServerName()).thenReturn("Proxy1"); + VotingPluginProxyTestImpl proxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(proxy).getVoteCacheHandler(); + + proxy.processQueue(); + + assertFalse(outbox.isMultiProxyCapabilityDiscoveryPending()); + assertEquals(java.util.Set.of("capable"), outbox.getMultiProxyRecipients()); + verify(multiProxyHandler).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.eq(java.util.Set.of("capable"))); + assertTrue(queue.contains(outbox)); + } + @Test void completedAckOutboxKeepsFenceForAlreadyScheduledListenerRetry() throws Exception { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); @@ -925,8 +1045,8 @@ void reconstructedQueuedVoteRetainsUnacknowledgedMultiProxyDeliveryState() throw restored.setMultiProxyOrigin(queued.getMultiProxyOrigin()); restored.setMultiProxyRecipients(queued.getMultiProxyRecipients()); restored.setMultiProxyAcknowledgedServers(queued.getMultiProxyAcknowledgedServers()); - // The failed post-publication update means durable storage still contains the - // pending legacy recipient captured before the first publish. + // The failed pre-publication update means no legacy copy was sent and durable + // storage still contains the pending recipient for restart recovery. restored.setMultiProxyLegacyPendingRecipients(java.util.Set.of("ProxyLegacy")); queue.clear(); queue.add(restored); @@ -939,8 +1059,8 @@ void reconstructedQueuedVoteRetainsUnacknowledgedMultiProxyDeliveryState() throw @SuppressWarnings({ "rawtypes", "unchecked" }) org.mockito.ArgumentCaptor> recipients = (org.mockito.ArgumentCaptor) org.mockito.ArgumentCaptor.forClass(java.util.Collection.class); - verify(multiProxyHandler, Mockito.times(3)).sendMultiProxyEnvelopeAccepted(Mockito.any(), recipients.capture()); - assertEquals(2, recipients.getAllValues().stream() + verify(multiProxyHandler, Mockito.times(2)).sendMultiProxyEnvelopeAccepted(Mockito.any(), recipients.capture()); + assertEquals(1, recipients.getAllValues().stream() .filter(value -> new java.util.HashSet<>(value).equals(java.util.Set.of("ProxyLegacy"))).count()); assertEquals(1, recipients.getAllValues().stream() .filter(value -> new java.util.HashSet<>(value).equals(java.util.Set.of("proxy2"))).count()); From 713e07c7311d7549b1798e4c8654dc9bfdef42ab Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:10:14 -0600 Subject: [PATCH 110/136] Harden durable proxy capability discovery --- .../votingplugin/proxy/VotingPluginProxy.java | 30 +++++++--- .../proxy/multiproxy/MultiProxyHandler.java | 57 +++++++++++++++++-- .../MultiProxyHandlerLifecycleTest.java | 45 +++++++++++++++ .../tests/VotingPluginProxyTest.java | 53 ++++++++++++++++- 4 files changed, 173 insertions(+), 12 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 2f386e69e..9c782e0e9 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -5625,13 +5625,18 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim // which targets are legacy. Stop before creating an ACK outbox that a legacy // peer could never complete; the handler emits the operator recovery message. if (multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked()) return false; - // Capability retries can be much more frequent than a normal vote. Start - // discovery promptly, but rate-limit repeated advertisements while the vote - // waits for a newly configured peer to identify itself. - multiProxyHandler.renewMultiProxyVoteCapabilityIfDue(); Set recipients = multiProxyHandler.getMultiProxyVoteRecipients(); Set renewingRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal(); Set discoveringRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery(); + // Query discovery before advertising. A retry that reaches the durable + // deadline must classify the peer rather than sending a final handshake and + // immediately falling through to legacy; live discovery instead renews early + // with a bounded settling interval for asynchronous Redis replies. + if (!discoveringRecipients.isEmpty()) { + multiProxyHandler.renewMultiProxyVoteCapabilityDiscoveryIfDue(); + } else if (!renewingRecipients.isEmpty()) { + multiProxyHandler.renewMultiProxyVoteCapabilityIfDue(); + } // Recording a first-observation deadline can itself fail and transition the // handler into recovery-blocked state. Recheck before interpreting an empty // discovery set as permission to use the lossy legacy route. @@ -5648,7 +5653,8 @@ private boolean beginMultiProxyForwarding(LiveVoteRetryState retryState, VoteTim // now would make the later legacy fallback hard to fence without a second, // duplicate-prone outbox. The durable peer deadline bounds this retry even // across a proxy restart. - admitMultiProxyDiscoveryOutbox(retryState, queuedVote, player, uuid, service, time, realVote, totals); + if (admitMultiProxyDiscoveryOutbox(retryState, queuedVote, player, uuid, service, time, realVote, totals) + != null) scheduleTimeVoteRetry(); return false; } Set durableRecipients = new LinkedHashSet<>(recipients); @@ -5778,10 +5784,14 @@ private VoteTimeQueue admitMultiProxyDiscoveryOutbox(LiveVoteRetryState retrySta /** Resolves a restored discovery-pending outbox and persists its recipient split before sending. */ private boolean resolveMultiProxyDiscoveryOutbox(VoteTimeQueue outbox) { if (multiProxyHandler == null || multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked()) return false; - multiProxyHandler.renewMultiProxyVoteCapabilityIfDue(); Set recipients = multiProxyHandler.getMultiProxyVoteRecipients(); Set renewingRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal(); Set discoveringRecipients = multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery(); + if (!discoveringRecipients.isEmpty()) { + multiProxyHandler.renewMultiProxyVoteCapabilityDiscoveryIfDue(); + } else if (!renewingRecipients.isEmpty()) { + multiProxyHandler.renewMultiProxyVoteCapabilityIfDue(); + } if (multiProxyHandler.isMultiProxyVoteCapabilityRecoveryBlocked() || !discoveringRecipients.isEmpty()) return false; Set configuredRecipients = multiProxyHandler.getConfiguredMultiProxyVoteRecipients(); if (configuredRecipients.isEmpty() && !recipients.isEmpty()) configuredRecipients.addAll(recipients); @@ -5861,7 +5871,13 @@ private boolean sendDurableMultiProxyOutbox(VoteTimeQueue outbox) { } pending.removeIf(recipient -> recipient != null && awaitingRenewal.contains(recipient.toLowerCase(Locale.ROOT))); - if (pending.isEmpty()) return false; + if (pending.isEmpty()) { + // This method is also reached directly from the listener path, where no + // enclosing queue loop schedules the next lease-renewal attempt. Keep the + // durable row live until the peer answers the bounded handshake. + scheduleTimeVoteRetry(); + return false; + } return multiProxyHandler.sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyVote(outbox.getName(), outbox.getUuid(), outbox.getService(), outbox.getTime(), false, outbox.isRealVote(), outbox.getTotals(), outbox.getVoteId(), false, false, 1, 1, outbox.getMultiProxyOrigin()), diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java index f533c96f9..bb4cb9ad8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java @@ -37,6 +37,13 @@ public abstract class MultiProxyHandler { static final long VOTE_CAPABILITY_LEASE_MILLIS = 5 * 60 * 1000L; /** Bound retry-driven capability advertisements while a durable outbox is waiting. */ static final long VOTE_CAPABILITY_RENEWAL_MIN_INTERVAL_MILLIS = 30 * 1000L; + /** + * A discovery reply can arrive asynchronously (notably through Redis). Renew + * before the bounded discovery window closes, leaving the final announcement a + * real interval to be observed instead of classifying a peer as legacy in the + * same retry that sent its last handshake. + */ + static final long VOTE_CAPABILITY_DISCOVERY_RENEWAL_MIN_INTERVAL_MILLIS = 10 * 1000L; /** * A newly configured peer gets this one bounded opportunity to answer the * capability handshake before it is treated as a legacy one-way peer. The @@ -62,6 +69,8 @@ public abstract class MultiProxyHandler { */ private boolean voteCapabilityRecoveryBlocked; private long lastVoteCapabilityAdvertisementMillis = Long.MIN_VALUE; + /** A newly persisted discovery deadline must cause an initial handshake promptly. */ + private boolean voteCapabilityDiscoveryAnnouncementRequired; /** Last wall-clock observation durably associated with discovery deadlines. */ private long lastVoteCapabilityObservationMillis; @@ -426,15 +435,37 @@ public synchronized Set getMultiProxyVoteRecipientsAwaitingCapabilityDis boolean changed = updated.keySet().removeIf(peer -> !configured.containsKey(peer) || knownVoteCapabilityPeers.contains(peer)); changed |= clockRolledBack && !updated.equals(voteCapabilityDiscoveryDeadlines); + boolean addedDiscoveryDeadline = false; for (String peer : configured.keySet()) { if (knownVoteCapabilityPeers.contains(peer) || updated.containsKey(peer)) continue; updated.put(peer, discoveryDeadline(now)); changed = true; + addedDiscoveryDeadline = true; } long observed = Math.max(0L, now); - boolean observationChanged = !updated.isEmpty() && observed != lastVoteCapabilityObservationMillis; + boolean activeDiscovery = false; + boolean discoveryExpiredSinceLastObservation = false; + for (Long deadline : updated.values()) { + if (deadline == null) continue; + if (deadline > now) { + activeDiscovery = true; + } else if (deadline > lastVoteCapabilityObservationMillis) { + // Preserve the first observation after expiry. Without it, a later + // clock rollback could make this already-consumed window live again. + discoveryExpiredSinceLastObservation = true; + } + } + // Keep a persisted high-water mark while discovery is live, but do so at + // the same bounded cadence as advertisements. Once an expired deadline is + // recorded, repeated legacy forwarding must not fsync this state per vote. + boolean observationChanged = !updated.isEmpty() && (clockRolledBack + || discoveryExpiredSinceLastObservation || (activeDiscovery + && (lastVoteCapabilityObservationMillis == 0L + || now - lastVoteCapabilityObservationMillis + >= VOTE_CAPABILITY_DISCOVERY_RENEWAL_MIN_INTERVAL_MILLIS))); if ((changed || observationChanged) && !saveVoteCapabilityState(knownVoteCapabilityPeers, updated, observed)) return Set.of(); + if (addedDiscoveryDeadline) voteCapabilityDiscoveryAnnouncementRequired = true; Set recipients = new LinkedHashSet<>(); for (Map.Entry configuredPeer : configured.entrySet()) { Long deadline = voteCapabilityDiscoveryDeadlines.get(configuredPeer.getKey()); @@ -448,6 +479,7 @@ public synchronized Set getMultiProxyVoteRecipientsAwaitingCapabilityDis /** Broadcasts this node's durable-ACK capability to configured peers. */ public synchronized void announceMultiProxyVoteCapability() { lastVoteCapabilityAdvertisementMillis = capabilityNowMillis(); + voteCapabilityDiscoveryAnnouncementRequired = false; sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyCapabilities(getMultiProxyServerName(), 1)); } @@ -459,13 +491,27 @@ public synchronized void announceMultiProxyVoteCapability() { * @return whether an advertisement was attempted */ public synchronized boolean renewMultiProxyVoteCapabilityIfDue() { + return renewMultiProxyVoteCapabilityIfDue(VOTE_CAPABILITY_RENEWAL_MIN_INTERVAL_MILLIS); + } + + /** + * Re-advertises while an initial discovery deadline is still live. The shorter + * cadence guarantees a settling interval before the durable deadline expires; + * callers must not invoke this after the discovery query says the window ended. + */ + public synchronized boolean renewMultiProxyVoteCapabilityDiscoveryIfDue() { + return renewMultiProxyVoteCapabilityIfDue(VOTE_CAPABILITY_DISCOVERY_RENEWAL_MIN_INTERVAL_MILLIS); + } + + private boolean renewMultiProxyVoteCapabilityIfDue(long minimumIntervalMillis) { long now = capabilityNowMillis(); - if (lastVoteCapabilityAdvertisementMillis != Long.MIN_VALUE + if (!voteCapabilityDiscoveryAnnouncementRequired && lastVoteCapabilityAdvertisementMillis != Long.MIN_VALUE && now >= lastVoteCapabilityAdvertisementMillis - && now - lastVoteCapabilityAdvertisementMillis < VOTE_CAPABILITY_RENEWAL_MIN_INTERVAL_MILLIS) { + && now - lastVoteCapabilityAdvertisementMillis < minimumIntervalMillis) { return false; } lastVoteCapabilityAdvertisementMillis = now; + voteCapabilityDiscoveryAnnouncementRequired = false; sendMultiProxyEnvelopeAccepted(VotingPluginWire.multiProxyCapabilities(getMultiProxyServerName(), 1)); return true; } @@ -536,7 +582,9 @@ private boolean acceptVoteCapabilityPeer(String normalized) { updated.add(normalized); Map updatedDeadlines = new HashMap<>(voteCapabilityDiscoveryDeadlines); updatedDeadlines.remove(normalized); - return saveVoteCapabilityState(updated, updatedDeadlines, lastVoteCapabilityObservationMillis); + boolean accepted = saveVoteCapabilityState(updated, updatedDeadlines, lastVoteCapabilityObservationMillis); + if (accepted) voteCapabilityDiscoveryAnnouncementRequired = false; + return accepted; } private long discoveryDeadline(long now) { @@ -583,6 +631,7 @@ public synchronized void loadMultiProxySupport() { voteCapabilityDiscoveryDeadlines.clear(); voteCapabilityRecoveryBlocked = false; lastVoteCapabilityAdvertisementMillis = Long.MIN_VALUE; + voteCapabilityDiscoveryAnnouncementRequired = false; lastVoteCapabilityObservationMillis = 0L; restoreVoteCapabilityPeers(); if (!getMultiProxySupportEnabled()) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java index 86e908d22..d3eb66581 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java @@ -361,6 +361,51 @@ void missedCapabilityAnnouncementFallsBackOnlyAfterItsBoundedPersistedWindow(@Te assertTrue(afterRestart.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery().isEmpty()); } + @Test + void expiredDiscoveryPersistsItsFinalObservationOnlyOnce(@TempDir Path dataDirectory) { + MultiProxyHandler handler = capabilityHandler(dataDirectory, "Legacy"); + org.mockito.Mockito.when(handler.capabilityNowMillis()).thenReturn(1_000L, + 1_000L + MultiProxyHandler.VOTE_CAPABILITY_DISCOVERY_WINDOW_MILLIS, + 1_000L + MultiProxyHandler.VOTE_CAPABILITY_DISCOVERY_WINDOW_MILLIS + 1L); + + assertEquals(java.util.Set.of("Legacy"), + handler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery()); + assertTrue(handler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery().isEmpty()); + assertEquals(1_000L + MultiProxyHandler.VOTE_CAPABILITY_DISCOVERY_WINDOW_MILLIS, + assertDoesNotThrow(() -> MultiProxyCapabilityStore.load(dataDirectory)).lastObservedMillis()); + + assertTrue(handler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery().isEmpty()); + assertEquals(1_000L + MultiProxyHandler.VOTE_CAPABILITY_DISCOVERY_WINDOW_MILLIS, + assertDoesNotThrow(() -> MultiProxyCapabilityStore.load(dataDirectory)).lastObservedMillis()); + } + + @Test + void discoveryRenewsBeforeItsBoundedFallbackInsteadOfAtExpiry() throws Exception { + MultiProxyHandler handler = mock(MultiProxyHandler.class, + org.mockito.Mockito.withSettings().useConstructor().defaultAnswer(org.mockito.Mockito.CALLS_REAL_METHODS)); + org.mockito.Mockito.when(handler.getMultiProxyMethod()).thenReturn(MultiProxyMethod.SOCKETS); + org.mockito.Mockito.when(handler.getMultiProxyServers()).thenReturn(List.of("Candidate")); + org.mockito.Mockito.when(handler.getMultiProxyServerName()).thenReturn("Primary"); + org.mockito.Mockito.when(handler.sendMultiProxyEnvelopeAccepted(org.mockito.ArgumentMatchers.any())).thenReturn(true); + org.mockito.Mockito.when(handler.capabilityNowMillis()).thenReturn(1_000L, 1_000L, + 11_000L, 11_000L, 21_000L, 21_000L, 30_000L, 31_000L); + org.mockito.Mockito.clearInvocations(handler); + + assertEquals(java.util.Set.of("Candidate"), handler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery()); + assertTrue(handler.renewMultiProxyVoteCapabilityDiscoveryIfDue()); + assertEquals(java.util.Set.of("Candidate"), handler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery()); + assertTrue(handler.renewMultiProxyVoteCapabilityDiscoveryIfDue()); + assertEquals(java.util.Set.of("Candidate"), handler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery()); + assertTrue(handler.renewMultiProxyVoteCapabilityDiscoveryIfDue()); + // This reply is asynchronous from the final early advertisement, but it is + // still accepted before the durable fallback boundary. + handleCapability(handler, "Candidate"); + assertTrue(handler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery().isEmpty()); + assertEquals(java.util.Set.of("Candidate"), handler.getMultiProxyVoteRecipients()); + org.mockito.Mockito.verify(handler, org.mockito.Mockito.times(3)) + .sendMultiProxyEnvelopeAccepted(org.mockito.ArgumentMatchers.any()); + } + @Test void clockRollbackExpiresDiscoveryInsteadOfExtendingItsPersistedWindow(@TempDir Path dataDirectory) { MultiProxyHandler handler = capabilityHandler(dataDirectory, "Candidate"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 0e13394ee..c2f3c27ad 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -553,6 +553,9 @@ void capabilityDiscoveryDefersVoteUntilExpiryThenSendsOneLegacyCopy() throws Exc "legacy recipients must be durably cleared before the non-deduplicating send"); return true; }); + java.util.concurrent.ScheduledExecutorService scheduler = Mockito + .mock(java.util.concurrent.ScheduledExecutorService.class); + votingPluginProxy.setSchedulerForTest(scheduler); VotingPluginProxyTestImpl proxy = Mockito.spy(votingPluginProxy); Mockito.doReturn(voteCache).when(proxy).getVoteCacheHandler(); @@ -578,6 +581,8 @@ void capabilityDiscoveryDefersVoteUntilExpiryThenSendsOneLegacyCopy() throws Exc assertEquals(1, queue.size()); assertTrue(queue.element().isMultiProxyCapabilityDiscoveryPending()); assertTrue(queue.element().isProcessed()); + verify(scheduler).schedule(Mockito.any(Runnable.class), Mockito.eq(5L), + Mockito.eq(java.util.concurrent.TimeUnit.SECONDS)); assertTrue((Boolean) begin.invoke(proxy, retry, queue.element(), "Player", "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); @@ -590,7 +595,7 @@ void capabilityDiscoveryDefersVoteUntilExpiryThenSendsOneLegacyCopy() throws Exc "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); verify(multiProxyHandler, Mockito.times(1)).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.eq(java.util.Set.of("Candidate"))); - verify(multiProxyHandler, Mockito.times(4)).renewMultiProxyVoteCapabilityIfDue(); + verify(multiProxyHandler).renewMultiProxyVoteCapabilityDiscoveryIfDue(); } @Test @@ -684,6 +689,52 @@ void expiredCapableMultiProxyPeerStaysInDurableOutboxUntilRenewal() throws Excep verify(multiProxyHandler, Mockito.never()).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any()); } + @Test + void deferredDurableMultiProxyLeaseRetrySchedulesThePersistedOutbox() throws Exception { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + java.util.Queue queue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + Mockito.when(voteCache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(voteCache.addTimeVoteToCache(Mockito.any())).thenAnswer(invocation -> { + queue.add(invocation.getArgument(0)); + return true; + }); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Collections.emptySet()); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityRenewal()) + .thenReturn(java.util.Set.of("Capable")); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipientsAwaitingCapabilityDiscovery()) + .thenReturn(java.util.Collections.emptySet()); + Mockito.when(multiProxyHandler.getConfiguredMultiProxyVoteRecipients()) + .thenReturn(new java.util.LinkedHashSet<>(java.util.Set.of("Capable"))); + Mockito.when(votingPluginProxy.getConfig().getProxyServerName()).thenReturn("Proxy1"); + java.util.concurrent.ScheduledExecutorService scheduler = Mockito + .mock(java.util.concurrent.ScheduledExecutorService.class); + votingPluginProxy.setSchedulerForTest(scheduler); + VotingPluginProxyTestImpl proxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(proxy).getVoteCacheHandler(); + + Class retryType = Class.forName("com.bencodez.votingplugin.proxy.VotingPluginProxy$LiveVoteRetryState"); + java.lang.reflect.Constructor constructor = retryType.getDeclaredConstructor(); + constructor.setAccessible(true); + Object retry = constructor.newInstance(); + java.util.UUID voteId = java.util.UUID.randomUUID(); + java.lang.reflect.Field retriesField = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); + retriesField.setAccessible(true); + @SuppressWarnings("unchecked") + java.util.Map retries = + (java.util.Map) retriesField.get(proxy); + retries.put(voteId, retry); + java.lang.reflect.Method begin = VotingPluginProxy.class.getDeclaredMethod("beginMultiProxyForwarding", + retryType, VoteTimeQueue.class, String.class, String.class, String.class, long.class, + boolean.class, com.bencodez.votingplugin.proxy.VoteTotalsSnapshot.class); + begin.setAccessible(true); + + assertFalse((Boolean) begin.invoke(proxy, retry, null, "Player", + "00000000-0000-0000-0000-000000000001", "Service", 100L, true, null)); + assertEquals(1, queue.size()); + verify(scheduler).schedule(Mockito.any(Runnable.class), Mockito.eq(5L), + Mockito.eq(java.util.concurrent.TimeUnit.SECONDS)); + } + @Test void persistedOutboxWaitsForFreshCapabilityLeaseAfterRestart() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); From 15eea849c33100033287bddb2b5fed92b2fbc6fb Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 15 Sep 2026 18:07:47 -0600 Subject: [PATCH 111/136] Cover durable proxy lease retry --- .../tests/VotingPluginProxyTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index c2f3c27ad..dd89bc90f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -765,6 +765,28 @@ void persistedOutboxWaitsForFreshCapabilityLeaseAfterRestart() { assertTrue(queue.contains(outbox)); } + @Test + void deferredPersistedDurableOutboxSchedulesItsOwnLeaseRetry() throws Exception { + VoteTimeQueue outbox = new VoteTimeQueue(java.util.UUID.randomUUID(), "Player", "Service", 100L, false, + java.util.Collections.emptySet(), java.util.Collections.emptySet(), "totals", true, + "00000000-0000-0000-0000-000000000001"); + outbox.requireMultiProxyAcknowledgements("Proxy1", java.util.Set.of("Capable")); + Mockito.when(multiProxyHandler.getMultiProxyVoteRecipients()).thenReturn(java.util.Collections.emptySet()); + java.util.concurrent.ScheduledExecutorService scheduler = Mockito + .mock(java.util.concurrent.ScheduledExecutorService.class); + votingPluginProxy.setSchedulerForTest(scheduler); + java.lang.reflect.Method retry = VotingPluginProxy.class.getDeclaredMethod("retryDurableMultiProxyOutbox", + VoteTimeQueue.class); + retry.setAccessible(true); + + assertFalse((Boolean) retry.invoke(votingPluginProxy, outbox)); + + verify(multiProxyHandler).renewMultiProxyVoteCapabilityIfDue(); + verify(multiProxyHandler, Mockito.never()).sendMultiProxyEnvelopeAccepted(Mockito.any(), Mockito.any()); + verify(scheduler).schedule(Mockito.any(Runnable.class), Mockito.eq(5L), + Mockito.eq(java.util.concurrent.TimeUnit.SECONDS)); + } + @Test void restartResumesDurableDiscoveryOutboxBeforePublishingIt() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); From 72d5183de1837082dd30173bbd203e45d9189d4c Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 15 Sep 2026 18:27:30 -0600 Subject: [PATCH 112/136] Recover interrupted plugin activation --- .../control/PluginDeploymentService.java | 23 ++++++++++++++++--- .../control/PluginDeploymentServiceTest.java | 22 ++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java index 4e58ff14c..3a7d74923 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java @@ -159,7 +159,7 @@ public Result deploy(Task task, URI endpoint, String nodeId, UUID sessionId, Str /** Package-visible for deterministic artifact-validation tests. */ Result stage(Task task, InputStream body, BooleanSupplier active) throws IOException { validate(task); - if (alreadyStaged(task)) return Result.restartRequired(); + if (alreadyStaged(task) || recoverInterruptedActivation(task)) return Result.restartRequired(); activeResponse.compareAndSet(null, body); Path temporary = Files.createTempFile(root, target.getFileName().toString() + ".", ".download"); Activation activation = null; @@ -357,11 +357,28 @@ private void writeMarker(Task task) throws IOException { private boolean alreadyStaged(Task task) throws IOException { if (!Files.isRegularFile(marker, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(marker) - || Files.size(marker) > 256 || !Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) - || Files.isSymbolicLink(target) || Files.size(target) != task.size()) return false; + || Files.size(marker) > 256) return false; String[] fields = Files.readString(marker, StandardCharsets.US_ASCII).split("\\R", -1); if (fields.length < 3 || !task.deploymentId().toString().equals(fields[0]) || !task.sha256().equals(fields[1]) || !Long.toString(task.size()).equals(fields[2])) return false; + return targetMatches(task); + } + + /** Complete the durable marker half of an activation interrupted after publish. */ + private boolean recoverInterruptedActivation(Task task) throws IOException { + // Proxy replacement creates a durable backup before publishing the target; + // that transaction shape lets a retry distinguish the activation crash window. + // Backend update folders have no such evidence and must still consume/verify + // the newly supplied body when a different deployment id is requested. + if (!replaceExisting) return false; + if (!targetMatches(task)) return false; + writeMarker(task); + return true; + } + + private boolean targetMatches(Task task) throws IOException { + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target) + || Files.size(target) != task.size()) return false; MessageDigest digest = sha256(); try (InputStream input = Files.newInputStream(target, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)) { byte[] bytes = new byte[BUFFER_BYTES]; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java index e04f5adfb..f7b6008a7 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java @@ -90,6 +90,28 @@ class PluginDeploymentServiceTest { assertArrayEquals(second, Files.readAllBytes(current)); } + @Test void proxyRecoversActivationInterruptedBeforeMarkerWithoutReplacingBackup() throws Exception { + Path current = directory.resolve("VotingPlugin.jar"); + Path backup = directory.resolve("VotingPlugin.jar.control-backup"); + Path marker = directory.resolve("VotingPlugin.jar.control-deployment"); + byte[] original = jar("name: VotingPlugin\nversion: original\n"); + byte[] candidate = jar("name: VotingPlugin\nversion: candidate\n"); + PluginDeploymentService.Task task = task(candidate); + Files.write(current, candidate); + Files.write(backup, original); + PluginDeploymentService service = PluginDeploymentService.proxy(current); + + assertEquals("RESTART_REQUIRED", + service.stage(task, new ByteArrayInputStream(new byte[0]), () -> true).code()); + + assertArrayEquals(candidate, Files.readAllBytes(current)); + assertArrayEquals(original, Files.readAllBytes(backup)); + String state = Files.readString(marker, StandardCharsets.US_ASCII); + assertTrue(state.contains(task.deploymentId().toString())); + assertTrue(state.contains(task.sha256())); + assertTrue(state.contains(Long.toString(task.size()))); + } + @Test void cancellationDuringCopyDoesNotPublish() throws Exception { byte[] artifact = jar("name: VotingPlugin\n"); PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); From b3cac6f857858721a673f1ec77a43e334e810304 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 18:00:06 -0600 Subject: [PATCH 113/136] Add verified VotingPlugin staging service --- .../control/PluginDeploymentService.java | 432 ++++++++++++++++++ 1 file changed, 432 insertions(+) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java new file mode 100644 index 000000000..3a7d74923 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java @@ -0,0 +1,432 @@ +package com.bencodez.votingplugin.control; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.charset.CodingErrorAction; +import java.nio.channels.FileChannel; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.time.Duration; +import java.util.HexFormat; +import java.util.Locale; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +/** + * Bounded, pull-only staging for the optional {@code plugin.deploy.v1} capability. + * It deliberately never reloads a plugin or starts a server. The small on-disk + * marker makes a leased task idempotent after a lost result acknowledgement. + */ +public final class PluginDeploymentService { + public static final String CAPABILITY = "plugin.deploy.v1"; + public static final long MAX_ARTIFACT_BYTES = 64L * 1024L * 1024L; + private static final int BUFFER_BYTES = 32 * 1024; + private static final int MAX_JAR_ENTRIES = 10_000; + private static final long MAX_INSPECTED_UNCOMPRESSED_BYTES = 256L * 1024L * 1024L; + private static final int MAX_PLUGIN_YML_BYTES = 64 * 1024; + private static final String MARKER = ".control-deployment"; + private static final ScheduledExecutorService BODY_DEADLINE_EXECUTOR = Executors.newSingleThreadScheduledExecutor(r -> { + Thread thread = new Thread(r, "VotingPlugin-control-artifact-deadline"); + thread.setDaemon(true); + return thread; + }); + + private final Path target; + private final Path root; + private final Path marker; + private final boolean replaceExisting; + private final AtomicBoolean staging = new AtomicBoolean(); + private final AtomicReference activeResponse = new AtomicReference<>(); + + private PluginDeploymentService(Path target, boolean replaceExisting) throws IOException { + this.target = target.toAbsolutePath().normalize(); + this.replaceExisting = replaceExisting; + Path parent = this.target.getParent(); + if (parent == null) throw new IOException("deployment target has no parent"); + Files.createDirectories(parent); + this.root = parent.toRealPath(LinkOption.NOFOLLOW_LINKS); + if (!Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(root)) { + throw new IOException("deployment staging directory is unsafe"); + } + if (!this.target.startsWith(root) || Files.isSymbolicLink(this.target)) { + throw new IOException("deployment target escapes staging directory"); + } + if (Files.exists(this.target, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(this.target, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("deployment target is not a regular file"); + } + this.marker = root.resolve(this.target.getFileName() + MARKER); + } + + /** Bukkit's update folder preserves the currently loaded plugin JAR. */ + public static PluginDeploymentService backend(Path updateDirectory) throws IOException { + if (updateDirectory == null) throw new IOException("Bukkit update folder is unavailable"); + return new PluginDeploymentService(updateDirectory.resolve("VotingPlugin.jar"), false); + } + + /** Proxies atomically replace their discovered plugin JAR only after a durable backup. */ + public static PluginDeploymentService proxy(Path currentPluginJar) throws IOException { + if (currentPluginJar == null || !currentPluginJar.getFileName().toString().endsWith(".jar")) { + throw new IOException("proxy plugin JAR is unavailable"); + } + if (!Files.isRegularFile(currentPluginJar, LinkOption.NOFOLLOW_LINKS) + || Files.isSymbolicLink(currentPluginJar)) throw new IOException("proxy plugin JAR is unsafe"); + return new PluginDeploymentService(currentPluginJar, true); + } + + public boolean isStaging() { return staging.get(); } + + /** Unblocks an in-progress response read during connector shutdown. */ + public void cancel() { + InputStream response = activeResponse.getAndSet(null); + if (response != null) { + try { response.close(); } catch (IOException ignored) { /* Shutdown is already in progress. */ } + } + } + + public Result deploy(Task task, URI endpoint, String nodeId, UUID sessionId, String credential, + HttpClient http, Duration timeout, BooleanSupplier active) { + if (!staging.compareAndSet(false, true)) return Result.failure("DEPLOYMENT_FAILED", "Another deployment is still staging"); + try { + validate(task); + if (alreadyStaged(task)) return Result.restartRequired(); + if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before download"); + URI artifact = endpoint.resolve("/api/v1/nodes/" + nodeId + "/deployments/" + task.deploymentId() + + "/artifact"); + HttpRequest request = HttpRequest.newBuilder(artifact).timeout(timeout) + .header("Authorization", "Bearer " + credential) + .header("X-Node-Session", sessionId.toString()) + .header("X-Deployment-Attempt", task.attemptId().toString()).GET().build(); + HttpResponse response = http.send(request, HttpResponse.BodyHandlers.ofInputStream()); + InputStream body = response.body(); + activeResponse.set(body); + ScheduledFuture bodyDeadline = BODY_DEADLINE_EXECUTOR.schedule(() -> { + try { body.close(); } catch (IOException ignored) { /* The staging operation reports the timeout. */ } + }, timeout.toMillis(), TimeUnit.MILLISECONDS); + try { + if (response.statusCode() != 200) { + body.close(); + return Result.failure("DOWNLOAD_FAILED", "Artifact download was rejected"); + } + String contentLength = response.headers().firstValue("Content-Length").orElse(null); + if (contentLength != null && (!contentLength.matches("[0-9]{1,9}") + || Long.parseLong(contentLength) != task.size())) { + body.close(); + return Result.failure("SIZE_MISMATCH", "Artifact size did not match the deployment task"); + } + if (!active.getAsBoolean()) { + body.close(); + return Result.failure("CANCELLED", "Deployment was cancelled before download"); + } + try (body) { + return stage(task, body, active); + } catch (IOException failure) { + if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging"); + return Result.failure("STAGING_FAILED", "Artifact could not be verified or staged on this node"); + } + } finally { + bodyDeadline.cancel(false); + activeResponse.compareAndSet(body, null); + } + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + return Result.failure("CANCELLED", "Deployment download was interrupted"); + } catch (IOException | RuntimeException failure) { + return Result.failure("DOWNLOAD_FAILED", "Artifact download failed"); + } finally { + staging.set(false); + } + } + + /** Package-visible for deterministic artifact-validation tests. */ + Result stage(Task task, InputStream body, BooleanSupplier active) throws IOException { + validate(task); + if (alreadyStaged(task) || recoverInterruptedActivation(task)) return Result.restartRequired(); + activeResponse.compareAndSet(null, body); + Path temporary = Files.createTempFile(root, target.getFileName().toString() + ".", ".download"); + Activation activation = null; + try { + MessageDigest digest = sha256(); + long written = copyExact(body, temporary, task.size(), digest, active); + force(temporary); + if (written != task.size()) return Result.failure("SIZE_MISMATCH", "Artifact size did not match the deployment task"); + if (!HexFormat.of().formatHex(digest.digest()).equals(task.sha256())) { + return Result.failure("HASH_MISMATCH", "Artifact digest did not match the deployment task"); + } + inspectJar(temporary); + if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging"); + activation = new Activation(); + activate(temporary, activation); + writeMarker(task); + activation.discard(); + return Result.restartRequired(); + } catch (CancelledDeploymentException failure) { + return Result.failure("CANCELLED", "Deployment was cancelled before staging"); + } catch (InvalidArtifactException failure) { + return Result.failure("INVALID_ARTIFACT", "Artifact is not a bounded VotingPlugin JAR"); + } catch (IOException failure) { + if (activation != null) { + try { activation.rollback(); } + catch (IOException rollbackFailure) { failure.addSuppressed(rollbackFailure); } + } + if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging"); + throw failure; + } finally { + activeResponse.compareAndSet(body, null); + Files.deleteIfExists(temporary); + } + } + + private long copyExact(InputStream input, Path temporary, long expected, MessageDigest digest, + BooleanSupplier active) throws IOException { + long total = 0; + byte[] buffer = new byte[BUFFER_BYTES]; + try (var output = Files.newOutputStream(temporary, StandardOpenOption.TRUNCATE_EXISTING)) { + for (;;) { + if (!active.getAsBoolean()) throw new CancelledDeploymentException(); + int read = input.read(buffer); + if (read == -1) break; + total += read; + if (total > expected || total > MAX_ARTIFACT_BYTES) throw new InvalidArtifactException(); + digest.update(buffer, 0, read); + output.write(buffer, 0, read); + } + } + return total; + } + + private void inspectJar(Path artifact) throws IOException { + try (JarFile jar = new JarFile(artifact.toFile(), false)) { + int entries = 0; + long uncompressed = 0; + JarEntry pluginYml = null; + var iterator = jar.entries(); + while (iterator.hasMoreElements()) { + JarEntry entry = iterator.nextElement(); + if (++entries > MAX_JAR_ENTRIES || unsafeEntry(entry.getName())) throw new InvalidArtifactException(); + long size = entry.getSize(); + long compressed = entry.getCompressedSize(); + if (size < 0 || compressed < 0 || compressed > 0 && size > compressed * 200L + || (uncompressed += size) > MAX_INSPECTED_UNCOMPRESSED_BYTES) { + throw new InvalidArtifactException(); + } + if ("plugin.yml".equals(entry.getName())) { + if (pluginYml != null || size > MAX_PLUGIN_YML_BYTES) throw new InvalidArtifactException(); + pluginYml = entry; + } + } + if (pluginYml == null || !isVotingPluginYml(jar, pluginYml)) throw new InvalidArtifactException(); + } catch (java.util.zip.ZipException failure) { + throw new InvalidArtifactException(); + } + } + + private static boolean unsafeEntry(String name) { + if (name == null || name.isEmpty() || name.length() > 512 || name.startsWith("/") + || name.startsWith("\\") || name.contains("\\") || name.indexOf('\0') >= 0) return true; + for (String component : name.split("/", -1)) { + if (component.equals(".") || component.equals("..")) return true; + } + return false; + } + + private static boolean isVotingPluginYml(JarFile jar, JarEntry entry) throws IOException { + byte[] bytes; + try (InputStream input = jar.getInputStream(entry)) { bytes = input.readNBytes(MAX_PLUGIN_YML_BYTES + 1); } + if (bytes.length > MAX_PLUGIN_YML_BYTES) return false; + String yml; + try { + yml = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT).decode(java.nio.ByteBuffer.wrap(bytes)).toString(); + } catch (java.nio.charset.CharacterCodingException failure) { + return false; + } + boolean found = false; + for (String line : yml.split("\\R")) { + if (!line.startsWith("name:")) continue; + if (found) return false; + found = true; + String value = line.substring(5).trim(); + int comment = value.indexOf('#'); + if (comment >= 0) value = value.substring(0, comment).trim(); + if (!"VotingPlugin".equals(unquote(value))) return false; + } + return found; + } + + private static String unquote(String value) { + return value.length() >= 2 && ((value.startsWith("\"") && value.endsWith("\"")) + || (value.startsWith("'") && value.endsWith("'"))) ? value.substring(1, value.length() - 1) : value; + } + + private void activate(Path temporary, Activation activation) throws IOException { + if (replaceExisting) { + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) { + throw new IOException("current proxy plugin JAR is unsafe"); + } + Path backup = root.resolve(target.getFileName() + ".control-backup"); + if (Files.exists(backup, LinkOption.NOFOLLOW_LINKS) + && (!Files.isRegularFile(backup, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(backup))) { + throw new IOException("proxy plugin backup is unsafe"); + } + Path backupTemp = Files.createTempFile(root, target.getFileName().toString() + ".", ".backup"); + try { + Files.copy(target, backupTemp, StandardCopyOption.REPLACE_EXISTING); + force(backupTemp); + move(backupTemp, backup); + forceDirectory(root); + } finally { Files.deleteIfExists(backupTemp); } + } + move(temporary, target); + activation.published = true; + force(target); + forceDirectory(root); + } + + /** Holds a private copy until the durable idempotency marker has been published. */ + private final class Activation { + private final Path previous; + private boolean published; + + private Activation() throws IOException { + if (!Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + previous = null; + return; + } + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) { + throw new IOException("deployment target is unsafe"); + } + previous = Files.createTempFile(root, target.getFileName().toString() + ".", ".rollback"); + Files.copy(target, previous, StandardCopyOption.REPLACE_EXISTING); + force(previous); + } + + private void rollback() throws IOException { + if (!published) { + discard(); + return; + } + if (previous == null) { + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) { + throw new IOException("deployed target cannot be safely removed"); + } + Files.delete(target); + } else { + move(previous, target); + force(target); + } + forceDirectory(root); + discard(); + } + + private void discard() { + if (previous == null) return; + try { Files.deleteIfExists(previous); } + catch (IOException ignored) { /* A private stale rollback copy is safer than a false deployment result. */ } + } + } + + private void writeMarker(Task task) throws IOException { + Path temporary = Files.createTempFile(root, target.getFileName().toString() + ".", ".marker"); + try { + Files.writeString(temporary, task.deploymentId() + "\n" + task.sha256() + "\n" + task.size() + "\n", + StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); + force(temporary); + move(temporary, marker); + forceDirectory(root); + } finally { Files.deleteIfExists(temporary); } + } + + private boolean alreadyStaged(Task task) throws IOException { + if (!Files.isRegularFile(marker, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(marker) + || Files.size(marker) > 256) return false; + String[] fields = Files.readString(marker, StandardCharsets.US_ASCII).split("\\R", -1); + if (fields.length < 3 || !task.deploymentId().toString().equals(fields[0]) + || !task.sha256().equals(fields[1]) || !Long.toString(task.size()).equals(fields[2])) return false; + return targetMatches(task); + } + + /** Complete the durable marker half of an activation interrupted after publish. */ + private boolean recoverInterruptedActivation(Task task) throws IOException { + // Proxy replacement creates a durable backup before publishing the target; + // that transaction shape lets a retry distinguish the activation crash window. + // Backend update folders have no such evidence and must still consume/verify + // the newly supplied body when a different deployment id is requested. + if (!replaceExisting) return false; + if (!targetMatches(task)) return false; + writeMarker(task); + return true; + } + + private boolean targetMatches(Task task) throws IOException { + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target) + || Files.size(target) != task.size()) return false; + MessageDigest digest = sha256(); + try (InputStream input = Files.newInputStream(target, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)) { + byte[] bytes = new byte[BUFFER_BYTES]; + for (int read; (read = input.read(bytes)) != -1;) digest.update(bytes, 0, read); + } + if (!task.sha256().equals(HexFormat.of().formatHex(digest.digest()))) return false; + try { + inspectJar(target); + return true; + } catch (InvalidArtifactException invalid) { + return false; + } + } + + private static void move(Path source, Path destination) throws IOException { + try { Files.move(source, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (AtomicMoveNotSupportedException ignored) { Files.move(source, destination, StandardCopyOption.REPLACE_EXISTING); } + } + + private static void force(Path file) throws IOException { + try (FileChannel channel = FileChannel.open(file, StandardOpenOption.WRITE)) { channel.force(true); } + } + + private static void forceDirectory(Path directory) throws IOException { + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { channel.force(true); } + catch (UnsupportedOperationException ignored) { /* Some supported filesystems cannot fsync directories. */ } + } + + private static MessageDigest sha256() { + try { return MessageDigest.getInstance("SHA-256"); } + catch (java.security.NoSuchAlgorithmException failure) { throw new IllegalStateException("SHA-256 is unavailable", failure); } + } + + private static void validate(Task task) { + if (task == null || task.deploymentId() == null || task.attemptId() == null || task.artifactId() == null + || !task.artifactId().matches("[A-Za-z0-9][A-Za-z0-9._-]{0,127}") || task.sha256() == null + || !task.sha256().matches("[0-9a-fA-F]{64}") || task.size() < 1 || task.size() > MAX_ARTIFACT_BYTES) { + throw new IllegalArgumentException("deployment task is invalid"); + } + } + + public record Task(UUID deploymentId, String artifactId, String sha256, long size, UUID attemptId) { + public Task { if (sha256 != null) sha256 = sha256.toLowerCase(Locale.ROOT); } + } + public record Result(boolean success, String code, String message) { + static Result restartRequired() { return new Result(true, "RESTART_REQUIRED", "Plugin update staged; restart is required"); } + static Result failure(String code, String message) { return new Result(false, code, message); } + } + private static final class InvalidArtifactException extends IOException { private static final long serialVersionUID = 1L; } + private static final class CancelledDeploymentException extends IOException { private static final long serialVersionUID = 1L; } +} From dec7cd1604d573d0101990a1fdf5097c773bbfd7 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 18:00:09 -0600 Subject: [PATCH 114/136] Test verified VotingPlugin staging --- .../control/PluginDeploymentServiceTest.java | 256 ++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java new file mode 100644 index 000000000..f7b6008a7 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java @@ -0,0 +1,256 @@ +package com.bencodez.votingplugin.control; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HexFormat; +import java.util.UUID; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class PluginDeploymentServiceTest { + @TempDir Path directory; + + @Test void backendStagesOnlyAnExactVerifiedVotingPluginJarAndIsIdempotent() throws Exception { + byte[] artifact = jar("name: VotingPlugin\nmain: example.Main\n"); + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + PluginDeploymentService.Task task = task(artifact); + + PluginDeploymentService.Result staged = service.stage(task, new ByteArrayInputStream(artifact), () -> true); + + assertTrue(staged.success()); + assertEquals("RESTART_REQUIRED", staged.code()); + assertEquals(artifact.length, Files.size(directory.resolve("update/VotingPlugin.jar"))); + assertTrue(service.stage(task, new ByteArrayInputStream(new byte[0]), () -> true).success()); + } + + @Test void backendIgnoresMatchingMarkerOnlyWhenTargetIsValidThenRestagesWhenCorrupted() throws Exception { + byte[] artifact = jar("name: VotingPlugin\n"); + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + PluginDeploymentService.Task task = task(artifact); + + service.stage(task, new ByteArrayInputStream(artifact), () -> true); + assertArrayEquals(artifact, Files.readAllBytes(directory.resolve("update/VotingPlugin.jar"))); + + byte[] corrupted = new byte[artifact.length]; + Arrays.fill(corrupted, (byte) 0x42); + Files.write(directory.resolve("update/VotingPlugin.jar"), corrupted); + assertArrayEquals(corrupted, Files.readAllBytes(directory.resolve("update/VotingPlugin.jar"))); + + assertTrue(service.stage(task, new ByteArrayInputStream(artifact), () -> true).success()); + assertArrayEquals(artifact, Files.readAllBytes(directory.resolve("update/VotingPlugin.jar"))); + } + + @Test void invalidPluginIdentityAndDigestNeverReachTheUpdateFolder() throws Exception { + byte[] wrongPlugin = jar("name: NotVotingPlugin\n"); + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + PluginDeploymentService.Result invalid = service.stage(task(wrongPlugin), new ByteArrayInputStream(wrongPlugin), () -> true); + assertFalse(invalid.success()); + assertEquals("INVALID_ARTIFACT", invalid.code()); + assertFalse(Files.exists(directory.resolve("update/VotingPlugin.jar"))); + + byte[] good = jar("name: VotingPlugin\n"); + PluginDeploymentService.Task mismatch = new PluginDeploymentService.Task(UUID.randomUUID(), "vp.jar", + "0".repeat(64), good.length, UUID.randomUUID()); + assertEquals("HASH_MISMATCH", service.stage(mismatch, new ByteArrayInputStream(good), () -> true).code()); + } + + @Test void proxyKeepsTheCurrentJarAsABackupBeforeReplacement() throws Exception { + Path current = directory.resolve("VotingPlugin.jar"); + byte[] original = jar("name: VotingPlugin\nversion: 0\n"); + byte[] first = jar("name: VotingPlugin\nversion: 1\n"); + byte[] second = jar("name: VotingPlugin\nversion: 2\n"); + Files.write(current, original); + PluginDeploymentService service = PluginDeploymentService.proxy(current); + + assertEquals("RESTART_REQUIRED", service.stage(task(first), new ByteArrayInputStream(first), () -> true).code()); + assertEquals("RESTART_REQUIRED", service.stage(task(second), new ByteArrayInputStream(second), () -> true).code()); + assertArrayEquals(first, Files.readAllBytes(directory.resolve("VotingPlugin.jar.control-backup"))); + assertArrayEquals(second, Files.readAllBytes(current)); + } + + @Test void proxyRecoversActivationInterruptedBeforeMarkerWithoutReplacingBackup() throws Exception { + Path current = directory.resolve("VotingPlugin.jar"); + Path backup = directory.resolve("VotingPlugin.jar.control-backup"); + Path marker = directory.resolve("VotingPlugin.jar.control-deployment"); + byte[] original = jar("name: VotingPlugin\nversion: original\n"); + byte[] candidate = jar("name: VotingPlugin\nversion: candidate\n"); + PluginDeploymentService.Task task = task(candidate); + Files.write(current, candidate); + Files.write(backup, original); + PluginDeploymentService service = PluginDeploymentService.proxy(current); + + assertEquals("RESTART_REQUIRED", + service.stage(task, new ByteArrayInputStream(new byte[0]), () -> true).code()); + + assertArrayEquals(candidate, Files.readAllBytes(current)); + assertArrayEquals(original, Files.readAllBytes(backup)); + String state = Files.readString(marker, StandardCharsets.US_ASCII); + assertTrue(state.contains(task.deploymentId().toString())); + assertTrue(state.contains(task.sha256())); + assertTrue(state.contains(Long.toString(task.size()))); + } + + @Test void cancellationDuringCopyDoesNotPublish() throws Exception { + byte[] artifact = jar("name: VotingPlugin\n"); + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + PluginDeploymentService.Task task = task(artifact); + + assertEquals("RESTART_REQUIRED", service.stage(task, new ByteArrayInputStream(artifact), () -> true).code()); + String markerBefore = Files.readString(directory.resolve("update/VotingPlugin.jar.control-deployment")); + byte[] targetBefore = Files.readAllBytes(directory.resolve("update/VotingPlugin.jar")); + + AtomicBoolean active = new AtomicBoolean(true); + InputStream body = new CancellingInputStream(new ByteArrayInputStream(artifact), active); + PluginDeploymentService.Task replacement = task(artifact); + assertEquals("CANCELLED", service.stage(replacement, body, active::get).code()); + + assertEquals(markerBefore, Files.readString(directory.resolve("update/VotingPlugin.jar.control-deployment"))); + assertArrayEquals(targetBefore, Files.readAllBytes(directory.resolve("update/VotingPlugin.jar"))); + assertFalse(active.get()); + } + + @Test void markerPublicationFailureRestoresThePreviousArtifact() throws Exception { + Path update = directory.resolve("update"); + Files.createDirectories(update); + byte[] original = jar("name: VotingPlugin\nversion: original\n"); + byte[] candidate = jar("name: VotingPlugin\nversion: candidate\n"); + Path target = update.resolve("VotingPlugin.jar"); + Files.write(target, original); + Path marker = update.resolve("VotingPlugin.jar.control-deployment"); + Files.createDirectory(marker); + Files.writeString(marker.resolve("keep"), "marker publication must fail"); + PluginDeploymentService service = PluginDeploymentService.backend(update); + + assertThrows(IOException.class, () -> service.stage(task(candidate), new ByteArrayInputStream(candidate), () -> true)); + + assertArrayEquals(original, Files.readAllBytes(target)); + } + + @Test void proxyMarkerPublicationFailureRestoresTheCurrentJarAndKeepsItsBackup() throws Exception { + byte[] original = jar("name: VotingPlugin\nversion: original\n"); + byte[] candidate = jar("name: VotingPlugin\nversion: candidate\n"); + Path target = directory.resolve("VotingPlugin.jar"); + Files.write(target, original); + Path marker = directory.resolve("VotingPlugin.jar.control-deployment"); + Files.createDirectory(marker); + Files.writeString(marker.resolve("keep"), "marker publication must fail"); + PluginDeploymentService service = PluginDeploymentService.proxy(target); + + assertThrows(IOException.class, () -> service.stage(task(candidate), new ByteArrayInputStream(candidate), () -> true)); + + assertArrayEquals(original, Files.readAllBytes(target)); + assertArrayEquals(original, Files.readAllBytes(directory.resolve("VotingPlugin.jar.control-backup"))); + } + + @Test void cancellationClosesAStalledArtifactStream() throws Exception { + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + AtomicBoolean active = new AtomicBoolean(true); + BlockingInputStream stalled = new BlockingInputStream(active); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future result = executor.submit(() -> service.stage( + task(new byte[] { 1 }), stalled, active::get)); + assertTrue(stalled.awaitRead()); + service.cancel(); + assertEquals("CANCELLED", result.get(5, TimeUnit.SECONDS).code()); + assertFalse(Files.exists(directory.resolve("update/VotingPlugin.jar"))); + } finally { + executor.shutdownNow(); + } + } + + private static PluginDeploymentService.Task task(byte[] artifact) throws Exception { + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(artifact)); + return new PluginDeploymentService.Task(UUID.randomUUID(), "votingplugin.jar", digest, artifact.length, + UUID.randomUUID()); + } + + private static byte[] jar(String pluginYml) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (JarOutputStream jar = new JarOutputStream(bytes)) { + jar.putNextEntry(new JarEntry("plugin.yml")); + jar.write(pluginYml.getBytes(StandardCharsets.UTF_8)); + jar.closeEntry(); + jar.putNextEntry(new JarEntry("example/Main.class")); + jar.write(new byte[] { 0, 1, 2 }); + jar.closeEntry(); + } + return bytes.toByteArray(); + } + + private static class CancellingInputStream extends InputStream { + private final InputStream delegate; + private final AtomicBoolean active; + private boolean cancelled; + + private CancellingInputStream(InputStream delegate, AtomicBoolean active) { + this.delegate = delegate; + this.active = active; + } + + @Override + public int read(byte[] bytes, int start, int length) throws IOException { + int read = delegate.read(bytes, start, length); + if (!cancelled && read > 0) { + cancelled = true; + active.set(false); + } + return read; + } + + @Override + public int read() throws IOException { return delegate.read(); } + } + + private static class BlockingInputStream extends InputStream { + private final AtomicBoolean active; + private final CountDownLatch reading = new CountDownLatch(1); + private boolean closed; + + private BlockingInputStream(AtomicBoolean active) { this.active = active; } + + private boolean awaitRead() throws InterruptedException { return reading.await(5, TimeUnit.SECONDS); } + + @Override + public synchronized int read(byte[] bytes, int start, int length) throws IOException { + reading.countDown(); + while (!closed) { + try { wait(); } + catch (InterruptedException failure) { Thread.currentThread().interrupt(); throw new IOException(failure); } + } + throw new IOException("stream closed"); + } + + @Override + public int read() throws IOException { return read(new byte[1], 0, 1); } + + @Override + public synchronized void close() { + closed = true; + active.set(false); + notifyAll(); + } + } +} From 157ddde85cb795ec3d96e53ad4b967017362768b Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 18:00:12 -0600 Subject: [PATCH 115/136] Enable verified Control staging on Bukkit nodes --- .../control/BackendControlConnector.java | 92 ++++++++++++++++++- 1 file changed, 90 insertions(+), 2 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 58a5a3209..2512b5418 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -61,6 +61,7 @@ public final class BackendControlConnector implements AutoCloseable { private final HttpClient http; private final BackendConfigurationService configurations; private final ControlInspectionService inspections; + private final PluginDeploymentService deployments; private final UUID sessionId = UUID.randomUUID(); private final Map completed = new LinkedHashMap<>(); private final boolean recovering; @@ -74,12 +75,14 @@ public final class BackendControlConnector implements AutoCloseable { private volatile boolean quickSetupsAccepted; private volatile boolean voteSitesSyncAccepted; private volatile boolean inspectionsAccepted; + private volatile boolean deploymentsAccepted; private volatile int inspectionFailures; private volatile long inspectionRetryAtNanos; private volatile int failures; private volatile ScheduledFuture scheduled; private volatile ScheduledFuture operationPolling; private volatile ScheduledFuture inspectionPolling; + private volatile ScheduledFuture deploymentPolling; private volatile Future activeReload; private volatile CompletableFuture activeOperation; @@ -108,6 +111,13 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set .followRedirects(HttpClient.Redirect.NEVER).build(); configurations = new BackendConfigurationService(plugin.getDataFolder().toPath(), this::reloadConfiguration); inspections = new ControlInspectionService(plugin); + PluginDeploymentService prepared = null; + try { + prepared = PluginDeploymentService.backend(plugin.getServer().getUpdateFolderFile().toPath()); + } catch (Exception failure) { + plugin.getLogger().warning("[Control] Plugin deployment staging is unavailable; capability not advertised"); + } + deployments = prepared; } private void reloadConfiguration(String fileName) throws Exception { @@ -190,6 +200,73 @@ public void start() { OPERATION_POLL_MILLIS, OPERATION_POLL_MILLIS, TimeUnit.MILLISECONDS); inspectionPolling = inspectionExecutor.scheduleWithFixedDelay(this::pollInspections, OPERATION_POLL_MILLIS, OPERATION_POLL_MILLIS, TimeUnit.MILLISECONDS); + if (deployments != null) { + deploymentPolling = executor.scheduleWithFixedDelay(this::pollDeployments, + OPERATION_POLL_MILLIS, OPERATION_POLL_MILLIS, TimeUnit.MILLISECONDS); + } + } + + /** Deployment is a separate leased lane; its large I/O never runs on Bukkit's primary thread. */ + private void pollDeployments() { + if (closed || deployments == null || !registered || failures != 0 || !deploymentsAccepted + || !running.compareAndSet(false, true)) return; + CompletableFuture operation = new CompletableFuture<>(); + synchronized (operationLifecycle) { + if (closed) { + running.set(false); + return; + } + activeOperation = operation; + } + try { + claimAndDeploy(); + } catch (Exception failure) { + registered = false; + failures = Math.min(30, failures + 1); + if (failures == 1 || failures % 10 == 0) { + plugin.getLogger().warning("[Control] Bukkit deployment polling unavailable; VotingPlugin remains active"); + } + ScheduledFuture heartbeat = scheduled; + if (heartbeat != null) heartbeat.cancel(false); + if (!closed) schedule(Math.min(TimeUnit.MINUTES.toMillis(5), + 1000L << Math.min(failures - 1, 8))); + } finally { + operation.complete(null); + synchronized (operationLifecycle) { + if (activeOperation == operation) activeOperation = null; + } + running.set(false); + } + } + + private void claimAndDeploy() throws Exception { + JsonObject body = new JsonObject(); + body.addProperty("sessionId", sessionId.toString()); + Response response = send("POST", "/api/v1/nodes/" + settings.nodeId() + "/deployments", body); + if (response.status() == 204 || closed) return; + JsonObject claimed = requireObject(response, 200); + PluginDeploymentService.Task task = deploymentTask(claimed); + PluginDeploymentService.Result result = deployments.deploy(task, settings.endpoint(), settings.nodeId(), sessionId, + credential, http, Duration.ofMillis(settings.requestTimeoutMillis()), () -> !closed); + if (closed) return; + JsonObject submitted = new JsonObject(); + submitted.addProperty("sessionId", sessionId.toString()); + submitted.addProperty("success", result.success()); + submitted.addProperty("code", result.code()); + submitted.addProperty("message", boundedResultMessage(result.message())); + submitted.addProperty("attemptId", task.attemptId().toString()); + requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/deployments/" + task.deploymentId() + + "/result", submitted), 200); + } + + private static PluginDeploymentService.Task deploymentTask(JsonObject task) { + try { + return new PluginDeploymentService.Task(UUID.fromString(string(task, "deploymentId")), + string(task, "artifactId"), string(task, "sha256"), Long.parseLong(string(task, "size")), + UUID.fromString(string(task, "attemptId"))); + } catch (RuntimeException failure) { + throw new IllegalArgumentException("deployment task is invalid"); + } } /** Polls the separately negotiated read-only lane on the connector worker. */ @@ -308,6 +385,8 @@ private void cycle() { voteSitesSyncAccepted = negotiatedCapability(node, "config.vote-sites-sync.v1", voteSitesSyncAccepted); boolean inspectionsWereAccepted = inspectionsAccepted; inspectionsAccepted = negotiatedCapability(node, "data.inspect.v1", inspectionsAccepted); + deploymentsAccepted = deployments != null + && negotiatedCapability(node, PluginDeploymentService.CAPABILITY, deploymentsAccepted); if (inspectionsAccepted && !inspectionsWereAccepted) { inspectionFailures = 0; inspectionRetryAtNanos = 0; @@ -358,6 +437,7 @@ private JsonObject register() throws Exception { quickSetupsAccepted = false; voteSitesSyncAccepted = false; inspectionsAccepted = false; + deploymentsAccepted = false; JsonObject body = sessionBody(); body.addProperty("nodeId", settings.nodeId()); body.addProperty("displayName", settings.nodeId()); @@ -368,13 +448,13 @@ private JsonObject register() throws Exception { .map(installed -> installed.getDescription().getName()).filter(name -> name != null && !name.isBlank()) .distinct().sorted(String.CASE_INSENSITIVE_ORDER).limit(128).forEach(detectedPlugins::add); body.add("detectedPlugins", detectedPlugins); - addCapabilities(body); + addCapabilities(body, deployments != null); return requireObject(send("POST", "/api/v1/nodes/register", body), 200, 201); } private JsonObject heartbeat() throws Exception { JsonObject body = sessionBody(); - addCapabilities(body); + addCapabilities(body, deployments != null); Response response = send("PUT", "/api/v1/nodes/" + settings.nodeId() + "/heartbeat", body); if (response.status() == 404) { registered = false; @@ -762,8 +842,13 @@ private JsonObject sessionBody() { } static void addCapabilities(JsonObject body) { + addCapabilities(body, false); + } + + static void addCapabilities(JsonObject body, boolean deploymentReady) { JsonArray capabilities = new JsonArray(); CAPABILITIES.stream().sorted().forEach(capabilities::add); + if (deploymentReady) capabilities.add(PluginDeploymentService.CAPABILITY); body.add("capabilities", capabilities); JsonArray required = new JsonArray(); required.add("config.files.v1"); @@ -831,6 +916,9 @@ public void close() { if (polling != null) polling.cancel(false); ScheduledFuture inspection = inspectionPolling; if (inspection != null) inspection.cancel(false); + ScheduledFuture deployment = deploymentPolling; + if (deployment != null) deployment.cancel(false); + if (deployments != null) deployments.cancel(); inspectionExecutor.shutdownNow(); if (reload != null && Bukkit.isPrimaryThread()) reload.cancel(false); awaitShutdown(executor, operation); From c3be44076922f66bed6f5e5f82019360ec1bedad Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 18:00:20 -0600 Subject: [PATCH 116/136] Cover deployment capability advertisement --- .../control/BackendControlConnectorProtocolTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java index c0452d01d..ffb06480b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java @@ -95,6 +95,18 @@ class BackendControlConnectorProtocolTest { .anyMatch(value -> "data.inspect.v1".equals(value.getAsString()))); } + @Test void deploymentCapabilityIsOnlyAddedWhenStagingWasPrepared() { + JsonObject unavailable = new JsonObject(); + BackendControlConnector.addCapabilities(unavailable); + assertFalse(unavailable.getAsJsonArray("capabilities").asList().stream() + .anyMatch(value -> PluginDeploymentService.CAPABILITY.equals(value.getAsString()))); + + JsonObject ready = new JsonObject(); + BackendControlConnector.addCapabilities(ready, true); + assertTrue(ready.getAsJsonArray("capabilities").asList().stream() + .anyMatch(value -> PluginDeploymentService.CAPABILITY.equals(value.getAsString()))); + } + @Test void heartbeatRetainsOmittedCapabilitiesAndHonorsExplicitReplacement() { JsonObject omitted = new JsonObject(); assertTrue(BackendControlConnector.negotiatedCapability(omitted, "config.files.v1", true)); From d89bdf51838390727c15a1820401eac09c06e59a Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 18:00:52 -0600 Subject: [PATCH 117/136] Enable verified Control staging on proxy nodes --- .../proxy/control/ControlConnector.java | 141 +++++++++++++++++- 1 file changed, 140 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index aa1f4fa40..9d24533fe 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; @@ -22,6 +23,8 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadLocalRandom; @@ -33,6 +36,7 @@ import java.util.function.Supplier; import java.util.regex.Pattern; +import com.bencodez.votingplugin.control.PluginDeploymentService; import com.bencodez.votingplugin.proxy.VotingPluginProxy; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; import com.bencodez.votingplugin.proxy.presence.BackendPresenceStatus; @@ -77,6 +81,10 @@ public final class ControlConnector implements AutoCloseable { private final ProxyRoutingConfigurationService configurationService; private final ProxyMethodConfigurationService methodConfigurationService; private final ProxyConfigurationFileService fileConfigurationService; + private final PluginDeploymentService deployments; + private final HttpClient deploymentHttp; + private final String deploymentCredential; + private final ExecutorService deploymentExecutor; private final Function> communicationTest; private final Runnable runtimeReplacement; private final Path dataDirectory; @@ -91,11 +99,13 @@ public final class ControlConnector implements AutoCloseable { private volatile boolean closed; private volatile boolean registered; private volatile boolean configurationAccepted; + private volatile boolean deploymentsAccepted; private volatile Set acceptedCapabilities = Set.of(); private volatile int failures; private volatile long snapshotSequence; private volatile ScheduledFuture scheduled; private volatile ScheduledFuture operationPolling; + private volatile ScheduledFuture deploymentPolling; private volatile CompletableFuture activeRequest; private volatile CompletableFuture activeOperation; private volatile Status status = Status.STARTING; @@ -130,6 +140,19 @@ private ControlConnector(Settings settings, ScheduledExecutorService scheduler, Function> communicationTest, ProxyMethodConfigurationService methodConfigurationService, Runnable runtimeReplacement, ProxyConfigurationFileService fileConfigurationService) { + this(settings, scheduler, transport, snapshotSource, logger, sessionId, jitterSource, configurationService, + dataDirectory, route, recovering, recoveryComplete, communicationTest, methodConfigurationService, + runtimeReplacement, fileConfigurationService, null, null, null); + } + + private ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, + Supplier> snapshotSource, Consumer logger, UUID sessionId, + LongSupplier jitterSource, ProxyRoutingConfigurationService configurationService, Path dataDirectory, + Route route, boolean recovering, Runnable recoveryComplete, + Function> communicationTest, + ProxyMethodConfigurationService methodConfigurationService, Runnable runtimeReplacement, + ProxyConfigurationFileService fileConfigurationService, PluginDeploymentService deployments, + HttpClient deploymentHttp, String deploymentCredential) { this.settings = Objects.requireNonNull(settings, "settings"); this.scheduler = Objects.requireNonNull(scheduler, "scheduler"); this.transport = Objects.requireNonNull(transport, "transport"); @@ -142,6 +165,14 @@ private ControlConnector(Settings settings, ScheduledExecutorService scheduler, this.communicationTest = communicationTest; this.runtimeReplacement = runtimeReplacement; this.fileConfigurationService = fileConfigurationService; + this.deployments = deployments; + this.deploymentHttp = deploymentHttp; + this.deploymentCredential = deploymentCredential; + this.deploymentExecutor = deployments == null ? null : Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "votingplugin-control-proxy-deployment"); + thread.setDaemon(true); + return thread; + }); this.dataDirectory = dataDirectory; this.route = route; this.recovering = recovering; @@ -210,12 +241,17 @@ public static ControlConnector create(VotingPluginProxy proxy) throws IOExceptio backends.sort(Comparator.comparing(ObservedBackend::backendId)); return List.copyOf(backends); }; + PluginDeploymentService deployments = prepareDeployment(proxy); + HttpClient deploymentHttp = deployments == null ? null : HttpClient.newBuilder() + .connectTimeout(Duration.ofMillis(settings.connectTimeoutMillis())) + .followRedirects(HttpClient.Redirect.NEVER).build(); ControlConnector connector = new ControlConnector(settings, proxy.getScheduler(), transport, snapshot, message -> proxy.log("[Control] " + message), UUID.randomUUID(), () -> ThreadLocalRandom.current().nextLong(), new ProxyRoutingConfigurationService(proxy), dataDirectory, route, recovering, proxy::restartControlServicesAfterRecovery, server -> proxy.testBackendCommunication(server, 5000L), new ProxyMethodConfigurationService(proxy), - () -> proxy.reloadCore(true), new ProxyConfigurationFileService(proxy)); + () -> proxy.reloadCore(true), new ProxyConfigurationFileService(proxy), + deployments, deploymentHttp, credential); if (recovered != null) connector.completedTasks.putAll(recovered.results()); return connector; } @@ -228,6 +264,92 @@ public void start() { schedule(0); operationPolling = scheduler.scheduleWithFixedDelay(this::pollOperations, OPERATION_POLL_MILLIS, OPERATION_POLL_MILLIS, TimeUnit.MILLISECONDS); + if (deployments != null) { + deploymentPolling = scheduler.scheduleWithFixedDelay(this::pollDeployments, + OPERATION_POLL_MILLIS, OPERATION_POLL_MILLIS, TimeUnit.MILLISECONDS); + } + } + + /** Keeps large artifact transfer and disk staging off proxy event threads and serializes it with operations. */ + private void pollDeployments() { + synchronized (operationLifecycle) { + if (closed || !registered || status != Status.CONNECTED || !deploymentsAccepted || deployments == null + || !inFlight.compareAndSet(false, true)) return; + } + CompletableFuture done = new CompletableFuture<>(); + activeOperation = done; + CompletableFuture work; + try { + CompletableFuture claim = transport.send(deploymentClaimRequest()); + activeRequest = claim; + work = claim.thenCompose(this::handleDeploymentClaim); + } catch (RuntimeException failure) { + work = CompletableFuture.failedFuture(failure); + } + work.whenComplete((ignored, failure) -> { + Throwable cause = failure == null ? null : unwrap(failure); + activeRequest = null; + if (cause == null) { + done.complete(null); + } else { + registered = false; + done.completeExceptionally(cause); + } + if (activeOperation == done) activeOperation = null; + finishCycle(); + if (cause != null && !closed) onFailure(cause); + }); + } + + private Request deploymentClaimRequest() { + JsonObject body = sessionBody(); + return new Request("POST", "/api/v1/nodes/" + settings.nodeId() + "/deployments", body.toString()); + } + + private CompletableFuture handleDeploymentClaim(Response response) { + if (response.statusCode == 204 || closed) return CompletableFuture.completedFuture(null); + if (response.statusCode == 404) { + registered = false; + throw new RegistryLostException(); + } + requireSuccess(response); + PluginDeploymentService.Task task = deploymentTask(parseObject(response.body)); + return CompletableFuture.supplyAsync(() -> deployments.deploy(task, settings.endpoint(), settings.nodeId(), sessionId, + deploymentCredential, deploymentHttp, Duration.ofMillis(settings.requestTimeoutMillis()), () -> !closed), + deploymentExecutor).thenCompose(result -> { + if (closed) return CompletableFuture.completedFuture(null); + JsonObject body = new JsonObject(); + body.addProperty("sessionId", sessionId.toString()); + body.addProperty("success", result.success()); + body.addProperty("code", result.code()); + body.addProperty("message", boundedResultMessage(result.message())); + body.addProperty("attemptId", task.attemptId().toString()); + CompletableFuture submitted = transport.send(new Request("POST", "/api/v1/nodes/" + + settings.nodeId() + "/deployments/" + task.deploymentId() + "/result", body.toString())); + activeRequest = submitted; + return submitted.thenAccept(ControlConnector::requireSuccess); + }); + } + + private static PluginDeploymentService.Task deploymentTask(JsonObject task) { + try { + return new PluginDeploymentService.Task(UUID.fromString(requireString(task, "deploymentId")), + requireString(task, "artifactId"), requireString(task, "sha256"), + Long.parseLong(requireString(task, "size")), UUID.fromString(requireString(task, "attemptId"))); + } catch (RuntimeException failure) { + throw new MalformedResponseException(); + } + } + + private static PluginDeploymentService prepareDeployment(VotingPluginProxy proxy) { + try { + var source = proxy.getClass().getProtectionDomain().getCodeSource(); + if (source == null || !"file".equalsIgnoreCase(source.getLocation().getProtocol())) return null; + return PluginDeploymentService.proxy(Path.of(source.getLocation().toURI())); + } catch (IOException | URISyntaxException | RuntimeException failure) { + proxy.log("[Control] Plugin deployment staging is unavailable; capability not advertised"); + return null; + } } /** Polls only the operation queue; heartbeat and presence retain their configured cadence. */ @@ -435,6 +557,8 @@ private void handlePrimaryResponse(Response response, boolean registration) { acceptedCapabilities = Set.copyOf(negotiated); configurationAccepted = acceptedCapabilities.stream().anyMatch(Set.of(CONFIGURATION_CAPABILITY, COMMUNICATION_TEST_CAPABILITY, PROXY_METHOD_CAPABILITY, PROXY_FILE_CAPABILITY)::contains); + deploymentsAccepted = deployments != null + && acceptedCapabilities.contains(PluginDeploymentService.CAPABILITY); } } @@ -1094,6 +1218,7 @@ private void addCapabilities(JsonObject body) { if (communicationTest != null) advertised.add(COMMUNICATION_TEST_CAPABILITY); if (methodConfigurationService != null) advertised.add(PROXY_METHOD_CAPABILITY); if (fileConfigurationService != null) advertised.add(PROXY_FILE_CAPABILITY); + if (deployments != null) advertised.add(PluginDeploymentService.CAPABILITY); body.add("capabilities", advertised); JsonArray required = new JsonArray(); required.add("presence.snapshot"); @@ -1133,6 +1258,10 @@ public void close() { } ScheduledFuture polling = operationPolling; if (polling != null) polling.cancel(false); + ScheduledFuture deployment = deploymentPolling; + if (deployment != null) deployment.cancel(false); + if (deployments != null) deployments.cancel(); + if (deploymentExecutor != null) deploymentExecutor.shutdownNow(); CompletableFuture request = activeRequest; if (request != null) { request.cancel(true); @@ -1150,6 +1279,16 @@ public void close() { throw new IllegalStateException("Interrupted while waiting for the Control operation", e); } } + if (deploymentExecutor != null) { + try { + if (!deploymentExecutor.awaitTermination(OPERATION_SHUTDOWN_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + throw new IllegalStateException("Control deployment worker did not stop before connector shutdown"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for the Control deployment worker", e); + } + } transport.close(); } From f623288b5b33ffec11218779c6fb831f185e6e05 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 18:01:17 -0600 Subject: [PATCH 118/136] Expose deterministic proxy capability advertisement --- .../proxy/control/ControlConnector.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 9d24533fe..70916c56f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -1212,13 +1212,19 @@ static List boundedResultChanges(List changes) { } private void addCapabilities(JsonObject body) { + addCapabilities(body, configurationService != null, communicationTest != null, methodConfigurationService != null, + fileConfigurationService != null, deployments != null); + } + + static void addCapabilities(JsonObject body, boolean configurationReady, boolean communicationReady, + boolean methodReady, boolean fileReady, boolean deploymentReady) { JsonArray advertised = new JsonArray(); BASE_CAPABILITIES.stream().sorted().forEach(advertised::add); - if (configurationService != null) advertised.add(CONFIGURATION_CAPABILITY); - if (communicationTest != null) advertised.add(COMMUNICATION_TEST_CAPABILITY); - if (methodConfigurationService != null) advertised.add(PROXY_METHOD_CAPABILITY); - if (fileConfigurationService != null) advertised.add(PROXY_FILE_CAPABILITY); - if (deployments != null) advertised.add(PluginDeploymentService.CAPABILITY); + if (configurationReady) advertised.add(CONFIGURATION_CAPABILITY); + if (communicationReady) advertised.add(COMMUNICATION_TEST_CAPABILITY); + if (methodReady) advertised.add(PROXY_METHOD_CAPABILITY); + if (fileReady) advertised.add(PROXY_FILE_CAPABILITY); + if (deploymentReady) advertised.add(PluginDeploymentService.CAPABILITY); body.add("capabilities", advertised); JsonArray required = new JsonArray(); required.add("presence.snapshot"); From 607129f31aacb7a6e9620dd3a7a8c673c90efbd9 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 18:01:22 -0600 Subject: [PATCH 119/136] Cover proxy deployment capability advertisement --- .../proxy/control/ControlConnectorTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index caffbbc47..d79a557f0 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -88,6 +88,18 @@ class ControlConnectorTest { assertEquals(1, secondSnapshot.get("sequence").getAsLong()); } + @Test void deploymentCapabilityIsAdvertisedOnlyWhenProxyStagingIsReady() { + JsonObject unavailable = new JsonObject(); + ControlConnector.addCapabilities(unavailable, true, true, true, true, false); + assertFalse(unavailable.getAsJsonArray("capabilities").asList().stream() + .anyMatch(value -> "plugin.deploy.v1".equals(value.getAsString()))); + + JsonObject ready = new JsonObject(); + ControlConnector.addCapabilities(ready, true, true, true, true, true); + assertTrue(ready.getAsJsonArray("capabilities").asList().stream() + .anyMatch(value -> "plugin.deploy.v1".equals(value.getAsString()))); + } + @Test void unavailableAuthenticationProtocolAndMalformedResponsesOnlyChangeConnectorState() { transport.nextPrimary = new Response(401, "{\"error\":{}}"); connector.cycle(); From 3369cb847f113052e638ba42007fddf56a2a6b69 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 18:13:59 -0600 Subject: [PATCH 120/136] Harden verified deployment transport and idempotency --- .../control/PluginDeploymentService.java | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java index 3a7d74923..bc8e37bc2 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java @@ -30,6 +30,8 @@ import java.util.jar.JarEntry; import java.util.jar.JarFile; +import com.bencodez.votingplugin.util.DurableFiles; + /** * Bounded, pull-only staging for the optional {@code plugin.deploy.v1} capability. * It deliberately never reloads a plugin or starts a server. The small on-disk @@ -102,11 +104,15 @@ public void cancel() { } } - public Result deploy(Task task, URI endpoint, String nodeId, UUID sessionId, String credential, - HttpClient http, Duration timeout, BooleanSupplier active) { + public Result deploy(Task task, URI endpoint, boolean directLocalHosted, String nodeId, UUID sessionId, + String credential, HttpClient http, Duration timeout, BooleanSupplier active) { if (!staging.compareAndSet(false, true)) return Result.failure("DEPLOYMENT_FAILED", "Another deployment is still staging"); try { validate(task); + if (!credentialEndpointAllowed(endpoint, directLocalHosted)) { + return Result.failure("INSECURE_ENDPOINT", + "Verified update staging requires HTTPS unless Control is hosted directly on this node"); + } if (alreadyStaged(task)) return Result.restartRequired(); if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before download"); URI artifact = endpoint.resolve("/api/v1/nodes/" + nodeId + "/deployments/" + task.deploymentId() @@ -361,6 +367,11 @@ private boolean alreadyStaged(Task task) throws IOException { String[] fields = Files.readString(marker, StandardCharsets.US_ASCII).split("\\R", -1); if (fields.length < 3 || !task.deploymentId().toString().equals(fields[0]) || !task.sha256().equals(fields[1]) || !Long.toString(task.size()).equals(fields[2])) return false; + if (!replaceExisting && !Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + // Bukkit removes the staged update JAR after consuming it on restart. The + // durable matching marker still proves this exact deployment was staged. + return true; + } return targetMatches(task); } @@ -403,8 +414,12 @@ private static void force(Path file) throws IOException { } private static void forceDirectory(Path directory) throws IOException { - try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { channel.force(true); } - catch (UnsupportedOperationException ignored) { /* Some supported filesystems cannot fsync directories. */ } + DurableFiles.forceDirectory(directory); + } + + static boolean credentialEndpointAllowed(URI endpoint, boolean directLocalHosted) { + return endpoint != null && ("https".equalsIgnoreCase(endpoint.getScheme()) + || directLocalHosted && "http".equalsIgnoreCase(endpoint.getScheme())); } private static MessageDigest sha256() { From 187db97bf39eddae3608a6eca80b5012d636dffd Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 18:14:11 -0600 Subject: [PATCH 121/136] Restrict backend deployment to active secure routes --- .../control/BackendControlConnector.java | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 2512b5418..be75f9873 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -31,6 +31,7 @@ import com.bencodez.votingplugin.VotingPluginMain; import com.bencodez.votingplugin.control.BackendControlResultStore.Route; import com.bencodez.votingplugin.control.BackendControlResultStore.StoredResult; +import com.bencodez.votingplugin.proxy.control.HostedControlManager; import com.bencodez.votingplugin.proxy.control.HostedControlManager.HostConfiguration; import com.bencodez.votingplugin.util.BoundedHttpBodyHandler; import com.bencodez.votingplugin.util.ControlCredentialFile; @@ -62,6 +63,7 @@ public final class BackendControlConnector implements AutoCloseable { private final BackendConfigurationService configurations; private final ControlInspectionService inspections; private final PluginDeploymentService deployments; + private final boolean directLocalDeploymentEndpoint; private final UUID sessionId = UUID.randomUUID(); private final Map completed = new LinkedHashMap<>(); private final boolean recovering; @@ -111,11 +113,19 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set .followRedirects(HttpClient.Redirect.NEVER).build(); configurations = new BackendConfigurationService(plugin.getDataFolder().toPath(), this::reloadConfiguration); inspections = new ControlInspectionService(plugin); + directLocalDeploymentEndpoint = HostedControlManager.isDirectLocalEndpoint( + settings.endpoint().toString(), hostedConfiguration); PluginDeploymentService prepared = null; - try { - prepared = PluginDeploymentService.backend(plugin.getServer().getUpdateFolderFile().toPath()); - } catch (Exception failure) { - plugin.getLogger().warning("[Control] Plugin deployment staging is unavailable; capability not advertised"); + boolean deploymentEndpointAllowed = PluginDeploymentService.credentialEndpointAllowed( + settings.endpoint(), directLocalDeploymentEndpoint); + if (!recovering && deploymentEndpointAllowed) { + try { + prepared = PluginDeploymentService.backend(plugin.getServer().getUpdateFolderFile().toPath()); + } catch (Exception failure) { + plugin.getLogger().warning("[Control] Plugin deployment staging is unavailable; capability not advertised"); + } + } else if (!recovering && !deploymentEndpointAllowed) { + plugin.getLogger().warning("[Control] Plugin deployment staging requires HTTPS unless Control is hosted directly on this node"); } deployments = prepared; } @@ -246,8 +256,9 @@ private void claimAndDeploy() throws Exception { if (response.status() == 204 || closed) return; JsonObject claimed = requireObject(response, 200); PluginDeploymentService.Task task = deploymentTask(claimed); - PluginDeploymentService.Result result = deployments.deploy(task, settings.endpoint(), settings.nodeId(), sessionId, - credential, http, Duration.ofMillis(settings.requestTimeoutMillis()), () -> !closed); + PluginDeploymentService.Result result = deployments.deploy(task, settings.endpoint(), directLocalDeploymentEndpoint, + settings.nodeId(), sessionId, credential, http, Duration.ofMillis(settings.requestTimeoutMillis()), + () -> !closed); if (closed) return; JsonObject submitted = new JsonObject(); submitted.addProperty("sessionId", sessionId.toString()); From bca3ec4d4c0aee2541d05a36afe7c6bb7429bca4 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 18:14:27 -0600 Subject: [PATCH 122/136] Fence and secure proxy deployment polling --- .../proxy/control/ControlConnector.java | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 70916c56f..f5681554d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -38,6 +38,7 @@ import com.bencodez.votingplugin.control.PluginDeploymentService; import com.bencodez.votingplugin.proxy.VotingPluginProxy; +import com.bencodez.votingplugin.proxy.control.HostedControlManager.HostConfiguration; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; import com.bencodez.votingplugin.proxy.presence.BackendPresenceStatus; import com.bencodez.votingplugin.proxy.control.ProxyControlResultStore.Route; @@ -84,6 +85,7 @@ public final class ControlConnector implements AutoCloseable { private final PluginDeploymentService deployments; private final HttpClient deploymentHttp; private final String deploymentCredential; + private final boolean directLocalDeploymentEndpoint; private final ExecutorService deploymentExecutor; private final Function> communicationTest; private final Runnable runtimeReplacement; @@ -142,7 +144,7 @@ private ControlConnector(Settings settings, ScheduledExecutorService scheduler, ProxyConfigurationFileService fileConfigurationService) { this(settings, scheduler, transport, snapshotSource, logger, sessionId, jitterSource, configurationService, dataDirectory, route, recovering, recoveryComplete, communicationTest, methodConfigurationService, - runtimeReplacement, fileConfigurationService, null, null, null); + runtimeReplacement, fileConfigurationService, null, null, null, false); } private ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, @@ -152,7 +154,7 @@ private ControlConnector(Settings settings, ScheduledExecutorService scheduler, Function> communicationTest, ProxyMethodConfigurationService methodConfigurationService, Runnable runtimeReplacement, ProxyConfigurationFileService fileConfigurationService, PluginDeploymentService deployments, - HttpClient deploymentHttp, String deploymentCredential) { + HttpClient deploymentHttp, String deploymentCredential, boolean directLocalDeploymentEndpoint) { this.settings = Objects.requireNonNull(settings, "settings"); this.scheduler = Objects.requireNonNull(scheduler, "scheduler"); this.transport = Objects.requireNonNull(transport, "transport"); @@ -168,6 +170,7 @@ private ControlConnector(Settings settings, ScheduledExecutorService scheduler, this.deployments = deployments; this.deploymentHttp = deploymentHttp; this.deploymentCredential = deploymentCredential; + this.directLocalDeploymentEndpoint = directLocalDeploymentEndpoint; this.deploymentExecutor = deployments == null ? null : Executors.newSingleThreadExecutor(runnable -> { Thread thread = new Thread(runnable, "votingplugin-control-proxy-deployment"); thread.setDaemon(true); @@ -223,6 +226,7 @@ public static ControlConnector create(VotingPluginProxy proxy) throws IOExceptio } } boolean recovering = recovered != null && recovered.routeRequired(); + boolean deploymentRouteCurrent = config.getControlEnabled() && !recovering; String credential = ControlCredentialFile.read(dataDirectory, credentialName); HttpControlTransport transport = new HttpControlTransport(settings.endpoint(), credential, settings.connectTimeoutMillis(), settings.requestTimeoutMillis()); @@ -241,7 +245,21 @@ public static ControlConnector create(VotingPluginProxy proxy) throws IOExceptio backends.sort(Comparator.comparing(ObservedBackend::backendId)); return List.copyOf(backends); }; - PluginDeploymentService deployments = prepareDeployment(proxy); + HostConfiguration hosted = new HostConfiguration(config.getControlHostedEnabled(), + config.getControlHostedAutoDownload(), config.getControlHostedAutoUpdate(), + config.getControlHostedDownloadUrl(), config.getControlHostedSha256(), + config.getControlHostedJarFile(), config.getControlHostedDataDirectory(), + config.getControlHostedHost(), config.getControlHostedPort(), + config.getControlHostedStartupTimeoutSeconds(), config.getControlHostedDownloadTimeoutSeconds()); + boolean directLocalDeploymentEndpoint = HostedControlManager.isDirectLocalEndpoint( + settings.endpoint().toString(), hosted); + boolean deploymentEndpointAllowed = PluginDeploymentService.credentialEndpointAllowed( + settings.endpoint(), directLocalDeploymentEndpoint); + PluginDeploymentService deployments = deploymentRouteCurrent && deploymentEndpointAllowed + ? prepareDeployment(proxy) : null; + if (deploymentRouteCurrent && !deploymentEndpointAllowed) { + proxy.log("[Control] Plugin deployment staging requires HTTPS unless Control is hosted directly on this node"); + } HttpClient deploymentHttp = deployments == null ? null : HttpClient.newBuilder() .connectTimeout(Duration.ofMillis(settings.connectTimeoutMillis())) .followRedirects(HttpClient.Redirect.NEVER).build(); @@ -251,7 +269,7 @@ public static ControlConnector create(VotingPluginProxy proxy) throws IOExceptio route, recovering, proxy::restartControlServicesAfterRecovery, server -> proxy.testBackendCommunication(server, 5000L), new ProxyMethodConfigurationService(proxy), () -> proxy.reloadCore(true), new ProxyConfigurationFileService(proxy), - deployments, deploymentHttp, credential); + deployments, deploymentHttp, credential, directLocalDeploymentEndpoint); if (recovered != null) connector.completedTasks.putAll(recovered.results()); return connector; } @@ -272,12 +290,12 @@ public void start() { /** Keeps large artifact transfer and disk staging off proxy event threads and serializes it with operations. */ private void pollDeployments() { + CompletableFuture done = new CompletableFuture<>(); synchronized (operationLifecycle) { if (closed || !registered || status != Status.CONNECTED || !deploymentsAccepted || deployments == null || !inFlight.compareAndSet(false, true)) return; + activeOperation = done; } - CompletableFuture done = new CompletableFuture<>(); - activeOperation = done; CompletableFuture work; try { CompletableFuture claim = transport.send(deploymentClaimRequest()); @@ -297,7 +315,11 @@ private void pollDeployments() { } if (activeOperation == done) activeOperation = null; finishCycle(); - if (cause != null && !closed) onFailure(cause); + if (cause != null && !closed) { + ScheduledFuture heartbeat = scheduled; + if (heartbeat != null) heartbeat.cancel(false); + onFailure(cause); + } }); } @@ -314,8 +336,9 @@ private CompletableFuture handleDeploymentClaim(Response response) { } requireSuccess(response); PluginDeploymentService.Task task = deploymentTask(parseObject(response.body)); - return CompletableFuture.supplyAsync(() -> deployments.deploy(task, settings.endpoint(), settings.nodeId(), sessionId, - deploymentCredential, deploymentHttp, Duration.ofMillis(settings.requestTimeoutMillis()), () -> !closed), + return CompletableFuture.supplyAsync(() -> deployments.deploy(task, settings.endpoint(), + directLocalDeploymentEndpoint, settings.nodeId(), sessionId, deploymentCredential, deploymentHttp, + Duration.ofMillis(settings.requestTimeoutMillis()), () -> !closed), deploymentExecutor).thenCompose(result -> { if (closed) return CompletableFuture.completedFuture(null); JsonObject body = new JsonObject(); From ad2c992279286eb2e77fb9963103b32d9cd6f0af Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 18:14:39 -0600 Subject: [PATCH 123/136] Cover secure deployment and consumed Bukkit updates --- .../control/PluginDeploymentServiceTest.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java index f7b6008a7..f8556353e 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java @@ -45,6 +45,31 @@ class PluginDeploymentServiceTest { assertTrue(service.stage(task, new ByteArrayInputStream(new byte[0]), () -> true).success()); } + @Test void backendMatchingMarkerSurvivesBukkitConsumingTheStagedJar() throws Exception { + byte[] artifact = jar("name: VotingPlugin\n"); + Path update = directory.resolve("update"); + PluginDeploymentService service = PluginDeploymentService.backend(update); + PluginDeploymentService.Task task = task(artifact); + + assertTrue(service.stage(task, new ByteArrayInputStream(artifact), () -> true).success()); + Files.delete(update.resolve("VotingPlugin.jar")); + + assertTrue(service.stage(task, new ByteArrayInputStream(new byte[0]), () -> true).success()); + assertFalse(Files.exists(update.resolve("VotingPlugin.jar")), + "a lost result acknowledgement after restart must not stage the consumed update again"); + } + + @Test void credentialedDeploymentRequiresHttpsUnlessSameNodeHostedHttpWasProven() { + assertTrue(PluginDeploymentService.credentialEndpointAllowed( + java.net.URI.create("https://control.example.test"), false)); + assertFalse(PluginDeploymentService.credentialEndpointAllowed( + java.net.URI.create("http://192.0.2.10:8080"), false)); + assertFalse(PluginDeploymentService.credentialEndpointAllowed( + java.net.URI.create("http://127.0.0.1:8080"), false)); + assertTrue(PluginDeploymentService.credentialEndpointAllowed( + java.net.URI.create("http://127.0.0.1:8080"), true)); + } + @Test void backendIgnoresMatchingMarkerOnlyWhenTargetIsValidThenRestagesWhenCorrupted() throws Exception { byte[] artifact = jar("name: VotingPlugin\n"); PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); From f938c9f88870e06f17ddfe8ec5002f2c78881bf4 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 18:14:58 -0600 Subject: [PATCH 124/136] Document verified deployment protocol --- docs/control-agent-contract.md | 59 ++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/docs/control-agent-contract.md b/docs/control-agent-contract.md index 8375e58f1..77c51edd0 100644 --- a/docs/control-agent-contract.md +++ b/docs/control-agent-contract.md @@ -107,6 +107,65 @@ redacted-editor task; turning `useMainMySQL` off alone does not invent credentia unknown fields. `READ` returns current typed state and its revision, `PREVIEW` computes changes and produces an approval, and only the approved `APPLY` writes atomically, reloads, and rolls back on reload failure, like every other quick setup. +## Verified plugin staging (`plugin.deploy.v1`) + +This optional capability stages one verified VotingPlugin JAR for the node's next process restart. It is never a hot reload +and never restarts a proxy or backend automatically. A node advertises it only when all of the following are true: + +- the connector is the currently enabled Control route, not a recovery-only connector draining an older durable result; +- a safe local staging target was prepared; +- the Control endpoint is HTTPS, or it is the already-proven direct same-node hosted HTTP listener. + +Arbitrary private-network HTTP does not qualify for deployment because the artifact request carries the node bearer +credential. The shared staging service enforces the same transport rule again before sending that credential. + +Control leases deployment work separately from configuration operations: + +```http +POST /api/v1/nodes/{nodeId}/deployments +Content-Type: application/json + +{"sessionId":""} +``` + +A `204` means no work. A `200` task contains `deploymentId`, `artifactId`, lowercase SHA-256 `sha256`, byte +`size`, and `attemptId`. Artifact size is limited to 64 MiB. The node downloads the exact leased artifact with: + +```http +GET /api/v1/nodes/{nodeId}/deployments/{deploymentId}/artifact +Authorization: Bearer +X-Node-Session: +X-Deployment-Attempt: +``` + +The node independently verifies the exact size and SHA-256, bounded ZIP/JAR structure, root `plugin.yml`, and +`name: VotingPlugin` before publication. Bukkit nodes stage to the server update folder; proxy nodes atomically replace +their discovered plugin JAR only after creating a durable `.control-backup`. Every successful deployment writes a small +`.control-deployment` marker so a lost result acknowledgement does not apply the same deployment twice. On Bukkit, that +marker remains authoritative after a restart consumes/removes the staged update JAR. On proxies, the target JAR must still +match the marker's digest; an interrupted publish-before-marker window can be recovered without overwriting the previous +backup. + +The result is posted to: + +```http +POST /api/v1/nodes/{nodeId}/deployments/{deploymentId}/result +Content-Type: application/json + +{ + "sessionId":"", + "attemptId":"", + "success":true, + "code":"RESTART_REQUIRED", + "message":"Plugin update staged; restart is required" +} +``` + +Failure codes include `CANCELLED`, `DOWNLOAD_FAILED`, `SIZE_MISMATCH`, `HASH_MISMATCH`, +`INVALID_ARTIFACT`, `STAGING_FAILED`, `DEPLOYMENT_FAILED`, and `INSECURE_ENDPOINT`. Deployment polling is +serialized with the connector lifecycle, uses the normal bounded failure backoff, and is cancelled during shutdown before +transport teardown. + ## Inspection transport An enrolled Bukkit node advertises `data.inspect.v1`. Once Control includes it in `acceptedCapabilities`, the node polls: From 632058fe3a6e09afc56b698dd3009c147c6a058d Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 18:15:01 -0600 Subject: [PATCH 125/136] Document verified deployment protocol --- docs/control-connector.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/control-connector.md b/docs/control-connector.md index 4e797fef5..14e1851a4 100644 --- a/docs/control-connector.md +++ b/docs/control-connector.md @@ -200,6 +200,29 @@ with the new settings. This prevents the old and new children from racing for th the currently running Control child unchanged. If the backend restarts with a pending result, the durable journal also restores the previous hosted settings until that result is acknowledged, preserving the recovery connector's endpoint. +## Verified VotingPlugin updates + +Nodes that negotiate `plugin.deploy.v1` can stage a VotingPlugin JAR supplied by Control for the next normal process +restart. This capability is deliberately separate from configuration control and from hosted-Control self-updates. +VotingPlugin never hot-reloads itself and never restarts the server or proxy automatically. + +Deployment is available only on the currently enabled Control route. Recovery-only connectors that exist solely to +acknowledge an older durable result never advertise or poll this capability. The node also requires a credential-safe +artifact transport: HTTPS is accepted generally; HTTP is accepted only when the existing hosted-Control checks prove the +endpoint is the direct same-node listener. A LAN/private HTTP endpoint may still be used for ordinary Control operations, +but it is intentionally ineligible for credentialed plugin-JAR staging. + +Control leases deployment work through `POST /api/v1/nodes/{nodeId}/deployments`. The node downloads the artifact through +the matching deployment artifact endpoint with its bearer credential plus exact session and attempt headers, then +independently verifies the 64 MiB size bound, SHA-256, JAR structure, and root `plugin.yml` identity. Bukkit stages the +verified JAR in the configured update folder; BungeeCord/Velocity retain a durable backup before atomically replacing the +running plugin JAR on disk. A durable deployment marker makes lost result acknowledgements idempotent, including the +post-restart Bukkit state where the server has already consumed the staged update JAR. + +Successful staging reports `RESTART_REQUIRED`. The administrator chooses when to restart. See +[the Control agent contract](control-agent-contract.md#verified-plugin-staging-plugindeployv1) for the exact task, download, +result, transport, and failure-code contract. + ## Discovery semantics Both platforms use the same connector implementation and protocol version `1`. Each proxy process creates a new session From a2b1381240bd1f8debd45ea3e63927aa2fb691c5 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Sep 2026 21:39:58 -0600 Subject: [PATCH 126/136] Fix verified deployment hardening build --- .../votingplugin/VotingPluginMain.java | 5 ++ .../control/BackendControlConnector.java | 3 +- .../control/PluginDeploymentService.java | 71 ++++++++++++++----- .../proxy/control/ControlConnector.java | 9 +++ .../control/PluginDeploymentServiceTest.java | 52 +++++++++++--- .../proxy/control/ControlConnectorTest.java | 7 ++ 6 files changed, 121 insertions(+), 26 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 35fb886a8..5ce3788fe 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -152,6 +152,11 @@ public class VotingPluginMain extends AdvancedCorePlugin { @Getter public static VotingPluginMain plugin; + /** Exact backend plugin file name Bukkit uses when consuming its update folder. */ + public File getLoadedPluginJarFile() { + return getFile(); + } + @Getter @Setter private ArrayList adminVoteCommand; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index be75f9873..499d313ad 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -120,7 +120,8 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set settings.endpoint(), directLocalDeploymentEndpoint); if (!recovering && deploymentEndpointAllowed) { try { - prepared = PluginDeploymentService.backend(plugin.getServer().getUpdateFolderFile().toPath()); + prepared = PluginDeploymentService.backend(plugin.getServer().getUpdateFolderFile().toPath(), + plugin.getLoadedPluginJarFile().toPath()); } catch (Exception failure) { plugin.getLogger().warning("[Control] Plugin deployment staging is unavailable; capability not advertised"); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java index bc8e37bc2..4196d4d39 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java @@ -54,12 +54,14 @@ public final class PluginDeploymentService { private final Path target; private final Path root; private final Path marker; + private final Path installedBackendJar; private final boolean replaceExisting; private final AtomicBoolean staging = new AtomicBoolean(); private final AtomicReference activeResponse = new AtomicReference<>(); - private PluginDeploymentService(Path target, boolean replaceExisting) throws IOException { + private PluginDeploymentService(Path target, Path installedBackendJar, boolean replaceExisting) throws IOException { this.target = target.toAbsolutePath().normalize(); + this.installedBackendJar = installedBackendJar == null ? null : installedBackendJar.toAbsolutePath().normalize(); this.replaceExisting = replaceExisting; Path parent = this.target.getParent(); if (parent == null) throw new IOException("deployment target has no parent"); @@ -78,10 +80,15 @@ private PluginDeploymentService(Path target, boolean replaceExisting) throws IOE this.marker = root.resolve(this.target.getFileName() + MARKER); } - /** Bukkit's update folder preserves the currently loaded plugin JAR. */ - public static PluginDeploymentService backend(Path updateDirectory) throws IOException { + /** Bukkit consumes an update only when its file name matches the currently loaded plugin JAR. */ + public static PluginDeploymentService backend(Path updateDirectory, Path currentPluginJar) throws IOException { if (updateDirectory == null) throw new IOException("Bukkit update folder is unavailable"); - return new PluginDeploymentService(updateDirectory.resolve("VotingPlugin.jar"), false); + if (currentPluginJar == null || currentPluginJar.getFileName() == null + || !currentPluginJar.getFileName().toString().toLowerCase(Locale.ROOT).endsWith(".jar")) { + throw new IOException("Bukkit plugin JAR is unavailable"); + } + return new PluginDeploymentService(updateDirectory.resolve(currentPluginJar.getFileName().toString()), + currentPluginJar, false); } /** Proxies atomically replace their discovered plugin JAR only after a durable backup. */ @@ -91,7 +98,7 @@ public static PluginDeploymentService proxy(Path currentPluginJar) throws IOExce } if (!Files.isRegularFile(currentPluginJar, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(currentPluginJar)) throw new IOException("proxy plugin JAR is unsafe"); - return new PluginDeploymentService(currentPluginJar, true); + return new PluginDeploymentService(currentPluginJar, null, true); } public boolean isStaging() { return staging.get(); } @@ -181,7 +188,7 @@ Result stage(Task task, InputStream body, BooleanSupplier active) throws IOExcep if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging"); activation = new Activation(); activate(temporary, activation); - writeMarker(task); + writeMarker(task, activation); activation.discard(); return Result.restartRequired(); } catch (CancelledDeploymentException failure) { @@ -311,6 +318,7 @@ private void activate(Path temporary, Activation activation) throws IOException private final class Activation { private final Path previous; private boolean published; + private boolean markerPublished; private Activation() throws IOException { if (!Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { @@ -330,6 +338,7 @@ private void rollback() throws IOException { discard(); return; } + if (markerPublished) DurableFiles.deleteIfExists(marker); if (previous == null) { if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) { throw new IOException("deployed target cannot be safely removed"); @@ -351,12 +360,17 @@ private void discard() { } private void writeMarker(Task task) throws IOException { + writeMarker(task, null); + } + + private void writeMarker(Task task, Activation activation) throws IOException { Path temporary = Files.createTempFile(root, target.getFileName().toString() + ".", ".marker"); try { Files.writeString(temporary, task.deploymentId() + "\n" + task.sha256() + "\n" + task.size() + "\n", StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); force(temporary); move(temporary, marker); + if (activation != null) activation.markerPublished = true; forceDirectory(root); } finally { Files.deleteIfExists(temporary); } } @@ -368,9 +382,10 @@ private boolean alreadyStaged(Task task) throws IOException { if (fields.length < 3 || !task.deploymentId().toString().equals(fields[0]) || !task.sha256().equals(fields[1]) || !Long.toString(task.size()).equals(fields[2])) return false; if (!replaceExisting && !Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { - // Bukkit removes the staged update JAR after consuming it on restart. The - // durable matching marker still proves this exact deployment was staged. - return true; + // Bukkit removes the staged update JAR after consuming it on restart. A + // missing staging file alone is not proof: it may have been deleted or + // quarantined. Confirm the installed backend JAR is the exact artifact. + return installedBackendJar != null && fileMatches(installedBackendJar, task); } return targetMatches(task); } @@ -388,16 +403,20 @@ private boolean recoverInterruptedActivation(Task task) throws IOException { } private boolean targetMatches(Task task) throws IOException { - if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target) - || Files.size(target) != task.size()) return false; + return fileMatches(target, task); + } + + private boolean fileMatches(Path candidate, Task task) throws IOException { + if (!Files.isRegularFile(candidate, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(candidate) + || Files.size(candidate) != task.size()) return false; MessageDigest digest = sha256(); - try (InputStream input = Files.newInputStream(target, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)) { + try (InputStream input = Files.newInputStream(candidate, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)) { byte[] bytes = new byte[BUFFER_BYTES]; for (int read; (read = input.read(bytes)) != -1;) digest.update(bytes, 0, read); } if (!task.sha256().equals(HexFormat.of().formatHex(digest.digest()))) return false; try { - inspectJar(target); + inspectJar(candidate); return true; } catch (InvalidArtifactException invalid) { return false; @@ -417,9 +436,29 @@ private static void forceDirectory(Path directory) throws IOException { DurableFiles.forceDirectory(directory); } - static boolean credentialEndpointAllowed(URI endpoint, boolean directLocalHosted) { - return endpoint != null && ("https".equalsIgnoreCase(endpoint.getScheme()) - || directLocalHosted && "http".equalsIgnoreCase(endpoint.getScheme())); + /** True when a deployment bearer credential may be sent to this Control endpoint. */ + public static boolean credentialEndpointAllowed(URI endpoint, boolean directLocalHosted) { + if (endpoint == null) return false; + if ("https".equalsIgnoreCase(endpoint.getScheme())) return true; + return directLocalHosted && "http".equalsIgnoreCase(endpoint.getScheme()) + && isLoopbackHost(endpoint.getHost()); + } + + private static boolean isLoopbackHost(String host) { + if (host == null) return false; + String normalized = host; + if (normalized.length() >= 2 && normalized.charAt(0) == '[' + && normalized.charAt(normalized.length() - 1) == ']') { + normalized = normalized.substring(1, normalized.length() - 1); + } + if ("localhost".equalsIgnoreCase(normalized) || "::1".equalsIgnoreCase(normalized) + || "0:0:0:0:0:0:0:1".equalsIgnoreCase(normalized)) return true; + String[] octets = normalized.split("\\.", -1); + if (octets.length != 4 || !"127".equals(octets[0])) return false; + for (int index = 1; index < octets.length; index++) { + if (!octets[index].matches("[0-9]{1,3}") || Integer.parseInt(octets[index]) > 255) return false; + } + return true; } private static MessageDigest sha256() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index f5681554d..c56ff7a01 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -45,6 +45,7 @@ import com.bencodez.votingplugin.proxy.control.ProxyControlResultStore.StoredResult; import com.bencodez.votingplugin.util.BoundedHttpBodyHandler; import com.bencodez.votingplugin.util.ControlCredentialFile; +import com.bencodez.votingplugin.util.DurableFiles; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -365,6 +366,10 @@ private static PluginDeploymentService.Task deploymentTask(JsonObject task) { } private static PluginDeploymentService prepareDeployment(VotingPluginProxy proxy) { + if (!proxyDeploymentSupported(System.getProperty("os.name", ""))) { + proxy.log("[Control] Plugin deployment staging is unavailable on Windows proxies; capability not advertised"); + return null; + } try { var source = proxy.getClass().getProtectionDomain().getCodeSource(); if (source == null || !"file".equalsIgnoreCase(source.getLocation().getProtocol())) return null; @@ -375,6 +380,10 @@ private static PluginDeploymentService prepareDeployment(VotingPluginProxy proxy } } + static boolean proxyDeploymentSupported(String osName) { + return !DurableFiles.isWindowsName(osName); + } + /** Polls only the operation queue; heartbeat and presence retain their configured cadence. */ void pollOperations() { CompletableFuture operationDone; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java index f8556353e..26bd3118c 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java @@ -34,7 +34,7 @@ class PluginDeploymentServiceTest { @Test void backendStagesOnlyAnExactVerifiedVotingPluginJarAndIsIdempotent() throws Exception { byte[] artifact = jar("name: VotingPlugin\nmain: example.Main\n"); - PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update"), Path.of("VotingPlugin.jar")); PluginDeploymentService.Task task = task(artifact); PluginDeploymentService.Result staged = service.stage(task, new ByteArrayInputStream(artifact), () -> true); @@ -45,18 +45,48 @@ class PluginDeploymentServiceTest { assertTrue(service.stage(task, new ByteArrayInputStream(new byte[0]), () -> true).success()); } - @Test void backendMatchingMarkerSurvivesBukkitConsumingTheStagedJar() throws Exception { + @Test void backendStagesUsingTheInstalledPluginJarFileName() throws Exception { byte[] artifact = jar("name: VotingPlugin\n"); Path update = directory.resolve("update"); - PluginDeploymentService service = PluginDeploymentService.backend(update); + PluginDeploymentService service = PluginDeploymentService.backend(update, + directory.resolve("plugins/VotingPlugin-7.1.2.jar")); + + assertEquals("RESTART_REQUIRED", service.stage(task(artifact), new ByteArrayInputStream(artifact), () -> true).code()); + assertArrayEquals(artifact, Files.readAllBytes(update.resolve("VotingPlugin-7.1.2.jar"))); + assertFalse(Files.exists(update.resolve("VotingPlugin.jar"))); + } + + @Test void backendMatchingMarkerSurvivesBukkitConsumingTheStagedJar() throws Exception { + byte[] artifact = jar("name: VotingPlugin\nversion: candidate\n"); + Path update = directory.resolve("update"); + Path installed = directory.resolve("VotingPlugin.jar"); + Files.write(installed, jar("name: VotingPlugin\nversion: old\n")); + PluginDeploymentService service = PluginDeploymentService.backend(update, installed); PluginDeploymentService.Task task = task(artifact); assertTrue(service.stage(task, new ByteArrayInputStream(artifact), () -> true).success()); + Files.write(installed, artifact); Files.delete(update.resolve("VotingPlugin.jar")); assertTrue(service.stage(task, new ByteArrayInputStream(new byte[0]), () -> true).success()); assertFalse(Files.exists(update.resolve("VotingPlugin.jar")), - "a lost result acknowledgement after restart must not stage the consumed update again"); + "a lost acknowledgement after restart must recognize the artifact Bukkit already consumed"); + } + + @Test void backendRestagesWhenTheUpdateJarDisappearsBeforeBukkitConsumesIt() throws Exception { + byte[] artifact = jar("name: VotingPlugin\nversion: candidate\n"); + Path update = directory.resolve("update"); + Path installed = directory.resolve("VotingPlugin.jar"); + Files.write(installed, jar("name: VotingPlugin\nversion: old\n")); + PluginDeploymentService service = PluginDeploymentService.backend(update, installed); + PluginDeploymentService.Task task = task(artifact); + + assertTrue(service.stage(task, new ByteArrayInputStream(artifact), () -> true).success()); + Files.delete(update.resolve("VotingPlugin.jar")); + + assertTrue(service.stage(task, new ByteArrayInputStream(artifact), () -> true).success()); + assertArrayEquals(artifact, Files.readAllBytes(update.resolve("VotingPlugin.jar")), + "a deleted or quarantined update must be staged again before restart"); } @Test void credentialedDeploymentRequiresHttpsUnlessSameNodeHostedHttpWasProven() { @@ -68,11 +98,15 @@ class PluginDeploymentServiceTest { java.net.URI.create("http://127.0.0.1:8080"), false)); assertTrue(PluginDeploymentService.credentialEndpointAllowed( java.net.URI.create("http://127.0.0.1:8080"), true)); + assertTrue(PluginDeploymentService.credentialEndpointAllowed( + java.net.URI.create("http://[::1]:8080"), true)); + assertFalse(PluginDeploymentService.credentialEndpointAllowed( + java.net.URI.create("http://127.example.com:8080"), true)); } @Test void backendIgnoresMatchingMarkerOnlyWhenTargetIsValidThenRestagesWhenCorrupted() throws Exception { byte[] artifact = jar("name: VotingPlugin\n"); - PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update"), Path.of("VotingPlugin.jar")); PluginDeploymentService.Task task = task(artifact); service.stage(task, new ByteArrayInputStream(artifact), () -> true); @@ -89,7 +123,7 @@ class PluginDeploymentServiceTest { @Test void invalidPluginIdentityAndDigestNeverReachTheUpdateFolder() throws Exception { byte[] wrongPlugin = jar("name: NotVotingPlugin\n"); - PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update"), Path.of("VotingPlugin.jar")); PluginDeploymentService.Result invalid = service.stage(task(wrongPlugin), new ByteArrayInputStream(wrongPlugin), () -> true); assertFalse(invalid.success()); assertEquals("INVALID_ARTIFACT", invalid.code()); @@ -139,7 +173,7 @@ class PluginDeploymentServiceTest { @Test void cancellationDuringCopyDoesNotPublish() throws Exception { byte[] artifact = jar("name: VotingPlugin\n"); - PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update"), Path.of("VotingPlugin.jar")); PluginDeploymentService.Task task = task(artifact); assertEquals("RESTART_REQUIRED", service.stage(task, new ByteArrayInputStream(artifact), () -> true).code()); @@ -166,7 +200,7 @@ class PluginDeploymentServiceTest { Path marker = update.resolve("VotingPlugin.jar.control-deployment"); Files.createDirectory(marker); Files.writeString(marker.resolve("keep"), "marker publication must fail"); - PluginDeploymentService service = PluginDeploymentService.backend(update); + PluginDeploymentService service = PluginDeploymentService.backend(update, Path.of("VotingPlugin.jar")); assertThrows(IOException.class, () -> service.stage(task(candidate), new ByteArrayInputStream(candidate), () -> true)); @@ -190,7 +224,7 @@ class PluginDeploymentServiceTest { } @Test void cancellationClosesAStalledArtifactStream() throws Exception { - PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update")); + PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update"), Path.of("VotingPlugin.jar")); AtomicBoolean active = new AtomicBoolean(true); BlockingInputStream stalled = new BlockingInputStream(active); ExecutorService executor = Executors.newSingleThreadExecutor(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index d79a557f0..e77b60165 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -891,6 +891,13 @@ class ControlConnectorTest { assertEquals(Status.STOPPED, connector.status()); } + @Test void proxyDeploymentIsNotAdvertisedOnWindowsWhereTheLiveJarCannotBeReplaced() { + assertFalse(ControlConnector.proxyDeploymentSupported("Windows 11")); + assertFalse(ControlConnector.proxyDeploymentSupported("Windows Server 2022")); + assertTrue(ControlConnector.proxyDeploymentSupported("Linux")); + assertTrue(ControlConnector.proxyDeploymentSupported("Darwin")); + } + @Test void backoffIsBoundedExponentialAndJitteredWithoutSleeping() { assertEquals(1000, ControlConnector.backoffMillis(1, 0)); assertEquals(2000, ControlConnector.backoffMillis(2, 0)); From b12bf791b1a9326dac00c88aedbd2207f3f2c53d Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 19 Sep 2026 15:07:33 -0600 Subject: [PATCH 127/136] Complete queued proxy deployment on connector shutdown --- .../proxy/control/ControlConnector.java | 46 +++++++++----- .../proxy/control/ControlConnectorTest.java | 60 +++++++++++++++++++ 2 files changed, 90 insertions(+), 16 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index c56ff7a01..8fe63463c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -110,6 +110,9 @@ public final class ControlConnector implements AutoCloseable { private volatile ScheduledFuture operationPolling; private volatile ScheduledFuture deploymentPolling; private volatile CompletableFuture activeRequest; + /* The deployment executor may discard queued AsyncSupply work during shutdown. + * Retain its future so close() can complete the dependent operation chain. */ + private volatile CompletableFuture activeDeploymentWork; private volatile CompletableFuture activeOperation; private volatile Status status = Status.STARTING; @@ -337,22 +340,31 @@ private CompletableFuture handleDeploymentClaim(Response response) { } requireSuccess(response); PluginDeploymentService.Task task = deploymentTask(parseObject(response.body)); - return CompletableFuture.supplyAsync(() -> deployments.deploy(task, settings.endpoint(), - directLocalDeploymentEndpoint, settings.nodeId(), sessionId, deploymentCredential, deploymentHttp, - Duration.ofMillis(settings.requestTimeoutMillis()), () -> !closed), - deploymentExecutor).thenCompose(result -> { - if (closed) return CompletableFuture.completedFuture(null); - JsonObject body = new JsonObject(); - body.addProperty("sessionId", sessionId.toString()); - body.addProperty("success", result.success()); - body.addProperty("code", result.code()); - body.addProperty("message", boundedResultMessage(result.message())); - body.addProperty("attemptId", task.attemptId().toString()); - CompletableFuture submitted = transport.send(new Request("POST", "/api/v1/nodes/" - + settings.nodeId() + "/deployments/" + task.deploymentId() + "/result", body.toString())); - activeRequest = submitted; - return submitted.thenAccept(ControlConnector::requireSuccess); - }); + final CompletableFuture deploymentWork; + synchronized (operationLifecycle) { + if (closed) return CompletableFuture.completedFuture(null); + deploymentWork = CompletableFuture.supplyAsync(() -> deployments.deploy(task, settings.endpoint(), + directLocalDeploymentEndpoint, settings.nodeId(), sessionId, deploymentCredential, deploymentHttp, + Duration.ofMillis(settings.requestTimeoutMillis()), () -> !closed), deploymentExecutor); + activeDeploymentWork = deploymentWork; + } + return deploymentWork.thenCompose(result -> { + if (closed) return CompletableFuture.completedFuture(null); + JsonObject body = new JsonObject(); + body.addProperty("sessionId", sessionId.toString()); + body.addProperty("success", result.success()); + body.addProperty("code", result.code()); + body.addProperty("message", boundedResultMessage(result.message())); + body.addProperty("attemptId", task.attemptId().toString()); + CompletableFuture submitted = transport.send(new Request("POST", "/api/v1/nodes/" + + settings.nodeId() + "/deployments/" + task.deploymentId() + "/result", body.toString())); + activeRequest = submitted; + return submitted.thenAccept(ControlConnector::requireSuccess); + }).whenComplete((ignored, failure) -> { + synchronized (operationLifecycle) { + if (activeDeploymentWork == deploymentWork) activeDeploymentWork = null; + } + }); } private static PluginDeploymentService.Task deploymentTask(JsonObject task) { @@ -1299,6 +1311,8 @@ public void close() { ScheduledFuture deployment = deploymentPolling; if (deployment != null) deployment.cancel(false); if (deployments != null) deployments.cancel(); + CompletableFuture deploymentWork = activeDeploymentWork; + if (deploymentWork != null) deploymentWork.cancel(true); if (deploymentExecutor != null) deploymentExecutor.shutdownNow(); CompletableFuture request = activeRequest; if (request != null) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index e77b60165..81e424fe2 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -1,5 +1,6 @@ package com.bencodez.votingplugin.proxy.control; +import com.bencodez.votingplugin.control.PluginDeploymentService; import com.bencodez.votingplugin.proxy.control.ControlConnector.ObservedBackend; import com.bencodez.votingplugin.proxy.control.ControlConnector.Request; import com.bencodez.votingplugin.proxy.control.ControlConnector.Response; @@ -12,6 +13,7 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.net.http.HttpClient; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -23,6 +25,7 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -891,6 +894,44 @@ class ControlConnectorTest { assertEquals(Status.STOPPED, connector.status()); } + @Test void closeCancelsARealQueuedDeploymentBeforeAwaitingItsOperation() throws Exception { + connector.close(); + Path update = dataDirectory.resolve("update"); + connector = deploymentConnector(PluginDeploymentService.backend(update, dataDirectory.resolve("VotingPlugin.jar"))); + setConnectorField("registered", true); + setConnectorField("deploymentsAccepted", true); + setConnectorField("status", Status.CONNECTED); + Field executorField = ControlConnector.class.getDeclaredField("deploymentExecutor"); + executorField.setAccessible(true); + ExecutorService deploymentExecutor = (ExecutorService) executorField.get(connector); + CountDownLatch workerStarted = new CountDownLatch(1); + deploymentExecutor.submit(() -> { + workerStarted.countDown(); + try { + new CountDownLatch(1).await(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(workerStarted.await(2, TimeUnit.SECONDS)); + UUID deploymentId = UUID.randomUUID(); + transport.nextPrimary = new Response(200, "{\"deploymentId\":\"" + deploymentId + + "\",\"artifactId\":\"VotingPlugin.jar\",\"sha256\":\"" + "a".repeat(64) + + "\",\"size\":\"1\",\"attemptId\":\"" + UUID.randomUUID() + "\"}"); + Method poll = ControlConnector.class.getDeclaredMethod("pollDeployments"); + poll.setAccessible(true); + poll.invoke(connector); + Field operationField = ControlConnector.class.getDeclaredField("activeOperation"); + operationField.setAccessible(true); + CompletableFuture operation = (CompletableFuture) operationField.get(connector); + assertNotNull(operation, "the actual deployment polling path must own the queued operation"); + + connector.close(); + + assertTrue(operation.isDone(), "shutdown must complete a deployment discarded before its executor starts it"); + assertFalse(Files.exists(update.resolve("VotingPlugin.jar")), "queued deployment work must not stage an artifact"); + } + @Test void proxyDeploymentIsNotAdvertisedOnWindowsWhereTheLiveJarCannotBeReplaced() { assertFalse(ControlConnector.proxyDeploymentSupported("Windows 11")); assertFalse(ControlConnector.proxyDeploymentSupported("Windows Server 2022")); @@ -928,6 +969,25 @@ private Settings settings() { URI.create("http://127.0.0.1:8080"), 30, 3000, 5000); } + private ControlConnector deploymentConnector(PluginDeploymentService deployments) throws Exception { + Constructor constructor = ControlConnector.class.getDeclaredConstructor(Settings.class, + ScheduledExecutorService.class, Transport.class, Supplier.class, Consumer.class, UUID.class, + LongSupplier.class, ProxyRoutingConfigurationService.class, Path.class, ProxyControlResultStore.Route.class, + boolean.class, Runnable.class, Function.class, ProxyMethodConfigurationService.class, Runnable.class, + ProxyConfigurationFileService.class, PluginDeploymentService.class, HttpClient.class, String.class, + boolean.class); + constructor.setAccessible(true); + return constructor.newInstance(settings(), scheduler, transport, (Supplier>) List::of, + (Consumer) logs::add, UUID.randomUUID(), (LongSupplier) () -> 0L, null, null, null, false, + null, null, null, null, null, deployments, HttpClient.newHttpClient(), "credential", false); + } + + private void setConnectorField(String name, Object value) throws Exception { + Field field = ControlConnector.class.getDeclaredField(name); + field.setAccessible(true); + field.set(connector, value); + } + private JsonObject submittedResult() { return JsonParser.parseString(transport.requests.stream().filter(request -> request.path().endsWith("/result")) .findFirst().orElseThrow().body()).getAsJsonObject(); From 5766dbf83c58e643eadd34135b83f4fc2ae7796a Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 19 Sep 2026 17:50:11 -0600 Subject: [PATCH 128/136] Recognize verified staged artifact across deployment retries --- .../control/PluginDeploymentService.java | 15 +++++++---- .../control/PluginDeploymentServiceTest.java | 27 +++++++++++++++++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java index 4196d4d39..54e13619a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java @@ -120,8 +120,8 @@ public Result deploy(Task task, URI endpoint, boolean directLocalHosted, String return Result.failure("INSECURE_ENDPOINT", "Verified update staging requires HTTPS unless Control is hosted directly on this node"); } - if (alreadyStaged(task)) return Result.restartRequired(); if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before download"); + if (alreadyStaged(task)) return Result.restartRequired(); URI artifact = endpoint.resolve("/api/v1/nodes/" + nodeId + "/deployments/" + task.deploymentId() + "/artifact"); HttpRequest request = HttpRequest.newBuilder(artifact).timeout(timeout) @@ -172,6 +172,7 @@ public Result deploy(Task task, URI endpoint, boolean directLocalHosted, String /** Package-visible for deterministic artifact-validation tests. */ Result stage(Task task, InputStream body, BooleanSupplier active) throws IOException { validate(task); + if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging"); if (alreadyStaged(task) || recoverInterruptedActivation(task)) return Result.restartRequired(); activeResponse.compareAndSet(null, body); Path temporary = Files.createTempFile(root, target.getFileName().toString() + ".", ".download"); @@ -379,8 +380,12 @@ private boolean alreadyStaged(Task task) throws IOException { if (!Files.isRegularFile(marker, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(marker) || Files.size(marker) > 256) return false; String[] fields = Files.readString(marker, StandardCharsets.US_ASCII).split("\\R", -1); - if (fields.length < 3 || !task.deploymentId().toString().equals(fields[0]) - || !task.sha256().equals(fields[1]) || !Long.toString(task.size()).equals(fields[2])) return false; + // A lost acknowledgement can cause Control to issue a new deployment ID + // for the same verified artifact after reconnect or restart. The marker + // only proves a completed prior stage; the target is checked below by + // exact size, digest and plugin identity before acknowledging the retry. + if (fields.length < 3 || !task.sha256().equals(fields[1]) + || !Long.toString(task.size()).equals(fields[2])) return false; if (!replaceExisting && !Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { // Bukkit removes the staged update JAR after consuming it on restart. A // missing staging file alone is not proof: it may have been deleted or @@ -394,8 +399,8 @@ private boolean alreadyStaged(Task task) throws IOException { private boolean recoverInterruptedActivation(Task task) throws IOException { // Proxy replacement creates a durable backup before publishing the target; // that transaction shape lets a retry distinguish the activation crash window. - // Backend update folders have no such evidence and must still consume/verify - // the newly supplied body when a different deployment id is requested. + // Backend update folders have no such evidence and must consume/verify + // a new body unless a matching durable marker and artifact remain. if (!replaceExisting) return false; if (!targetMatches(task)) return false; writeMarker(task); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java index 26bd3118c..c9f6a5cde 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java @@ -73,6 +73,28 @@ class PluginDeploymentServiceTest { "a lost acknowledgement after restart must recognize the artifact Bukkit already consumed"); } + @Test void retryWithNewDeploymentIdAcknowledgesOnlyTheSameVerifiedArtifact() throws Exception { + byte[] artifact = jar("name: VotingPlugin\nversion: candidate\n"); + Path update = directory.resolve("update"); + Path installed = directory.resolve("VotingPlugin.jar"); + Files.write(installed, jar("name: VotingPlugin\nversion: old\n")); + PluginDeploymentService service = PluginDeploymentService.backend(update, installed); + PluginDeploymentService.Task original = task(artifact); + PluginDeploymentService.Task retry = task(artifact); + assertTrue(service.stage(original, new ByteArrayInputStream(artifact), () -> true).success()); + assertFalse(original.deploymentId().equals(retry.deploymentId())); + + assertEquals("RESTART_REQUIRED", + service.stage(retry, new ByteArrayInputStream(new byte[0]), () -> true).code()); + assertArrayEquals(artifact, Files.readAllBytes(update.resolve("VotingPlugin.jar"))); + + Files.write(installed, artifact); + Files.delete(update.resolve("VotingPlugin.jar")); + assertEquals("RESTART_REQUIRED", + service.stage(retry, new ByteArrayInputStream(new byte[0]), () -> true).code()); + assertFalse(Files.exists(update.resolve("VotingPlugin.jar"))); + } + @Test void backendRestagesWhenTheUpdateJarDisappearsBeforeBukkitConsumesIt() throws Exception { byte[] artifact = jar("name: VotingPlugin\nversion: candidate\n"); Path update = directory.resolve("update"); @@ -181,8 +203,9 @@ class PluginDeploymentServiceTest { byte[] targetBefore = Files.readAllBytes(directory.resolve("update/VotingPlugin.jar")); AtomicBoolean active = new AtomicBoolean(true); - InputStream body = new CancellingInputStream(new ByteArrayInputStream(artifact), active); - PluginDeploymentService.Task replacement = task(artifact); + byte[] replacementArtifact = jar("name: VotingPlugin\nversion: replacement\n"); + InputStream body = new CancellingInputStream(new ByteArrayInputStream(replacementArtifact), active); + PluginDeploymentService.Task replacement = task(replacementArtifact); assertEquals("CANCELLED", service.stage(replacement, body, active::get).code()); assertEquals(markerBefore, Files.readString(directory.resolve("update/VotingPlugin.jar.control-deployment"))); From ef2be5253ccf4d62e03bb66384d960aeff1f822f Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 19 Sep 2026 17:57:38 -0600 Subject: [PATCH 129/136] Publish deployment marker before proxy activation --- .../control/PluginDeploymentService.java | 28 ++++++++----------- .../control/PluginDeploymentServiceTest.java | 25 +++++++++++++++-- docs/control-agent-contract.md | 10 +++---- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java index 54e13619a..9a1bff3f4 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java @@ -173,7 +173,7 @@ public Result deploy(Task task, URI endpoint, boolean directLocalHosted, String Result stage(Task task, InputStream body, BooleanSupplier active) throws IOException { validate(task); if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging"); - if (alreadyStaged(task) || recoverInterruptedActivation(task)) return Result.restartRequired(); + if (alreadyStaged(task)) return Result.restartRequired(); activeResponse.compareAndSet(null, body); Path temporary = Files.createTempFile(root, target.getFileName().toString() + ".", ".download"); Activation activation = null; @@ -188,8 +188,9 @@ Result stage(Task task, InputStream body, BooleanSupplier active) throws IOExcep inspectJar(temporary); if (!active.getAsBoolean()) return Result.failure("CANCELLED", "Deployment was cancelled before staging"); activation = new Activation(); - activate(temporary, activation); + prepareProxyBackup(); writeMarker(task, activation); + activate(temporary, activation); activation.discard(); return Result.restartRequired(); } catch (CancelledDeploymentException failure) { @@ -291,7 +292,7 @@ private static String unquote(String value) { || (value.startsWith("'") && value.endsWith("'"))) ? value.substring(1, value.length() - 1) : value; } - private void activate(Path temporary, Activation activation) throws IOException { + private void prepareProxyBackup() throws IOException { if (replaceExisting) { if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) { throw new IOException("current proxy plugin JAR is unsafe"); @@ -309,6 +310,9 @@ private void activate(Path temporary, Activation activation) throws IOException forceDirectory(root); } finally { Files.deleteIfExists(backupTemp); } } + } + + private void activate(Path temporary, Activation activation) throws IOException { move(temporary, target); activation.published = true; force(target); @@ -335,11 +339,11 @@ private Activation() throws IOException { } private void rollback() throws IOException { + if (markerPublished) DurableFiles.deleteIfExists(marker); if (!published) { discard(); return; } - if (markerPublished) DurableFiles.deleteIfExists(marker); if (previous == null) { if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) { throw new IOException("deployed target cannot be safely removed"); @@ -379,6 +383,10 @@ private void writeMarker(Task task, Activation activation) throws IOException { private boolean alreadyStaged(Task task) throws IOException { if (!Files.isRegularFile(marker, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(marker) || Files.size(marker) > 256) return false; + if (replaceExisting) { + Path backup = root.resolve(target.getFileName() + ".control-backup"); + if (!Files.isRegularFile(backup, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(backup)) return false; + } String[] fields = Files.readString(marker, StandardCharsets.US_ASCII).split("\\R", -1); // A lost acknowledgement can cause Control to issue a new deployment ID // for the same verified artifact after reconnect or restart. The marker @@ -395,18 +403,6 @@ private boolean alreadyStaged(Task task) throws IOException { return targetMatches(task); } - /** Complete the durable marker half of an activation interrupted after publish. */ - private boolean recoverInterruptedActivation(Task task) throws IOException { - // Proxy replacement creates a durable backup before publishing the target; - // that transaction shape lets a retry distinguish the activation crash window. - // Backend update folders have no such evidence and must consume/verify - // a new body unless a matching durable marker and artifact remain. - if (!replaceExisting) return false; - if (!targetMatches(task)) return false; - writeMarker(task); - return true; - } - private boolean targetMatches(Task task) throws IOException { return fileMatches(target, task); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java index c9f6a5cde..e854316e6 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java @@ -171,7 +171,7 @@ class PluginDeploymentServiceTest { assertArrayEquals(second, Files.readAllBytes(current)); } - @Test void proxyRecoversActivationInterruptedBeforeMarkerWithoutReplacingBackup() throws Exception { + @Test void proxyMissingMarkerRequiresNormalVerifiedStaging() throws Exception { Path current = directory.resolve("VotingPlugin.jar"); Path backup = directory.resolve("VotingPlugin.jar.control-backup"); Path marker = directory.resolve("VotingPlugin.jar.control-deployment"); @@ -182,17 +182,36 @@ class PluginDeploymentServiceTest { Files.write(backup, original); PluginDeploymentService service = PluginDeploymentService.proxy(current); - assertEquals("RESTART_REQUIRED", + assertEquals("SIZE_MISMATCH", service.stage(task, new ByteArrayInputStream(new byte[0]), () -> true).code()); + assertFalse(Files.exists(marker)); + assertArrayEquals(original, Files.readAllBytes(backup)); + assertEquals("RESTART_REQUIRED", + service.stage(task, new ByteArrayInputStream(candidate), () -> true).code()); assertArrayEquals(candidate, Files.readAllBytes(current)); - assertArrayEquals(original, Files.readAllBytes(backup)); + assertArrayEquals(candidate, Files.readAllBytes(backup)); String state = Files.readString(marker, StandardCharsets.US_ASCII); assertTrue(state.contains(task.deploymentId().toString())); assertTrue(state.contains(task.sha256())); assertTrue(state.contains(Long.toString(task.size()))); } + @Test void proxyDoesNotAcknowledgeAStagedArtifactWithoutItsBackup() throws Exception { + Path current = directory.resolve("VotingPlugin.jar"); + Path backup = directory.resolve("VotingPlugin.jar.control-backup"); + byte[] original = jar("name: VotingPlugin\nversion: original\n"); + byte[] candidate = jar("name: VotingPlugin\nversion: candidate\n"); + Files.write(current, original); + PluginDeploymentService service = PluginDeploymentService.proxy(current); + PluginDeploymentService.Task task = task(candidate); + assertEquals("RESTART_REQUIRED", service.stage(task, new ByteArrayInputStream(candidate), () -> true).code()); + Files.delete(backup); + assertEquals("SIZE_MISMATCH", service.stage(task, new ByteArrayInputStream(new byte[0]), () -> true).code()); + assertEquals("RESTART_REQUIRED", service.stage(task, new ByteArrayInputStream(candidate), () -> true).code()); + assertTrue(Files.isRegularFile(backup)); + } + @Test void cancellationDuringCopyDoesNotPublish() throws Exception { byte[] artifact = jar("name: VotingPlugin\n"); PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update"), Path.of("VotingPlugin.jar")); diff --git a/docs/control-agent-contract.md b/docs/control-agent-contract.md index d56c37f4e..dc1e5d6ad 100644 --- a/docs/control-agent-contract.md +++ b/docs/control-agent-contract.md @@ -176,11 +176,11 @@ X-Deployment-Attempt: The node independently verifies the exact size and SHA-256, bounded ZIP/JAR structure, root `plugin.yml`, and `name: VotingPlugin` before publication. Bukkit nodes stage to the server update folder; proxy nodes atomically replace -their discovered plugin JAR only after creating a durable `.control-backup`. Every successful deployment writes a small -`.control-deployment` marker so a lost result acknowledgement does not apply the same deployment twice. On Bukkit, that -marker remains authoritative after a restart consumes/removes the staged update JAR. On proxies, the target JAR must still -match the marker's digest; an interrupted publish-before-marker window can be recovered without overwriting the previous -backup. +their discovered plugin JAR only after creating a durable `.control-backup`. A small durable +`.control-deployment` marker is published before the verified target is moved into place, so a lost result +acknowledgement does not apply the same artifact twice. On Bukkit, a consumed staged update must match the installed JAR. +On proxies, both a safe backup and a target JAR matching the marker's digest are required for acknowledgement; a missing +marker requires a fresh verified download and staging attempt. The result is posted to: From 9bd840f8bc0be44dcf54ea05e53bf010de87755f Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 19 Sep 2026 18:05:56 -0600 Subject: [PATCH 130/136] Reject unsafe Bukkit update targets and unknown deployment fields --- .../control/BackendControlConnector.java | 5 ++++- .../control/PluginDeploymentService.java | 11 +++++++++-- .../proxy/control/ControlConnector.java | 5 ++++- .../BackendControlConnectorProtocolTest.java | 16 ++++++++++++++++ .../control/PluginDeploymentServiceTest.java | 5 +++++ .../proxy/control/ControlConnectorTest.java | 11 +++++++++++ 6 files changed, 49 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 4f143bb41..ac5a1bc20 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -54,6 +54,7 @@ public final class BackendControlConnector implements AutoCloseable { private static final Set CAPABILITIES = Set.of("config.files.v1", "config.file-comments.v1", "config.quick-setup.v1", "config.quick-setup.v2", "config.vote-sites-sync.v1", "config.proxy-method.v1", "config.reward-files.v1", "data.inspect.v1"); + private static final Set DEPLOYMENT_TASK_FIELDS = Set.of("deploymentId", "artifactId", "sha256", "size", "attemptId"); private final VotingPluginMain plugin; private final Path dataDirectory; @@ -349,8 +350,10 @@ private void claimAndDeploy() throws Exception { + "/result", submitted), 200); } - private static PluginDeploymentService.Task deploymentTask(JsonObject task) { + static PluginDeploymentService.Task deploymentTask(JsonObject task) { try { + if (task == null || task.size() != DEPLOYMENT_TASK_FIELDS.size() + || !DEPLOYMENT_TASK_FIELDS.containsAll(task.keySet())) throw new IllegalArgumentException(); return new PluginDeploymentService.Task(UUID.fromString(string(task, "deploymentId")), string(task, "artifactId"), string(task, "sha256"), Long.parseLong(string(task, "size")), UUID.fromString(string(task, "attemptId"))); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java index 9a1bff3f4..cf76174b3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java @@ -87,8 +87,15 @@ public static PluginDeploymentService backend(Path updateDirectory, Path current || !currentPluginJar.getFileName().toString().toLowerCase(Locale.ROOT).endsWith(".jar")) { throw new IOException("Bukkit plugin JAR is unavailable"); } - return new PluginDeploymentService(updateDirectory.resolve(currentPluginJar.getFileName().toString()), - currentPluginJar, false); + Path target = updateDirectory.resolve(currentPluginJar.getFileName().toString()).toAbsolutePath().normalize(); + Path installed = currentPluginJar.toAbsolutePath().normalize(); + // An empty/disabled Bukkit update folder can resolve to the loaded JAR's + // directory. Staging there would overwrite a file that Bukkit currently has open. + if (target.equals(installed) || Files.exists(target, LinkOption.NOFOLLOW_LINKS) + && Files.exists(installed, LinkOption.NOFOLLOW_LINKS) && Files.isSameFile(target, installed)) { + throw new IOException("Bukkit update folder resolves to the loaded plugin JAR"); + } + return new PluginDeploymentService(target, currentPluginJar, false); } /** Proxies atomically replace their discovered plugin JAR only after a durable backup. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 8fe63463c..a0576e7be 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -69,6 +69,7 @@ public final class ControlConnector implements AutoCloseable { private static final String PROXY_METHOD_PRESET = "proxy-method"; private static final String INTERNAL_OPERATION_TYPE = "_controlOperationType"; private static final String INTERNAL_REQUIRED_CAPABILITY = "_controlRequiredCapability"; + private static final Set DEPLOYMENT_TASK_FIELDS = Set.of("deploymentId", "artifactId", "sha256", "size", "attemptId"); private static final long OPERATION_POLL_MILLIS = 1000; private static final long MAX_BACKOFF_MILLIS = TimeUnit.MINUTES.toMillis(5); private static final long OPERATION_SHUTDOWN_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(65); @@ -367,8 +368,10 @@ private CompletableFuture handleDeploymentClaim(Response response) { }); } - private static PluginDeploymentService.Task deploymentTask(JsonObject task) { + static PluginDeploymentService.Task deploymentTask(JsonObject task) { try { + if (task == null || task.size() != DEPLOYMENT_TASK_FIELDS.size() + || !DEPLOYMENT_TASK_FIELDS.containsAll(task.keySet())) throw new IllegalArgumentException(); return new PluginDeploymentService.Task(UUID.fromString(requireString(task, "deploymentId")), requireString(task, "artifactId"), requireString(task, "sha256"), Long.parseLong(requireString(task, "size")), UUID.fromString(requireString(task, "attemptId"))); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java index 8670fa3dc..15e655cec 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java @@ -41,6 +41,22 @@ class BackendControlConnectorProtocolTest { "{\"error\":{\"code\":\"NODE_NOT_FOUND\"}}"))); } + @Test void deploymentTaskRejectsUnknownV1Fields() { + JsonObject task = deploymentTask(); + task.addProperty("unexpected", "value"); + assertThrows(IllegalArgumentException.class, () -> BackendControlConnector.deploymentTask(task)); + } + + private static JsonObject deploymentTask() { + JsonObject task = new JsonObject(); + task.addProperty("deploymentId", "00000000-0000-0000-0000-000000000001"); + task.addProperty("artifactId", "VotingPlugin.jar"); + task.addProperty("sha256", "a".repeat(64)); + task.addProperty("size", "1"); + task.addProperty("attemptId", "00000000-0000-0000-0000-000000000002"); + return task; + } + @Test void abandonedBackendIntentBecomesATerminalRecoveryResult() { JsonObject anticipated = new JsonObject(); anticipated.addProperty("attemptId", "00000000-0000-0000-0000-000000000199"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java index e854316e6..32afa1b40 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java @@ -56,6 +56,11 @@ class PluginDeploymentServiceTest { assertFalse(Files.exists(update.resolve("VotingPlugin.jar"))); } + @Test void backendRejectsAnEmptyOrDisabledBukkitUpdateFolderThatResolvesToTheLoadedJar() { + assertThrows(IOException.class, + () -> PluginDeploymentService.backend(Path.of(""), Path.of("VotingPlugin.jar"))); + } + @Test void backendMatchingMarkerSurvivesBukkitConsumingTheStagedJar() throws Exception { byte[] artifact = jar("name: VotingPlugin\nversion: candidate\n"); Path update = directory.resolve("update"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index 81e424fe2..315447dfe 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -51,6 +51,17 @@ class ControlConnectorTest { @Test void responseBudgetCanCarryTheLargestEscapedManagedFileTask() { assertTrue(ControlConnector.MAX_RESPONSE_BYTES >= ProxyConfigurationFileService.MAX_BYTES * 6); } + + @Test void deploymentTaskRejectsUnknownV1Fields() { + JsonObject task = new JsonObject(); + task.addProperty("deploymentId", "00000000-0000-0000-0000-000000000001"); + task.addProperty("artifactId", "VotingPlugin.jar"); + task.addProperty("sha256", "a".repeat(64)); + task.addProperty("size", "1"); + task.addProperty("attemptId", "00000000-0000-0000-0000-000000000002"); + task.addProperty("unexpected", "value"); + assertThrows(RuntimeException.class, () -> ControlConnector.deploymentTask(task)); + } private ControlConnector connector; @BeforeEach void setUp() { From 0c5b725b1aa3862cb7577604222a9981985e4a82 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 19 Sep 2026 18:13:57 -0600 Subject: [PATCH 131/136] Require atomic deployment publication --- .../control/PluginDeploymentService.java | 8 ++++---- .../control/PluginDeploymentServiceTest.java | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java index cf76174b3..2b5256738 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/PluginDeploymentService.java @@ -9,7 +9,6 @@ import java.nio.charset.StandardCharsets; import java.nio.charset.CodingErrorAction; import java.nio.channels.FileChannel; -import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; @@ -431,9 +430,10 @@ private boolean fileMatches(Path candidate, Task task) throws IOException { } } - private static void move(Path source, Path destination) throws IOException { - try { Files.move(source, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } - catch (AtomicMoveNotSupportedException ignored) { Files.move(source, destination, StandardCopyOption.REPLACE_EXISTING); } + static void move(Path source, Path destination) throws IOException { + // The proxy target is executable on its next startup. A copy/delete fallback + // can leave it missing or partial after a crash, even with a durable backup. + Files.move(source, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } private static void force(Path file) throws IOException { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java index 32afa1b40..423e46db8 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/PluginDeploymentServiceTest.java @@ -19,6 +19,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.FileSystems; +import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Path; import java.security.MessageDigest; import java.util.HexFormat; @@ -32,6 +34,21 @@ class PluginDeploymentServiceTest { @TempDir Path directory; + @Test void publicationRefusesAProviderWithoutAtomicMoves() throws Exception { + Path source = directory.resolve("VotingPlugin.jar"); + byte[] original = jar("name: VotingPlugin\nversion: old\n"); + Files.write(source, original); + Path archive = directory.resolve("non-atomic.zip"); + try (var zip = FileSystems.newFileSystem(java.net.URI.create("jar:" + archive.toUri()), + java.util.Map.of("create", "true"))) { + Path destination = zip.getPath("/VotingPlugin.jar"); + assertThrows(AtomicMoveNotSupportedException.class, + () -> PluginDeploymentService.move(source, destination)); + assertArrayEquals(original, Files.readAllBytes(source)); + assertFalse(Files.exists(destination)); + } + } + @Test void backendStagesOnlyAnExactVerifiedVotingPluginJarAndIsIdempotent() throws Exception { byte[] artifact = jar("name: VotingPlugin\nmain: example.Main\n"); PluginDeploymentService service = PluginDeploymentService.backend(directory.resolve("update"), Path.of("VotingPlugin.jar")); From 43fbf0b8a057165f22d516d4b4cd154ef8984610 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 19 Sep 2026 18:55:44 -0600 Subject: [PATCH 132/136] Continue rollover queue after retryable auxiliary broadcast --- .../votingplugin/proxy/VotingPluginProxy.java | 17 ++++-- .../VotingPluginProxyQueueProgressTest.java | 61 +++++++++++++++++++ 2 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyQueueProgressTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 9c782e0e9..cbcef0759 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -3409,10 +3409,10 @@ public synchronized void processQueue() { } VoteTotalsSnapshot queuedTotals = forwardedTotals[1].isEmpty() ? null : VoteTotalsSnapshot.parseStorage(forwardedTotals[1]); - QueuedVoteResult result = vote(vote.getName(), vote.getService(), queuedRealVote, false, vote.getTime(), queuedTotals, - vote.getUuid(), vote); - if (result == QueuedVoteResult.RETRY) { + QueuedVoteResult result = replayQueuedVote(vote, queuedTotals, queuedRealVote); + if (result == QueuedVoteResult.RETRY || result == QueuedVoteResult.RETRY_NONBLOCKING) { scheduleTimeVoteRetry(); + if (result == QueuedVoteResult.RETRY_NONBLOCKING) continue; return; } if ((result == QueuedVoteResult.SUCCESS || result == QueuedVoteResult.TERMINAL) @@ -4950,8 +4950,12 @@ public synchronized void abandonLiveVoteRetry(UUID voteId) { if (voteId != null) liveVoteRetries.remove(voteId); } - private enum QueuedVoteResult { - SUCCESS, RETRY, TERMINAL + enum QueuedVoteResult { + SUCCESS, RETRY, RETRY_NONBLOCKING, TERMINAL + } + + protected QueuedVoteResult replayQueuedVote(VoteTimeQueue vote, VoteTotalsSnapshot totals, boolean realVote) { + return vote(vote.getName(), vote.getService(), realVote, false, vote.getTime(), totals, vote.getUuid(), vote); } private synchronized QueuedVoteResult vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, @@ -5460,7 +5464,8 @@ && getGlobalDataHandler().isTimeChangedHappened()) { String deliveryId = stableLiveHttpBroadcastDeliveryId(voteId, targetServer); if (!sendStableHttpEnvelope(targetServer, deliveryId, broadcast)) { retryState.broadcastForwardedServers.addAll(broadcastForwardedServers); - return QueuedVoteResult.RETRY; + return queuedVote == null ? QueuedVoteResult.RETRY + : QueuedVoteResult.RETRY_NONBLOCKING; } broadcastForwardedServers.add(targetServer); retryState.broadcastForwardedServers.add(targetServer); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyQueueProgressTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyQueueProgressTest.java new file mode 100644 index 000000000..c97a78ad3 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyQueueProgressTest.java @@ -0,0 +1,61 @@ +package com.bencodez.votingplugin.proxy; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Collections; +import java.util.Queue; +import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import com.bencodez.votingplugin.proxy.cache.VoteCacheHandler; +import com.bencodez.votingplugin.tests.VotingPluginProxyTestImpl; +import com.bencodez.votingplugin.timequeue.VoteTimeQueue; + +class VotingPluginProxyQueueProgressTest { + @Test + void nonblockingQueuedRetryAllowsLaterProcessedRowToDrain() { + VoteCacheHandler cache = Mockito.mock(VoteCacheHandler.class); + Queue queue = new ConcurrentLinkedQueue<>(); + VoteTimeQueue first = new VoteTimeQueue(UUID.randomUUID(), "Player", "Service", 100L); + VoteTimeQueue second = new VoteTimeQueue(UUID.randomUUID(), "Player", "Service", 101L); + second.setProcessed(true); + queue.add(first); + queue.add(second); + Mockito.when(cache.getTimeChangeQueue()).thenReturn(queue); + Mockito.when(cache.removeTimeVote(second)).thenAnswer(invocation -> queue.remove(second)); + VotingPluginProxy proxy = Mockito.spy(new VotingPluginProxyTestImpl()); + Mockito.doReturn(cache).when(proxy).getVoteCacheHandler(); + Mockito.doReturn(VotingPluginProxy.QueuedVoteResult.RETRY_NONBLOCKING) + .when(proxy).replayQueuedVote(Mockito.eq(first), Mockito.isNull(), Mockito.anyBoolean()); + + proxy.processQueue(); + + assertTrue(queue.contains(first)); + assertFalse(queue.contains(second)); + } + + @Test + void ordinaryQueuedRetryStillStopsBeforeLaterRow() { + VoteCacheHandler cache = Mockito.mock(VoteCacheHandler.class); + Queue queue = new ConcurrentLinkedQueue<>(); + VoteTimeQueue first = new VoteTimeQueue(UUID.randomUUID(), "Player", "Service", 100L); + VoteTimeQueue second = new VoteTimeQueue(UUID.randomUUID(), "Player", "Service", 101L); + second.setProcessed(true); + queue.add(first); + queue.add(second); + Mockito.when(cache.getTimeChangeQueue()).thenReturn(queue); + VotingPluginProxy proxy = Mockito.spy(new VotingPluginProxyTestImpl()); + Mockito.doReturn(cache).when(proxy).getVoteCacheHandler(); + Mockito.doReturn(VotingPluginProxy.QueuedVoteResult.RETRY) + .when(proxy).replayQueuedVote(Mockito.eq(first), Mockito.isNull(), Mockito.anyBoolean()); + + proxy.processQueue(); + + assertTrue(queue.contains(first)); + assertTrue(queue.contains(second)); + } +} From bc9dac6303626e1757e4b5570b6fba548399fd27 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 19 Sep 2026 19:10:16 -0600 Subject: [PATCH 133/136] Require proxy method v2 for HTTP quick apply --- .../control/BackendControlConnector.java | 11 ++++++++ .../BackendControlConnectorProtocolTest.java | 27 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index defc577b3..f9cbd4093 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -84,6 +84,7 @@ public final class BackendControlConnector implements AutoCloseable { private volatile boolean registered; private volatile boolean operationsAccepted; private volatile boolean quickSetupsAccepted; + private volatile boolean proxyMethodV2Accepted; private volatile boolean votePartySetupsAccepted; private volatile boolean voteSitesSyncAccepted; private volatile boolean rewardFilesAccepted; @@ -626,6 +627,7 @@ private void cycle() { } operationsAccepted = negotiatedCapability(node, "config.files.v1", operationsAccepted); quickSetupsAccepted = negotiatedCapability(node, "config.quick-setup.v1", quickSetupsAccepted); + proxyMethodV2Accepted = negotiatedCapability(node, "config.proxy-method.v2", proxyMethodV2Accepted); votePartySetupsAccepted = negotiatedCapability(node, "config.quick-setup.v2", votePartySetupsAccepted); voteSitesSyncAccepted = negotiatedCapability(node, "config.vote-sites-sync.v1", voteSitesSyncAccepted); rewardFilesAccepted = configurations.supportsNamedRewardFiles() @@ -1079,6 +1081,11 @@ private TaskResult executeQuick(UUID operationId, String type, JsonObject config throws IOException { String preset = string(configuration, "preset"); Map options = options(configuration.getAsJsonObject("options")); + if ("APPLY".equals(type) && "proxy-method".equals(preset) + && !proxyMethodApplyCapabilityAccepted(options.getOrDefault("method", "PLUGINMESSAGING"), + proxyMethodV2Accepted)) { + return TaskResult.failure("UNSUPPORTED_TASK", "The HTTP proxy method capability was not negotiated"); + } if (!quickSetupCapabilityAccepted(preset, quickSetupsAccepted, votePartySetupsAccepted, voteSitesSyncAccepted, options)) { return TaskResult.failure("UNSUPPORTED_TASK", "The required quick setup capability was not negotiated"); @@ -1144,6 +1151,10 @@ static boolean quickSetupCapabilityAccepted(String preset, boolean quickSetupsAc voteSitesSyncAccepted); } + static boolean proxyMethodApplyCapabilityAccepted(String method, boolean proxyMethodV2Accepted) { + return !"HTTP".equalsIgnoreCase(method) || proxyMethodV2Accepted; + } + private Response send(String method, String path, JsonObject body) throws Exception { HttpRequest request = HttpRequest.newBuilder(settings.endpoint().resolve(path)) .timeout(Duration.ofMillis(settings.requestTimeoutMillis())).header("Content-Type", "application/json") diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java index 1b2566103..f76f0c414 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java @@ -298,6 +298,33 @@ private static JsonObject deploymentTask() { assertTrue(BackendControlConnector.quickSetupCapabilityAccepted("common-settings", true, false, false)); } + @Test void httpProxyMethodApplyRequiresNegotiatedV2Capability() { + assertFalse(BackendControlConnector.proxyMethodApplyCapabilityAccepted("HTTP", false)); + assertTrue(BackendControlConnector.proxyMethodApplyCapabilityAccepted("HTTP", true)); + assertTrue(BackendControlConnector.proxyMethodApplyCapabilityAccepted("PLUGINMESSAGING", false)); + } + + @Test void httpQuickApplyRejectsMissingV2BeforeTouchingConfiguration() throws Exception { + BackendControlConnector connector = org.mockito.Mockito.mock(BackendControlConnector.class, + org.mockito.Mockito.CALLS_REAL_METHODS); + var quick = BackendControlConnector.class.getDeclaredField("quickSetupsAccepted"); + quick.setAccessible(true); + quick.setBoolean(connector, true); + JsonObject configuration = JsonParser.parseString( + "{\"preset\":\"proxy-method\",\"options\":{\"method\":\"HTTP\"}}") + .getAsJsonObject(); + var execute = BackendControlConnector.class.getDeclaredMethod("executeQuick", java.util.UUID.class, + String.class, JsonObject.class, JsonObject.class); + execute.setAccessible(true); + Object result = execute.invoke(connector, java.util.UUID.randomUUID(), "APPLY", configuration, + new JsonObject()); + var json = result.getClass().getDeclaredMethod("json"); + json.setAccessible(true); + JsonObject response = (JsonObject) json.invoke(result); + assertFalse(response.get("success").getAsBoolean()); + assertEquals("UNSUPPORTED_TASK", response.get("code").getAsString()); + } + @Test void votePartyRequiresItsVersionedCapability() { assertFalse(BackendControlConnector.quickSetupCapabilityAccepted("vote-party", true, false, false)); assertFalse(BackendControlConnector.quickSetupCapabilityAccepted("vote-party", false, true, false)); From 4c990459491fea581817a359c60f8591cc991058 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:12:43 -0600 Subject: [PATCH 134/136] Preserve deferred proxy method and gate HTTP previews --- .../votingplugin/VotingPluginMain.java | 13 +++++++--- .../control/BackendControlConnector.java | 2 +- ...PluginMainBackendProxyPublicationTest.java | 26 +++++++++++++++++++ .../BackendControlConnectorProtocolTest.java | 18 +++++++------ 4 files changed, 47 insertions(+), 12 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 17e0f36a3..824bf0af0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1250,6 +1250,7 @@ public void restartBackendProxyHandler(long validationDeadlineNanos) { public static final class BackendProxyRestart { private final BackendProxyHandler previous; private final BackendProxyHandler replacement; + private BungeeMethod replacementMethod; private final boolean disabled; private final boolean previousRequiresPreparation; private volatile boolean previousPrepared; @@ -1277,9 +1278,14 @@ private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler re /** Only exclusive same-method transports may be restored by the Control worker. */ public boolean requiresWorkerRollback() { return previousPrepared && previous != null && replacement != null - && ((previous.getMethod() == BungeeMethod.SOCKETS && replacement.getMethod() == BungeeMethod.SOCKETS) + && ((previous.getMethod() == BungeeMethod.SOCKETS && effectiveReplacementMethod() == BungeeMethod.SOCKETS) || (previous.getMethod() == BungeeMethod.MQTT - && replacement.getMethod() == BungeeMethod.MQTT)); + && effectiveReplacementMethod() == BungeeMethod.MQTT)); + } + + private BungeeMethod effectiveReplacementMethod() { + BungeeMethod loadedMethod = replacement.getMethod(); + return loadedMethod != null ? loadedMethod : replacementMethod; } } @@ -1324,6 +1330,7 @@ public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { } } BackendProxyRestart restart = new BackendProxyRestart(previous, replacement, false, previousRequiresPreparation); + restart.replacementMethod = replacementMethod; restart.replacementLoadDeferred = deferredReplacementLoad; return restart; } @@ -1343,7 +1350,7 @@ public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long // worker join fails. Mark the restart first so abort/await still owns the // restoration path after a partially completed preparation. restart.previousPrepared = true; - BungeeMethod replacementMethod = restart.replacement == null ? null : restart.replacement.getMethod(); + BungeeMethod replacementMethod = restart.replacement == null ? null : restart.effectiveReplacementMethod(); if (restart.replacement != null) restart.replacement.beginPreparedHttpHandoff(); if (!restart.previous.prepareForReplacement(replacementMethod, validationDeadlineNanos)) throw new IllegalStateException("Previous proxy transport could not be prepared for replacement"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index f9cbd4093..82fbffe0c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -1081,7 +1081,7 @@ private TaskResult executeQuick(UUID operationId, String type, JsonObject config throws IOException { String preset = string(configuration, "preset"); Map options = options(configuration.getAsJsonObject("options")); - if ("APPLY".equals(type) && "proxy-method".equals(preset) + if ("proxy-method".equals(preset) && !proxyMethodApplyCapabilityAccepted(options.getOrDefault("method", "PLUGINMESSAGING"), proxyMethodV2Accepted)) { return TaskResult.failure("UNSUPPORTED_TASK", "The HTTP proxy method capability was not negotiated"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java index be99106a3..00caae4ab 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainBackendProxyPublicationTest.java @@ -31,6 +31,32 @@ import com.bencodez.votingplugin.proxy.BungeeMethod; class VotingPluginMainBackendProxyPublicationTest { + @Test + void deferredSameMethodRestartPreparesBeforeLoadingReplacement() throws Exception { + for (BungeeMethod method : new BungeeMethod[] { BungeeMethod.SOCKETS, BungeeMethod.MQTT }) { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + BackendProxyHandler previous = mock(BackendProxyHandler.class); + BungeeSettings settings = mock(BungeeSettings.class); + setBackendProxyHandler(plugin, previous); + setField(plugin, "bungeeSettings", settings); + when(settings.isUseBungeecoord()).thenReturn(true); + when(settings.getBungeeMethod()).thenReturn(method.name()); + when(previous.getMethod()).thenReturn(method); + when(previous.prepareForReplacement(org.mockito.ArgumentMatchers.eq(method), + org.mockito.ArgumentMatchers.anyLong())).thenReturn(true); + try (var constructed = org.mockito.Mockito.mockConstruction(BackendProxyHandler.class)) { + VotingPluginMain.BackendProxyRestart restart = plugin.prepareBackendProxyHandlerRestart(); + BackendProxyHandler replacement = constructed.constructed().get(0); + verify(replacement, never()).loadForReplacement(); + plugin.validateBackendProxyHandlerRestart(restart, System.nanoTime() + 1_000_000_000L); + verify(previous).prepareForReplacement(org.mockito.ArgumentMatchers.eq(method), + org.mockito.ArgumentMatchers.anyLong()); + verify(replacement).loadForReplacement(); + assertTrue(restart.requiresWorkerRollback()); + } + } + } + @Test void proxyMethodControlPreparationReloadsOnlyNarrowSettingsBeforeReturningTheStagedRestart() throws Exception { VotingPluginMain plugin = mock(VotingPluginMain.class); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java index f76f0c414..8509f9e7f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java @@ -304,7 +304,7 @@ private static JsonObject deploymentTask() { assertTrue(BackendControlConnector.proxyMethodApplyCapabilityAccepted("PLUGINMESSAGING", false)); } - @Test void httpQuickApplyRejectsMissingV2BeforeTouchingConfiguration() throws Exception { + @Test void httpQuickRequestsRejectMissingV2BeforeTouchingConfiguration() throws Exception { BackendControlConnector connector = org.mockito.Mockito.mock(BackendControlConnector.class, org.mockito.Mockito.CALLS_REAL_METHODS); var quick = BackendControlConnector.class.getDeclaredField("quickSetupsAccepted"); @@ -316,13 +316,15 @@ private static JsonObject deploymentTask() { var execute = BackendControlConnector.class.getDeclaredMethod("executeQuick", java.util.UUID.class, String.class, JsonObject.class, JsonObject.class); execute.setAccessible(true); - Object result = execute.invoke(connector, java.util.UUID.randomUUID(), "APPLY", configuration, - new JsonObject()); - var json = result.getClass().getDeclaredMethod("json"); - json.setAccessible(true); - JsonObject response = (JsonObject) json.invoke(result); - assertFalse(response.get("success").getAsBoolean()); - assertEquals("UNSUPPORTED_TASK", response.get("code").getAsString()); + for (String type : new String[] { "READ", "PREVIEW", "APPLY" }) { + Object result = execute.invoke(connector, java.util.UUID.randomUUID(), type, configuration, + new JsonObject()); + var json = result.getClass().getDeclaredMethod("json"); + json.setAccessible(true); + JsonObject response = (JsonObject) json.invoke(result); + assertFalse(response.get("success").getAsBoolean()); + assertEquals("UNSUPPORTED_TASK", response.get("code").getAsString()); + } } @Test void votePartyRequiresItsVersionedCapability() { From 88e785870fa1fc0609b35414d40a22f7837ee0f9 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:31:03 -0600 Subject: [PATCH 135/136] Keep live vote timestamps stable across retries --- .../votingplugin/proxy/VotingPluginProxy.java | 5 +++-- .../votingplugin/tests/VotingPluginProxyTest.java | 13 ++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index cbcef0759..be7053e6b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -5066,8 +5066,9 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea // Cache online state/server once (IMPORTANT for broadcast logic correctness) final boolean playerOnline = isPlayerOnlineForVoteRouting(player); final String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(player) : null; - long time = queueTime != 0 ? queueTime - : LocalDateTime.now().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + long time = retryState != null ? retryState.time + : (queueTime != 0 ? queueTime + : LocalDateTime.now().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()); Set broadcastTargets = queuedVote == null ? new LinkedHashSet<>() : new LinkedHashSet<>(queuedVote.getBroadcastTargets()); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index dd89bc90f..3b77956d2 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -324,7 +324,7 @@ void liveVotePropagatesRetryWhenProcessingCannotBeMadeDurable() { } @Test - void liveVoteRetryDoesNotApplyVotePartyTwice() { + void liveVoteRetryKeepsTimestampAndDoesNotApplyVotePartyTwice() throws Exception { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); Mockito.when(votingPluginProxy.getConfig().getSendVotesToAllServers()).thenReturn(true); votingPluginProxy.setMethod(BungeeMethod.HTTP); @@ -340,12 +340,19 @@ void liveVoteRetryDoesNotApplyVotePartyTwice() { java.util.UUID voteId = java.util.UUID.randomUUID(); assertThrows(VotingPluginProxy.VoteRetryException.class, - () -> spyProxy.vote("Player", "Service", true, false, 100L, null, + () -> spyProxy.vote("Player", "Service", true, false, 0L, null, "00000000-0000-0000-0000-000000000001", voteId)); + java.lang.reflect.Field retriesField = VotingPluginProxy.class.getDeclaredField("liveVoteRetries"); + retriesField.setAccessible(true); + Object retryState = ((java.util.Map) retriesField.get(spyProxy)).get(voteId); + java.lang.reflect.Field timeField = retryState.getClass().getDeclaredField("time"); + timeField.setAccessible(true); + timeField.setLong(retryState, 123456789L); spyProxy.setAvailableServers("Server1", "Server2", "Server3"); - spyProxy.vote("Player", "Service", true, false, 100L, null, + spyProxy.vote("Player", "Service", true, false, 0L, null, "00000000-0000-0000-0000-000000000001", voteId); + assertEquals("123456789", spyProxy.getLastVoteEnvelope().getFields().get(VotingPluginWire.K_TIME)); verify(spyProxy).addVoteParty(); verify(voteCache, Mockito.times(3)).addServerVoteDurably(Mockito.anyString(), Mockito.any()); } From f95f2a34de855768d720ffadd46ef385f4b77ed5 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:50:00 -0600 Subject: [PATCH 136/136] Normalize HTTP proxy capability selection --- .../proxy/control/ControlConnector.java | 3 ++- .../proxy/control/ControlConnectorTest.java | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 225d0d295..cd8b8c27f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -1177,7 +1177,8 @@ private static String requiredCapability(JsonObject configuration) { private static String proxyMethodCapability(JsonObject configuration) { JsonObject options = configuration == null ? null : configuration.getAsJsonObject("options"); - return options != null && options.has("method") && "HTTP".equals(options.get("method").getAsString()) + return options != null && options.has("method") + && "HTTP".equalsIgnoreCase(options.get("method").getAsString().trim()) ? PROXY_METHOD_HTTP_CAPABILITY : PROXY_METHOD_CAPABILITY; } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index 315447dfe..124dae39c 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -320,6 +320,31 @@ class ControlConnectorTest { assertFalse(ControlConnector.requiresRuntimeReplacement(new StoredResult(result, true, false))); } + @Test void lowercaseHttpProxyMethodRequiresV2ForEveryOperationPhase() throws Exception { + Field accepted = ControlConnector.class.getDeclaredField("acceptedCapabilities"); + accepted.setAccessible(true); + Method execute = ControlConnector.class.getDeclaredMethod("executeTask", UUID.class, JsonObject.class); + execute.setAccessible(true); + for (String method : List.of("http", " HTTP ")) { + for (String type : List.of("READ", "PREVIEW", "APPLY")) { + JsonObject task = JsonParser.parseString("{\"configuration\":{\"domain\":\"quick-setup\"," + + "\"preset\":\"proxy-method\",\"options\":{\"method\":\"" + method + "\"}}}") + .getAsJsonObject(); + task.addProperty("type", type); + accepted.set(connector, Set.of("config.proxy-method.v1")); + Object rejected = ((CompletableFuture) execute.invoke(connector, UUID.randomUUID(), task)).join(); + Method json = rejected.getClass().getDeclaredMethod("json"); + json.setAccessible(true); + assertTrue(((JsonObject) json.invoke(rejected)).get("message").getAsString() + .contains("not negotiated")); + accepted.set(connector, Set.of("config.proxy-method.v2")); + Object acceptedResult = ((CompletableFuture) execute.invoke(connector, UUID.randomUUID(), task)).join(); + assertTrue(((JsonObject) json.invoke(acceptedResult)).get("message").getAsString() + .contains("unavailable")); + } + } + } + @Test void proxyFileCapabilityAdvertisesAndDispatchesMaskedReadResults() throws Exception { Path file = dataDirectory.resolve(ProxyConfigurationFileService.FILE_NAME); Files.writeString(file, "Database:\n Password: local-secret\nProxy:\n Enabled: true\n");