From 567531eeef58ba49bb96d16953032f952651db35 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 30 Aug 2026 22:32:25 -0600 Subject: [PATCH 01/36] Add secure single-port HTTP proxy transport --- VotingPlugin/pom.xml | 17 +- .../http/HttpBackendTransportConnector.java | 204 +++++++++++ .../http/HttpClientCredentialStore.java | 162 +++++++++ .../backendproxy/http/HttpConnectionCode.java | 97 ++++++ .../http/HttpEnrollmentAuthority.java | 175 ++++++++++ .../backendproxy/http/HttpPinnedTls.java | 94 +++++ .../http/HttpProxyTransportServer.java | 252 ++++++++++++++ .../backendproxy/http/HttpTlsIdentity.java | 321 ++++++++++++++++++ .../http/HttpTransportProtocol.java | 174 ++++++++++ .../http/HttpTransportSecrets.java | 64 ++++ .../BackendProxyTransportManager.java | 3 + .../transport/HttpBackendProxyTransport.java | 124 +++++++ .../votingplugin/config/BungeeSettings.java | 6 +- .../control/BackendConfigurationService.java | 13 +- .../votingplugin/listeners/VotiferEvent.java | 5 +- .../votingplugin/proxy/BungeeMethod.java | 8 +- .../votingplugin/proxy/VotingPluginProxy.java | 69 +++- .../proxy/VotingPluginProxyCommand.java | 20 ++ .../proxy/VotingPluginProxyConfig.java | 15 + .../proxy/bungee/BungeeConfig.java | 21 +- .../ProxyMethodConfigurationService.java | 15 + .../proxy/velocity/VelocityConfig.java | 15 + .../src/main/resources/BungeeSettings.yml | 15 +- .../src/main/resources/bungeeconfig.yml | 22 +- .../http/HttpTransportRuntimeTest.java | 146 ++++++++ .../http/HttpTransportSecurityTest.java | 100 ++++++ .../BackendConfigurationServiceTest.java | 13 + .../ProxyMethodConfigurationServiceTest.java | 13 + .../votingplugin/tests/BungeeMethodTest.java | 9 + docs/http-transport.md | 50 +++ 30 files changed, 2221 insertions(+), 21 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecrets.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java create mode 100644 docs/http-transport.md diff --git a/VotingPlugin/pom.xml b/VotingPlugin/pom.xml index 465e2f743..646b1e8ac 100644 --- a/VotingPlugin/pom.xml +++ b/VotingPlugin/pom.xml @@ -15,6 +15,7 @@ 21 21 3.4.0 + 1.85 @@ -151,6 +152,10 @@ ${project.groupId}.votingplugin.bstats + + org.bouncycastle + ${project.groupId}.votingplugin.bouncycastle + xyz.upperlevel.spigot @@ -254,6 +259,16 @@ + + org.bouncycastle + bcpkix-jdk18on + ${bouncycastle.version} + + + org.bouncycastle + bcprov-jdk18on + ${bouncycastle.version} + org.spigotmc spigot-api @@ -692,4 +707,4 @@ - \ No newline at end of file + diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java new file mode 100644 index 000000000..67030e90b --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -0,0 +1,204 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.KeyStore; +import java.security.cert.X509Certificate; +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; + +/** Backend-side, persistent HTTP/1.1 long-poll connector. */ +public final class HttpBackendTransportConnector implements AutoCloseable { + public static final Duration CLIENT_TIMEOUT = Duration.ofSeconds(35); + private final HttpClientCredentialStore.HttpClientProfile profile; + private final String serverId; + private final Consumer onEnvelope; + private final HttpClient client; + private final URI transportEndpoint; + private final ThreadPoolExecutor callbackExecutor; + private final AtomicBoolean running = new AtomicBoolean(); + private final Object state = new Object(); + private final LinkedHashMap outgoing = new LinkedHashMap<>(); + private final Set received = new LinkedHashSet<>(), processing = new LinkedHashSet<>(); + private final ArrayDeque acknowledgements = new ArrayDeque<>(); + private final String session = UUID.randomUUID().toString(); + private volatile Thread poller; + private long sequence; + + public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, + HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { + this(profile(code, serverId), credential, onEnvelope); + } + + /** Starts normal transport from the non-secret profile persisted by enrollment. */ + public HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope) throws Exception { + this(enrolled == null ? null : enrolled.profile(), enrolled == null ? null : enrolled.credential(), onEnvelope); + } + + public HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { + if (profile == null || credential == null || onEnvelope == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); + if (!matchesCredential(profile, credential)) throw new IllegalArgumentException("HTTP client certificate does not match transport profile"); + this.profile = profile; this.serverId = profile.serverId(); this.onEnvelope = onEnvelope; + client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); + transportEndpoint = profile.endpoint().resolve("v1/transport"); + callbackExecutor = executor("VotingPlugin-HTTP-callback", 2, 128); + } + + /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ + public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, Path credentials, + Consumer onEnvelope) throws Exception { + this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope); + if (code == null || !profile(code, serverId).equals(this.profile)) throw new IllegalArgumentException("HTTP transport profile does not match connection code"); + } + + /** Starts normal transport using only the persisted certificate and non-secret profile. */ + public HttpBackendTransportConnector(Path credentials, Consumer onEnvelope) throws Exception { + this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope); + } + + /** Performs enrollment network I/O; call this from a connector/setup worker, never a platform main thread. */ + public static HttpClientCredentialStore.ClientCredential enroll(HttpConnectionCode code, String serverId, Path credentials) throws Exception { + if (code == null || credentials == null || serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) throw new IllegalArgumentException("Enrollment configuration is invalid"); + code.requireActive(Clock.systemUTC()); + byte[] payload = ("{\"server\":\"" + serverId + "\",\"token\":\"" + code.enrollmentToken() + "\"}").getBytes(StandardCharsets.UTF_8); + HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(Duration.ofSeconds(5)).sslContext(HttpPinnedTls.clientContext(code)).build(); + HttpRequest request = HttpRequest.newBuilder(code.endpoint().resolve("v1/enroll")).timeout(CLIENT_TIMEOUT) + .header("Content-Type", "application/json").header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(payload)).build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() != 201 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) throw new IllegalArgumentException("Enrollment was rejected"); + HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); + HttpClientCredentialStore.saveEnrolled(credentials, code, issued); return HttpClientCredentialStore.load(credentials); + } + + public void start() { + if (!running.compareAndSet(false, true)) return; + poller = new Thread(this::pollLoop, "VotingPlugin-HTTP-poll"); poller.setDaemon(true); poller.start(); + } + /** + * Inserts an in-memory at-least-once delivery. It survives retry/lost responses while this process remains alive; + * callers needing restart durability must retain the application operation independently. + */ + public boolean send(JsonEnvelope envelope) { + if (envelope == null || !running.get()) return false; + try { HttpTransportProtocol.validateEnvelope(envelope); } + catch (IllegalArgumentException invalid) { return false; } + synchronized (state) { + if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; + String id = UUID.randomUUID().toString(); outgoing.put(id, new HttpTransportProtocol.Delivery(id, envelope)); return true; + } + } + /** A synchronous single poll, useful for lifecycle-controlled integrations and tests. */ + public boolean pollOnce() { + if (!running.get()) return false; + try { + List acks; List messages; long requestSequence; + synchronized (state) { + acks = first(acknowledgements); requestSequence = sequence++; + messages = HttpTransportProtocol.fittingMessages(serverId, session, requestSequence, acks, outgoing.values()); + for (int index = 0; index < acks.size(); index++) acknowledgements.removeFirst(); + } + HttpRequest request = HttpRequest.newBuilder(transportEndpoint).timeout(CLIENT_TIMEOUT).header("Content-Type", "application/json") + .header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(HttpTransportProtocol.request(serverId, session, requestSequence, acks, messages))).build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() != 200 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) return false; + HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(response.body()); + if (!serverId.equals(packet.server()) || !session.equals(packet.session()) || packet.sequence() != requestSequence) return false; + synchronized (state) { for (String ack : packet.acks()) outgoing.remove(ack); } + for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); + return true; + } catch (Exception failure) { return false; } + } + @Override public void close() { + running.getAndSet(false); + Thread current = poller; if (current != null) current.interrupt(); + callbackExecutor.shutdown(); try { if (!callbackExecutor.awaitTermination(5, TimeUnit.SECONDS)) callbackExecutor.shutdownNow(); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); callbackExecutor.shutdownNow(); } + } + + private void pollLoop() { + long retry = 1000L; + while (running.get()) { if (pollOnce()) { retry = 1000L; continue; } try { Thread.sleep(retry); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); break; } retry = Math.min(30_000L, retry * 2); } + } + List accept(List deliveries) { + synchronized (state) { + List accepted = new java.util.ArrayList<>(); + for (HttpTransportProtocol.Delivery delivery : deliveries) { + if (received.contains(delivery.id())) { queueAck(delivery.id()); continue; } + if (!processing.contains(delivery.id())) { if (received.size() >= HttpTransportProtocol.MAX_QUEUE) break; processing.add(delivery.id()); accepted.add(delivery); } + } + return accepted; + } + } + void dispatch(HttpTransportProtocol.Delivery delivery) { + try { callbackExecutor.execute(() -> { boolean success = false; try { onEnvelope.accept(delivery.envelope()); success = true; } catch (RuntimeException ignored) { } + synchronized (state) { processing.remove(delivery.id()); if (success) { received.add(delivery.id()); while (received.size() > HttpTransportProtocol.MAX_QUEUE) received.remove(received.iterator().next()); queueAck(delivery.id()); } } + }); } catch (RejectedExecutionException rejected) { synchronized (state) { processing.remove(delivery.id()); } } + } + private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } + int queuedOutgoing() { synchronized (state) { return outgoing.size(); } } + List drainAcknowledgements() { synchronized (state) { return drain(acknowledgements); } } + private static List first(Collection values) { List output = new java.util.ArrayList<>(); for (T value : values) { output.add(value); if (output.size() == HttpTransportProtocol.MAX_BATCH) break; } return output; } + private static List drain(ArrayDeque values) { List output = new java.util.ArrayList<>(); while (!values.isEmpty() && output.size() < HttpTransportProtocol.MAX_BATCH) output.add(values.remove()); return output; } + private static ThreadPoolExecutor executor(String name, int threads, int queue) { ThreadFactory factory = task -> { Thread thread = new Thread(task, name); thread.setDaemon(true); return thread; }; return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); } + private static HttpClientCredentialStore.HttpClientProfile profile(HttpConnectionCode code, String serverId) { + if (code == null || serverId == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); + if (!code.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalArgumentException("HTTP connection code belongs to a different backend"); + return new HttpClientCredentialStore.HttpClientProfile(serverId, code.endpoint(), code.serverCertificatePin(), code.caCertificatePin()); + } + private static boolean matchesCredential(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential) { + try { + credential.certificate().checkValidity(); credential.certificate().verify(credential.caCertificate().getPublicKey()); + String expected = "urn:votingplugin:http-backend:" + profile.serverId(); + var names = credential.certificate().getSubjectAlternativeNames(); if (names == null) return false; + for (java.util.List name : names) if (name.size() == 2 && Integer.valueOf(6).equals(name.get(0)) && expected.equals(name.get(1))) return true; + return false; + } catch (Exception invalid) { return false; } + } + private static SSLContext clientContext(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { + KeyStore store = KeyStore.getInstance("PKCS12"); store.load(null, new char[0]); + store.setKeyEntry("client", credential.privateKey(), credential.password(), new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); + KeyManagerFactory keys = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keys.init(store, credential.password()); + SSLContext context = SSLContext.getInstance("TLS"); context.init(keys.getKeyManagers(), new javax.net.ssl.TrustManager[] { new PinnedTrustManager(profile) }, null); return context; + } + private static final class PinnedTrustManager implements javax.net.ssl.X509TrustManager { + private final HttpClientCredentialStore.HttpClientProfile profile; PinnedTrustManager(HttpClientCredentialStore.HttpClientProfile profile) { this.profile = profile; } + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } + @Override public void checkServerTrusted(X509Certificate[] chain, String authType) throws java.security.cert.CertificateException { + if (chain == null || chain.length < 2) throw new java.security.cert.CertificateException("Pinned HTTPS server chain is incomplete"); + chain[0].checkValidity(); chain[chain.length - 1].checkValidity(); + try { chain[0].verify(chain[chain.length - 1].getPublicKey()); } + catch (java.security.GeneralSecurityException invalid) { throw new java.security.cert.CertificateException("Pinned HTTPS server signature is invalid", invalid); } + if (chain[chain.length - 1].getBasicConstraints() < 0 || !HttpTransportSecrets.constantTimeEquals(profile.serverCertificatePin().getBytes(StandardCharsets.US_ASCII), HttpTransportSecrets.certificatePin(chain[0]).getBytes(StandardCharsets.US_ASCII))) throw new java.security.cert.CertificateException("Pinned HTTPS server mismatch"); + String caPin = HttpTransportSecrets.certificatePin(chain[chain.length - 1]); + if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), caPin.getBytes(StandardCharsets.US_ASCII))) + throw new java.security.cert.CertificateException("Pinned HTTPS authority mismatch"); + } + @Override public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java new file mode 100644 index 000000000..ae801ae0a --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -0,0 +1,162 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.net.URI; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.Base64; +import java.util.EnumSet; +import java.util.Properties; +import com.bencodez.votingplugin.util.DurableFiles; + +/** Owner-only persistence for the client certificate bundle returned by enrollment. */ +public final class HttpClientCredentialStore { + private static final String BUNDLE_FILE = "http-transport-client.p12"; + private static final String PASSWORD_FILE = "http-transport-client-password"; + private static final String PROFILE_FILE = "http-transport-profile.properties"; + private HttpClientCredentialStore() { } + + public static void save(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws IOException { + if (issued == null) throw new IllegalArgumentException("Issued credential is required"); + Files.createDirectories(directory); + byte[] bundle = issued.pkcs12(); + try { writePrivate(safe(directory.resolve(BUNDLE_FILE)), bundle); } + finally { java.util.Arrays.fill(bundle, (byte) 0); } + char[] password = issued.password(); + try { writePrivate(safe(directory.resolve(PASSWORD_FILE)), asciiBytes(password)); } + finally { java.util.Arrays.fill(password, '\0'); } + } + + /** Persists the certificate plus the non-secret normal-transport profile after enrollment. */ + public static void saveEnrolled(Path directory, HttpConnectionCode code, HttpTlsIdentity.IssuedClientCertificate issued) + throws IOException { + if (code == null || issued == null) throw new IllegalArgumentException("Connection code and credential are required"); + save(directory, issued); + HttpClientProfile profile = new HttpClientProfile(HttpTlsIdentity.canonicalServerId(issued.serverId()), code.endpoint(), + code.serverCertificatePin(), code.caCertificatePin()); + Properties properties = new Properties(); + properties.setProperty("version", "1"); + properties.setProperty("serverId", profile.serverId()); + properties.setProperty("endpoint", profile.endpoint().toASCIIString()); + properties.setProperty("serverPin", profile.serverCertificatePin()); + properties.setProperty("caPin", profile.caCertificatePin()); + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + properties.store(bytes, "VotingPlugin HTTP transport profile"); + writePrivate(safe(directory.resolve(PROFILE_FILE)), bytes.toByteArray()); + } + + public static ClientCredential load(Path directory) throws Exception { + Path bundle = safe(directory.resolve(BUNDLE_FILE)); + Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); + if (!Files.isRegularFile(bundle, LinkOption.NOFOLLOW_LINKS) || !Files.isRegularFile(passwordFile, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP client certificate has not been enrolled"); + byte[] passwordBytes = Files.readAllBytes(passwordFile); + if (passwordBytes.length < 40 || passwordBytes.length > 128) throw new IOException("HTTP client password is invalid"); + char[] password = new String(passwordBytes, StandardCharsets.US_ASCII).toCharArray(); + java.util.Arrays.fill(passwordBytes, (byte) 0); + try { + KeyStore store = KeyStore.getInstance("PKCS12"); + try (var input = Files.newInputStream(bundle, LinkOption.NOFOLLOW_LINKS)) { store.load(input, password); } + PrivateKey privateKey = (PrivateKey) store.getKey("client", password); + java.security.cert.Certificate[] chain = store.getCertificateChain("client"); + if (privateKey == null || chain == null || chain.length != 2 + || !(chain[0] instanceof X509Certificate client) || !(chain[1] instanceof X509Certificate authority)) + throw new IOException("HTTP client certificate bundle is invalid"); + return new ClientCredential(privateKey, client, authority, password); + } finally { java.util.Arrays.fill(password, '\0'); } + } + + public static HttpClientProfile loadProfile(Path directory) throws IOException { + Path profile = safe(directory.resolve(PROFILE_FILE)); + if (!Files.isRegularFile(profile, LinkOption.NOFOLLOW_LINKS) || Files.size(profile) > 8192) + throw new IOException("HTTP transport profile has not been enrolled"); + Properties properties = new Properties(); + try (var input = Files.newInputStream(profile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } + if (properties.size() != 5 || !"1".equals(properties.getProperty("version"))) + throw new IOException("HTTP transport profile is invalid"); + try { + return new HttpClientProfile(properties.getProperty("serverId"), URI.create(properties.getProperty("endpoint")), + properties.getProperty("serverPin"), properties.getProperty("caPin")); + } catch (IllegalArgumentException failure) { throw new IOException("HTTP transport profile is invalid", failure); } + } + + /** Loads and cross-checks the persisted client key material and bound normal-transport profile. */ + public static EnrolledClient loadEnrolled(Path directory) throws Exception { + ClientCredential credential = load(directory); + HttpClientProfile profile = loadProfile(directory); + if (!matchesProfile(credential, profile)) throw new IOException("HTTP client certificate does not match its profile"); + return new EnrolledClient(profile, credential); + } + + public record ClientCredential(PrivateKey privateKey, X509Certificate certificate, X509Certificate caCertificate, char[] password) { + public ClientCredential { password = password.clone(); } + @Override public char[] password() { return password.clone(); } + } + + public record HttpClientProfile(String serverId, URI endpoint, String serverCertificatePin, String caCertificatePin) { + public HttpClientProfile { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + HttpConnectionCode validation = new HttpConnectionCode(serverId, endpoint, serverCertificatePin, caCertificatePin, + java.time.Instant.now().plusSeconds(1), HttpTransportSecrets.randomToken()); + endpoint = validation.endpoint(); + serverCertificatePin = validation.serverCertificatePin(); + caCertificatePin = validation.caCertificatePin(); + } + } + + public record EnrolledClient(HttpClientProfile profile, ClientCredential credential) { } + + private static boolean matchesProfile(ClientCredential credential, HttpClientProfile profile) { + try { + credential.certificate().checkValidity(); + credential.certificate().verify(credential.caCertificate().getPublicKey()); + java.util.List usage = credential.certificate().getExtendedKeyUsage(); + boolean[] keyUsage = credential.certificate().getKeyUsage(); + if (usage == null || !usage.contains(org.bouncycastle.asn1.x509.KeyPurposeId.id_kp_clientAuth.getId()) + || keyUsage == null || !keyUsage[0]) return false; + String expected = "urn:votingplugin:http-backend:" + profile.serverId(); + var names = credential.certificate().getSubjectAlternativeNames(); + if (names == null) return false; + for (java.util.List name : names) if (name.size() == 2 + && Integer.valueOf(6).equals(name.get(0)) && expected.equals(name.get(1))) return true; + return false; + } catch (Exception invalid) { return false; } + } + + private static Path safe(Path file) throws IOException { + if (Files.isSymbolicLink(file)) throw new IOException("Refusing unsafe HTTP credential path"); + return file.toAbsolutePath().normalize(); + } + + private static void writePrivate(Path file, byte[] contents) throws IOException { + Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); + try { + setOwnerOnly(temporary); + Files.write(temporary, contents, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); } + setOwnerOnly(file); + DurableFiles.forceDirectory(file.getParent()); + } finally { Files.deleteIfExists(temporary); } + } + + private static void setOwnerOnly(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { } + } + + private static byte[] asciiBytes(char[] characters) { + byte[] output = new byte[characters.length]; + for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; + return output; + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java new file mode 100644 index 000000000..812751426 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java @@ -0,0 +1,97 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.Instant; +import java.util.Base64; +import java.util.Locale; + +/** + * A copy/paste connection code. It deliberately includes no long-lived credential: + * its only secret is a short-lived, single-use enrollment token. The trailing MAC is a + * corruption check keyed by that included token; it is not a proxy signature and cannot stop + * someone who can replace the whole code from replacing it with another valid code. + */ +public record HttpConnectionCode(String serverId, URI endpoint, String serverCertificatePin, String caCertificatePin, + Instant expiresAt, String enrollmentToken) { + private static final String VERSION = "VPH1"; + private static final int MAX_CODE_LENGTH = 4096; + + public HttpConnectionCode { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + endpoint = validateEndpoint(endpoint); + serverCertificatePin = validatePin(serverCertificatePin, "server certificate pin"); + caCertificatePin = validatePin(caCertificatePin, "CA certificate pin"); + if (expiresAt == null) throw new IllegalArgumentException("Expiry is required"); + enrollmentToken = validateToken(enrollmentToken); + } + + public String encode() { + String endpointPart = Base64.getUrlEncoder().withoutPadding() + .encodeToString(endpoint.toASCIIString().getBytes(StandardCharsets.UTF_8)); + String unsigned = String.join(".", VERSION, serverId, endpointPart, serverCertificatePin, caCertificatePin, + Long.toString(expiresAt.getEpochSecond()), enrollmentToken); + byte[] token = Base64.getUrlDecoder().decode(enrollmentToken); + return unsigned + "." + HttpTransportSecrets.hmacSha256Url(token, unsigned); + } + + public boolean isActive(Clock clock) { + return expiresAt.isAfter(clock.instant()); + } + + public void requireActive(Clock clock) { + if (!isActive(clock)) throw new IllegalArgumentException("Connection code has expired"); + } + + public static HttpConnectionCode parse(String code) { + if (code == null || code.length() > MAX_CODE_LENGTH || code.indexOf('\n') >= 0 || code.indexOf('\r') >= 0) + throw new IllegalArgumentException("Connection code is invalid"); + String[] parts = code.split("\\.", -1); + if (parts.length != 8 || !VERSION.equals(parts[0])) throw new IllegalArgumentException("Connection code is invalid"); + try { + String endpoint = new String(Base64.getUrlDecoder().decode(parts[2]), StandardCharsets.UTF_8); + String unsigned = String.join(".", parts[0], parts[1], parts[2], parts[3], parts[4], parts[5], parts[6]); + byte[] token = Base64.getUrlDecoder().decode(parts[6]); + String expected = HttpTransportSecrets.hmacSha256Url(token, unsigned); + if (!HttpTransportSecrets.constantTimeEquals(expected.getBytes(StandardCharsets.US_ASCII), + parts[7].getBytes(StandardCharsets.US_ASCII))) throw new IllegalArgumentException("Connection code is invalid"); + return new HttpConnectionCode(parts[1], new URI(endpoint), parts[3], parts[4], Instant.ofEpochSecond(Long.parseLong(parts[5])), + parts[6]); + } catch (IllegalArgumentException | URISyntaxException failure) { + throw new IllegalArgumentException("Connection code is invalid", failure); + } + } + + private static URI validateEndpoint(URI value) { + if (value == null || !"https".equalsIgnoreCase(value.getScheme()) || value.getHost() == null + || value.getUserInfo() != null || value.getFragment() != null || value.getRawQuery() != null) + throw new IllegalArgumentException("Endpoint must be an absolute HTTPS URL without credentials or query"); + if (value.getPort() > 65535 || value.getPort() < -1) throw new IllegalArgumentException("Endpoint port is invalid"); + String path = value.getRawPath(); + if (path == null || path.isEmpty()) path = "/"; + if (!path.endsWith("/")) path += "/"; + try { + return new URI("https", null, value.getHost().toLowerCase(Locale.ROOT), value.getPort(), path, null, null); + } catch (URISyntaxException failure) { + throw new IllegalArgumentException("Endpoint is invalid", failure); + } + } + + private static String validatePin(String pin, String name) { + if (pin == null || !pin.matches("[0-9a-fA-F]{64}")) throw new IllegalArgumentException(name + " is invalid"); + return pin.toLowerCase(Locale.ROOT); + } + + private static String validateToken(String token) { + if (token == null || token.length() < 43 || token.length() > 128 || !token.matches("[A-Za-z0-9_-]+")) + throw new IllegalArgumentException("Enrollment token is invalid"); + try { + if (Base64.getUrlDecoder().decode(token).length < 32) throw new IllegalArgumentException("Enrollment token is invalid"); + return token; + } catch (IllegalArgumentException failure) { + throw new IllegalArgumentException("Enrollment token is invalid", failure); + } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java new file mode 100644 index 000000000..5d16a0eeb --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java @@ -0,0 +1,175 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.Properties; +import java.util.Base64; +import java.util.EnumSet; +import com.bencodez.votingplugin.util.DurableFiles; + +/** + * Single-use enrollment tokens and client-certificate binding. Token material is never retained; + * only SHA-256 hashes are kept until expiry. This type is thread-safe. + */ +public final class HttpEnrollmentAuthority { + private static final Duration MAX_ENROLLMENT_LIFETIME = Duration.ofMinutes(15); + private final HttpTlsIdentity identity; + private final Clock clock; + private final Path stateFile; + private final Map enrollments = new HashMap<>(); + private final Map bindings = new HashMap<>(); + private final Set revokedCertificatePins = new HashSet<>(); + private boolean persistenceFailure; + + /** Creates a restart-safe authority. State contains only public certificate pins and revocations. */ + public HttpEnrollmentAuthority(HttpTlsIdentity identity, Path stateDirectory) throws java.io.IOException { + this(identity, Clock.systemUTC(), stateFile(stateDirectory)); + loadState(); + } + + HttpEnrollmentAuthority(HttpTlsIdentity identity, Clock clock) { + this(identity, clock, null); + } + + HttpEnrollmentAuthority(HttpTlsIdentity identity, Clock clock, Path stateFile) { + if (identity == null || clock == null) throw new IllegalArgumentException("Identity and clock are required"); + this.identity = identity; + this.clock = clock; + this.stateFile = stateFile; + } + + public synchronized HttpConnectionCode createConnectionCode(String serverId, URI endpoint, Duration lifetime) { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + if (lifetime == null || lifetime.isNegative() || lifetime.isZero() || lifetime.compareTo(MAX_ENROLLMENT_LIFETIME) > 0) + throw new IllegalArgumentException("Enrollment lifetime must be between one second and fifteen minutes"); + expireEnrollments(); + Instant expiresAt = clock.instant().plus(lifetime); + String token = HttpTransportSecrets.randomToken(); + byte[] tokenHash = HttpTransportSecrets.sha256(token.getBytes(StandardCharsets.US_ASCII)); + String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(tokenHash); + enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId)); + return new HttpConnectionCode(serverId, endpoint, identity.serverCertificatePin(), identity.caCertificatePin(), expiresAt, token); + } + + public synchronized HttpTlsIdentity.IssuedClientCertificate enroll(String serverId, String enrollmentToken) throws Exception { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + if (enrollmentToken == null || enrollmentToken.length() > 128) throw new IllegalArgumentException("Enrollment was rejected"); + expireEnrollments(); + byte[] suppliedHash = HttpTransportSecrets.sha256(enrollmentToken.getBytes(StandardCharsets.US_ASCII)); + String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(suppliedHash); + Enrollment enrollment = enrollments.remove(lookup); // consume before issuing: failed retries require a fresh code. + if (enrollment == null || !HttpTransportSecrets.constantTimeEquals(enrollment.tokenHash(), suppliedHash)) + throw new IllegalArgumentException("Enrollment was rejected"); + if (!serverId.equals(enrollment.serverId())) throw new IllegalArgumentException("Enrollment was rejected"); + ClientBinding existing = bindings.get(serverId); + if (existing != null && !existing.revoked()) throw new IllegalStateException("Server id is already enrolled"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); + bindings.put(serverId, new ClientBinding(HttpTransportSecrets.certificatePin(issued.certificate()), false)); + try { persistState(); } + catch (java.io.IOException failure) { persistenceFailure = true; throw failure; } + return issued; + } + + public synchronized boolean authenticate(String serverId, java.security.cert.X509Certificate certificate) { + if (persistenceFailure || serverId == null || certificate == null) return false; + try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } + catch (IllegalArgumentException invalid) { return false; } + if (!identity.validClientCertificate(serverId, certificate)) return false; + ClientBinding binding = bindings.get(serverId); + String pin = HttpTransportSecrets.certificatePin(certificate); + return binding != null && !binding.revoked() && !revokedCertificatePins.contains(pin) + && HttpTransportSecrets.constantTimeEquals(binding.certificatePin().getBytes(StandardCharsets.US_ASCII), + pin.getBytes(StandardCharsets.US_ASCII)); + } + + public synchronized void revoke(String serverId) { + try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } + catch (IllegalArgumentException invalid) { return; } + ClientBinding binding = bindings.get(serverId); + if (binding != null) { + bindings.put(serverId, new ClientBinding(binding.certificatePin(), true)); + revokedCertificatePins.add(binding.certificatePin()); + try { persistState(); } + catch (java.io.IOException failure) { persistenceFailure = true; throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); } + } + } + + private synchronized void loadState() throws java.io.IOException { + if (stateFile == null || !Files.exists(stateFile, LinkOption.NOFOLLOW_LINKS)) return; + if (!Files.isRegularFile(stateFile, LinkOption.NOFOLLOW_LINKS) || Files.size(stateFile) > 65536) + throw new java.io.IOException("HTTP enrollment state is invalid"); + Properties properties = new Properties(); + try (var input = Files.newInputStream(stateFile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } + for (String key : properties.stringPropertyNames()) { + if (key.startsWith("binding.")) { + String serverId = new String(Base64.getUrlDecoder().decode(key.substring("binding.".length())), StandardCharsets.UTF_8); + serverId = HttpTlsIdentity.canonicalServerId(serverId); + String[] value = properties.getProperty(key, "").split(":", -1); + if (value.length != 2 || !value[0].matches("[0-9a-f]{64}") || !("0".equals(value[1]) || "1".equals(value[1]))) + throw new java.io.IOException("HTTP enrollment state is invalid"); + bindings.put(serverId, new ClientBinding(value[0], "1".equals(value[1]))); + if ("1".equals(value[1])) revokedCertificatePins.add(value[0]); + } else if (!"version".equals(key)) throw new java.io.IOException("HTTP enrollment state is invalid"); + } + if (!"1".equals(properties.getProperty("version"))) throw new java.io.IOException("HTTP enrollment state is invalid"); + } + + private synchronized void persistState() throws java.io.IOException { + if (stateFile == null) return; + Properties properties = new Properties(); + properties.setProperty("version", "1"); + for (Map.Entry entry : bindings.entrySet()) { + String key = Base64.getUrlEncoder().withoutPadding().encodeToString(entry.getKey().getBytes(StandardCharsets.UTF_8)); + properties.setProperty("binding." + key, entry.getValue().certificatePin() + ":" + (entry.getValue().revoked() ? "1" : "0")); + } + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + properties.store(bytes, "VotingPlugin HTTP certificate bindings"); + Path temporary = Files.createTempFile(stateFile.getParent(), stateFile.getFileName().toString(), ".tmp"); + try { + setOwnerOnly(temporary); + Files.write(temporary, bytes.toByteArray(), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, stateFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, stateFile, StandardCopyOption.REPLACE_EXISTING); } + setOwnerOnly(stateFile); + DurableFiles.forceDirectory(stateFile.getParent()); + } finally { Files.deleteIfExists(temporary); } + } + + private static Path stateFile(Path directory) throws java.io.IOException { + if (directory == null) throw new IllegalArgumentException("State directory is required"); + Files.createDirectories(directory); + Path file = directory.toAbsolutePath().normalize().resolve("http-transport-clients.properties"); + if (Files.isSymbolicLink(file)) throw new java.io.IOException("Refusing unsafe HTTP enrollment state path"); + return file; + } + + private static void setOwnerOnly(Path path) throws java.io.IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { } + } + + private void expireEnrollments() { + Instant now = clock.instant(); + enrollments.entrySet().removeIf(entry -> !entry.getValue().expiresAt().isAfter(now)); + } + + private record Enrollment(byte[] tokenHash, Instant expiresAt, String serverId) { + private Enrollment { tokenHash = tokenHash.clone(); } + @Override public byte[] tokenHash() { return tokenHash.clone(); } + } + private record ClientBinding(String certificatePin, boolean revoked) { } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java new file mode 100644 index 000000000..1aaa04cab --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java @@ -0,0 +1,94 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import java.security.cert.X509Certificate; +import java.security.KeyStore; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +/** Builds the backend TLS context. A public CA store is intentionally not consulted. */ +public final class HttpPinnedTls { + private HttpPinnedTls() { } + + public static SSLContext clientContext(HttpConnectionCode code) throws Exception { + if (code == null) throw new IllegalArgumentException("Connection code is required"); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, new TrustManager[] { new PinnedServerTrustManager(code.serverCertificatePin(), code.caCertificatePin()) }, null); + return context; + } + + /** + * Normal transport context: presents the enrolled client certificate and accepts only the + * proxy leaf/CA pins in the connection code. Callers must not override the HttpClient default + * endpoint-identification settings; hostname verification remains enabled. + */ + public static SSLContext mutualTlsContext(HttpConnectionCode code, HttpClientCredentialStore.ClientCredential credential) + throws Exception { + if (code == null || credential == null || credential.privateKey() == null || credential.certificate() == null || credential.caCertificate() == null) + throw new IllegalArgumentException("Enrolled client credential is required"); + char[] password = credential.password(); + try { + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, new char[0]); + store.setKeyEntry("client", credential.privateKey(), password, + new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); + KeyManagerFactory managers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + managers.init(store, password); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(managers.getKeyManagers(), new TrustManager[] { + new PinnedServerTrustManager(code.serverCertificatePin(), code.caCertificatePin()) }, null); + return context; + } finally { java.util.Arrays.fill(password, '\0'); } + } + + public static boolean matchesServerPin(HttpConnectionCode code, X509Certificate certificate) { + if (code == null || certificate == null) return false; + String actual = HttpTransportSecrets.certificatePin(certificate); + return HttpTransportSecrets.constantTimeEquals(code.serverCertificatePin() + .getBytes(java.nio.charset.StandardCharsets.US_ASCII), actual.getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + } + + /** TLS 1.3 is used where the runtime exposes it; hostname verification is deliberately left enabled. */ + public static SSLParameters secureParameters(SSLContext context) { + SSLParameters parameters = context.getDefaultSSLParameters(); + for (String protocol : context.getSupportedSSLParameters().getProtocols()) { + if ("TLSv1.3".equals(protocol)) { + parameters.setProtocols(new String[] { "TLSv1.3" }); + break; + } + } + return parameters; + } + + private static final class PinnedServerTrustManager implements X509TrustManager { + private final String expectedPin; + private final String expectedCaPin; + private PinnedServerTrustManager(String expectedPin, String expectedCaPin) { + this.expectedPin = expectedPin; + this.expectedCaPin = expectedCaPin; + } + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } + @Override public void checkServerTrusted(X509Certificate[] chain, String authType) throws java.security.cert.CertificateException { + if (chain == null || chain.length < 2) throw new java.security.cert.CertificateException("Server certificate chain is incomplete"); + chain[0].checkValidity(); + chain[chain.length - 1].checkValidity(); + try { chain[0].verify(chain[chain.length - 1].getPublicKey()); } + catch (java.security.GeneralSecurityException invalid) { + throw new java.security.cert.CertificateException("Server certificate signature is invalid", invalid); + } + if (chain[chain.length - 1].getBasicConstraints() < 0) + throw new java.security.cert.CertificateException("Server certificate authority is invalid"); + String actual = HttpTransportSecrets.certificatePin(chain[0]); + if (!HttpTransportSecrets.constantTimeEquals(expectedPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII), + actual.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) + throw new java.security.cert.CertificateException("Server certificate pin does not match"); + String issuer = HttpTransportSecrets.certificatePin(chain[chain.length - 1]); + if (!HttpTransportSecrets.constantTimeEquals(expectedCaPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII), + issuer.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) + throw new java.security.cert.CertificateException("Server certificate authority pin does not match"); + } + @Override public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java new file mode 100644 index 000000000..542d1ab6a --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -0,0 +1,252 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsExchange; +import com.sun.net.httpserver.HttpsParameters; +import com.sun.net.httpserver.HttpsServer; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLPeerUnverifiedException; + +/** + * One HTTPS listener for enrollment and the backend-to-proxy long-poll transport. + * Every normal request is certificate-authenticated in the handler, rather than relying on TLS WANT auth. + */ +public final class HttpProxyTransportServer implements AutoCloseable { + static { + // JDK HttpServer reads these once when its internal server configuration is initialized. + // Set conservative process-wide bounds before this transport creates its listener. + setDefault("sun.net.httpserver.maxReqTime", "10"); + setDefault("sun.net.httpserver.maxRspTime", "10"); + setDefault("jdk.httpserver.maxConnections", "144"); + setDefault("sun.net.httpserver.maxReqHeaders", "32"); + setDefault("sun.net.httpserver.maxReqHeaderSize", "16384"); + } + // Keep an idle request open long enough to reuse the TLS connection, but bound backend-origin + // latency when a message is queued immediately after the request body has already been sent. + public static final Duration LONG_POLL = Duration.ofSeconds(2); + private final HttpTlsIdentity identity; + private final HttpEnrollmentAuthority authority; + private final HttpsServer server; + private final ThreadPoolExecutor listenerExecutor; + private final ThreadPoolExecutor handlerExecutor; + private final Semaphore admission = new Semaphore(64); + private final Map backends = new HashMap<>(); + private final Consumer onEnvelope; + private volatile boolean closed; + + public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Consumer onEnvelope) throws Exception { + if (bind == null || identity == null || authority == null || onEnvelope == null) throw new IllegalArgumentException("HTTP transport configuration is required"); + this.identity = identity; this.authority = authority; this.onEnvelope = onEnvelope; + server = HttpsServer.create(bind, 32); + server.setHttpsConfigurator(new HttpsConfigurator(identity.serverContext()) { + @Override public void configure(HttpsParameters parameters) { + SSLParameters ssl = HttpPinnedTls.secureParameters(getSSLContext()); + ssl.setWantClientAuth(true); parameters.setSSLParameters(ssl); + } + }); + // Long polls are blocking by design. Capacity is bounded by admission, while enough workers + // remain available for all admitted polls plus setup requests. + listenerExecutor = executor("VotingPlugin-HTTP-listener", 72, 72); + handlerExecutor = executor("VotingPlugin-HTTP-handler", 4, 128); + server.setExecutor(listenerExecutor); + server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); + server.createContext("/v1/transport", exchange -> transport((HttpsExchange) exchange)); + } + + public void start() { if (closed) throw new IllegalStateException("HTTP transport is closed"); server.start(); } + public int port() { return server.getAddress().getPort(); } + public URI endpoint(String host) { return URI.create("https://" + host + ":" + port() + "/"); } + + /** Queues an in-memory proxy-origin envelope for a specific authenticated backend; this queue is not restart-durable. */ + public boolean send(String serverId, JsonEnvelope envelope) { + if (closed || serverId == null || envelope == null) return false; + try { serverId = HttpTlsIdentity.canonicalServerId(serverId); HttpTransportProtocol.validateEnvelope(envelope); } + catch (IllegalArgumentException invalid) { return false; } + synchronized (backends) { + BackendState backend = backends.computeIfAbsent(serverId, ignored -> new BackendState()); + boolean accepted = backend.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), envelope)); + if (accepted) backend.signal(); + return accepted; + } + } + + @Override public void close() { + if (closed) return; closed = true; server.stop(1); + shutdown(handlerExecutor); shutdown(listenerExecutor); + synchronized (backends) { for (BackendState backend : backends.values()) backend.signal(); backends.clear(); } + } + + private void enroll(HttpsExchange exchange) throws IOException { + if (!"/v1/enroll".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } + if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } + if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } + if (!boundedFixedBody(exchange, 8192) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + try { + HttpTransportProtocol.Enrollment request = HttpTransportProtocol.parseEnrollment(read(exchange.getRequestBody(), 8192)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll(request.server(), request.token()); + reply(exchange, 201, HttpTransportProtocol.enrollmentResponse(issued)); + } catch (Exception rejected) { reply(exchange, 403, new byte[0]); } + finally { admission.release(); } + } + + private void transport(HttpsExchange exchange) throws IOException { + if (!"/v1/transport".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } + if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } + if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } + if (!boundedFixedBody(exchange, HttpTransportProtocol.MAX_BODY_BYTES) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + try { + HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(read(exchange.getRequestBody(), HttpTransportProtocol.MAX_BODY_BYTES)); + X509Certificate certificate = peerCertificate(exchange); + if (certificate == null || !authority.authenticate(packet.server(), certificate)) { reply(exchange, 401, new byte[0]); return; } + BackendState backend; + synchronized (backends) { backend = backends.computeIfAbsent(packet.server(), ignored -> new BackendState()); } + if (!backend.beginPoll(packet.session())) { reply(exchange, 409, new byte[0]); return; } + try { + handlePacket(packet, backend); + Response response = backend.await(packet.server(), packet.session(), packet.sequence()); + reply(exchange, 200, HttpTransportProtocol.response(packet.server(), packet.session(), packet.sequence(), response.acks(), response.messages())); + } finally { backend.endPoll(); } + } catch (IllegalArgumentException rejected) { reply(exchange, 400, new byte[0]); + } catch (Exception failure) { reply(exchange, 503, new byte[0]); + } finally { admission.release(); } + } + + private void handlePacket(HttpTransportProtocol.Packet packet, BackendState backend) { + List accepted; + synchronized (backend) { + if (!backend.allowRequest()) throw new IllegalArgumentException("transport rate limited"); + if (!backend.acceptSession(packet.session(), packet.sequence())) throw new IllegalArgumentException("stale session request"); + backend.acknowledge(packet.acks()); accepted = backend.acceptIncoming(packet.messages()); + } + for (HttpTransportProtocol.Delivery delivery : accepted) dispatch(packet.server(), backend, delivery); + } + private void dispatch(String serverId, BackendState backend, HttpTransportProtocol.Delivery delivery) { + try { handlerExecutor.execute(() -> { + boolean success = false; + try { onEnvelope.accept(new ReceivedEnvelope(serverId, delivery.id(), normalizeBackendIdentity(serverId, delivery.envelope()))); success = true; } + catch (RuntimeException ignored) { } + synchronized (backend) { backend.completeIncoming(delivery.id(), success); } + }); } catch (RejectedExecutionException rejected) { synchronized (backend) { backend.completeIncoming(delivery.id(), false); } } + } + private static JsonEnvelope normalizeBackendIdentity(String serverId, JsonEnvelope envelope) { + // The authenticated TLS identity is authoritative; never forward a forged `server` field. + return envelope.toBuilder().put("server", serverId).build(); + } + private static X509Certificate peerCertificate(HttpsExchange exchange) { + try { Certificate[] peer = exchange.getSSLSession().getPeerCertificates(); + return peer.length > 0 && peer[0] instanceof X509Certificate certificate ? certificate : null; + } catch (SSLPeerUnverifiedException absent) { return null; } + } + private static byte[] read(InputStream input, int maximum) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); byte[] buffer = new byte[4096]; int total = 0, read; + while ((read = input.read(buffer)) >= 0) { total += read; if (total > maximum) throw new IllegalArgumentException("HTTP body is too large"); output.write(buffer, 0, read); } + return output.toByteArray(); + } + private static void reply(HttpsExchange exchange, int status, byte[] body) throws IOException { + Headers headers = exchange.getResponseHeaders(); headers.set("Cache-Control", "no-store"); headers.set("Content-Type", "application/json; charset=utf-8"); + exchange.sendResponseHeaders(status, body.length); try (var output = exchange.getResponseBody()) { output.write(body); } + } + private static boolean json(HttpsExchange exchange) { + String contentType = exchange.getRequestHeaders().getFirst("Content-Type"); + return contentType != null && contentType.toLowerCase(java.util.Locale.ROOT).matches("application/json(?:\\s*;.*)?"); + } + private static boolean boundedFixedBody(HttpsExchange exchange, int maximum) { + if (exchange.getRequestHeaders().getFirst("Transfer-Encoding") != null) return false; + String value = exchange.getRequestHeaders().getFirst("Content-Length"); + try { long length = Long.parseLong(value); return length > 0L && length <= maximum; } + catch (RuntimeException invalid) { return false; } + } + private static ThreadPoolExecutor executor(String name, int threads, int queue) { + ThreadFactory factory = task -> { Thread thread = new Thread(task, name); thread.setDaemon(true); return thread; }; + return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); + } + private static void setDefault(String name, String value) { if (System.getProperty(name) == null) System.setProperty(name, value); } + private static void shutdown(ExecutorService executor) { executor.shutdown(); try { if (!executor.awaitTermination(5, TimeUnit.SECONDS)) executor.shutdownNow(); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); executor.shutdownNow(); } } + + public record ReceivedEnvelope(String serverId, String messageId, JsonEnvelope envelope) { } + static record Response(Collection acks, Collection messages) { } + static final class BackendState { + private String session; private long sequence = -1L; + private final LinkedHashMap outgoing = new LinkedHashMap<>(); + private final Set seen = new LinkedHashSet<>(); private final Set processing = new LinkedHashSet<>(); + private final ArrayDeque acknowledgements = new ArrayDeque<>(); + private final Set delivered = new LinkedHashSet<>(); + private long lastDeliveryNanos; + private double requestTokens = 24.0d; + private long lastTokenNanos = System.nanoTime(); + private boolean activePoll; + private boolean beginPoll(String requestedSession) { synchronized (this) { if (activePoll) return false; activePoll = true; return true; } } + private void endPoll() { synchronized (this) { activePoll = false; notifyAll(); } } + private boolean allowRequest() { + long now = System.nanoTime(); requestTokens = Math.min(24.0d, requestTokens + ((now - lastTokenNanos) / 1_000_000_000.0d) * 2.0d); + lastTokenNanos = now; if (requestTokens < 1.0d) return false; requestTokens -= 1.0d; return true; + } + boolean acceptSession(String requested, long requestedSequence) { + if (!requested.equals(session)) { session = requested; sequence = -1L; delivered.clear(); lastDeliveryNanos = 0L; } + // The connector allocates a fresh monotonic sequence for every attempt. Rejecting equality + // prevents a captured request from being replayed with altered ACKs or a new payload. + if (requestedSequence <= sequence) return false; sequence = requestedSequence; return true; + } + private boolean enqueue(HttpTransportProtocol.Delivery delivery) { if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; outgoing.put(delivery.id(), delivery); return true; } + private void acknowledge(Collection acks) { for (String id : acks) { outgoing.remove(id); delivered.remove(id); } } + List acceptIncoming(List received) { + List accepted = new java.util.ArrayList<>(); + for (HttpTransportProtocol.Delivery delivery : received) { + if (seen.contains(delivery.id())) { queueAck(delivery.id()); continue; } + if (!processing.contains(delivery.id())) { if (seen.size() >= HttpTransportProtocol.MAX_QUEUE) throw new IllegalArgumentException("dedup queue exhausted"); processing.add(delivery.id()); accepted.add(delivery); } + } + return accepted; + } + synchronized void completeIncoming(String id, boolean success) { processing.remove(id); if (success) { seen.add(id); while (seen.size() > HttpTransportProtocol.MAX_QUEUE) seen.remove(seen.iterator().next()); queueAck(id); signal(); } } + private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } + synchronized Response await(String serverId, String requestedSession, long requestedSequence) { + long deadline = System.nanoTime() + LONG_POLL.toNanos(); + while (acknowledgements.isEmpty() && !hasUndelivered() && !redeliveryDue()) { + long remaining = deadline - System.nanoTime(); if (remaining <= 0) break; + try { TimeUnit.NANOSECONDS.timedWait(this, remaining); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); break; } + } + List acks = new java.util.ArrayList<>(); while (!acknowledgements.isEmpty() && acks.size() < HttpTransportProtocol.MAX_BATCH) acks.add(acknowledgements.remove()); + List candidates = new java.util.ArrayList<>(); + if (hasUndelivered() || redeliveryDue()) for (HttpTransportProtocol.Delivery delivery : outgoing.values()) { + if (!delivered.contains(delivery.id()) || redeliveryDue()) candidates.add(delivery); + if (candidates.size() == HttpTransportProtocol.MAX_BATCH) break; + } + List messages = HttpTransportProtocol.fittingMessages(serverId, requestedSession, + requestedSequence, acks, candidates); + for (HttpTransportProtocol.Delivery delivery : messages) delivered.add(delivery.id()); + if (!messages.isEmpty()) lastDeliveryNanos = System.nanoTime(); + return new Response(acks, messages); + } + private boolean hasUndelivered() { for (String id : outgoing.keySet()) if (!delivered.contains(id)) return true; return false; } + private boolean redeliveryDue() { return !outgoing.isEmpty() && lastDeliveryNanos > 0L && System.nanoTime() - lastDeliveryNanos >= LONG_POLL.toNanos(); } + private synchronized void signal() { notifyAll(); } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java new file mode 100644 index 000000000..f3db354b9 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -0,0 +1,321 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import java.io.IOException; +import java.io.OutputStream; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.Security; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.util.Date; +import java.util.EnumSet; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x509.BasicConstraints; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.GeneralName; +import org.bouncycastle.asn1.x509.GeneralNames; +import org.bouncycastle.asn1.x509.KeyUsage; +import org.bouncycastle.asn1.x509.ExtendedKeyUsage; +import org.bouncycastle.asn1.x509.KeyPurposeId; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.X509v3CertificateBuilder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import com.bencodez.votingplugin.util.DurableFiles; + +/** Durable private CA plus server identity used by the proxy HTTP listener. */ +public final class HttpTlsIdentity { + private static final String CA_FILE = "http-transport-ca.p12"; + private static final String SERVER_FILE = "http-transport-server.p12"; + private static final String PASSWORD_FILE = "http-transport-password"; + private static final char[] EMPTY_PASSWORD = new char[0]; + private final PrivateKey caKey; + private final X509Certificate caCertificate; + private final PrivateKey serverKey; + private final X509Certificate serverCertificate; + private final char[] password; + + private HttpTlsIdentity(PrivateKey caKey, X509Certificate caCertificate, PrivateKey serverKey, + X509Certificate serverCertificate, char[] password) { + this.caKey = caKey; + this.caCertificate = caCertificate; + this.serverKey = serverKey; + this.serverCertificate = serverCertificate; + this.password = password.clone(); + } + + public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost) throws Exception { + if (advertisedHost == null || advertisedHost.isBlank() || advertisedHost.length() > 253) + throw new IllegalArgumentException("Advertised HTTPS host is invalid"); + Files.createDirectories(directory); + Path caFile = safe(directory.resolve(CA_FILE)); + Path serverFile = safe(directory.resolve(SERVER_FILE)); + Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); + boolean caExists = Files.exists(caFile, LinkOption.NOFOLLOW_LINKS); + boolean serverExists = Files.exists(serverFile, LinkOption.NOFOLLOW_LINKS); + boolean passwordExists = Files.exists(passwordFile, LinkOption.NOFOLLOW_LINKS); + if (caExists || serverExists || passwordExists) { + if (!(caExists && serverExists && passwordExists)) throw new IOException("HTTP TLS identity files are incomplete"); + char[] password = readPassword(passwordFile); + try { + KeyStore ca = load(caFile, password); + KeyStore server = load(serverFile, password); + PrivateKey caKey = (PrivateKey) ca.getKey("ca", password); + X509Certificate caCertificate = (X509Certificate) ca.getCertificate("ca"); + PrivateKey serverKey = (PrivateKey) server.getKey("server", password); + X509Certificate serverCertificate = (X509Certificate) server.getCertificate("server"); + if (caKey == null || caCertificate == null || serverKey == null || serverCertificate == null) + throw new IOException("HTTP TLS identity files are invalid"); + if (!hasServerName(serverCertificate, advertisedHost)) { + ensureBouncyCastle(); + KeyPair serverPair = keyPair(); + serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, caKey, + CertificateRole.SERVER, advertisedHost); + serverKey = serverPair.getPrivate(); + server = KeyStore.getInstance("PKCS12"); + server.load(null, EMPTY_PASSWORD); + server.setKeyEntry("server", serverKey, password, new Certificate[] { serverCertificate, caCertificate }); + writeStore(serverFile, server, password); + } + return new HttpTlsIdentity(caKey, caCertificate, serverKey, serverCertificate, password); + } finally { Arrays.fill(password, '\0'); } + } + ensureBouncyCastle(); + char[] password = HttpTransportSecrets.randomToken().toCharArray(); + try { + KeyPair caPair = keyPair(); + X509Certificate caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, CertificateRole.CA, null); + KeyPair serverPair = keyPair(); + X509Certificate serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, + caPair.getPrivate(), CertificateRole.SERVER, advertisedHost); + KeyStore ca = KeyStore.getInstance("PKCS12"); + ca.load(null, EMPTY_PASSWORD); + ca.setKeyEntry("ca", caPair.getPrivate(), password, new Certificate[] { caCertificate }); + KeyStore server = KeyStore.getInstance("PKCS12"); + server.load(null, EMPTY_PASSWORD); + server.setKeyEntry("server", serverPair.getPrivate(), password, new Certificate[] { serverCertificate, caCertificate }); + writeStore(caFile, ca, password); + writeStore(serverFile, server, password); + byte[] passwordBytes = asciiBytes(password); + try { writePrivate(passwordFile, passwordBytes); } + finally { Arrays.fill(passwordBytes, (byte) 0); } + return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password); + } finally { Arrays.fill(password, '\0'); } + } + + public String serverCertificatePin() { return HttpTransportSecrets.certificatePin(serverCertificate); } + public String caCertificatePin() { return HttpTransportSecrets.certificatePin(caCertificate); } + public X509Certificate caCertificate() { return caCertificate; } + public X509Certificate serverCertificate() { return serverCertificate; } + + /** + * The listener requests a client certificate but validates its issuance/binding in the HTTP handler. + * This is necessary to share the enrollment and normal endpoints on one JDK HttpsServer listener. + */ + public SSLContext serverContext() throws Exception { + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, EMPTY_PASSWORD); + store.setKeyEntry("server", serverKey, password, new Certificate[] { serverCertificate, caCertificate }); + KeyManagerFactory keyManagers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagers.init(store, password); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(keyManagers.getKeyManagers(), new TrustManager[] { new EnrollmentAwareTrustManager(caCertificate) }, null); + return context; + } + + public IssuedClientCertificate issueClientCertificate(String serverId) throws Exception { + serverId = canonicalServerId(serverId); + ensureBouncyCastle(); + KeyPair pair = keyPair(); + X509Certificate certificate = certificate("CN=" + serverId, pair, caCertificate, caKey, CertificateRole.CLIENT, + "urn:votingplugin:http-backend:" + serverId); + char[] clientPassword = HttpTransportSecrets.randomToken().toCharArray(); + try { + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, EMPTY_PASSWORD); + store.setKeyEntry("client", pair.getPrivate(), clientPassword, new Certificate[] { certificate, caCertificate }); + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + store.store(bytes, clientPassword); + return new IssuedClientCertificate(serverId, certificate, bytes.toByteArray(), clientPassword); + } finally { Arrays.fill(clientPassword, '\0'); } + } + + public static String canonicalServerId(String serverId) { + if (serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) + throw new IllegalArgumentException("Server id is invalid"); + return serverId.toLowerCase(java.util.Locale.ROOT); + } + + public boolean issuedByThisCa(X509Certificate certificate) { + if (certificate == null) return false; + try { + certificate.checkValidity(); + certificate.verify(caCertificate.getPublicKey()); + return true; + } catch (Exception failure) { + return false; + } + } + + public boolean validClientCertificate(String expectedServerId, X509Certificate certificate) { + if (!issuedByThisCa(certificate)) return false; + try { + List usage = certificate.getExtendedKeyUsage(); + boolean[] keyUsage = certificate.getKeyUsage(); + if (usage == null || !usage.contains(KeyPurposeId.id_kp_clientAuth.getId()) || keyUsage == null || !keyUsage[0]) return false; + String expectedUri = "urn:votingplugin:http-backend:" + canonicalServerId(expectedServerId); + Collection> names = certificate.getSubjectAlternativeNames(); + if (names == null) return false; + for (List name : names) { + if (name.size() == 2 && Integer.valueOf(GeneralName.uniformResourceIdentifier).equals(name.get(0)) + && expectedUri.equals(name.get(1))) return true; + } + return false; + } catch (Exception failure) { return false; } + } + + public record IssuedClientCertificate(String serverId, X509Certificate certificate, byte[] pkcs12, char[] password) { + public IssuedClientCertificate { + pkcs12 = pkcs12.clone(); + password = password.clone(); + } + @Override public byte[] pkcs12() { return pkcs12.clone(); } + @Override public char[] password() { return password.clone(); } + } + + private static KeyPair keyPair() throws Exception { + KeyPairGenerator generator = KeyPairGenerator.getInstance("EC"); + generator.initialize(new java.security.spec.ECGenParameterSpec("secp256r1")); + return generator.generateKeyPair(); + } + + private static X509Certificate certificate(String subject, KeyPair subjectKey, X509Certificate issuer, PrivateKey issuerKey, + CertificateRole role, String subjectAlternativeName) throws Exception { + Instant now = Instant.now(); + X500Name issuerName = issuer == null ? new X500Name(subject) : new X500Name(issuer.getSubjectX500Principal().getName()); + X509v3CertificateBuilder builder = new JcaX509v3CertificateBuilder(issuerName, + new BigInteger(160, new java.security.SecureRandom()).setBit(159), Date.from(now.minusSeconds(300)), + Date.from(now.plusSeconds(role == CertificateRole.CA ? 315360000L : 31536000L)), new X500Name(subject), subjectKey.getPublic()); + builder.addExtension(Extension.basicConstraints, true, new BasicConstraints(role == CertificateRole.CA)); + builder.addExtension(Extension.keyUsage, true, new KeyUsage(role == CertificateRole.CA ? KeyUsage.keyCertSign | KeyUsage.cRLSign + : KeyUsage.digitalSignature)); + if (role == CertificateRole.SERVER) builder.addExtension(Extension.extendedKeyUsage, false, + new ExtendedKeyUsage(KeyPurposeId.id_kp_serverAuth)); + if (role == CertificateRole.CLIENT) builder.addExtension(Extension.extendedKeyUsage, false, + new ExtendedKeyUsage(KeyPurposeId.id_kp_clientAuth)); + if (role == CertificateRole.SERVER && subjectAlternativeName != null) { + GeneralName name; + if (subjectAlternativeName.matches("(?:\\d{1,3}\\.){3}\\d{1,3}") || subjectAlternativeName.indexOf(':') >= 0) + name = new GeneralName(GeneralName.iPAddress, subjectAlternativeName); + else name = new GeneralName(GeneralName.dNSName, subjectAlternativeName); + builder.addExtension(Extension.subjectAlternativeName, false, new GeneralNames(name)); + } + if (role == CertificateRole.CLIENT) builder.addExtension(Extension.subjectAlternativeName, false, + new GeneralNames(new GeneralName(GeneralName.uniformResourceIdentifier, subjectAlternativeName))); + ContentSigner signer = new JcaContentSignerBuilder("SHA256withECDSA").setProvider("BC") + .build(issuerKey == null ? subjectKey.getPrivate() : issuerKey); + X509CertificateHolder holder = builder.build(signer); + return new JcaX509CertificateConverter().setProvider("BC").getCertificate(holder); + } + + private static void ensureBouncyCastle() { + if (Security.getProvider("BC") == null) Security.addProvider(new BouncyCastleProvider()); + } + + private static String certificateName(String host) { + return host.replaceAll("[^A-Za-z0-9 ._-]", "_"); + } + + private static boolean hasServerName(X509Certificate certificate, String advertisedHost) { + try { + Collection> names = certificate.getSubjectAlternativeNames(); + if (names == null) return false; + for (List name : names) { + if (name.size() != 2 || !(name.get(1) instanceof String value)) continue; + if ((Integer.valueOf(GeneralName.dNSName).equals(name.get(0)) || Integer.valueOf(GeneralName.iPAddress).equals(name.get(0))) + && advertisedHost.equalsIgnoreCase(value)) return true; + } + return false; + } catch (Exception failure) { return false; } + } + + private static Path safe(Path file) throws IOException { + Path parent = file.toAbsolutePath().normalize().getParent(); + if (parent == null || Files.isSymbolicLink(file)) throw new IOException("Refusing unsafe HTTP TLS identity path"); + return file.toAbsolutePath().normalize(); + } + + private static KeyStore load(Path path, char[] password) throws Exception { + KeyStore store = KeyStore.getInstance("PKCS12"); + try (var input = Files.newInputStream(path, LinkOption.NOFOLLOW_LINKS)) { store.load(input, password); } + return store; + } + + private static char[] readPassword(Path path) throws IOException { + byte[] bytes = Files.readAllBytes(path); + if (bytes.length < 40 || bytes.length > 128) throw new IOException("HTTP TLS password file is invalid"); + try { return new String(bytes, java.nio.charset.StandardCharsets.US_ASCII).toCharArray(); } + finally { Arrays.fill(bytes, (byte) 0); } + } + + private static void writeStore(Path file, KeyStore store, char[] password) throws Exception { + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + store.store(bytes, password); + byte[] contents = bytes.toByteArray(); + try { writePrivate(file, contents); } + finally { Arrays.fill(contents, (byte) 0); } + } + + private static void writePrivate(Path file, byte[] contents) throws IOException { + Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); + try { + setOwnerOnly(temporary); + Files.write(temporary, contents, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); } + setOwnerOnly(file); + DurableFiles.forceDirectory(file.getParent()); + } finally { Files.deleteIfExists(temporary); } + } + + private static byte[] asciiBytes(char[] characters) { + byte[] output = new byte[characters.length]; + for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; + return output; + } + + private static void setOwnerOnly(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { /* Windows ACLs are inherited; never make the file world-readable. */ } + } + + private static final class EnrollmentAwareTrustManager implements X509TrustManager { + private final X509Certificate[] acceptedIssuers; + private EnrollmentAwareTrustManager(X509Certificate caCertificate) { this.acceptedIssuers = new X509Certificate[] { caCertificate }; } + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { } + @Override public void checkServerTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } + @Override public X509Certificate[] getAcceptedIssuers() { return acceptedIssuers.clone(); } + } + private enum CertificateRole { CA, SERVER, CLIENT } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java new file mode 100644 index 000000000..0224dad7d --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java @@ -0,0 +1,174 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collection; +import java.util.List; +import java.util.UUID; + +/** Strict, versioned HTTP transport envelope. Payloads remain canonical JsonEnvelopeCodec values. */ +final class HttpTransportProtocol { + static final int VERSION = 1; + static final int MAX_BODY_BYTES = 256 * 1024; + static final int MAX_BATCH = 64; + static final int MAX_ENVELOPE_BYTES = 48 * 1024; + static final int MAX_QUEUE = 1024; + static final long MAX_CLOCK_SKEW_MILLIS = 90_000L; + + private HttpTransportProtocol() { } + + static void validateEnvelope(JsonEnvelope envelope) { + if (envelope == null || JsonEnvelopeCodec.encode(envelope).getBytes(StandardCharsets.UTF_8).length > MAX_ENVELOPE_BYTES) throw bad(); + } + + static byte[] request(String server, String session, long sequence, Collection acks, + Collection messages) { + JsonObject root = base(server, session, sequence); + root.add("acks", ids(acks)); + root.add("messages", messages(messages)); + byte[] encoded = root.toString().getBytes(StandardCharsets.UTF_8); + if (encoded.length > MAX_BODY_BYTES) throw bad(); + return encoded; + } + + static List fittingMessages(String server, String session, long sequence, Collection acks, + Collection candidates) { + List output = new ArrayList<>(); + for (Delivery candidate : candidates) { + if (output.size() == MAX_BATCH) break; + output.add(candidate); + try { request(server, session, sequence, acks, output); } + catch (IllegalArgumentException tooLarge) { output.remove(output.size() - 1); break; } + } + return output; + } + + static byte[] response(String server, String session, long sequence, Collection acks, + Collection messages) { + return request(server, session, sequence, acks, messages); + } + + static Packet parsePacket(byte[] body) { + if (body == null || body.length == 0 || body.length > MAX_BODY_BYTES) throw bad(); + try { + JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw bad(); + JsonObject root = parsed.getAsJsonObject(); + requireOnly(root, "v", "server", "session", "sequence", "timestamp", "acks", "messages"); + if (integer(root, "v") != VERSION) throw bad(); + String server = HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); + String session = uuid(root, "session"); + long sequence = nonNegative(root, "sequence"); + long timestamp = root.get("timestamp").getAsLong(); + if (Math.abs(Instant.now().toEpochMilli() - timestamp) > MAX_CLOCK_SKEW_MILLIS) throw bad(); + List acks = parseIds(root.get("acks")); + List messages = parseMessages(root.get("messages")); + return new Packet(server, session, sequence, acks, messages); + } catch (RuntimeException invalid) { throw bad(); } + } + + static byte[] enrollmentResponse(HttpTlsIdentity.IssuedClientCertificate certificate) { + JsonObject output = new JsonObject(); + byte[] bundle = certificate.pkcs12(); + try { output.addProperty("bundle", Base64.getUrlEncoder().withoutPadding().encodeToString(bundle)); } + finally { java.util.Arrays.fill(bundle, (byte) 0); } + char[] password = certificate.password(); + try { output.addProperty("password", new String(password)); } + finally { java.util.Arrays.fill(password, '\0'); } + return output.toString().getBytes(StandardCharsets.UTF_8); + } + + static Enrollment parseEnrollment(byte[] body) { + if (body == null || body.length == 0 || body.length > 8192) throw bad(); + try { + JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw bad(); + JsonObject root = parsed.getAsJsonObject(); + requireOnly(root, "server", "token"); + String server = HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); + String token = string(root, "token", 128); + if (!token.matches("[A-Za-z0-9_-]{43,128}")) throw bad(); + return new Enrollment(server, token); + } catch (RuntimeException invalid) { throw bad(); } + } + + static HttpTlsIdentity.IssuedClientCertificate parseEnrollmentResponse(String server, byte[] body) { + if (body == null || body.length == 0 || body.length > MAX_BODY_BYTES) throw bad(); + try { + JsonObject root = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)).getAsJsonObject(); + requireOnly(root, "bundle", "password"); + byte[] bundle = Base64.getUrlDecoder().decode(string(root, "bundle", MAX_BODY_BYTES * 2)); + char[] password = string(root, "password", 128).toCharArray(); + if (bundle.length == 0 || password.length < 40) throw bad(); + try { return new HttpTlsIdentity.IssuedClientCertificate(HttpTlsIdentity.canonicalServerId(server), null, bundle, password); } + finally { java.util.Arrays.fill(bundle, (byte) 0); java.util.Arrays.fill(password, '\0'); } + } catch (RuntimeException invalid) { throw bad(); } + } + + private static JsonObject base(String server, String session, long sequence) { + JsonObject root = new JsonObject(); + root.addProperty("v", VERSION); root.addProperty("server", server); root.addProperty("session", session); + root.addProperty("sequence", sequence); root.addProperty("timestamp", Instant.now().toEpochMilli()); + return root; + } + private static JsonArray ids(Collection values) { + if (values == null || values.size() > MAX_BATCH) throw bad(); + JsonArray output = new JsonArray(); + for (String value : values) { validId(value); output.add(value); } + return output; + } + private static JsonArray messages(Collection values) { + if (values == null || values.size() > MAX_BATCH) throw bad(); + JsonArray output = new JsonArray(); + for (Delivery delivery : values) { + validId(delivery.id()); + String encoded = JsonEnvelopeCodec.encode(delivery.envelope()); + byte[] bytes = encoded.getBytes(StandardCharsets.UTF_8); + if (bytes.length > MAX_ENVELOPE_BYTES) throw bad(); + JsonObject item = new JsonObject(); item.addProperty("id", delivery.id()); + item.addProperty("payload", Base64.getUrlEncoder().withoutPadding().encodeToString(bytes)); output.add(item); + } + return output; + } + private static List parseIds(JsonElement value) { + if (value == null || !value.isJsonArray() || value.getAsJsonArray().size() > MAX_BATCH) throw bad(); + List output = new ArrayList<>(); + for (JsonElement item : value.getAsJsonArray()) { if (!item.isJsonPrimitive()) throw bad(); String id = item.getAsString(); validId(id); output.add(id); } + return output; + } + private static List parseMessages(JsonElement value) { + if (value == null || !value.isJsonArray() || value.getAsJsonArray().size() > MAX_BATCH) throw bad(); + List output = new ArrayList<>(); + for (JsonElement item : value.getAsJsonArray()) { + if (!item.isJsonObject()) throw bad(); JsonObject object = item.getAsJsonObject(); requireOnly(object, "id", "payload"); + String id = string(object, "id", 64); validId(id); + byte[] payload = Base64.getUrlDecoder().decode(string(object, "payload", MAX_ENVELOPE_BYTES * 2)); + if (payload.length == 0 || payload.length > MAX_ENVELOPE_BYTES) throw bad(); + JsonEnvelope envelope = JsonEnvelopeCodec.decode(new String(payload, StandardCharsets.UTF_8)); + output.add(new Delivery(id, envelope)); + } + return output; + } + private static void requireOnly(JsonObject object, String... names) { + for (String name : object.keySet()) { boolean found = false; for (String allowed : names) if (allowed.equals(name)) { found = true; break; } if (!found) throw bad(); } + for (String name : names) if (!object.has(name) || object.get(name).isJsonNull()) throw bad(); + } + private static String string(JsonObject object, String name, int max) { JsonElement v = object.get(name); if (!v.isJsonPrimitive() || !v.getAsJsonPrimitive().isString()) throw bad(); String value = v.getAsString(); if (value.isEmpty() || value.length() > max) throw bad(); return value; } + private static long integer(JsonObject object, String name) { try { return object.get(name).getAsLong(); } catch (RuntimeException failure) { throw bad(); } } + private static long nonNegative(JsonObject object, String name) { long n = integer(object, name); if (n < 0) throw bad(); return n; } + private static String uuid(JsonObject object, String name) { String value = string(object, name, 64); try { return UUID.fromString(value).toString(); } catch (IllegalArgumentException invalid) { throw bad(); } } + private static void validId(String id) { if (id == null || id.length() > 64) throw bad(); try { UUID.fromString(id); } catch (IllegalArgumentException invalid) { throw bad(); } } + private static IllegalArgumentException bad() { return new IllegalArgumentException("Invalid HTTP transport message"); } + + record Delivery(String id, JsonEnvelope envelope) { } + record Packet(String server, String session, long sequence, List acks, List messages) { } + record Enrollment(String server, String token) { } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecrets.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecrets.java new file mode 100644 index 000000000..f1a8ec04b --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecrets.java @@ -0,0 +1,64 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import java.util.Base64; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +/** Small, deliberately dependency-free cryptographic helpers for the HTTP transport. */ +final class HttpTransportSecrets { + private static final SecureRandom RANDOM = new SecureRandom(); + + private HttpTransportSecrets() { } + + static byte[] randomBytes(int length) { + if (length < 16) throw new IllegalArgumentException("Secret length is too small"); + byte[] value = new byte[length]; + RANDOM.nextBytes(value); + return value; + } + + static String randomToken() { + return Base64.getUrlEncoder().withoutPadding().encodeToString(randomBytes(32)); + } + + static byte[] sha256(byte[] value) { + try { + return MessageDigest.getInstance("SHA-256").digest(value); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } + } + + static String sha256Hex(byte[] value) { + StringBuilder output = new StringBuilder(64); + for (byte part : sha256(value)) output.append(String.format("%02x", part & 0xff)); + return output.toString(); + } + + static String certificatePin(X509Certificate certificate) { + try { + return sha256Hex(certificate.getEncoded()); + } catch (Exception failure) { + throw new IllegalArgumentException("Could not encode certificate", failure); + } + } + + static boolean constantTimeEquals(byte[] first, byte[] second) { + return first != null && second != null && MessageDigest.isEqual(first, second); + } + + static String hmacSha256Url(byte[] key, String value) { + try { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return Base64.getUrlEncoder().withoutPadding().encodeToString(mac.doFinal(value.getBytes(StandardCharsets.US_ASCII))); + } catch (Exception failure) { + throw new IllegalStateException("HMAC-SHA-256 is unavailable", failure); + } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 4d7b20872..43b54c128 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -41,6 +41,9 @@ public void start(BungeeMethod method, GlobalMessageHandler messageHandler) { case SOCKETS: transport = new SocketBackendProxyTransport(plugin); break; + case HTTP: + transport = new HttpBackendProxyTransport(plugin); + break; case REDIS: transport = new RedisBackendProxyTransport(plugin, processedVoteCache); break; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java new file mode 100644 index 000000000..9240b93c8 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -0,0 +1,124 @@ +package com.bencodez.votingplugin.backendproxy.transport; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.concurrent.TimeUnit; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; +import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.backendproxy.http.HttpBackendTransportConnector; +import com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore; +import com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode; + +/** Backend adapter for the secure outbound-only HTTP proxy transport. */ +public final class HttpBackendProxyTransport implements BackendProxyTransport { + private static final int MAX_STARTUP_QUEUE = 1024; + private final VotingPluginMain plugin; + private final Object lifecycle = new Object(); + private final ArrayDeque startupQueue = new ArrayDeque<>(); + private volatile HttpBackendTransportConnector connector; + private volatile Thread worker; + private volatile RuntimeException startupFailure; + private volatile boolean closed; + private final java.util.concurrent.atomic.AtomicBoolean queueWarning = new java.util.concurrent.atomic.AtomicBoolean(); + + public HttpBackendProxyTransport(VotingPluginMain plugin) { + this.plugin = plugin; + } + + @Override + public void start(GlobalMessageHandler messageHandler) { + Path directory = plugin.getDataFolder().toPath().resolve("http"); + String serverId = plugin.getBungeeSettings().getServer(); + String connectionCode = plugin.getBungeeSettings().getHttpConnectionCode(); + worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler), + "VotingPlugin-HTTP-Backend-Setup"); + worker.setDaemon(true); + worker.start(); + } + + private void initialize(Path directory, String serverId, String configuredCode, + GlobalMessageHandler messageHandler) { + try { + if (!Files.isRegularFile(directory.resolve("http-transport-profile.properties"))) { + HttpConnectionCode code = HttpConnectionCode.parse(configuredCode); + HttpBackendTransportConnector.enroll(code, serverId, directory); + } + HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); + if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) + throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); + HttpBackendTransportConnector replacement = new HttpBackendTransportConnector(enrolled, messageHandler::onMessage); + synchronized (lifecycle) { + if (closed) { + replacement.close(); + return; + } + connector = replacement; + replacement.start(); + while (!startupQueue.isEmpty()) { + if (!replacement.send(startupQueue.removeFirst())) { + throw new IllegalStateException("HTTP startup queue could not be transferred"); + } + } + } + } catch (Exception failure) { + startupFailure = new IllegalStateException("Secure HTTP backend enrollment or connection failed", failure); + plugin.getLogger().severe("Secure HTTP backend transport is unavailable; check the connection code and proxy endpoint"); + } + } + + @Override + public void send(JsonEnvelope envelope) { + synchronized (lifecycle) { + if (closed) return; + HttpBackendTransportConnector active = connector; + if (active != null) { + if (!active.send(envelope) && queueWarning.compareAndSet(false, true)) + plugin.getLogger().severe("Secure HTTP transport queue is full or rejected an oversized message; delivery was not accepted"); + } else if (startupQueue.size() < MAX_STARTUP_QUEUE) { + startupQueue.addLast(envelope); + } + } + } + + @Override + public void validate() { + RuntimeException failure = startupFailure; + if (failure != null) throw failure; + String serverId = plugin.getBungeeSettings().getServer(); + if (serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) { + throw new IllegalStateException("HTTP requires a valid unique backend Server name"); + } + Path directory = plugin.getDataFolder().toPath().resolve("http"); + if (!Files.isRegularFile(directory.resolve("http-transport-profile.properties")) + && (plugin.getBungeeSettings().getHttpConnectionCode() == null + || plugin.getBungeeSettings().getHttpConnectionCode().isBlank())) { + throw new IllegalStateException("HTTP requires a temporary ConnectionCode for initial enrollment"); + } + } + + @Override + public void close() { + Thread setup; + HttpBackendTransportConnector active; + synchronized (lifecycle) { + closed = true; + startupQueue.clear(); + setup = worker; + worker = null; + active = connector; + connector = null; + } + if (setup != null) { + setup.interrupt(); + try { + setup.join(TimeUnit.SECONDS.toMillis(5)); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + } + if (active != null) active.close(); + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/config/BungeeSettings.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/config/BungeeSettings.java index b9f8b1afc..5c8a05ab7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/config/BungeeSettings.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/config/BungeeSettings.java @@ -88,7 +88,11 @@ public class BungeeSettings extends YMLFile { @ConfigDataInt(path = "BungeeServer.Port") @Getter - private int bungeeServerPort = 1297; + private int bungeeServerPort = 1297; + + @ConfigDataString(path = "HTTP.ConnectionCode") + @Getter + private String httpConnectionCode = ""; @ConfigDataBoolean(path = "PerServerPoints") @Getter diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index fc00e3e9b..0bb8d29f4 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -47,7 +47,7 @@ public final class BackendConfigurationService { "WaitUntilVoteDelay", "PermissionToView", "IgnoreCanVote", "VoteDelayDailyHour", "VoteDelayMin", "GiveOffline"); private static final Pattern COMMENT_SECRET = Pattern.compile( - "(?i)([\"']?\\b(?:[\\w-]*(?:password|secret)[\\w-]*|token|api[ _.-]?key|authorization|[\\w.-]*webhook[ _.-]?url)" + "(?i)([\"']?\\b(?:[\\w-]*(?:password|secret)[\\w-]*|token|connection[ _.-]?code|api[ _.-]?key|authorization|[\\w.-]*webhook[ _.-]?url)" + "\\b[\"']?\\s*[:=]\\s*)(.*)$"); private static final Pattern SECRET_PATH_URL = Pattern.compile("(?i)([\"']?\\burl\\b[\"']?\\s*[:=]\\s*)(.*)$"); private static final Pattern BLOCK_SCALAR_INDICATOR = Pattern.compile("[|>](?:[+-][1-9]?|[1-9][+-]?)?"); @@ -592,6 +592,15 @@ private void validateProxyMethod(BungeeMethod method, YamlConfiguration settings case SOCKETS: configuredHostAndPort(settings, "BungeeServer.Host", "BungeeServer.Port", 1297, "BungeeServer"); break; + case HTTP: + String connectionCode = settings.getString("HTTP.ConnectionCode", ""); + if (connectionCode == null || connectionCode.isBlank()) { + Path identity = dataDirectory.resolve("http").resolve("http-transport-client.p12"); + if (!Files.isRegularFile(identity)) { + throw new IllegalArgumentException("HTTP.ConnectionCode must be set for initial enrollment"); + } + } + break; case MYSQL: try { YamlConfiguration main = parse(readRaw(resolve("Config.yml"), false)); @@ -957,7 +966,7 @@ private static List restoreCommentSecrets(List proposed, List clientHandles; private SocketHandler socketHandler; + private HttpProxyTransportServer httpTransportServer; + private HttpEnrollmentAuthority httpEnrollmentAuthority; @Getter @Setter @@ -566,6 +571,8 @@ protected boolean sendProxyBroadcastEnvelopeNow(String server, JsonEnvelope enve // envelopes. This preserves the socket connection and its delivery // acknowledgement instead of creating a second short-lived socket. return sendSocketEnvelope(server, envelope); + case HTTP: + return sendHttpEnvelope(server, envelope); default: return false; } @@ -1328,6 +1335,8 @@ public void onReceiveEnvelope(JsonEnvelope envelope) { }); rebuildSocketClients(); + } else if (method.equals(BungeeMethod.HTTP)) { + startHttpTransport(); } else if (method.equals(BungeeMethod.REDIS)) { redisHandler = new RedisHandler(getConfig().getRedisHost(), getConfig().getRedisPort(), getConfig().getRedisUsername(), getConfig().getRedisPassword(), getConfig().getRedisDbIndex(), @@ -1391,6 +1400,9 @@ public void sendMessage(String server, int delay, JsonEnvelope envelope) { case SOCKETS: sendSocketEnvelope(server, envelope); break; + case HTTP: + sendHttpEnvelope(server, envelope); + break; default: break; } @@ -2418,6 +2430,7 @@ public void completeRuntimeReplacementShutdown() { if (socketHandler != null) socketHandler.closeConnection(); }); runCleanup("socket clients", this::closeSocketClients); + runCleanup("HTTP transport", this::closeHttpTransport); runCleanup("Redis subscriber", () -> { if (redisHandler != null) redisHandler.close(); }); @@ -2653,6 +2666,60 @@ private synchronized boolean sendSocketEnvelope(String server, JsonEnvelope enve } } + private synchronized boolean sendHttpEnvelope(String server, JsonEnvelope envelope) { + HttpProxyTransportServer transport = httpTransportServer; + return transport != null && transport.send(server, envelope); + } + + private void startHttpTransport() { + try { + URI endpoint = URI.create(getConfig().getHttpPublicEndpoint()); + if (!"https".equalsIgnoreCase(endpoint.getScheme()) || endpoint.getHost() == null + || endpoint.getUserInfo() != null || endpoint.getQuery() != null || endpoint.getFragment() != null + || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) { + throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); + } + File directory = new File(getDataFolderPlugin(), "http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.toPath(), endpoint.getHost()); + httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); + httpTransportServer = new HttpProxyTransportServer( + new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, + httpEnrollmentAuthority, received -> { + GlobalMessageProxyHandler handler = globalMessageProxyHandler; + if (handler == null) throw new IllegalStateException("HTTP message router is not ready"); + handler.onMessage(received.envelope()); + }); + httpTransportServer.start(); + logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" + + httpTransportServer.port() + "; use /votingpluginbungee httpcode for each backend"); + } catch (Exception failure) { + closeHttpTransport(); + throw new IllegalStateException("HTTP transport could not start securely", failure); + } + } + + private synchronized void closeHttpTransport() { + HttpProxyTransportServer transport = httpTransportServer; + httpTransportServer = null; + httpEnrollmentAuthority = null; + if (transport != null) transport.close(); + } + + public String createHttpConnectionCode(String serverId) { + HttpEnrollmentAuthority authority = httpEnrollmentAuthority; + if (method != BungeeMethod.HTTP || authority == null) { + throw new IllegalStateException("The HTTP transport is not running"); + } + return authority.createConnectionCode(serverId, URI.create(getConfig().getHttpPublicEndpoint()), Duration.ofMinutes(15)) + .encode(); + } + + public void revokeHttpBackend(String serverId) { + HttpEnrollmentAuthority authority = httpEnrollmentAuthority; + if (method != BungeeMethod.HTTP || authority == null) throw new IllegalStateException("The HTTP transport is not running"); + authority.revoke(HttpTlsIdentity.canonicalServerId(serverId)); + } + private synchronized void closeSocketClients() { HashMap clients = clientHandles; clientHandles = null; @@ -2673,7 +2740,7 @@ static void stopSocketClients(Map clients) { private void warnUnsupportedDedicatedVotingProxyMode() { if (getConfig().getDedicatedVotingProxy() && (method == null || !method.supportsBackendPresence())) { - logSevere("DedicatedVotingProxy requires MYSQL, REDIS, MQTT, or SOCKETS; PLUGINMESSAGING is disabled for " + logSevere("DedicatedVotingProxy requires MYSQL, REDIS, MQTT, SOCKETS, or HTTP; PLUGINMESSAGING is disabled for " + "dedicated-proxy routing. Falling back to normal proxy routing."); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java index 7420f0a49..069796fb6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyCommand.java @@ -47,6 +47,24 @@ public String execute(String[] args) { case "status": return handleStatusCommand(); + case "httpcode": + if (args.length != 2) return "&cUsage: httpcode "; + try { + return "&aTemporary HTTP backend connection code (expires in 15 minutes and works once):\n&f" + + plugin.createHttpConnectionCode(args[1]); + } catch (IllegalArgumentException | IllegalStateException failure) { + return "&cThe secure HTTP transport is not running."; + } + + case "httprevoke": + if (args.length != 2) return "&cUsage: httprevoke "; + try { + plugin.revokeHttpBackend(args[1]); + return "&aRevoked HTTP backend identity for " + args[1] + ". Generate a new connection code to re-enroll it."; + } catch (IllegalArgumentException | IllegalStateException failure) { + return "&cCould not revoke that HTTP backend identity."; + } + case "multiproxystatus": plugin.getMultiProxyHandler().sendStatus(); return "&aSent status message across multi-proxy"; @@ -92,6 +110,8 @@ private String getHelpMessage() { helpBuilder.append("/votingplugin vote - Send a vote\n"); helpBuilder.append("/votingplugin forcetimechange - Force a time change\n"); helpBuilder.append("/votingplugin status - Check connection status\n"); + helpBuilder.append("/votingplugin httpcode - Create a node-bound one-time HTTP connection code\n"); + helpBuilder.append("/votingplugin httprevoke - Revoke a backend identity before re-enrollment\n"); helpBuilder.append("/votingplugin multiproxystatus - Send status message across proxies\n"); helpBuilder.append("/votingplugin voteparty - Trigger or modify vote party\n"); return helpBuilder.toString(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java index 3c5eeff94..cd00c91e0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java @@ -241,6 +241,21 @@ default List getProxyBroadcastOfflineForwardServers() { */ public int getBungeePort(); + /** Bind address for the single-port HTTP transport listener. */ + default String getHttpHost() { + return "0.0.0.0"; + } + + /** Public HTTPS origin embedded in newly generated backend connection codes. */ + default String getHttpPublicEndpoint() { + return ""; + } + + /** Listener port for the single-port HTTP transport. */ + default int getHttpPort() { + return 1297; + } + /** * Gets the plugin message channel. * diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index ee1ff540a..4aa394cca 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -209,9 +209,24 @@ public String getBungeeMethod() { } @Override - public int getBungeePort() { - return getData().getInt("BungeeServer.Port", 1297); - } + public int getBungeePort() { + return getData().getInt("BungeeServer.Port", 1297); + } + + @Override + public String getHttpHost() { + return getData().getString("HTTP.Host", "0.0.0.0"); + } + + @Override + public String getHttpPublicEndpoint() { + return getData().getString("HTTP.PublicEndpoint", ""); + } + + @Override + public int getHttpPort() { + return getData().getInt("HTTP.Port", 1297); + } @Override public boolean getDebug() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java index c98f054c8..f2c72515b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java @@ -58,6 +58,21 @@ private void validate(ProxyMethodConfiguration proposal, VotingPluginProxyConfig } } break; + case HTTP: + if (blank(config.getHttpHost()) || config.getHttpPort() < 1 || config.getHttpPort() > 65535) { + throw new IllegalArgumentException("HTTP.Host and HTTP.Port must be set"); + } + String endpoint = config.getHttpPublicEndpoint(); + if (blank(endpoint)) throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); + URI httpEndpoint; + try { httpEndpoint = URI.create(endpoint); } + catch (RuntimeException invalid) { throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); } + if (!"https".equalsIgnoreCase(httpEndpoint.getScheme()) || httpEndpoint.getHost() == null + || httpEndpoint.getUserInfo() != null || httpEndpoint.getQuery() != null || httpEndpoint.getFragment() != null + || (httpEndpoint.getPath() != null && !httpEndpoint.getPath().isEmpty() && !"/".equals(httpEndpoint.getPath()))) { + throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); + } + break; case MYSQL: if (!config.hasDatabaseConfigured()) { throw new IllegalArgumentException("The proxy database Host must be configured for MYSQL"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index 008aa7716..9b6077896 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -360,6 +360,21 @@ public int getBungeePort() { return getInt(getNode("BungeeServer", "Port"), 1297); } + @Override + public String getHttpHost() { + return getString(getNode("HTTP", "Host"), "0.0.0.0"); + } + + @Override + public String getHttpPublicEndpoint() { + return getString(getNode("HTTP", "PublicEndpoint"), ""); + } + + @Override + public int getHttpPort() { + return getInt(getNode("HTTP", "Port"), 1297); + } + @Override public boolean getDebug() { return getBoolean(getNode("Debug"), false); diff --git a/VotingPlugin/src/main/resources/BungeeSettings.yml b/VotingPlugin/src/main/resources/BungeeSettings.yml index 036566edc..a8310126d 100644 --- a/VotingPlugin/src/main/resources/BungeeSettings.yml +++ b/VotingPlugin/src/main/resources/BungeeSettings.yml @@ -10,7 +10,7 @@ # 📑 Config Index (matches this file) # ───────────────────────────────────────────────────────────────────────────── # • Core proxy toggle (UseBungeecord) + method selection (BungeeMethod) -# • Method configs (PLUGINMESSAGING / REDIS / MQTT / MYSQL / SOCKETS) +# • Method configs (PLUGINMESSAGING / HTTP / REDIS / MQTT / MYSQL / SOCKETS) # • Broadcast behavior (BungeeBroadcast / DisableBroadcast / Always) # • Per-server behavior (PerServerRewards / Milestones / Points) # • VoteParty (proxy rewards + global commands) @@ -61,12 +61,19 @@ UseBungeecord: false # Requires restart and set on all servers # https://github.com/BenCodez/VotingPlugin/wiki/Bungeecord-Setups # Available: -# PLUGINMESSAGING (Recommended) -# REDIS (2nd Most Recommended) +# PLUGINMESSAGING (Recommended) +# HTTP (secure direct connection; one proxy port) +# REDIS (2nd Most Recommended) # MQTT # MYSQL # SOCKETS (Not recommended) -BungeeMethod: PLUGINMESSAGING +BungeeMethod: PLUGINMESSAGING + +# Secure direct transport. The proxy generates a temporary ConnectionCode. +# Paste it here once, start the backend, then remove it after enrollment succeeds. +# The backend stores its private client identity in the VotingPlugin data folder. +HTTP: + ConnectionCode: '' # Use Redis for between server communication, set BungeeMethod to REDIS to use this Redis: diff --git a/VotingPlugin/src/main/resources/bungeeconfig.yml b/VotingPlugin/src/main/resources/bungeeconfig.yml index 0c37b46fa..b500d4d7b 100644 --- a/VotingPlugin/src/main/resources/bungeeconfig.yml +++ b/VotingPlugin/src/main/resources/bungeeconfig.yml @@ -249,17 +249,31 @@ WhiteListedServers: [] # Requires restart and set on all servers # https://github.com/BenCodez/VotingPlugin/wiki/Bungeecoord-Setups # Available: -# PLUGINMESSAGING (Recommended) -# REDIS +# PLUGINMESSAGING (Recommended) +# HTTP (secure direct connection; one proxy port) +# REDIS # MQTT # SOCKETS # MYSQL (Not recommended) # PLUGINMESSAGING uses this proxy's player/server state and disables backend # presence messages. Other methods trust each backend's configured Server name. -BungeeMethod: PLUGINMESSAGING +BungeeMethod: PLUGINMESSAGING ###################################################################################### -# Settings for each setup below +# Settings for each setup below + +########################################### +# HTTP Settings +########################################### + +# HTTP uses one encrypted listener on the proxy. Backends connect outbound, so +# they do not need an exposed port. PublicEndpoint must be the HTTPS URL that +# backend servers can reach. VotingPlugin creates the TLS identity, enrollment +# codes, and per-backend client certificates automatically. +HTTP: + Host: '0.0.0.0' + Port: 1297 + PublicEndpoint: '' ########################################### # PLUGINMESSAGING settings diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java new file mode 100644 index 000000000..2273959d1 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -0,0 +1,146 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.net.InetSocketAddress; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpTransportRuntimeTest { + @TempDir Path directory; + + @Test + void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + CountDownLatch proxyReceived = new CountDownLatch(1), backendReceived = new CountDownLatch(1); + AtomicReference received = new AtomicReference<>(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, + message -> { received.set(message); proxyReceived.countDown(); })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, "lobby-1", directory.resolve("client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", credential, + envelope -> backendReceived.countDown())) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("to-proxy").put("server", "forged").build())); + assertTrue(proxyReceived.await(8, TimeUnit.SECONDS)); + assertEquals("lobby-1", received.get().serverId()); + assertEquals("lobby-1", received.get().envelope().getFields().get("server")); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (connector.queuedOutgoing() != 0 && System.nanoTime() < deadline) Thread.sleep(10); + assertEquals(0, connector.queuedOutgoing(), "proxy ACK must remove the exact outbound delivery ID"); + assertTrue(server.send("lobby-1", JsonEnvelope.builder("to-backend").build())); + assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); + } + } + } + + @Test + void normalTransportRejectsAClientWithoutCertificate() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpClient client = HttpClient.newBuilder().sslContext(HttpPinnedTls.clientContext(code)).build(); + byte[] body = HttpTransportProtocol.request("lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(), java.util.List.of()); + HttpResponse response = client.send(HttpRequest.newBuilder(code.endpoint().resolve("v1/transport")) + .timeout(Duration.ofSeconds(5)).header("Content-Type", "application/json").POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(), HttpResponse.BodyHandlers.ofByteArray()); + assertEquals(401, response.statusCode()); + } + } + + @Test + void boundedQueuesFailClosed() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { + for (int i = 0; i < HttpTransportProtocol.MAX_QUEUE; i++) assertTrue(server.send("lobby-1", JsonEnvelope.builder("x").build())); + assertFalse(server.send("lobby-1", JsonEnvelope.builder("x").build())); + assertFalse(server.send("lobby-1", JsonEnvelope.builder("x").put("large", "x".repeat(HttpTransportProtocol.MAX_ENVELOPE_BYTES)).build())); + } + } + + @Test + void aggregatePacketBudgetSplitsLargeValidEnvelopes() { + java.util.List candidates = new java.util.ArrayList<>(); + for (int index = 0; index < 12; index++) candidates.add(new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("large").put("value", "x".repeat(40_000)).build())); + String session = java.util.UUID.randomUUID().toString(); + java.util.List fitted = HttpTransportProtocol.fittingMessages( + "lobby-1", session, 0, java.util.List.of(), candidates); + assertTrue(fitted.size() > 0 && fitted.size() < candidates.size()); + assertTrue(HttpTransportProtocol.request("lobby-1", session, 0, java.util.List.of(), fitted).length + <= HttpTransportProtocol.MAX_BODY_BYTES); + } + + @Test + void persistedProfileStartsAfterTheEnrollmentCodeExpires() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { + server.start(); + HttpConnectionCode active = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(active, "lobby-1", directory.resolve("client")); + HttpConnectionCode expired = new HttpConnectionCode(active.serverId(), active.endpoint(), active.serverCertificatePin(), active.caCertificatePin(), + java.time.Instant.now().minusSeconds(1), active.enrollmentToken()); + try (HttpBackendTransportConnector ignored = new HttpBackendTransportConnector(expired, "lobby-1", directory.resolve("client"), message -> { })) { + assertTrue(true); + } + try (HttpBackendTransportConnector ignored = new HttpBackendTransportConnector(directory.resolve("client"), message -> { })) { + assertTrue(true); + } + } + } + + @Test + void duplicateInboundDeliveryIsReAcknowledgedWithoutSecondDispatch() { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); + String session = java.util.UUID.randomUUID().toString(); + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + assertTrue(state.acceptSession(session, 0)); + assertEquals(1, state.acceptIncoming(java.util.List.of(delivery)).size()); + state.completeIncoming(id, true); + assertEquals(java.util.List.of(id), state.await("lobby-1", session, 0).acks()); + assertTrue(state.acceptSession(session, 1)); + assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty()); + assertEquals(java.util.List.of(id), state.await("lobby-1", session, 1).acks()); + String replacementSession = java.util.UUID.randomUUID().toString(); + assertTrue(state.acceptSession(replacementSession, 0)); + assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty()); + assertEquals(java.util.List.of(id), state.await("lobby-1", replacementSession, 0).acks()); + } + + @Test + void backendReAcknowledgesLostAckDuplicateWithoutSecondCallback() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + CountDownLatch callback = new CountDownLatch(1); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("client")), envelope -> callback.countDown())) { + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + connector.dispatch(delivery); + assertTrue(callback.await(2, TimeUnit.SECONDS)); + assertEquals(java.util.List.of(id), connector.drainAcknowledgements()); + assertTrue(connector.accept(java.util.List.of(delivery)).isEmpty()); + assertEquals(java.util.List.of(id), connector.drainAcknowledgements()); + } + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java new file mode 100644 index 000000000..5b62bdda3 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -0,0 +1,100 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpTransportSecurityTest { + @TempDir Path directory; + + @Test + void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { + HttpConnectionCode original = new HttpConnectionCode("lobby", URI.create("https://Proxy.Example.test:8443/http"), pin('a'), pin('b'), + Instant.parse("2030-01-01T00:00:00Z"), HttpTransportSecrets.randomToken()); + String encoded = original.encode(); + HttpConnectionCode parsed = HttpConnectionCode.parse(encoded); + assertEquals(URI.create("https://proxy.example.test:8443/http/"), parsed.endpoint()); + assertEquals(original.serverCertificatePin(), parsed.serverCertificatePin()); + char last = encoded.charAt(encoded.length() - 1); + assertThrows(IllegalArgumentException.class, () -> HttpConnectionCode.parse(encoded.substring(0, encoded.length() - 1) + + (last == 'A' ? 'B' : 'A'))); + assertThrows(IllegalArgumentException.class, () -> HttpConnectionCode.parse("http://not-a-code")); + } + + @Test + void expiredCodesAreNotActive() { + HttpConnectionCode code = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test/"), pin('a'), pin('b'), + Instant.parse("2029-12-31T23:59:59Z"), HttpTransportSecrets.randomToken()); + assertFalse(code.isActive(Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC))); + assertThrows(IllegalArgumentException.class, () -> code.requireActive(Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC))); + } + + @Test + void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { + HttpTlsIdentity created = HttpTlsIdentity.loadOrCreate(directory, "localhost"); + HttpTlsIdentity loaded = HttpTlsIdentity.loadOrCreate(directory, "localhost"); + assertEquals(created.serverCertificatePin(), loaded.serverCertificatePin()); + assertEquals(created.caCertificatePin(), loaded.caCertificatePin()); + HttpConnectionCode correct = new HttpConnectionCode("lobby", URI.create("https://localhost:8443/"), created.serverCertificatePin(), + created.caCertificatePin(), Instant.now().plusSeconds(60), HttpTransportSecrets.randomToken()); + HttpConnectionCode incorrect = new HttpConnectionCode("lobby", URI.create("https://localhost:8443/"), pin('0'), created.caCertificatePin(), + Instant.now().plusSeconds(60), HttpTransportSecrets.randomToken()); + assertTrue(HttpPinnedTls.matchesServerPin(correct, created.serverCertificate())); + assertFalse(HttpPinnedTls.matchesServerPin(incorrect, created.serverCertificate())); + assertTrue(Files.exists(directory.resolve("http-transport-ca.p12"))); + HttpTlsIdentity rotated = HttpTlsIdentity.loadOrCreate(directory, "127.0.0.1"); + assertEquals(created.caCertificatePin(), rotated.caCertificatePin()); + assertNotEquals(created.serverCertificatePin(), rotated.serverCertificatePin()); + } + + @Test + void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory, "localhost"); + Clock clock = Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock); + HttpConnectionCode wrongTargetCode = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + assertThrows(IllegalArgumentException.class, () -> authority.enroll("attacker", wrongTargetCode.enrollmentToken())); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", code.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", issued.certificate())); + assertTrue(identity.validClientCertificate("LOBBY-1", issued.certificate())); + assertFalse(authority.authenticate("lobby-2", issued.certificate())); + assertThrows(IllegalArgumentException.class, () -> authority.enroll("lobby-2", code.enrollmentToken())); + authority.revoke("lobby-1"); + assertFalse(authority.authenticate("lobby-1", issued.certificate())); + HttpConnectionCode replacementCode = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate replacement = authority.enroll("lobby-1", replacementCode.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", replacement.certificate())); + assertFalse(authority.authenticate("lobby-1", issued.certificate())); + HttpClientCredentialStore.saveEnrolled(directory.resolve("client"), code, issued); + HttpClientCredentialStore.ClientCredential restored = HttpClientCredentialStore.load(directory.resolve("client")); + assertEquals(HttpTransportSecrets.certificatePin(issued.certificate()), HttpTransportSecrets.certificatePin(restored.certificate())); + HttpClientCredentialStore.HttpClientProfile profile = HttpClientCredentialStore.loadProfile(directory.resolve("client")); + assertEquals("lobby-1", profile.serverId()); + assertEquals(code.endpoint(), profile.endpoint()); + assertEquals("lobby-1", HttpClientCredentialStore.loadEnrolled(directory.resolve("client")).profile().serverId()); + assertNotEquals(null, HttpPinnedTls.mutualTlsContext(code, restored)); + Files.writeString(directory.resolve("client").resolve("http-transport-profile.properties"), "version=1\nserverId=lobby-1\n"); + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.loadProfile(directory.resolve("client"))); + HttpEnrollmentAuthority durable = new HttpEnrollmentAuthority(identity, directory.resolve("state")); + HttpConnectionCode durableCode = durable.createConnectionCode("survival", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate durableIssued = durable.enroll("survival", durableCode.enrollmentToken()); + assertTrue(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); + durable.revoke("survival"); + assertFalse(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); + } + + private static String pin(char character) { return String.valueOf(character).repeat(64); } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index bd282b5f8..05e92b462 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -679,6 +679,19 @@ class BackendConfigurationServiceTest { BackendConfigurationService.QuickPreview mqtt = service.previewQuickSetup("proxy-backend", Map.of("server", "lobby", "method", "mqtt")); assertTrue(mqtt.proposal().content().contains("BungeeMethod: MQTT")); + BackendConfigurationService.QuickPreview http = service.previewQuickSetup("proxy-backend", + Map.of("server", "lobby", "method", "http")); + assertTrue(http.proposal().content().contains("BungeeMethod: HTTP")); + } + + @Test void httpConnectionCodeIsRedactedFromManagedConfiguration() throws Exception { + Files.writeString(directory.resolve("BungeeSettings.yml"), + "HTTP:\n ConnectionCode: VPH1-sensitive-enrollment-code\n"); + BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); + + BackendConfigurationService.Document read = service.read("BungeeSettings.yml"); + assertFalse(read.content().contains("VPH1-sensitive-enrollment-code")); + assertTrue(read.content().contains(BackendConfigurationService.REDACTED)); } @Test void proxyMethodSwitchPreflightsRequiredBackendSettings() throws Exception { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java index 4e6a25da7..164c7a7ea 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java @@ -59,6 +59,19 @@ void validatesRequiredSettingsForEveryTransport() { when(config.getSpigotServerConfiguration("lobby")).thenReturn(Map.of("Host", "localhost")); assertDoesNotThrow(() -> service.validate(new ProxyMethodConfiguration(BungeeMethod.SOCKETS))); + assertThrows(IllegalArgumentException.class, + () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + when(config.getHttpHost()).thenReturn("0.0.0.0"); + when(config.getHttpPort()).thenReturn(1297); + when(config.getHttpPublicEndpoint()).thenReturn("http://proxy.example.test:1297"); + assertThrows(IllegalArgumentException.class, + () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test:1297/terminated-path"); + assertThrows(IllegalArgumentException.class, + () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test:1297"); + assertDoesNotThrow(() -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + assertThrows(IllegalArgumentException.class, () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.MYSQL))); when(config.hasDatabaseConfigured()).thenReturn(true); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/BungeeMethodTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/BungeeMethodTest.java index 9d77bd7df..df9c09bc2 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/BungeeMethodTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/BungeeMethodTest.java @@ -1,6 +1,7 @@ package com.bencodez.votingplugin.tests; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; @@ -22,4 +23,12 @@ public void standaloneTransportsSupportBackendPresence() { } } } + + @Test + public void httpMethodNameIsCaseInsensitive() { + assertEquals(BungeeMethod.HTTP, BungeeMethod.getByName("http")); + assertEquals(BungeeMethod.HTTP, BungeeMethod.getByName("HTTP")); + assertFalse(BungeeMethod.HTTP.requiresPlayerOnline()); + assertTrue(BungeeMethod.HTTP.supportsBackendPresence()); + } } diff --git a/docs/http-transport.md b/docs/http-transport.md new file mode 100644 index 000000000..63e1da0a5 --- /dev/null +++ b/docs/http-transport.md @@ -0,0 +1,50 @@ +# Secure HTTP proxy transport + +The `HTTP` bungee method gives every backend an outbound encrypted connection to one HTTPS listener on the proxy. Only the proxy port is opened; backend servers need no inbound transport ports. + +## Quick setup + +1. On the proxy, set the following in `bungeeconfig.yml`: + + ```yaml + BungeeMethod: HTTP + HTTP: + Host: '0.0.0.0' + Port: 1297 + PublicEndpoint: 'https://proxy.example.com:1297/' + ``` + +2. Allow TCP port `1297` to the proxy. `PublicEndpoint` must resolve directly to this VotingPlugin listener. +3. Restart the proxy and run `/votingpluginbungee httpcode `. The name must exactly identify the intended backend; generate a separate code for each backend. +4. On a backend, set a unique `Server`, enable bungee mode, select `HTTP`, and paste its code into `BungeeSettings.yml`: + + ```yaml + UseBungeecord: true + Server: lobby-1 + BungeeMethod: HTTP + HTTP: + ConnectionCode: 'paste-code-here' + ``` + +5. Restart the backend. Once enrollment succeeds, remove `ConnectionCode` from the configuration. The backend's private identity is stored in its VotingPlugin data folder and is reused automatically. + +Connection codes expire after 15 minutes and can be used only once. Treat a fresh code like a temporary password: transfer it privately and do not publish it in logs, tickets, or chat rooms. + +## Security model + +- TLS keys and a private certificate authority are generated automatically on the proxy. No shared transport password or public CA setup is required. +- The backend pins both the exact proxy certificate and its private authority during enrollment. Normal traffic uses a distinct client certificate bound to that backend's canonical `Server` name. +- Every normal request is authenticated again at the application boundary. A payload cannot claim another backend identity, and redirect following is disabled. +- TLS 1.3 is required and weak protocols are disabled. +- Enrollment tokens are 256-bit random capabilities, single-use, short-lived, and retained by the proxy only as hashes. +- Credentials, keys, pins, and revocation state are written atomically with owner-only permissions where the operating system supports them. +- Request bodies, queues, batches, worker pools, concurrent requests, per-backend polling, and request rates are bounded. Exact paths, methods, and JSON content types are enforced. +- Delivery IDs, acknowledgements, and duplicate suppression provide in-process at-least-once delivery. Vote caching remains responsible for application-level restart durability. + +The listener must terminate TLS itself because client-certificate authentication is part of the protocol. Do not put an HTTP TLS-terminating reverse proxy or CDN in front of it. A TCP/L4 proxy that passes TLS through unchanged is suitable. Internet-facing installations should also use the host firewall or provider firewall for volumetric denial-of-service protection; an application cannot fully absorb a link or TCP flood. + +If a backend host or its private credential is compromised, run `/votingpluginbungee httprevoke ` on the proxy before generating a new connection code. Revocation takes effect on the next request and permits a replacement identity to enroll under that server name. Keep the proxy's `http` data directory backed up and private: it contains the transport authority. + +## Performance + +The connector reuses HTTP/1.1 TLS connections, batches messages and acknowledgements, and performs all network and certificate work off the game thread. A two-second bounded long poll avoids busy polling while limiting the worst-case delay for a backend message queued just after an idle request began. All queues are bounded to prevent traffic bursts from causing unbounded memory growth. From 1fd3be90b51150a52c2d390b049bce913250cfbb Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 16:27:15 -0600 Subject: [PATCH 02/36] Validate HTTP client certificates at TLS boundary --- .../backendproxy/http/HttpTlsIdentity.java | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java index f3db354b9..b0c34ae81 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -25,6 +25,7 @@ import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509TrustManager; import org.bouncycastle.asn1.x500.X500Name; import org.bouncycastle.asn1.x509.BasicConstraints; @@ -129,8 +130,9 @@ public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost public X509Certificate serverCertificate() { return serverCertificate; } /** - * The listener requests a client certificate but validates its issuance/binding in the HTTP handler. - * This is necessary to share the enrollment and normal endpoints on one JDK HttpsServer listener. + * The listener requests an optional client certificate so enrollment can share the same port. + * Any certificate that is presented must chain to this transport's private CA; normal requests + * additionally validate the certificate's persisted backend binding in the HTTP handler. */ public SSLContext serverContext() throws Exception { KeyStore store = KeyStore.getInstance("PKCS12"); @@ -311,11 +313,24 @@ private static void setOwnerOnly(Path path) throws IOException { } private static final class EnrollmentAwareTrustManager implements X509TrustManager { - private final X509Certificate[] acceptedIssuers; - private EnrollmentAwareTrustManager(X509Certificate caCertificate) { this.acceptedIssuers = new X509Certificate[] { caCertificate }; } - @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { } + private final X509TrustManager delegate; + private EnrollmentAwareTrustManager(X509Certificate caCertificate) throws Exception { + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + trustStore.load(null, EMPTY_PASSWORD); + trustStore.setCertificateEntry("http-transport-ca", caCertificate); + TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + factory.init(trustStore); + this.delegate = Arrays.stream(factory.getTrustManagers()) + .filter(X509TrustManager.class::isInstance) + .map(X509TrustManager.class::cast) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No X.509 trust manager is available")); + } + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) throws java.security.cert.CertificateException { + delegate.checkClientTrusted(chain, authType); + } @Override public void checkServerTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } - @Override public X509Certificate[] getAcceptedIssuers() { return acceptedIssuers.clone(); } + @Override public X509Certificate[] getAcceptedIssuers() { return delegate.getAcceptedIssuers(); } } private enum CertificateRole { CA, SERVER, CLIENT } } From d27636bdcfe17b61c59b24163b209d62f0759311 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 16:59:14 -0600 Subject: [PATCH 03/36] Harden HTTP transport lifecycle --- .../http/HttpBackendTransportConnector.java | 109 +++++++++++++----- .../http/HttpClientCredentialStore.java | 98 +++++++++++++++- .../http/HttpEnrollmentAuthority.java | 61 +++++++--- .../backendproxy/http/HttpPinnedTls.java | 19 ++- .../http/HttpProxyTransportServer.java | 21 +++- .../backendproxy/http/HttpTlsIdentity.java | 106 +++++++++++++---- .../http/HttpTransportProtocol.java | 17 +++ .../transport/HttpBackendProxyTransport.java | 28 +++-- .../http/HttpTransportRuntimeTest.java | 105 +++++++++++++++++ .../http/HttpTransportSecurityTest.java | 66 ++++++++++- .../HttpBackendProxyTransportTest.java | 49 ++++++++ docs/http-transport.md | 3 +- 12 files changed, 602 insertions(+), 80 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 67030e90b..1c318f6b6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -35,17 +35,21 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private final HttpClientCredentialStore.HttpClientProfile profile; private final String serverId; private final Consumer onEnvelope; - private final HttpClient client; + private volatile HttpClient client; + private volatile HttpClientCredentialStore.ClientCredential credential; + private final Path credentialDirectory; private final URI transportEndpoint; private final ThreadPoolExecutor callbackExecutor; private final AtomicBoolean running = new AtomicBoolean(); private final Object state = new Object(); + private final Object renewal = new Object(); private final LinkedHashMap outgoing = new LinkedHashMap<>(); private final Set received = new LinkedHashSet<>(), processing = new LinkedHashSet<>(); private final ArrayDeque acknowledgements = new ArrayDeque<>(); private final String session = UUID.randomUUID().toString(); private volatile Thread poller; private long sequence; + private volatile long nextRenewalCheckNanos; public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { @@ -54,16 +58,27 @@ public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, /** Starts normal transport from the non-secret profile persisted by enrollment. */ public HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope) throws Exception { - this(enrolled == null ? null : enrolled.profile(), enrolled == null ? null : enrolled.credential(), onEnvelope); + this(enrolled, onEnvelope, null); } public HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { + this(profile, credential, onEnvelope, null); + } + + private HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope, + Path credentialDirectory) throws Exception { + this(enrolled == null ? null : enrolled.profile(), enrolled == null ? null : enrolled.credential(), onEnvelope, credentialDirectory); + } + + private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope, Path credentialDirectory) throws Exception { if (profile == null || credential == null || onEnvelope == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); if (!matchesCredential(profile, credential)) throw new IllegalArgumentException("HTTP client certificate does not match transport profile"); this.profile = profile; this.serverId = profile.serverId(); this.onEnvelope = onEnvelope; - client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) - .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); + this.credential = credential; + this.credentialDirectory = credentialDirectory; + client = client(profile, credential); transportEndpoint = profile.endpoint().resolve("v1/transport"); callbackExecutor = executor("VotingPlugin-HTTP-callback", 2, 128); } @@ -71,18 +86,20 @@ public HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, Path credentials, Consumer onEnvelope) throws Exception { - this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope); + this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope, credentials); if (code == null || !profile(code, serverId).equals(this.profile)) throw new IllegalArgumentException("HTTP transport profile does not match connection code"); } /** Starts normal transport using only the persisted certificate and non-secret profile. */ public HttpBackendTransportConnector(Path credentials, Consumer onEnvelope) throws Exception { - this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope); + this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope, credentials); } /** Performs enrollment network I/O; call this from a connector/setup worker, never a platform main thread. */ public static HttpClientCredentialStore.ClientCredential enroll(HttpConnectionCode code, String serverId, Path credentials) throws Exception { if (code == null || credentials == null || serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) throw new IllegalArgumentException("Enrollment configuration is invalid"); + if (!code.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) + throw new IllegalArgumentException("HTTP connection code belongs to a different backend"); code.requireActive(Clock.systemUTC()); byte[] payload = ("{\"server\":\"" + serverId + "\",\"token\":\"" + code.enrollmentToken() + "\"}").getBytes(StandardCharsets.UTF_8); HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) @@ -113,9 +130,10 @@ public boolean send(JsonEnvelope envelope) { } } /** A synchronous single poll, useful for lifecycle-controlled integrations and tests. */ - public boolean pollOnce() { + public synchronized boolean pollOnce() { if (!running.get()) return false; try { + maybeRenewCredential(); List acks; List messages; long requestSequence; synchronized (state) { acks = first(acknowledgements); requestSequence = sequence++; @@ -149,16 +167,22 @@ List accept(List List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : deliveries) { if (received.contains(delivery.id())) { queueAck(delivery.id()); continue; } - if (!processing.contains(delivery.id())) { if (received.size() >= HttpTransportProtocol.MAX_QUEUE) break; processing.add(delivery.id()); accepted.add(delivery); } + if (!processing.contains(delivery.id())) { + processing.add(delivery.id()); accepted.add(delivery); + } } return accepted; } } void dispatch(HttpTransportProtocol.Delivery delivery) { try { callbackExecutor.execute(() -> { boolean success = false; try { onEnvelope.accept(delivery.envelope()); success = true; } catch (RuntimeException ignored) { } - synchronized (state) { processing.remove(delivery.id()); if (success) { received.add(delivery.id()); while (received.size() > HttpTransportProtocol.MAX_QUEUE) received.remove(received.iterator().next()); queueAck(delivery.id()); } } - }); } catch (RejectedExecutionException rejected) { synchronized (state) { processing.remove(delivery.id()); } } + completeIncoming(delivery.id(), success); + }); } catch (RejectedExecutionException rejected) { completeIncoming(delivery.id(), false); } } + void completeIncoming(String id, boolean success) { synchronized (state) { + processing.remove(id); + if (success) { received.add(id); while (received.size() > HttpTransportProtocol.MAX_QUEUE) received.remove(received.iterator().next()); queueAck(id); } + } } private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } int queuedOutgoing() { synchronized (state) { return outgoing.size(); } } List drainAcknowledgements() { synchronized (state) { return drain(acknowledgements); } } @@ -180,25 +204,52 @@ private static boolean matchesCredential(HttpClientCredentialStore.HttpClientPro return false; } catch (Exception invalid) { return false; } } - private static SSLContext clientContext(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { - KeyStore store = KeyStore.getInstance("PKCS12"); store.load(null, new char[0]); - store.setKeyEntry("client", credential.privateKey(), credential.password(), new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); - KeyManagerFactory keys = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keys.init(store, credential.password()); - SSLContext context = SSLContext.getInstance("TLS"); context.init(keys.getKeyManagers(), new javax.net.ssl.TrustManager[] { new PinnedTrustManager(profile) }, null); return context; - } - private static final class PinnedTrustManager implements javax.net.ssl.X509TrustManager { - private final HttpClientCredentialStore.HttpClientProfile profile; PinnedTrustManager(HttpClientCredentialStore.HttpClientProfile profile) { this.profile = profile; } - @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } - @Override public void checkServerTrusted(X509Certificate[] chain, String authType) throws java.security.cert.CertificateException { - if (chain == null || chain.length < 2) throw new java.security.cert.CertificateException("Pinned HTTPS server chain is incomplete"); - chain[0].checkValidity(); chain[chain.length - 1].checkValidity(); - try { chain[0].verify(chain[chain.length - 1].getPublicKey()); } - catch (java.security.GeneralSecurityException invalid) { throw new java.security.cert.CertificateException("Pinned HTTPS server signature is invalid", invalid); } - if (chain[chain.length - 1].getBasicConstraints() < 0 || !HttpTransportSecrets.constantTimeEquals(profile.serverCertificatePin().getBytes(StandardCharsets.US_ASCII), HttpTransportSecrets.certificatePin(chain[0]).getBytes(StandardCharsets.US_ASCII))) throw new java.security.cert.CertificateException("Pinned HTTPS server mismatch"); - String caPin = HttpTransportSecrets.certificatePin(chain[chain.length - 1]); - if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), caPin.getBytes(StandardCharsets.US_ASCII))) - throw new java.security.cert.CertificateException("Pinned HTTPS authority mismatch"); + private void maybeRenewCredential() { + synchronized (renewal) { + Path directory = credentialDirectory; + if (directory == null || !HttpTlsIdentity.needsRenewal(credential.certificate(), Clock.systemUTC())) return; + long now = System.nanoTime(); + if (nextRenewalCheckNanos != 0L && now - nextRenewalCheckNanos < 0L) return; + nextRenewalCheckNanos = now + Duration.ofHours(6).toNanos(); + try { + byte[] body = HttpTransportProtocol.renewalRequest(serverId); + HttpRequest request = HttpRequest.newBuilder(profile.endpoint().resolve("v1/renew")).timeout(CLIENT_TIMEOUT) + .header("Content-Type", "application/json").header("Cache-Control", "no-store") + .POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() != 201 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) return; + HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(directory, issued); + HttpClientCredentialStore.ClientCredential replacement = staged.credential(); + if (!matchesCredential(profile, replacement)) throw new IllegalArgumentException("Renewed HTTP certificate is invalid"); + HttpClient replacementClient = client(profile, replacement); + HttpClientCredentialStore.activateReplacement(directory, staged); + client = replacementClient; + credential = replacement; + } catch (Exception ignored) { /* The active generation is unchanged; retry on the bounded schedule. */ } } - @Override public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } + } + private static HttpClient client(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential) throws Exception { + return HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); + } + private static SSLContext clientContext(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { + String caPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); + if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), + caPin.getBytes(StandardCharsets.US_ASCII))) throw new IllegalArgumentException("HTTP authority does not match transport profile"); + char[] password = credential.password(); + try { + KeyStore store = KeyStore.getInstance("PKCS12"); store.load(null, new char[0]); + store.setKeyEntry("client", credential.privateKey(), password, + new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); + KeyManagerFactory keys = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keys.init(store, password); + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); trustStore.load(null, new char[0]); + trustStore.setCertificateEntry("http-transport-ca", credential.caCertificate()); + javax.net.ssl.TrustManagerFactory trusts = javax.net.ssl.TrustManagerFactory.getInstance( + javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()); + trusts.init(trustStore); + SSLContext context = SSLContext.getInstance("TLS"); context.init(keys.getKeyManagers(), trusts.getTrustManagers(), null); return context; + } finally { java.util.Arrays.fill(password, '\0'); } } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index ae801ae0a..459a2c360 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -22,6 +22,8 @@ public final class HttpClientCredentialStore { private static final String BUNDLE_FILE = "http-transport-client.p12"; private static final String PASSWORD_FILE = "http-transport-client-password"; private static final String PROFILE_FILE = "http-transport-profile.properties"; + private static final String GENERATIONS_DIRECTORY = "http-transport-client-generations"; + private static final String CURRENT_FILE = "http-transport-client-current"; private HttpClientCredentialStore() { } public static void save(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws IOException { @@ -39,9 +41,16 @@ public static void save(Path directory, HttpTlsIdentity.IssuedClientCertificate public static void saveEnrolled(Path directory, HttpConnectionCode code, HttpTlsIdentity.IssuedClientCertificate issued) throws IOException { if (code == null || issued == null) throw new IllegalArgumentException("Connection code and credential are required"); - save(directory, issued); HttpClientProfile profile = new HttpClientProfile(HttpTlsIdentity.canonicalServerId(issued.serverId()), code.endpoint(), code.serverCertificatePin(), code.caCertificatePin()); + try { + StagedCredential staged = stage(directory, issued, profile); + activateReplacement(directory, staged); + } catch (IOException failure) { throw failure; + } catch (Exception failure) { throw new IOException("Could not persist HTTP client credential", failure); } + } + + private static void writeProfile(Path directory, HttpClientProfile profile) throws IOException { Properties properties = new Properties(); properties.setProperty("version", "1"); properties.setProperty("serverId", profile.serverId()); @@ -54,6 +63,10 @@ public static void saveEnrolled(Path directory, HttpConnectionCode code, HttpTls } public static ClientCredential load(Path directory) throws Exception { + return loadCredential(activeDirectory(directory)); + } + + private static ClientCredential loadCredential(Path directory) throws Exception { Path bundle = safe(directory.resolve(BUNDLE_FILE)); Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); if (!Files.isRegularFile(bundle, LinkOption.NOFOLLOW_LINKS) || !Files.isRegularFile(passwordFile, LinkOption.NOFOLLOW_LINKS)) @@ -74,7 +87,56 @@ public static ClientCredential load(Path directory) throws Exception { } finally { java.util.Arrays.fill(password, '\0'); } } + /** Writes and validates a replacement generation without touching the active credential. */ + static StagedCredential stageReplacement(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws Exception { + return stage(directory, issued, loadProfile(directory)); + } + + private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClientCertificate issued, + HttpClientProfile profile) throws Exception { + if (directory == null || issued == null) throw new IllegalArgumentException("Credential replacement is required"); + Path generations = directory.toAbsolutePath().normalize().resolve(GENERATIONS_DIRECTORY); + Files.createDirectories(generations); + if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential generation directory is unsafe"); + String name = java.util.UUID.randomUUID().toString(); + Path generation = generations.resolve(name); + Files.createDirectory(generation); + setOwnerOnlyDirectory(generation); + try { + save(generation, issued); + writeProfile(generation, profile); + EnrolledClient enrolled = loadEnrolled(generation); + return new StagedCredential(name, enrolled.credential()); + } catch (Exception failure) { + try { Files.deleteIfExists(generation.resolve(BUNDLE_FILE)); Files.deleteIfExists(generation.resolve(PASSWORD_FILE)); + Files.deleteIfExists(generation.resolve(PROFILE_FILE)); Files.deleteIfExists(generation); } + catch (IOException cleanup) { failure.addSuppressed(cleanup); } + throw failure; + } + } + + /** Atomically makes a fully validated generation durable and active. */ + static void activateReplacement(Path directory, StagedCredential staged) throws IOException { + if (directory == null || staged == null || !staged.name().matches("[0-9a-f-]{36}")) + throw new IllegalArgumentException("Staged credential is invalid"); + Path generations = directory.toAbsolutePath().normalize().resolve(GENERATIONS_DIRECTORY); + Path generation = generations.resolve(staged.name()).normalize(); + if (!generation.getParent().equals(generations) || Files.isSymbolicLink(generation) + || !Files.isRegularFile(generation.resolve(BUNDLE_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(generation.resolve(PASSWORD_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(generation.resolve(PROFILE_FILE), LinkOption.NOFOLLOW_LINKS)) + throw new IOException("Staged HTTP credential is incomplete"); + writePrivate(safe(directory.resolve(CURRENT_FILE)), staged.name().getBytes(StandardCharsets.US_ASCII)); + } + + static record StagedCredential(String name, ClientCredential credential) { } + public static HttpClientProfile loadProfile(Path directory) throws IOException { + return loadProfileFile(activeDirectory(directory)); + } + + private static HttpClientProfile loadProfileFile(Path directory) throws IOException { Path profile = safe(directory.resolve(PROFILE_FILE)); if (!Files.isRegularFile(profile, LinkOption.NOFOLLOW_LINKS) || Files.size(profile) > 8192) throw new IOException("HTTP transport profile has not been enrolled"); @@ -88,10 +150,16 @@ public static HttpClientProfile loadProfile(Path directory) throws IOException { } catch (IllegalArgumentException failure) { throw new IOException("HTTP transport profile is invalid", failure); } } + public static boolean hasEnrolledProfile(Path directory) { + try { loadProfile(directory); return true; } + catch (Exception unavailable) { return false; } + } + /** Loads and cross-checks the persisted client key material and bound normal-transport profile. */ public static EnrolledClient loadEnrolled(Path directory) throws Exception { - ClientCredential credential = load(directory); - HttpClientProfile profile = loadProfile(directory); + Path active = activeDirectory(directory); + ClientCredential credential = loadCredential(active); + HttpClientProfile profile = loadProfileFile(active); if (!matchesProfile(credential, profile)) throw new IOException("HTTP client certificate does not match its profile"); return new EnrolledClient(profile, credential); } @@ -116,6 +184,9 @@ public record EnrolledClient(HttpClientProfile profile, ClientCredential credent private static boolean matchesProfile(ClientCredential credential, HttpClientProfile profile) { try { + String authorityPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); + if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), + authorityPin.getBytes(StandardCharsets.US_ASCII))) return false; credential.certificate().checkValidity(); credential.certificate().verify(credential.caCertificate().getPublicKey()); java.util.List usage = credential.certificate().getExtendedKeyUsage(); @@ -136,6 +207,21 @@ private static Path safe(Path file) throws IOException { return file.toAbsolutePath().normalize(); } + private static Path activeDirectory(Path directory) throws IOException { + Path root = directory.toAbsolutePath().normalize(); + Path current = safe(root.resolve(CURRENT_FILE)); + if (!Files.exists(current, LinkOption.NOFOLLOW_LINKS)) return root; + if (!Files.isRegularFile(current, LinkOption.NOFOLLOW_LINKS) || Files.size(current) > 64) + throw new IOException("HTTP client credential pointer is invalid"); + String name = Files.readString(current, StandardCharsets.US_ASCII); + if (!name.matches("[0-9a-f-]{36}")) throw new IOException("HTTP client credential pointer is invalid"); + Path generations = root.resolve(GENERATIONS_DIRECTORY); + Path generation = generations.resolve(name).normalize(); + if (!generation.getParent().equals(generations) || Files.isSymbolicLink(generation) || !Files.isDirectory(generation, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP client credential generation is invalid"); + return generation; + } + private static void writePrivate(Path file, byte[] contents) throws IOException { Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); try { @@ -154,6 +240,12 @@ private static void setOwnerOnly(Path path) throws IOException { catch (UnsupportedOperationException ignored) { } } + private static void setOwnerOnlyDirectory(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } + catch (UnsupportedOperationException ignored) { } + } + private static byte[] asciiBytes(char[] characters) { byte[] output = new byte[characters.length]; for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java index 5d16a0eeb..9ac9b0c51 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java @@ -70,14 +70,15 @@ public synchronized HttpTlsIdentity.IssuedClientCertificate enroll(String server expireEnrollments(); byte[] suppliedHash = HttpTransportSecrets.sha256(enrollmentToken.getBytes(StandardCharsets.US_ASCII)); String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(suppliedHash); - Enrollment enrollment = enrollments.remove(lookup); // consume before issuing: failed retries require a fresh code. + Enrollment enrollment = enrollments.get(lookup); if (enrollment == null || !HttpTransportSecrets.constantTimeEquals(enrollment.tokenHash(), suppliedHash)) throw new IllegalArgumentException("Enrollment was rejected"); if (!serverId.equals(enrollment.serverId())) throw new IllegalArgumentException("Enrollment was rejected"); + enrollments.remove(lookup); // consume only after the token and its intended backend both match. ClientBinding existing = bindings.get(serverId); if (existing != null && !existing.revoked()) throw new IllegalStateException("Server id is already enrolled"); HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); - bindings.put(serverId, new ClientBinding(HttpTransportSecrets.certificatePin(issued.certificate()), false)); + bindings.put(serverId, new ClientBinding(HttpTransportSecrets.certificatePin(issued.certificate()), null, false)); try { persistState(); } catch (java.io.IOException failure) { persistenceFailure = true; throw failure; } return issued; @@ -90,9 +91,28 @@ public synchronized boolean authenticate(String serverId, java.security.cert.X50 if (!identity.validClientCertificate(serverId, certificate)) return false; ClientBinding binding = bindings.get(serverId); String pin = HttpTransportSecrets.certificatePin(certificate); - return binding != null && !binding.revoked() && !revokedCertificatePins.contains(pin) - && HttpTransportSecrets.constantTimeEquals(binding.certificatePin().getBytes(StandardCharsets.US_ASCII), - pin.getBytes(StandardCharsets.US_ASCII)); + if (binding == null || binding.revoked() || revokedCertificatePins.contains(pin)) return false; + if (samePin(binding.certificatePin(), pin)) return true; + if (!samePin(binding.pendingCertificatePin(), pin)) return false; + bindings.put(serverId, new ClientBinding(pin, null, false)); + revokedCertificatePins.add(binding.certificatePin()); + try { persistState(); return true; } + catch (java.io.IOException failure) { persistenceFailure = true; return false; } + } + + /** Issues a replacement while the currently bound certificate is still valid. The old binding remains active + * until the replacement successfully authenticates, making a lost renewal response safe to retry. */ + public synchronized HttpTlsIdentity.IssuedClientCertificate renew(String serverId, + java.security.cert.X509Certificate currentCertificate) throws Exception { + if (!authenticate(serverId, currentCertificate)) throw new IllegalArgumentException("Certificate renewal was rejected"); + serverId = HttpTlsIdentity.canonicalServerId(serverId); + ClientBinding binding = bindings.get(serverId); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); + bindings.put(serverId, new ClientBinding(binding.certificatePin(), + HttpTransportSecrets.certificatePin(issued.certificate()), false)); + try { persistState(); } + catch (java.io.IOException failure) { persistenceFailure = true; throw failure; } + return issued; } public synchronized void revoke(String serverId) { @@ -100,8 +120,9 @@ public synchronized void revoke(String serverId) { catch (IllegalArgumentException invalid) { return; } ClientBinding binding = bindings.get(serverId); if (binding != null) { - bindings.put(serverId, new ClientBinding(binding.certificatePin(), true)); + bindings.put(serverId, new ClientBinding(binding.certificatePin(), binding.pendingCertificatePin(), true)); revokedCertificatePins.add(binding.certificatePin()); + if (binding.pendingCertificatePin() != null) revokedCertificatePins.add(binding.pendingCertificatePin()); try { persistState(); } catch (java.io.IOException failure) { persistenceFailure = true; throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); } } @@ -118,22 +139,31 @@ private synchronized void loadState() throws java.io.IOException { String serverId = new String(Base64.getUrlDecoder().decode(key.substring("binding.".length())), StandardCharsets.UTF_8); serverId = HttpTlsIdentity.canonicalServerId(serverId); String[] value = properties.getProperty(key, "").split(":", -1); - if (value.length != 2 || !value[0].matches("[0-9a-f]{64}") || !("0".equals(value[1]) || "1".equals(value[1]))) + if (!((value.length == 2 && "1".equals(properties.getProperty("version"))) + || (value.length == 3 && "2".equals(properties.getProperty("version")))) + || !value[0].matches("[0-9a-f]{64}")) + throw new java.io.IOException("HTTP enrollment state is invalid"); + String pending = value.length == 3 && !"-".equals(value[1]) ? value[1] : null; + String revoked = value[value.length - 1]; + if ((pending != null && !pending.matches("[0-9a-f]{64}")) || !("0".equals(revoked) || "1".equals(revoked))) throw new java.io.IOException("HTTP enrollment state is invalid"); - bindings.put(serverId, new ClientBinding(value[0], "1".equals(value[1]))); - if ("1".equals(value[1])) revokedCertificatePins.add(value[0]); + bindings.put(serverId, new ClientBinding(value[0], pending, "1".equals(revoked))); + if ("1".equals(revoked)) { revokedCertificatePins.add(value[0]); if (pending != null) revokedCertificatePins.add(pending); } } else if (!"version".equals(key)) throw new java.io.IOException("HTTP enrollment state is invalid"); } - if (!"1".equals(properties.getProperty("version"))) throw new java.io.IOException("HTTP enrollment state is invalid"); + if (!("1".equals(properties.getProperty("version")) || "2".equals(properties.getProperty("version")))) + throw new java.io.IOException("HTTP enrollment state is invalid"); } private synchronized void persistState() throws java.io.IOException { if (stateFile == null) return; Properties properties = new Properties(); - properties.setProperty("version", "1"); + properties.setProperty("version", "2"); for (Map.Entry entry : bindings.entrySet()) { String key = Base64.getUrlEncoder().withoutPadding().encodeToString(entry.getKey().getBytes(StandardCharsets.UTF_8)); - properties.setProperty("binding." + key, entry.getValue().certificatePin() + ":" + (entry.getValue().revoked() ? "1" : "0")); + properties.setProperty("binding." + key, entry.getValue().certificatePin() + ":" + + (entry.getValue().pendingCertificatePin() == null ? "-" : entry.getValue().pendingCertificatePin()) + + ":" + (entry.getValue().revoked() ? "1" : "0")); } java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); properties.store(bytes, "VotingPlugin HTTP certificate bindings"); @@ -171,5 +201,10 @@ private record Enrollment(byte[] tokenHash, Instant expiresAt, String serverId) private Enrollment { tokenHash = tokenHash.clone(); } @Override public byte[] tokenHash() { return tokenHash.clone(); } } - private record ClientBinding(String certificatePin, boolean revoked) { } + private static boolean samePin(String expected, String actual) { + return expected != null && actual != null && HttpTransportSecrets.constantTimeEquals( + expected.getBytes(StandardCharsets.US_ASCII), actual.getBytes(StandardCharsets.US_ASCII)); + } + + private record ClientBinding(String certificatePin, String pendingCertificatePin, boolean revoked) { } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java index 1aaa04cab..9da95637d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpPinnedTls.java @@ -20,9 +20,9 @@ public static SSLContext clientContext(HttpConnectionCode code) throws Exception } /** - * Normal transport context: presents the enrolled client certificate and accepts only the - * proxy leaf/CA pins in the connection code. Callers must not override the HttpClient default - * endpoint-identification settings; hostname verification remains enabled. + * Normal transport context: presents the enrolled client certificate and trusts only the + * pinned private authority. Callers must not override the HttpClient default endpoint-identification + * settings; hostname verification remains enabled and the proxy leaf may renew under the same CA. */ public static SSLContext mutualTlsContext(HttpConnectionCode code, HttpClientCredentialStore.ClientCredential credential) throws Exception { @@ -30,15 +30,24 @@ public static SSLContext mutualTlsContext(HttpConnectionCode code, HttpClientCre throw new IllegalArgumentException("Enrolled client credential is required"); char[] password = credential.password(); try { + String authorityPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); + if (!HttpTransportSecrets.constantTimeEquals(code.caCertificatePin().getBytes(java.nio.charset.StandardCharsets.US_ASCII), + authorityPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) + throw new IllegalArgumentException("HTTP authority does not match connection code"); KeyStore store = KeyStore.getInstance("PKCS12"); store.load(null, new char[0]); store.setKeyEntry("client", credential.privateKey(), password, new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); KeyManagerFactory managers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); managers.init(store, password); + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + trustStore.load(null, new char[0]); + trustStore.setCertificateEntry("http-transport-ca", credential.caCertificate()); + javax.net.ssl.TrustManagerFactory trusts = javax.net.ssl.TrustManagerFactory.getInstance( + javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()); + trusts.init(trustStore); SSLContext context = SSLContext.getInstance("TLS"); - context.init(managers.getKeyManagers(), new TrustManager[] { - new PinnedServerTrustManager(code.serverCertificatePin(), code.caCertificatePin()) }, null); + context.init(managers.getKeyManagers(), trusts.getTrustManagers(), null); return context; } finally { java.util.Arrays.fill(password, '\0'); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index 542d1ab6a..18bb284d2 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -78,9 +78,26 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity handlerExecutor = executor("VotingPlugin-HTTP-handler", 4, 128); server.setExecutor(listenerExecutor); server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); + server.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); server.createContext("/v1/transport", exchange -> transport((HttpsExchange) exchange)); } + private void renew(HttpsExchange exchange) throws IOException { + if (!"/v1/renew".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } + if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } + if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } + if (!boundedFixedBody(exchange, 1024) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + try { + String serverId = HttpTransportProtocol.parseRenewal(read(exchange.getRequestBody(), 1024)); + X509Certificate certificate = peerCertificate(exchange); + if (certificate == null || !authority.authenticate(serverId, certificate)) { reply(exchange, 401, new byte[0]); return; } + HttpTlsIdentity.IssuedClientCertificate issued = authority.renew(serverId, certificate); + reply(exchange, 201, HttpTransportProtocol.enrollmentResponse(issued)); + } catch (IllegalArgumentException rejected) { reply(exchange, 403, new byte[0]); + } catch (Exception failure) { reply(exchange, 503, new byte[0]); + } finally { admission.release(); } + } + public void start() { if (closed) throw new IllegalStateException("HTTP transport is closed"); server.start(); } public int port() { return server.getAddress().getPort(); } public URI endpoint(String host) { return URI.create("https://" + host + ":" + port() + "/"); } @@ -221,7 +238,9 @@ List acceptIncoming(List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : received) { if (seen.contains(delivery.id())) { queueAck(delivery.id()); continue; } - if (!processing.contains(delivery.id())) { if (seen.size() >= HttpTransportProtocol.MAX_QUEUE) throw new IllegalArgumentException("dedup queue exhausted"); processing.add(delivery.id()); accepted.add(delivery); } + if (!processing.contains(delivery.id())) { + processing.add(delivery.id()); accepted.add(delivery); + } } return accepted; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java index b0c34ae81..afdbb34c3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -13,19 +13,25 @@ import java.security.KeyPairGenerator; import java.security.KeyStore; import java.security.PrivateKey; +import java.security.Principal; import java.security.Security; import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.time.Instant; +import java.time.Clock; +import java.time.Duration; import java.util.Date; import java.util.EnumSet; import java.util.Arrays; import java.util.Collection; import java.util.List; -import javax.net.ssl.KeyManagerFactory; +import java.net.Socket; +import javax.net.ssl.KeyManager; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509ExtendedKeyManager; import javax.net.ssl.X509TrustManager; import org.bouncycastle.asn1.x500.X500Name; import org.bouncycastle.asn1.x509.BasicConstraints; @@ -50,24 +56,34 @@ public final class HttpTlsIdentity { private static final String SERVER_FILE = "http-transport-server.p12"; private static final String PASSWORD_FILE = "http-transport-password"; private static final char[] EMPTY_PASSWORD = new char[0]; + static final Duration RENEW_BEFORE = Duration.ofDays(30); private final PrivateKey caKey; private final X509Certificate caCertificate; - private final PrivateKey serverKey; - private final X509Certificate serverCertificate; + private volatile PrivateKey serverKey; + private volatile X509Certificate serverCertificate; private final char[] password; + private final Path serverFile; + private final String advertisedHost; private HttpTlsIdentity(PrivateKey caKey, X509Certificate caCertificate, PrivateKey serverKey, - X509Certificate serverCertificate, char[] password) { + X509Certificate serverCertificate, char[] password, Path serverFile, String advertisedHost) { this.caKey = caKey; this.caCertificate = caCertificate; this.serverKey = serverKey; this.serverCertificate = serverCertificate; this.password = password.clone(); + this.serverFile = serverFile; + this.advertisedHost = advertisedHost; } public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost) throws Exception { + return loadOrCreate(directory, advertisedHost, Clock.systemUTC()); + } + + static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock clock) throws Exception { if (advertisedHost == null || advertisedHost.isBlank() || advertisedHost.length() > 253) throw new IllegalArgumentException("Advertised HTTPS host is invalid"); + if (clock == null) throw new IllegalArgumentException("Clock is required"); Files.createDirectories(directory); Path caFile = safe(directory.resolve(CA_FILE)); Path serverFile = safe(directory.resolve(SERVER_FILE)); @@ -87,28 +103,29 @@ public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost X509Certificate serverCertificate = (X509Certificate) server.getCertificate("server"); if (caKey == null || caCertificate == null || serverKey == null || serverCertificate == null) throw new IOException("HTTP TLS identity files are invalid"); - if (!hasServerName(serverCertificate, advertisedHost)) { + if (!hasServerName(serverCertificate, advertisedHost) || needsRenewal(serverCertificate, clock)) { ensureBouncyCastle(); KeyPair serverPair = keyPair(); serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, caKey, - CertificateRole.SERVER, advertisedHost); + CertificateRole.SERVER, advertisedHost, clock.instant()); serverKey = serverPair.getPrivate(); server = KeyStore.getInstance("PKCS12"); server.load(null, EMPTY_PASSWORD); server.setKeyEntry("server", serverKey, password, new Certificate[] { serverCertificate, caCertificate }); writeStore(serverFile, server, password); } - return new HttpTlsIdentity(caKey, caCertificate, serverKey, serverCertificate, password); + return new HttpTlsIdentity(caKey, caCertificate, serverKey, serverCertificate, password, serverFile, advertisedHost); } finally { Arrays.fill(password, '\0'); } } ensureBouncyCastle(); char[] password = HttpTransportSecrets.randomToken().toCharArray(); try { KeyPair caPair = keyPair(); - X509Certificate caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, CertificateRole.CA, null); + X509Certificate caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, CertificateRole.CA, null, + clock.instant()); KeyPair serverPair = keyPair(); X509Certificate serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, - caPair.getPrivate(), CertificateRole.SERVER, advertisedHost); + caPair.getPrivate(), CertificateRole.SERVER, advertisedHost, clock.instant()); KeyStore ca = KeyStore.getInstance("PKCS12"); ca.load(null, EMPTY_PASSWORD); ca.setKeyEntry("ca", caPair.getPrivate(), password, new Certificate[] { caCertificate }); @@ -120,11 +137,16 @@ public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost byte[] passwordBytes = asciiBytes(password); try { writePrivate(passwordFile, passwordBytes); } finally { Arrays.fill(passwordBytes, (byte) 0); } - return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password); + return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password, + serverFile, advertisedHost); } finally { Arrays.fill(password, '\0'); } } - public String serverCertificatePin() { return HttpTransportSecrets.certificatePin(serverCertificate); } + public String serverCertificatePin() { + try { renewServerCertificateIfNeeded(); } + catch (Exception failure) { throw new IllegalStateException("Could not renew HTTP server certificate", failure); } + return HttpTransportSecrets.certificatePin(serverCertificate); + } public String caCertificatePin() { return HttpTransportSecrets.certificatePin(caCertificate); } public X509Certificate caCertificate() { return caCertificate; } public X509Certificate serverCertificate() { return serverCertificate; } @@ -135,22 +157,38 @@ public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost * additionally validate the certificate's persisted backend binding in the HTTP handler. */ public SSLContext serverContext() throws Exception { - KeyStore store = KeyStore.getInstance("PKCS12"); - store.load(null, EMPTY_PASSWORD); - store.setKeyEntry("server", serverKey, password, new Certificate[] { serverCertificate, caCertificate }); - KeyManagerFactory keyManagers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagers.init(store, password); + renewServerCertificateIfNeeded(); SSLContext context = SSLContext.getInstance("TLS"); - context.init(keyManagers.getKeyManagers(), new TrustManager[] { new EnrollmentAwareTrustManager(caCertificate) }, null); + context.init(new KeyManager[] { new RotatingServerKeyManager() }, + new TrustManager[] { new EnrollmentAwareTrustManager(caCertificate) }, null); return context; } + private synchronized void renewServerCertificateIfNeeded() throws Exception { + if (!needsRenewal(serverCertificate, Clock.systemUTC())) return; + ensureBouncyCastle(); + KeyPair pair = keyPair(); + X509Certificate replacement = certificate("CN=" + certificateName(advertisedHost), pair, caCertificate, caKey, + CertificateRole.SERVER, advertisedHost, Instant.now()); + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, EMPTY_PASSWORD); + store.setKeyEntry("server", pair.getPrivate(), password, new Certificate[] { replacement, caCertificate }); + writeStore(serverFile, store, password); + serverKey = pair.getPrivate(); + serverCertificate = replacement; + } + public IssuedClientCertificate issueClientCertificate(String serverId) throws Exception { + return issueClientCertificate(serverId, Instant.now()); + } + + IssuedClientCertificate issueClientCertificate(String serverId, Instant issuedAt) throws Exception { serverId = canonicalServerId(serverId); + if (issuedAt == null) throw new IllegalArgumentException("Certificate issuance time is required"); ensureBouncyCastle(); KeyPair pair = keyPair(); X509Certificate certificate = certificate("CN=" + serverId, pair, caCertificate, caKey, CertificateRole.CLIENT, - "urn:votingplugin:http-backend:" + serverId); + "urn:votingplugin:http-backend:" + serverId, issuedAt); char[] clientPassword = HttpTransportSecrets.randomToken().toCharArray(); try { KeyStore store = KeyStore.getInstance("PKCS12"); @@ -212,8 +250,7 @@ private static KeyPair keyPair() throws Exception { } private static X509Certificate certificate(String subject, KeyPair subjectKey, X509Certificate issuer, PrivateKey issuerKey, - CertificateRole role, String subjectAlternativeName) throws Exception { - Instant now = Instant.now(); + CertificateRole role, String subjectAlternativeName, Instant now) throws Exception { X500Name issuerName = issuer == null ? new X500Name(subject) : new X500Name(issuer.getSubjectX500Principal().getName()); X509v3CertificateBuilder builder = new JcaX509v3CertificateBuilder(issuerName, new BigInteger(160, new java.security.SecureRandom()).setBit(159), Date.from(now.minusSeconds(300)), @@ -240,6 +277,10 @@ private static X509Certificate certificate(String subject, KeyPair subjectKey, X return new JcaX509CertificateConverter().setProvider("BC").getCertificate(holder); } + static boolean needsRenewal(X509Certificate certificate, Clock clock) { + return certificate == null || !certificate.getNotAfter().toInstant().isAfter(clock.instant().plus(RENEW_BEFORE)); + } + private static void ensureBouncyCastle() { if (Security.getProvider("BC") == null) Security.addProvider(new BouncyCastleProvider()); } @@ -312,6 +353,31 @@ private static void setOwnerOnly(Path path) throws IOException { catch (UnsupportedOperationException ignored) { /* Windows ACLs are inherited; never make the file world-readable. */ } } + private final class RotatingServerKeyManager extends X509ExtendedKeyManager { + private static final String ALIAS = "server"; + private void refresh() { + try { renewServerCertificateIfNeeded(); } + catch (Exception failure) { throw new IllegalStateException("Could not renew HTTP server certificate", failure); } + } + private String alias(String keyType) { + refresh(); + return keyType != null && ("EC".equalsIgnoreCase(keyType) || keyType.toUpperCase(java.util.Locale.ROOT).startsWith("EC_")) + ? ALIAS : null; + } + @Override public String[] getClientAliases(String keyType, Principal[] issuers) { return null; } + @Override public String chooseClientAlias(String[] keyTypes, Principal[] issuers, Socket socket) { return null; } + @Override public String[] getServerAliases(String keyType, Principal[] issuers) { + return alias(keyType) == null ? null : new String[] { ALIAS }; + } + @Override public String chooseServerAlias(String keyType, Principal[] issuers, Socket socket) { return alias(keyType); } + @Override public String chooseEngineServerAlias(String keyType, Principal[] issuers, SSLEngine engine) { return alias(keyType); } + @Override public X509Certificate[] getCertificateChain(String alias) { + refresh(); + return ALIAS.equals(alias) ? new X509Certificate[] { serverCertificate, caCertificate } : null; + } + @Override public PrivateKey getPrivateKey(String alias) { refresh(); return ALIAS.equals(alias) ? serverKey : null; } + } + private static final class EnrollmentAwareTrustManager implements X509TrustManager { private final X509TrustManager delegate; private EnrollmentAwareTrustManager(X509Certificate caCertificate) throws Exception { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java index 0224dad7d..f275e9b76 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java @@ -100,6 +100,23 @@ static Enrollment parseEnrollment(byte[] body) { } catch (RuntimeException invalid) { throw bad(); } } + static byte[] renewalRequest(String server) { + JsonObject root = new JsonObject(); + root.addProperty("server", HttpTlsIdentity.canonicalServerId(server)); + return root.toString().getBytes(StandardCharsets.UTF_8); + } + + static String parseRenewal(byte[] body) { + if (body == null || body.length == 0 || body.length > 1024) throw bad(); + try { + JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw bad(); + JsonObject root = parsed.getAsJsonObject(); + requireOnly(root, "server"); + return HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); + } catch (RuntimeException invalid) { throw bad(); } + } + static HttpTlsIdentity.IssuedClientCertificate parseEnrollmentResponse(String server, byte[] body) { if (body == null || body.length == 0 || body.length > MAX_BODY_BYTES) throw bad(); try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 9240b93c8..1b8e86569 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -1,7 +1,7 @@ package com.bencodez.votingplugin.backendproxy.transport; -import java.nio.file.Files; import java.nio.file.Path; +import java.time.Clock; import java.util.ArrayDeque; import java.util.concurrent.TimeUnit; @@ -33,6 +33,7 @@ public void start(GlobalMessageHandler messageHandler) { Path directory = plugin.getDataFolder().toPath().resolve("http"); String serverId = plugin.getBungeeSettings().getServer(); String connectionCode = plugin.getBungeeSettings().getHttpConnectionCode(); + validateConfiguration(directory, serverId, connectionCode); worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler), "VotingPlugin-HTTP-Backend-Setup"); worker.setDaemon(true); @@ -42,14 +43,14 @@ public void start(GlobalMessageHandler messageHandler) { private void initialize(Path directory, String serverId, String configuredCode, GlobalMessageHandler messageHandler) { try { - if (!Files.isRegularFile(directory.resolve("http-transport-profile.properties"))) { + if (!HttpClientCredentialStore.hasEnrolledProfile(directory)) { HttpConnectionCode code = HttpConnectionCode.parse(configuredCode); HttpBackendTransportConnector.enroll(code, serverId, directory); } HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); - HttpBackendTransportConnector replacement = new HttpBackendTransportConnector(enrolled, messageHandler::onMessage); + HttpBackendTransportConnector replacement = new HttpBackendTransportConnector(directory, messageHandler::onMessage); synchronized (lifecycle) { if (closed) { replacement.close(); @@ -92,10 +93,23 @@ public void validate() { throw new IllegalStateException("HTTP requires a valid unique backend Server name"); } Path directory = plugin.getDataFolder().toPath().resolve("http"); - if (!Files.isRegularFile(directory.resolve("http-transport-profile.properties")) - && (plugin.getBungeeSettings().getHttpConnectionCode() == null - || plugin.getBungeeSettings().getHttpConnectionCode().isBlank())) { - throw new IllegalStateException("HTTP requires a temporary ConnectionCode for initial enrollment"); + validateConfiguration(directory, serverId, plugin.getBungeeSettings().getHttpConnectionCode()); + } + + private static void validateConfiguration(Path directory, String serverId, String configuredCode) { + try { serverId = com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId); } + catch (IllegalArgumentException invalid) { throw new IllegalStateException("HTTP requires a valid unique backend Server name", invalid); } + if (!HttpClientCredentialStore.hasEnrolledProfile(directory)) { + if (configuredCode == null || configuredCode.isBlank()) + throw new IllegalStateException("HTTP requires a temporary ConnectionCode for initial enrollment"); + try { + HttpConnectionCode code = HttpConnectionCode.parse(configuredCode); + code.requireActive(Clock.systemUTC()); + if (!code.serverId().equals(serverId)) + throw new IllegalArgumentException("Connection code belongs to a different backend"); + } catch (IllegalArgumentException invalid) { + throw new IllegalStateException("HTTP ConnectionCode is invalid, expired, or belongs to a different backend", invalid); + } } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 2273959d1..77d1ac512 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -11,6 +11,7 @@ import java.net.http.HttpResponse; import java.nio.file.Path; import java.time.Duration; +import java.time.Instant; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -105,6 +106,71 @@ void persistedProfileStartsAfterTheEnrollmentCodeExpires() throws Exception { } } + @Test + void persistedBackendConnectsAfterAutomaticServerLeafRotation() throws Exception { + Instant now = Instant.now(); + Path proxyDirectory = directory.resolve("proxy"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost", + java.time.Clock.fixed(now.minus(Duration.ofDays(340)), java.time.ZoneOffset.UTC)); + String originalServerPin = HttpTransportSecrets.certificatePin(original.serverCertificate()); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(original, directory.resolve("authority")); + HttpTlsIdentity rotated = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost"); + CountDownLatch received = new CountDownLatch(1); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), rotated, + authority, ignored -> received.countDown())) { + HttpConnectionCode activeCode = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", activeCode.enrollmentToken()); + HttpConnectionCode oldProfileCode = new HttpConnectionCode(activeCode.serverId(), activeCode.endpoint(), originalServerPin, + activeCode.caCertificatePin(), activeCode.expiresAt(), activeCode.enrollmentToken()); + HttpClientCredentialStore.saveEnrolled(directory.resolve("client"), oldProfileCode, issued); + assertFalse(oldProfileCode.serverCertificatePin().equals(rotated.serverCertificatePin())); + server.start(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(directory.resolve("client"), ignored -> { })) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("after-rotation").build())); + assertTrue(received.await(8, TimeUnit.SECONDS)); + } + } + } + + @Test + void backendRenewsClientCertificateBeforeExpiryWithoutNewConnectionCode() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate expiring = identity.issueClientCertificate("lobby-1", + Instant.now().minus(Duration.ofDays(340))); + String originalPin = HttpTransportSecrets.certificatePin(expiring.certificate()); + Path authorityDirectory = directory.resolve("authority"); + java.nio.file.Files.createDirectories(authorityDirectory); + String key = java.util.Base64.getUrlEncoder().withoutPadding() + .encodeToString("lobby-1".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + java.nio.file.Files.writeString(authorityDirectory.resolve("http-transport-clients.properties"), + "version=2\nbinding." + key + "=" + originalPin + ":-:0\n"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, ignored -> { })) { + HttpConnectionCode profileCode = new HttpConnectionCode("lobby-1", server.endpoint("localhost"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); + Path clientDirectory = directory.resolve("client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, profileCode, expiring); + server.start(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { + connector.start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(8); + String renewedPin = originalPin; + while (renewedPin.equals(originalPin) && System.nanoTime() < deadline) { + Thread.sleep(25); + renewedPin = HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(clientDirectory).certificate()); + } + assertFalse(renewedPin.equals(originalPin)); + HttpClientCredentialStore.ClientCredential renewed = HttpClientCredentialStore.load(clientDirectory); + long promotionDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (authority.authenticate("lobby-1", expiring.certificate()) && System.nanoTime() < promotionDeadline) Thread.sleep(25); + assertTrue(authority.authenticate("lobby-1", renewed.certificate())); + assertFalse(authority.authenticate("lobby-1", expiring.certificate())); + } + } + } + @Test void duplicateInboundDeliveryIsReAcknowledgedWithoutSecondDispatch() { HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); @@ -124,6 +190,24 @@ void duplicateInboundDeliveryIsReAcknowledgedWithoutSecondDispatch() { assertEquals(java.util.List.of(id), state.await("lobby-1", replacementSession, 0).acks()); } + @Test + void proxyDedupWindowEvictsOldestCompletedDeliveryAtCapacity() { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); + String oldest = null, newest = null; + for (int index = 0; index < HttpTransportProtocol.MAX_QUEUE + 1; index++) { + String id = java.util.UUID.randomUUID().toString(); + if (index == 0) oldest = id; + newest = id; + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + assertEquals(1, state.acceptIncoming(java.util.List.of(delivery)).size()); + state.completeIncoming(id, true); + } + HttpTransportProtocol.Delivery evicted = new HttpTransportProtocol.Delivery(oldest, JsonEnvelope.builder("x").build()); + HttpTransportProtocol.Delivery retained = new HttpTransportProtocol.Delivery(newest, JsonEnvelope.builder("x").build()); + assertEquals(1, state.acceptIncoming(java.util.List.of(evicted)).size()); + assertTrue(state.acceptIncoming(java.util.List.of(retained)).isEmpty()); + } + @Test void backendReAcknowledgesLostAckDuplicateWithoutSecondCallback() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); @@ -143,4 +227,25 @@ void backendReAcknowledgesLostAckDuplicateWithoutSecondCallback() throws Excepti assertEquals(java.util.List.of(id), connector.drainAcknowledgements()); } } + + @Test + void backendDedupWindowContinuesAfterCapacity() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("client")), envelope -> { })) { + for (int index = 0; index < HttpTransportProtocol.MAX_QUEUE; index++) { + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + assertEquals(1, connector.accept(java.util.List.of(delivery)).size()); + connector.completeIncoming(id, true); + } + HttpTransportProtocol.Delivery next = new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").build()); + assertEquals(1, connector.accept(java.util.List.of(next)).size()); + } + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 5b62bdda3..996eda9be 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -66,6 +66,9 @@ void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock); HttpConnectionCode wrongTargetCode = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); assertThrows(IllegalArgumentException.class, () -> authority.enroll("attacker", wrongTargetCode.enrollmentToken())); + assertTrue(authority.authenticate("lobby-1", authority.enroll("lobby-1", wrongTargetCode.enrollmentToken()).certificate()), + "a wrong backend must not consume another backend's connection code"); + authority.revoke("lobby-1"); HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", code.enrollmentToken()); assertTrue(authority.authenticate("lobby-1", issued.certificate())); @@ -86,7 +89,10 @@ void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { assertEquals(code.endpoint(), profile.endpoint()); assertEquals("lobby-1", HttpClientCredentialStore.loadEnrolled(directory.resolve("client")).profile().serverId()); assertNotEquals(null, HttpPinnedTls.mutualTlsContext(code, restored)); - Files.writeString(directory.resolve("client").resolve("http-transport-profile.properties"), "version=1\nserverId=lobby-1\n"); + Path clientDirectory = directory.resolve("client"); + String generation = Files.readString(clientDirectory.resolve("http-transport-client-current")); + Files.writeString(clientDirectory.resolve("http-transport-client-generations").resolve(generation) + .resolve("http-transport-profile.properties"), "version=1\nserverId=lobby-1\n"); assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.loadProfile(directory.resolve("client"))); HttpEnrollmentAuthority durable = new HttpEnrollmentAuthority(identity, directory.resolve("state")); HttpConnectionCode durableCode = durable.createConnectionCode("survival", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); @@ -96,5 +102,63 @@ void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { assertFalse(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); } + @Test + void renewalKeepsOldCredentialUntilReplacementAuthenticates() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("state")); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate original = authority.enroll("lobby-1", code.enrollmentToken()); + HttpTlsIdentity.IssuedClientCertificate replacement = authority.renew("lobby-1", original.certificate()); + + assertTrue(authority.authenticate("lobby-1", original.certificate()), "lost renewal responses must leave the old credential usable"); + assertTrue(authority.authenticate("lobby-1", replacement.certificate()), "first replacement request promotes the pending binding"); + assertFalse(authority.authenticate("lobby-1", original.certificate()), "promotion revokes the superseded credential"); + assertTrue(new HttpEnrollmentAuthority(identity, directory.resolve("state")) + .authenticate("lobby-1", replacement.certificate()), "promoted renewal must survive restart"); + } + + @Test + void serverLeafRotatesInsideRenewalWindowAndPreservesAuthority() throws Exception { + Instant now = Instant.now(); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(directory, "localhost", + Clock.fixed(now.minus(Duration.ofDays(340)), ZoneOffset.UTC)); + String originalPin = HttpTransportSecrets.certificatePin(original.serverCertificate()); + HttpTlsIdentity renewed = HttpTlsIdentity.loadOrCreate(directory, "localhost", Clock.fixed(now, ZoneOffset.UTC)); + assertNotEquals(originalPin, renewed.serverCertificatePin()); + assertEquals(original.caCertificatePin(), renewed.caCertificatePin()); + assertFalse(HttpTlsIdentity.needsRenewal(renewed.serverCertificate(), Clock.fixed(now, ZoneOffset.UTC))); + } + + @Test + void activeTlsContextRotatesServerLeafInsideRenewalWindow() throws Exception { + Instant now = Instant.now(); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory, "localhost", + Clock.fixed(now.minus(Duration.ofDays(340)), ZoneOffset.UTC)); + String expiringPin = HttpTransportSecrets.certificatePin(identity.serverCertificate()); + identity.serverContext(); + assertNotEquals(expiringPin, identity.serverCertificatePin()); + assertFalse(HttpTlsIdentity.needsRenewal(identity.serverCertificate(), Clock.systemUTC())); + } + + @Test + void stagedCredentialDoesNotReplaceActiveGenerationUntilAtomicActivation() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + Path client = directory.resolve("client"); + HttpTlsIdentity.IssuedClientCertificate original = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, code, original); + String originalPin = HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate()); + HttpTlsIdentity.IssuedClientCertificate replacement = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(client, replacement); + assertEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); + HttpClientCredentialStore.activateReplacement(client, staged); + assertNotEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); + HttpTlsIdentity.IssuedClientCertificate manuallyReenrolled = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, code, manuallyReenrolled); + assertEquals(HttpTransportSecrets.certificatePin(manuallyReenrolled.certificate()), + HttpTransportSecrets.certificatePin(HttpClientCredentialStore.loadEnrolled(client).credential().certificate())); + } + private static String pin(char character) { return String.valueOf(character).repeat(64); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java new file mode 100644 index 000000000..dd899304c --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -0,0 +1,49 @@ +package com.bencodez.votingplugin.backendproxy.transport; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode; +import com.bencodez.votingplugin.config.BungeeSettings; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; +import java.net.URI; +import java.nio.file.Path; +import java.time.Instant; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpBackendProxyTransportTest { + @TempDir Path directory; + + @Test + void validatesInitialConnectionCodeSynchronously() { + VotingPluginMain plugin = mock(VotingPluginMain.class); + BungeeSettings settings = mock(BungeeSettings.class); + when(plugin.getDataFolder()).thenReturn(directory.toFile()); + when(plugin.getBungeeSettings()).thenReturn(settings); + when(settings.getServer()).thenReturn("lobby-1"); + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(plugin); + + when(settings.getHttpConnectionCode()).thenReturn("malformed"); + assertThrows(IllegalStateException.class, transport::validate); + + when(settings.getHttpConnectionCode()).thenReturn(code("lobby-1", Instant.now().minusSeconds(1)).encode()); + assertThrows(IllegalStateException.class, transport::validate); + + when(settings.getHttpConnectionCode()).thenReturn(code("survival", Instant.now().plusSeconds(60)).encode()); + assertThrows(IllegalStateException.class, transport::validate); + assertThrows(IllegalStateException.class, () -> transport.start(mock(GlobalMessageHandler.class)), + "invalid configuration must fail before the enrollment worker starts"); + + when(settings.getHttpConnectionCode()).thenReturn(code("lobby-1", Instant.now().plusSeconds(60)).encode()); + assertDoesNotThrow(transport::validate); + } + + private static HttpConnectionCode code(String serverId, Instant expiry) { + return new HttpConnectionCode(serverId, URI.create("https://proxy.example.test:1297/"), "a".repeat(64), + "b".repeat(64), expiry, "A".repeat(43)); + } +} diff --git a/docs/http-transport.md b/docs/http-transport.md index 63e1da0a5..7e9bbc4c1 100644 --- a/docs/http-transport.md +++ b/docs/http-transport.md @@ -33,7 +33,8 @@ Connection codes expire after 15 minutes and can be used only once. Treat a fres ## Security model - TLS keys and a private certificate authority are generated automatically on the proxy. No shared transport password or public CA setup is required. -- The backend pins both the exact proxy certificate and its private authority during enrollment. Normal traffic uses a distinct client certificate bound to that backend's canonical `Server` name. +- Enrollment pins both the exact proxy certificate and its private authority. Normal traffic trusts only that pinned private authority and keeps HTTPS hostname verification enabled, allowing the proxy leaf certificate to rotate safely without trusting public certificate authorities. It uses a distinct client certificate bound to that backend's canonical `Server` name. +- Proxy and backend leaf certificates renew automatically during a 30-day pre-expiry window. Backend renewal is authenticated by the still-valid mTLS identity, persisted before use, and switches the proxy binding only after the replacement successfully connects; no new connection code is needed. - Every normal request is authenticated again at the application boundary. A payload cannot claim another backend identity, and redirect following is disabled. - TLS 1.3 is required and weak protocols are disabled. - Enrollment tokens are 256-bit random capabilities, single-use, short-lived, and retained by the proxy only as hashes. From 1d30c9420158266503c7e5c2750bc5d45838a9ca Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 17:11:00 -0600 Subject: [PATCH 04/36] Address HTTP transport review findings --- .../http/HttpClientCredentialStore.java | 38 ++++++++++++++++--- .../http/HttpProxyTransportServer.java | 14 +++---- .../backendproxy/http/HttpTlsIdentity.java | 33 +++++----------- .../transport/HttpBackendProxyTransport.java | 24 +++++++----- .../control/BackendConfigurationService.java | 4 +- .../http/HttpTransportRuntimeTest.java | 8 +++- .../http/HttpTransportSecurityTest.java | 19 ++++++++++ .../HttpBackendProxyTransportTest.java | 22 +++++++++++ .../BackendConfigurationServiceTest.java | 16 ++++++++ docs/http-transport.md | 2 +- 10 files changed, 132 insertions(+), 48 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index 459a2c360..f0d7fe681 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -22,6 +22,7 @@ public final class HttpClientCredentialStore { private static final String BUNDLE_FILE = "http-transport-client.p12"; private static final String PASSWORD_FILE = "http-transport-client-password"; private static final String PROFILE_FILE = "http-transport-profile.properties"; + private static final String CONNECTION_CODE_DIGEST_FILE = "http-transport-connection-code.sha256"; private static final String GENERATIONS_DIRECTORY = "http-transport-client-generations"; private static final String CURRENT_FILE = "http-transport-client-current"; private HttpClientCredentialStore() { } @@ -44,7 +45,7 @@ public static void saveEnrolled(Path directory, HttpConnectionCode code, HttpTls HttpClientProfile profile = new HttpClientProfile(HttpTlsIdentity.canonicalServerId(issued.serverId()), code.endpoint(), code.serverCertificatePin(), code.caCertificatePin()); try { - StagedCredential staged = stage(directory, issued, profile); + StagedCredential staged = stage(directory, issued, profile, connectionCodeDigest(code)); activateReplacement(directory, staged); } catch (IOException failure) { throw failure; } catch (Exception failure) { throw new IOException("Could not persist HTTP client credential", failure); } @@ -89,11 +90,12 @@ private static ClientCredential loadCredential(Path directory) throws Exception /** Writes and validates a replacement generation without touching the active credential. */ static StagedCredential stageReplacement(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws Exception { - return stage(directory, issued, loadProfile(directory)); + Path active = activeDirectory(directory); + return stage(directory, issued, loadProfileFile(active), readConnectionCodeDigest(active)); } private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClientCertificate issued, - HttpClientProfile profile) throws Exception { + HttpClientProfile profile, String connectionCodeDigest) throws Exception { if (directory == null || issued == null) throw new IllegalArgumentException("Credential replacement is required"); Path generations = directory.toAbsolutePath().normalize().resolve(GENERATIONS_DIRECTORY); Files.createDirectories(generations); @@ -106,11 +108,14 @@ private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClie try { save(generation, issued); writeProfile(generation, profile); + if (connectionCodeDigest != null) writePrivate(safe(generation.resolve(CONNECTION_CODE_DIGEST_FILE)), + connectionCodeDigest.getBytes(StandardCharsets.US_ASCII)); EnrolledClient enrolled = loadEnrolled(generation); return new StagedCredential(name, enrolled.credential()); } catch (Exception failure) { try { Files.deleteIfExists(generation.resolve(BUNDLE_FILE)); Files.deleteIfExists(generation.resolve(PASSWORD_FILE)); - Files.deleteIfExists(generation.resolve(PROFILE_FILE)); Files.deleteIfExists(generation); } + Files.deleteIfExists(generation.resolve(PROFILE_FILE)); Files.deleteIfExists(generation.resolve(CONNECTION_CODE_DIGEST_FILE)); + Files.deleteIfExists(generation); } catch (IOException cleanup) { failure.addSuppressed(cleanup); } throw failure; } @@ -151,10 +156,19 @@ private static HttpClientProfile loadProfileFile(Path directory) throws IOExcept } public static boolean hasEnrolledProfile(Path directory) { - try { loadProfile(directory); return true; } + try { loadEnrolled(directory); return true; } catch (Exception unavailable) { return false; } } + /** Returns whether this exact one-time code created the active credential, without persisting the code itself. */ + public static boolean matchesEnrollmentCode(Path directory, HttpConnectionCode code) throws IOException { + if (code == null) throw new IllegalArgumentException("Connection code is required"); + String stored = readConnectionCodeDigest(activeDirectory(directory)); + if (stored == null) return false; + return HttpTransportSecrets.constantTimeEquals(stored.getBytes(StandardCharsets.US_ASCII), + connectionCodeDigest(code).getBytes(StandardCharsets.US_ASCII)); + } + /** Loads and cross-checks the persisted client key material and bound normal-transport profile. */ public static EnrolledClient loadEnrolled(Path directory) throws Exception { Path active = activeDirectory(directory); @@ -222,6 +236,20 @@ private static Path activeDirectory(Path directory) throws IOException { return generation; } + private static String readConnectionCodeDigest(Path directory) throws IOException { + Path digest = safe(directory.resolve(CONNECTION_CODE_DIGEST_FILE)); + if (!Files.exists(digest, LinkOption.NOFOLLOW_LINKS)) return null; + if (!Files.isRegularFile(digest, LinkOption.NOFOLLOW_LINKS) || Files.size(digest) != 64) + throw new IOException("HTTP connection-code marker is invalid"); + String value = Files.readString(digest, StandardCharsets.US_ASCII); + if (!value.matches("[0-9a-f]{64}")) throw new IOException("HTTP connection-code marker is invalid"); + return value; + } + + private static String connectionCodeDigest(HttpConnectionCode code) { + return HttpTransportSecrets.sha256Hex(code.encode().getBytes(StandardCharsets.US_ASCII)); + } + private static void writePrivate(Path file, byte[] contents) throws IOException { Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index 18bb284d2..7d415bdc6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -107,12 +107,9 @@ public boolean send(String serverId, JsonEnvelope envelope) { if (closed || serverId == null || envelope == null) return false; try { serverId = HttpTlsIdentity.canonicalServerId(serverId); HttpTransportProtocol.validateEnvelope(envelope); } catch (IllegalArgumentException invalid) { return false; } - synchronized (backends) { - BackendState backend = backends.computeIfAbsent(serverId, ignored -> new BackendState()); - boolean accepted = backend.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), envelope)); - if (accepted) backend.signal(); - return accepted; - } + BackendState backend; + synchronized (backends) { backend = backends.computeIfAbsent(serverId, ignored -> new BackendState()); } + return backend.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), envelope)); } @Override public void close() { @@ -232,7 +229,10 @@ boolean acceptSession(String requested, long requestedSequence) { // prevents a captured request from being replayed with altered ACKs or a new payload. if (requestedSequence <= sequence) return false; sequence = requestedSequence; return true; } - private boolean enqueue(HttpTransportProtocol.Delivery delivery) { if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; outgoing.put(delivery.id(), delivery); return true; } + synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { + if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; + outgoing.put(delivery.id(), delivery); signal(); return true; + } private void acknowledge(Collection acks) { for (String id : acks) { outgoing.remove(id); delivered.remove(id); } } List acceptIncoming(List received) { List accepted = new java.util.ArrayList<>(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java index afdbb34c3..eb40dda21 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -32,7 +32,6 @@ import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509ExtendedKeyManager; -import javax.net.ssl.X509TrustManager; import org.bouncycastle.asn1.x500.X500Name; import org.bouncycastle.asn1.x509.BasicConstraints; import org.bouncycastle.asn1.x509.Extension; @@ -159,11 +158,19 @@ public String serverCertificatePin() { public SSLContext serverContext() throws Exception { renewServerCertificateIfNeeded(); SSLContext context = SSLContext.getInstance("TLS"); - context.init(new KeyManager[] { new RotatingServerKeyManager() }, - new TrustManager[] { new EnrollmentAwareTrustManager(caCertificate) }, null); + context.init(new KeyManager[] { new RotatingServerKeyManager() }, trustManagers(caCertificate), null); return context; } + static TrustManager[] trustManagers(X509Certificate caCertificate) throws Exception { + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + trustStore.load(null, EMPTY_PASSWORD); + trustStore.setCertificateEntry("http-transport-ca", caCertificate); + TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + factory.init(trustStore); + return factory.getTrustManagers(); + } + private synchronized void renewServerCertificateIfNeeded() throws Exception { if (!needsRenewal(serverCertificate, Clock.systemUTC())) return; ensureBouncyCastle(); @@ -378,25 +385,5 @@ private String alias(String keyType) { @Override public PrivateKey getPrivateKey(String alias) { refresh(); return ALIAS.equals(alias) ? serverKey : null; } } - private static final class EnrollmentAwareTrustManager implements X509TrustManager { - private final X509TrustManager delegate; - private EnrollmentAwareTrustManager(X509Certificate caCertificate) throws Exception { - KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); - trustStore.load(null, EMPTY_PASSWORD); - trustStore.setCertificateEntry("http-transport-ca", caCertificate); - TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - factory.init(trustStore); - this.delegate = Arrays.stream(factory.getTrustManagers()) - .filter(X509TrustManager.class::isInstance) - .map(X509TrustManager.class::cast) - .findFirst() - .orElseThrow(() -> new IllegalStateException("No X.509 trust manager is available")); - } - @Override public void checkClientTrusted(X509Certificate[] chain, String authType) throws java.security.cert.CertificateException { - delegate.checkClientTrusted(chain, authType); - } - @Override public void checkServerTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } - @Override public X509Certificate[] getAcceptedIssuers() { return delegate.getAcceptedIssuers(); } - } private enum CertificateRole { CA, SERVER, CLIENT } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 1b8e86569..1e436fbf1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -43,10 +43,8 @@ public void start(GlobalMessageHandler messageHandler) { private void initialize(Path directory, String serverId, String configuredCode, GlobalMessageHandler messageHandler) { try { - if (!HttpClientCredentialStore.hasEnrolledProfile(directory)) { - HttpConnectionCode code = HttpConnectionCode.parse(configuredCode); - HttpBackendTransportConnector.enroll(code, serverId, directory); - } + HttpConnectionCode code = enrollmentCode(directory, serverId, configuredCode); + if (code != null) HttpBackendTransportConnector.enroll(code, serverId, directory); HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); @@ -97,20 +95,28 @@ public void validate() { } private static void validateConfiguration(Path directory, String serverId, String configuredCode) { + enrollmentCode(directory, serverId, configuredCode); + } + + static HttpConnectionCode enrollmentCode(Path directory, String serverId, String configuredCode) { try { serverId = com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId); } catch (IllegalArgumentException invalid) { throw new IllegalStateException("HTTP requires a valid unique backend Server name", invalid); } - if (!HttpClientCredentialStore.hasEnrolledProfile(directory)) { - if (configuredCode == null || configuredCode.isBlank()) - throw new IllegalStateException("HTTP requires a temporary ConnectionCode for initial enrollment"); + boolean enrolled = HttpClientCredentialStore.hasEnrolledProfile(directory); + if (configuredCode != null && !configuredCode.isBlank()) { try { HttpConnectionCode code = HttpConnectionCode.parse(configuredCode); - code.requireActive(Clock.systemUTC()); if (!code.serverId().equals(serverId)) throw new IllegalArgumentException("Connection code belongs to a different backend"); - } catch (IllegalArgumentException invalid) { + if (enrolled && HttpClientCredentialStore.matchesEnrollmentCode(directory, code)) return null; + code.requireActive(Clock.systemUTC()); + return code; + } catch (Exception invalid) { throw new IllegalStateException("HTTP ConnectionCode is invalid, expired, or belongs to a different backend", invalid); } } + if (!enrolled) + throw new IllegalStateException("HTTP requires a temporary ConnectionCode for initial enrollment"); + return null; } @Override diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 0bb8d29f4..3309b7b93 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -28,6 +28,7 @@ import org.bukkit.configuration.InvalidConfigurationException; import org.bukkit.configuration.file.YamlConfiguration; +import com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore; import com.bencodez.votingplugin.proxy.BungeeMethod; import com.bencodez.votingplugin.util.DurableFiles; @@ -595,8 +596,7 @@ private void validateProxyMethod(BungeeMethod method, YamlConfiguration settings case HTTP: String connectionCode = settings.getString("HTTP.ConnectionCode", ""); if (connectionCode == null || connectionCode.isBlank()) { - Path identity = dataDirectory.resolve("http").resolve("http-transport-client.p12"); - if (!Files.isRegularFile(identity)) { + if (!HttpClientCredentialStore.hasEnrolledProfile(dataDirectory.resolve("http"))) { throw new IllegalArgumentException("HTTP.ConnectionCode must be set for initial enrollment"); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 77d1ac512..9c722a08c 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -222,7 +222,13 @@ void backendReAcknowledgesLostAckDuplicateWithoutSecondCallback() throws Excepti HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); connector.dispatch(delivery); assertTrue(callback.await(2, TimeUnit.SECONDS)); - assertEquals(java.util.List.of(id), connector.drainAcknowledgements()); + java.util.List acknowledgements = java.util.List.of(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (acknowledgements.isEmpty() && System.nanoTime() < deadline) { + acknowledgements = connector.drainAcknowledgements(); + if (acknowledgements.isEmpty()) Thread.sleep(5); + } + assertEquals(java.util.List.of(id), acknowledgements); assertTrue(connector.accept(java.util.List.of(delivery)).isEmpty()); assertEquals(java.util.List.of(id), connector.drainAcknowledgements()); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 996eda9be..cb67b5c1e 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -1,5 +1,6 @@ package com.bencodez.votingplugin.backendproxy.http; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -13,6 +14,8 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; +import java.util.Arrays; +import javax.net.ssl.X509TrustManager; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -140,6 +143,20 @@ void activeTlsContextRotatesServerLeafInsideRenewalWindow() throws Exception { assertFalse(HttpTlsIdentity.needsRenewal(identity.serverCertificate(), Clock.systemUTC())); } + @Test + void serverTlsUsesPrivateCaTrustAndRejectsForeignClients() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity foreign = HttpTlsIdentity.loadOrCreate(directory.resolve("foreign"), "localhost"); + X509TrustManager trust = Arrays.stream(HttpTlsIdentity.trustManagers(identity.caCertificate())) + .filter(X509TrustManager.class::isInstance).map(X509TrustManager.class::cast).findFirst().orElseThrow(); + HttpTlsIdentity.IssuedClientCertificate accepted = identity.issueClientCertificate("lobby-1"); + HttpTlsIdentity.IssuedClientCertificate rejected = foreign.issueClientCertificate("lobby-1"); + assertDoesNotThrow(() -> trust.checkClientTrusted( + new java.security.cert.X509Certificate[] { accepted.certificate(), identity.caCertificate() }, "EC")); + assertThrows(java.security.cert.CertificateException.class, () -> trust.checkClientTrusted( + new java.security.cert.X509Certificate[] { rejected.certificate(), foreign.caCertificate() }, "EC")); + } + @Test void stagedCredentialDoesNotReplaceActiveGenerationUntilAtomicActivation() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); @@ -154,6 +171,8 @@ void stagedCredentialDoesNotReplaceActiveGenerationUntilAtomicActivation() throw assertEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); HttpClientCredentialStore.activateReplacement(client, staged); assertNotEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, code), + "automatic certificate renewal must retain the consumed-code marker"); HttpTlsIdentity.IssuedClientCertificate manuallyReenrolled = identity.issueClientCertificate("lobby-1"); HttpClientCredentialStore.saveEnrolled(client, code, manuallyReenrolled); assertEquals(HttpTransportSecrets.certificatePin(manuallyReenrolled.certificate()), diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index dd899304c..0531cb055 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -1,12 +1,16 @@ package com.bencodez.votingplugin.backendproxy.transport; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore; import com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode; +import com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity; import com.bencodez.votingplugin.config.BungeeSettings; import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; import java.net.URI; @@ -42,6 +46,24 @@ void validatesInitialConnectionCodeSynchronously() { assertDoesNotThrow(transport::validate); } + @Test + void freshConnectionCodeOverridesAnExistingEnrollment() throws Exception { + Path credentials = directory.resolve("http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "proxy.example.test"); + HttpConnectionCode original = new HttpConnectionCode("lobby-1", URI.create("https://proxy.example.test:1297/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().minusSeconds(1), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(credentials, original, identity.issueClientCertificate("lobby-1")); + assertNull(HttpBackendProxyTransport.enrollmentCode(credentials, "lobby-1", "")); + assertNull(HttpBackendProxyTransport.enrollmentCode(credentials, "lobby-1", original.encode()), + "the already-consumed code must not be retried, even after it expires"); + + HttpConnectionCode replacement = new HttpConnectionCode("lobby-1", original.endpoint(), original.serverCertificatePin(), + original.caCertificatePin(), Instant.now().plusSeconds(60), "B".repeat(43)); + assertEquals(replacement.encode(), HttpBackendProxyTransport.enrollmentCode(credentials, "lobby-1", replacement.encode()).encode()); + assertThrows(IllegalStateException.class, + () -> HttpBackendProxyTransport.enrollmentCode(credentials, "lobby-1", "malformed")); + } + private static HttpConnectionCode code(String serverId, Instant expiry) { return new HttpConnectionCode(serverId, URI.create("https://proxy.example.test:1297/"), "a".repeat(64), "b".repeat(64), expiry, "A".repeat(43)); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index 05e92b462..23a0dc8ec 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -694,6 +694,22 @@ class BackendConfigurationServiceTest { assertTrue(read.content().contains(BackendConfigurationService.REDACTED)); } + @Test void httpMethodAcceptsGenerationBasedEnrolledProfile() throws Exception { + Files.writeString(directory.resolve("BungeeSettings.yml"), + "UseBungeecord: true\nServer: lobby-1\nBungeeMethod: PLUGINMESSAGING\nPluginMessageChannel: vp:vp\n"); + com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity identity = + com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode code = + new com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode("lobby-1", + java.net.URI.create("https://localhost:1297/"), identity.serverCertificatePin(), + identity.caCertificatePin(), java.time.Instant.now().plusSeconds(60), "A".repeat(43)); + com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore.saveEnrolled(directory.resolve("http"), code, + identity.issueClientCertificate("lobby-1")); + + BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); + assertDoesNotThrow(() -> service.previewQuickSetup("proxy-method", Map.of("method", "HTTP"))); + } + @Test void proxyMethodSwitchPreflightsRequiredBackendSettings() throws Exception { Path settings = directory.resolve("BungeeSettings.yml"); Files.writeString(settings, "UseBungeecord: true\nServer: lobby\nBungeeMethod: PLUGINMESSAGING\n" diff --git a/docs/http-transport.md b/docs/http-transport.md index 7e9bbc4c1..80bd99391 100644 --- a/docs/http-transport.md +++ b/docs/http-transport.md @@ -26,7 +26,7 @@ The `HTTP` bungee method gives every backend an outbound encrypted connection to ConnectionCode: 'paste-code-here' ``` -5. Restart the backend. Once enrollment succeeds, remove `ConnectionCode` from the configuration. The backend's private identity is stored in its VotingPlugin data folder and is reused automatically. +5. Restart the backend. Once enrollment succeeds, remove `ConnectionCode` from the configuration. The backend's private identity is stored in its VotingPlugin data folder and is reused automatically. A different nonblank code is treated as an explicit re-enrollment request; a digest lets harmless restarts recognize the already-consumed code without storing its secret token. Connection codes expire after 15 minutes and can be used only once. Treat a fresh code like a temporary password: transfer it privately and do not publish it in logs, tickets, or chat rooms. From e9bbe03612d276dd3d5a0cfc6ccdad944b4a8e1c Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 17:22:19 -0600 Subject: [PATCH 05/36] Harden HTTP shutdown and protocol parsing --- .../http/HttpTransportProtocol.java | 15 ++++++-- .../transport/HttpBackendProxyTransport.java | 37 +++++++++++-------- .../http/HttpTransportRuntimeTest.java | 22 +++++++++++ .../HttpBackendProxyTransportTest.java | 25 +++++++++++++ 4 files changed, 80 insertions(+), 19 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java index f275e9b76..a846c410b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java @@ -67,8 +67,9 @@ static Packet parsePacket(byte[] body) { String server = HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); String session = uuid(root, "session"); long sequence = nonNegative(root, "sequence"); - long timestamp = root.get("timestamp").getAsLong(); - if (Math.abs(Instant.now().toEpochMilli() - timestamp) > MAX_CLOCK_SKEW_MILLIS) throw bad(); + long timestamp = integer(root, "timestamp"); + long now = Instant.now().toEpochMilli(); + if (timestamp < now - MAX_CLOCK_SKEW_MILLIS || timestamp > now + MAX_CLOCK_SKEW_MILLIS) throw bad(); List acks = parseIds(root.get("acks")); List messages = parseMessages(root.get("messages")); return new Packet(server, session, sequence, acks, messages); @@ -179,7 +180,15 @@ private static void requireOnly(JsonObject object, String... names) { for (String name : names) if (!object.has(name) || object.get(name).isJsonNull()) throw bad(); } private static String string(JsonObject object, String name, int max) { JsonElement v = object.get(name); if (!v.isJsonPrimitive() || !v.getAsJsonPrimitive().isString()) throw bad(); String value = v.getAsString(); if (value.isEmpty() || value.length() > max) throw bad(); return value; } - private static long integer(JsonObject object, String name) { try { return object.get(name).getAsLong(); } catch (RuntimeException failure) { throw bad(); } } + private static long integer(JsonObject object, String name) { + try { + JsonElement value = object.get(name); + if (!value.isJsonPrimitive() || !value.getAsJsonPrimitive().isNumber()) throw bad(); + String token = value.getAsString(); + if (!token.matches("-?(?:0|[1-9][0-9]*)")) throw bad(); + return Long.parseLong(token); + } catch (RuntimeException failure) { throw bad(); } + } private static long nonNegative(JsonObject object, String name) { long n = integer(object, name); if (n < 0) throw bad(); return n; } private static String uuid(JsonObject object, String name) { String value = string(object, name, 64); try { return UUID.fromString(value).toString(); } catch (IllegalArgumentException invalid) { throw bad(); } } private static void validId(String id) { if (id == null || id.length() > 64) throw bad(); try { UUID.fromString(id); } catch (IllegalArgumentException invalid) { throw bad(); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 1e436fbf1..ac077a074 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -49,19 +49,21 @@ private void initialize(Path directory, String serverId, String configuredCode, if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); HttpBackendTransportConnector replacement = new HttpBackendTransportConnector(directory, messageHandler::onMessage); + boolean discard = false; synchronized (lifecycle) { if (closed) { - replacement.close(); - return; - } - connector = replacement; - replacement.start(); - while (!startupQueue.isEmpty()) { - if (!replacement.send(startupQueue.removeFirst())) { - throw new IllegalStateException("HTTP startup queue could not be transferred"); + discard = true; + } else { + connector = replacement; + replacement.start(); + while (!startupQueue.isEmpty()) { + if (!replacement.send(startupQueue.removeFirst())) { + throw new IllegalStateException("HTTP startup queue could not be transferred"); + } } } } + if (discard) replacement.close(); } catch (Exception failure) { startupFailure = new IllegalStateException("Secure HTTP backend enrollment or connection failed", failure); plugin.getLogger().severe("Secure HTTP backend transport is unavailable; check the connection code and proxy endpoint"); @@ -124,6 +126,7 @@ public void close() { Thread setup; HttpBackendTransportConnector active; synchronized (lifecycle) { + if (closed) return; closed = true; startupQueue.clear(); setup = worker; @@ -131,14 +134,16 @@ public void close() { active = connector; connector = null; } - if (setup != null) { - setup.interrupt(); - try { - setup.join(TimeUnit.SECONDS.toMillis(5)); - } catch (InterruptedException interrupted) { - Thread.currentThread().interrupt(); - } - } + if (setup != null) setup.interrupt(); + if (setup == null && active == null) return; + Thread cleanup = new Thread(() -> drain(setup, active), "VotingPlugin-HTTP-Backend-Cleanup"); + cleanup.setDaemon(true); + cleanup.start(); + } + + private static void drain(Thread setup, HttpBackendTransportConnector active) { + if (setup != null) try { setup.join(TimeUnit.SECONDS.toMillis(5)); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } if (active != null) active.close(); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 9c722a08c..75cf657ad 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -1,7 +1,9 @@ package com.bencodez.votingplugin.backendproxy.http; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; @@ -87,6 +89,26 @@ void aggregatePacketBudgetSplitsLargeValidEnvelopes() { <= HttpTransportProtocol.MAX_BODY_BYTES); } + @Test + void packetNumbersMustUseCanonicalJsonIntegerTokens() { + com.google.gson.JsonObject packet = com.google.gson.JsonParser.parseString(new String(HttpTransportProtocol.request( + "lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(), java.util.List.of()), + java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); + assertDoesNotThrow(() -> HttpTransportProtocol.parsePacket(packet.toString() + .getBytes(java.nio.charset.StandardCharsets.UTF_8))); + String timestamp = packet.get("timestamp").getAsString(); + java.util.Map> invalid = java.util.Map.of( + "v", java.util.List.of("\"1\"", "1.0", "1e0"), + "sequence", java.util.List.of("\"0\"", "0.0", "0e0"), + "timestamp", java.util.List.of("\"" + timestamp + "\"", timestamp + ".0", timestamp + "e0")); + for (var field : invalid.entrySet()) for (String token : field.getValue()) { + com.google.gson.JsonObject rejected = packet.deepCopy(); + rejected.add(field.getKey(), com.google.gson.JsonParser.parseString(token)); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + rejected.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8)), field.getKey() + "=" + token); + } + } + @Test void persistedProfileStartsAfterTheEnrollmentCodeExpires() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 0531cb055..d11aea72f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -16,6 +17,8 @@ import java.net.URI; import java.nio.file.Path; import java.time.Instant; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -64,6 +67,28 @@ void freshConnectionCodeOverridesAnExistingEnrollment() throws Exception { () -> HttpBackendProxyTransport.enrollmentCode(credentials, "lobby-1", "malformed")); } + @Test + void closeNeverWaitsForSetupOnTheCallingThread() throws Exception { + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(mock(VotingPluginMain.class)); + CountDownLatch started = new CountDownLatch(1), release = new CountDownLatch(1); + Thread blocked = new Thread(() -> { + started.countDown(); + while (release.getCount() != 0) try { release.await(); } + catch (InterruptedException ignored) { /* Simulate setup I/O that has not unwound yet. */ } + }); + blocked.start(); + assertTrue(started.await(1, TimeUnit.SECONDS)); + java.lang.reflect.Field worker = HttpBackendProxyTransport.class.getDeclaredField("worker"); + worker.setAccessible(true); + worker.set(transport, blocked); + + long startedAt = System.nanoTime(); + transport.close(); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt); + try { assertTrue(elapsedMillis < 500, "close blocked the calling thread for " + elapsedMillis + " ms"); } + finally { release.countDown(); blocked.join(TimeUnit.SECONDS.toMillis(1)); } + } + private static HttpConnectionCode code(String serverId, Instant expiry) { return new HttpConnectionCode(serverId, URI.create("https://proxy.example.test:1297/"), "a".repeat(64), "b".repeat(64), expiry, "A".repeat(43)); From 38a37ddb1b00f4b1f19918d86a2eb63f0da0c728 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 17:31:49 -0600 Subject: [PATCH 06/36] fix(http): retain votes rejected by transport queue --- .../backendproxy/http/HttpConnectionCode.java | 3 +- .../votingplugin/proxy/VotingPluginProxy.java | 63 ++++++++++++++++--- .../ProxyMethodConfigurationService.java | 1 + .../http/HttpTransportSecurityTest.java | 7 +++ .../ProxyMethodConfigurationServiceTest.java | 8 +++ .../tests/VotingPluginProxyTest.java | 43 +++++++++++++ .../tests/VotingPluginProxyTestImpl.java | 13 ++++ 7 files changed, 128 insertions(+), 10 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java index 812751426..5700147fb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java @@ -68,7 +68,8 @@ private static URI validateEndpoint(URI value) { if (value == null || !"https".equalsIgnoreCase(value.getScheme()) || value.getHost() == null || value.getUserInfo() != null || value.getFragment() != null || value.getRawQuery() != null) throw new IllegalArgumentException("Endpoint must be an absolute HTTPS URL without credentials or query"); - if (value.getPort() > 65535 || value.getPort() < -1) throw new IllegalArgumentException("Endpoint port is invalid"); + if (value.getPort() == 0 || value.getPort() > 65535 || value.getPort() < -1) + throw new IllegalArgumentException("Endpoint port is invalid"); String path = value.getRawPath(); if (path == null || path.isEmpty()) path = "/"; if (!path.endsWith("/")) path += "/"; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 9815b6a14..441aaac27 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -578,6 +578,24 @@ protected boolean sendProxyBroadcastEnvelopeNow(String server, JsonEnvelope enve } } + /** + * Sends a reward-bearing vote envelope and reports whether the selected + * transport accepted it. Legacy transports retain their existing asynchronous + * semantics; HTTP exposes its bounded-queue result so a vote is never discarded + * when the queue is full. + */ + protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope) { + if (method == BungeeMethod.HTTP) { + return sendHttpEnvelope(server, envelope); + } + GlobalMessageProxyHandler handler = globalMessageProxyHandler; + if (handler == null) { + return false; + } + handler.sendMessage(server, delay, envelope); + return true; + } + public synchronized void checkCachedVotes(String server) { int delay = 1; if (isServerValid(server)) { @@ -626,11 +644,14 @@ && getConfig().getProxyBroadcastEnabled()) { broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); } - globalMessageProxyHandler.sendMessage(server, delay, + if (!sendVoteEnvelopeAccepted(server, delay, VotingPluginWire.vote(cache.getPlayerName(), cache.getUuid(), cache.getService(), cache.getTime(), false, cache.isRealVote(), cache.getText(), cache.getVoteId(), getConfig().getBungeeManageTotals(), - broadcastHere, num, numberOfVotes)); + broadcastHere, num, numberOfVotes))) { + debug("Retaining cached vote because the transport rejected delivery for " + server); + continue; + } delay++; num++; removed.add(cache); @@ -688,10 +709,14 @@ && getConfig().getProxyBroadcastEnabled()) { } if (!cache.isRewardDelivered()) { - globalMessageProxyHandler.sendMessage(server, delay, + if (!sendVoteEnvelopeAccepted(server, delay, VotingPluginWire.voteOnline(cache.getPlayerName(), cache.getUuid(), cache.getService(), cache.getTime(), false, cache.isRealVote(), cache.getText(), cache.getVoteId(), - getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes)); + getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes))) { + debug("Retaining online vote because the transport rejected delivery for " + server); + retained.add(cache); + continue; + } // The normal envelope is also a valid broadcast delivery for the // current target. Record it so a previously pending standalone // retry cannot announce the same vote again later. @@ -2675,6 +2700,7 @@ private void startHttpTransport() { try { URI endpoint = URI.create(getConfig().getHttpPublicEndpoint()); if (!"https".equalsIgnoreCase(endpoint.getScheme()) || endpoint.getHost() == null + || endpoint.getPort() == 0 || endpoint.getPort() > 65535 || endpoint.getUserInfo() != null || endpoint.getQuery() != null || endpoint.getFragment() != null || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) { throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); @@ -3461,9 +3487,18 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea broadcastHere = proxyBroadcastDecider.shouldBroadcast(s, targets); } - globalMessageProxyHandler.sendMessage(s, 2, + if (!sendVoteEnvelopeAccepted(s, 2, VotingPluginWire.vote(player, uuid, service, time, true, realVote, text.toString(), - voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1)); + voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1))) { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addServerVote(s, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, + text.toString(), broadcastForwarded, standaloneProxyBroadcast, + proxyBroadcastTargets, broadcastForwardedServers, false)); + debug("Caching vote after the transport rejected delivery for " + s); + } } } } else { @@ -3479,11 +3514,21 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); } - globalMessageProxyHandler.sendMessage(server, 1, + boolean rewardAccepted = sendVoteEnvelopeAccepted(server, 1, VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(), voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1)); + if (!rewardAccepted) { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addOnlineVote(uuid, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), + broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, + broadcastForwardedServers, false)); + debug("Caching online vote after the transport rejected delivery for " + server); + } - if (canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled() + if (rewardAccepted && canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled() && !standaloneProxyBroadcast) { Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); @@ -3505,7 +3550,7 @@ private synchronized QueuedVoteResult vote(String player, String service, boolea } // multiproxy: envelope-only clear vote - if (getConfig().getMultiProxySupport() && getConfig().getMultiProxyOneGlobalReward()) { + if (rewardAccepted && getConfig().getMultiProxySupport() && getConfig().getMultiProxyOneGlobalReward()) { multiProxyHandler.sendClearVote(uuid, player); } } else { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java index f2c72515b..d58041214 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationService.java @@ -68,6 +68,7 @@ private void validate(ProxyMethodConfiguration proposal, VotingPluginProxyConfig try { httpEndpoint = URI.create(endpoint); } catch (RuntimeException invalid) { throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); } if (!"https".equalsIgnoreCase(httpEndpoint.getScheme()) || httpEndpoint.getHost() == null + || httpEndpoint.getPort() == 0 || httpEndpoint.getPort() > 65535 || httpEndpoint.getUserInfo() != null || httpEndpoint.getQuery() != null || httpEndpoint.getFragment() != null || (httpEndpoint.getPath() != null && !httpEndpoint.getPath().isEmpty() && !"/".equals(httpEndpoint.getPath()))) { throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index cb67b5c1e..d85574604 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -20,6 +20,13 @@ import org.junit.jupiter.api.io.TempDir; class HttpTransportSecurityTest { + @Test + void connectionCodeRejectsExplicitZeroPort() { + assertThrows(IllegalArgumentException.class, + () -> new HttpConnectionCode("lobby", URI.create("https://proxy.example.test:0/"), pin('a'), + pin('b'), Instant.now().plusSeconds(60), "token")); + } + @TempDir Path directory; @Test diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java index 164c7a7ea..f49600dce 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyMethodConfigurationServiceTest.java @@ -69,6 +69,14 @@ void validatesRequiredSettingsForEveryTransport() { when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test:1297/terminated-path"); assertThrows(IllegalArgumentException.class, () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test:0"); + assertThrows(IllegalArgumentException.class, + () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test:65536"); + assertThrows(IllegalArgumentException.class, + () -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); + when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test"); + assertDoesNotThrow(() -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); when(config.getHttpPublicEndpoint()).thenReturn("https://proxy.example.test:1297"); assertDoesNotThrow(() -> service.validate(new ProxyMethodConfiguration(BungeeMethod.HTTP))); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index b6c587a5a..4f8b711fb 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -604,6 +604,49 @@ void pendingServerBroadcastRetriesBeforeOfflineRewardDelivery() { verify(voteCache).updateServerVote("Server1", vote); } + @Test + void rejectedHttpQueueDeliveryRetainsCachedServerVote() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote vote = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "player-uuid", + "Service", 100L, true, "totals"); + Mockito.when(voteCache.hasVotes("Server1")).thenReturn(true); + Mockito.when(voteCache.getVotes("Server1")) + .thenReturn(new java.util.ArrayList<>(java.util.List.of(vote))); + Mockito.when(votingPluginProxy.getConfig().getBlockedServers()) + .thenReturn(java.util.Collections.emptyList()); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.checkCachedVotes("Server1"); + + verify(voteCache).removeServerVotes(Mockito.eq("Server1"), + Mockito.argThat(java.util.List::isEmpty)); + } + + @Test + void rejectedHttpQueueDeliveryRetainsCachedOnlineVote() { + VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); + OfflineBungeeVote vote = new OfflineBungeeVote(java.util.UUID.randomUUID(), "Player", "player-uuid", + "Service", 100L, true, "totals"); + Mockito.when(voteCache.hasOnlineVotes("player-uuid")).thenReturn(true); + Mockito.when(voteCache.getOnlineVotes("player-uuid")) + .thenReturn(new java.util.ArrayList<>(java.util.List.of(vote))); + Mockito.when(votingPluginProxy.getConfig().getBlockedServers()) + .thenReturn(java.util.Collections.emptyList()); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(voteCache).when(spyProxy).getVoteCacheHandler(); + spyProxy.checkOnlineVotes("Player", "player-uuid", "Server1"); + + assertFalse(vote.isRewardDelivered()); + verify(voteCache).addOnlineVote("player-uuid", vote); + verify(multiProxyHandler, never()).sendClearVote(Mockito.anyString(), Mockito.anyString()); + } + @Test void pendingOnlineBroadcastRetriesWhenTargetGainsAnyCarrier() { VoteCacheHandler voteCache = Mockito.mock(VoteCacheHandler.class); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index ecdd7e6d1..dd22c32ed 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -23,6 +23,7 @@ public class VotingPluginProxyTestImpl extends VotingPluginProxy { private final List warnings = new ArrayList<>(); private VotingPluginProxyConfig config; private boolean pluginMessageDeliveryResult = true; + private boolean voteEnvelopeDeliveryResult = true; private boolean communicationTestDeliveryResult = true; private JsonEnvelope lastCommunicationTestEnvelope; private boolean playerOnline = true; @@ -212,6 +213,18 @@ public boolean sendProxyBroadcastImmediately(String server, JsonEnvelope envelop return sendProxyBroadcastEnvelopeNow(server, envelope); } + @Override + protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope) { + if (getMethod() == com.bencodez.votingplugin.proxy.BungeeMethod.HTTP) { + return voteEnvelopeDeliveryResult; + } + return super.sendVoteEnvelopeAccepted(server, delay, envelope); + } + + public void setVoteEnvelopeDeliveryResult(boolean voteEnvelopeDeliveryResult) { + this.voteEnvelopeDeliveryResult = voteEnvelopeDeliveryResult; + } + @Override protected boolean sendCommunicationTestEnvelopeNow(String server, JsonEnvelope envelope) { lastCommunicationTestEnvelope = envelope; From 57226856ab49dcf52815d27a78fc6a7a02cb0a7f Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 17:42:49 -0600 Subject: [PATCH 07/36] fix(http): persist outbound deliveries until ack --- .../http/HttpProxyTransportServer.java | 146 +++++++++++++++++- .../http/HttpTransportProtocol.java | 24 +++ .../votingplugin/proxy/VotingPluginProxy.java | 2 +- .../http/HttpTransportRuntimeTest.java | 37 +++++ 4 files changed, 205 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index 7d415bdc6..a70ef95ae 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -1,6 +1,7 @@ package com.bencodez.votingplugin.backendproxy.http; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.votingplugin.util.DurableFiles; import com.sun.net.httpserver.Headers; import com.sun.net.httpserver.HttpsConfigurator; import com.sun.net.httpserver.HttpsExchange; @@ -11,6 +12,12 @@ import java.io.InputStream; import java.net.InetSocketAddress; import java.net.URI; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.time.Duration; @@ -58,13 +65,26 @@ public final class HttpProxyTransportServer implements AutoCloseable { private final ThreadPoolExecutor handlerExecutor; private final Semaphore admission = new Semaphore(64); private final Map backends = new HashMap<>(); + private final DurableOutgoingQueue durableOutgoing; private final Consumer onEnvelope; private volatile boolean closed; public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, Consumer onEnvelope) throws Exception { + this(bind, identity, authority, null, onEnvelope); + } + + public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Path outgoingDirectory, Consumer onEnvelope) throws Exception { if (bind == null || identity == null || authority == null || onEnvelope == null) throw new IllegalArgumentException("HTTP transport configuration is required"); this.identity = identity; this.authority = authority; this.onEnvelope = onEnvelope; + durableOutgoing = outgoingDirectory == null ? null : new DurableOutgoingQueue(outgoingDirectory); + if (durableOutgoing != null) for (Map.Entry> pending + : durableOutgoing.load().entrySet()) { + BackendState state = new BackendState(pending.getKey(), durableOutgoing); + state.restore(pending.getValue()); + backends.put(pending.getKey(), state); + } server = HttpsServer.create(bind, 32); server.setHttpsConfigurator(new HttpsConfigurator(identity.serverContext()) { @Override public void configure(HttpsParameters parameters) { @@ -102,13 +122,15 @@ private void renew(HttpsExchange exchange) throws IOException { public int port() { return server.getAddress().getPort(); } public URI endpoint(String host) { return URI.create("https://" + host + ":" + port() + "/"); } - /** Queues an in-memory proxy-origin envelope for a specific authenticated backend; this queue is not restart-durable. */ + /** Queues a proxy-origin envelope durably before reporting acceptance. */ public boolean send(String serverId, JsonEnvelope envelope) { if (closed || serverId == null || envelope == null) return false; try { serverId = HttpTlsIdentity.canonicalServerId(serverId); HttpTransportProtocol.validateEnvelope(envelope); } catch (IllegalArgumentException invalid) { return false; } BackendState backend; - synchronized (backends) { backend = backends.computeIfAbsent(serverId, ignored -> new BackendState()); } + final String canonicalServerId = serverId; + synchronized (backends) { backend = backends.computeIfAbsent(serverId, + ignored -> new BackendState(canonicalServerId, durableOutgoing)); } return backend.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), envelope)); } @@ -208,6 +230,8 @@ private static ThreadPoolExecutor executor(String name, int threads, int queue) public record ReceivedEnvelope(String serverId, String messageId, JsonEnvelope envelope) { } static record Response(Collection acks, Collection messages) { } static final class BackendState { + private final String serverId; + private final DurableOutgoingQueue durableOutgoing; private String session; private long sequence = -1L; private final LinkedHashMap outgoing = new LinkedHashMap<>(); private final Set seen = new LinkedHashSet<>(); private final Set processing = new LinkedHashSet<>(); @@ -217,6 +241,13 @@ static final class BackendState { private double requestTokens = 24.0d; private long lastTokenNanos = System.nanoTime(); private boolean activePoll; + BackendState() { this(null, null); } + private BackendState(String serverId, DurableOutgoingQueue durableOutgoing) { + this.serverId = serverId; this.durableOutgoing = durableOutgoing; + } + private synchronized void restore(Collection deliveries) { + for (HttpTransportProtocol.Delivery delivery : deliveries) outgoing.put(delivery.id(), delivery); + } private boolean beginPoll(String requestedSession) { synchronized (this) { if (activePoll) return false; activePoll = true; return true; } } private void endPoll() { synchronized (this) { activePoll = false; notifyAll(); } } private boolean allowRequest() { @@ -231,9 +262,18 @@ boolean acceptSession(String requested, long requestedSequence) { } synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; + if (durableOutgoing != null) try { durableOutgoing.persist(serverId, delivery); } + catch (IOException failure) { return false; } outgoing.put(delivery.id(), delivery); signal(); return true; } - private void acknowledge(Collection acks) { for (String id : acks) { outgoing.remove(id); delivered.remove(id); } } + private void acknowledge(Collection acks) { + for (String id : acks) { + if (!outgoing.containsKey(id)) continue; + if (durableOutgoing != null) try { durableOutgoing.remove(serverId, id); } + catch (IOException failure) { continue; } + outgoing.remove(id); delivered.remove(id); + } + } List acceptIncoming(List received) { List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : received) { @@ -268,4 +308,104 @@ synchronized Response await(String serverId, String requestedSession, long reque private boolean redeliveryDue() { return !outgoing.isEmpty() && lastDeliveryNanos > 0L && System.nanoTime() - lastDeliveryNanos >= LONG_POLL.toNanos(); } private synchronized void signal() { notifyAll(); } } + + private static final class DurableOutgoingQueue { + private static final String FILE_PATTERN = "[0-9]{20}-[0-9a-f-]{36}\\.json"; + private final Path root; + private final Map> files = new HashMap<>(); + private long sequence; + + private DurableOutgoingQueue(Path root) throws IOException { + this.root = root.toAbsolutePath().normalize(); + Files.createDirectories(this.root); + if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue directory is invalid"); + ownerOnlyDirectory(this.root); + } + + private synchronized Map> load() throws IOException { + Map> loaded = new LinkedHashMap<>(); + try (DirectoryStream servers = Files.newDirectoryStream(root)) { + for (Path directory : servers) { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue contains an invalid entry"); + String serverId; + try { serverId = HttpTlsIdentity.canonicalServerId(directory.getFileName().toString()); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue server is invalid", invalid); } + if (!serverId.equals(directory.getFileName().toString())) + throw new IOException("HTTP outgoing queue server is not canonical"); + List entries = new java.util.ArrayList<>(); + try (DirectoryStream messages = Files.newDirectoryStream(directory)) { + for (Path message : messages) entries.add(message); + } + entries.sort(java.util.Comparator.comparing(path -> path.getFileName().toString())); + List deliveries = new java.util.ArrayList<>(); + Map serverFiles = files.computeIfAbsent(serverId, ignored -> new HashMap<>()); + for (Path message : entries) { + String name = message.getFileName().toString(); + if (name.startsWith(".pending-") && name.endsWith(".tmp") + && !Files.isSymbolicLink(message) && Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS)) { + DurableFiles.deleteIfExists(message); + continue; + } + if (Files.isSymbolicLink(message) || !Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS) + || !name.matches(FILE_PATTERN) || Files.size(message) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L) + throw new IOException("HTTP outgoing queue message is invalid"); + HttpTransportProtocol.Delivery delivery; + try { delivery = HttpTransportProtocol.parseStoredDelivery(Files.readAllBytes(message)); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue message is invalid", invalid); } + if (!name.endsWith("-" + delivery.id() + ".json") || serverFiles.put(delivery.id(), message) != null) + throw new IOException("HTTP outgoing queue message id is invalid"); + deliveries.add(delivery); + if (deliveries.size() > HttpTransportProtocol.MAX_QUEUE) + throw new IOException("HTTP outgoing queue exceeds its bound"); + sequence = Math.max(sequence, Long.parseLong(name.substring(0, 20))); + } + if (!deliveries.isEmpty()) loaded.put(serverId, deliveries); + } + } + return loaded; + } + + private synchronized void persist(String serverId, HttpTransportProtocol.Delivery delivery) throws IOException { + Path directory = root.resolve(serverId).normalize(); + if (!directory.getParent().equals(root)) throw new IOException("HTTP outgoing queue server is invalid"); + Files.createDirectories(directory); ownerOnlyDirectory(directory); + if (sequence == Long.MAX_VALUE) throw new IOException("HTTP outgoing queue sequence is exhausted"); + String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); + Path target = directory.resolve(name); + Path temporary = Files.createTempFile(directory, ".pending-", ".tmp"); + try { + ownerOnlyFile(temporary); + Files.write(temporary, HttpTransportProtocol.storedDelivery(delivery), StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } + ownerOnlyFile(target); DurableFiles.forceDirectory(directory); + files.computeIfAbsent(serverId, ignored -> new HashMap<>()).put(delivery.id(), target); + } finally { Files.deleteIfExists(temporary); } + } + + private synchronized void remove(String serverId, String id) throws IOException { + Map serverFiles = files.get(serverId); + Path file = serverFiles == null ? null : serverFiles.get(id); + if (file == null) throw new IOException("HTTP outgoing queue acknowledgement is unknown"); + DurableFiles.deleteIfExists(file); + serverFiles.remove(id); + } + + private static void ownerOnlyFile(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, java.util.EnumSet.of( + java.nio.file.attribute.PosixFilePermission.OWNER_READ, + java.nio.file.attribute.PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { } + } + private static void ownerOnlyDirectory(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, java.util.EnumSet.of( + java.nio.file.attribute.PosixFilePermission.OWNER_READ, + java.nio.file.attribute.PosixFilePermission.OWNER_WRITE, + java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE)); } + catch (UnsupportedOperationException ignored) { } + } + } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java index a846c410b..3fa01f878 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java @@ -51,6 +51,30 @@ static List fittingMessages(String server, String session, long sequen return output; } + static byte[] storedDelivery(Delivery delivery) { + validId(delivery.id()); + validateEnvelope(delivery.envelope()); + JsonObject root = new JsonObject(); + root.addProperty("v", VERSION); + root.addProperty("id", delivery.id()); + root.addProperty("payload", Base64.getUrlEncoder().withoutPadding().encodeToString( + JsonEnvelopeCodec.encode(delivery.envelope()).getBytes(StandardCharsets.UTF_8))); + return root.toString().getBytes(StandardCharsets.UTF_8); + } + + static Delivery parseStoredDelivery(byte[] body) { + if (body == null || body.length == 0 || body.length > MAX_ENVELOPE_BYTES * 2) throw bad(); + try { + JsonObject root = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)).getAsJsonObject(); + requireOnly(root, "v", "id", "payload"); + if (integer(root, "v") != VERSION) throw bad(); + String id = string(root, "id", 64); validId(id); + byte[] payload = Base64.getUrlDecoder().decode(string(root, "payload", MAX_ENVELOPE_BYTES * 2)); + if (payload.length == 0 || payload.length > MAX_ENVELOPE_BYTES) throw bad(); + return new Delivery(id, JsonEnvelopeCodec.decode(new String(payload, StandardCharsets.UTF_8))); + } catch (RuntimeException invalid) { throw bad(); } + } + static byte[] response(String server, String session, long sequence, Collection acks, Collection messages) { return request(server, session, sequence, acks, messages); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 441aaac27..b7dc11054 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -2710,7 +2710,7 @@ private void startHttpTransport() { httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); httpTransportServer = new HttpProxyTransportServer( new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, - httpEnrollmentAuthority, received -> { + httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), received -> { GlobalMessageProxyHandler handler = globalMessageProxyHandler; if (handler == null) throw new IllegalStateException("HTTP message router is not ready"); handler.onMessage(received.envelope()); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 75cf657ad..a75bb4c1a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -76,6 +76,43 @@ void boundedQueuesFailClosed() throws Exception { } } + @Test + void proxyOutgoingQueueSurvivesRestartUntilBackendAcknowledges() throws Exception { + Path proxyDirectory = directory.resolve("proxy"); + Path authorityDirectory = directory.resolve("authority"); + Path queueDirectory = directory.resolve("outgoing"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); + HttpProxyTransportServer first = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueDirectory, ignored -> { }); + assertTrue(first.send("lobby-1", JsonEnvelope.builder("durable").build())); + first.close(); + + CountDownLatch received = new CountDownLatch(1); + try (HttpProxyTransportServer restarted = new HttpProxyTransportServer( + new InetSocketAddress("localhost", 0), identity, authority, queueDirectory, ignored -> { })) { + restarted.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", restarted.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, + "lobby-1", directory.resolve("durable-client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", + credential, envelope -> received.countDown())) { + connector.start(); + assertTrue(received.await(8, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (countRegularFiles(queueDirectory) != 0L && System.nanoTime() < deadline) Thread.sleep(20); + assertEquals(0L, countRegularFiles(queueDirectory), "backend ACK must durably remove the delivery"); + } + } + } + + private static long countRegularFiles(Path root) throws Exception { + try (java.util.stream.Stream paths = java.nio.file.Files.walk(root)) { + return paths.filter(path -> java.nio.file.Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS)).count(); + } + } + @Test void aggregatePacketBudgetSplitsLargeValidEnvelopes() { java.util.List candidates = new java.util.ArrayList<>(); From aa0f8d8ba5ef5eddeba87d99c511c9001a837593 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 17:51:08 -0600 Subject: [PATCH 08/36] fix(http): persist poll-created backend queues --- .../http/HttpProxyTransportServer.java | 3 +- .../http/HttpTransportRuntimeTest.java | 34 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index a70ef95ae..c77b8a3f0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -163,7 +163,8 @@ private void transport(HttpsExchange exchange) throws IOException { X509Certificate certificate = peerCertificate(exchange); if (certificate == null || !authority.authenticate(packet.server(), certificate)) { reply(exchange, 401, new byte[0]); return; } BackendState backend; - synchronized (backends) { backend = backends.computeIfAbsent(packet.server(), ignored -> new BackendState()); } + synchronized (backends) { backend = backends.computeIfAbsent(packet.server(), + ignored -> new BackendState(packet.server(), durableOutgoing)); } if (!backend.beginPoll(packet.session())) { reply(exchange, 409, new byte[0]); return; } try { handlePacket(packet, backend); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index a75bb4c1a..ac95b1f8b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -107,6 +107,40 @@ void proxyOutgoingQueueSurvivesRestartUntilBackendAcknowledges() throws Exceptio } } + @Test + void pollCreatedBackendStateUsesDurableOutgoingQueue() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("poll-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("poll-authority")); + Path queueDirectory = directory.resolve("poll-outgoing"); + CountDownLatch proxyReceived = new CountDownLatch(1), backendReceived = new CountDownLatch(1); + CountDownLatch releaseBackendCallback = new CountDownLatch(1); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueDirectory, ignored -> proxyReceived.countDown())) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, + "lobby-1", directory.resolve("poll-client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", + credential, envelope -> { + backendReceived.countDown(); + try { releaseBackendCallback.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + })) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("establish-poll").build())); + assertTrue(proxyReceived.await(8, TimeUnit.SECONDS)); + assertTrue(server.send("lobby-1", JsonEnvelope.builder("durable-after-poll").build())); + assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); + assertEquals(1L, countRegularFiles(queueDirectory), + "a poll-created backend state must persist before reporting acceptance"); + releaseBackendCallback.countDown(); + } + } finally { + releaseBackendCallback.countDown(); + } + } + private static long countRegularFiles(Path root) throws Exception { try (java.util.stream.Stream paths = java.nio.file.Files.walk(root)) { return paths.filter(path -> java.nio.file.Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS)).count(); From 63e8f9ddd9bc538392f6fcf6563cacffb69eb4b4 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 17:59:43 -0600 Subject: [PATCH 09/36] fix(http): validate enrollment codes in Control --- .../transport/HttpBackendProxyTransport.java | 2 +- .../control/BackendConfigurationService.java | 9 +++---- .../BackendConfigurationServiceTest.java | 27 +++++++++++++++++++ 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index ac077a074..81f520ed7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -96,7 +96,7 @@ public void validate() { validateConfiguration(directory, serverId, plugin.getBungeeSettings().getHttpConnectionCode()); } - private static void validateConfiguration(Path directory, String serverId, String configuredCode) { + public static void validateConfiguration(Path directory, String serverId, String configuredCode) { enrollmentCode(directory, serverId, configuredCode); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 3309b7b93..3c84caeb0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -28,7 +28,7 @@ import org.bukkit.configuration.InvalidConfigurationException; import org.bukkit.configuration.file.YamlConfiguration; -import com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore; +import com.bencodez.votingplugin.backendproxy.transport.HttpBackendProxyTransport; import com.bencodez.votingplugin.proxy.BungeeMethod; import com.bencodez.votingplugin.util.DurableFiles; @@ -595,11 +595,8 @@ private void validateProxyMethod(BungeeMethod method, YamlConfiguration settings break; case HTTP: String connectionCode = settings.getString("HTTP.ConnectionCode", ""); - if (connectionCode == null || connectionCode.isBlank()) { - if (!HttpClientCredentialStore.hasEnrolledProfile(dataDirectory.resolve("http"))) { - throw new IllegalArgumentException("HTTP.ConnectionCode must be set for initial enrollment"); - } - } + try { HttpBackendProxyTransport.validateConfiguration(dataDirectory.resolve("http"), server, connectionCode); } + catch (IllegalStateException invalid) { throw new IllegalArgumentException(invalid.getMessage(), invalid); } break; case MYSQL: try { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index 23a0dc8ec..a2d111189 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -710,6 +710,33 @@ class BackendConfigurationServiceTest { assertDoesNotThrow(() -> service.previewQuickSetup("proxy-method", Map.of("method", "HTTP"))); } + @Test void httpMethodPreflightRejectsInvalidExpiredAndWrongServerConnectionCodes() throws Exception { + Path settings = directory.resolve("BungeeSettings.yml"); + Files.writeString(settings, "UseBungeecord: true\nServer: lobby-1\nBungeeMethod: PLUGINMESSAGING\n" + + "PluginMessageChannel: vp:vp\nHTTP:\n ConnectionCode: malformed\n"); + BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); + assertThrows(IllegalArgumentException.class, + () -> service.previewQuickSetup("proxy-method", Map.of("method", "HTTP"))); + + com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity identity = + com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.loadOrCreate(directory.resolve("code-proxy"), "localhost"); + com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode expired = + new com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode("lobby-1", + java.net.URI.create("https://localhost:1297/"), identity.serverCertificatePin(), + identity.caCertificatePin(), java.time.Instant.now().minusSeconds(1), "A".repeat(43)); + Files.writeString(settings, Files.readString(settings).replace("malformed", expired.encode())); + assertThrows(IllegalArgumentException.class, + () -> service.previewQuickSetup("proxy-method", Map.of("method", "HTTP"))); + + com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode wrongServer = + new com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode("survival", + expired.endpoint(), expired.serverCertificatePin(), expired.caCertificatePin(), + java.time.Instant.now().plusSeconds(60), "B".repeat(43)); + Files.writeString(settings, Files.readString(settings).replace(expired.encode(), wrongServer.encode())); + assertThrows(IllegalArgumentException.class, + () -> service.previewQuickSetup("proxy-method", Map.of("method", "HTTP"))); + } + @Test void proxyMethodSwitchPreflightsRequiredBackendSettings() throws Exception { Path settings = directory.resolve("BungeeSettings.yml"); Files.writeString(settings, "UseBungeecord: true\nServer: lobby\nBungeeMethod: PLUGINMESSAGING\n" From 5b666dc501aad8675cf6a061effa455a5f84f433 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 19:18:53 -0600 Subject: [PATCH 10/36] fix(http): serialize callbacks and renew transport CA --- .../http/HttpBackendTransportConnector.java | 13 ++-- .../http/HttpClientCredentialStore.java | 7 +- .../backendproxy/http/HttpTlsIdentity.java | 67 ++++++++++++++----- .../http/HttpTransportRuntimeTest.java | 31 +++++++++ .../http/HttpTransportSecurityTest.java | 34 ++++++++++ 5 files changed, 130 insertions(+), 22 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 1c318f6b6..8733c44a5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -32,7 +32,7 @@ /** Backend-side, persistent HTTP/1.1 long-poll connector. */ public final class HttpBackendTransportConnector implements AutoCloseable { public static final Duration CLIENT_TIMEOUT = Duration.ofSeconds(35); - private final HttpClientCredentialStore.HttpClientProfile profile; + private volatile HttpClientCredentialStore.HttpClientProfile profile; private final String serverId; private final Consumer onEnvelope; private volatile HttpClient client; @@ -80,7 +80,10 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil this.credentialDirectory = credentialDirectory; client = client(profile, credential); transportEndpoint = profile.endpoint().resolve("v1/transport"); - callbackExecutor = executor("VotingPlugin-HTTP-callback", 2, 128); + // GlobalMessageHandler routes mutate backend vote state and must observe the + // wire order. One bounded lane preserves batch ordering without running work on + // the long-poll thread. + callbackExecutor = executor("VotingPlugin-HTTP-callback", 1, 128); } /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ @@ -221,9 +224,11 @@ private void maybeRenewCredential() { HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(directory, issued); HttpClientCredentialStore.ClientCredential replacement = staged.credential(); - if (!matchesCredential(profile, replacement)) throw new IllegalArgumentException("Renewed HTTP certificate is invalid"); - HttpClient replacementClient = client(profile, replacement); + HttpClientCredentialStore.HttpClientProfile replacementProfile = staged.profile(); + if (!matchesCredential(replacementProfile, replacement)) throw new IllegalArgumentException("Renewed HTTP certificate is invalid"); + HttpClient replacementClient = client(replacementProfile, replacement); HttpClientCredentialStore.activateReplacement(directory, staged); + profile = replacementProfile; client = replacementClient; credential = replacement; } catch (Exception ignored) { /* The active generation is unchanged; retry on the bounded schedule. */ } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index f0d7fe681..32b80b8f6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -107,11 +107,14 @@ private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClie setOwnerOnlyDirectory(generation); try { save(generation, issued); + ClientCredential replacement = loadCredential(generation); + profile = new HttpClientProfile(profile.serverId(), profile.endpoint(), profile.serverCertificatePin(), + HttpTransportSecrets.certificatePin(replacement.caCertificate())); writeProfile(generation, profile); if (connectionCodeDigest != null) writePrivate(safe(generation.resolve(CONNECTION_CODE_DIGEST_FILE)), connectionCodeDigest.getBytes(StandardCharsets.US_ASCII)); EnrolledClient enrolled = loadEnrolled(generation); - return new StagedCredential(name, enrolled.credential()); + return new StagedCredential(name, enrolled.credential(), enrolled.profile()); } catch (Exception failure) { try { Files.deleteIfExists(generation.resolve(BUNDLE_FILE)); Files.deleteIfExists(generation.resolve(PASSWORD_FILE)); Files.deleteIfExists(generation.resolve(PROFILE_FILE)); Files.deleteIfExists(generation.resolve(CONNECTION_CODE_DIGEST_FILE)); @@ -135,7 +138,7 @@ static void activateReplacement(Path directory, StagedCredential staged) throws writePrivate(safe(directory.resolve(CURRENT_FILE)), staged.name().getBytes(StandardCharsets.US_ASCII)); } - static record StagedCredential(String name, ClientCredential credential) { } + static record StagedCredential(String name, ClientCredential credential, HttpClientProfile profile) { } public static HttpClientProfile loadProfile(Path directory) throws IOException { return loadProfileFile(activeDirectory(directory)); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java index eb40dda21..41522eacc 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -56,21 +56,24 @@ public final class HttpTlsIdentity { private static final String PASSWORD_FILE = "http-transport-password"; private static final char[] EMPTY_PASSWORD = new char[0]; static final Duration RENEW_BEFORE = Duration.ofDays(30); + static final Duration CA_RENEW_BEFORE = Duration.ofDays(365); private final PrivateKey caKey; - private final X509Certificate caCertificate; + private volatile X509Certificate caCertificate; private volatile PrivateKey serverKey; private volatile X509Certificate serverCertificate; private final char[] password; + private final Path caFile; private final Path serverFile; private final String advertisedHost; private HttpTlsIdentity(PrivateKey caKey, X509Certificate caCertificate, PrivateKey serverKey, - X509Certificate serverCertificate, char[] password, Path serverFile, String advertisedHost) { + X509Certificate serverCertificate, char[] password, Path caFile, Path serverFile, String advertisedHost) { this.caKey = caKey; this.caCertificate = caCertificate; this.serverKey = serverKey; this.serverCertificate = serverCertificate; this.password = password.clone(); + this.caFile = caFile; this.serverFile = serverFile; this.advertisedHost = advertisedHost; } @@ -102,7 +105,18 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock X509Certificate serverCertificate = (X509Certificate) server.getCertificate("server"); if (caKey == null || caCertificate == null || serverKey == null || serverCertificate == null) throw new IOException("HTTP TLS identity files are invalid"); - if (!hasServerName(serverCertificate, advertisedHost) || needsRenewal(serverCertificate, clock)) { + boolean caRenewed = needsCaRenewal(caCertificate, clock); + if (caRenewed) { + ensureBouncyCastle(); + KeyPair caPair = new KeyPair(caCertificate.getPublicKey(), caKey); + caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, + CertificateRole.CA, null, clock.instant()); + ca = KeyStore.getInstance("PKCS12"); + ca.load(null, EMPTY_PASSWORD); + ca.setKeyEntry("ca", caKey, password, new Certificate[] { caCertificate }); + writeStore(caFile, ca, password); + } + if (caRenewed || !hasServerName(serverCertificate, advertisedHost) || needsRenewal(serverCertificate, clock)) { ensureBouncyCastle(); KeyPair serverPair = keyPair(); serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, caKey, @@ -113,7 +127,8 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock server.setKeyEntry("server", serverKey, password, new Certificate[] { serverCertificate, caCertificate }); writeStore(serverFile, server, password); } - return new HttpTlsIdentity(caKey, caCertificate, serverKey, serverCertificate, password, serverFile, advertisedHost); + return new HttpTlsIdentity(caKey, caCertificate, serverKey, serverCertificate, password, caFile, serverFile, + advertisedHost); } finally { Arrays.fill(password, '\0'); } } ensureBouncyCastle(); @@ -137,16 +152,15 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock try { writePrivate(passwordFile, passwordBytes); } finally { Arrays.fill(passwordBytes, (byte) 0); } return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password, - serverFile, advertisedHost); + caFile, serverFile, advertisedHost); } finally { Arrays.fill(password, '\0'); } } public String serverCertificatePin() { - try { renewServerCertificateIfNeeded(); } - catch (Exception failure) { throw new IllegalStateException("Could not renew HTTP server certificate", failure); } + refreshIdentity(); return HttpTransportSecrets.certificatePin(serverCertificate); } - public String caCertificatePin() { return HttpTransportSecrets.certificatePin(caCertificate); } + public String caCertificatePin() { refreshIdentity(); return HttpTransportSecrets.certificatePin(caCertificate); } public X509Certificate caCertificate() { return caCertificate; } public X509Certificate serverCertificate() { return serverCertificate; } @@ -156,7 +170,7 @@ public String serverCertificatePin() { * additionally validate the certificate's persisted backend binding in the HTTP handler. */ public SSLContext serverContext() throws Exception { - renewServerCertificateIfNeeded(); + renewIdentityIfNeeded(); SSLContext context = SSLContext.getInstance("TLS"); context.init(new KeyManager[] { new RotatingServerKeyManager() }, trustManagers(caCertificate), null); return context; @@ -171,16 +185,34 @@ static TrustManager[] trustManagers(X509Certificate caCertificate) throws Except return factory.getTrustManagers(); } - private synchronized void renewServerCertificateIfNeeded() throws Exception { - if (!needsRenewal(serverCertificate, Clock.systemUTC())) return; + private void refreshIdentity() { + try { renewIdentityIfNeeded(); } + catch (Exception failure) { throw new IllegalStateException("Could not renew HTTP TLS identity", failure); } + } + + private synchronized void renewIdentityIfNeeded() throws Exception { + Clock clock = Clock.systemUTC(); + boolean renewCa = needsCaRenewal(caCertificate, clock); + if (!renewCa && !needsRenewal(serverCertificate, clock)) return; ensureBouncyCastle(); + X509Certificate replacementCa = caCertificate; + if (renewCa) { + KeyPair caPair = new KeyPair(caCertificate.getPublicKey(), caKey); + replacementCa = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, + CertificateRole.CA, null, clock.instant()); + KeyStore caStore = KeyStore.getInstance("PKCS12"); + caStore.load(null, EMPTY_PASSWORD); + caStore.setKeyEntry("ca", caKey, password, new Certificate[] { replacementCa }); + writeStore(caFile, caStore, password); + } KeyPair pair = keyPair(); - X509Certificate replacement = certificate("CN=" + certificateName(advertisedHost), pair, caCertificate, caKey, - CertificateRole.SERVER, advertisedHost, Instant.now()); + X509Certificate replacement = certificate("CN=" + certificateName(advertisedHost), pair, replacementCa, caKey, + CertificateRole.SERVER, advertisedHost, clock.instant()); KeyStore store = KeyStore.getInstance("PKCS12"); store.load(null, EMPTY_PASSWORD); - store.setKeyEntry("server", pair.getPrivate(), password, new Certificate[] { replacement, caCertificate }); + store.setKeyEntry("server", pair.getPrivate(), password, new Certificate[] { replacement, replacementCa }); writeStore(serverFile, store, password); + caCertificate = replacementCa; serverKey = pair.getPrivate(); serverCertificate = replacement; } @@ -288,6 +320,10 @@ static boolean needsRenewal(X509Certificate certificate, Clock clock) { return certificate == null || !certificate.getNotAfter().toInstant().isAfter(clock.instant().plus(RENEW_BEFORE)); } + static boolean needsCaRenewal(X509Certificate certificate, Clock clock) { + return certificate == null || !certificate.getNotAfter().toInstant().isAfter(clock.instant().plus(CA_RENEW_BEFORE)); + } + private static void ensureBouncyCastle() { if (Security.getProvider("BC") == null) Security.addProvider(new BouncyCastleProvider()); } @@ -363,8 +399,7 @@ private static void setOwnerOnly(Path path) throws IOException { private final class RotatingServerKeyManager extends X509ExtendedKeyManager { private static final String ALIAS = "server"; private void refresh() { - try { renewServerCertificateIfNeeded(); } - catch (Exception failure) { throw new IllegalStateException("Could not renew HTTP server certificate", failure); } + refreshIdentity(); } private String alias(String keyType) { refresh(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index ac95b1f8b..7363f5334 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -327,6 +327,37 @@ void backendReAcknowledgesLostAckDuplicateWithoutSecondCallback() throws Excepti } } + @Test + void backendCallbacksAreSerializedInDeliveryOrder() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ordered-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("ordered-client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1), secondStarted = new CountDownLatch(1); + java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("ordered-client")), envelope -> { + String marker = String.valueOf(envelope.getFields().get("marker")); + order.add(marker); + if ("first".equals(marker)) { + firstStarted.countDown(); + try { releaseFirst.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } else secondStarted.countDown(); + })) { + connector.dispatch(new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").put("marker", "first").build())); + connector.dispatch(new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").put("marker", "second").build())); + assertTrue(firstStarted.await(2, TimeUnit.SECONDS)); + assertFalse(secondStarted.await(150, TimeUnit.MILLISECONDS)); + releaseFirst.countDown(); + assertTrue(secondStarted.await(2, TimeUnit.SECONDS)); + assertEquals(java.util.List.of("first", "second"), order); + } finally { releaseFirst.countDown(); } + } + @Test void backendDedupWindowContinuesAfterCapacity() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index d85574604..1d5a57b72 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -139,6 +139,40 @@ void serverLeafRotatesInsideRenewalWindowAndPreservesAuthority() throws Exceptio assertFalse(HttpTlsIdentity.needsRenewal(renewed.serverCertificate(), Clock.fixed(now, ZoneOffset.UTC))); } + @Test + void runningPrivateCaRollsOverBeforeExpiryWithoutStrandingExistingClients() throws Exception { + Instant now = Instant.now(); + Clock originalClock = Clock.fixed(now.minus(Duration.ofDays(9 * 365L + 30L)), ZoneOffset.UTC); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(directory, "localhost", originalClock); + X509Certificate originalCa = original.caCertificate(); + HttpTlsIdentity.IssuedClientCertificate existingClient = original.issueClientCertificate("lobby-1", now); + Path client = directory.resolve("client"); + HttpConnectionCode oldCode = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + HttpTransportSecrets.certificatePin(original.serverCertificate()), HttpTransportSecrets.certificatePin(originalCa), + now.plusSeconds(60), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, oldCode, existingClient); + + String renewedPin = original.caCertificatePin(); + assertNotEquals(HttpTransportSecrets.certificatePin(originalCa), renewedPin); + assertEquals(originalCa.getPublicKey(), original.caCertificate().getPublicKey(), + "certificate rollover keeps the private authority key so old and new trust anchors overlap"); + assertFalse(HttpTlsIdentity.needsCaRenewal(original.caCertificate(), Clock.fixed(now, ZoneOffset.UTC))); + assertTrue(original.validClientCertificate("lobby-1", existingClient.certificate())); + + X509TrustManager oldClientTrust = Arrays.stream(HttpTlsIdentity.trustManagers(originalCa)) + .filter(X509TrustManager.class::isInstance).map(X509TrustManager.class::cast).findFirst().orElseThrow(); + assertDoesNotThrow(() -> oldClientTrust.checkServerTrusted( + new X509Certificate[] { original.serverCertificate(), original.caCertificate() }, "ECDHE_ECDSA")); + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(client, + original.issueClientCertificate("lobby-1", now)); + assertEquals(HttpTransportSecrets.certificatePin(originalCa), HttpClientCredentialStore.loadProfile(client).caCertificatePin()); + assertEquals(renewedPin, staged.profile().caCertificatePin()); + HttpClientCredentialStore.activateReplacement(client, staged); + assertEquals(renewedPin, HttpClientCredentialStore.loadProfile(client).caCertificatePin()); + assertEquals(renewedPin, HttpTlsIdentity.loadOrCreate(directory, "localhost").caCertificatePin(), + "live CA rollover must survive restart"); + } + @Test void activeTlsContextRotatesServerLeafInsideRenewalWindow() throws Exception { Instant now = Instant.now(); From dbf416f19bfd017d81cd82be11289706181e602b Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 19:25:00 -0600 Subject: [PATCH 11/36] fix(http): verify backend presence against proxy route --- .../votingplugin/proxy/VotingPluginProxy.java | 31 ++++++-- .../http/HttpTransportSecurityTest.java | 1 + .../tests/VotingPluginProxyTest.java | 71 +++++++++++++++++++ .../tests/VotingPluginProxyTestImpl.java | 5 ++ 4 files changed, 103 insertions(+), 5 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index b7dc11054..428f3e151 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -2710,11 +2710,7 @@ private void startHttpTransport() { httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); httpTransportServer = new HttpProxyTransportServer( new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, - httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), received -> { - GlobalMessageProxyHandler handler = globalMessageProxyHandler; - if (handler == null) throw new IllegalStateException("HTTP message router is not ready"); - handler.onMessage(received.envelope()); - }); + httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope); httpTransportServer.start(); logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" + httpTransportServer.port() + "; use /votingpluginbungee httpcode for each backend"); @@ -2724,6 +2720,31 @@ private void startHttpTransport() { } } + /** Keeps the authenticated mTLS backend identity attached to security-sensitive proxy routing. */ + protected void handleHttpTransportEnvelope(HttpProxyTransportServer.ReceivedEnvelope received) { + if (!isAuthenticatedHttpEnvelopeAllowed(received)) { + debug("Ignored HTTP envelope whose player-presence claim did not match its authenticated backend"); + return; + } + GlobalMessageProxyHandler handler = globalMessageProxyHandler; + if (handler == null) throw new IllegalStateException("HTTP message router is not ready"); + handler.onMessage(received.envelope()); + } + + private boolean isAuthenticatedHttpEnvelopeAllowed(HttpProxyTransportServer.ReceivedEnvelope received) { + if (received == null || received.envelope() == null || received.serverId() == null) return false; + String stampedServer = received.envelope().getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + if (!received.serverId().equalsIgnoreCase(stampedServer)) return false; + if (!VotingPluginWire.SUB_LOGIN.equals(received.envelope().getSubChannel())) return true; + VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(received.envelope()); + boolean modern = event.connectionId != null || event.backendIncarnationId != null + || event.backendStartedAt != 0L || event.presenceTimestamp != 0L; + if (!modern || isDedicatedVotingProxyEnabled()) return true; + // A player-facing proxy has a stronger authority than any backend: its live + // player connection supplies both the current route and (in online mode) UUID. + return isLegacyLoginDestinationAuthoritative(event.player, event.uuid, received.serverId()); + } + private synchronized void closeHttpTransport() { HttpProxyTransportServer transport = httpTransportServer; httpTransportServer = null; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 1d5a57b72..bcb614215 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -10,6 +10,7 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; +import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; import java.time.Instant; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 4f8b711fb..6b3fb402d 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -16,6 +16,7 @@ import org.mockito.MockitoAnnotations; import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalDataHandlerProxy; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; import com.bencodez.votingplugin.proxy.BungeeMethod; import com.bencodez.votingplugin.proxy.OfflineBungeeVote; @@ -350,6 +351,76 @@ void pluginMessagingIgnoresExtendedPresenceLogin() { assertEquals(0, spyProxy.getBackendPlayerPresenceTracker().getOnlinePlayerCount()); } + @Test + void httpModernPresenceRejectsAuthenticatedBackendThatDoesNotMatchProxyRoute() { + String uuid = java.util.UUID.randomUUID().toString(); + Mockito.when(votingPluginProxy.getConfig().getOnlineMode()).thenReturn(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(uuid).when(spyProxy).getUUID("Player"); + var handler = Mockito.mock(com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler.class); + spyProxy.setGlobalMessageProxyHandlerForTest(handler); + JsonEnvelope envelope = VotingPluginWire.login("Player", uuid, "Server2", java.util.UUID.randomUUID(), + java.util.UUID.randomUUID(), 1000L, 1100L); + + spyProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer.ReceivedEnvelope( + "Server2", java.util.UUID.randomUUID().toString(), envelope)); + + verify(handler, never()).onMessage(Mockito.any()); + assertEquals(0, spyProxy.getBackendPlayerPresenceTracker().getOnlinePlayerCount()); + } + + @Test + void httpModernPresenceAcceptsAuthenticatedBackendMatchingProxyRouteAndUuid() { + String uuid = java.util.UUID.randomUUID().toString(); + Mockito.when(votingPluginProxy.getConfig().getOnlineMode()).thenReturn(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(uuid).when(spyProxy).getUUID("Player"); + var handler = Mockito.mock(com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler.class); + spyProxy.setGlobalMessageProxyHandlerForTest(handler); + JsonEnvelope envelope = VotingPluginWire.login("Player", uuid, "Server1", java.util.UUID.randomUUID(), + java.util.UUID.randomUUID(), 1000L, 1100L); + + spyProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer.ReceivedEnvelope( + "Server1", java.util.UUID.randomUUID().toString(), envelope)); + + verify(handler).onMessage(envelope); + } + + @Test + void httpModernPresenceRejectsUuidThatDoesNotMatchProxyPlayer() { + String authoritativeUuid = java.util.UUID.randomUUID().toString(); + String claimedUuid = java.util.UUID.randomUUID().toString(); + Mockito.when(votingPluginProxy.getConfig().getOnlineMode()).thenReturn(true); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + VotingPluginProxyTestImpl spyProxy = Mockito.spy(votingPluginProxy); + Mockito.doReturn(authoritativeUuid).when(spyProxy).getUUID("Player"); + var handler = Mockito.mock(com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler.class); + spyProxy.setGlobalMessageProxyHandlerForTest(handler); + JsonEnvelope envelope = VotingPluginWire.login("Player", claimedUuid, "Server1", java.util.UUID.randomUUID(), + java.util.UUID.randomUUID(), 1000L, 1100L); + + spyProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer.ReceivedEnvelope( + "Server1", java.util.UUID.randomUUID().toString(), envelope)); + + verify(handler, never()).onMessage(Mockito.any()); + assertEquals(0, spyProxy.getBackendPlayerPresenceTracker().getOnlinePlayerCount()); + } + + @Test + void httpEnvelopeRejectsServerFieldThatDoesNotMatchAuthenticatedBackend() { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + var handler = Mockito.mock(com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler.class); + votingPluginProxy.setGlobalMessageProxyHandlerForTest(handler); + JsonEnvelope envelope = JsonEnvelope.builder("vote").put(VotingPluginWire.K_SERVER, "Server2").build(); + + votingPluginProxy.handleHttpTransportEnvelopeForTest(new com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer.ReceivedEnvelope( + "Server1", java.util.UUID.randomUUID().toString(), envelope)); + + verify(handler, never()).onMessage(Mockito.any()); + } + @Test void dedicatedVotingProxyRoutesUsingConfirmedBackendPresence() { Mockito.when(votingPluginProxy.getConfig().getDedicatedVotingProxy()).thenReturn(true); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index dd22c32ed..a2d18b1c1 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -17,6 +17,7 @@ import com.bencodez.votingplugin.proxy.OfflineBungeeVote; import com.bencodez.votingplugin.proxy.VotingPluginProxy; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; +import com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer; import com.bencodez.votingplugin.timequeue.VoteTimeQueue; public class VotingPluginProxyTestImpl extends VotingPluginProxy { @@ -243,6 +244,10 @@ public void handleLoginMessageForTest(JsonEnvelope envelope) { handleLoginMessage(envelope); } + public void handleHttpTransportEnvelopeForTest(HttpProxyTransportServer.ReceivedEnvelope received) { + handleHttpTransportEnvelope(received); + } + public void handleStatusOkayForTest(JsonEnvelope envelope) { handleStatusOkay(envelope); } From ac6a947ae5d684d3bf46d49a074fca814e2befba Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 19:27:49 -0600 Subject: [PATCH 12/36] fix(http): preserve proxy callback order --- .../http/HttpProxyTransportServer.java | 4 ++- .../http/HttpTransportRuntimeTest.java | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index c77b8a3f0..bdbbaaccd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -95,7 +95,9 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity // Long polls are blocking by design. Capacity is bounded by admission, while enough workers // remain available for all admitted polls plus setup requests. listenerExecutor = executor("VotingPlugin-HTTP-listener", 72, 72); - handlerExecutor = executor("VotingPlugin-HTTP-handler", 4, 128); + // The proxy router mutates shared presence, vote, and reward state. A separate + // bounded FIFO lane keeps wire order without blocking long-poll workers. + handlerExecutor = executor("VotingPlugin-HTTP-handler", 1, 128); server.setExecutor(listenerExecutor); server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); server.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 7363f5334..399f9be88 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -358,6 +358,40 @@ void backendCallbacksAreSerializedInDeliveryOrder() throws Exception { } finally { releaseFirst.countDown(); } } + @Test + void proxyCallbacksAreSerializedInDeliveryOrder() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ordered-proxy-server"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("ordered-authority")); + CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1), secondStarted = new CountDownLatch(1); + java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, received -> { + String marker = String.valueOf(received.envelope().getFields().get("marker")); + order.add(marker); + if ("first".equals(marker)) { + firstStarted.countDown(); + try { releaseFirst.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } else secondStarted.countDown(); + })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, "lobby-1", + directory.resolve("ordered-proxy-client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", credential, + ignored -> { })) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("x").put("marker", "first").build())); + assertTrue(connector.send(JsonEnvelope.builder("x").put("marker", "second").build())); + assertTrue(firstStarted.await(3, TimeUnit.SECONDS)); + assertFalse(secondStarted.await(150, TimeUnit.MILLISECONDS)); + releaseFirst.countDown(); + assertTrue(secondStarted.await(3, TimeUnit.SECONDS)); + assertEquals(java.util.List.of("first", "second"), order); + } + } finally { releaseFirst.countDown(); } + } + @Test void backendDedupWindowContinuesAfterCapacity() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); From f104e61d83a94febf73a4cb3059804d34ed05b8f Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:04:12 -0600 Subject: [PATCH 13/36] fix(http): preserve callback order and crash dedup --- .../http/HttpBackendTransportConnector.java | 84 +++++++++++-- .../http/HttpInboundDeliveryStore.java | 119 ++++++++++++++++++ .../http/HttpProxyTransportServer.java | 18 +-- .../http/HttpTransportRuntimeTest.java | 112 ++++++++++++++++- .../http/HttpTransportSecurityTest.java | 19 +++ docs/http-transport.md | 6 +- 6 files changed, 335 insertions(+), 23 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 8733c44a5..6747e1990 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -1,6 +1,7 @@ package com.bencodez.votingplugin.backendproxy.http; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -32,12 +33,14 @@ /** Backend-side, persistent HTTP/1.1 long-poll connector. */ public final class HttpBackendTransportConnector implements AutoCloseable { public static final Duration CLIENT_TIMEOUT = Duration.ofSeconds(35); + static final int CALLBACK_QUEUE_CAPACITY = 128; private volatile HttpClientCredentialStore.HttpClientProfile profile; private final String serverId; private final Consumer onEnvelope; private volatile HttpClient client; private volatile HttpClientCredentialStore.ClientCredential credential; private final Path credentialDirectory; + private final HttpInboundDeliveryStore inboundDeliveries; private final URI transportEndpoint; private final ThreadPoolExecutor callbackExecutor; private final AtomicBoolean running = new AtomicBoolean(); @@ -51,17 +54,19 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private long sequence; private volatile long nextRenewalCheckNanos; - public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, + /** In-memory test constructor; production transport must use a directory-backed constructor. */ + HttpBackendTransportConnector(HttpConnectionCode code, String serverId, HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { this(profile(code, serverId), credential, onEnvelope); } - /** Starts normal transport from the non-secret profile persisted by enrollment. */ - public HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope) throws Exception { + /** In-memory test constructor; production transport must use a directory-backed constructor. */ + HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope) throws Exception { this(enrolled, onEnvelope, null); } - public HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, + /** In-memory test constructor; production transport must use a directory-backed constructor. */ + HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { this(profile, credential, onEnvelope, null); } @@ -78,12 +83,17 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil this.profile = profile; this.serverId = profile.serverId(); this.onEnvelope = onEnvelope; this.credential = credential; this.credentialDirectory = credentialDirectory; + inboundDeliveries = credentialDirectory == null ? null : new HttpInboundDeliveryStore(credentialDirectory); + if (inboundDeliveries != null) for (String id : inboundDeliveries.snapshot()) { + received.add(id); + queueAck(id); + } client = client(profile, credential); transportEndpoint = profile.endpoint().resolve("v1/transport"); // GlobalMessageHandler routes mutate backend vote state and must observe the // wire order. One bounded lane preserves batch ordering without running work on - // the long-poll thread. - callbackExecutor = executor("VotingPlugin-HTTP-callback", 1, 128); + // the long-poll thread; bounded admission below backpressures this poller. + callbackExecutor = executor("VotingPlugin-HTTP-callback", 1, CALLBACK_QUEUE_CAPACITY); } /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ @@ -135,9 +145,10 @@ public boolean send(JsonEnvelope envelope) { /** A synchronous single poll, useful for lifecycle-controlled integrations and tests. */ public synchronized boolean pollOnce() { if (!running.get()) return false; + List acks = List.of(); boolean acknowledgementsConfirmed = false; try { maybeRenewCredential(); - List acks; List messages; long requestSequence; + List messages; long requestSequence; synchronized (state) { acks = first(acknowledgements); requestSequence = sequence++; messages = HttpTransportProtocol.fittingMessages(serverId, session, requestSequence, acks, outgoing.values()); @@ -149,10 +160,13 @@ public synchronized boolean pollOnce() { if (response.statusCode() != 200 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) return false; HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(response.body()); if (!serverId.equals(packet.server()) || !session.equals(packet.session()) || packet.sequence() != requestSequence) return false; + confirmAcknowledgements(acks); + acknowledgementsConfirmed = true; synchronized (state) { for (String ack : packet.acks()) outgoing.remove(ack); } for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); return true; - } catch (Exception failure) { return false; } + } catch (Exception failure) { return false; + } finally { if (!acknowledgementsConfirmed) requeueAcknowledgements(acks); } } @Override public void close() { running.getAndSet(false); @@ -169,7 +183,9 @@ List accept(List synchronized (state) { List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : deliveries) { - if (received.contains(delivery.id())) { queueAck(delivery.id()); continue; } + if (received.contains(delivery.id()) || inboundDeliveries != null && inboundDeliveries.contains(delivery.id())) { + received.add(delivery.id()); queueAck(delivery.id()); continue; + } if (!processing.contains(delivery.id())) { processing.add(delivery.id()); accepted.add(delivery); } @@ -178,20 +194,66 @@ List accept(List } } void dispatch(HttpTransportProtocol.Delivery delivery) { - try { callbackExecutor.execute(() -> { boolean success = false; try { onEnvelope.accept(delivery.envelope()); success = true; } catch (RuntimeException ignored) { } + Runnable callback = () -> { + boolean success = false, reserved = false; + try { + if (inboundDeliveries != null) { inboundDeliveries.reserve(delivery.id()); reserved = true; } + onEnvelope.accept(delivery.envelope()); + success = true; + } catch (IOException persistenceFailure) { + // Never run a side-effecting callback without first publishing its replay fence. + } catch (RuntimeException callbackFailure) { + if (reserved) try { inboundDeliveries.remove(delivery.id()); } + catch (IOException removalFailure) { + // A callback can fail after partial effects. If the fence cannot be removed, + // fail closed as processed rather than risk awarding again on a retry. + success = true; + } + } completeIncoming(delivery.id(), success); - }); } catch (RejectedExecutionException rejected) { completeIncoming(delivery.id(), false); } + }; + if (!executeOrdered(callbackExecutor, callback)) completeIncoming(delivery.id(), false); } void completeIncoming(String id, boolean success) { synchronized (state) { processing.remove(id); if (success) { received.add(id); while (received.size() > HttpTransportProtocol.MAX_QUEUE) received.remove(received.iterator().next()); queueAck(id); } } } private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } + private void requeueAcknowledgements(List ids) { synchronized (state) { + for (int index = ids.size() - 1; index >= 0; index--) { + String id = ids.get(index); + if (!acknowledgements.contains(id)) { + while (acknowledgements.size() >= HttpTransportProtocol.MAX_QUEUE) acknowledgements.removeLast(); + acknowledgements.addFirst(id); + } + } + } } + private void confirmAcknowledgements(Collection ids) { + for (String id : ids) { + if (inboundDeliveries != null) try { inboundDeliveries.remove(id); } + catch (IOException cleanupFailure) { continue; } + synchronized (state) { received.remove(id); } + } + } int queuedOutgoing() { synchronized (state) { return outgoing.size(); } } List drainAcknowledgements() { synchronized (state) { return drain(acknowledgements); } } private static List first(Collection values) { List output = new java.util.ArrayList<>(); for (T value : values) { output.add(value); if (output.size() == HttpTransportProtocol.MAX_BATCH) break; } return output; } private static List drain(ArrayDeque values) { List output = new java.util.ArrayList<>(); while (!values.isEmpty() && output.size() < HttpTransportProtocol.MAX_BATCH) output.add(values.remove()); return output; } private static ThreadPoolExecutor executor(String name, int threads, int queue) { ThreadFactory factory = task -> { Thread thread = new Thread(task, name); thread.setDaemon(true); return thread; }; return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); } + static boolean executeOrdered(ThreadPoolExecutor executor, Runnable task) { + try { executor.execute(task); return true; } + catch (RejectedExecutionException fullOrClosed) { + if (executor.isShutdown()) return false; + try { + while (!executor.isShutdown()) { + if (!executor.getQueue().offer(task, 100L, TimeUnit.MILLISECONDS)) continue; + if (executor.isShutdown() && executor.remove(task)) return false; + return true; + } + } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + return false; + } + } private static HttpClientCredentialStore.HttpClientProfile profile(HttpConnectionCode code, String serverId) { if (code == null || serverId == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); if (!code.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalArgumentException("HTTP connection code belongs to a different backend"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java new file mode 100644 index 000000000..9a03c87c6 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java @@ -0,0 +1,119 @@ +package com.bencodez.votingplugin.backendproxy.http; + +import com.bencodez.votingplugin.util.DurableFiles; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.util.EnumSet; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.UUID; + +/** + * Crash-durable fence for proxy deliveries that may already have caused backend side effects. + * A delivery is reserved before its callback runs and removed only after the proxy confirms its ACK. + */ +final class HttpInboundDeliveryStore { + private static final String DIRECTORY = "http-transport-inbound-deliveries"; + private static final String SUFFIX = ".seen"; + private static final int MAX_ENTRIES = HttpTransportProtocol.MAX_QUEUE; + private final Path root; + private final Set entries = new LinkedHashSet<>(); + + HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { + Path credentials = credentialDirectory.toAbsolutePath().normalize(); + if (Files.isSymbolicLink(credentials) || !Files.isDirectory(credentials, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential directory is unsafe"); + ownerOnlyDirectory(credentials); + root = credentials.resolve(DIRECTORY).normalize(); + if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); + Files.createDirectories(root); + if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery directory is unsafe"); + ownerOnlyDirectory(root); + load(); + } + + synchronized boolean contains(String id) { return entries.contains(canonical(id)); } + + synchronized void reserve(String id) throws IOException { + id = canonical(id); + if (entries.contains(id)) return; + if (entries.size() >= MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence is full"); + requireRoot(); + Path target = root.resolve(id + SUFFIX); + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery fence is inconsistent"); + Path temporary = Files.createTempFile(root, ".pending-", ".tmp"); + try { + ownerOnlyFile(temporary); + Files.writeString(temporary, id, StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } + ownerOnlyFile(target); + DurableFiles.forceDirectory(root); + entries.add(id); + } finally { Files.deleteIfExists(temporary); } + } + + synchronized boolean remove(String id) throws IOException { + id = canonical(id); + if (!entries.contains(id)) return true; + requireRoot(); + DurableFiles.deleteIfExists(root.resolve(id + SUFFIX)); + entries.remove(id); + return true; + } + + synchronized Set snapshot() { return Set.copyOf(entries); } + + private void load() throws IOException { + try (DirectoryStream files = Files.newDirectoryStream(root)) { + for (Path file : files) { + String name = file.getFileName().toString(); + if (name.startsWith(".pending-") && name.endsWith(".tmp") && !Files.isSymbolicLink(file) + && Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) { + DurableFiles.deleteIfExists(file); + continue; + } + if (Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) + || !name.endsWith(SUFFIX) || Files.size(file) > 64L) + throw new IOException("HTTP inbound delivery fence contains an invalid entry"); + String id = canonical(name.substring(0, name.length() - SUFFIX.length())); + if (!name.equals(id + SUFFIX) || !Files.readString(file, StandardCharsets.US_ASCII).equals(id) + || !entries.add(id)) + throw new IOException("HTTP inbound delivery fence entry is invalid"); + if (entries.size() > MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence exceeds its bound"); + } + } + } + + private static String canonical(String id) { + if (id == null) throw new IllegalArgumentException("HTTP delivery id is invalid"); + String canonical = UUID.fromString(id).toString(); + if (!canonical.equals(id)) throw new IllegalArgumentException("HTTP delivery id is not canonical"); + return canonical; + } + + private void requireRoot() throws IOException { + if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery directory is unsafe"); + } + + private static void ownerOnlyFile(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { } + } + private static void ownerOnlyDirectory(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } + catch (UnsupportedOperationException ignored) { } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index bdbbaaccd..2d3825a44 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -32,7 +32,6 @@ import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; @@ -97,7 +96,7 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity listenerExecutor = executor("VotingPlugin-HTTP-listener", 72, 72); // The proxy router mutates shared presence, vote, and reward state. A separate // bounded FIFO lane keeps wire order without blocking long-poll workers. - handlerExecutor = executor("VotingPlugin-HTTP-handler", 1, 128); + handlerExecutor = executor("VotingPlugin-HTTP-handler", 1, HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY); server.setExecutor(listenerExecutor); server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); server.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); @@ -178,7 +177,7 @@ private void transport(HttpsExchange exchange) throws IOException { } finally { admission.release(); } } - private void handlePacket(HttpTransportProtocol.Packet packet, BackendState backend) { + private void handlePacket(HttpTransportProtocol.Packet packet, BackendState backend) throws IOException { List accepted; synchronized (backend) { if (!backend.allowRequest()) throw new IllegalArgumentException("transport rate limited"); @@ -188,12 +187,14 @@ private void handlePacket(HttpTransportProtocol.Packet packet, BackendState back for (HttpTransportProtocol.Delivery delivery : accepted) dispatch(packet.server(), backend, delivery); } private void dispatch(String serverId, BackendState backend, HttpTransportProtocol.Delivery delivery) { - try { handlerExecutor.execute(() -> { + Runnable callback = () -> { boolean success = false; try { onEnvelope.accept(new ReceivedEnvelope(serverId, delivery.id(), normalizeBackendIdentity(serverId, delivery.envelope()))); success = true; } catch (RuntimeException ignored) { } synchronized (backend) { backend.completeIncoming(delivery.id(), success); } - }); } catch (RejectedExecutionException rejected) { synchronized (backend) { backend.completeIncoming(delivery.id(), false); } } + }; + if (!HttpBackendTransportConnector.executeOrdered(handlerExecutor, callback)) + synchronized (backend) { backend.completeIncoming(delivery.id(), false); } } private static JsonEnvelope normalizeBackendIdentity(String serverId, JsonEnvelope envelope) { // The authenticated TLS identity is authoritative; never forward a forged `server` field. @@ -269,11 +270,12 @@ synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { catch (IOException failure) { return false; } outgoing.put(delivery.id(), delivery); signal(); return true; } - private void acknowledge(Collection acks) { + private void acknowledge(Collection acks) throws IOException { for (String id : acks) { if (!outgoing.containsKey(id)) continue; - if (durableOutgoing != null) try { durableOutgoing.remove(serverId, id); } - catch (IOException failure) { continue; } + // A 200 response is the backend's proof that its durable replay fence may + // be deleted. Never return success while the proxy delivery still exists. + if (durableOutgoing != null) durableOutgoing.remove(serverId, id); outgoing.remove(id); delivered.remove(id); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 399f9be88..1a1250b61 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -33,8 +33,8 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti message -> { received.set(message); proxyReceived.countDown(); })) { server.start(); HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); - HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, "lobby-1", directory.resolve("client")); - try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", credential, + HttpBackendTransportConnector.enroll(code, "lobby-1", directory.resolve("client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(directory.resolve("client"), envelope -> backendReceived.countDown())) { connector.start(); assertTrue(connector.send(JsonEnvelope.builder("to-proxy").put("server", "forged").build())); @@ -46,6 +46,10 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti assertEquals(0, connector.queuedOutgoing(), "proxy ACK must remove the exact outbound delivery ID"); assertTrue(server.send("lobby-1", JsonEnvelope.builder("to-backend").build())); assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); + Path inboundFence = directory.resolve("client").resolve("http-transport-inbound-deliveries"); + long fenceDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (countRegularFiles(inboundFence) != 0L && System.nanoTime() < fenceDeadline) Thread.sleep(20); + assertEquals(0L, countRegularFiles(inboundFence), "a confirmed ACK must remove the backend replay fence"); } } } @@ -358,6 +362,110 @@ void backendCallbacksAreSerializedInDeliveryOrder() throws Exception { } finally { releaseFirst.countDown(); } } + @Test + void backendCallbackQueueBackpressuresWithoutBreakingFifo() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("backpressure-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("backpressure-client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1); + int deliveries = HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY + 2; + CountDownLatch completed = new CountDownLatch(deliveries), overflowSubmitted = new CountDownLatch(1); + java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("backpressure-client")), envelope -> { + int marker = Integer.parseInt(envelope.getFields().get("marker")); + order.add(marker); + if (marker == 0) { + firstStarted.countDown(); + try { releaseFirst.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } + completed.countDown(); + })) { + connector.dispatch(delivery(0)); + assertTrue(firstStarted.await(2, TimeUnit.SECONDS)); + for (int marker = 1; marker <= HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY; marker++) + connector.dispatch(delivery(marker)); + Thread overflow = new Thread(() -> { + connector.dispatch(delivery(deliveries - 1)); + overflowSubmitted.countDown(); + }, "HTTP-overflow-submitter"); + overflow.start(); + assertFalse(overflowSubmitted.await(150, TimeUnit.MILLISECONDS), "a full ordered lane must backpressure its producer"); + releaseFirst.countDown(); + assertTrue(overflowSubmitted.await(2, TimeUnit.SECONDS)); + assertTrue(completed.await(5, TimeUnit.SECONDS)); + assertEquals(java.util.stream.IntStream.range(0, deliveries).boxed().toList(), order); + } finally { releaseFirst.countDown(); } + } + + @Test + void durableBackendFencePreventsCallbackReplayAfterRestartBeforeAck() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("fence-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "A".repeat(43)); + Path clientDirectory = directory.resolve("fence-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + java.util.concurrent.atomic.AtomicInteger callbacks = new java.util.concurrent.atomic.AtomicInteger(); + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("vote").build()); + try (HttpBackendTransportConnector first = new HttpBackendTransportConnector(clientDirectory, + ignored -> callbacks.incrementAndGet())) { + first.dispatch(delivery); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (callbacks.get() != 1 && System.nanoTime() < deadline) Thread.sleep(5); + assertEquals(1, callbacks.get()); + } + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, + ignored -> callbacks.incrementAndGet())) { + assertEquals(java.util.List.of(id), restarted.drainAcknowledgements(), + "restart must retain and acknowledge the pre-callback delivery fence"); + assertTrue(restarted.accept(java.util.List.of(delivery)).isEmpty()); + assertEquals(1, callbacks.get(), "a durable proxy replay must not award twice"); + } + } + + @Test + void failedBackendCallbackRemovesFenceAndRemainsRetryable() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-fence-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "B".repeat(43)); + Path clientDirectory = directory.resolve("retry-fence-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + java.util.concurrent.atomic.AtomicInteger attempts = new java.util.concurrent.atomic.AtomicInteger(); + CountDownLatch failed = new CountDownLatch(1), succeeded = new CountDownLatch(1); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("vote").build()); + try (HttpBackendTransportConnector first = new HttpBackendTransportConnector(clientDirectory, ignored -> { + attempts.incrementAndGet(); failed.countDown(); throw new IllegalStateException("retry"); + })) { + first.dispatch(delivery); + assertTrue(failed.await(2, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (countRegularFiles(clientDirectory.resolve("http-transport-inbound-deliveries")) != 0L + && System.nanoTime() < deadline) Thread.sleep(5); + assertTrue(first.drainAcknowledgements().isEmpty()); + } + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, ignored -> { + attempts.incrementAndGet(); succeeded.countDown(); + })) { + java.util.List accepted = restarted.accept(java.util.List.of(delivery)); + assertEquals(1, accepted.size()); + restarted.dispatch(accepted.get(0)); + assertTrue(succeeded.await(2, TimeUnit.SECONDS)); + assertEquals(2, attempts.get()); + } + } + + private static HttpTransportProtocol.Delivery delivery(int marker) { + return new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").put("marker", marker).build()); + } + @Test void proxyCallbacksAreSerializedInDeliveryOrder() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ordered-proxy-server"), "localhost"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index bcb614215..7f02d54ad 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -52,6 +52,25 @@ void expiredCodesAreNotActive() { assertThrows(IllegalArgumentException.class, () -> code.requireActive(Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC))); } + @Test + void inboundDeliveryFenceRejectsCorruptionAndPathReplacement() throws Exception { + Path corruptCredentials = directory.resolve("corrupt-client"); + Path corruptFence = corruptCredentials.resolve("http-transport-inbound-deliveries"); + Files.createDirectories(corruptFence); + Files.writeString(corruptFence.resolve("not-a-delivery.seen"), "not-a-delivery"); + assertThrows(java.io.IOException.class, () -> new HttpInboundDeliveryStore(corruptCredentials)); + + Path replacedCredentials = directory.resolve("replaced-client"); + Files.createDirectories(replacedCredentials); + HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(replacedCredentials); + Path fence = replacedCredentials.resolve("http-transport-inbound-deliveries"); + Path outside = directory.resolve("outside-fence"); + Files.createDirectory(outside); + Files.delete(fence); + Files.createSymbolicLink(fence, outside); + assertThrows(java.io.IOException.class, () -> store.reserve(java.util.UUID.randomUUID().toString())); + } + @Test void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { HttpTlsIdentity created = HttpTlsIdentity.loadOrCreate(directory, "localhost"); diff --git a/docs/http-transport.md b/docs/http-transport.md index 80bd99391..78587e159 100644 --- a/docs/http-transport.md +++ b/docs/http-transport.md @@ -40,7 +40,9 @@ Connection codes expire after 15 minutes and can be used only once. Treat a fres - Enrollment tokens are 256-bit random capabilities, single-use, short-lived, and retained by the proxy only as hashes. - Credentials, keys, pins, and revocation state are written atomically with owner-only permissions where the operating system supports them. - Request bodies, queues, batches, worker pools, concurrent requests, per-backend polling, and request rates are bounded. Exact paths, methods, and JSON content types are enforced. -- Delivery IDs, acknowledgements, and duplicate suppression provide in-process at-least-once delivery. Vote caching remains responsible for application-level restart durability. +- Proxy-to-backend messages stay in an owner-only durable proxy queue until the backend acknowledgement is durably applied. Before a backend callback can change vote or reward state, the backend fsyncs that delivery ID in its credential directory; a restart therefore acknowledges a replay without awarding it twice. The fence is deleted only after a matching authenticated HTTP 200 confirms the proxy removed the queued message. +- This crash fence deliberately favors preventing duplicate rewards: a process or host failure in the very small interval after the fence is persisted but before the callback begins can suppress that callback after restart. Fully transactional exactly-once reward execution is not possible because arbitrary reward commands and external plugin effects cannot share a transaction with the transport journal. +- Backend-to-proxy messages retain the existing bounded in-process retry semantics. Vote caching remains responsible for application-level durability in that direction. The listener must terminate TLS itself because client-certificate authentication is part of the protocol. Do not put an HTTP TLS-terminating reverse proxy or CDN in front of it. A TCP/L4 proxy that passes TLS through unchanged is suitable. Internet-facing installations should also use the host firewall or provider firewall for volumetric denial-of-service protection; an application cannot fully absorb a link or TCP flood. @@ -48,4 +50,4 @@ If a backend host or its private credential is compromised, run `/votingpluginbu ## Performance -The connector reuses HTTP/1.1 TLS connections, batches messages and acknowledgements, and performs all network and certificate work off the game thread. A two-second bounded long poll avoids busy polling while limiting the worst-case delay for a backend message queued just after an idle request began. All queues are bounded to prevent traffic bursts from causing unbounded memory growth. +The connector reuses HTTP/1.1 TLS connections, batches messages and acknowledgements, and performs all network, certificate, and delivery-journal work off the game thread. A two-second bounded long poll avoids busy polling while limiting the worst-case delay for a backend message queued just after an idle request began. Callback lanes are single-worker FIFO queues; when their fixed capacity is reached, bounded listener/poller workers apply backpressure instead of dropping an older callback and admitting newer work out of order. All queues and admission pools remain bounded to prevent traffic bursts from causing unbounded memory growth. From 13fd2b7b09194c9cc7240215b1d6407c3b5640c5 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:07:03 -0600 Subject: [PATCH 14/36] fix(http): normalize corrupt fence errors --- .../backendproxy/http/HttpInboundDeliveryStore.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java index 9a03c87c6..9fce3ebd0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java @@ -86,7 +86,11 @@ private void load() throws IOException { if (Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) || !name.endsWith(SUFFIX) || Files.size(file) > 64L) throw new IOException("HTTP inbound delivery fence contains an invalid entry"); - String id = canonical(name.substring(0, name.length() - SUFFIX.length())); + String id; + try { id = canonical(name.substring(0, name.length() - SUFFIX.length())); } + catch (IllegalArgumentException invalid) { + throw new IOException("HTTP inbound delivery fence entry is invalid", invalid); + } if (!name.equals(id + SUFFIX) || !Files.readString(file, StandardCharsets.US_ASCII).equals(id) || !entries.add(id)) throw new IOException("HTTP inbound delivery fence entry is invalid"); From 71f5ca4d6daa2c1fbfc287a08ae2a70016ece6cc Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:17:28 -0600 Subject: [PATCH 15/36] fix(http): revoke pending enrollment codes --- .../http/HttpEnrollmentAuthority.java | 2 ++ .../http/HttpTransportSecurityTest.java | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java index 9ac9b0c51..719028472 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpEnrollmentAuthority.java @@ -118,6 +118,8 @@ public synchronized HttpTlsIdentity.IssuedClientCertificate renew(String serverI public synchronized void revoke(String serverId) { try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } catch (IllegalArgumentException invalid) { return; } + final String revokedServer = serverId; + enrollments.entrySet().removeIf(entry -> revokedServer.equals(entry.getValue().serverId())); ClientBinding binding = bindings.get(serverId); if (binding != null) { bindings.put(serverId, new ClientBinding(binding.certificatePin(), binding.pendingCertificatePin(), true)); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 7f02d54ad..455ecae7d 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -132,6 +132,28 @@ void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { assertFalse(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); } + @Test + void revocationInvalidatesEveryPendingCodeForTheBackend() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("revoke-pending"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, + Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC)); + URI endpoint = URI.create("https://localhost:8443/"); + HttpConnectionCode beforeEnrollment = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + authority.revoke("LOBBY-1"); + assertThrows(IllegalArgumentException.class, + () -> authority.enroll("lobby-1", beforeEnrollment.enrollmentToken())); + + HttpConnectionCode active = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + authority.enroll("lobby-1", active.enrollmentToken()); + HttpConnectionCode firstPending = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + HttpConnectionCode secondPending = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + authority.revoke("lobby-1"); + assertThrows(IllegalArgumentException.class, + () -> authority.enroll("lobby-1", firstPending.enrollmentToken())); + assertThrows(IllegalArgumentException.class, + () -> authority.enroll("lobby-1", secondPending.enrollmentToken())); + } + @Test void renewalKeepsOldCredentialUntilReplacementAuthenticates() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); From 6216a04cc35d39c7e7473d357e72a4c118a5d99b Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:30:04 -0600 Subject: [PATCH 16/36] fix(http): distinguish incomplete inbound deliveries --- .../http/HttpBackendTransportConnector.java | 34 ++++--- .../http/HttpInboundDeliveryStore.java | 92 +++++++++++++------ .../http/HttpTransportRuntimeTest.java | 60 ++++++++++-- docs/http-transport.md | 4 +- 4 files changed, 137 insertions(+), 53 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 6747e1990..69aa4a6f8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -84,9 +84,11 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil this.credential = credential; this.credentialDirectory = credentialDirectory; inboundDeliveries = credentialDirectory == null ? null : new HttpInboundDeliveryStore(credentialDirectory); - if (inboundDeliveries != null) for (String id : inboundDeliveries.snapshot()) { - received.add(id); - queueAck(id); + if (inboundDeliveries != null) for (var entry : inboundDeliveries.snapshot().entrySet()) { + if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { + received.add(entry.getKey()); + queueAck(entry.getKey()); + } } client = client(profile, credential); transportEndpoint = profile.endpoint().resolve("v1/transport"); @@ -183,9 +185,14 @@ List accept(List synchronized (state) { List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : deliveries) { - if (received.contains(delivery.id()) || inboundDeliveries != null && inboundDeliveries.contains(delivery.id())) { + HttpInboundDeliveryStore.State persisted = inboundDeliveries == null ? null : inboundDeliveries.state(delivery.id()); + if (received.contains(delivery.id()) || persisted == HttpInboundDeliveryStore.State.COMPLETED) { received.add(delivery.id()); queueAck(delivery.id()); continue; } + // A callback that was running when the process stopped may already have + // produced external side effects. Keep the proxy copy without replaying or + // acknowledging it; arbitrary plugin callbacks cannot share this journal. + if (persisted == HttpInboundDeliveryStore.State.RUNNING) continue; if (!processing.contains(delivery.id())) { processing.add(delivery.id()); accepted.add(delivery); } @@ -195,20 +202,21 @@ List accept(List } void dispatch(HttpTransportProtocol.Delivery delivery) { Runnable callback = () -> { - boolean success = false, reserved = false; + boolean success = false; try { - if (inboundDeliveries != null) { inboundDeliveries.reserve(delivery.id()); reserved = true; } + if (inboundDeliveries != null) { + if (inboundDeliveries.state(delivery.id()) == null) inboundDeliveries.reserve(delivery.id()); + inboundDeliveries.markRunning(delivery.id()); + } onEnvelope.accept(delivery.envelope()); + if (inboundDeliveries != null) inboundDeliveries.markCompleted(delivery.id()); success = true; } catch (IOException persistenceFailure) { - // Never run a side-effecting callback without first publishing its replay fence. + // Never run before RUNNING is durable and never acknowledge until + // COMPLETED is durable. An uncertain transition stays fail-closed. } catch (RuntimeException callbackFailure) { - if (reserved) try { inboundDeliveries.remove(delivery.id()); } - catch (IOException removalFailure) { - // A callback can fail after partial effects. If the fence cannot be removed, - // fail closed as processed rather than risk awarding again on a retry. - success = true; - } + // The callback may have failed after partial external effects. Leave RUNNING + // unacknowledged so a restart cannot silently lose or duplicate the delivery. } completeIncoming(delivery.id(), success); }; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java index 9fce3ebd0..960d02ba5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java @@ -11,20 +11,16 @@ import java.nio.file.StandardOpenOption; import java.nio.file.attribute.PosixFilePermission; import java.util.EnumSet; -import java.util.LinkedHashSet; -import java.util.Set; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.UUID; -/** - * Crash-durable fence for proxy deliveries that may already have caused backend side effects. - * A delivery is reserved before its callback runs and removed only after the proxy confirms its ACK. - */ +/** Crash-durable state for proxy deliveries around a non-transactional application callback. */ final class HttpInboundDeliveryStore { private static final String DIRECTORY = "http-transport-inbound-deliveries"; - private static final String SUFFIX = ".seen"; private static final int MAX_ENTRIES = HttpTransportProtocol.MAX_QUEUE; private final Path root; - private final Set entries = new LinkedHashSet<>(); + private final Map entries = new LinkedHashMap<>(); HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { Path credentials = credentialDirectory.toAbsolutePath().normalize(); @@ -34,20 +30,21 @@ final class HttpInboundDeliveryStore { root = credentials.resolve(DIRECTORY).normalize(); if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); Files.createDirectories(root); - if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP inbound delivery directory is unsafe"); + requireRoot(); ownerOnlyDirectory(root); load(); } - synchronized boolean contains(String id) { return entries.contains(canonical(id)); } + synchronized State state(String id) { return entries.get(canonical(id)); } synchronized void reserve(String id) throws IOException { id = canonical(id); - if (entries.contains(id)) return; + State existing = entries.get(id); + if (existing == State.RESERVED) return; + if (existing != null) throw new IOException("HTTP inbound delivery fence is already active"); if (entries.size() >= MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence is full"); requireRoot(); - Path target = root.resolve(id + SUFFIX); + Path target = file(id, State.RESERVED); if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP inbound delivery fence is inconsistent"); Path temporary = Files.createTempFile(root, ".pending-", ".tmp"); @@ -55,24 +52,39 @@ synchronized void reserve(String id) throws IOException { ownerOnlyFile(temporary); Files.writeString(temporary, id, StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); DurableFiles.forceFile(temporary); - try { Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); } - catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } + move(temporary, target); ownerOnlyFile(target); DurableFiles.forceDirectory(root); - entries.add(id); + entries.put(id, State.RESERVED); } finally { Files.deleteIfExists(temporary); } } - synchronized boolean remove(String id) throws IOException { + synchronized void markRunning(String id) throws IOException { transition(id, State.RESERVED, State.RUNNING); } + synchronized void markCompleted(String id) throws IOException { transition(id, State.RUNNING, State.COMPLETED); } + + synchronized void remove(String id) throws IOException { id = canonical(id); - if (!entries.contains(id)) return true; + State state = entries.get(id); + if (state == null) return; requireRoot(); - DurableFiles.deleteIfExists(root.resolve(id + SUFFIX)); + DurableFiles.deleteIfExists(file(id, state)); entries.remove(id); - return true; } - synchronized Set snapshot() { return Set.copyOf(entries); } + synchronized Map snapshot() { return Map.copyOf(entries); } + + private void transition(String id, State expected, State replacement) throws IOException { + id = canonical(id); + if (entries.get(id) != expected) throw new IOException("HTTP inbound delivery fence state is invalid"); + requireRoot(); + Path source = file(id, expected), target = file(id, replacement); + if (Files.isSymbolicLink(source) || !Files.isRegularFile(source, LinkOption.NOFOLLOW_LINKS) + || Files.exists(target, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery fence state is unsafe"); + move(source, target); + DurableFiles.forceDirectory(root); + entries.put(id, replacement); + } private void load() throws IOException { try (DirectoryStream files = Files.newDirectoryStream(root)) { @@ -83,34 +95,48 @@ private void load() throws IOException { DurableFiles.deleteIfExists(file); continue; } - if (Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) - || !name.endsWith(SUFFIX) || Files.size(file) > 64L) + State state = State.fromFileName(name); + if (state == null || Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) + || Files.size(file) > 64L) throw new IOException("HTTP inbound delivery fence contains an invalid entry"); String id; - try { id = canonical(name.substring(0, name.length() - SUFFIX.length())); } + try { id = canonical(name.substring(0, name.length() - state.suffix.length())); } catch (IllegalArgumentException invalid) { throw new IOException("HTTP inbound delivery fence entry is invalid", invalid); } - if (!name.equals(id + SUFFIX) || !Files.readString(file, StandardCharsets.US_ASCII).equals(id) - || !entries.add(id)) + if (!name.equals(id + state.suffix) || !Files.readString(file, StandardCharsets.US_ASCII).equals(id)) throw new IOException("HTTP inbound delivery fence entry is invalid"); + State existing = entries.get(id); + if (existing == null) entries.put(id, state); + else { + // A provider without atomic moves may expose both names after an + // interrupted transition. Preserve the furthest fail-closed state: + // RUNNING never replays, and COMPLETED alone may be acknowledged. + State retained = existing.ordinal() >= state.ordinal() ? existing : state; + State obsolete = retained == existing ? state : existing; + DurableFiles.deleteIfExists(file(id, obsolete)); + entries.put(id, retained); + } if (entries.size() > MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence exceeds its bound"); } } } + private Path file(String id, State state) { return root.resolve(id + state.suffix); } + private static void move(Path source, Path target) throws IOException { + try { Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(source, target); } + } private static String canonical(String id) { if (id == null) throw new IllegalArgumentException("HTTP delivery id is invalid"); String canonical = UUID.fromString(id).toString(); if (!canonical.equals(id)) throw new IllegalArgumentException("HTTP delivery id is not canonical"); return canonical; } - private void requireRoot() throws IOException { if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP inbound delivery directory is unsafe"); } - private static void ownerOnlyFile(Path path) throws IOException { try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } catch (UnsupportedOperationException ignored) { } @@ -120,4 +146,14 @@ private static void ownerOnlyDirectory(Path path) throws IOException { PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } catch (UnsupportedOperationException ignored) { } } + + enum State { + RESERVED(".reserved"), RUNNING(".running"), COMPLETED(".completed"); + private final String suffix; + State(String suffix) { this.suffix = suffix; } + private static State fromFileName(String name) { + for (State state : values()) if (name.endsWith(state.suffix)) return state; + return null; + } + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 1a1250b61..b4623ed67 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -429,7 +429,7 @@ void durableBackendFencePreventsCallbackReplayAfterRestartBeforeAck() throws Exc } @Test - void failedBackendCallbackRemovesFenceAndRemainsRetryable() throws Exception { + void failedBackendCallbackRemainsUnacknowledgedAndIsNotReplayed() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-fence-proxy"), "localhost"); HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), @@ -437,7 +437,7 @@ void failedBackendCallbackRemovesFenceAndRemainsRetryable() throws Exception { Path clientDirectory = directory.resolve("retry-fence-client"); HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); java.util.concurrent.atomic.AtomicInteger attempts = new java.util.concurrent.atomic.AtomicInteger(); - CountDownLatch failed = new CountDownLatch(1), succeeded = new CountDownLatch(1); + CountDownLatch failed = new CountDownLatch(1); HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("vote").build()); try (HttpBackendTransportConnector first = new HttpBackendTransportConnector(clientDirectory, ignored -> { @@ -445,22 +445,62 @@ void failedBackendCallbackRemovesFenceAndRemainsRetryable() throws Exception { })) { first.dispatch(delivery); assertTrue(failed.await(2, TimeUnit.SECONDS)); - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); - while (countRegularFiles(clientDirectory.resolve("http-transport-inbound-deliveries")) != 0L - && System.nanoTime() < deadline) Thread.sleep(5); assertTrue(first.drainAcknowledgements().isEmpty()); } - try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, ignored -> { - attempts.incrementAndGet(); succeeded.countDown(); - })) { + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, + ignored -> attempts.incrementAndGet())) { + assertTrue(restarted.drainAcknowledgements().isEmpty()); + java.util.List accepted = restarted.accept(java.util.List.of(delivery)); + assertTrue(accepted.isEmpty(), "an ambiguous callback must not be awarded twice"); + assertEquals(1, attempts.get()); + } + } + + @Test + void reservedButNotStartedDeliveryResumesAfterRestart() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("reserved-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "C".repeat(43)); + Path clientDirectory = directory.resolve("reserved-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + String id = java.util.UUID.randomUUID().toString(); + new HttpInboundDeliveryStore(clientDirectory).reserve(id); + CountDownLatch completed = new CountDownLatch(1); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("vote").build()); + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, ignored -> completed.countDown())) { + assertTrue(restarted.drainAcknowledgements().isEmpty(), "a reservation alone must never be acknowledged"); java.util.List accepted = restarted.accept(java.util.List.of(delivery)); assertEquals(1, accepted.size()); restarted.dispatch(accepted.get(0)); - assertTrue(succeeded.await(2, TimeUnit.SECONDS)); - assertEquals(2, attempts.get()); + assertTrue(completed.await(2, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (new HttpInboundDeliveryStore(clientDirectory).state(id) != HttpInboundDeliveryStore.State.COMPLETED + && System.nanoTime() < deadline) Thread.sleep(5); + assertEquals(java.util.List.of(id), restarted.drainAcknowledgements()); } } + @Test + void interruptedStateRenameRetainsTheFurthestSafeState() throws Exception { + Path clientDirectory = directory.resolve("interrupted-state-client"); + String id = java.util.UUID.randomUUID().toString(); + String completedId = java.util.UUID.randomUUID().toString(); + Path states = clientDirectory.resolve("http-transport-inbound-deliveries"); + Files.createDirectories(states); + Files.writeString(states.resolve(id + ".reserved"), id); + Files.writeString(states.resolve(id + ".running"), id); + Files.writeString(states.resolve(completedId + ".running"), completedId); + Files.writeString(states.resolve(completedId + ".completed"), completedId); + HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(clientDirectory); + assertEquals(HttpInboundDeliveryStore.State.RUNNING, store.state(id)); + assertEquals(HttpInboundDeliveryStore.State.COMPLETED, store.state(completedId)); + assertFalse(Files.exists(states.resolve(id + ".reserved"))); + assertTrue(Files.exists(states.resolve(id + ".running"))); + assertFalse(Files.exists(states.resolve(completedId + ".running"))); + assertTrue(Files.exists(states.resolve(completedId + ".completed"))); + } + private static HttpTransportProtocol.Delivery delivery(int marker) { return new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("x").put("marker", marker).build()); diff --git a/docs/http-transport.md b/docs/http-transport.md index 78587e159..740911e45 100644 --- a/docs/http-transport.md +++ b/docs/http-transport.md @@ -40,8 +40,8 @@ Connection codes expire after 15 minutes and can be used only once. Treat a fres - Enrollment tokens are 256-bit random capabilities, single-use, short-lived, and retained by the proxy only as hashes. - Credentials, keys, pins, and revocation state are written atomically with owner-only permissions where the operating system supports them. - Request bodies, queues, batches, worker pools, concurrent requests, per-backend polling, and request rates are bounded. Exact paths, methods, and JSON content types are enforced. -- Proxy-to-backend messages stay in an owner-only durable proxy queue until the backend acknowledgement is durably applied. Before a backend callback can change vote or reward state, the backend fsyncs that delivery ID in its credential directory; a restart therefore acknowledges a replay without awarding it twice. The fence is deleted only after a matching authenticated HTTP 200 confirms the proxy removed the queued message. -- This crash fence deliberately favors preventing duplicate rewards: a process or host failure in the very small interval after the fence is persisted but before the callback begins can suppress that callback after restart. Fully transactional exactly-once reward execution is not possible because arbitrary reward commands and external plugin effects cannot share a transaction with the transport journal. +- Proxy-to-backend messages stay in an owner-only durable proxy queue until the backend acknowledgement is durably applied. The backend fsyncs three delivery states around each callback: a reserved callback resumes after restart, a completed callback is acknowledged without running twice, and a running callback with an uncertain outcome is neither replayed nor acknowledged. The proxy therefore retains the durable source copy instead of silently losing it. The completed entry is deleted only after a matching authenticated HTTP 200 confirms the proxy removed the queued message. +- A host failure while a callback is running has an inherently ambiguous result because arbitrary reward commands and external plugin effects cannot share a transaction with the transport journal. Such a delivery remains quarantined for operator investigation, avoiding both an automatic duplicate reward and an acknowledgement that could hide a missed reward. - Backend-to-proxy messages retain the existing bounded in-process retry semantics. Vote caching remains responsible for application-level durability in that direction. The listener must terminate TLS itself because client-certificate authentication is part of the protocol. Do not put an HTTP TLS-terminating reverse proxy or CDN in front of it. A TCP/L4 proxy that passes TLS through unchanged is suitable. Internet-facing installations should also use the host firewall or provider firewall for volumetric denial-of-service protection; an application cannot fully absorb a link or TCP flood. From 2a9fcc6d847493e55f3a06ce1c727d5477bc35b5 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:31:45 -0600 Subject: [PATCH 17/36] test(http): import filesystem helper --- .../votingplugin/backendproxy/http/HttpTransportRuntimeTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index b4623ed67..4b97184e7 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -11,6 +11,7 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; From 9eae8b5a35a5486e6eb93f70cbbd9c449a2bfb8b Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:42:05 -0600 Subject: [PATCH 18/36] fix(http): bound responses and persist queue roots --- .../http/HttpBackendTransportConnector.java | 29 +++++++++++++++---- .../http/HttpProxyTransportServer.java | 13 ++++++++- .../http/HttpTransportSecurityTest.java | 9 ++++++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 69aa4a6f8..7359e136d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -2,6 +2,7 @@ import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -121,8 +122,8 @@ public static HttpClientCredentialStore.ClientCredential enroll(HttpConnectionCo .connectTimeout(Duration.ofSeconds(5)).sslContext(HttpPinnedTls.clientContext(code)).build(); HttpRequest request = HttpRequest.newBuilder(code.endpoint().resolve("v1/enroll")).timeout(CLIENT_TIMEOUT) .header("Content-Type", "application/json").header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(payload)).build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); - if (response.statusCode() != 201 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) throw new IllegalArgumentException("Enrollment was rejected"); + LimitedResponse response = sendLimited(client, request); + if (response.statusCode() != 201) throw new IllegalArgumentException("Enrollment was rejected"); HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); HttpClientCredentialStore.saveEnrolled(credentials, code, issued); return HttpClientCredentialStore.load(credentials); } @@ -158,8 +159,8 @@ public synchronized boolean pollOnce() { } HttpRequest request = HttpRequest.newBuilder(transportEndpoint).timeout(CLIENT_TIMEOUT).header("Content-Type", "application/json") .header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(HttpTransportProtocol.request(serverId, session, requestSequence, acks, messages))).build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); - if (response.statusCode() != 200 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) return false; + LimitedResponse response = sendLimited(client, request); + if (response.statusCode() != 200) return false; HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(response.body()); if (!serverId.equals(packet.server()) || !session.equals(packet.session()) || packet.sequence() != requestSequence) return false; confirmAcknowledgements(acks); @@ -289,8 +290,8 @@ private void maybeRenewCredential() { HttpRequest request = HttpRequest.newBuilder(profile.endpoint().resolve("v1/renew")).timeout(CLIENT_TIMEOUT) .header("Content-Type", "application/json").header("Cache-Control", "no-store") .POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); - if (response.statusCode() != 201 || response.body().length > HttpTransportProtocol.MAX_BODY_BYTES) return; + LimitedResponse response = sendLimited(client, request); + if (response.statusCode() != 201) return; HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(directory, issued); HttpClientCredentialStore.ClientCredential replacement = staged.credential(); @@ -309,6 +310,22 @@ private static HttpClient client(HttpClientCredentialStore.HttpClientProfile pro return HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); } + private static LimitedResponse sendLimited(HttpClient client, HttpRequest request) throws IOException, InterruptedException { + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); + try (InputStream body = response.body()) { + long declaredLength = response.headers().firstValueAsLong("Content-Length").orElse(-1L); + if (declaredLength > HttpTransportProtocol.MAX_BODY_BYTES) + throw new IOException("HTTP transport response exceeds its limit"); + return new LimitedResponse(response.statusCode(), readLimited(body)); + } + } + static byte[] readLimited(InputStream body) throws IOException { + byte[] bytes = body.readNBytes(HttpTransportProtocol.MAX_BODY_BYTES + 1); + if (bytes.length > HttpTransportProtocol.MAX_BODY_BYTES) + throw new IOException("HTTP transport response exceeds its limit"); + return bytes; + } + private record LimitedResponse(int statusCode, byte[] body) { } private static SSLContext clientContext(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { String caPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index 2d3825a44..b9e62536c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -375,7 +375,18 @@ private synchronized Map> load() th private synchronized void persist(String serverId, HttpTransportProtocol.Delivery delivery) throws IOException { Path directory = root.resolve(serverId).normalize(); if (!directory.getParent().equals(root)) throw new IOException("HTTP outgoing queue server is invalid"); - Files.createDirectories(directory); ownerOnlyDirectory(directory); + boolean created = false; + try { Files.createDirectory(directory); created = true; } + catch (java.nio.file.FileAlreadyExistsException existing) { } + try { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue server directory is invalid"); + ownerOnlyDirectory(directory); + } finally { + // The child fsync below cannot make this newly published name durable in + // its parent. Persist the root entry before accepting the first message. + if (created) DurableFiles.forceDirectory(root); + } if (sequence == Long.MAX_VALUE) throw new IOException("HTTP outgoing queue sequence is exhausted"); String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); Path target = directory.resolve(name); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 455ecae7d..094d752c9 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -30,6 +30,15 @@ void connectionCodeRejectsExplicitZeroPort() { @TempDir Path directory; + @Test + void backendResponseReaderRejectsBodiesBeyondTheWireLimit() throws Exception { + byte[] maximum = new byte[HttpTransportProtocol.MAX_BODY_BYTES]; + assertEquals(maximum.length, HttpBackendTransportConnector.readLimited( + new java.io.ByteArrayInputStream(maximum)).length); + assertThrows(java.io.IOException.class, () -> HttpBackendTransportConnector.readLimited( + new java.io.ByteArrayInputStream(new byte[HttpTransportProtocol.MAX_BODY_BYTES + 1]))); + } + @Test void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { HttpConnectionCode original = new HttpConnectionCode("lobby", URI.create("https://Proxy.Example.test:8443/http"), pin('a'), pin('b'), From b13ce28b450e071bcedd72cd6ba6393be994bb69 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:44:33 -0600 Subject: [PATCH 19/36] test(http): await resumed delivery acknowledgement --- .../backendproxy/http/HttpTransportRuntimeTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 4b97184e7..fbd30301f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -476,9 +476,12 @@ void reservedButNotStartedDeliveryResumesAfterRestart() throws Exception { restarted.dispatch(accepted.get(0)); assertTrue(completed.await(2, TimeUnit.SECONDS)); long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); - while (new HttpInboundDeliveryStore(clientDirectory).state(id) != HttpInboundDeliveryStore.State.COMPLETED - && System.nanoTime() < deadline) Thread.sleep(5); - assertEquals(java.util.List.of(id), restarted.drainAcknowledgements()); + java.util.List acknowledgements = java.util.List.of(); + while (acknowledgements.isEmpty() && System.nanoTime() < deadline) { + acknowledgements = restarted.drainAcknowledgements(); + if (acknowledgements.isEmpty()) Thread.sleep(5); + } + assertEquals(java.util.List.of(id), acknowledgements); } } From 0a938cf3ad6002bbdb709ee72a9a4991916d7b17 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 20:53:47 -0600 Subject: [PATCH 20/36] fix(http): persist transport directory entries --- .../http/HttpClientCredentialStore.java | 3 +++ .../http/HttpInboundDeliveryStore.java | 12 +++++++++--- .../http/HttpProxyTransportServer.java | 14 ++++++++++---- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index 32b80b8f6..b100cc5a7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -114,6 +114,9 @@ private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClie if (connectionCodeDigest != null) writePrivate(safe(generation.resolve(CONNECTION_CODE_DIGEST_FILE)), connectionCodeDigest.getBytes(StandardCharsets.US_ASCII)); EnrolledClient enrolled = loadEnrolled(generation); + // Each file is durable within the generation, but the generation name is + // published by its parent. Persist it before CURRENT can activate it. + DurableFiles.forceDirectory(generations); return new StagedCredential(name, enrolled.credential(), enrolled.profile()); } catch (Exception failure) { try { Files.deleteIfExists(generation.resolve(BUNDLE_FILE)); Files.deleteIfExists(generation.resolve(PASSWORD_FILE)); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java index 960d02ba5..40dcb4101 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java @@ -29,9 +29,15 @@ final class HttpInboundDeliveryStore { ownerOnlyDirectory(credentials); root = credentials.resolve(DIRECTORY).normalize(); if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); - Files.createDirectories(root); - requireRoot(); - ownerOnlyDirectory(root); + boolean created = false; + try { Files.createDirectory(root); created = true; } + catch (java.nio.file.FileAlreadyExistsException existing) { } + try { + requireRoot(); + ownerOnlyDirectory(root); + } finally { + if (created) DurableFiles.forceDirectory(credentials); + } load(); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index b9e62536c..8e1ecc925 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -322,10 +322,16 @@ private static final class DurableOutgoingQueue { private DurableOutgoingQueue(Path root) throws IOException { this.root = root.toAbsolutePath().normalize(); - Files.createDirectories(this.root); - if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP outgoing queue directory is invalid"); - ownerOnlyDirectory(this.root); + boolean created = false; + try { Files.createDirectory(this.root); created = true; } + catch (java.nio.file.FileAlreadyExistsException existing) { } + try { + if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue directory is invalid"); + ownerOnlyDirectory(this.root); + } finally { + if (created) DurableFiles.forceDirectory(this.root.getParent()); + } } private synchronized Map> load() throws IOException { From 5949e113bc7c338c3c6de8c5ce1496503e8f8719 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 21:12:40 -0600 Subject: [PATCH 21/36] fix(http): coordinate transport lifecycle handoffs --- .../votingplugin/VotingPluginMain.java | 7 +- .../backendproxy/BackendProxyHandler.java | 7 +- .../http/HttpBackendTransportConnector.java | 4 + .../http/HttpInboundDeliveryStore.java | 8 + .../BackendProxyTransportManager.java | 7 +- .../transport/HttpBackendProxyTransport.java | 64 +- .../control/BackendControlConnector.java | 3 +- .../votingplugin/proxy/VotingPluginProxy.java | 3682 +---------------- .../http/HttpTransportSecurityTest.java | 13 + .../HttpBackendProxyTransportTest.java | 39 + 10 files changed, 139 insertions(+), 3695 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 06cf263bd..a2d243fa5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1215,6 +1215,11 @@ public String getBackendHostedControlStatus() { /** Recreates proxy transports after Control applies BungeeSettings.yml. */ public synchronized void restartBackendProxyHandler() { + restartBackendProxyHandler(System.nanoTime() + TimeUnit.SECONDS.toNanos(25)); + } + + /** Recreates proxy transports while preserving the caller's end-to-end validation deadline. */ + public synchronized void restartBackendProxyHandler(long validationDeadlineNanos) { BackendProxyHandler previous = backendProxyHandler; if (!bungeeSettings.isUseBungeecoord()) { backendProxyHandler = null; @@ -1229,7 +1234,7 @@ public synchronized void restartBackendProxyHandler() { BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); try { replacement.load(); - replacement.validateTransport(); + replacement.validateTransport(validationDeadlineNanos); if (previous != null) previous.completeRedisHandoff(replacement); } catch (RuntimeException failure) { replacement.close(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 8395c38b4..f485374bb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -110,10 +110,15 @@ public void prepareForReplacement(BungeeMethod replacementMethod) { /** Fails a configuration apply when its selected transport did not initialize. */ public void validateTransport() { + validateTransport(System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(25)); + } + + /** Validates transport startup without extending the caller's existing deadline. */ + public void validateTransport(long deadlineNanos) { if (method == null || globalMessageHandler == null || presenceManager == null) { throw new IllegalStateException("Backend proxy handler initialization failed"); } - transportManager.validate(); + transportManager.validate(deadlineNanos); } /** Completes the no-loss/no-duplicate same-Redis subscriber handoff after validation. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 7359e136d..820d4ab5e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -173,6 +173,10 @@ public synchronized boolean pollOnce() { } @Override public void close() { running.getAndSet(false); + // Revoke this connector's journal writer before a replacement snapshots it. + // In-flight transitions serialize with seal(): either COMPLETED is already + // durable, or the delivery remains durably RUNNING and fail-closed. + if (inboundDeliveries != null) inboundDeliveries.seal(); Thread current = poller; if (current != null) current.interrupt(); callbackExecutor.shutdown(); try { if (!callbackExecutor.awaitTermination(5, TimeUnit.SECONDS)) callbackExecutor.shutdownNow(); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); callbackExecutor.shutdownNow(); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java index 40dcb4101..ba04a31ec 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpInboundDeliveryStore.java @@ -21,6 +21,7 @@ final class HttpInboundDeliveryStore { private static final int MAX_ENTRIES = HttpTransportProtocol.MAX_QUEUE; private final Path root; private final Map entries = new LinkedHashMap<>(); + private boolean sealed; HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { Path credentials = credentialDirectory.toAbsolutePath().normalize(); @@ -44,6 +45,7 @@ final class HttpInboundDeliveryStore { synchronized State state(String id) { return entries.get(canonical(id)); } synchronized void reserve(String id) throws IOException { + requireWritable(); id = canonical(id); State existing = entries.get(id); if (existing == State.RESERVED) return; @@ -67,8 +69,10 @@ synchronized void reserve(String id) throws IOException { synchronized void markRunning(String id) throws IOException { transition(id, State.RESERVED, State.RUNNING); } synchronized void markCompleted(String id) throws IOException { transition(id, State.RUNNING, State.COMPLETED); } + synchronized void seal() { sealed = true; } synchronized void remove(String id) throws IOException { + requireWritable(); id = canonical(id); State state = entries.get(id); if (state == null) return; @@ -80,6 +84,7 @@ synchronized void remove(String id) throws IOException { synchronized Map snapshot() { return Map.copyOf(entries); } private void transition(String id, State expected, State replacement) throws IOException { + requireWritable(); id = canonical(id); if (entries.get(id) != expected) throw new IOException("HTTP inbound delivery fence state is invalid"); requireRoot(); @@ -129,6 +134,9 @@ private void load() throws IOException { } private Path file(String id, State state) { return root.resolve(id + state.suffix); } + private void requireWritable() throws IOException { + if (sealed) throw new IOException("HTTP inbound delivery store ownership has ended"); + } private static void move(Path source, Path target) throws IOException { try { Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(source, target); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 43b54c128..fecb46302 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -70,8 +70,13 @@ public void close() { } public void validate() { + validate(System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(25)); + } + + public void validate(long deadlineNanos) { if (transport == null) throw new IllegalStateException("Backend proxy transport was not initialized"); - transport.validate(); + if (transport instanceof HttpBackendProxyTransport http) http.validate(deadlineNanos); + else transport.validate(); } public void prepareForReplacement() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 81f520ed7..944557439 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -3,6 +3,9 @@ import java.nio.file.Path; import java.time.Clock; import java.util.ArrayDeque; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; @@ -15,13 +18,18 @@ /** Backend adapter for the secure outbound-only HTTP proxy transport. */ public final class HttpBackendProxyTransport implements BackendProxyTransport { private static final int MAX_STARTUP_QUEUE = 1024; + private static final long DEFAULT_STARTUP_VALIDATION_SECONDS = 25L; + private static final ConcurrentHashMap DIRECTORY_OWNERS = new ConcurrentHashMap<>(); private final VotingPluginMain plugin; private final Object lifecycle = new Object(); + private final CountDownLatch startupComplete = new CountDownLatch(1); private final ArrayDeque startupQueue = new ArrayDeque<>(); private volatile HttpBackendTransportConnector connector; private volatile Thread worker; private volatile RuntimeException startupFailure; + private volatile boolean started; private volatile boolean closed; + private Semaphore directoryOwner; private final java.util.concurrent.atomic.AtomicBoolean queueWarning = new java.util.concurrent.atomic.AtomicBoolean(); public HttpBackendProxyTransport(VotingPluginMain plugin) { @@ -34,6 +42,7 @@ public void start(GlobalMessageHandler messageHandler) { String serverId = plugin.getBungeeSettings().getServer(); String connectionCode = plugin.getBungeeSettings().getHttpConnectionCode(); validateConfiguration(directory, serverId, connectionCode); + started = true; worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler), "VotingPlugin-HTTP-Backend-Setup"); worker.setDaemon(true); @@ -42,31 +51,46 @@ public void start(GlobalMessageHandler messageHandler) { private void initialize(Path directory, String serverId, String configuredCode, GlobalMessageHandler messageHandler) { + Path ownerKey = directory.toAbsolutePath().normalize(); + Semaphore owner = DIRECTORY_OWNERS.computeIfAbsent(ownerKey, ignored -> new Semaphore(1)); + boolean acquired = false, installed = false; + HttpBackendTransportConnector replacement = null; try { + owner.acquire(); + acquired = true; + synchronized (lifecycle) { if (closed) return; } HttpConnectionCode code = enrollmentCode(directory, serverId, configuredCode); if (code != null) HttpBackendTransportConnector.enroll(code, serverId, directory); HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); - HttpBackendTransportConnector replacement = new HttpBackendTransportConnector(directory, messageHandler::onMessage); + replacement = new HttpBackendTransportConnector(directory, messageHandler::onMessage); boolean discard = false; synchronized (lifecycle) { if (closed) { discard = true; } else { - connector = replacement; replacement.start(); while (!startupQueue.isEmpty()) { if (!replacement.send(startupQueue.removeFirst())) { throw new IllegalStateException("HTTP startup queue could not be transferred"); } } + connector = replacement; + directoryOwner = owner; + installed = true; } } if (discard) replacement.close(); } catch (Exception failure) { startupFailure = new IllegalStateException("Secure HTTP backend enrollment or connection failed", failure); plugin.getLogger().severe("Secure HTTP backend transport is unavailable; check the connection code and proxy endpoint"); + } finally { + if (!installed) { + if (replacement != null) replacement.close(); + if (acquired) owner.release(); + } + startupComplete.countDown(); } } @@ -86,14 +110,28 @@ public void send(JsonEnvelope envelope) { @Override public void validate() { - RuntimeException failure = startupFailure; - if (failure != null) throw failure; + validate(System.nanoTime() + TimeUnit.SECONDS.toNanos(DEFAULT_STARTUP_VALIDATION_SECONDS)); + } + + void validate(long deadlineNanos) { String serverId = plugin.getBungeeSettings().getServer(); if (serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) { throw new IllegalStateException("HTTP requires a valid unique backend Server name"); } Path directory = plugin.getDataFolder().toPath().resolve("http"); validateConfiguration(directory, serverId, plugin.getBungeeSettings().getHttpConnectionCode()); + if (!started) return; + try { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L || !startupComplete.await(remaining, TimeUnit.NANOSECONDS)) + throw new IllegalStateException("Secure HTTP backend setup did not finish within the validation deadline"); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Secure HTTP backend setup validation was interrupted", interrupted); + } + RuntimeException failure = startupFailure; + if (failure != null) throw failure; + if (closed || connector == null) throw new IllegalStateException("Secure HTTP backend transport did not become ready"); } public static void validateConfiguration(Path directory, String serverId, String configuredCode) { @@ -125,6 +163,7 @@ static HttpConnectionCode enrollmentCode(Path directory, String serverId, String public void close() { Thread setup; HttpBackendTransportConnector active; + Semaphore owner; synchronized (lifecycle) { if (closed) return; closed = true; @@ -133,17 +172,22 @@ public void close() { worker = null; active = connector; connector = null; + owner = directoryOwner; + directoryOwner = null; } + startupComplete.countDown(); if (setup != null) setup.interrupt(); - if (setup == null && active == null) return; - Thread cleanup = new Thread(() -> drain(setup, active), "VotingPlugin-HTTP-Backend-Cleanup"); + if (setup == null && active == null && owner == null) return; + Thread cleanup = new Thread(() -> drain(setup, active, owner), "VotingPlugin-HTTP-Backend-Cleanup"); cleanup.setDaemon(true); cleanup.start(); } - private static void drain(Thread setup, HttpBackendTransportConnector active) { - if (setup != null) try { setup.join(TimeUnit.SECONDS.toMillis(5)); } - catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } - if (active != null) active.close(); + private static void drain(Thread setup, HttpBackendTransportConnector active, Semaphore owner) { + try { + if (setup != null) try { setup.join(TimeUnit.SECONDS.toMillis(5)); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + if (active != null) active.close(); + } finally { if (owner != null) owner.release(); } } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 2b903ccea..00899595c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -112,11 +112,12 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set private void reloadConfiguration(String fileName) throws Exception { Future reload; + long validationDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(29); synchronized (operationLifecycle) { if (closed) throw new IllegalStateException("Bukkit Control connector is stopping"); reload = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { plugin.reloadFromControl(); - if ("BungeeSettings.yml".equals(fileName)) plugin.restartBackendProxyHandler(); + if ("BungeeSettings.yml".equals(fileName)) plugin.restartBackendProxyHandler(validationDeadline); return null; }); activeReload = reload; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 428f3e151..36b185f2d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1,3681 +1 @@ -package com.bencodez.votingplugin.proxy; - -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.sql.SQLException; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; - -import javax.net.ssl.SSLParameters; - -import org.eclipse.paho.client.mqttv3.MqttException; - -import com.bencodez.advancedcore.api.time.TimeType; -import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalDataHandlerProxy; -import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalMySQL; -import com.bencodez.advancedcore.bungeeapi.time.BungeeTimeChecker; -import com.bencodez.simpleapi.encryption.EncryptionHandler; -import com.bencodez.simpleapi.json.JsonParser; -import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; -import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec; -import com.bencodez.simpleapi.servercomm.global.GlobalMessageListener; -import com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler; -import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; -import com.bencodez.simpleapi.servercomm.mqtt.MqttServerComm; -import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; -import com.bencodez.simpleapi.servercomm.redis.RedisHandler; -import com.bencodez.simpleapi.servercomm.redis.RedisListener; -import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; -import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; -import com.bencodez.simpleapi.servercomm.sockets.SocketReceiver; -import com.bencodez.simpleapi.sql.Column; -import com.bencodez.simpleapi.sql.DataType; -import com.bencodez.simpleapi.sql.data.DataValue; -import com.bencodez.simpleapi.sql.data.DataValueBoolean; -import com.bencodez.simpleapi.sql.data.DataValueInt; -import com.bencodez.simpleapi.sql.data.DataValueString; -import com.bencodez.simpleapi.sql.mysql.config.MysqlConfig; -import com.bencodez.votingplugin.backendproxy.http.HttpEnrollmentAuthority; -import com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer; -import com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity; -import com.bencodez.votingplugin.proxy.broadcast.ProxyBroadcastDecider; -import com.bencodez.votingplugin.proxy.cache.IVoteCache; -import com.bencodez.votingplugin.proxy.cache.VoteCacheHandler; -import com.bencodez.votingplugin.proxy.cache.nonvoted.INonVotedPlayersStorage; -import com.bencodez.votingplugin.proxy.cache.nonvoted.NonVotedPlayersCache; -import com.bencodez.votingplugin.proxy.control.ControlConnector; -import com.bencodez.votingplugin.proxy.control.HostedControlManager; -import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyHandler; -import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyMethod; -import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfiguration; -import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfigurationBungee; -import com.bencodez.votingplugin.proxy.presence.BackendPlayerPresenceTracker; -import com.bencodez.votingplugin.proxy.presence.PlayerPresence; -import com.bencodez.votingplugin.timequeue.VoteTimeQueue; -import com.bencodez.votingplugin.topvoter.TopVoter; -import com.bencodez.votingplugin.util.MinecraftUsernameValidator; -import com.bencodez.votingplugin.util.ServiceSiteValidator; -import com.bencodez.votingplugin.votelog.VoteLogMysqlTable; -import com.bencodez.votingplugin.votelog.VoteLogMysqlTable.VoteLogStatus; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; - -import redis.clients.jedis.DefaultJedisClientConfig; -import redis.clients.jedis.HostAndPort; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; - -import lombok.Getter; -import lombok.Setter; - -public abstract class VotingPluginProxy { - private static final long PRESENCE_HANDOFF_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(2); - private static final long PRESENCE_STARTUP_RESYNC_DELAY_SECONDS = 5L; - private static final long PRESENCE_MAINTENANCE_INTERVAL_SECONDS = 30L; - private static final long PRESENCE_BACKEND_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(90); - private static final long CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10); - - @Getter - @Setter - private int votePartyVotes = 0; - - @Getter - @Setter - private int currentVotePartyVotesRequired = 0; - - @Getter - @Setter - private ProxyMysqlUserTable proxyMySQL; - - private EncryptionHandler encryptionHandler; - - private HashMap clientHandles; - - private SocketHandler socketHandler; - private HttpProxyTransportServer httpTransportServer; - private HttpEnrollmentAuthority httpEnrollmentAuthority; - - @Getter - @Setter - private boolean votifierEnabled = true; - - @Getter - private ConcurrentHashMap uuidPlayerNameCache = new ConcurrentHashMap<>(); - - @Getter - @Setter - private GlobalDataHandlerProxy globalDataHandler; - - @Getter - private RedisHandler redisHandler; - private JedisPool redisPublisherPool; - private volatile long redisPublisherRetryAfter; - private boolean timeVoteRetryScheduled; - private boolean timeVoteDeliveryRetryScheduled; - private boolean cachedVoteDeliveryRetryScheduled; - - private boolean enabled; - - @Getter - @Setter - private MultiProxyHandler multiProxyHandler; - - @Getter - private BungeeTimeChecker bungeeTimeChecker; - - @Getter - @Setter - private BungeeMethod method; - - @Getter - private MqttHandler mqttHandler; - - @Getter - private GlobalMessageProxyHandler globalMessageProxyHandler; - - @Getter - @Setter - private MySqlMessenger proxyMysqlMessenger; - - @Getter - private VoteCacheHandler voteCacheHandler; - - @Getter - private NonVotedPlayersCache nonVotedPlayersCache; - - @Getter - private final BackendPlayerPresenceTracker backendPlayerPresenceTracker = new BackendPlayerPresenceTracker(); - private final Map pendingPresenceHandoffs = new HashMap<>(); - private final Set pendingBackendRecoverySnapshots = ConcurrentHashMap.newKeySet(); - private final Map controlEnrollmentNextAllowed = new ConcurrentHashMap<>(); - private final Map pendingCommunicationTests = new ConcurrentHashMap<>(); - private volatile ControlConnector controlConnector; - private volatile HostedControlManager hostedControlManager; - private final Object controlLifecycleLock = new Object(); - private final AtomicLong controlServicesGeneration = new AtomicLong(); - private final ExecutorService controlLifecycleExecutor = Executors.newSingleThreadExecutor(task -> { - Thread thread = new Thread(task, "votingplugin-control-lifecycle"); - thread.setDaemon(true); - return thread; - }); - - public VotingPluginProxy() { - enabled = true; - - bungeeTimeChecker = new BungeeTimeChecker(getConfig().getTimeZone(), getConfig().getTimeHourOffSet(), - getConfig().getTimeWeekOffSet()) { - - @Override - public void debug(String text) { - debug2(text); - } - - @Override - public long getLastUpdated() { - return getVoteCacheLastUpdated(); - } - - @Override - public int getPrevDay() { - return getVoteCachePrevDay(); - } - - @Override - public String getPrevMonth() { - return getVoteCachePrevMonth(); - } - - @Override - public int getPrevWeek() { - return getVoteCachePrevWeek(); - } - - @Override - public void info(String text) { - log(text); - } - - @Override - public boolean isEnabled() { - return enabled; - } - - @Override - public boolean isIgnoreTime() { - return isVoteCacheIgnoreTime(); - } - - @Override - public void setIgnoreTime(boolean ignore) { - setVoteCacheVoteCacheIgnoreTime(ignore); - } - - @Override - public void setLastUpdated() { - setVoteCacheLastUpdated(); - } - - @Override - public void setPrevDay(int day) { - setVoteCachePrevDay(day); - } - - @Override - public void setPrevMonth(String text) { - setVoteCachePrevMonth(text); - } - - @Override - public void setPrevWeek(int week) { - setVoteCachePrevWeek(week); - } - - @Override - public void timeChanged(TimeType type, boolean fake, boolean pre, boolean post) { - if (getConfig().getVoteCacheTime() > 0) { - getVoteCacheHandler().checkVoteCacheTime(getConfig().getVoteCacheTime()); - } - if (!getConfig().getGlobalDataEnabled()) { - warn("Global data not enabled, ignoring time change event"); - return; - } - int delay = 1; - for (String s : getAllAvailableServers()) { - if (getGlobalDataHandler().getGlobalMysql().containsKey(s)) { - String lastOnlineStr = getGlobalDataHandler().getString(s, "LastOnline"); - long lastOnline = 0; - try { - lastOnline = Long.valueOf(lastOnlineStr); - } catch (NumberFormatException e) { - // ignore - } - - if (LocalDateTime.now().atZone(ZoneOffset.UTC).toInstant().toEpochMilli() - lastOnline < 1000 - * 60 * 60 * 12) { - HashMap dataToSet = new HashMap<>(); - dataToSet.put("LastUpdated", new DataValueString( - "" + LocalDateTime.now().atZone(ZoneOffset.UTC).toInstant().toEpochMilli())); - dataToSet.put("FinishedProcessing", new DataValueBoolean(false)); - dataToSet.put(type.toString(), new DataValueBoolean(true)); - getGlobalDataHandler().setData(s, dataToSet); - - globalMessageProxyHandler.sendMessage(s, delay, VotingPluginWire.bungeeTimeChange()); - delay++; - } else { - warn("Server " + s + " hasn't been online recently"); - } - } else { - warn("Server " + s + " global data handler disabled?"); - } - } - globalDataHandler.onTimeChange(type); - } - - @Override - public void warning(String text) { - warn(text); - } - }; - } - - public void onTimeChangedFailed(String srv, TimeType type) { - getGlobalDataHandler().setBoolean(srv, type.toString(), false); - getGlobalDataHandler().setBoolean(srv, "FinishedProcessing", true); - getGlobalDataHandler().setBoolean(srv, "Processing", false); - } - - public void onTimeChangedFinished(TimeType type) { - if (type.equals(TimeType.MONTH)) { - getProxyMySQL().copyColumnData(TopVoter.Monthly.getColumnName(), "LastMonthTotal"); - } - getProxyMySQL().wipeColumnData(TopVoter.of(type).getColumnName(), DataType.INTEGER); - - if (!getConfig().getGlobalDataEnabled()) { - return; - } - for (String s : getAllAvailableServers()) { - getGlobalDataHandler().setBoolean(s, "ForceUpdate", true); - getGlobalMessageProxyHandler().sendMessage(s, 1, VotingPluginWire.bungeeTimeChange()); - } - processQueue(); - } - - /** - * Load MySQL + global data handler. - */ - public void loadMysql(MysqlConfig mysqlConfig, MysqlConfig globalDataMysqlConfig) { - if (mysqlConfig.getHostName().isEmpty() || mysqlConfig.getDatabase().isEmpty()) { - logSevere("MySQL is not configured correctly. " + "Missing host/database. host=" + mysqlConfig.getHostName() - + " db=" + mysqlConfig.getDatabase()); - setProxyMySQL(null); - return; - } - - setProxyMySQL(new ProxyMysqlUserTable("VotingPlugin_Users", mysqlConfig, getConfig().getDebug()) { - - @Override - public void debug(SQLException e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void logSevere(String string) { - VotingPluginProxy.this.logSevere(string); - } - - @Override - public void logInfo(String string) { - VotingPluginProxy.this.logInfo(string); - } - - @Override - public void debug(Throwable t) { - if (getConfig().getDebug()) { - t.printStackTrace(); - } - } - - @Override - public void debug(String str) { - debug2(str); - } - }); - - ArrayList servers = new ArrayList(getAllAvailableServers()); - - if (getConfig().getGlobalDataEnabled()) { - if (getConfig().getGlobalDataUseMainMySQL()) { - setGlobalDataHandler(new GlobalDataHandlerProxy( - new GlobalMySQL("VotingPlugin_GlobalData", getProxyMySQL().getMysql()) { - - @Override - public void debugEx(Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void debugLog(String text) { - debug(text); - } - - @Override - public void info(String text) { - logInfo(text); - } - - @Override - public void logSevere(String text) { - VotingPluginProxy.this.logSevere(text); - } - - @Override - public void warning(String text) { - warn(text); - } - }, servers) { - - @Override - public void onTimeChangedFailed(String srv, TimeType type) { - VotingPluginProxy.this.onTimeChangedFailed(srv, type); - } - - @Override - public void onTimeChangedFinished(TimeType type) { - VotingPluginProxy.this.onTimeChangedFinished(type); - } - }); - } else { - setGlobalDataHandler( - new GlobalDataHandlerProxy(new GlobalMySQL("VotingPlugin_GlobalData", globalDataMysqlConfig) { - - @Override - public void debugEx(Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void debugLog(String text) { - debug(text); - } - - @Override - public void info(String text) { - logInfo(text); - } - - @Override - public void logSevere(String text) { - VotingPluginProxy.this.logSevere(text); - } - - @Override - public void warning(String text) { - warn(text); - } - }, servers) { - - @Override - public void onTimeChangedFailed(String srv, TimeType type) { - VotingPluginProxy.this.onTimeChangedFailed(srv, type); - } - - @Override - public void onTimeChangedFinished(TimeType type) { - VotingPluginProxy.this.onTimeChangedFinished(type); - } - }); - } - - // update global schema columns (unchanged from original) - getGlobalDataHandler().getGlobalMysql().alterColumnType("IgnoreTime", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("MONTH", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("WEEK", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("DAY", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("FinishedProcessing", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("Processing", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("ForceUpdate", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("LastUpdated", "MEDIUMTEXT"); - } - - // column types (unchanged from original) - getProxyMySQL().alterColumnType("TopVoterIgnore", "VARCHAR(5)"); - getProxyMySQL().alterColumnType("CheckWorld", "VARCHAR(5)"); - getProxyMySQL().alterColumnType("Reminded", "VARCHAR(5)"); - getProxyMySQL().alterColumnType("DisableBroadcast", "VARCHAR(5)"); - getProxyMySQL().alterColumnType("LastOnline", "VARCHAR(20)"); - getProxyMySQL().alterColumnType("PlayerName", "VARCHAR(30)"); - getProxyMySQL().alterColumnType("DailyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("WeeklyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("DayVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("BestDayVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("WeekVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("BestWeekVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("VotePartyVotes", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("MonthVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("Points", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("HighestDailyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("AllTimeTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("HighestMonthlyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("MonthTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("HighestWeeklyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("LastMonthTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("LastWeeklyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("LastDailyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("OfflineRewards", "MEDIUMTEXT"); - getProxyMySQL().alterColumnType("DayVoteStreakLastUpdate", "MEDIUMTEXT"); - - if (getConfig().getStoreMonthTotalsWithDate()) { - getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now()), "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now().plusMonths(1)), - "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now().plusMonths(2)), - "INT DEFAULT '0'"); - } - } - - public void addCurrentVotePartyVotes(int amount) { - votePartyVotes += amount; - setVoteCacheVotePartyCurrentVotes(votePartyVotes); - debug("Current vote party total: " + votePartyVotes); - } - - public void addNonVotedPlayer(String uuid, String playerName) { - nonVotedPlayersCache.addPlayer(uuid, playerName); - } - - public void addVoteParty() { - if (getConfig().getVotePartyEnabled()) { - addCurrentVotePartyVotes(1); - checkVoteParty(); - } - } - - public abstract void broadcast(String message); - - private Set sendProxyBroadcast(Set targets, String uuid, String player, String service, long time, - String text, boolean wasOnline) { - Set forwarded = new LinkedHashSet<>(); - for (String targetServer : targets) { - JsonEnvelope envelope = VotingPluginWire.voteBroadcast(uuid, player, service, time, text, wasOnline); - if (sendProxyBroadcastEnvelopeNow(targetServer, envelope)) { - forwarded.add(targetServer); - } - } - return forwarded; - } - - /** - * Sends a standalone proxy broadcast through the selected transport and reports - * whether that transport accepted the message. - * - * @param server target backend server - * @param envelope standalone broadcast envelope - * @return true only when the transport accepted the message - */ - protected boolean sendProxyBroadcastEnvelopeNow(String server, JsonEnvelope envelope) { - switch (method) { - case MQTT: - return sendMqttEnvelopeServer(server, envelope); - case MYSQL: - if (proxyMysqlMessenger == null) { - return false; - } - try { - proxyMysqlMessenger.sendToBackend(server, envelope); - return true; - } catch (SQLException e) { - debug(e.getMessage()); - return false; - } - case PLUGINMESSAGING: - return sendPluginMessageServerNow(server, envelope); - case REDIS: - return sendRedisEnvelopeServer(server, envelope, true); - case SOCKETS: - // Standalone broadcasts use the same initialized client as normal - // envelopes. This preserves the socket connection and its delivery - // acknowledgement instead of creating a second short-lived socket. - return sendSocketEnvelope(server, envelope); - case HTTP: - return sendHttpEnvelope(server, envelope); - default: - return false; - } - } - - /** - * Sends a reward-bearing vote envelope and reports whether the selected - * transport accepted it. Legacy transports retain their existing asynchronous - * semantics; HTTP exposes its bounded-queue result so a vote is never discarded - * when the queue is full. - */ - protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope) { - if (method == BungeeMethod.HTTP) { - return sendHttpEnvelope(server, envelope); - } - GlobalMessageProxyHandler handler = globalMessageProxyHandler; - if (handler == null) { - return false; - } - handler.sendMessage(server, delay, envelope); - return true; - } - - public synchronized void checkCachedVotes(String server) { - int delay = 1; - if (isServerValid(server)) { - if (isSomeoneOnlineServerForVoteRouting(server)) { - if (getVoteCacheHandler().hasVotes(server) && !getConfig().getBlockedServers().contains(server)) { - ArrayList c = getVoteCacheHandler().getVotes(server); - ArrayList removed = new ArrayList<>(); - if (!c.isEmpty()) { - int num = 1; - int numberOfVotes = c.size(); - for (OfflineBungeeVote cache : c) { - if (cache.isDeliveryStateDirty() && !persistServerVoteDelivery(server, cache)) { - continue; - } - if (cache.isProxyBroadcastHandled() && cache.needsBroadcastOn(server)) { - Set forwarded = sendProxyBroadcast(Collections.singleton(server), - cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), - cache.getText(), false); - if (cache.getBroadcastForwardedServers().addAll(forwarded)) { - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - if (!persistServerVoteDelivery(server, cache)) { - continue; - } - } - } - - boolean toSend = true; - if (getConfig().getWaitForUserOnline()) { - if (!isPlayerOnlineForVoteRouting(cache.getPlayerName())) { - toSend = false; - } else if (isPlayerOnlineForVoteRouting(cache.getPlayerName()) - && !getCurrentPlayerServerForVoteRouting(cache.getPlayerName()).equals(server)) { - toSend = false; - } - } - if (toSend) { - boolean broadcastHere = cache.needsBroadcastOn(server); - if (!cache.isProxyBroadcastHandled() && broadcastHere - && getConfig().getProxyBroadcastEnabled()) { - boolean playerOnline = isPlayerOnlineForVoteRouting(cache.getPlayerName()); - String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(cache.getPlayerName()) - : null; - - Set targets = proxyBroadcastDecider.resolveTargets(playerOnline, - playerServer); - broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); - } - - if (!sendVoteEnvelopeAccepted(server, delay, - VotingPluginWire.vote(cache.getPlayerName(), cache.getUuid(), - cache.getService(), cache.getTime(), false, cache.isRealVote(), - cache.getText(), cache.getVoteId(), getConfig().getBungeeManageTotals(), - broadcastHere, num, numberOfVotes))) { - debug("Retaining cached vote because the transport rejected delivery for " + server); - continue; - } - delay++; - num++; - removed.add(cache); - } else { - debug("Not sending vote because user isn't on server " + server + ": " - + cache.toString()); - } - } - getVoteCacheHandler().removeServerVotes(server, removed); - } else { - debug("No cached votes for server: " + server); - } - } else { - debug("No cached votes for server: " + server); - } - } - } else { - debug("Server not valid: " + server); - } - } - - public synchronized void checkOnlineVotes(String player, String uuid, String server) { - int delay = 1; - if (isPlayerOnlineForVoteRouting(player) && getVoteCacheHandler().hasOnlineVotes(uuid)) { - ArrayList c = getVoteCacheHandler().getOnlineVotes(uuid); - if (!c.isEmpty()) { - if (server == null) { - server = getCurrentPlayerServerForVoteRouting(player); - } - if (!getConfig().getBlockedServers().contains(server)) { - int num = 1; - int numberOfVotes = (int) c.stream().filter(vote -> !vote.isRewardDelivered()).count(); - boolean deliveredReward = false; - ArrayList retained = new ArrayList<>(); - for (OfflineBungeeVote cache : c) { - if (cache.isProxyBroadcastHandled()) { - Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); - pendingTargets.removeAll(cache.getBroadcastForwardedServers()); - List blockedServers = getConfig().getBlockedServers(); - if (blockedServers != null) { - pendingTargets.removeAll(blockedServers); - } - cache.getBroadcastForwardedServers().addAll(sendProxyBroadcast(pendingTargets, - cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), - cache.getText(), false)); - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - } - boolean broadcastHere = cache.needsBroadcastOn(server); - if (!cache.isProxyBroadcastHandled() && broadcastHere - && getConfig().getProxyBroadcastEnabled()) { - String playerServer = (server != null) ? server : getCurrentPlayerServerForVoteRouting(player); - - Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); - broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); - } - - if (!cache.isRewardDelivered()) { - if (!sendVoteEnvelopeAccepted(server, delay, - VotingPluginWire.voteOnline(cache.getPlayerName(), cache.getUuid(), cache.getService(), - cache.getTime(), false, cache.isRealVote(), cache.getText(), cache.getVoteId(), - getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes))) { - debug("Retaining online vote because the transport rejected delivery for " + server); - retained.add(cache); - continue; - } - // The normal envelope is also a valid broadcast delivery for the - // current target. Record it so a previously pending standalone - // retry cannot announce the same vote again later. - if (cache.isProxyBroadcastHandled() && broadcastHere) { - cache.getBroadcastForwardedServers().add(server); - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - } - cache.setRewardDelivered(true); - deliveredReward = true; - delay++; - num++; - } - - if (cache.isProxyBroadcastHandled() && !cache.isProxyBroadcastComplete()) { - retained.add(cache); - } - } - getVoteCacheHandler().removeOnlineVotes(uuid); - for (OfflineBungeeVote pending : retained) { - getVoteCacheHandler().addOnlineVote(uuid, pending); - } - - // multiproxy: envelope-only - if (deliveredReward && getConfig().getMultiProxySupport() - && getConfig().getMultiProxyOneGlobalReward()) { - multiProxyHandler.sendClearVote(uuid, player); - } - } - } - } - } - - /** - * Retries voter-keyed standalone broadcasts when any player makes a target - * backend available as a plugin-message carrier. - * - * @param server backend server that gained a carrier - */ - protected synchronized void retryPendingOnlineBroadcasts(String server) { - List blockedServers = getConfig().getBlockedServers(); - if (server == null || (blockedServers != null && blockedServers.contains(server))) { - return; - } - for (String cachedUuid : getVoteCacheHandler().getOnlineVoteUUIDs()) { - for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { - if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { - continue; - } - if (!cache.isProxyBroadcastHandled() || !cache.needsBroadcastOn(server)) { - continue; - } - Set forwarded = sendProxyBroadcast(Collections.singleton(server), cache.getUuid(), - cache.getPlayerName(), cache.getService(), cache.getTime(), cache.getText(), false); - if (cache.getBroadcastForwardedServers().addAll(forwarded)) { - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { - getVoteCacheHandler().removeOnlineVote(cachedUuid, cache); - } else { - persistOnlineVoteDelivery(cachedUuid, cache); - } - } - } - } - } - - protected synchronized void retryPendingTimeBroadcasts(String server) { - List blockedServers = getConfig().getBlockedServers(); - if (server == null || (blockedServers != null && blockedServers.contains(server))) { - return; - } - if (getVoteCacheHandler().getTimeChangeQueue() == null) { - return; - } - for (VoteTimeQueue vote : new ArrayList<>(getVoteCacheHandler().getTimeChangeQueue())) { - if (vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { - continue; - } - if (!vote.isProxyBroadcastHandled() || vote.getUuid().isEmpty() || !vote.getBroadcastTargets().contains(server) - || vote.getBroadcastForwardedServers().contains(server)) { - continue; - } - Set forwarded = sendProxyBroadcast(Collections.singleton(server), vote.getUuid(), vote.getName(), - vote.getService(), vote.getTime(), vote.getTotals(), false); - if (vote.getBroadcastForwardedServers().addAll(forwarded)) { - persistTimeVoteDelivery(vote); - } - } - } - - /** - * Periodically retries every pending voter-keyed standalone broadcast. This is - * required for broker transports whose recovery does not produce a player-login - * carrier event. - */ - public synchronized void retryPendingOnlineBroadcasts() { - for (String cachedUuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { - for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { - if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { - continue; - } - if (!cache.isProxyBroadcastHandled() || cache.isProxyBroadcastComplete()) { - continue; - } - Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); - pendingTargets.removeAll(cache.getBroadcastForwardedServers()); - List blockedServers = getConfig().getBlockedServers(); - if (blockedServers != null) { - pendingTargets.removeAll(blockedServers); - } - Set forwarded = sendProxyBroadcast(pendingTargets, cache.getUuid(), cache.getPlayerName(), - cache.getService(), cache.getTime(), cache.getText(), false); - if (cache.getBroadcastForwardedServers().addAll(forwarded)) { - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { - getVoteCacheHandler().removeOnlineVote(cachedUuid, cache); - } else { - persistOnlineVoteDelivery(cachedUuid, cache); - } - } - } - } - retryPendingTimeBroadcasts(); - } - - public synchronized void retryPendingTimeBroadcasts() { - if (getVoteCacheHandler().getTimeChangeQueue() == null) { - return; - } - for (VoteTimeQueue vote : new ArrayList<>(getVoteCacheHandler().getTimeChangeQueue())) { - if (vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { - continue; - } - if (!vote.isProxyBroadcastHandled() || vote.getUuid().isEmpty()) { - continue; - } - Set pendingTargets = new LinkedHashSet<>(vote.getBroadcastTargets()); - pendingTargets.removeAll(vote.getBroadcastForwardedServers()); - List blockedServers = getConfig().getBlockedServers(); - if (blockedServers != null) { - pendingTargets.removeAll(blockedServers); - } - Set forwarded = sendProxyBroadcast(pendingTargets, vote.getUuid(), vote.getName(), vote.getService(), - vote.getTime(), vote.getTotals(), false); - if (vote.getBroadcastForwardedServers().addAll(forwarded)) { - persistTimeVoteDelivery(vote); - } - } - } - - protected synchronized boolean persistTimeVoteDelivery(VoteTimeQueue vote) { - if (getVoteCacheHandler().updateTimeVote(vote)) { - vote.setDeliveryStateDirty(false); - return true; - } - vote.setDeliveryStateDirty(true); - scheduleTimeVoteDeliveryRetry(); - return false; - } - - private void scheduleTimeVoteDeliveryRetry() { - if (timeVoteDeliveryRetryScheduled || getScheduler() == null) { - return; - } - timeVoteDeliveryRetryScheduled = true; - try { - getScheduler().schedule(() -> { - synchronized (VotingPluginProxy.this) { - timeVoteDeliveryRetryScheduled = false; - } - retryPendingTimeBroadcasts(); - }, 5, TimeUnit.SECONDS); - } catch (RuntimeException e) { - timeVoteDeliveryRetryScheduled = false; - debug("Unable to schedule timed broadcast state retry: " + e.getMessage()); - } - } - - protected synchronized boolean persistServerVoteDelivery(String server, OfflineBungeeVote vote) { - if (getVoteCacheHandler().updateServerVote(server, vote)) { - vote.setDeliveryStateDirty(false); - return true; - } - vote.setDeliveryStateDirty(true); - scheduleCachedVoteDeliveryRetry(); - return false; - } - - protected synchronized boolean persistOnlineVoteDelivery(String uuid, OfflineBungeeVote vote) { - if (getVoteCacheHandler().updateOnlineVote(uuid, vote)) { - vote.setDeliveryStateDirty(false); - return true; - } - vote.setDeliveryStateDirty(true); - scheduleCachedVoteDeliveryRetry(); - return false; - } - - private void scheduleCachedVoteDeliveryRetry() { - if (cachedVoteDeliveryRetryScheduled || getScheduler() == null) { - return; - } - cachedVoteDeliveryRetryScheduled = true; - try { - getScheduler().schedule(() -> { - synchronized (VotingPluginProxy.this) { - cachedVoteDeliveryRetryScheduled = false; - } - retryCachedVoteDeliveryPersistence(); - }, 5, TimeUnit.SECONDS); - } catch (RuntimeException e) { - cachedVoteDeliveryRetryScheduled = false; - debug("Unable to schedule cached broadcast state retry: " + e.getMessage()); - } - } - - private synchronized void retryCachedVoteDeliveryPersistence() { - for (String server : getVoteCacheHandler().getCachedVotesServers()) { - for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getVotes(server))) { - if (vote.isDeliveryStateDirty()) { - persistServerVoteDelivery(server, vote); - } - } - } - for (String uuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { - for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(uuid))) { - if (vote.isDeliveryStateDirty()) { - persistOnlineVoteDelivery(uuid, vote); - } - } - } - } - - public void checkVoteParty() { - if (getConfig().getVotePartyEnabled()) { - if (votePartyVotes >= currentVotePartyVotesRequired) { - debug("Vote party reached"); - addCurrentVotePartyVotes(-currentVotePartyVotesRequired); - - currentVotePartyVotesRequired += getConfig().getVotePartyIncreaseVotesRequired(); - setVoteCacheVotePartyIncreaseVotesRequired( - getVoteCacheVotePartyIncreaseVotesRequired() + getConfig().getVotePartyIncreaseVotesRequired()); - - if (!getConfig().getVotePartyBroadcast().isEmpty()) { - broadcast(getConfig().getVotePartyBroadcast()); - } - - for (String command : getConfig().getVotePartyBungeeCommands()) { - runConsoleCommand(command); - } - - if (getConfig().getVotePartySendToAllServers()) { - for (String server : getAllAvailableServers()) { - sendVoteParty(server); - } - } else { - for (String server : getConfig().getVotePartyServersToSend()) { - sendVoteParty(server); - } - } - } - saveVoteCacheFile(); - } - } - - public abstract void debug(String str); - - private void debug2(String message) { - debug(message); - } - - /** - * HTTP client used for Mojang API requests. - */ - private final HttpClient httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); - - /** - * Fetches a player's UUID from the Mojang API. - * - * @param playerName player name - * @return player UUID, or {@code null} if not found - * @throws IOException if the request fails - * @throws InterruptedException if interrupted while waiting for the response - */ - public UUID fetchUUID(String playerName) throws IOException, InterruptedException { - if (playerName == null || playerName.equalsIgnoreCase("null")) { - return null; - } - - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create("https://api.mojang.com/users/profiles/minecraft/" + playerName)).GET() - .timeout(Duration.ofSeconds(5)).build(); - - HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - - if (response.statusCode() == 400 || response.statusCode() == 404) { - log("There is no player with the name \"" + playerName + "\"!"); - return null; - } - - if (response.statusCode() < 200 || response.statusCode() >= 300) { - throw new IOException("Failed to fetch UUID for " + playerName + ", HTTP " + response.statusCode()); - } - - JsonElement element = JsonParser.parseString(response.body()); - if (element == null || !element.isJsonObject()) { - return null; - } - - JsonObject object = element.getAsJsonObject(); - if (!object.has("id") || object.get("id").isJsonNull()) { - return null; - } - - String uuidAsString = object.get("id").getAsString(); - return parseUUIDFromString(uuidAsString); - } - - public abstract Set getAllAvailableServers(); - - /** Complete platform server set before whitelist/blocked routing filters. */ - public abstract Set getAllConfiguredServers(); - - public abstract VotingPluginProxyConfig getConfig(); - - public abstract String getCurrentPlayerServer(String player); - - /** - * Resolves a player's server for vote routing. A dedicated voting proxy has no - * local players, so it uses the backend presence tracker instead. - */ - protected String getCurrentPlayerServerForVoteRouting(String player) { - if (isDedicatedVotingProxyEnabled()) { - return backendPlayerPresenceTracker.getPlayer(player).map(presence -> presence.getServer()).orElse(null); - } - return getCurrentPlayerServer(player); - } - - /** - * Dedicated routing is intentionally unavailable on plugin messaging: that - * transport is attached to a player-facing proxy and does not carry backend - * presence snapshots. - */ - protected boolean isDedicatedVotingProxyEnabled() { - return getConfig().getDedicatedVotingProxy() && method != null && method.supportsBackendPresence(); - } - - public abstract File getDataFolderPlugin(); - - public String getMonthTotalsWithDatePath() { - LocalDateTime cTime = getBungeeTimeChecker().getTime(); - return getMonthTotalsWithDatePath(cTime); - } - - public String getMonthTotalsWithDatePath(LocalDateTime cTime) { - return "MonthTotal-" + cTime.getMonth().toString() + "-" + cTime.getYear(); - } - - public abstract String getProperName(String uuid, String playerName); - - public abstract String getUUID(String playerName); - - private int getValue(ArrayList cols, String column, int toAdd) { - for (Column d : cols) { - if (d.getName().equalsIgnoreCase(column)) { - DataValue value = d.getValue(); - int num = 0; - if (value == null) { - return toAdd; - } - if (value.isInt()) { - num = value.getInt(); - } else if (value.isString()) { - try { - num = Integer.parseInt(value.getString()); - } catch (Exception e) { - // ignore - } - } - return num + toAdd; - } - } - return toAdd; - } - - private VoteTotalsSnapshot getProjectedRolloverTotals(ArrayList data, String player) { - List timeChanges = getGlobalDataHandler().getTimeChanges(); - boolean resetMonth = timeChanges.contains(TimeType.MONTH); - boolean resetWeek = timeChanges.contains(TimeType.WEEK); - boolean resetDay = timeChanges.contains(TimeType.DAY); - int acceptedQueuedVotes = 0; - int acceptedGlobalQueuedVotes = 0; - for (VoteTimeQueue queued : getVoteCacheHandler().getTimeChangeQueue()) { - if (!queued.isProcessed()) { - acceptedGlobalQueuedVotes++; - } - if (!queued.isProcessed() && queued.getName() != null && queued.getName().equalsIgnoreCase(player)) { - acceptedQueuedVotes++; - } - } - int voteIncrement = acceptedQueuedVotes + 1; - - int allTimeTotal = getValue(data, "AllTimeTotal", voteIncrement); - int monthTotal = resetMonth ? voteIncrement : getValue(data, "MonthTotal", voteIncrement); - int weeklyTotal = resetWeek ? voteIncrement : getValue(data, "WeeklyTotal", voteIncrement); - int dailyTotal = resetDay ? voteIncrement : getValue(data, "DailyTotal", voteIncrement); - int points = getValue(data, "Points", voteIncrement * getConfig().getPointsOnVote()); - - int maxVotes = getConfig().getMaxAmountOfVotesPerDay(); - if (maxVotes > 0) { - int days = getBungeeTimeChecker().getTime().getDayOfMonth(); - if (monthTotal > days * maxVotes) { - monthTotal = days * maxVotes; - } - } - if (getConfig().getLimitVotePoints() > 0 && points > getConfig().getLimitVotePoints()) { - points = getConfig().getLimitVotePoints(); - } - - int dateMonthTotal = -1; - if (getConfig().getStoreMonthTotalsWithDate()) { - if (getConfig().getUseMonthDateTotalsAsPrimaryTotal()) { - dateMonthTotal = resetMonth ? voteIncrement - : getValue(data, getMonthTotalsWithDatePath(), voteIncrement); - } else { - dateMonthTotal = monthTotal; - } - } - - int[] projectedVoteParty = getProjectedVotePartyState(acceptedGlobalQueuedVotes + 1); - return new VoteTotalsSnapshot(allTimeTotal, monthTotal, weeklyTotal, dailyTotal, points, - projectedVoteParty[0], projectedVoteParty[1], dateMonthTotal); - } - - protected boolean canForwardStandaloneBroadcast(boolean managesTotals) { - return managesTotals; - } - - protected int[] getProjectedVotePartyState(int acceptedVotes) { - int current = votePartyVotes; - int required = currentVotePartyVotesRequired; - if (!getConfig().getVotePartyEnabled()) { - return new int[] { current, required }; - } - - int increase = getConfig().getVotePartyIncreaseVotesRequired(); - for (int i = 0; i < acceptedVotes; i++) { - current++; - if (current >= required) { - current -= required; - required += increase; - } - } - return new int[] { current, required }; - } - - public abstract String getPluginVersion(); - - public abstract int getVoteCacheCurrentVotePartyVotes(); - - public abstract long getVoteCacheLastUpdated(); - - public abstract int getVoteCachePrevDay(); - - public abstract String getVoteCachePrevMonth(); - - public abstract int getVoteCachePrevWeek(); - - public abstract int getVoteCacheVotePartyIncreaseVotesRequired(); - - public abstract boolean isPlayerOnline(String playerName); - - /** - * Checks online state for vote routing, using backend presence only when this - * proxy is explicitly configured as the dedicated voting proxy. - */ - protected boolean isPlayerOnlineForVoteRouting(String playerName) { - return isDedicatedVotingProxyEnabled() ? backendPlayerPresenceTracker.getPlayer(playerName).isPresent() - : isPlayerOnline(playerName); - } - - public abstract boolean isServerValid(String server); - - public abstract boolean isSomeoneOnlineServer(String server); - - protected boolean isSomeoneOnlineServerForVoteRouting(String server) { - if (!isDedicatedVotingProxyEnabled()) { - return isSomeoneOnlineServer(server); - } - com.bencodez.votingplugin.proxy.presence.BackendPresenceStatus status = backendPlayerPresenceTracker - .getBackendStatus(server); - return status != null && status.isAvailable() && status.getPlayerCount() > 0; - } - - public abstract boolean isVoteCacheIgnoreTime(); - - public abstract MysqlConfig getVoteCacheMySQLConfig(); - - public abstract MysqlConfig getNonVotedCacheMySQLConfig(); - - public abstract MysqlConfig getVoteLoggingMySQLConfig(); - - /** - * Shutdown MySQL-related resources safely. - */ - public void shutdownMySql() { - if (getProxyMysqlMessenger() != null) { - getProxyMysqlMessenger().shutdown(); - setProxyMysqlMessenger(null); - } - - if (getProxyMySQL() != null) { - getProxyMySQL().shutdown(); - setProxyMySQL(null); - } - } - - public void load(IVoteCache jsonStorage, INonVotedPlayersStorage nonVotedCacheJson) { - method = BungeeMethod.getByName(getConfig().getBungeeMethod()); - if (getMethod() == null) { - method = BungeeMethod.PLUGINMESSAGING; - } - warnUnsupportedDedicatedVotingProxyMode(); - uuidPlayerNameCache = getProxyMySQL().getRowsUUIDNameQuery(); - - bungeeTimeChecker.setTimeChangeFailSafeBypass(getConfig().getTimeChangeFailSafeBypass()); - bungeeTimeChecker.loadTimer(); - - voteCacheHandler = new VoteCacheHandler(getVoteCacheMySQLConfig(), getConfig().getVoteCacheUseMySQL(), - getConfig().getVoteCacheUseMainMySQL(), getProxyMySQL().getMysql(), getConfig().getDebug(), - jsonStorage) { - - @Override - public void logInfo1(String msg) { - logInfo(msg); - } - - @Override - public void logSevere1(String msg) { - logSevere(msg); - } - - @Override - public void debug1(Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void debug1(String msg) { - if (getConfig().getDebug()) { - debug(msg); - } - } - - @Override - public void debug1(Throwable e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - }; - voteCacheHandler.load(); - - nonVotedPlayersCache = new NonVotedPlayersCache(getNonVotedCacheMySQLConfig(), - getConfig().getNonVotedCacheUseMySQL(), getConfig().getNonVotedCacheUseMainMySQL(), - getProxyMySQL().getMysql(), nonVotedCacheJson, getConfig().getDebug()) { - - @Override - public boolean userExists(String uuid) { - return getProxyMySQL().containsKeyQuery(uuid); - } - - @Override - public void logInfo1(String msg) { - logInfo(msg); - } - - @Override - public void logSevere1(String msg) { - logSevere(msg); - } - - @Override - public void debug1(Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void debug1(String msg) { - if (getConfig().getDebug()) { - debug(msg); - } - } - - @Override - public Set getAllUUIDs() { - return getProxyMySQL().getUuids(); - } - }; - - if (method.equals(BungeeMethod.MYSQL)) { - try { - proxyMysqlMessenger = new MySqlMessenger("VotingPlugin", - getProxyMySQL().getMysql().getConnectionManager().getDataSource(), MySqlMessenger.Mode.PROXY, - null, // no serverId in PROXY mode - msg -> { - if (getConfig().getDebug()) { - debug("Got from " + msg.source + ": " + msg.envelope.getSubChannel() + " " - + msg.envelope.getFields()); - } - globalMessageProxyHandler.onMessage(msg.envelope); - }); - } catch (SQLException e) { - e.printStackTrace(); - } - } else if (method.equals(BungeeMethod.PLUGINMESSAGING)) { - if (getConfig().getPluginMessageEncryption()) { - encryptionHandler = new EncryptionHandler("VotingPlugin", - new File(getDataFolderPlugin(), "secretkey.key")); - } - } else if (method.equals(BungeeMethod.SOCKETS)) { - encryptionHandler = new EncryptionHandler("VotingPlugin", new File(getDataFolderPlugin(), "secretkey.key")); - - socketHandler = new SocketHandler(getPluginVersion(), getConfig().getBungeeHost(), - getConfig().getBungeePort(), encryptionHandler, getConfig().getDebug()) { - - @Override - public void log(String str) { - logInfo(str); - } - }; - - socketHandler.add(new SocketReceiver() { - @Override - public void onReceiveEnvelope(JsonEnvelope envelope) { - globalMessageProxyHandler.onMessage(envelope); - } - }); - - rebuildSocketClients(); - } else if (method.equals(BungeeMethod.HTTP)) { - startHttpTransport(); - } else if (method.equals(BungeeMethod.REDIS)) { - redisHandler = new RedisHandler(getConfig().getRedisHost(), getConfig().getRedisPort(), - getConfig().getRedisUsername(), getConfig().getRedisPassword(), getConfig().getRedisDbIndex(), - getConfig().getRedisSsl()) { - - @Override - public void debug(String message) { - debug2(message); - } - }; - redisPublisherPool = new JedisPool(new HostAndPort(getConfig().getRedisHost(), getConfig().getRedisPort()), - buildRedisClientConfig(getConfig())); - - runAsync(() -> { - RedisListener listener = redisHandler.createEnvelopeListener( - getConfig().getRedisPrefix() + "VotingPlugin", - (ch, env) -> globalMessageProxyHandler.onMessage(env)); - redisHandler.loadListener(listener); - }); - - } else if (method.equals(BungeeMethod.MQTT)) { - try { - mqttHandler = new MqttHandler(new MqttServerComm(getConfig().getMqttClientID(), - getConfig().getMqttBrokerURL(), getConfig().getMqttUsername(), getConfig().getMqttPassword()), - 2); - - mqttHandler.subscribeEnvelopes(getConfig().getMqttPrefix() + "votingplugin/servers/proxy", - (topic, env) -> globalMessageProxyHandler.onMessage(env)); - - } catch (MqttException e) { - e.printStackTrace(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - currentVotePartyVotesRequired = getConfig().getVotePartyVotesRequired() - + getVoteCacheVotePartyIncreaseVotesRequired(); - votePartyVotes = getVoteCacheCurrentVotePartyVotes(); - - globalMessageProxyHandler = new GlobalMessageProxyHandler() { - @Override - public void sendMessage(String server, int delay, JsonEnvelope envelope) { - switch (method) { - case MQTT: - sendMqttEnvelopeServer(server, envelope); - break; - case MYSQL: - try { - proxyMysqlMessenger.sendToBackend(server, envelope); - } catch (SQLException e) { - e.printStackTrace(); - } - break; - case PLUGINMESSAGING: - sendPluginMessageServer(server, delay, envelope); - break; - case REDIS: - sendRedisEnvelopeServer(server, envelope); - break; - case SOCKETS: - sendSocketEnvelope(server, envelope); - break; - case HTTP: - sendHttpEnvelope(server, envelope); - break; - default: - break; - } - } - }; - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_LOGIN) { - @Override - public void onReceive(JsonEnvelope message) { - handleLoginMessage(message); - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_LOGOUT) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(message); - if (!isPresenceServerValid(event.server, VotingPluginWire.SUB_LOGOUT) - || !isPresenceGenerationValid(event.backendIncarnationId, event.backendStartedAt, - event.presenceTimestamp, - VotingPluginWire.SUB_LOGOUT)) { - return; - } - if (!backendPlayerPresenceTracker.playerOffline(event.uuid, event.server, event.connectionId, - event.backendIncarnationId, event.backendStartedAt, event.presenceTimestamp, - System.currentTimeMillis())) { - debug("Ignored invalid or stale logout envelope: " + message.getFields()); - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_STARTED) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); - long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); - long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); - if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_STARTED) - && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, - VotingPluginWire.SUB_BACKEND_STARTED)) { - if (backendPlayerPresenceTracker.backendStarted(server, backendIncarnationId, backendStartedAt, - presenceTimestamp, System.currentTimeMillis())) { - discardPendingPresenceHandoffs(server); - pendingBackendRecoverySnapshots.add(presenceServerKey(server)); - requestBackendPresenceSnapshot(server); - } - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_STOPPED) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); - long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); - long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); - if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_STOPPED) - && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, - VotingPluginWire.SUB_BACKEND_STOPPED)) { - if (backendPlayerPresenceTracker.backendStopped(server, backendIncarnationId, backendStartedAt, - presenceTimestamp, System.currentTimeMillis())) { - discardPendingPresenceHandoffs(server); - pendingBackendRecoverySnapshots.remove(presenceServerKey(server)); - } - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_HEARTBEAT) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); - long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); - long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); - if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_HEARTBEAT) - && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, - VotingPluginWire.SUB_BACKEND_HEARTBEAT)) { - backendPlayerPresenceTracker.heartbeat(server, backendIncarnationId, backendStartedAt, - presenceTimestamp, System.currentTimeMillis()); - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_PRESENCE_SNAPSHOT) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - if (!isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_SNAPSHOT)) { - return; - } - VotingPluginWire.PresenceSnapshot snapshot = VotingPluginWire.readPresenceSnapshot(message); - long now = System.currentTimeMillis(); - boolean accepted = snapshot.valid - && isPresenceGenerationValid(snapshot.backendIncarnationId, snapshot.backendStartedAt, - snapshot.presenceTimestamp, - VotingPluginWire.SUB_PRESENCE_SNAPSHOT) - && backendPlayerPresenceTracker.applySnapshotChunk(snapshot.server, - snapshot.requestId, snapshot.chunkIndex, snapshot.chunkCount, snapshot.players, - snapshot.backendIncarnationId, snapshot.backendStartedAt, - snapshot.presenceTimestamp, now); - if (!accepted) { - debug("Ignored invalid or unexpected presence snapshot from " + snapshot.server); - if (backendPlayerPresenceTracker.getPendingSnapshotRequestId(snapshot.server, now) == null) { - discardPendingPresenceHandoffs(snapshot.requestId); - } - } else if (backendPlayerPresenceTracker.getPendingSnapshotRequestId(snapshot.server, now) == null) { - pendingBackendRecoverySnapshots.remove(presenceServerKey(snapshot.server)); - Set handoffPlayers = completePendingPresenceHandoffs(snapshot.requestId, snapshot.server, - snapshot.backendIncarnationId, snapshot.backendStartedAt, now); - processDedicatedSnapshotLogins(snapshot.server, handoffPlayers); - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_STATUS_OKAY) { - @Override - public void onReceive(JsonEnvelope message) { - handleStatusOkay(message); - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener("voteupdate") { - @Override - public void onReceive(JsonEnvelope message) { - int delay = 1; - for (String send : getAllAvailableServers()) { - globalMessageProxyHandler.sendMessage(send, delay, message); - delay++; - } - } - }); - - proxyBroadcastDecider = new ProxyBroadcastDecider(() -> getConfig(), () -> getAllAvailableServers(), - s -> isServerValid(s), - s -> getConfig().getBlockedServers() != null && getConfig().getBlockedServers().contains(s)); - - loadMultiProxySupport(); - loadVoteLoggingMySQL(); - if (method.supportsBackendPresence()) { - scheduleBackendPresenceStartupResync(); - loadTaskTimer(this::maintainBackendPresence, PRESENCE_MAINTENANCE_INTERVAL_SECONDS, - PRESENCE_MAINTENANCE_INTERVAL_SECONDS); - } - startControlServices(); - - debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); - } - - private void startControlServices() { - synchronized (controlLifecycleLock) { - ControlConnector predecessor = controlConnector; - if (predecessor != null && predecessor.deferReplacementUntilSafe(this::restartControlServicesAsync)) { - log("[Control] service restart deferred until the current result is acknowledged"); - return; - } - stopControlServicesLocked(true); - startControlServicesLocked(); - } - } - - /** Keeps potentially long hosted-Control handoffs off proxy command/event threads. */ - private void restartControlServicesAsync() { - final long generation = controlServicesGeneration.incrementAndGet(); - try { - controlLifecycleExecutor.execute(() -> { - try { - synchronized (controlLifecycleLock) { - if (!enabled || generation != controlServicesGeneration.get()) return; - ControlConnector predecessor = controlConnector; - if (predecessor != null - && predecessor.deferReplacementUntilSafe(this::restartControlServicesAsync)) { - log("[Control] service restart deferred until the current result is acknowledged"); - return; - } - stopControlServicesLocked(true); - startControlServicesLocked(); - } - } catch (RuntimeException failure) { - if (generation == controlServicesGeneration.get()) { - logSevere("[Control] asynchronous service restart failed: " + failure.getMessage()); - } - } - }); - } catch (RuntimeException failure) { - logSevere("[Control] services were not restarted because async scheduling failed"); - } - } - - /** Rebuilds a recovery connector from current settings after its durable result is acknowledged. */ - public final void restartControlServicesAfterRecovery() { - restartControlServicesAsync(); - } - - private void stopControlServices(boolean waitForHosted) { - synchronized (controlLifecycleLock) { - stopControlServicesLocked(waitForHosted); - } - } - - private void startControlServicesLocked() { - if (getConfig().getControlHostedEnabled()) { - try { - hostedControlManager = HostedControlManager.create(this); - if (hostedControlManager != null) hostedControlManager.start(); - } catch (IOException | IllegalArgumentException e) { - hostedControlManager = null; - logSevere("[Control Host] configuration or automatic enrollment is invalid; VotingPlugin remains unaffected"); - } - } - try { - controlConnector = ControlConnector.create(this); - if (controlConnector != null) controlConnector.start(); - } catch (IOException | IllegalArgumentException e) { - controlConnector = null; - logSevere("[Control] connector configuration or credential is invalid; voting remains unaffected"); - } - } - - private void stopControlServicesLocked(boolean waitForHosted) { - ControlConnector connector = controlConnector; - if (connector != null) { - try { - connector.close(); - if (controlConnector == connector) controlConnector = null; - } catch (RuntimeException failure) { - if (waitForHosted) throw failure; - if (controlConnector == connector) controlConnector = null; - logSevere("[Control] connector did not stop cleanly; proxy cleanup will continue"); - } - } - HostedControlManager manager = hostedControlManager; - if (manager != null) { - try { - if (waitForHosted) { - manager.closeAndWait(); - } else { - manager.close(); - } - if (hostedControlManager == manager) hostedControlManager = null; - } catch (RuntimeException failure) { - if (waitForHosted) throw failure; - if (hostedControlManager == manager) hostedControlManager = null; - logSevere("[Control Host] manager did not stop cleanly; proxy cleanup will continue"); - } - } - } - - public String getControlConnectorStatus() { - ControlConnector connector = controlConnector; - return connector == null ? "DISABLED" : connector.status().name(); - } - - public String getHostedControlStatus() { - HostedControlManager manager = hostedControlManager; - return manager == null ? "DISABLED" : manager.status().name(); - } - - /** - * Handles both the original login notification and extended presence logins. - * Kept protected so transport-policy behavior can be regression tested without - * initializing a live proxy transport. - * - * @param message login envelope - */ - protected void handleLoginMessage(JsonEnvelope message) { - VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(message); - String player = event.player; - String uuid = event.uuid; - String server = event.server; - - if (player.isEmpty() || uuid.isEmpty()) { - logSevere("Invalid login envelope received: " + message.getFields()); - return; - } - boolean legacy = event.connectionId == null && event.backendIncarnationId == null - && event.backendStartedAt == 0L && event.presenceTimestamp == 0L; - boolean accepted = false; - String deliveryServer = server; - if (legacy) { - if (method == BungeeMethod.PLUGINMESSAGING) { - String proxyServer = getCurrentPlayerServer(player); - accepted = isLegacyLoginDestinationAuthoritative(player, uuid, proxyServer); - if (accepted) { - deliveryServer = proxyServer; - } - } else if (method != null && method.supportsBackendPresence() - && isPresenceServerValid(server, VotingPluginWire.SUB_LOGIN)) { - accepted = isLegacyLoginDestinationAuthoritative(player, uuid, server); - } - } else if (method != null && method.supportsBackendPresence() && event.connectionId != null - && isPresenceServerValid(server, VotingPluginWire.SUB_LOGIN) - && isPresenceGenerationValid(event.backendIncarnationId, event.backendStartedAt, - event.presenceTimestamp, VotingPluginWire.SUB_LOGIN)) { - BackendPlayerPresenceTracker.PlayerOnlineResult result = backendPlayerPresenceTracker.playerOnlineResult( - player, uuid, server, event.connectionId, - event.backendIncarnationId, event.backendStartedAt, event.presenceTimestamp, - System.currentTimeMillis()); - accepted = result.isAccepted(); - if (result.isConflictingPresence()) { - requestBackendPresenceSnapshot(server, - new PendingPresenceHandoff(player, uuid, server, event.connectionId, - event.backendIncarnationId, event.backendStartedAt, - result.getConflictSequence(), System.currentTimeMillis())); - } - } - - debug("Login: " + player + "/" + uuid + " " + server); - if (accepted) { - discardPendingPresenceHandoff(uuid); - login(player, uuid, deliveryServer); - } else { - debug("Ignored invalid or stale login envelope: " + message.getFields()); - } - } - - /** - * Validates a legacy login against an authority independent of the envelope. - * Player-facing proxies use their native live route and UUID. A dedicated - * voting proxy has no native player session, so it requires an exact modern - * presence match for the claimed destination. - */ - private boolean isLegacyLoginDestinationAuthoritative(String player, String uuid, String server) { - if (server == null || server.isBlank()) { - return false; - } - - UUID claimedUuid; - try { - claimedUuid = UUID.fromString(uuid.trim()); - } catch (RuntimeException e) { - return false; - } - - if (isDedicatedVotingProxyEnabled()) { - PlayerPresence presence = backendPlayerPresenceTracker.getPlayer(player).orElse(null); - return presence != null && presence.getServer().equalsIgnoreCase(server) - && (!getConfig().getOnlineMode() || presence.getUuid().equals(claimedUuid)); - } - - if (!isPlayerOnline(player)) { - return false; - } - String proxyServer = getCurrentPlayerServer(player); - if (proxyServer == null || !proxyServer.equalsIgnoreCase(server)) { - return false; - } - if (!getConfig().getOnlineMode()) { - return true; - } - - String authoritativeUuid = getUUID(player); - if (authoritativeUuid == null || authoritativeUuid.isBlank()) { - return false; - } - try { - return claimedUuid.equals(UUID.fromString(authoritativeUuid.trim())); - } catch (IllegalArgumentException e) { - return false; - } - } - - private VoteLogMysqlTable voteLogMysqlTable; - - @Getter - private ProxyBroadcastDecider proxyBroadcastDecider; - - public void loadVoteLoggingMySQL() { - if (getConfig().getVoteLoggingEnabled()) { - if (getConfig().getVoteLoggingUseMainMySQL()) { - voteLogMysqlTable = new VoteLogMysqlTable("votingplugin_votelog", getProxyMySQL().getMysql(), - getVoteLoggingMySQLConfig(), getConfig().getDebug()) { - - @Override - public void logSevere(String string) { - VotingPluginProxy.this.logSevere(string); - } - - @Override - public void logInfo(String string) { - VotingPluginProxy.this.logInfo(string); - } - - @Override - public void debug(Throwable e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public String getServerName() { - return "Proxy"; - } - }; - } else { - voteLogMysqlTable = new VoteLogMysqlTable("votingplugin_votelog", getVoteLoggingMySQLConfig(), - getConfig().getDebug()) { - - @Override - public void logSevere(String string) { - VotingPluginProxy.this.logSevere(string); - } - - @Override - public void logInfo(String string) { - VotingPluginProxy.this.logInfo(string); - } - - @Override - public void debug(Throwable e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public String getServerName() { - return "Proxy"; - } - }; - } - - if (getConfig().getVoteLoggingPurgeDays() > 0) { - loadTaskTimer(() -> voteLogMysqlTable.purgeOlderThanDays(getConfig().getVoteLoggingPurgeDays(), 100), - 60, 60 * 60); - } - - debug("Vote logging MySQL enabled"); - } else { - debug("Vote logging MySQL disabled"); - } - } - - public abstract void loadTaskTimer(Runnable runnable, long delaySeconds, long repeatSeconds); - - public void loadMultiProxySupport() { - if (multiProxyHandler != null) { - multiProxyHandler.close(); - } - multiProxyHandler = new MultiProxyHandler() { - - @Override - public void addNonVotedPlayerCache(String uuid, String player) { - addNonVotedPlayer(uuid, player); - } - - @Override - public void clearVote(String uuid) { - getVoteCacheHandler().clearOnlineVoteRewards(uuid); - } - - @Override - public boolean getDebug() { - return getConfig().getDebug(); - } - - @Override - public EncryptionHandler getEncryptionHandler() { - return encryptionHandler; - } - - @Override - public MultiProxyMethod getMultiProxyMethod() { - return MultiProxyMethod.getByName(getConfig().getMultiProxyMethod()); - } - - @Override - public String getMultiProxyPassword() { - return getConfig().getMultiProxyRedisPassword(); - } - - @Override - public String getMultiProxyRedisHost() { - return getConfig().getMultiProxyRedisHost(); - } - - @Override - public int getMultiProxyRedisPort() { - return getConfig().getMultiProxyRedisPort(); - } - - @Override - public boolean getMultiProxyRedisSsl() { - return getConfig().getMultiProxyRedisSsl(); - } - - @Override - public int getMultiProxyRedisDbIndex() { - return getConfig().getMultiProxyRedisDbIndex(); - } - - @Override - public boolean getMultiProxyRedisUseExistingConnection() { - return getConfig().getMultiProxyRedisUseExistingConnection(); - } - - @Override - public String getMultiProxyServerName() { - return getConfig().getProxyServerName(); - } - - @Override - public Collection getMultiProxyServers() { - return getConfig().getMultiProxyServers(); - } - - @Override - public MultiProxyServerSocketConfiguration getMultiProxyServersConfiguration(String s) { - return new MultiProxyServerSocketConfigurationBungee(s, - getConfig().getMultiProxyServersConfiguration(s)); - } - - @Override - public String getMultiProxySocketHostHost() { - return getConfig().getMultiProxySocketHostHost(); - } - - @Override - public int getMultiProxySocketHostPort() { - return getConfig().getMultiProxySocketHostPort(); - } - - @Override - public boolean getMultiProxySupportEnabled() { - return getConfig().getMultiProxySupport(); - } - - @Override - public String getMultiProxyUsername() { - return getConfig().getMultiProxyRedisUsername(); - } - - @Override - public File getPluginDataFolder() { - return getDataFolderPlugin(); - } - - @Override - public boolean getPrimaryServer() { - return getConfig().getPrimaryServer(); - } - - @Override - public List getProxyServers() { - return getConfig().getProxyServers(); - } - - @Override - public RedisHandler getRedisHandler() { - return redisHandler; - } - - @Override - public String getVersion() { - return getPluginVersion(); - } - - @Override - public void logInfo(String msg) { - log(msg); - } - - @Override - public void runAsnc(Runnable runnable) { - runAsync(runnable); - } - - @Override - public void setEncryptionHandler(EncryptionHandler encryptionHandler1) { - encryptionHandler = encryptionHandler1; - } - - @Override - public void triggerVote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, - VoteTotalsSnapshot text, String uuid) { - vote(player, service, realVote, timeQueue, queueTime, text, uuid); - } - }; - multiProxyHandler.loadMultiProxySupport(); - } - - public abstract void log(String message); - - /** - * Requests a complete player-presence snapshot from one backend server. - * - * @param server configured backend server name - * @return new or already-active request identifier, or null when the server is - * invalid or is inside the snapshot-request cooldown - */ - public UUID requestBackendPresenceSnapshot(String server) { - return requestBackendPresenceSnapshot(server, null); - } - - private UUID requestBackendPresenceSnapshot(String server, PendingPresenceHandoff handoff) { - return requestBackendPresenceSnapshot(server, handoff, System.currentTimeMillis(), false); - } - - private UUID requestBackendPresenceSnapshot(String server, PendingPresenceHandoff handoff, long now, - boolean handoffAlreadyQueued) { - if (method == null || !method.supportsBackendPresence() || globalMessageProxyHandler == null - || !isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_SNAPSHOT_REQUEST)) { - return null; - } - long backendStartedAt = backendPlayerPresenceTracker.getBackendStartedAt(server); - UUID backendIncarnationId = backendPlayerPresenceTracker.getBackendIncarnationId(server); - if (backendStartedAt <= 0L || backendIncarnationId == null) { - return null; - } - if (handoff != null && (!server.equalsIgnoreCase(handoff.server) - || !backendIncarnationId.equals(handoff.backendIncarnationId) - || backendStartedAt != handoff.backendStartedAt)) { - return null; - } - if (handoff != null && (handoffAlreadyQueued ? !isPendingPresenceHandoff(handoff, now) - : !queuePendingPresenceHandoff(handoff, now))) { - return null; - } - UUID requestId = handoff == null - ? backendPlayerPresenceTracker.beginSnapshot(server, UUID.randomUUID(), backendIncarnationId, - backendStartedAt, now) - : backendPlayerPresenceTracker.beginSnapshotForDestinationClaim(server, UUID.randomUUID(), - backendIncarnationId, backendStartedAt, handoff.playerUuid, handoff.conflictSequence, now); - boolean created = requestId != null; - if (!created) { - requestId = handoff == null ? backendPlayerPresenceTracker.getPendingSnapshotRequestId(server, now) - : backendPlayerPresenceTracker.getPendingSnapshotRequestIdForDestinationClaim(server, - handoff.playerUuid, handoff.conflictSequence, now); - } - if (requestId == null) { - if (handoff != null && !backendPlayerPresenceTracker.isCurrentDestinationClaim(handoff.playerUuid, - handoff.server, handoff.conflictSequence)) { - discardPendingPresenceHandoff(handoff); - } - // A handoff stays unassigned while the destination is inside its snapshot - // cooldown. Presence maintenance will attach it to the next allowed snapshot. - return null; - } - if (handoff != null) { - assignPendingPresenceHandoff(handoff, requestId, now); - } - if (created) { - JsonEnvelope request = VotingPluginWire.presenceSnapshotRequest(server, requestId, backendIncarnationId, - backendStartedAt, now); - globalMessageProxyHandler.sendMessage(server, 1, request); - } - return requestId; - } - - private boolean queuePendingPresenceHandoff(PendingPresenceHandoff handoff, long now) { - if (!isPresenceHandoffValid(handoff, now)) { - return false; - } - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - PendingPresenceHandoff current = pendingPresenceHandoffs.get(handoff.playerUuid); - if (current != null && current.conflictSequence > handoff.conflictSequence) { - return false; - } - handoff.requestId = null; - pendingPresenceHandoffs.put(handoff.playerUuid, handoff); - return true; - } - } - - private boolean isPendingPresenceHandoff(PendingPresenceHandoff handoff, long now) { - if (!isPresenceHandoffValid(handoff, now)) { - return false; - } - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - return pendingPresenceHandoffs.get(handoff.playerUuid) == handoff; - } - } - - private void assignPendingPresenceHandoff(PendingPresenceHandoff handoff, UUID requestId, long now) { - if (requestId == null || !isPresenceHandoffValid(handoff, now)) { - return; - } - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - if (pendingPresenceHandoffs.get(handoff.playerUuid) == handoff) { - handoff.requestId = requestId; - } - } - } - - private boolean isPresenceHandoffValid(PendingPresenceHandoff handoff, long now) { - return handoff != null && handoff.playerUuid != null && handoff.connectionId != null - && handoff.conflictSequence > 0L - && now >= handoff.createdAt && now - handoff.createdAt <= PRESENCE_HANDOFF_TIMEOUT_MILLIS; - } - - private Set completePendingPresenceHandoffs(UUID requestId, String server, UUID backendIncarnationId, - long backendStartedAt, long now) { - List completed = new ArrayList<>(); - Set completedPlayers = new LinkedHashSet<>(); - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - pendingPresenceHandoffs.entrySet().removeIf(entry -> { - PendingPresenceHandoff handoff = entry.getValue(); - if (!requestId.equals(handoff.requestId)) { - return false; - } - if (handoff.server.equalsIgnoreCase(server) - && handoff.backendIncarnationId.equals(backendIncarnationId) - && handoff.backendStartedAt == backendStartedAt) { - completed.add(handoff); - } - return true; - }); - } - for (PendingPresenceHandoff handoff : completed) { - PlayerPresence presence = backendPlayerPresenceTracker.getPlayer(handoff.playerUuid).orElse(null); - if (presence != null && presence.getServer().equalsIgnoreCase(handoff.server) - && presence.getConnectionId().equals(handoff.connectionId)) { - login(handoff.playerName, handoff.uuid, handoff.server); - completedPlayers.add(handoff.playerUuid); - } - releaseDestinationClaim(handoff); - } - return completedPlayers; - } - - /** - * Drains voter-keyed cached rewards when a complete recovery snapshot first - * confirms a player on a dedicated voting proxy. Cross-backend handoffs are - * already processed by their token-bound completion path and are excluded to - * avoid a second login callback. - */ - protected void processDedicatedSnapshotLogins(String server, Set handoffPlayers) { - if (!isDedicatedVotingProxyEnabled() || server == null || server.isBlank()) { - return; - } - Set excluded = handoffPlayers == null ? Collections.emptySet() : handoffPlayers; - for (PlayerPresence presence : backendPlayerPresenceTracker.getOnlinePlayers()) { - if (presence.getServer().equalsIgnoreCase(server) && !excluded.contains(presence.getUuid())) { - login(presence.getPlayerName(), presence.getUuid().toString(), presence.getServer()); - } - } - } - - private void discardPendingPresenceHandoff(String uuid) { - try { - UUID playerUuid = UUID.fromString(uuid.trim()); - PendingPresenceHandoff removed; - synchronized (pendingPresenceHandoffs) { - removed = pendingPresenceHandoffs.remove(playerUuid); - } - releaseDestinationClaim(removed); - } catch (Exception ignored) { - // Invalid identities are rejected by the presence tracker. - } - } - - private void discardPendingPresenceHandoff(PendingPresenceHandoff handoff) { - boolean removed = false; - synchronized (pendingPresenceHandoffs) { - if (handoff != null && pendingPresenceHandoffs.get(handoff.playerUuid) == handoff) { - pendingPresenceHandoffs.remove(handoff.playerUuid); - removed = true; - } - } - if (removed) { - releaseDestinationClaim(handoff); - } - } - - private void discardPendingPresenceHandoffs(String server) { - synchronized (pendingPresenceHandoffs) { - pendingPresenceHandoffs.entrySet().removeIf(entry -> { - if (!entry.getValue().server.equalsIgnoreCase(server)) { - return false; - } - releaseDestinationClaim(entry.getValue()); - return true; - }); - } - } - - private void discardPendingPresenceHandoffs(UUID requestId) { - if (requestId == null) { - return; - } - synchronized (pendingPresenceHandoffs) { - pendingPresenceHandoffs.entrySet().removeIf(entry -> { - if (!requestId.equals(entry.getValue().requestId)) { - return false; - } - releaseDestinationClaim(entry.getValue()); - return true; - }); - } - } - - private void prunePendingPresenceHandoffs(long now) { - pendingPresenceHandoffs.entrySet().removeIf(entry -> { - PendingPresenceHandoff handoff = entry.getValue(); - if (now >= handoff.createdAt && now - handoff.createdAt <= PRESENCE_HANDOFF_TIMEOUT_MILLIS) { - return false; - } - releaseDestinationClaim(handoff); - return true; - }); - } - - private void releaseDestinationClaim(PendingPresenceHandoff handoff) { - if (handoff != null) { - backendPlayerPresenceTracker.releaseDestinationClaim(handoff.playerUuid, handoff.server, - handoff.conflictSequence); - } - } - - protected void retryPendingPresenceHandoffs(long now) { - List retry = new ArrayList<>(); - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - for (PendingPresenceHandoff handoff : pendingPresenceHandoffs.values()) { - UUID activeRequestId = backendPlayerPresenceTracker.getPendingSnapshotRequestId(handoff.server, now); - if (handoff.requestId != null && !handoff.requestId.equals(activeRequestId)) { - handoff.requestId = null; - } - if (handoff.requestId == null) { - retry.add(handoff); - } - } - } - for (PendingPresenceHandoff handoff : retry) { - requestBackendPresenceSnapshot(handoff.server, handoff, now, true); - } - } - - protected int getPendingPresenceHandoffCount() { - synchronized (pendingPresenceHandoffs) { - return pendingPresenceHandoffs.size(); - } - } - - protected void scheduleBackendPresenceStartupResync() { - ScheduledExecutorService scheduler = getScheduler(); - if (method == null || !method.supportsBackendPresence() || scheduler == null) { - return; - } - scheduler.schedule(this::requestBackendPresenceStartupResync, - PRESENCE_STARTUP_RESYNC_DELAY_SECONDS, TimeUnit.SECONDS); - } - - protected void requestBackendPresenceStartupResync() { - if (!enabled || method == null || !method.supportsBackendPresence() || globalMessageProxyHandler == null) { - return; - } - long requestedAt = System.currentTimeMillis(); - int delay = 1; - for (String server : getAllAvailableServers()) { - if (!isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_RESYNC_REQUEST)) { - continue; - } - globalMessageProxyHandler.sendMessage(server, delay++, - VotingPluginWire.presenceResyncRequest(server, UUID.randomUUID(), requestedAt)); - } - } - - private void maintainBackendPresence() { - if (!enabled || method == null || !method.supportsBackendPresence()) { - return; - } - expireBackendPresence(PRESENCE_BACKEND_TIMEOUT_MILLIS); - for (String server : getAllAvailableServers()) { - if (pendingBackendRecoverySnapshots.contains(presenceServerKey(server))) { - requestBackendPresenceSnapshot(server); - } - } - retryPendingPresenceHandoffs(System.currentTimeMillis()); - } - - private String presenceServerKey(String server) { - return server == null ? "" : server.trim().toLowerCase(java.util.Locale.ROOT); - } - - private boolean isPresenceServerValid(String server, String subChannel) { - // The presence protocol's trust boundary is the configured backend set. The - // selected transport must only be accessible to backend servers trusted not to - // impersonate one another. - if (server == null || server.isBlank() || !isServerValid(server)) { - debug("Ignored " + subChannel + " presence envelope for an unconfigured server"); - return false; - } - return true; - } - - private boolean isPresenceGenerationValid(UUID backendIncarnationId, long backendStartedAt, - long presenceTimestamp, String subChannel) { - if (backendIncarnationId == null || backendStartedAt <= 0L || presenceTimestamp < backendStartedAt) { - debug("Ignored " + subChannel + " presence envelope with an invalid backend generation"); - return false; - } - return true; - } - - /** - * Removes presence owned by backends that have stopped reporting heartbeats. - * Scheduling and timeout configuration are intentionally left to dedicated - * proxy mode. - * - * @param timeoutMillis maximum backend silence before expiry - * @return expired backend server names - */ - public Set expireBackendPresence(long timeoutMillis) { - if (method == null || !method.supportsBackendPresence()) { - return Collections.emptySet(); - } - long now = System.currentTimeMillis(); - Set expired = backendPlayerPresenceTracker.expireBackends(now, timeoutMillis); - for (String server : expired) { - discardPendingPresenceHandoffs(server); - // Keep recovery pending while this generation is unavailable. If the same - // backend process resumes, its heartbeat can mark it available again and the - // maintenance task will request a fresh snapshot of players who stayed online. - pendingBackendRecoverySnapshots.add(presenceServerKey(server)); - } - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - } - return expired; - } - - public void login(String playerName, String uuid, String serverName) { - if (!getConfig().getOnlineMode()) { - uuid = getUUID(playerName); - } - - try { - if (uuid != null && !uuid.isEmpty() && !uuid.equalsIgnoreCase("null")) { - uuid = UUID.fromString(uuid.trim()).toString(); - } - } catch (Exception ignored) { - // ignore - } - - if (getConfig().getOnlineMode()) { - addNonVotedPlayer(uuid, playerName); - } - if (isPlayerOnlineForVoteRouting(playerName)) { - if (getConfig().getGlobalDataEnabled()) { - if (getGlobalDataHandler().isTimeChangedHappened()) { - getGlobalDataHandler().checkForFinishedTimeChanges(); - } - } - - checkCachedVotes(serverName); - retryPendingOnlineBroadcasts(serverName); - retryPendingTimeBroadcasts(serverName); - checkOnlineVotes(playerName, uuid, serverName); - multiProxyHandler.login(uuid, playerName); - } - } - - private void logInfo(String msg) { - log(msg); - } - - public abstract void logSevere(String message); - - public void onDisable() { - onDisable(false); - } - - /** Full runtime replacement waits for hosted workers; final proxy stop remains non-blocking. */ - public void onDisable(boolean waitForHosted) { - if (waitForHosted) { - prepareForRuntimeReplacement(); - } else { - controlServicesGeneration.incrementAndGet(); - controlLifecycleExecutor.shutdownNow(); - stopControlServices(false); - } - completeRuntimeReplacementShutdown(); - } - - /** Fail-closed gate that must complete before a replacement proxy runtime is created. */ - public void prepareForRuntimeReplacement() { - controlServicesGeneration.incrementAndGet(); - synchronized (controlLifecycleLock) { - ControlConnector connector = controlConnector; - if (connector != null && !connector.reserveRuntimeReplacement()) { - throw new IllegalStateException("Control result must be acknowledged before proxy runtime replacement"); - } - controlLifecycleExecutor.shutdown(); - stopControlServicesLocked(true); - } - } - - /** Best-effort remainder of runtime teardown after the Control overlap gate has succeeded. */ - public void completeRuntimeReplacementShutdown() { - cancelCommunicationTests("Proxy runtime stopped before the backend replied"); - runCleanup("vote cache", () -> getVoteCacheHandler().saveVoteCache()); - runCleanup("proxy MySQL messenger", () -> { - if (getProxyMysqlMessenger() != null) getProxyMysqlMessenger().shutdown(); - }); - runCleanup("proxy MySQL", () -> { - if (getProxyMySQL() != null) getProxyMySQL().shutdown(); - }); - runCleanup("multi-proxy handler", () -> { - if (multiProxyHandler != null) multiProxyHandler.close(); - }); - runCleanup("socket listener", () -> { - if (socketHandler != null) socketHandler.closeConnection(); - }); - runCleanup("socket clients", this::closeSocketClients); - runCleanup("HTTP transport", this::closeHttpTransport); - runCleanup("Redis subscriber", () -> { - if (redisHandler != null) redisHandler.close(); - }); - runCleanup("Redis publisher", () -> { - JedisPool pool = redisPublisherPool; - try { - if (pool != null) pool.close(); - } finally { - if (redisPublisherPool == pool) redisPublisherPool = null; - } - }); - runCleanup("MQTT transport", () -> { - if (mqttHandler != null) mqttHandler.disconnect(); - }); - runCleanup("time checker", () -> bungeeTimeChecker.shutdown()); - runCleanup("global data", () -> { - if (getGlobalDataHandler() != null) getGlobalDataHandler().shutdown(); - }); - enabled = false; - } - - private void runCleanup(String service, CleanupAction cleanup) { - try { - cleanup.run(); - } catch (Exception failure) { - logSevere("Unable to stop " + service + "; remaining proxy cleanup will continue"); - } - } - - @FunctionalInterface - private interface CleanupAction { void run() throws Exception; } - - public void onPluginMessageReceived(DataInputStream in) { - onPluginMessageReceived(in, null); - } - - /** Receives a plugin message bound to the backend server connection that sent it. */ - public void onPluginMessageReceived(DataInputStream in, String sourceServer) { - runAsync(() -> { - try { - final String headerSub; - if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { - headerSub = encryptionHandler.decrypt(in.readUTF()); - } else { - headerSub = in.readUTF(); - } - - int size = in.readInt(); // sanity only - - if (getConfig().getDebug()) { - debug("Received plugin message header=" + headerSub + " size=" + size); - } - - String payload = ""; - if (size > 0) { - if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { - payload = encryptionHandler.decrypt(in.readUTF()); - } else { - payload = in.readUTF(); - } - } - - JsonEnvelope envelope = JsonEnvelopeCodec.decode(payload); - - if (!headerSub.equalsIgnoreCase(envelope.getSubChannel())) { - if (getConfig().getDebug()) { - warn("PluginMessage subChannel mismatch: header=" + headerSub + " env=" - + envelope.getSubChannel()); - } - return; - } - - if (VotingPluginWire.SUB_CONTROL_ENROLLMENT_REQUEST.equals(envelope.getSubChannel())) { - handleControlEnrollmentRequest(sourceServer, envelope); - return; - } - - globalMessageProxyHandler.onMessage(envelope); - } catch (Exception e) { - e.printStackTrace(); - } - }); - } - - private void handleControlEnrollmentRequest(String sourceServer, JsonEnvelope envelope) { - VotingPluginWire.ControlEnrollmentRequest request = VotingPluginWire.readControlEnrollmentRequest(envelope); - if (!request.valid || sourceServer == null || sourceServer.isBlank()) return; - if (!sourceServer.equals(request.nodeId)) { - sendPluginMessageServer(sourceServer, 0, - VotingPluginWire.controlEnrollmentResult(sourceServer, request.requestId, false)); - return; - } - long now = System.nanoTime(); - AtomicBoolean allowed = new AtomicBoolean(); - controlEnrollmentNextAllowed.compute(sourceServer, (ignored, nextAllowed) -> { - if (nextAllowed == null || now >= nextAllowed) { - allowed.set(true); - return now + CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS; - } - return nextAllowed; - }); - if (!allowed.get()) return; - HostedControlManager manager = hostedControlManager; - if (manager == null) { - sendPluginMessageServer(sourceServer, 0, - VotingPluginWire.controlEnrollmentResult(sourceServer, request.requestId, false)); - return; - } - manager.installNodeVerifier(sourceServer, request.verifier, request.endpoint).whenComplete((installed, failure) -> { - boolean success = failure == null && Boolean.TRUE.equals(installed); - sendPluginMessageServer(sourceServer, 0, - VotingPluginWire.controlEnrollmentResult(sourceServer, request.requestId, success)); - if (success) log("[Control] automatically enrolled backend node " + sourceServer); - }); - } - - private UUID parseUUIDFromString(String uuidAsString) { - String[] parts = { "0x" + uuidAsString.substring(0, 8), "0x" + uuidAsString.substring(8, 12), - "0x" + uuidAsString.substring(12, 16), "0x" + uuidAsString.substring(16, 20), - "0x" + uuidAsString.substring(20, 32) }; - - long mostSigBits = Long.decode(parts[0]).longValue(); - mostSigBits <<= 16; - mostSigBits |= Long.decode(parts[1]).longValue(); - mostSigBits <<= 16; - mostSigBits |= Long.decode(parts[2]).longValue(); - - long leastSigBits = Long.decode(parts[3]).longValue(); - leastSigBits <<= 48; - leastSigBits |= Long.decode(parts[4]).longValue(); - - return new UUID(mostSigBits, leastSigBits); - } - - public synchronized void processQueue() { - while (getVoteCacheHandler().getTimeChangeQueue().size() > 0) { - VoteTimeQueue vote = getVoteCacheHandler().getTimeChangeQueue().element(); - if (!vote.isProcessed()) { - VoteTotalsSnapshot queuedTotals = vote.getTotals() == null || vote.getTotals().isEmpty() ? null - : VoteTotalsSnapshot.parseStorage(vote.getTotals()); - QueuedVoteResult result = vote(vote.getName(), vote.getService(), true, false, vote.getTime(), queuedTotals, - vote.getUuid(), vote); - if (result == QueuedVoteResult.RETRY) { - scheduleTimeVoteRetry(); - return; - } - if (result == QueuedVoteResult.TERMINAL) { - warn("Removing terminal rollover vote " + vote.getVoteId() + " for " + vote.getName() + "/" - + ServiceSiteValidator.sanitizeForLog(vote.getService())); - } - } - if (!getVoteCacheHandler().removeTimeVote(vote)) { - scheduleTimeVoteRetry(); - return; - } - } - } - - private void scheduleTimeVoteRetry() { - if (timeVoteRetryScheduled || getScheduler() == null) { - return; - } - timeVoteRetryScheduled = true; - try { - getScheduler().schedule(() -> { - synchronized (VotingPluginProxy.this) { - timeVoteRetryScheduled = false; - } - processQueue(); - }, 5, TimeUnit.SECONDS); - } catch (RuntimeException e) { - timeVoteRetryScheduled = false; - debug("Unable to schedule rollover vote retry: " + e.getMessage()); - } - } - - public void reload() { - reloadRuntime(true); - } - - /** Applies a Control-originated configuration reload without stopping its connector or hosted service. */ - public void reloadFromControl() { - reloadRuntime(false); - } - - private void reloadRuntime(boolean restartControlServices) { - method = BungeeMethod.getByName(getConfig().getBungeeMethod()); - if (getMethod() == null) { - method = BungeeMethod.PLUGINMESSAGING; - } - warnUnsupportedDedicatedVotingProxyMode(); - if (!restartControlServices && method == BungeeMethod.SOCKETS) { - rebuildSocketClients(); - } - - setCurrentVotePartyVotesRequired( - getConfig().getVotePartyVotesRequired() + getVoteCacheVotePartyIncreaseVotesRequired()); - if (restartControlServices) { - loadMultiProxySupport(); - restartControlServicesAsync(); - } - } - - private synchronized void rebuildSocketClients() { - HashMap rebuilt = new HashMap<>(); - try { - List blocked = getConfig().getBlockedServers(); - for (String server : getConfig().getSpigotServers()) { - if (blocked.contains(server)) continue; - Map data = getConfig().getSpigotServerConfiguration(server); - String host = data.containsKey("Host") ? (String) data.get("Host") : ""; - int port = data.containsKey("Port") ? (int) data.get("Port") : 1298; - rebuilt.put(server, new ClientHandler(host, port, encryptionHandler, getConfig().getDebug())); - } - } catch (RuntimeException failure) { - stopSocketClients(rebuilt); - throw failure; - } - HashMap previous = clientHandles; - clientHandles = rebuilt; - stopSocketClients(previous); - } - - private synchronized boolean sendSocketEnvelope(String server, JsonEnvelope envelope) { - ClientHandler socketClient = clientHandles == null ? null : clientHandles.get(server); - if (socketClient == null) return false; - try { - socketClient.sendEnvelope(envelope); - return true; - } catch (RuntimeException e) { - debug(e.getMessage()); - return false; - } - } - - private synchronized boolean sendHttpEnvelope(String server, JsonEnvelope envelope) { - HttpProxyTransportServer transport = httpTransportServer; - return transport != null && transport.send(server, envelope); - } - - private void startHttpTransport() { - try { - URI endpoint = URI.create(getConfig().getHttpPublicEndpoint()); - if (!"https".equalsIgnoreCase(endpoint.getScheme()) || endpoint.getHost() == null - || endpoint.getPort() == 0 || endpoint.getPort() > 65535 - || endpoint.getUserInfo() != null || endpoint.getQuery() != null || endpoint.getFragment() != null - || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) { - throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); - } - File directory = new File(getDataFolderPlugin(), "http"); - HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.toPath(), endpoint.getHost()); - httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); - httpTransportServer = new HttpProxyTransportServer( - new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, - httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope); - httpTransportServer.start(); - logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" - + httpTransportServer.port() + "; use /votingpluginbungee httpcode for each backend"); - } catch (Exception failure) { - closeHttpTransport(); - throw new IllegalStateException("HTTP transport could not start securely", failure); - } - } - - /** Keeps the authenticated mTLS backend identity attached to security-sensitive proxy routing. */ - protected void handleHttpTransportEnvelope(HttpProxyTransportServer.ReceivedEnvelope received) { - if (!isAuthenticatedHttpEnvelopeAllowed(received)) { - debug("Ignored HTTP envelope whose player-presence claim did not match its authenticated backend"); - return; - } - GlobalMessageProxyHandler handler = globalMessageProxyHandler; - if (handler == null) throw new IllegalStateException("HTTP message router is not ready"); - handler.onMessage(received.envelope()); - } - - private boolean isAuthenticatedHttpEnvelopeAllowed(HttpProxyTransportServer.ReceivedEnvelope received) { - if (received == null || received.envelope() == null || received.serverId() == null) return false; - String stampedServer = received.envelope().getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - if (!received.serverId().equalsIgnoreCase(stampedServer)) return false; - if (!VotingPluginWire.SUB_LOGIN.equals(received.envelope().getSubChannel())) return true; - VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(received.envelope()); - boolean modern = event.connectionId != null || event.backendIncarnationId != null - || event.backendStartedAt != 0L || event.presenceTimestamp != 0L; - if (!modern || isDedicatedVotingProxyEnabled()) return true; - // A player-facing proxy has a stronger authority than any backend: its live - // player connection supplies both the current route and (in online mode) UUID. - return isLegacyLoginDestinationAuthoritative(event.player, event.uuid, received.serverId()); - } - - private synchronized void closeHttpTransport() { - HttpProxyTransportServer transport = httpTransportServer; - httpTransportServer = null; - httpEnrollmentAuthority = null; - if (transport != null) transport.close(); - } - - public String createHttpConnectionCode(String serverId) { - HttpEnrollmentAuthority authority = httpEnrollmentAuthority; - if (method != BungeeMethod.HTTP || authority == null) { - throw new IllegalStateException("The HTTP transport is not running"); - } - return authority.createConnectionCode(serverId, URI.create(getConfig().getHttpPublicEndpoint()), Duration.ofMinutes(15)) - .encode(); - } - - public void revokeHttpBackend(String serverId) { - HttpEnrollmentAuthority authority = httpEnrollmentAuthority; - if (method != BungeeMethod.HTTP || authority == null) throw new IllegalStateException("The HTTP transport is not running"); - authority.revoke(HttpTlsIdentity.canonicalServerId(serverId)); - } - - private synchronized void closeSocketClients() { - HashMap clients = clientHandles; - clientHandles = null; - stopSocketClients(clients); - } - - static void stopSocketClients(Map clients) { - if (clients == null) return; - for (ClientHandler client : clients.values()) { - if (client == null) continue; - try { - client.stopConnection(); - } catch (RuntimeException ignored) { - // Best effort: one broken client must not prevent the remaining sockets from closing. - } - } - } - - private void warnUnsupportedDedicatedVotingProxyMode() { - if (getConfig().getDedicatedVotingProxy() && (method == null || !method.supportsBackendPresence())) { - logSevere("DedicatedVotingProxy requires MYSQL, REDIS, MQTT, SOCKETS, or HTTP; PLUGINMESSAGING is disabled for " - + "dedicated-proxy routing. Falling back to normal proxy routing."); - } - } - - public abstract void runAsync(Runnable run); - - /** Platform name used only for the transport-neutral Control discovery contract. */ - public abstract String getProxyPlatform(); - - public abstract void runConsoleCommand(String command); - - public abstract void saveVoteCacheFile(); - - public abstract void reloadCore(boolean mysql); - - /** Strict Control reload path; failures propagate so the caller can restore its backup. */ - public abstract void reloadControlConfiguration() throws Exception; - - public abstract boolean sendPluginMessageData(String server, String channel, byte[] data, boolean queue); - - private static final int PLUGIN_MESSAGE_HARD_LIMIT = 32767; - private static final int PLUGIN_MESSAGE_SOFT_LIMIT = 30000; - - public void sendPluginMessageServer(String server, int delay, JsonEnvelope envelope) { - getScheduler().schedule(() -> sendPluginMessageServerNow(server, envelope), delay * 5L, TimeUnit.MILLISECONDS); - } - - /** - * Sends a plugin-message envelope immediately and reports whether the proxy - * accepted it for delivery. - * - * @param server target backend server - * @param envelope envelope to send - * @return true when the proxy accepted the message for delivery - */ - protected boolean sendPluginMessageServerNow(String server, JsonEnvelope envelope) { - final String subChannel = envelope.getSubChannel(); - final String payload = JsonEnvelopeCodec.encode(envelope); - - final byte[] subChannelBytes = subChannel.getBytes(java.nio.charset.StandardCharsets.UTF_8); - final byte[] payloadBytes = payload.getBytes(java.nio.charset.StandardCharsets.UTF_8); - - // Estimate bytes written: - // - writeUTF adds 2-byte length prefix + UTF-8 bytes - // - writeInt is 4 bytes - int estimatedSize = 2 + subChannelBytes.length + // subChannel UTF (len prefix + bytes) - 4 + // payload length int - 2 + payloadBytes.length; // payload UTF (len prefix + bytes) - - if (estimatedSize > PLUGIN_MESSAGE_SOFT_LIMIT) { - debug("[PluginMessage] Payload nearing limit (" + estimatedSize + " bytes) server=" + server - + " subChannel=" + subChannel + " — consider Redis instead"); - } - - if (estimatedSize > PLUGIN_MESSAGE_HARD_LIMIT) { - debug("[PluginMessage] Payload TOO LARGE (" + estimatedSize + " bytes, max=" + PLUGIN_MESSAGE_HARD_LIMIT - + ") server=" + server + " subChannel=" + subChannel + " — NOT sent"); - return false; - } - - try (ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(); - DataOutputStream out = new DataOutputStream(byteOutStream)) { - if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { - out.writeUTF(encryptionHandler.encrypt(subChannel)); - } else { - out.writeUTF(subChannel); - } - - // sanity only: MUST be bytes, not chars - out.writeInt(payloadBytes.length); - - if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { - out.writeUTF(encryptionHandler.encrypt(payload)); - } else { - out.writeUTF(payload); - } - out.flush(); - - boolean sent = sendPluginMessageData(server, getConfig().getPluginMessageChannel().toLowerCase(), - byteOutStream.toByteArray(), false); - if (getConfig().getDebug()) { - debug((sent ? "Sent" : "Could not send") + " plugin envelope (" + estimatedSize + " bytes) " + server - + " " + subChannel + " " + envelope.getFields()); - } - return sent; - } catch (Exception e) { - e.printStackTrace(); - return false; - } - } - - static DefaultJedisClientConfig buildRedisClientConfig(VotingPluginProxyConfig configSource) { - DefaultJedisClientConfig.Builder config = DefaultJedisClientConfig.builder() - .database(configSource.getRedisDbIndex()).ssl(configSource.getRedisSsl()).connectionTimeoutMillis(2000) - .socketTimeoutMillis(2000); - if (configSource.getRedisSsl()) { - SSLParameters sslParameters = new SSLParameters(); - sslParameters.setEndpointIdentificationAlgorithm("HTTPS"); - config.sslParameters(sslParameters); - } - if (configSource.getRedisUsername() != null && !configSource.getRedisUsername().isEmpty()) { - config.user(configSource.getRedisUsername()); - } - if (configSource.getRedisPassword() != null && !configSource.getRedisPassword().isEmpty()) { - config.password(configSource.getRedisPassword()); - } - return config.build(); - } - - public boolean sendRedisEnvelopeServer(String server, JsonEnvelope envelope) { - return sendRedisEnvelopeServer(server, envelope, false); - } - - private boolean sendRedisEnvelopeServer(String server, JsonEnvelope envelope, boolean useRetryCooldown) { - JedisPool publisherPool = redisPublisherPool; - if (publisherPool == null || (useRetryCooldown && System.currentTimeMillis() < redisPublisherRetryAfter)) { - return false; - } - - try (Jedis jedis = publisherPool.getResource()) { - String channel = getConfig().getRedisPrefix() + "VotingPlugin_" + server; - long subscribers = jedis.publish(channel, - JsonEnvelopeCodec.encode(VotingPluginWire.withRedisDeliveryId(envelope))); - redisPublisherRetryAfter = 0L; - return subscribers > 0; - } catch (Exception e) { - if (useRetryCooldown) { - // Standalone broadcasts remain queued, so their retries can be throttled safely. - redisPublisherRetryAfter = System.currentTimeMillis() + 2000L; - } - debug(e.getMessage()); - return false; - } - } - - public boolean sendMqttEnvelopeServer(String server, JsonEnvelope envelope) { - if (mqttHandler == null) { - return false; - } - try { - mqttHandler.publishEnvelope(getConfig().getMqttPrefix() + "votingplugin/servers/" + server, envelope); - return true; - } catch (Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - return false; - } - } - - public boolean sendSocketEnvelopeServer(String server, JsonEnvelope envelope) { - Map configuration = getConfig().getSpigotServerConfiguration(server); - if (configuration == null) { - return false; - } - String host = configuration.get("Host") instanceof String ? (String) configuration.get("Host") : ""; - int port = configuration.get("Port") instanceof Number ? ((Number) configuration.get("Port")).intValue() : 1298; - if (host.isEmpty()) { - return false; - } - - String payload = JsonEnvelopeCodec.encode(envelope); - String encoded = encryptionHandler != null ? encryptionHandler.encrypt(payload) : payload; - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(host, port), 2000); - try (DataOutputStream output = new DataOutputStream(socket.getOutputStream())) { - output.writeUTF(encoded); - output.flush(); - } - return true; - } catch (Exception e) { - debug(e.getMessage()); - return false; - } - } - - public void sendServerNameMessage() { - for (String s : getAllAvailableServers()) { - sendPluginMessageServer(s, 1, VotingPluginWire.serverName(s)); - } - } - - public void sendVoteParty(String server) { - if (isSomeoneOnlineServerForVoteRouting(server)) { - globalMessageProxyHandler.sendMessage(server, 1, VotingPluginWire.votePartyBungee()); - } - } - - public void setCurrentVotePartyVotes(int amount) { - votePartyVotes = amount; - setVoteCacheVotePartyCurrentVotes(amount); - debug("Current vote party total: " + votePartyVotes); - } - - public abstract void setVoteCacheLastUpdated(); - - public abstract void setVoteCachePrevDay(int day); - - public abstract void setVoteCachePrevMonth(String text); - - public abstract void setVoteCachePrevWeek(int week); - - public abstract void setVoteCacheVoteCacheIgnoreTime(boolean ignore); - - public abstract void setVoteCacheVotePartyCurrentVotes(int votes); - - public abstract void setVoteCacheVotePartyIncreaseVotesRequired(int votes); - - public void status() { - for (String s : getAllAvailableServers()) { - if (!isSomeoneOnlineServerForVoteRouting(s)) { - log("No players on server " + s + " to send test status message, please retest with someone online"); - } else { - log("Sending request for status message on " + s); - globalMessageProxyHandler.sendMessage(s, 1, VotingPluginWire.status(s)); - } - } - } - - /** Runs a correlated, non-vote round trip over the active backend transport. */ - public CompletableFuture testBackendCommunication(String requestedServer, - long timeoutMillis) { - String server = requestedServer == null ? "" : requestedServer.trim(); - BungeeMethod activeMethod = method; - if (server.isEmpty() || !getAllAvailableServers().contains(server)) { - return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, - "UNKNOWN_BACKEND", "The backend is not configured on this proxy")); - } - if (activeMethod == null || globalMessageProxyHandler == null) { - return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, - "TRANSPORT_UNAVAILABLE", "The proxy communication transport is not running")); - } - if (activeMethod == BungeeMethod.PLUGINMESSAGING && !isSomeoneOnlineServerForVoteRouting(server)) { - return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, - "PLAYER_REQUIRED", "Plugin messaging requires an online player on the selected backend")); - } - ScheduledExecutorService scheduler = getScheduler(); - if (scheduler == null) { - return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, - "TRANSPORT_UNAVAILABLE", "The proxy scheduler is not running")); - } - long boundedTimeout = Math.max(500L, Math.min(timeoutMillis, 30000L)); - UUID requestId = UUID.randomUUID(); - CompletableFuture result = new CompletableFuture<>(); - PendingCommunicationTest pending = new PendingCommunicationTest(server, activeMethod, System.nanoTime(), result); - pendingCommunicationTests.put(requestId, pending); - result.whenComplete((ignored, failure) -> pendingCommunicationTests.remove(requestId, pending)); - try { - if (!sendCommunicationTestEnvelopeNow(server, VotingPluginWire.status(server, requestId))) { - result.complete(CommunicationTestResult.failure(server, activeMethod, "TRANSPORT_UNAVAILABLE", - "The active transport could not accept the communication test")); - return result; - } - scheduler.schedule(() -> result.complete(CommunicationTestResult.failure(server, activeMethod, - "TIMEOUT", "No correlated reply arrived before the timeout")), boundedTimeout, TimeUnit.MILLISECONDS); - } catch (RuntimeException failure) { - result.complete(CommunicationTestResult.failure(server, activeMethod, "SEND_FAILED", - "The proxy could not send the communication test")); - } - return result; - } - - /** Sends a diagnostic immediately and reports whether the active transport accepted it. */ - protected boolean sendCommunicationTestEnvelopeNow(String server, JsonEnvelope envelope) { - return sendProxyBroadcastEnvelopeNow(server, envelope); - } - - protected void handleStatusOkay(JsonEnvelope message) { - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - String request = message.getFields().getOrDefault(VotingPluginWire.K_REQUEST_ID, ""); - if (request.isEmpty()) { - log("Status okay for " + server); - return; - } - UUID requestId; - try { - requestId = UUID.fromString(request); - } catch (IllegalArgumentException ignored) { - debug("Ignored status reply with an invalid request ID from " + server); - return; - } - PendingCommunicationTest pending = pendingCommunicationTests.get(requestId); - if (pending == null || !pending.server().equals(server)) { - debug("Ignored unexpected status reply from " + server); - return; - } - long roundTripMillis = Math.max(0L, - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - pending.startedAtNanos())); - pending.result().complete(CommunicationTestResult.success(server, pending.method(), roundTripMillis)); - } - - private void cancelCommunicationTests(String message) { - pendingCommunicationTests.forEach((requestId, pending) -> pending.result().complete( - CommunicationTestResult.failure(pending.server(), pending.method(), "TRANSPORT_STOPPED", message))); - pendingCommunicationTests.clear(); - } - - public record CommunicationTestResult(boolean success, String code, String message, String server, - String method, long roundTripMillis) { - private static CommunicationTestResult success(String server, BungeeMethod method, long roundTripMillis) { - return new CommunicationTestResult(true, "OK", "Backend replied over the active transport", server, - method == null ? "" : method.name(), roundTripMillis); - } - - private static CommunicationTestResult failure(String server, BungeeMethod method, String code, String message) { - return new CommunicationTestResult(false, code, message, server, - method == null ? "" : method.name(), -1L); - } - } - - private record PendingCommunicationTest(String server, BungeeMethod method, long startedAtNanos, - CompletableFuture result) { } - - private void sendVoteDelayRejected(String player, String uuid, String service, boolean playerOnline, - String playerServer) { - if (!playerOnline || playerServer == null || !getAllAvailableServers().contains(playerServer)) { - debug("Not sending vote delay rejection for " + player + " because the player is offline"); - return; - } - - globalMessageProxyHandler.sendMessage(playerServer, 1, - VotingPluginWire.voteDelayRejected(player, uuid, service, true)); - } - - public String getWaitUntilDelaySiteFromService(String service) { - for (String site : getConfig().getWaitUntilVoteDelaySites()) { - if (getConfig().getWaitUntilVoteDelayService(site).equalsIgnoreCase(service)) { - return site; - } - } - return ""; - } - - private long getLastVotesTime(String uuid, ArrayList cols, String site, String service, String player, - boolean includeTimeChangeQueue) { - long mostRecentTime = 0; - - if (getVoteCacheHandler().hasOnlineVotes(uuid)) { - ArrayList onlineVotes = getVoteCacheHandler().getOnlineVotes(uuid); - for (OfflineBungeeVote vote : onlineVotes) { - if (vote.getService().equalsIgnoreCase(service)) { - mostRecentTime = Math.max(mostRecentTime, vote.getTime()); - } - } - } - - for (String server : getAllAvailableServers()) { - for (OfflineBungeeVote vote : getVoteCacheHandler().getVotes(server)) { - if (vote.getUuid().equals(uuid) && vote.getService().equalsIgnoreCase(service)) { - mostRecentTime = Math.max(mostRecentTime, vote.getTime()); - } - } - } - - if (includeTimeChangeQueue && player != null) { - for (VoteTimeQueue queuedVote : getVoteCacheHandler().getTimeChangeQueue()) { - if (queuedVote.getName().equalsIgnoreCase(player) - && queuedVote.getService().equalsIgnoreCase(service)) { - mostRecentTime = Math.max(mostRecentTime, queuedVote.getTime()); - } - } - } - - for (Column d : cols) { - if (d.getName().equalsIgnoreCase("LastVotes")) { - DataValue value = d.getValue(); - String[] list = value.getString().split("%line%"); - for (String str : list) { - String[] data = str.split("//"); - if (data[0].equalsIgnoreCase(site)) { - mostRecentTime = Math.max(mostRecentTime, Long.valueOf(data[1])); - } - } - } - } - return mostRecentTime; - } - - public boolean checkVoteDelay(String uuid, String service, ArrayList data) { - return checkVoteDelay(uuid, null, service, data, false); - } - - /** - * Checks the configured vote delay, optionally including accepted votes waiting - * for a GlobalData time change to finish. - * - * @param uuid player UUID - * @param player player name used by the time-change queue - * @param service vote service - * @param data current player data - * @param includeTimeChangeQueue whether queued votes reserve their delay slot - * @return true when the vote may be accepted - */ - public boolean checkVoteDelay(String uuid, String player, String service, ArrayList data, - boolean includeTimeChangeQueue) { - String site = getWaitUntilDelaySiteFromService(service); - if (site.isEmpty()) { - debug("No service site set for " + service + ", skipping vote delay check"); - return true; - } - - int voteDelay = getConfig().getWaitUntilVoteDelayVoteDelay(site); - int voteDelayMin = getConfig().getWaitUntilVoteDelayVoteDelayMin(site); - - long lastVote = getLastVotesTime(uuid, data, site, service, player, includeTimeChangeQueue); - if (lastVote == 0) { - debug("No last vote time found for " + uuid + "/" + service + ", skipping vote delay check"); - return true; - } - - try { - LocalDateTime now = getBungeeTimeChecker().getTime(); - LocalDateTime lastVoteTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastVote), ZoneId.systemDefault()) - .plusHours(getConfig().getTimeHourOffSet()); - - if (!getConfig().getWaitUntilVoteDelayVoteDelayDaily(site)) { - if (voteDelay == 0 && voteDelayMin == 0) { - debug("Vote delay is 0 for " + site + ", skipping vote delay check"); - return true; - } - - LocalDateTime nextvote = lastVoteTime.plusHours((long) voteDelay).plusMinutes((long) voteDelayMin); - return now.isAfter(nextvote); - } - LocalDateTime resetTime = lastVoteTime.withHour(getConfig().getWaitUntilVoteDelayVoteDelayHour(site)) - .withMinute(0).withSecond(0); - LocalDateTime resetTimeTomorrow = resetTime.plusHours(24); - - if (lastVoteTime.isBefore(resetTime)) { - if (now.isAfter(resetTime)) { - debug("Vote delay is met for " + uuid + "/" + service + ", vote can be processed"); - return true; - } - } else { - if (now.isAfter(resetTimeTomorrow)) { - debug("Vote delay is met for " + uuid + "/" + service + ", vote can be processed"); - return true; - } - } - } catch (Exception e) { - e.printStackTrace(); - } - - debug("Vote delay is not met for " + uuid + "/" + service + ", skipping vote"); - return false; - } - - public synchronized void vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, - VoteTotalsSnapshot text, String uuid) { - vote(player, service, realVote, timeQueue, queueTime, text, uuid, null); - } - - private enum QueuedVoteResult { - SUCCESS, RETRY, TERMINAL - } - - private synchronized QueuedVoteResult vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, - VoteTotalsSnapshot text, String uuid, VoteTimeQueue queuedVote) { - try { - if (!ServiceSiteValidator.isValid(service)) { - warn("Rejected vote with invalid service site '" + ServiceSiteValidator.sanitizeForLog(service) + "'"); - return QueuedVoteResult.TERMINAL; - } - if (!MinecraftUsernameValidator.isValid(player, getConfig().getBedrockPlayerPrefix())) { - warn("Rejected vote with invalid Minecraft username '" - + MinecraftUsernameValidator.sanitizeForLog(player) + "' from service '" - + MinecraftUsernameValidator.sanitizeForLog(service) + "'"); - return QueuedVoteResult.TERMINAL; - } - - UUID voteId = queuedVote == null ? null : queuedVote.getVoteId(); - if (voteId == null) { - voteId = UUID.randomUUID(); - } - - // UUID resolution - if (!getConfig().getOnlineMode()) { - uuid = getUUID(player); - } - - if (uuid == null || uuid.isEmpty()) { - uuid = getUUID(player); - - // Bedrock prefix auto-detect - if (uuid.isEmpty() && !getConfig().getBedrockPlayerPrefix().isEmpty() - && !player.startsWith(getConfig().getBedrockPlayerPrefix())) { - String uuid1 = getUUID(getConfig().getBedrockPlayerPrefix() + player); - if (!uuid1.isEmpty()) { - debug("Detected bedrock player without prefix, adjusting..."); - player = getConfig().getBedrockPlayerPrefix() + player; - uuid = uuid1; - } - } - } - - if (uuid.isEmpty()) { - if (player.startsWith(getConfig().getBedrockPlayerPrefix())) { - log("Ignoring vote since unable to get UUID of bedrock player"); - return QueuedVoteResult.TERMINAL; - } - if (!getConfig().getAllowUnJoined()) { - log("Ignoring vote from " + player + " since player hasn't joined before"); - return QueuedVoteResult.TERMINAL; - } - if (!getConfig().getUUIDLookup()) { - log("Failed to get uuid for " + player); - return QueuedVoteResult.TERMINAL; - } - - debug("Fetching UUID online, since allowunjoined is enabled"); - UUID u = null; - try { - if (getConfig().getOnlineMode()) { - u = fetchUUID(player); - } - } catch (Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - if (u == null) { - debug("Failed to get uuid for " + player); - return QueuedVoteResult.TERMINAL; - } - uuid = u.toString(); - } - - // Normalize UUID string if possible - try { - if (uuid != null && !uuid.isEmpty() && !uuid.equalsIgnoreCase("null")) { - uuid = UUID.fromString(uuid.trim()).toString(); - } - } catch (Exception ignored) { - // ignore - } - - player = getProperName(uuid, player); - - // Cache online state/server once (IMPORTANT for broadcast logic correctness) - final boolean playerOnline = isPlayerOnlineForVoteRouting(player); - final String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(player) : null; - long time = queueTime != 0 ? queueTime - : LocalDateTime.now().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); - - Set broadcastTargets = queuedVote == null ? new LinkedHashSet<>() - : new LinkedHashSet<>(queuedVote.getBroadcastTargets()); - Set broadcastForwardedServers = queuedVote == null ? new LinkedHashSet<>() - : new LinkedHashSet<>(queuedVote.getBroadcastForwardedServers()); - boolean proxyBroadcastHandled = queuedVote != null && queuedVote.isProxyBroadcastHandled(); - boolean processesTotals = getConfig().getPrimaryServer() || !getConfig().getMultiProxySupport(); - boolean managesTotals = processesTotals && getConfig().getBungeeManageTotals(); - boolean canValidateStandaloneBroadcast = canForwardStandaloneBroadcast(managesTotals); - ArrayList data = null; - boolean queueForTimeChange = false; - - // A completion callback can wipe totals and replay older queued votes. Run it - // before loading this vote's database snapshot so the calculations below use - // the post-rollover state. - if (getConfig().getGlobalDataEnabled() && getGlobalDataHandler().isTimeChangedHappened()) { - getGlobalDataHandler().checkForFinishedTimeChanges(); - queueForTimeChange = timeQueue && getGlobalDataHandler().isTimeChangedHappened(); - } - - // Validate the vote before any immediate announcement. This keeps duplicate - // votes rejected by the delay check out of the GlobalData rollover queue and - // prevents announcing a vote that will not be processed. - if (managesTotals) { - if (getProxyMySQL() == null) { - logSevere("Mysql is not loaded correctly, stopping vote processing"); - return QueuedVoteResult.RETRY; - } - - if (!getProxyMySQL().containsKeyQuery(uuid)) { - getProxyMySQL().update(uuid, "PlayerName", new DataValueString(player)); - getProxyMySQL().getUuids().add(uuid); - } - - data = getProxyMySQL().getExactQuery(new Column("uuid", new DataValueString(uuid))); - if (!checkVoteDelay(uuid, player, service, data, queuedVote == null)) { - log("Vote delay is not met for " + player + "/" + service + ", skipping vote"); - sendVoteDelayRejected(player, uuid, service, playerOnline, playerServer); - return QueuedVoteResult.TERMINAL; - } - } - - // Forward an accepted offline broadcast before the still-active GlobalData - // change queues the reward/totals work. The queued delivery state prevents - // replaying broadcasts that already reached a backend. - if (queueForTimeChange) { - VoteTotalsSnapshot projectedTotals = managesTotals ? getProjectedRolloverTotals(data, player) : text; - if (canValidateStandaloneBroadcast && proxyBroadcastDecider.usesImmediateForwarding(playerOnline)) { - broadcastTargets.addAll(proxyBroadcastDecider.resolveTargets(false, null)); - proxyBroadcastHandled = true; - } - VoteTimeQueue delayedVote = new VoteTimeQueue(voteId, player, service, time, - proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, - projectedTotals == null ? "" : projectedTotals.toString(), false, uuid); - if (!getVoteCacheHandler().addTimeVoteToCache(delayedVote)) { - logSevere("Unable to persist queued rollover vote for " + player + "/" + service - + "; skipping proxy broadcast"); - return QueuedVoteResult.RETRY; - } - if (proxyBroadcastHandled) { - for (String target : broadcastTargets) { - Set forwarded = sendProxyBroadcast(Collections.singleton(target), uuid, player, - service, time, projectedTotals == null ? "" : projectedTotals.toString(), false); - if (delayedVote.getBroadcastForwardedServers().addAll(forwarded)) { - broadcastForwardedServers.addAll(forwarded); - persistTimeVoteDelivery(delayedVote); - } - } - } - log("Caching vote from " + player + "/" + service - + " because time change is happening right now"); - return QueuedVoteResult.SUCCESS; - } - - addVoteParty(); - - // Totals processing (primary server OR no multiproxy) - if (processesTotals) { - if (managesTotals) { - int allTimeTotal = getValue(data, "AllTimeTotal", 1); - int monthTotal = getValue(data, "MonthTotal", 1); - - int dateMonthTotal = -1; - if (getConfig().getStoreMonthTotalsWithDate()) { - if (getConfig().getUseMonthDateTotalsAsPrimaryTotal()) { - dateMonthTotal = getValue(data, getMonthTotalsWithDatePath(), 1); - } else { - dateMonthTotal = monthTotal; - } - } - - int weeklyTotal = getValue(data, "WeeklyTotal", 1); - int dailyTotal = getValue(data, "DailyTotal", 1); - int points = getValue(data, "Points", getConfig().getPointsOnVote()); - - int maxVotes = getConfig().getMaxAmountOfVotesPerDay(); - if (maxVotes > 0) { - LocalDateTime cTime = getBungeeTimeChecker().getTime(); - int days = cTime.getDayOfMonth(); - if (monthTotal > days * maxVotes) { - monthTotal = days * maxVotes; - } - } - - if (getConfig().getLimitVotePoints() > 0 && points > getConfig().getLimitVotePoints()) { - points = getConfig().getLimitVotePoints(); - } - - text = new VoteTotalsSnapshot(allTimeTotal, monthTotal, weeklyTotal, dailyTotal, points, - votePartyVotes, currentVotePartyVotesRequired, dateMonthTotal); - - ArrayList update = new ArrayList<>(); - update.add(new Column("AllTimeTotal", new DataValueInt(allTimeTotal))); - update.add(new Column("MonthTotal", new DataValueInt(monthTotal))); - if (getConfig().getStoreMonthTotalsWithDate()) { - update.add(new Column(getMonthTotalsWithDatePath(), new DataValueInt(dateMonthTotal))); - } - update.add(new Column("WeeklyTotal", new DataValueInt(weeklyTotal))); - update.add(new Column("DailyTotal", new DataValueInt(dailyTotal))); - update.add(new Column("Points", new DataValueInt(points))); - - debug("Setting totals " + text.toString() + ", voteId=" + voteId + " for " + player + "/" - + service); - getProxyMySQL().update(uuid, update); - } else { - text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); - } - } - if (text == null) { - text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); - } - - VoteLogStatus voteStatus = VoteLogStatus.IMMEDIATE; - boolean standaloneProxyBroadcast = canValidateStandaloneBroadcast && (proxyBroadcastHandled - || proxyBroadcastDecider.usesImmediateForwarding(playerOnline)); - Set proxyBroadcastTargets = Collections.emptySet(); - if (standaloneProxyBroadcast) { - // A handled queued broadcast was necessarily sampled while the player was - // offline. Retry only targets that did not previously accept delivery. - proxyBroadcastTargets = proxyBroadcastHandled ? new LinkedHashSet<>(broadcastTargets) - : proxyBroadcastDecider.resolveTargets(false, null); - Set remainingTargets = new LinkedHashSet<>(proxyBroadcastTargets); - remainingTargets.removeAll(broadcastForwardedServers); - broadcastForwardedServers.addAll(sendProxyBroadcast(remainingTargets, uuid, player, service, time, - text == null ? "" : text.toString(), false)); - } - - // =========================== - // Send vote(s) to backend(s) - // =========================== - if (getConfig().getSendVotesToAllServers()) { - for (String s : getAllAvailableServers()) { - - boolean forceCache = getConfig().getWaitForUserOnline() - && (!playerOnline || playerServer == null || !playerServer.equalsIgnoreCase(s)); - - if (forceCache) { - debug("Forcing vote to cache for server " + s); - } - - if ((!isSomeoneOnlineServerForVoteRouting(s) && method.requiresPlayerOnline()) || forceCache) { - voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addServerVote(s, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, - text.toString(), broadcastForwarded, standaloneProxyBroadcast, - proxyBroadcastTargets, broadcastForwardedServers, false)); - debug("Caching vote for " + player + " on " + service + " for " + s); - } else { - boolean broadcastHere = !broadcastForwardedServers.contains(s); - if (broadcastHere && getConfig().getProxyBroadcastEnabled()) { - Set targets = standaloneProxyBroadcast ? proxyBroadcastTargets - : proxyBroadcastDecider.resolveTargets(playerOnline, playerServer); - broadcastHere = proxyBroadcastDecider.shouldBroadcast(s, targets); - } - - if (!sendVoteEnvelopeAccepted(s, 2, - VotingPluginWire.vote(player, uuid, service, time, true, realVote, text.toString(), - voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1))) { - voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addServerVote(s, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, - text.toString(), broadcastForwarded, standaloneProxyBroadcast, - proxyBroadcastTargets, broadcastForwardedServers, false)); - debug("Caching vote after the transport rejected delivery for " + s); - } - } - } - } else { - // Single-server mode: online goes to player server; otherwise queue as "online - // vote" - if (playerOnline && playerServer != null && getAllAvailableServers().contains(playerServer)) { - String server = playerServer; - - boolean broadcastHere = !broadcastForwardedServers.contains(server); - if (broadcastHere && getConfig().getProxyBroadcastEnabled()) { - Set targets = standaloneProxyBroadcast ? proxyBroadcastTargets - : proxyBroadcastDecider.resolveTargets(true, playerServer); - broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); - } - - boolean rewardAccepted = sendVoteEnvelopeAccepted(server, 1, - VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(), - voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1)); - if (!rewardAccepted) { - voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addOnlineVote(uuid, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), - broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, - broadcastForwardedServers, false)); - debug("Caching online vote after the transport rejected delivery for " + server); - } - - if (rewardAccepted && canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled() - && !standaloneProxyBroadcast) { - Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); - - int bDelay = 2; - for (String targetServer : targets) { - // avoid double-broadcast on the same server that already got the voteOnline - if (targetServer.equalsIgnoreCase(server)) { - continue; - } - if (getConfig().getBlockedServers().contains(targetServer)) { - continue; - } - - globalMessageProxyHandler.sendMessage(targetServer, bDelay, - VotingPluginWire.voteBroadcast(uuid, player, service, time, - text == null ? "" : text.toString(), true)); - bDelay++; - } - } - - // multiproxy: envelope-only clear vote - if (rewardAccepted && getConfig().getMultiProxySupport() && getConfig().getMultiProxyOneGlobalReward()) { - multiProxyHandler.sendClearVote(uuid, player); - } - } else { - voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addOnlineVote(uuid, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), - broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, - broadcastForwardedServers, false)); - debug("Caching online vote for " + player + " on " + service); - } - - int delay = 2; - for (String s : getAllAvailableServers()) { - globalMessageProxyHandler.sendMessage(s, delay + 1, VotingPluginWire.voteUpdate(uuid, - votePartyVotes, currentVotePartyVotesRequired, service, time, text.toString())); - delay += 2; - } - } - - // Vote logging - if (voteLogMysqlTable != null && getConfig().getVoteLoggingEnabled()) { - voteLogMysqlTable.logVote(voteId, voteStatus, service, uuid, player, time, - getVoteCacheHandler().getProxyCachedTotal(uuid)); - } - - // =========================== - // Multiproxy forwarding - // =========================== - if (getConfig().getMultiProxySupport() && getConfig().getPrimaryServer()) { - if (!getConfig().getMultiProxyOneGlobalReward()) { - debug("Sending global proxy vote envelope"); - multiProxyHandler.sendMultiProxyEnvelope(VotingPluginWire.vote(player, uuid, service, time, false, - realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); - } else { - // Only send to other proxies if the player DID NOT already receive reward on a - // backend - boolean shouldSend = true; - if (playerOnline && playerServer != null) { - if (!getConfig().getBlockedServers().contains(playerServer)) { - shouldSend = false; - } - } - - if (shouldSend) { - debug("Sending global proxy voteonline envelope"); - multiProxyHandler - .sendMultiProxyEnvelope(VotingPluginWire.voteOnline(player, uuid, service, time, false, - realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); - } else { - debug("Not sending global proxy message for voteonline, player already got reward"); - } - } - } - if (queuedVote != null) { - queuedVote.setProcessed(true); - if (!getVoteCacheHandler().updateTimeVote(queuedVote)) { - warn("Unable to persist completed rollover vote " + queuedVote.getVoteId() - + "; attempting durable removal immediately"); - } - } - return QueuedVoteResult.SUCCESS; - } catch (Exception e) { - e.printStackTrace(); - return QueuedVoteResult.RETRY; - } - } - - private static final class PendingPresenceHandoff { - private UUID requestId; - private final UUID playerUuid; - private final String playerName; - private final String uuid; - private final String server; - private final UUID connectionId; - private final UUID backendIncarnationId; - private final long backendStartedAt; - private final long conflictSequence; - private final long createdAt; - - private PendingPresenceHandoff(String playerName, String uuid, String server, UUID connectionId, - UUID backendIncarnationId, long backendStartedAt, long conflictSequence, long createdAt) { - this.playerUuid = parsePlayerUuid(uuid); - this.playerName = playerName; - this.uuid = uuid; - this.server = server; - this.connectionId = connectionId; - this.backendIncarnationId = backendIncarnationId; - this.backendStartedAt = backendStartedAt; - this.conflictSequence = conflictSequence; - this.createdAt = createdAt; - } - - private static UUID parsePlayerUuid(String uuid) { - try { - return UUID.fromString(uuid.trim()); - } catch (Exception ignored) { - return null; - } - } - } - - public abstract void warn(String message); - - public abstract ScheduledExecutorService getScheduler(); -} +YªçŠx-®éÜj×¢ëiºÚ+Š§j[h‘éÜ¢éíã}å:-jZ.¶›­–)Þ³W6¶vR6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡“° ¦–×÷'B¦fæ–òä'—FT'&”÷WGWE7G&VÓ°¦–×÷'B¦fæ–òäFF–çWE7G&VÓ°¦–×÷'B¦fæ–òäFF÷WGWE7G&VÓ°¦–×÷'B¦fæ–òäf–ÆS°¦–×÷'B¦fæ–òä”ôW†6WF–öã°¦–×÷'B¦fææWBä–æWE6ö6¶WDFG&W73°¦–×÷'B¦fææWBå6ö6¶WC°¦–×÷'B¦fææWBåU$“°¦–×÷'B¦fææWBæ‡GGä‡GG6Æ–VçC°¦–×÷'B¦fææWBæ‡GGä‡GG&WVW7C°¦–×÷'B¦fææWBæ‡GGä‡GG&W7öç6S°¦–×÷'B¦fç7Âå5ÄW†6WF–öã°¦–×÷'B¦fçF–ÖRäGW&F–öã°¦–×÷'B¦fçF–ÖRä–ç7FçC°¦–×÷'B¦fçF–ÖRäÆö6ÄFFUF–ÖS°¦–×÷'B¦fçF–ÖRå¦öæT–C°¦–×÷'B¦fçF–ÖRå¦öæTöfg6WC°¦–×÷'B¦fçWF–Âä'&”Æ—7C°¦–×÷'B¦fçWF–Âä6öÆÆV7F–öã°¦–×÷'B¦fçWF–Âä6öÆÆV7F–öç3°¦–×÷'B¦fçWF–Âä†6„Ö°¦–×÷'B¦fçWF–ÂäÆ–æ¶VD†6…6WC°¦–×÷'B¦fçWF–ÂäÆ—7C°¦–×÷'B¦fçWF–ÂäÖ°¦–×÷'B¦fçWF–Âå6WC°¦–×÷'B¦fçWF–ÂåUT”C°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBä6ö×ÆWF&ÆTgWGW&S°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBä6öæ7W'&VçD†6„Ö°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBäW†V7WF÷%6W'f–6S°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBäW†V7WF÷'3°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBå66†VGVÆVDW†V7WF÷%6W'f–6S°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBåF–ÖUVæ—C°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBæFöÖ–2äFöÖ–4&ööÆVã°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBæFöÖ–2äFöÖ–4Æöæs° ¦–×÷'B¦f‚ææWBç76Âå54Å&ÖWFW'3° ¦–×÷'B÷&ræV6Æ—6Rç†òæ6Æ–VçBæ×GGc2ä×GDW†6WF–öã° ¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ’çF–ÖRåF–ÖUG—S°¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ'VævVV’ævÆö&ÆFFävÆö&ÄFF†æFÆW%&÷‡“°¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ'VævVV’ævÆö&ÆFFävÆö&Äו5ð¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ'VævVV’çF–ÖRä'VævVUF–ÖT6†V6¶W#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’æVæ7'—F–öâäVæ7'—F–öä†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’æ§6öâä§6öå'6W#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ6öFV2ä§6öäVçfVÆ÷S°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ6öFV2ä§6öäVçfVÆ÷T6öFV3°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒævÆö&ÂävÆö&ÄÖW76vTÆ—7FVæW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒævÆö&ÂävÆö&ÄÖW76vU&÷‡”†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ×GBä×GD†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ×GBä×GE6W'fW$6öÖÓ°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ×—7Âäו7ÄÖW76VævW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç&VF—2å&VF—4†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç&VF—2å&VF—4Æ—7FVæW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç6ö6¶WG2ä6Æ–VçD†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç6ö6¶WG2å6ö6¶WD†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç6ö6¶WG2å6ö6¶WE&V6V—fW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7Âä6öÇVÖã°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂäFFG—S°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVS°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVT&ööÆVã°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVT–çC°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVU7G&–æs°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7Âæ×—7Âæ6öæf–rä×—7Ä6öæf–s°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âæ&6¶VæG&÷‡’æ‡GGä‡GGVç&öÆÆÖVçDWF†÷&—G“°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âæ&6¶VæG&÷‡’æ‡GGä‡GG&÷‡•G&ç7÷'E6W'fW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âæ&6¶VæG&÷‡’æ‡GGä‡GGFÇ4–FVçF—G“°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ'&öF67Bå&÷‡”'&öF67DFV6–FW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Rä•f÷FT66†S°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Råf÷FT66†T†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Rææöçf÷FVBä”æöåf÷FVEÆ–W'57F÷&vS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Rææöçf÷FVBäæöåf÷FVEÆ–W'466†S°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ6öçG&öÂä6öçG&öÄ6öææV7F÷#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ6öçG&öÂä†÷7FVD6öçG&öÄÖævW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡”†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡”ÖWF†öC°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡•6W'fW%6ö6¶WD6öæf–wW&F–öã°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡•6W'fW%6ö6¶WD6öæf–wW&F–öä'VævVS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’ç&W6Væ6Rä&6¶VæEÆ–W%&W6Væ6UG&6¶W#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’ç&W6Væ6RåÆ–W%&W6Væ6S°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçF–ÖWVWVRåf÷FUF–ÖUVWVS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçF÷f÷FW"åF÷f÷FW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçWF–ÂäÖ–æV7&gEW6W&æÖUfÆ–FF÷#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçWF–Âå6W'f–6U6—FUfÆ–FF÷#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçf÷FVÆöråf÷FTÆöt×—7ÅF&ÆS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçf÷FVÆöråf÷FTÆöt×—7ÅF&ÆRåf÷FTÆöu7FGW3°¦–×÷'B6öÒævöövÆRæw6öâä§6öäVÆVÖVçC°¦–×÷'B6öÒævöövÆRæw6öâä§6öäö&¦V7C° ¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2äFVfVÇD¦VF—46Æ–VçD6öæf–s°¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2ä†÷7DæE÷'C°¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2ä¦VF—3°¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2ä¦VF—5ööð ¦–×÷'BÆöÖ&ö²ävWGFW#°¦–×÷'BÆöÖ&ö²å6WGFW#° §V&Æ–2'7G&7B6Æ72f÷F–æuÇVv–å&÷‡’° —&—fFR7FF–2f–æÂÆöær$U4Tä4Uô„äDôdeõD”ÔTõUEôÔ”ÄÄ•2ÒF–ÖUVæ—BäÔ”åUDU2çFôÖ–ÆÆ—2ƒ"“° —&—fFR7FF–2f–æÂÆöær$U4Tä4Uõ5D%EUõ$U5”ä5ôDTÄ•õ4T4ôäE2ÒTð —&—fFR7FF–2f–æÂÆöær$U4Tä4UôÔ”åDTää4Uô”åDU%dÅõ4T4ôäE2Ò3ð —&—fFR7FF–2f–æÂÆöær$U4Tä4Uô$4´TäEõD”ÔTõUEôÔ”ÄÄ•2ÒF–ÖUVæ—Bå4T4ôäE2çFôÖ–ÆÆ—2ƒ““° —&—fFR7FF–2f–æÂÆöær4ôåE$ôÅôTå$ôÄÄÔTåEôÔ”åô”åDU%dÅôääõ2ÒF–ÖUVæ—Bå4T4ôäE2çFôææ÷2ƒ“°  ”vWGFW  ”6WGFW  —&—fFR–çBf÷FU'G•f÷FW2Ò°  ”vWGFW  ”6WGFW  —&—fFR–çB7W'&VçEf÷FU'G•f÷FW5&WV—&VBÒ°  ”vWGFW  ”6WGFW  —&—fFR&÷‡”×—7ÅW6W%F&ÆR&÷‡”ו5ð  —&—fFRVæ7'—F–öä†æFÆW"Væ7'—F–öä†æFÆW#°  —&—fFR†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â6Æ–VçD†æFÆW3°  —&—fFR6ö6¶WD†æFÆW"6ö6¶WD†æFÆW#° —&—fFR‡GG&÷‡•G&ç7÷'E6W'fW"‡GGG&ç7÷'E6W'fW#° —&—fFR‡GGVç&öÆÆÖVçDWF†÷&—G’‡GGVç&öÆÆÖVçDWF†÷&—G“°  ”vWGFW  ”6WGFW  —&—fFR&ööÆVâf÷F–f–W$Væ&ÆVBÒG'VS°  ”vWGFW  —&—fFR6öæ7W'&VçD†6„ÖÅUT”BÂ7G&–æsâWV–EÆ–W$æÖT66†RÒæWr6öæ7W'&VçD†6„ÖÃâ‚“°  ”vWGFW  ”6WGFW  —&—fFRvÆö&ÄFF†æFÆW%&÷‡’vÆö&ÄFF†æFÆW#°  ”vWGFW  —&—fFR&VF—4†æFÆW"&VF—4†æFÆW#° —&—fFR¦VF—5ööÂ&VF—5V&Æ—6†W%ööð —&—fFRföÆF–ÆRÆöær&VF—5V&Æ—6†W%&WG'”gFW#° —&—fFR&ööÆVâF–ÖUf÷FU&WG'•66†VGVÆVC° —&—fFR&ööÆVâF–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVC° —&—fFR&ööÆVâ66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVC°  —&—fFR&ööÆVâVæ&ÆVC°  ”vWGFW  ”6WGFW  —&—fFR×VÇF•&÷‡”†æFÆW"×VÇF•&÷‡”†æFÆW#°  ”vWGFW  —&—fFR'VævVUF–ÖT6†V6¶W"'VævVUF–ÖT6†V6¶W#°  ”vWGFW  ”6WGFW  —&—fFR'VævVTÖWF†öBÖWF†öC°  ”vWGFW  —&—fFR×GD†æFÆW"×GD†æFÆW#°  ”vWGFW  —&—fFRvÆö&ÄÖW76vU&÷‡”†æFÆW"vÆö&ÄÖW76vU&÷‡”†æFÆW#°  ”vWGFW  ”6WGFW  —&—fFRו7ÄÖW76VævW"&÷‡”×—7ÄÖW76VævW#°  ”vWGFW  —&—fFRf÷FT66†T†æFÆW"f÷FT66†T†æFÆW#°  ”vWGFW  —&—fFRæöåf÷FVEÆ–W'466†Ræöåf÷FVEÆ–W'466†S°  ”vWGFW  —&—fFRf–æÂ&6¶VæEÆ–W%&W6Væ6UG&6¶W"&6¶VæEÆ–W%&W6Væ6UG&6¶W"ÒæWr&6¶VæEÆ–W%&W6Væ6UG&6¶W"‚“° —&—fFRf–æÂÖÅUT”BÂVæF–æu&W6Væ6T†æFöfcâVæF–æu&W6Væ6T†æFöfg2ÒæWr†6„ÖÃâ‚“° —&—fFRf–æÂ6WCÅ7G&–æsâVæF–æt&6¶VæE&V6÷fW'•6æ6†÷G2Ò6öæ7W'&VçD†6„ÖææWt¶W•6WB‚“° —&—fFRf–æÂÖÅ7G&–ærÂÆöæsâ6öçG&öÄVç&öÆÆÖVçDæW‡DÆÆ÷vVBÒæWr6öæ7W'&VçD†6„ÖÃâ‚“° —&—fFRf–æÂÖÅUT”BÂVæF–æt6öÖ×Væ–6F–öåFW7CâVæF–æt6öÖ×Væ–6F–öåFW7G2ÒæWr6öæ7W'&VçD†6„ÖÃâ‚“° —&—fFRföÆF–ÆR6öçG&öÄ6öææV7F÷"6öçG&öÄ6öææV7F÷#° —&—fFRföÆF–ÆR†÷7FVD6öçG&öÄÖævW"†÷7FVD6öçG&öÄÖævW#° —&—fFRf–æÂö&¦V7B6öçG&öÄÆ–fV7–6ÆTÆö6²ÒæWrö&¦V7B‚“° —&—fFRf–æÂFöÖ–4Æöær6öçG&öÅ6W'f–6W4vVæW&F–öâÒæWrFöÖ–4Æöær‚“° —&—fFRf–æÂW†V7WF÷%6W'f–6R6öçG&öÄÆ–fV7–6ÆTW†V7WF÷"ÒW†V7WF÷'2ææWu6–ævÆUF‡&VDW†V7WF÷"‡F6²Óâ° •F‡&VBF‡&VBÒæWrF‡&VB‡F6²Â'f÷F–æwÇVv–âÖ6öçG&öÂÖÆ–fV7–6ÆR"“° —F‡&VBç6WDFVÖöâ‡G'VR“° —&WGW&âF‡&VC° —Ò“°  —V&Æ–2f÷F–æuÇVv–å&÷‡’‚’° –Væ&ÆVBÒG'VS°  –'VævVUF–ÖT6†V6¶W"ÒæWr'VævVUF–ÖT6†V6¶W"†vWD6öæf–r‚’ævWEF–ÖU¦öæR‚’ÂvWD6öæf–r‚’ævWEF–ÖT†÷W$öfe6WB‚’À –vWD6öæf–r‚’ævWEF–ÖUvVV´öfe6WB‚’’°  ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…7G&–ærFW‡B’° –FV'Vs"‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2ÆöærvWDÆ7EWFFVB‚’° —&WGW&âvWEf÷FT66†TÆ7EWFFVB‚“° —Р ”÷fW'&–FP —V&Æ–2–çBvWE&WdF’‚’° —&WGW&âvWEf÷FT66†U&WdF’‚“° —Р ”÷fW'&–FP —V&Æ–27G&–ærvWE&WdÖöçF‚‚’° —&WGW&âvWEf÷FT66†U&WdÖöçF‚‚“° —Р ”÷fW'&–FP —V&Æ–2–çBvWE&WevVV²‚’° —&WGW&âvWEf÷FT66†U&WevVV²‚“° —Р ”÷fW'&–FP —V&Æ–2fö–B–æfò…7G&–ærFW‡B’° –Æör‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2&ööÆVâ—4Væ&ÆVB‚’° —&WGW&âVæ&ÆVC° —Р ”÷fW'&–FP —V&Æ–2&ööÆVâ—4–væ÷&UF–ÖR‚’° —&WGW&â—5f÷FT66†T–væ÷&UF–ÖR‚“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WD–væ÷&UF–ÖR†&ööÆVâ–væ÷&R’° —6WEf÷FT66†Uf÷FT66†T–væ÷&UF–ÖR†–væ÷&R“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WDÆ7EWFFVB‚’° —6WEf÷FT66†TÆ7EWFFVB‚“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WE&WdF’†–çBF’’° —6WEf÷FT66†U&WdF’†F’“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WE&WdÖöçF‚…7G&–ærFW‡B’° —6WEf÷FT66†U&WdÖöçF‚‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WE&WevVV²†–çBvVV²’° —6WEf÷FT66†U&WevVV²‡vVV²“° —Р ”÷fW'&–FP —V&Æ–2fö–BF–ÖT6†ævVB…F–ÖUG—RG—RÂ&ööÆVâf¶RÂ&ööÆVâ&RÂ&ööÆVâ÷7B’° ––b†vWD6öæf–r‚’ævWEf÷FT66†UF–ÖR‚’â’° –vWEf÷FT66†T†æFÆW"‚’æ6†V6µf÷FT66†UF–ÖR†vWD6öæf–r‚’ævWEf÷FT66†UF–ÖR‚’“° —Р––b‚vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’’° —v&â‚$vÆö&ÂFFæ÷BVæ&ÆVB–væ÷&–ærF–ÖR6†ævRWfVçB"“° —&WGW&ã° —Р––çBFVÆ’Ò° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° ––b†vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æ6öçF–ç4¶W’‡2’’° •7G&–ærÆ7DöæÆ–æU7G"ÒvWDvÆö&ÄFF†æFÆW"‚’ævWE7G&–ær‡2Â$Æ7DöæÆ–æR"“° –ÆöærÆ7DöæÆ–æRÒ° —G'’° –Æ7DöæÆ–æRÒÆöærçfÇVTöb†Æ7DöæÆ–æU7G"“° —Ò6F6‚„çVÖ&W$f÷&ÖDW†6WF–öâR’° ’òò–væ÷&P —Р ––b„Æö6ÄFFUF–ÖRææ÷r‚’æE¦öæR…¦öæTöfg6WBåUD2’çFô–ç7FçB‚’çFôWö6„Ö–ÆÆ’‚’ÒÆ7DöæÆ–æR ’¢c¢c¢"’° ”†6„ÖÅ7G&–ærÂFFfÇVSâFFFõ6WBÒæWr†6„ÖÃâ‚“° –FFFõ6WBçWB‚$Æ7EWFFVB"ÂæWrFFfÇVU7G&–ær€ ’""²Æö6ÄFFUF–ÖRææ÷r‚’æE¦öæR…¦öæTöfg6WBåUD2’çFô–ç7FçB‚’çFôWö6„Ö–ÆÆ’‚’’“° –FFFõ6WBçWB‚$f–æ—6†VE&ö6W76–ær"ÂæWrFFfÇVT&ööÆVâ†fÇ6R’“° –FFFõ6WBçWB‡G—RçFõ7G&–ær‚’ÂæWrFFfÇVT&ööÆVâ‡G'VR’“° –vWDvÆö&ÄFF†æFÆW"‚’ç6WDFF‡2ÂFFFõ6WB“°  –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡2ÂFVÆ’Âf÷F–æuÇVv–åv—&Ræ'VævVUF–ÖT6†ævR‚’“° –FVÆ’²³° —ÒVÇ6R° —v&â‚%6W'fW""²2²"†6âwB&VVâöæÆ–æR&V6VçFÇ’"“° —Р—ÒVÇ6R° —v&â‚%6W'fW""²2²"vÆö&ÂFF†æFÆW"F—6&ÆVCò"“° —Р—Р–vÆö&ÄFF†æFÆW"æöåF–ÖT6†ævR‡G—R“° —Р ”÷fW'&–FP —V&Æ–2fö–Bv&æ–ær…7G&–ærFW‡B’° —v&â‡FW‡B“° —Р—Ó° —Р —V&Æ–2fö–BöåF–ÖT6†ævVDf–ÆVB…7G&–ær7'bÂF–ÖUG—RG—R’° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡7'bÂG—RçFõ7G&–ær‚’ÂfÇ6R“° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡7'bÂ$f–æ—6†VE&ö6W76–ær"ÂG'VR“° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡7'bÂ%&ö6W76–ær"ÂfÇ6R“° —Р —V&Æ–2fö–BöåF–ÖT6†ævVDf–æ—6†VB…F–ÖUG—RG—R’° ––b‡G—RæWVÇ2…F–ÖUG—RäÔôåD‚’’° –vWE&÷‡”ו5‚’æ6÷”6öÇVÖäFF…F÷f÷FW"äÖöçF†Ç’ævWD6öÇVÖäæÖR‚’Â$Æ7DÖöçF…F÷FÂ"“° —Р–vWE&÷‡”ו5‚’çv—T6öÇVÖäFF…F÷f÷FW"æöb‡G—R’ævWD6öÇVÖäæÖR‚’ÂFFG—Rä”åDTtU"“°  ––b‚vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’’° —&WGW&ã° —Р–f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡2Â$f÷&6UWFFR"ÂG'VR“° –vWDvÆö&ÄÖW76vU&÷‡”†æFÆW"‚’ç6VæDÖW76vR‡2ÂÂf÷F–æuÇVv–åv—&Ræ'VævVUF–ÖT6†ævR‚’“° —Р—&ö6W75VWVR‚“° —Р ’ò¢  ’¢ÆöBו5²vÆö&ÂFF†æFÆW"à ’¢ð —V&Æ–2fö–BÆöD×—7„ח7Ä6öæf–r×—7Ä6öæf–rÂ×—7Ä6öæf–rvÆö&ÄFF×—7Ä6öæf–r’° ––b†×—7Ä6öæf–rævWD†÷7DæÖR‚’æ—4V×G’‚’ÇÂ×—7Ä6öæf–rævWDFF&6R‚’æ—4V×G’‚’’° –Æöu6WfW&R‚$ו5—2æ÷B6öæf–wW&VB6÷'&V7FÇ’â"²$Ö—76–ær†÷7BöFF&6Râ†÷7CÒ"²×—7Ä6öæf–rævWD†÷7DæÖR‚ ’²"F#Ò"²×—7Ä6öæf–rævWDFF&6R‚’“° —6WE&÷‡”ו5†çVÆ“° —&WGW&ã° —Р —6WE&÷‡”ו5†æWr&÷‡”×—7ÅW6W%F&ÆR‚%f÷F–æuÇVv–åõW6W'2"Â×—7Ä6öæf–rÂvWD6öæf–r‚’ævWDFV'Vr‚’’°  ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…5ÄW†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&R…7G&–ær7G&–ær’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöu6WfW&R‡7G&–ær“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöt–æfò…7G&–ær7G&–ær’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöt–æfò‡7G&–ær“° —Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…F‡&÷v&ÆRB’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° —Bç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…7G&–ær7G"’° –FV'Vs"‡7G"“° —Р—Ò“°  ”'&”Æ—7CÅ7G&–æsâ6W'fW'2ÒæWr'&”Æ—7CÅ7G&–æsâ†vWDÆÄf–Æ&ÆU6W'fW'2‚’“°  ––b†vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’’° ––b†vWD6öæf–r‚’ævWDvÆö&ÄFFW6TÖ–äו5‚’’° —6WDvÆö&ÄFF†æFÆW"†æWrvÆö&ÄFF†æFÆW%&÷‡’€ –æWrvÆö&Äו5‚%f÷F–æuÇVv–åôvÆö&ÄFF"ÂvWE&÷‡”ו5‚’ævWD×—7‚’’°  ”÷fW'&–FP —V&Æ–2fö–BFV'VtW‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'VtÆör…7G&–ærFW‡B’° –FV'Vr‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–B–æfò…7G&–ærFW‡B’° –Æöt–æfò‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&R…7G&–ærFW‡B’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöu6WfW&R‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–Bv&æ–ær…7G&–ærFW‡B’° —v&â‡FW‡B“° —Р—ÒÂ6W'fW'2’°  ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–ÆVB…7G&–ær7'bÂF–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–ÆVB‡7'bÂG—R“° —Р ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–æ—6†VB…F–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–æ—6†VB‡G—R“° —Р—Ò“° —ÒVÇ6R° —6WDvÆö&ÄFF†æFÆW"€ –æWrvÆö&ÄFF†æFÆW%&÷‡’†æWrvÆö&Äו5‚%f÷F–æuÇVv–åôvÆö&ÄFF"ÂvÆö&ÄFF×—7Ä6öæf–r’°  ”÷fW'&–FP —V&Æ–2fö–BFV'VtW‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'VtÆör…7G&–ærFW‡B’° –FV'Vr‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–B–æfò…7G&–ærFW‡B’° –Æöt–æfò‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&R…7G&–ærFW‡B’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöu6WfW&R‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–Bv&æ–ær…7G&–ærFW‡B’° —v&â‡FW‡B“° —Р—ÒÂ6W'fW'2’°  ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–ÆVB…7G&–ær7'bÂF–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–ÆVB‡7'bÂG—R“° —Р ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–æ—6†VB…F–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–æ—6†VB‡G—R“° —Р—Ò“° —Р ’òòWFFRvÆö&Â66†VÖ6öÇVÖç2‡Væ6†ævVBg&öÒ÷&–v–æ –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$–væ÷&UF–ÖR"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$ÔôåD‚"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚%tTT²"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$D’"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$f–æ—6†VE&ö6W76–ær"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚%&ö6W76–ær"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$f÷&6UWFFR"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$Æ7EWFFVB"Â$ÔTD•TÕDU…B"“° —Р ’òò6öÇVÖâG—W2‡Væ6†ævVBg&öÒ÷&–v–æ –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%F÷f÷FW$–væ÷&R"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$6†V6µv÷&ÆB"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%&VÖ–æFVB"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F—6&ÆT'&öF67B"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7DöæÆ–æR"Â%d$4„"ƒ#’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%Æ–W$æÖR"Â%d$4„"ƒ3’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F–Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%vVV¶Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F•f÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$&W7DF•f÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%vVVµf÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$&W7EvVVµf÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%f÷FU'G•f÷FW2"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$ÖöçF…f÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%ö–çG2"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$†–v†W7DF–Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$ÆÅF–ÖUF÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$†–v†W7DÖöçF†Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$ÖöçF…F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$†–v†W7EvVV¶Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7DÖöçF…F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7EvVV¶Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7DF–Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$öffÆ–æU&Wv&G2"Â$ÔTD•TÕDU…B"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F•f÷FU7G&V´Æ7EWFFR"Â$ÔTD•TÕDU…B"“°  ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R†vWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖRææ÷r‚’’Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R†vWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖRææ÷r‚’çÇW4ÖöçF‡2ƒ’’À ’$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R†vWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖRææ÷r‚’çÇW4ÖöçF‡2ƒ"’’À ’$”åBDTdTÅBsr"“° —Р—Р —V&Æ–2fö–BFD7W'&VçEf÷FU'G•f÷FW2†–çBÖ÷VçB’° —f÷FU'G•f÷FW2³ÒÖ÷VçC° —6WEf÷FT66†Uf÷FU'G”7W'&VçEf÷FW2‡f÷FU'G•f÷FW2“° –FV'Vr‚$7W'&VçBf÷FR'G’F÷Fâ"²f÷FU'G•f÷FW2“° —Р —V&Æ–2fö–BFDæöåf÷FVEÆ–W"…7G&–ærWV–BÂ7G&–ærÆ–W$æÖR’° –æöåf÷FVEÆ–W'466†RæFEÆ–W"‡WV–BÂÆ–W$æÖR“° —Р —V&Æ–2fö–BFEf÷FU'G’‚’° ––b†vWD6öæf–r‚’ævWEf÷FU'G”Væ&ÆVB‚’’° –FD7W'&VçEf÷FU'G•f÷FW2ƒ“° –6†V6µf÷FU'G’‚“° —Р—Р —V&Æ–2'7G&7Bfö–B'&öF67B…7G&–ærÖW76vR“°  —&—fFR6WCÅ7G&–æsâ6VæE&÷‡”'&öF67B…6WCÅ7G&–æsâF&vWG2Â7G&–ærWV–BÂ7G&–ærÆ–W"Â7G&–ær6W'f–6RÂÆöærF–ÖRÀ •7G&–ærFW‡BÂ&ööÆVâv4öæÆ–æR’° •6WCÅ7G&–æsâf÷'v&FVBÒæWrÆ–æ¶VD†6…6WCÃâ‚“° –f÷"…7G&–ærF&vWE6W'fW"¢F&vWG2’° ”§6öäVçfVÆ÷RVçfVÆ÷RÒf÷F–æuÇVv–åv—&Rçf÷FT'&öF67B‡WV–BÂÆ–W"Â6W'f–6RÂF–ÖRÂFW‡BÂv4öæÆ–æR“° ––b‡6VæE&÷‡”'&öF67DVçfVÆ÷Tæ÷r‡F&vWE6W'fW"ÂVçfVÆ÷R’’° –f÷'v&FVBæFB‡F&vWE6W'fW"“° —Р—Р—&WGW&âf÷'v&FVC° —Р ’ò¢  ’¢6VæG27FæFÆöæR&÷‡’'&öF67BF‡&÷Vv‚F†R6VÆV7FVBG&ç7÷'BæB&W÷'G0 ’¢v†WF†W"F†BG&ç7÷'B66WFVBF†RÖW76vRà ’  ’¢&Ò6W'fW"F&vWB&6¶VæB6W'fW  ’¢&ÒVçfVÆ÷R7FæFÆöæR'&öF67BVçfVÆ÷P ’¢&WGW&âG'VRöæÇ’v†VâF†RG&ç7÷'B66WFVBF†RÖW76vP ’¢ð —&÷FV7FVB&ööÆVâ6VæE&÷‡”'&öF67DVçfVÆ÷Tæ÷r…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° —7v—F6‚†ÖWF†öB’° –66RÕEC  —&WGW&â6VæD×GDVçfVÆ÷U6W'fW"‡6W'fW"ÂVçfVÆ÷R“° –66RÕ•5à ––b‡&÷‡”×—7ÄÖW76VævW"ÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р—G'’° —&÷‡”×—7ÄÖW76VævW"ç6VæEFô&6¶VæB‡6W'fW"ÂVçfVÆ÷R“° —&WGW&âG'VS° —Ò6F6‚…5ÄW†6WF–öâR’° –FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р–66RÅTt”äÔU54t”äs  —&WGW&â6VæEÇVv–äÖW76vU6W'fW$æ÷r‡6W'fW"ÂVçfVÆ÷R“° –66R$TD•3  —&WGW&â6VæE&VF—4VçfVÆ÷U6W'fW"‡6W'fW"ÂVçfVÆ÷RÂG'VR“° –66R4ô4´UE3  ’òò7FæFÆöæR'&öF67G2W6RF†R6ÖR–æ—F–Æ—¦VB6Æ–VçB2æ÷&ÖÀ ’òòVçfVÆ÷W2âF†—2&W6W'fW2F†R6ö6¶WB6öææV7F–öâæB—G2FVÆ—fW' ’òò6¶æ÷vÆVFvVÖVçB–ç7FVBöb7&VF–ær6V6öæB6†÷'BÖÆ—fVB6ö6¶WBà —&WGW&â6VæE6ö6¶WDVçfVÆ÷R‡6W'fW"ÂVçfVÆ÷R“° –66R…EE  —&WGW&â6VæD‡GGVçfVÆ÷R‡6W'fW"ÂVçfVÆ÷R“° –FVfVÇC  —&WGW&âfÇ6S° —Р—Р ’ò¢  ’¢6VæG2&Wv&BÖ&V&–ærf÷FRVçfVÆ÷RæB&W÷'G2v†WF†W"F†R6VÆV7FV@ ’¢G&ç7÷'B66WFVB—BâÆVv7’G&ç7÷'G2&WF–âF†V—"W†—7F–ær7–æ6‡&öæ÷W0 ’¢6VÖçF–73²…EEW‡÷6W2—G2&÷VæFVB×VWVR&W7VÇB6òf÷FR—2æWfW"F—66&FV@ ’¢v†VâF†RVWVR—2gVÆÂà ’¢ð —&÷FV7FVB&ööÆVâ6VæEf÷FTVçfVÆ÷T66WFVB…7G&–ær6W'fW"–çBFVĤ6öäVçfVÆ÷RVçfVÆ÷R’° ––b†ÖWF†öBÓÒ'VævVTÖWF†öBä…EE’° —&WGW&â6VæD‡GGVçfVÆ÷R‡6W'fW"ÂVçfVÆ÷R“° —Р”vÆö&ÄÖW76vU&÷‡”†æFÆW"†æFÆW"ÒvÆö&ÄÖW76vU&÷‡”†æFÆW#° ––b††æFÆW"ÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р–†æFÆW"ç6VæDÖW76vR‡6W'fW"ÂFVÆ’ÂVçfVÆ÷R“° —&WGW&âG'VS° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–B6†V6´66†VEf÷FW2…7G&–ær6W'fW"’° ––çBFVÆ’Ò° ––b†—56W'fW%fÆ–B‡6W'fW"’’° ––b†—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡6W'fW"’’° ––b†vWEf÷FT66†T†æFÆW"‚’æ†5f÷FW2‡6W'fW"’bbvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ2ÒvWEf÷FT66†T†æFÆW"‚’ævWEf÷FW2‡6W'fW"“° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ&VÖ÷fVBÒæWr'&”Æ—7CÃâ‚“° ––b‚2æ—4V×G’‚’’° ––çBçVÒÒ° ––çBçVÖ&W$öef÷FW2Ò2ç6—¦R‚“° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢2’° ––b†66†Ræ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7E6W'fW%f÷FTFVÆ—fW'’‡6W'fW"Â66†R’’° –6öçF–çVS° —Р––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb66†RææVVG4'&öF67Döâ‡6W'fW"’’° •6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡6W'fW"’À –66†RævWEWV–B‚’Â66†RævWEÆ–W$æÖR‚’Â66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’À –66†RævWEFW‡B‚’ÂfÇ6R“° ––b†66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° ––b‚W'6—7E6W'fW%f÷FTFVÆ—fW'’‡6W'fW"Â66†R’’° –6öçF–çVS° —Р—Р—Р –&ööÆVâFõ6VæBÒG'VS° ––b†vWD6öæf–r‚’ævWEv—Df÷%W6W$öæÆ–æR‚’’° ––b‚—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’’’° —Fõ6VæBÒfÇ6S° —ÒVÇ6R–b†—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’ ’bbvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’’æWVÇ2‡6W'fW"’’° —Fõ6VæBÒfÇ6S° —Р—Р––b‡Fõ6VæB’° –&ööÆVâ'&öF67D†W&RÒ66†RææVVG4'&öF67Döâ‡6W'fW"“° ––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb'&öF67D†W&P ’bbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° –&ööÆVâÆ–W$öæÆ–æRÒ—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’“° •7G&–ærÆ–W%6W'fW"ÒÆ–W$öæÆ–æRòvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’ “¢çVÆð  •6WCÅ7G&–æsâF&vWG2Ò&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡Æ–W$öæÆ–æRÀ —Æ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡6W'fW"ÂF&vWG2“° —Р ––b‚6VæEf÷FTVçfVÆ÷T66WFVB‡6W'fW"ÂFVÆ’À •f÷F–æuÇVv–åv—&Rçf÷FR†66†RævWEÆ–W$æÖR‚’Â66†RævWEWV–B‚’À –66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’ÂfÇ6RÂ66†Ræ—5&VÅf÷FR‚’À –66†RævWEFW‡B‚’Â66†RævWEf÷FT–B‚’ÂvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’À –'&öF67D†W&RÂçVÒÂçVÖ&W$öef÷FW2’’’° –FV'Vr‚%&WF–æ–ær66†VBf÷FR&V6W6RF†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²6W'fW"“° –6öçF–çVS° —Р–FVÆ’²³° –çVÒ²³° —&VÖ÷fVBæFB†66†R“° —ÒVÇ6R° –FV'Vr‚$æ÷B6VæF–ærf÷FR&V6W6RW6W"—6âwBöâ6W'fW""²6W'fW"²#¢  ’²66†RçFõ7G&–ær‚’“° —Р—Р–vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fU6W'fW%f÷FW2‡6W'fW"Â&VÖ÷fVB“° —ÒVÇ6R° –FV'Vr‚$æò66†VBf÷FW2f÷"6W'fW#¢"²6W'fW"“° —Р—ÒVÇ6R° –FV'Vr‚$æò66†VBf÷FW2f÷"6W'fW#¢"²6W'fW"“° —Р—Р—ÒVÇ6R° –FV'Vr‚%6W'fW"æ÷BfÆ–C¢"²6W'fW"“° —Р—Р —V&Æ–27–æ6‡&öæ—¦VBfö–B6†V6´öæÆ–æUf÷FW2…7G&–ærÆ–W"Â7G&–ærWV–BÂ7G&–ær6W'fW"’° ––çBFVÆ’Ò° ––b†—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær‡Æ–W"’bbvWEf÷FT66†T†æFÆW"‚’æ†4öæÆ–æUf÷FW2‡WV–B’’° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ2ÒvWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2‡WV–B“° ––b‚2æ—4V×G’‚’’° ––b‡6W'fW"ÓÒçVÆÂ’° —6W'fW"ÒvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær‡Æ–W"“° —Р––b‚vWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° ––çBçVÒÒ° ––çBçVÖ&W$öef÷FW2Ò†–çB’2ç7G&VÒ‚’æf–ÇFW"‡f÷FRÓâf÷FRæ—5&Wv&DFVÆ—fW&VB‚’’æ6÷VçB‚“° –&ööÆVâFVÆ—fW&VE&Wv&BÒfÇ6S° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ&WF–æVBÒæWr'&”Æ—7CÃâ‚“° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢2’° ––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’’° •6WCÅ7G&–æsâVæF–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ†66†RævWD'&öF67EF&vWG2‚’“° —VæF–æuF&vWG2ç&VÖ÷fTƆ66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b†&Æö6¶VE6W'fW'2ÒçVÆÂ’° —VæF–æuF&vWG2ç&VÖ÷fTƆ&Æö6¶VE6W'fW'2“° —Р–66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƇ6VæE&÷‡”'&öF67B‡VæF–æuF&vWG2À –66†RævWEWV–B‚’Â66†RævWEÆ–W$æÖR‚’Â66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’À –66†RævWEFW‡B‚’ÂfÇ6R’“° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° —Р–&ööÆVâ'&öF67D†W&RÒ66†RææVVG4'&öF67Döâ‡6W'fW"“° ––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb'&öF67D†W&P ’bbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° •7G&–ærÆ–W%6W'fW"Ò‡6W'fW"ÒçVÆÂ’ò6W'fW"¢vWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær‡Æ–W"“°  •6WCÅ7G&–æsâF&vWG2Ò&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡G'VRÂÆ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡6W'fW"ÂF&vWG2“° —Р ––b‚66†Ræ—5&Wv&DFVÆ—fW&VB‚’’° ––b‚6VæEf÷FTVçfVÆ÷T66WFVB‡6W'fW"ÂFVÆ’À •f÷F–æuÇVv–åv—&Rçf÷FTöæÆ–æR†66†RævWEÆ–W$æÖR‚’Â66†RævWEWV–B‚’Â66†RævWE6W'f–6R‚’À –66†RævWEF–ÖR‚’ÂfÇ6RÂ66†Ræ—5&VÅf÷FR‚’Â66†RævWEFW‡B‚’Â66†RævWEf÷FT–B‚’À –vWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’Â'&öF67D†W&RÂçVÒÂçVÖ&W$öef÷FW2’’’° –FV'Vr‚%&WF–æ–æröæÆ–æRf÷FR&V6W6RF†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²6W'fW"“° —&WF–æVBæFB†66†R“° –6öçF–çVS° —Р’òòF†Ræ÷&ÖÂVçfVÆ÷R—2Ç6òfÆ–B'&öF67BFVÆ—fW'’f÷"F†P ’òò7W'&VçBF&vWBâ&V6÷&B—B6ò&Wf–÷W6Ç’VæF–ær7FæFÆöæP ’òò&WG'’6ææ÷Bææ÷Væ6RF†R6ÖRf÷FRv–âÆFW"à ––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb'&öF67D†W&R’° –66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFB‡6W'fW"“° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° —Р–66†Rç6WE&Wv&DFVÆ—fW&VB‡G'VR“° –FVÆ—fW&VE&Wv&BÒG'VS° –FVÆ’²³° –çVÒ²³° —Р ––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° —&WF–æVBæFB†66†R“° —Р—Р–vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fTöæÆ–æUf÷FW2‡WV–B“° –f÷"„öffÆ–æT'VævVUf÷FRVæF–ær¢&WF–æVB’° –vWEf÷FT66†T†æFÆW"‚’æFDöæÆ–æUf÷FR‡WV–BÂVæF–ær“° —Р ’òò×VÇF—&÷‡“¢VçfVÆ÷RÖöæÇ ––b†FVÆ—fW&VE&Wv&BbbvWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚ ’bbvWD6öæf–r‚’ævWD×VÇF•&÷‡”öæTvÆö&Å&Wv&B‚’’° –×VÇF•&÷‡”†æFÆW"ç6VæD6ÆV%f÷FR‡WV–BÂÆ–W"“° —Р—Р—Р—Р—Р ’ò¢  ’¢&WG&–W2f÷FW"Ö¶W–VB7FæFÆöæR'&öF67G2v†Vâç’Æ–W"Ö¶W2F&vW@ ’¢&6¶VæBf–Æ&ÆR2ÇVv–âÖÖW76vR6'&–W"à ’  ’¢&Ò6W'fW"&6¶VæB6W'fW"F†Bv–æVB6'&–W  ’¢ð —&÷FV7FVB7–æ6‡&öæ—¦VBfö–B&WG'•VæF–ætöæÆ–æT'&öF67G2…7G&–ær6W'fW"’° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b‡6W'fW"ÓÒçVÆÂdž&Æö6¶VE6W'fW'2ÒçVÆÂbb&Æö6¶VE6W'fW'2æ6öçF–ç2‡6W'fW"’’’° —&WGW&ã° —Р–f÷"…7G&–ær66†VEWV–B¢vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FUUT”G2‚’’° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2†66†VEWV–B’’’° ––b†66†Ræ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R’’° –6öçF–çVS° —Р––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂ66†RææVVG4'&öF67Döâ‡6W'fW"’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡6W'fW"’Â66†RævWEWV–B‚’À –66†RævWEÆ–W$æÖR‚’Â66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’Â66†RævWEFW‡B‚’ÂfÇ6R“° ––b†66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° ––b†66†Ræ—5&Wv&DFVÆ—fW&VB‚’bb66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° –vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fTöæÆ–æUf÷FR†66†VEWV–BÂ66†R“° —ÒVÇ6R° —W'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R“° —Р—Р—Р—Р—Р —&÷FV7FVB7–æ6‡&öæ—¦VBfö–B&WG'•VæF–æuF–ÖT'&öF67G2…7G&–ær6W'fW"’° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b‡6W'fW"ÓÒçVÆÂdž&Æö6¶VE6W'fW'2ÒçVÆÂbb&Æö6¶VE6W'fW'2æ6öçF–ç2‡6W'fW"’’’° —&WGW&ã° —Р––b†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’ÓÒçVÆÂ’° —&WGW&ã° —Р–f÷"…f÷FUF–ÖUVWVRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR’’° –6öçF–çVS° —Р––b‚f÷FRæ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂf÷FRævWEWV–B‚’æ—4V×G’‚’ÇÂf÷FRævWD'&öF67EF&vWG2‚’æ6öçF–ç2‡6W'fW" —ÇÂf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡6W'fW"’Âf÷FRævWEWV–B‚’Âf÷FRævWDæÖR‚’À —f÷FRævWE6W'f–6R‚’Âf÷FRævWEF–ÖR‚’Âf÷FRævWEF÷FÇ2‚’ÂfÇ6R“° ––b‡f÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° —W'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR“° —Р—Р—Р ’ò¢  ’¢W&–öF–6ÆÇ’&WG&–W2WfW'’VæF–ærf÷FW"Ö¶W–VB7FæFÆöæR'&öF67BâF†—2—0 ’¢&WV—&VBf÷"'&ö¶W"G&ç7÷'G2v†÷6R&V6÷fW'’FöW2æ÷B&öGV6RÆ–W"ÖÆöv–à ’¢6'&–W"WfVçBà ’¢ð —V&Æ–27–æ6‡&öæ—¦VBfö–B&WG'•VæF–ætöæÆ–æT'&öF67G2‚’° –f÷"…7G&–ær66†VEWV–B¢æWrÆ–æ¶VD†6…6WCÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FUUT”G2‚’’’° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2†66†VEWV–B’’’° ––b†66†Ræ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R’’° –6öçF–çVS° —Р––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂ66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâVæF–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ†66†RævWD'&öF67EF&vWG2‚’“° —VæF–æuF&vWG2ç&VÖ÷fTƆ66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b†&Æö6¶VE6W'fW'2ÒçVÆÂ’° —VæF–æuF&vWG2ç&VÖ÷fTƆ&Æö6¶VE6W'fW'2“° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B‡VæF–æuF&vWG2Â66†RævWEWV–B‚’Â66†RævWEÆ–W$æÖR‚’À –66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’Â66†RævWEFW‡B‚’ÂfÇ6R“° ––b†66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° ––b†66†Ræ—5&Wv&DFVÆ—fW&VB‚’bb66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° –vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fTöæÆ–æUf÷FR†66†VEWV–BÂ66†R“° —ÒVÇ6R° —W'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R“° —Р—Р—Р—Р—&WG'•VæF–æuF–ÖT'&öF67G2‚“° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–B&WG'•VæF–æuF–ÖT'&öF67G2‚’° ––b†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’ÓÒçVÆÂ’° —&WGW&ã° —Р–f÷"…f÷FUF–ÖUVWVRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR’’° –6öçF–çVS° —Р––b‚f÷FRæ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂf÷FRævWEWV–B‚’æ—4V×G’‚’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâVæF–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ‡f÷FRævWD'&öF67EF&vWG2‚’“° —VæF–æuF&vWG2ç&VÖ÷fTƇf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b†&Æö6¶VE6W'fW'2ÒçVÆÂ’° —VæF–æuF&vWG2ç&VÖ÷fTƆ&Æö6¶VE6W'fW'2“° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B‡VæF–æuF&vWG2Âf÷FRævWEWV–B‚’Âf÷FRævWDæÖR‚’Âf÷FRævWE6W'f–6R‚’À —f÷FRævWEF–ÖR‚’Âf÷FRævWEF÷FÇ2‚’ÂfÇ6R“° ––b‡f÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° —W'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR“° —Р—Р—Р —&÷FV7FVB7–æ6‡&öæ—¦VB&ööÆVâW'6—7EF–ÖUf÷FTFVÆ—fW'’…f÷FUF–ÖUVWVRf÷FR’° ––b†vWEf÷FT66†T†æFÆW"‚’çWFFUF–ÖUf÷FR‡f÷FR’’° —f÷FRç6WDFVÆ—fW'•7FFTF—'G’†fÇ6R“° —&WGW&âG'VS° —Р—f÷FRç6WDFVÆ—fW'•7FFTF—'G’‡G'VR“° —66†VGVÆUF–ÖUf÷FTFVÆ—fW'•&WG'’‚“° —&WGW&âfÇ6S° —Р —&—fFRfö–B66†VGVÆUF–ÖUf÷FTFVÆ—fW'•&WG'’‚’° ––b‡F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÇÂvWE66†VGVÆW"‚’ÓÒçVÆÂ’° —&WGW&ã° —Р—F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒG'VS° —G'’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ° —7–æ6‡&öæ—¦VB…f÷F–æuÇVv–å&÷‡’çF†—2’° —F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° —Р—&WG'•VæF–æuF–ÖT'&öF67G2‚“° —ÒÂRÂF–ÖUVæ—Bå4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° —F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° –FV'Vr‚%Væ&ÆRFò66†VGVÆRF–ÖVB'&öF67B7FFR&WG'“¢"²RævWDÖW76vR‚’“° —Р—Р —&÷FV7FVB7–æ6‡&öæ—¦VB&ööÆVâW'6—7E6W'fW%f÷FTFVÆ—fW'’…7G&–ær6W'fW"ÂöffÆ–æT'VævVUf÷FRf÷FR’° ––b†vWEf÷FT66†T†æFÆW"‚’çWFFU6W'fW%f÷FR‡6W'fW"Âf÷FR’’° —f÷FRç6WDFVÆ—fW'•7FFTF—'G’†fÇ6R“° —&WGW&âG'VS° —Р—f÷FRç6WDFVÆ—fW'•7FFTF—'G’‡G'VR“° —66†VGVÆT66†VEf÷FTFVÆ—fW'•&WG'’‚“° —&WGW&âfÇ6S° —Р —&÷FV7FVB7–æ6‡&öæ—¦VB&ööÆVâW'6—7DöæÆ–æUf÷FTFVÆ—fW'’…7G&–ærWV–BÂöffÆ–æT'VævVUf÷FRf÷FR’° ––b†vWEf÷FT66†T†æFÆW"‚’çWFFTöæÆ–æUf÷FR‡WV–BÂf÷FR’’° —f÷FRç6WDFVÆ—fW'•7FFTF—'G’†fÇ6R“° —&WGW&âG'VS° —Р—f÷FRç6WDFVÆ—fW'•7FFTF—'G’‡G'VR“° —66†VGVÆT66†VEf÷FTFVÆ—fW'•&WG'’‚“° —&WGW&âfÇ6S° —Р —&—fFRfö–B66†VGVÆT66†VEf÷FTFVÆ—fW'•&WG'’‚’° ––b†66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÇÂvWE66†VGVÆW"‚’ÓÒçVÆÂ’° —&WGW&ã° —Р–66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒG'VS° —G'’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ° —7–æ6‡&öæ—¦VB…f÷F–æuÇVv–å&÷‡’çF†—2’° –66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° —Р—&WG'”66†VEf÷FTFVÆ—fW'•W'6—7FVæ6R‚“° —ÒÂRÂF–ÖUVæ—Bå4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° –66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° –FV'Vr‚%Væ&ÆRFò66†VGVÆR66†VB'&öF67B7FFR&WG'“¢"²RævWDÖW76vR‚’“° —Р—Р —&—fFR7–æ6‡&öæ—¦VBfö–B&WG'”66†VEf÷FTFVÆ—fW'•W'6—7FVæ6R‚’° –f÷"…7G&–ær6W'fW"¢vWEf÷FT66†T†æFÆW"‚’ævWD66†VEf÷FW56W'fW'2‚’’° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWEf÷FW2‡6W'fW"’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’’° —W'6—7E6W'fW%f÷FTFVÆ—fW'’‡6W'fW"Âf÷FR“° —Р—Р—Р–f÷"…7G&–ærWV–B¢æWrÆ–æ¶VD†6…6WCÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FUUT”G2‚’’’° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2‡WV–B’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’’° —W'6—7DöæÆ–æUf÷FTFVÆ—fW'’‡WV–BÂf÷FR“° —Р—Р—Р—Р —V&Æ–2fö–B6†V6µf÷FU'G’‚’° ––b†vWD6öæf–r‚’ævWEf÷FU'G”Væ&ÆVB‚’’° ––b‡f÷FU'G•f÷FW2ãÒ7W'&VçEf÷FU'G•f÷FW5&WV—&VB’° –FV'Vr‚%f÷FR'G’&V6†VB"“° –FD7W'&VçEf÷FU'G•f÷FW2‚Ö7W'&VçEf÷FU'G•f÷FW5&WV—&VB“°  –7W'&VçEf÷FU'G•f÷FW5&WV—&VB³ÒvWD6öæf–r‚’ævWEf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚“° —6WEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB€ –vWEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚’²vWD6öæf–r‚’ævWEf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚’“°  ––b‚vWD6öæf–r‚’ævWEf÷FU'G”'&öF67B‚’æ—4V×G’‚’’° –'&öF67B†vWD6öæf–r‚’ævWEf÷FU'G”'&öF67B‚’“° —Р –f÷"…7G&–ær6öÖÖæB¢vWD6öæf–r‚’ævWEf÷FU'G”'VævVT6öÖÖæG2‚’’° —'Vä6öç6öÆT6öÖÖæB†6öÖÖæB“° —Р ––b†vWD6öæf–r‚’ævWEf÷FU'G•6VæEFôÆÅ6W'fW'2‚’’° –f÷"…7G&–ær6W'fW"¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° —6VæEf÷FU'G’‡6W'fW"“° —Р—ÒVÇ6R° –f÷"…7G&–ær6W'fW"¢vWD6öæf–r‚’ævWEf÷FU'G•6W'fW'5Fõ6VæB‚’’° —6VæEf÷FU'G’‡6W'fW"“° —Р—Р—Р—6fUf÷FT66†Tf–ÆR‚“° —Р—Р —V&Æ–2'7G&7Bfö–BFV'Vr…7G&–ær7G"“°  —&—fFRfö–BFV'Vs"…7G&–ærÖW76vR’° –FV'Vr†ÖW76vR“° —Р ’ò¢  ’¢…EE6Æ–VçBW6VBf÷"Öö¦ær’&WVW7G2à ’¢ð —&—fFRf–æ‡GG6Æ–VçB‡GG6Æ–VçBÒ‡GG6Æ–VçBææWt'V–ÆFW"‚’æ6öææV7EF–ÖV÷WB„GW&F–öâæöe6V6öæG2ƒR’’æ'V–ÆB‚“°  ’ò¢  ’¢fWF6†W2Æ–W"w2UT”Bg&öÒF†RÖö¦ær’à ’  ’¢&ÒÆ–W$æÖRÆ–W"æÖP ’¢&WGW&âÆ–W"UT”BÂ÷"´6öFRçVÆÇÒ–bæ÷Bf÷Væ@ ’¢F‡&÷w2”ôW†6WF–öâ–bF†R&WVW7Bf–Ç0 ’¢F‡&÷w2–çFW''WFVDW†6WF–öâ–b–çFW''WFVBv†–ÆRv—F–ærf÷"F†R&W7öç6P ’¢ð —V&Æ–2UT”BfWF6…UT”B…7G&–ærÆ–W$æÖR’F‡&÷w2”ôW†6WF–öâ–çFW''WFVDW†6WF–öâ° ––b‡Æ–W$æÖRÓÒçVÆÂÇÂÆ–W$æÖRæWVÇ4–væ÷&T66R‚&çVÆÂ"’’° —&WGW&âçVÆð —Р ”‡GG&WVW7B&WVW7BÒ‡GG&WVW7BææWt'V–ÆFW"‚ ’çW&’…U$’æ7&VFR‚&‡GG3¢òö’æÖö¦æræ6öÒ÷W6W'2÷&öf–ÆW2öÖ–æV7&gBò"²Æ–W$æÖR’’ätUB‚ ’çF–ÖV÷WB„GW&F–öâæöe6V6öæG2ƒR’’æ'V–ÆB‚“°  ”‡GG&W7öç6SÅ7G&–æsâ&W7öç6RÒ‡GG6Æ–VçBç6VæB‡&WVW7B‡GG&W7öç6Rä&öG”†æFÆW'2æöe7G&–ær‚’“°  ––b‡&W7öç6Rç7FGW46öFR‚’ÓÒCÇÂ&W7öç6Rç7FGW46öFR‚’ÓÒCB’° –Æör‚%F†W&R—2æòÆ–W"v—F‚F†RæÖRÂ""²Æ–W$æÖR²%Â""“° —&WGW&âçVÆð —Р ––b‡&W7öç6Rç7FGW46öFR‚’Â#ÇÂ&W7öç6Rç7FGW46öFR‚’ãÒ3’° —F‡&÷ræWr”ôW†6WF–öâ‚$f–ÆVBFòfWF6‚UT”Bf÷""²Æ–W$æÖR²"Â…EE"²&W7öç6Rç7FGW46öFR‚’“° —Р ”§6öäVÆVÖVçBVÆVÖVçBÒ§6öå'6W"ç'6U7G&–ær‡&W7öç6Ræ&öG’‚’“° ––b†VÆVÖVçBÓÒçVÆÂÇÂVÆVÖVçBæ—4§6öäö&¦V7B‚’’° —&WGW&âçVÆð —Р ”§6öäö&¦V7Bö&¦V7BÒVÆVÖVçBævWD4§6öäö&¦V7B‚“° ––b‚ö&¦V7Bæ†2‚&–B"’ÇÂö&¦V7BævWB‚&–B"’æ—4§6öäçVÆ‚’’° —&WGW&âçVÆð —Р •7G&–ærWV–D57G&–ærÒö&¦V7BævWB‚&–B"’ævWD57G&–ær‚“° —&WGW&â'6UUT”Dg&öÕ7G&–ær‡WV–D57G&–ær“° —Р —V&Æ–2'7G&7B6WCÅ7G&–æsâvWDÆÄf–Æ&ÆU6W'fW'2‚“°  ’ò¢¢6ö×ÆWFRÆFf÷&Ò6W'fW"6WB&Vf÷&Rv†—FVÆ—7Bö&Æö6¶VB&÷WF–ærf–ÇFW'2â¢ð —V&Æ–2'7G&7B6WCÅ7G&–æsâvWDÆÄ6öæf–wW&VE6W'fW'2‚“°  —V&Æ–2'7G&7Bf÷F–æuÇVv–å&÷‡”6öæf–rvWD6öæf–r‚“°  —V&Æ–2'7G&7B7G&–ærvWD7W'&VçEÆ–W%6W'fW"…7G&–ærÆ–W"“°  ’ò¢  ’¢&W6öÇfW2Æ–W"w26W'fW"f÷"f÷FR&÷WF–ærâFVF–6FVBf÷F–ær&÷‡’†2æð ’¢Æö6ÂÆ–W'2Â6ò—BW6W2F†R&6¶VæB&W6Væ6RG&6¶W"–ç7FVBà ’¢ð —&÷FV7FVB7G&–ærvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær…7G&–ærÆ–W"’° ––b†—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’’° —&WGW&â&6¶VæEÆ–W%&W6Væ6UG&6¶W"ævWEÆ–W"‡Æ–W"’æÖ‡&W6Væ6RÓâ&W6Væ6RævWE6W'fW"‚’’æ÷$VÇ6R†çVÆ“° —Р—&WGW&âvWD7W'&VçEÆ–W%6W'fW"‡Æ–W"“° —Р ’ò¢  ’¢FVF–6FVB&÷WF–ær—2–çFVçF–öæÆÇ’Væf–Æ&ÆRöâÇVv–âÖW76v–æs¢F†@ ’¢G&ç7÷'B—2GF6†VBFòÆ–W"Öf6–ær&÷‡’æBFöW2æ÷B6''’&6¶Væ@ ’¢&W6Væ6R6æ6†÷G2à ’¢ð —&÷FV7FVB&ööÆVâ—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’° —&WGW&âvWD6öæf–r‚’ævWDFVF–6FVEf÷F–æu&÷‡’‚’bbÖWF†öBÒçVÆÂbbÖWF†öBç7W÷'G4&6¶VæE&W6Væ6R‚“° —Р —V&Æ–2'7G&7Bf–ÆRvWDFFföÆFW%ÇVv–â‚“°  —V&Æ–27G&–ærvWDÖöçF…F÷FÇ5v—F„FFUF‚‚’° ”Æö6ÄFFUF–ÖR5F–ÖRÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚“° —&WGW&âvWDÖöçF…F÷FÇ5v—F„FFUF‚†5F–ÖR“° —Р —V&Æ–27G&–ærvWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖR5F–ÖR’° —&WGW&â$ÖöçF…F÷FÂÒ"²5F–ÖRævWDÖöçF‚‚’çFõ7G&–ær‚’²"Ò"²5F–ÖRævWE–V"‚“° —Р —V&Æ–2'7G&7B7G&–ærvWE&÷W$æÖR…7G&–ærWV–BÂ7G&–ærÆ–W$æÖR“°  —V&Æ–2'7G&7B7G&–ærvWEUT”B…7G&–ærÆ–W$æÖR“°  —&—fFR–çBvWEfÇVR„'&”Æ—7CÄ6öÇVÖãâ6öÇ2Â7G&–ær6öÇVÖâ–çBFôFB’° –f÷"„6öÇVÖâB¢6öÇ2’° ––b†BævWDæÖR‚’æWVÇ4–væ÷&T66R†6öÇVÖâ’’° ”FFfÇVRfÇVRÒBævWEfÇVR‚“° ––çBçVÒÒ° ––b‡fÇVRÓÒçVÆÂ’° —&WGW&âFôFC° —Р––b‡fÇVRæ—4–çB‚’’° –çVÒÒfÇVRævWD–çB‚“° —ÒVÇ6R–b‡fÇVRæ—57G&–ær‚’’° —G'’° –çVÒÒ–çFVvW"ç'6T–çB‡fÇVRævWE7G&–ær‚’“° —Ò6F6‚„W†6WF–öâR’° ’òò–væ÷&P —Р—Р—&WGW&âçVÒ²FôFC° —Р—Р—&WGW&âFôFC° —Р —&—fFRf÷FUF÷FÇ56æ6†÷BvWE&ö¦V7FVE&öÆÆ÷fW%F÷FÇ2„'&”Æ—7CÄ6öÇVÖãâFFÂ7G&–ærÆ–W"’° ”Æ—7CÅF–ÖUG—SâF–ÖT6†ævW2ÒvWDvÆö&ÄFF†æFÆW"‚’ævWEF–ÖT6†ævW2‚“° –&ööÆVâ&W6WDÖöçF‚ÒF–ÖT6†ævW2æ6öçF–ç2…F–ÖUG—RäÔôåD‚“° –&ööÆVâ&W6WEvVV²ÒF–ÖT6†ævW2æ6öçF–ç2…F–ÖUG—RåtTT²“° –&ööÆVâ&W6WDF’ÒF–ÖT6†ævW2æ6öçF–ç2…F–ÖUG—RäD’“° ––çB66WFVEVWVVEf÷FW2Ò° ––çB66WFVDvÆö&ÅVWVVEf÷FW2Ò° –f÷"…f÷FUF–ÖUVWVRVWVVB¢vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’° ––b‚VWVVBæ—5&ö6W76VB‚’’° –66WFVDvÆö&ÅVWVVEf÷FW2²³° —Р––b‚VWVVBæ—5&ö6W76VB‚’bbVWVVBævWDæÖR‚’ÒçVÆÂbbVWVVBævWDæÖR‚’æWVÇ4–væ÷&T66R‡Æ–W"’’° –66WFVEVWVVEf÷FW2²³° —Р—Р––çBf÷FT–æ7&VÖVçBÒ66WFVEVWVVEf÷FW2²°  ––çBÆÅF–ÖUF÷FÂÒvWEfÇVR†FFÂ$ÆÅF–ÖUF÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBÖöçF…F÷FÂÒ&W6WDÖöçF‚òf÷FT–æ7&VÖVçB¢vWEfÇVR†FFÂ$ÖöçF…F÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBvVV¶Ç•F÷FÂÒ&W6WEvVV²òf÷FT–æ7&VÖVçB¢vWEfÇVR†FFÂ%vVV¶Ç•F÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBF–Ç•F÷FÂÒ&W6WDF’òf÷FT–æ7&VÖVçB¢vWEfÇVR†FFÂ$F–Ç•F÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBö–çG2ÒvWEfÇVR†FFÂ%ö–çG2"Âf÷FT–æ7&VÖVçB¢vWD6öæf–r‚’ævWEö–çG4öåf÷FR‚’“°  ––çBÖ…f÷FW2ÒvWD6öæf–r‚’ævWDÖ„Ö÷VçDöef÷FW5W$F’‚“° ––b†Ö…f÷FW2â’° ––çBF—2ÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚’ævWDF”ödÖöçF‚‚“° ––b†ÖöçF…F÷FÂâF—2¢Ö…f÷FW2’° –ÖöçF…F÷FÂÒF—2¢Ö…f÷FW3° —Р—Р––b†vWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’âbbö–çG2âvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’’° —ö–çG2ÒvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚“° —Р ––çBFFTÖöçF…F÷FÂÒÓ° ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° ––b†vWD6öæf–r‚’ævWEW6TÖöçF„FFUF÷FÇ45&–Ö'•F÷F‚’’° –FFTÖöçF…F÷FÂÒ&W6WDÖöçF‚òf÷FT–æ7&VÖVç@ “¢vWEfÇVR†FFÂvWDÖöçF…F÷FÇ5v—F„FFUF‚‚’Âf÷FT–æ7&VÖVçB“° —ÒVÇ6R° –FFTÖöçF…F÷FÂÒÖöçF…F÷Fð —Р—Р ––çEµÒ&ö¦V7FVEf÷FU'G’ÒvWE&ö¦V7FVEf÷FU'G•7FFR†66WFVDvÆö&ÅVWVVEf÷FW2²“° —&WGW&âæWrf÷FUF÷FÇ56æ6†÷B†ÆÅF–ÖUF÷FÂÂÖöçF…F÷FÂÂvVV¶Ç•F÷FÂÂF–Ç•F÷FÂÂö–çG2À —&ö¦V7FVEf÷FU'G•³ÒÂ&ö¦V7FVEf÷FU'G•³ÒÂFFTÖöçF…F÷F“° —Р —&÷FV7FVB&ööÆVâ6äf÷'v&E7FæFÆöæT'&öF67B†&ööÆVâÖævW5F÷FÇ2’° —&WGW&âÖævW5F÷FÇ3° —Р —&÷FV7FVB–çEµÒvWE&ö¦V7FVEf÷FU'G•7FFR†–çB66WFVEf÷FW2’° ––çB7W'&VçBÒf÷FU'G•f÷FW3° ––çB&WV—&VBÒ7W'&VçEf÷FU'G•f÷FW5&WV—&VC° ––b‚vWD6öæf–r‚’ævWEf÷FU'G”Væ&ÆVB‚’’° —&WGW&âæWr–çEµÒ²7W'&VçBÂ&WV—&VBÓ° —Р ––çB–æ7&V6RÒvWD6öæf–r‚’ævWEf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚“° –f÷"†–çB’Ò²’Â66WFVEf÷FW3²’²²’° –7W'&VçB²³° ––b†7W'&VçBãÒ&WV—&VB’° –7W'&VçBÓÒ&WV—&VC° —&WV—&VB³Ò–æ7&V6S° —Р—Р—&WGW&âæWr–çEµÒ²7W'&VçBÂ&WV—&VBÓ° —Р —V&Æ–2'7G&7B7G&–ærvWEÇVv–åfW'6–öâ‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†T7W'&VçEf÷FU'G•f÷FW2‚“°  —V&Æ–2'7G&7BÆöærvWEf÷FT66†TÆ7EWFFVB‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†U&WdF’‚“°  —V&Æ–2'7G&7B7G&–ærvWEf÷FT66†U&WdÖöçF‚‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†U&WevVV²‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚“°  —V&Æ–2'7G&7B&ööÆVâ—5Æ–W$öæÆ–æR…7G&–ærÆ–W$æÖR“°  ’ò¢  ’¢6†V6·2öæÆ–æR7FFRf÷"f÷FR&÷WF–ærÂW6–ær&6¶VæB&W6Væ6RöæÇ’v†VâF†—0 ’¢&÷‡’—2W‡Æ–6—FÇ’6öæf–wW&VB2F†RFVF–6FVBf÷F–ær&÷‡’à ’¢ð —&÷FV7FVB&ööÆVâ—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær…7G&–ærÆ–W$æÖR’° —&WGW&â—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’ò&6¶VæEÆ–W%&W6Væ6UG&6¶W"ævWEÆ–W"‡Æ–W$æÖR’æ—5&W6VçB‚ “¢—5Æ–W$öæÆ–æR‡Æ–W$æÖR“° —Р —V&Æ–2'7G&7B&ööÆVâ—56W'fW%fÆ–B…7G&–ær6W'fW"“°  —V&Æ–2'7G&7B&ööÆVâ—56öÖVöæTöæÆ–æU6W'fW"…7G&–ær6W'fW"“°  —&÷FV7FVB&ööÆVâ—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær…7G&–ær6W'fW"’° ––b‚—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’’° —&WGW&â—56öÖVöæTöæÆ–æU6W'fW"‡6W'fW"“° —Р–6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’ç&W6Væ6Rä&6¶VæE&W6Væ6U7FGW27FGW2Ò&6¶VæEÆ–W%&W6Væ6UG&6¶W  ’ævWD&6¶VæE7FGW2‡6W'fW"“° —&WGW&â7FGW2ÒçVÆÂbb7FGW2æ—4f–Æ&ÆR‚’bb7FGW2ævWEÆ–W$6÷VçB‚’â° —Р —V&Æ–2'7G&7B&ööÆVâ—5f÷FT66†T–væ÷&UF–ÖR‚“°  —V&Æ–2'7G&7B×—7Ä6öæf–rvWEf÷FT66†Tו5Ä6öæf–r‚“°  —V&Æ–2'7G&7B×—7Ä6öæf–rvWDæöåf÷FVD66†Tו5Ä6öæf–r‚“°  —V&Æ–2'7G&7B×—7Ä6öæf–rvWEf÷FTÆövv–ætו5Ä6öæf–r‚“°  ’ò¢  ’¢6‡WFF÷vâו5Â×&VÆFVB&W6÷W&6W26fVÇ’à ’¢ð —V&Æ–2fö–B6‡WFF÷väו7‚’° ––b†vWE&÷‡”×—7ÄÖW76VævW"‚’ÒçVÆÂ’° –vWE&÷‡”×—7ÄÖW76VævW"‚’ç6‡WFF÷vâ‚“° —6WE&÷‡”×—7ÄÖW76VævW"†çVÆ“° —Р ––b†vWE&÷‡”ו5‚’ÒçVÆÂ’° –vWE&÷‡”ו5‚’ç6‡WFF÷vâ‚“° —6WE&÷‡”ו5†çVÆ“° —Р—Р —V&Æ–2fö–BÆöB„•f÷FT66†R§6öå7F÷&vR”æöåf÷FVEÆ–W'57F÷&vRæöåf÷FVD66†T§6öâ’° –ÖWF†öBÒ'VævVTÖWF†öBævWD'”æÖR†vWD6öæf–r‚’ævWD'VævVTÖWF†öB‚’“° ––b†vWDÖWF†öB‚’ÓÒçVÆÂ’° –ÖWF†öBÒ'VævVTÖWF†öBåÅTt”äÔU54t”äs° —Р—v&åVç7W÷'FVDFVF–6FVEf÷F–æu&÷‡”ÖöFR‚“° —WV–EÆ–W$æÖT66†RÒvWE&÷‡”ו5‚’ævWE&÷w5UT”DæÖUVW'’‚“°  –'VævVUF–ÖT6†V6¶W"ç6WEF–ÖT6†ævTf–Å6fT'—72†vWD6öæf–r‚’ævWEF–ÖT6†ævTf–Å6fT'—72‚’“° –'VævVUF–ÖT6†V6¶W"æÆöEF–ÖW"‚“°  —f÷FT66†T†æFÆW"ÒæWrf÷FT66†T†æFÆW"†vWEf÷FT66†Tו5Ä6öæf–r‚’ÂvWD6öæf–r‚’ævWEf÷FT66†UW6Tו5‚’À –vWD6öæf–r‚’ævWEf÷FT66†UW6TÖ–äו5‚’ÂvWE&÷‡”ו5‚’ævWD×—7‚’ÂvWD6öæf–r‚’ævWDFV'Vr‚’À –§6öå7F÷&vR’°  ”÷fW'&–FP —V&Æ–2fö–BÆöt–æfó…7G&–ær×6r’° –Æöt–æfò†×6r“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&S…7G&–ær×6r’° –Æöu6WfW&R†×6r“° —Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vs„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vs…7G&–ær×6r’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –FV'Vr†×6r“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vs…F‡&÷v&ÆRR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р—Ó° —f÷FT66†T†æFÆW"æÆöB‚“°  –æöåf÷FVEÆ–W'466†RÒæWræöåf÷FVEÆ–W'466†R†vWDæöåf÷FVD66†Tו5Ä6öæf–r‚’À –vWD6öæf–r‚’ævWDæöåf÷FVD66†UW6Tו5‚’ÂvWD6öæf–r‚’ævWDæöåf÷FVD66†UW6TÖ–äו5‚’À –vWE&÷‡”ו5‚’ævWD×—7‚’Âæöåf÷FVD66†T§6öâÂvWD6öæf–r‚’ævWDFV'Vr‚’’°  ”÷fW'&–FP —V&Íx÷Þm¢G§²ÚîÆ­yÔ”B†Ö÷7E6–t&—G2ÂÆV7E6–t&—G2“° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–B&ö6W75VWVR‚’° —v†–ÆR†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’ç6—¦R‚’â’° •f÷FUF–ÖUVWVRf÷FRÒvWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’æVÆVÖVçB‚“° ––b‚f÷FRæ—5&ö6W76VB‚’’° •f÷FUF÷FÇ56æ6†÷BVWVVEF÷FÇ2Òf÷FRævWEF÷FÇ2‚’ÓÒçVÆÂÇÂf÷FRævWEF÷FÇ2‚’æ—4V×G’‚’òçVÆÀ “¢f÷FUF÷FÇ56æ6†÷Bç'6U7F÷&vR‡f÷FRævWEF÷FÇ2‚’“° •VWVVEf÷FU&W7VÇB&W7VÇBÒf÷FR‡f÷FRævWDæÖR‚’Âf÷FRævWE6W'f–6R‚’ÂG'VRÂfÇ6RÂf÷FRævWEF–ÖR‚’ÂVWVVEF÷FÇ2À —f÷FRævWEWV–B‚’Âf÷FR“° ––b‡&W7VÇBÓÒVWVVEf÷FU&W7VÇBå$UE%’’° —66†VGVÆUF–ÖUf÷FU&WG'’‚“° —&WGW&ã° —Р––b‡&W7VÇBÓÒVWVVEf÷FU&W7VÇBåDU$Ô”äÂ’° —v&â‚%&VÖ÷f–ærFW&Ö–æÂ&öÆÆ÷fW"f÷FR"²f÷FRævWEf÷FT–B‚’²"f÷""²f÷FRævWDæÖR‚’²"ò  ’²6W'f–6U6—FUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡f÷FRævWE6W'f–6R‚’’“° —Р—Р––b‚vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fUF–ÖUf÷FR‡f÷FR’’° —66†VGVÆUF–ÖUf÷FU&WG'’‚“° —&WGW&ã° —Р—Р—Р —&—fFRfö–B66†VGVÆUF–ÖUf÷FU&WG'’‚’° ––b‡F–ÖUf÷FU&WG'•66†VGVÆVBÇÂvWE66†VGVÆW"‚’ÓÒçVÆÂ’° —&WGW&ã° —Р—F–ÖUf÷FU&WG'•66†VGVÆVBÒG'VS° —G'’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ° —7–æ6‡&öæ—¦VB…f÷F–æuÇVv–å&÷‡’çF†—2’° —F–ÖUf÷FU&WG'•66†VGVÆVBÒfÇ6S° —Р—&ö6W75VWVR‚“° —ÒÂRÂF–ÖUVæ—Bå4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° —F–ÖUf÷FU&WG'•66†VGVÆVBÒfÇ6S° –FV'Vr‚%Væ&ÆRFò66†VGVÆR&öÆÆ÷fW"f÷FR&WG'“¢"²RævWDÖW76vR‚’“° —Р—Р —V&Æ–2fö–B&VÆöB‚’° —&VÆöE'VçF–ÖR‡G'VR“° —Р ’ò¢¢Æ–W26öçG&öÂÖ÷&–v–æFVB6öæf–wW&F–öâ&VÆöBv—F†÷WB7F÷–ær—G26öææV7F÷"÷"†÷7FVB6W'f–6Râ¢ð —V&Æ–2fö–B&VÆöDg&öÔ6öçG&ö‚’° —&VÆöE'VçF–ÖR†fÇ6R“° —Р —&—fFRfö–B&VÆöE'VçF–ÖR†&ööÆVâ&W7F'D6öçG&öÅ6W'f–6W2’° –ÖWF†öBÒ'VævVTÖWF†öBævWD'”æÖR†vWD6öæf–r‚’ævWD'VævVTÖWF†öB‚’“° ––b†vWDÖWF†öB‚’ÓÒçVÆÂ’° –ÖWF†öBÒ'VævVTÖWF†öBåÅTt”äÔU54t”äs° —Р—v&åVç7W÷'FVDFVF–6FVEf÷F–æu&÷‡”ÖöFR‚“° ––b‚&W7F'D6öçG&öÅ6W'f–6W2bbÖWF†öBÓÒ'VævVTÖWF†öBå4ô4´UE2’° —&V'V–ÆE6ö6¶WD6Æ–VçG2‚“° —Р —6WD7W'&VçEf÷FU'G•f÷FW5&WV—&VB€ –vWD6öæf–r‚’ævWEf÷FU'G•f÷FW5&WV—&VB‚’²vWEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚’“° ––b‡&W7F'D6öçG&öÅ6W'f–6W2’° –ÆöD×VÇF•&÷‡•7W÷'B‚“° —&W7F'D6öçG&öÅ6W'f–6W47–æ2‚“° —Р—Р —&—fFR7–æ6‡&öæ—¦VBfö–B&V'V–ÆE6ö6¶WD6Æ–VçG2‚’° ”†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â&V'V–ÇBÒæWr†6„ÖÃâ‚“° —G'’° ”Æ—7CÅ7G&–æsâ&Æö6¶VBÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° –f÷"…7G&–ær6W'fW"¢vWD6öæf–r‚’ævWE7–v÷E6W'fW'2‚’’° ––b†&Æö6¶VBæ6öçF–ç2‡6W'fW"’’6öçF–çVS° ”ÖÅ7G&–ærÂö&¦V7CâFFÒvWD6öæf–r‚’ævWE7–v÷E6W'fW$6öæf–wW&F–öâ‡6W'fW"“° •7G&–ær†÷7BÒFFæ6öçF–ç4¶W’‚$†÷7B"’ò…7G&–ær’FFævWB‚$†÷7B"’¢"#° ––çB÷'BÒFFæ6öçF–ç4¶W’‚%÷'B"’ò†–çB’FFævWB‚%÷'B"’¢#“ƒ° —&V'V–ÇBçWB‡6W'fW"ÂæWr6Æ–VçD†æFÆW"††÷7BÂ÷'BÂVæ7'—F–öä†æFÆW"ÂvWD6öæf–r‚’ævWDFV'Vr‚’’“° —Р—Ò6F6‚…'VçF–ÖTW†6WF–öâf–ÇW&R’° —7F÷6ö6¶WD6Æ–VçG2‡&V'V–ÇB“° —F‡&÷rf–ÇW&S° —Р”†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â&Wf–÷W2Ò6Æ–VçD†æFÆW3° –6Æ–VçD†æFÆW2Ò&V'V–ÇC° —7F÷6ö6¶WD6Æ–VçG2‡&Wf–÷W2“° —Р —&—fFR7–æ6‡&öæ—¦VB&ööÆVâ6VæE6ö6¶WDVçfVÆ÷R…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ”6Æ–VçD†æFÆW"6ö6¶WD6Æ–VçBÒ6Æ–VçD†æFÆW2ÓÒçVÆÂòçVÆ¢6Æ–VçD†æFÆW2ævWB‡6W'fW"“° ––b‡6ö6¶WD6Æ–VçBÓÒçVÆÂ’&WGW&âfÇ6S° —G'’° —6ö6¶WD6Æ–VçBç6VæDVçfVÆ÷R†VçfVÆ÷R“° —&WGW&âG'VS° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° –FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р—Р —&—fFR7–æ6‡&öæ—¦VB&ööÆVâ6VæD‡GGVçfVÆ÷R…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ”‡GG&÷‡•G&ç7÷'E6W'fW"G&ç7÷'BÒ‡GGG&ç7÷'E6W'fW#° —&WGW&âG&ç7÷'BÒçVÆÂbbG&ç7÷'Bç6VæB‡6W'fW"ÂVçfVÆ÷R“° —Р —&—fFRfö–B7F'D‡GGG&ç7÷'B‚’° —G'’° •U$’VæGö–çBÒU$’æ7&VFR†vWD6öæf–r‚’ævWD‡GGV&Æ–4VæGö–çB‚’“° ––b‚&‡GG2"æWVÇ4–væ÷&T66R†VæGö–çBævWE66†VÖR‚’’ÇÂVæGö–çBævWD†÷7B‚’ÓÒçVÆÀ —ÇÂVæGö–çBævWE÷'B‚’ÓÒÇÂVæGö–çBævWE÷'B‚’âcSS3P —ÇÂVæGö–çBævWEW6W$–æfò‚’ÒçVÆÂÇÂVæGö–çBævWEVW'’‚’ÒçVÆÂÇÂVæGö–çBævWDg&vÖVçB‚’ÒçVÆÀ —džVæGö–çBævWEF‚‚’ÒçVÆÂbbVæGö–çBævWEF‚‚’æ—4V×G’‚’bb"ò"æWVÇ2†VæGö–çBævWEF‚‚’’’’° —F‡&÷ræWr–ÆÆVvÄ&wVÖVçDW†6WF–öâ‚$…EEåV&Æ–4VæGö–çB×W7B&Râ…EE2÷&–v–â"“° —Р”f–ÆRF—&V7F÷'’ÒæWrf–ÆR†vWDFFföÆFW%ÇVv–â‚’Â&‡GG"“° ”‡GGFÇ4–FVçF—G’–FVçF—G’Ò‡GGFÇ4–FVçF—G’æÆöD÷$7&VFR†F—&V7F÷'’çFõF‚‚’ÂVæGö–çBævWD†÷7B‚’“° –‡GGVç&öÆÆÖVçDWF†÷&—G’ÒæWr‡GGVç&öÆÆÖVçDWF†÷&—G’†–FVçF—G’ÂF—&V7F÷'’çFõF‚‚’“° –‡GGG&ç7÷'E6W'fW"ÒæWr‡GG&÷‡•G&ç7÷'E6W'fW"€ –æWr–æWE6ö6¶WDFG&W72†vWD6öæf–r‚’ævWD‡GG†÷7B‚’ÂvWD6öæf–r‚’ævWD‡GG÷'B‚’’–FVçF—G’À –‡GGVç&öÆÆÖVçDWF†÷&—G’ÂF—&V7F÷'’çFõF‚‚’ç&W6öÇfR‚&÷WFvö–ær×c"’ÂF†—3£¦†æFÆT‡GGG&ç7÷'DVçfVÆ÷R“° –‡GGG&ç7÷'E6W'fW"ç7F'B‚“° –Æöt–æfò‚$…EEG&ç7÷'BÆ—7FVæ–ær6V7W&VÇ’öâ"²vWD6öæf–r‚’ævWD‡GG†÷7B‚’²#¢  ’²‡GGG&ç7÷'E6W'fW"ç÷'B‚’²#²W6R÷f÷F–æwÇVv–æ'VævVR‡GG6öFRÇ6W'fW#âf÷"V6‚&6¶VæB"“° —Ò6F6‚„W†6WF–öâf–ÇW&R’° –6Æ÷6T‡GGG&ç7÷'B‚“° —F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚$…EEG&ç7÷'B6÷VÆBæ÷B7F'B6V7W&VÇ’"Âf–ÇW&R“° —Р—Р ’ò¢¢¶VW2F†RWF†VçF–6FVBÕDÅ2&6¶VæB–FVçF—G’GF6†VBFò6V7W&—G’×6Vç6—F—fR&÷‡’&÷WF–ærâ¢ð —&÷FV7FVBfö–B†æFÆT‡GGG&ç7÷'DVçfVÆ÷R„‡GG&÷‡•G&ç7÷'E6W'fW"å&V6V—fVDVçfVÆ÷R&V6V—fVB’° ––b‚—4WF†VçF–6FVD‡GGVçfVÆ÷TÆÆ÷vVB‡&V6V—fVB’’° –FV'Vr‚$–væ÷&VB…EEVçfVÆ÷Rv†÷6RÆ–W"×&W6Væ6R6Æ–ÒF–Bæ÷BÖF6‚—G2WF†VçF–6FVB&6¶VæB"“° —&WGW&ã° —Р”vÆö&ÄÖW76vU&÷‡”†æFÆW"†æFÆW"ÒvÆö&ÄÖW76vU&÷‡”†æFÆW#° ––b††æFÆW"ÓÒçVÆÂ’F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚$…EEÖW76vR&÷WFW"—2æ÷B&VG’"“° –†æFÆW"æöäÖW76vR‡&V6V—fVBæVçfVÆ÷R‚’“° —Р —&—fFR&ööÆVâ—4WF†VçF–6FVD‡GGVçfVÆ÷TÆÆ÷vVB„‡GG&÷‡•G&ç7÷'E6W'fW"å&V6V—fVDVçfVÆ÷R&V6V—fVB’° ––b‡&V6V—fVBÓÒçVÆÂÇÂ&V6V—fVBæVçfVÆ÷R‚’ÓÒçVÆÂÇÂ&V6V—fVBç6W'fW$–B‚’ÓÒçVÆÂ’&WGW&âfÇ6S° •7G&–ær7F×VE6W'fW"Ò&V6V—fVBæVçfVÆ÷R‚’ævWDf–VÆG2‚’ævWD÷$FVfVÇB…f÷F–æuÇVv–åv—&Räµõ4U%dU"Â""“° ––b‚&V6V—fVBç6W'fW$–B‚’æWVÇ4–væ÷&T66R‡7F×VE6W'fW"’’&WGW&âfÇ6S° ––b‚f÷F–æuÇVv–åv—&Rå5T%ôÄôt”âæWVÇ2‡&V6V—fVBæVçfVÆ÷R‚’ævWE7V$6†ææV‚’’’&WGW&âG'VS° •f÷F–æuÇVv–åv—&RåÆ–W%&W6Væ6TWfVçBWfVçBÒf÷F–æuÇVv–åv—&Rç&VEÆ–W%&W6Væ6TWfVçB‡&V6V—fVBæVçfVÆ÷R‚’“° –&ööÆVâÖöFW&âÒWfVçBæ6öææV7F–öä–BÒçVÆÂÇÂWfVçBæ&6¶VæD–æ6&æF–öä–BÒçVÆÀ —ÇÂWfVçBæ&6¶VæE7F'FVDBÒÂÇÂWfVçBç&W6Væ6UF–ÖW7F×Òð ––b‚ÖöFW&âÇ—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’’&WGW&âG'VS° ’òòÆ–W"Öf6–ær&÷‡’†27G&öævW"WF†÷&—G’F†âç’&6¶VæC¢—G2Æ—fP ’òòÆ–W"6öææV7F–öâ7WÆ–W2&÷F‚F†R7W'&VçB&÷WFRæB†–âöæÆ–æRÖöFR’UT”Bà —&WGW&â—4ÆVv7”Æöv–äFW7F–æF–öäWF†÷&—FF—fR†WfVçBçÆ–W"ÂWfVçBçWV–BÂ&V6V—fVBç6W'fW$–B‚’“° —Р —&—fFR7–æ6‡&öæ—¦VBfö–B6Æ÷6T‡GGG&ç7÷'B‚’° ”‡GG&÷‡•G&ç7÷'E6W'fW"G&ç7÷'BÒ‡GGG&ç7÷'E6W'fW#° –‡GGG&ç7÷'E6W'fW"ÒçVÆð –‡GGVç&öÆÆÖVçDWF†÷&—G’ÒçVÆð ––b‡G&ç7÷'BÒçVÆÂ’G&ç7÷'Bæ6Æ÷6R‚“° —Р —V&Æ–27G&–ær7&VFT‡GG6öææV7F–öä6öFR…7G&–ær6W'fW$–B’° ”‡GGVç&öÆÆÖVçDWF†÷&—G’WF†÷&—G’Ò‡GGVç&öÆÆÖVçDWF†÷&—G“° ––b†ÖWF†öBÒ'VævVTÖWF†öBä…EEÇÂWF†÷&—G’ÓÒçVÆÂ’° —F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚%F†R…EEG&ç7÷'B—2æ÷B'Vææ–ær"“° —Р—&WGW&âWF†÷&—G’æ7&VFT6öææV7F–öä6öFR‡6W'fW$–BÂU$’æ7&VFR†vWD6öæf–r‚’ævWD‡GGV&Æ–4VæGö–çB‚’’ÂGW&F–öâæödÖ–çWFW2ƒR’ ’æVæ6öFR‚“° —Р —V&Æ–2fö–B&Wfö¶T‡GG&6¶VæB…7G&–ær6W'fW$–B’° ”‡GGVç&öÆÆÖVçDWF†÷&—G’WF†÷&—G’Ò‡GGVç&öÆÆÖVçDWF†÷&—G“° ––b†ÖWF†öBÒ'VævVTÖWF†öBä…EEÇÂWF†÷&—G’ÓÒçVÆÂ’F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚%F†R…EEG&ç7÷'B—2æ÷B'Vææ–ær"“° –WF†÷&—G’ç&Wfö¶R„‡GGFÇ4–FVçF—G’æ6æöæ–6Å6W'fW$–B‡6W'fW$–B’“° —Р —&—fFR7–æ6‡&öæ—¦VBfö–B6Æ÷6U6ö6¶WD6Æ–VçG2‚’° ”†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â6Æ–VçG2Ò6Æ–VçD†æFÆW3° –6Æ–VçD†æFÆW2ÒçVÆð —7F÷6ö6¶WD6Æ–VçG2†6Æ–VçG2“° —Р —7FF–2fö–B7F÷6ö6¶WD6Æ–VçG2„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â6Æ–VçG2’° ––b†6Æ–VçG2ÓÒçVÆÂ’&WGW&ã° –f÷"„6Æ–VçD†æFÆW"6Æ–VçB¢6Æ–VçG2çfÇVW2‚’’° ––b†6Æ–VçBÓÒçVÆÂ’6öçF–çVS° —G'’° –6Æ–VçBç7F÷6öææV7F–öâ‚“° —Ò6F6‚…'VçF–ÖTW†6WF–öâ–væ÷&VB’° ’òò&W7BVff÷'C¢öæR'&ö¶Vâ6Æ–VçB×W7Bæ÷B&WfVçBF†R&VÖ–æ–ær6ö6¶WG2g&öÒ6Æ÷6–ærࠗР—Р—Р —&—fFRfö–Bv&åVç7W÷'FVDFVF–6FVEf÷F–æu&÷‡”ÖöFR‚’° ––b†vWD6öæf–r‚’ævWDFVF–6FVEf÷F–æu&÷‡’‚’bb†ÖWF†öBÓÒçVÆÂÇÂÖWF†öBç7W÷'G4&6¶VæE&W6Væ6R‚’’’° –Æöu6WfW&R‚$FVF–6FVEf÷F–æu&÷‡’&WV—&W2Õ•5ÂÂ$TD•2ÂÕEBÂ4ô4´UE2Â÷"…EE²ÅTt”äÔU54t”är—2F—6&ÆVBf÷"  ’²&FVF–6FVB×&÷‡’&÷WF–ærâfÆÆ–ær&6²Fòæ÷&ÖÂ&÷‡’&÷WF–ærâ"“° —Р—Р —V&Æ–2'7G&7Bfö–B'Vä7–æ2…'Vææ&ÆR'Vâ“°  ’ò¢¢ÆFf÷&ÒæÖRW6VBöæÇ’f÷"F†RG&ç7÷'BÖæWWG&Â6öçG&öÂF—66÷fW'’6öçG&7Bâ¢ð —V&Æ–2'7G&7B7G&–ærvWE&÷‡•ÆFf÷&Ò‚“°  —V&Æ–2'7G&7Bfö–B'Vä6öç6öÆT6öÖÖæB…7G&–ær6öÖÖæB“°  —V&Æ–2'7G&7Bfö–B6fUf÷FT66†Tf–ÆR‚“°  —V&Æ–2'7G&7Bfö–B&VÆöD6÷&R†&ööÆVâ×—7“°  ’ò¢¢7G&–7B6öçG&öÂ&VÆöBFƒ²f–ÇW&W2&÷vFR6òF†R6ÆÆW"6â&W7F÷&R—G2&6·Wâ¢ð —V&Æ–2'7G&7Bfö–B&VÆöD6öçG&öÄ6öæf–wW&F–öâ‚’F‡&÷w2W†6WF–öã°  —V&Æ–2'7G&7B&ööÆVâ6VæEÇVv–äÖW76vTFF…7G&–ær6W'fW"Â7G&–ær6†ææVÂÂ'—FUµÒFFÂ&ööÆVâVWVR“°  —&—fFR7FF–2f–æ–çBÅTt”åôÔU54tUô„$EôĔԕBÒ3#scs° —&—fFR7FF–2f–æ–çBÅTt”åôÔU54tUõ4ôeEôĔԕBÒ3°  —V&Æ–2fö–B6VæEÇVv–äÖW76vU6W'fW"…7G&–ær6W'fW"–çBFVĤ6öäVçfVÆ÷RVçfVÆ÷R’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ6VæEÇVv–äÖW76vU6W'fW$æ÷r‡6W'fW"ÂVçfVÆ÷R’ÂFVÆ’¢TÂÂF–ÖUVæ—BäÔ”ÄÄ•4T4ôäE2“° —Р ’ò¢  ’¢6VæG2ÇVv–âÖÖW76vRVçfVÆ÷R–ÖÖVF–FVÇ’æB&W÷'G2v†WF†W"F†R&÷‡ ’¢66WFVB—Bf÷"FVÆ—fW'’à ’  ’¢&Ò6W'fW"F&vWB&6¶VæB6W'fW  ’¢&ÒVçfVÆ÷RVçfVÆ÷RFò6Væ@ ’¢&WGW&âG'VRv†VâF†R&÷‡’66WFVBF†RÖW76vRf÷"FVÆ—fW' ’¢ð —&÷FV7FVB&ööÆVâ6VæEÇVv–äÖW76vU6W'fW$æ÷r…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° –f–æÂ7G&–ær7V$6†ææVÂÒVçfVÆ÷RævWE7V$6†ææV‚“° –f–æÂ7G&–ær–ÆöBÒ§6öäVçfVÆ÷T6öFV2æVæ6öFR†VçfVÆ÷R“°  –f–æÂ'—FUµÒ7V$6†ææVÄ'—FW2Ò7V$6†ææVÂævWD'—FW2†¦fææ–òæ6†'6WBå7FæF&D6†'6WG2åUDeó‚“° –f–æÂ'—FUµÒ–ÆöD'—FW2Ò–ÆöBævWD'—FW2†¦fææ–òæ6†'6WBå7FæF&D6†'6WG2åUDeó‚“°  ’òòW7F–ÖFR'—FW2w&—GFVã  ’òòÒw&—FUUDbFG2"Ö'—FRÆVæwF‚&Vf—‚²UDbÓ‚'—FW0 ’òòÒw&—FT–çB—2B'—FW0 ––çBW7F–ÖFVE6—¦RÒ"²7V$6†ææVÄ'—FW2æÆVæwF‚²òò7V$6†ææVÂUDb†ÆVâ&Vf—‚²'—FW2 “B²òò–ÆöBÆVæwF‚–ç@ “"²–ÆöD'—FW2æÆVæwFƒ²òò–ÆöBUDb†ÆVâ&Vf—‚²'—FW2  ––b†W7F–ÖFVE6—¦RâÅTt”åôÔU54tUõ4ôeEôĔԕB’° –FV'Vr‚%µÇVv–äÖW76vUÒ–ÆöBæV&–ærÆ–Ö—B‚"²W7F–ÖFVE6—¦R²"'—FW2’6W'fW#Ò"²6W'fW  ’²"7V$6†ææVÃÒ"²7V$6†ææV²"( B6öç6–FW"&VF—2–ç7FVB"“° —Р ––b†W7F–ÖFVE6—¦RâÅTt”åôÔU54tUô„$EôĔԕB’° –FV'Vr‚%µÇVv–äÖW76vUÒ–ÆöBDôòÄ$tR‚"²W7F–ÖFVE6—¦R²"'—FW2ÂÖƒÒ"²ÅTt”åôÔU54tUô„$EôĔԕ@ ’²"’6W'fW#Ò"²6W'fW"²"7V$6†ææVÃÒ"²7V$6†ææV²"( BäõB6VçB"“° —&WGW&âfÇ6S° —Р —G'’„'—FT'&”÷WGWE7G&VÒ'—FT÷WE7G&VÒÒæWr'—FT'&”÷WGWE7G&VÒ‚“° ”FF÷WGWE7G&VÒ÷WBÒæWrFF÷WGWE7G&VÒ†'—FT÷WE7G&VÒ’’° ––b†vWD6öæf–r‚’ævWEÇVv–äÖW76vTVæ7'—F–öâ‚’bbVæ7'—F–öä†æFÆW"ÒçVÆÂ’° –÷WBçw&—FUUDb†Væ7'—F–öä†æFÆW"æVæ7'—B‡7V$6†ææVÂ’“° —ÒVÇ6R° –÷WBçw&—FUUDb‡7V$6†ææV“° —Р ’òò6æ—G’öæÇ“¢ÕU5B&R'—FW2Âæ÷B6†'0 –÷WBçw&—FT–çB‡–ÆöD'—FW2æÆVæwF‚“°  ––b†vWD6öæf–r‚’ævWEÇVv–äÖW76vTVæ7'—F–öâ‚’bbVæ7'—F–öä†æFÆW"ÒçVÆÂ’° –÷WBçw&—FUUDb†Væ7'—F–öä†æFÆW"æVæ7'—B‡–ÆöB’“° —ÒVÇ6R° –÷WBçw&—FUUDb‡–ÆöB“° —Р–÷WBæfÇW6‚‚“°  –&ööÆVâ6VçBÒ6VæEÇVv–äÖW76vTFF‡6W'fW"ÂvWD6öæf–r‚’ævWEÇVv–äÖW76vT6†ææV‚’çFôÆ÷vW$66R‚’À –'—FT÷WE7G&VÒçFô'—FT'&’‚’ÂfÇ6R“° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –FV'Vr‚‡6VçBò%6VçB"¢$6÷VÆBæ÷B6VæB"’²"ÇVv–âVçfVÆ÷R‚"²W7F–ÖFVE6—¦R²"'—FW2’"²6W'fW  ’²""²7V$6†ææV²""²VçfVÆ÷RævWDf–VÆG2‚’“° —Р—&WGW&â6VçC° —Ò6F6‚„W†6WF–öâR’° –Rç&–çE7F6µG&6R‚“° —&WGW&âfÇ6S° —Р—Р —7FF–2FVfVÇD¦VF—46Æ–VçD6öæf–r'V–ÆE&VF—46Æ–VçD6öæf–r…f÷F–æuÇVv–å&÷‡”6öæf–r6öæf–u6÷W&6R’° ”FVfVÇD¦VF—46Æ–VçD6öæf–rä'V–ÆFW"6öæf–rÒFVfVÇD¦VF—46Æ–VçD6öæf–ræ'V–ÆFW"‚ ’æFF&6R†6öæf–u6÷W&6RævWE&VF—4F$–æFW‚‚’’ç76†6öæf–u6÷W&6RævWE&VF—576‚’’æ6öææV7F–öåF–ÖV÷WDÖ–ÆÆ—2ƒ# ’ç6ö6¶WEF–ÖV÷WDÖ–ÆÆ—2ƒ#“° ––b†6öæf–u6÷W&6RævWE&VF—576‚’’° •54Å&ÖWFW'276Å&ÖWFW'2ÒæWr54Å&ÖWFW'2‚“° —76Å&ÖWFW'2ç6WDVæGö–çD–FVçF–f–6F–öäÆv÷&—F†Ò‚$…EE2"“° –6öæf–rç76Å&ÖWFW'2‡76Å&ÖWFW'2“° —Р––b†6öæf–u6÷W&6RævWE&VF—5W6W&æÖR‚’ÒçVÆÂbb6öæf–u6÷W&6RævWE&VF—5W6W&æÖR‚’æ—4V×G’‚’’° –6öæf–rçW6W"†6öæf–u6÷W&6RævWE&VF—5W6W&æÖR‚’“° —Р––b†6öæf–u6÷W&6RævWE&VF—577v÷&B‚’ÒçVÆÂbb6öæf–u6÷W&6RævWE&VF—577v÷&B‚’æ—4V×G’‚’’° –6öæf–rç77v÷&B†6öæf–u6÷W&6RævWE&VF—577v÷&B‚’“° —Р—&WGW&â6öæf–ræ'V–ÆB‚“° —Р —V&Æ–2&ööÆVâ6VæE&VF—4VçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° —&WGW&â6VæE&VF—4VçfVÆ÷U6W'fW"‡6W'fW"ÂVçfVÆ÷RÂfÇ6R“° —Р —&—fFR&ööÆVâ6VæE&VF—4VçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷RÂ&ööÆVâW6U&WG'”6ööÆF÷vâ’° ”¦VF—5ööÂV&Æ—6†W%ööÂÒ&VF—5V&Æ—6†W%ööð ––b‡V&Æ—6†W%ööÂÓÒçVÆÂLJW6U&WG'”6ööÆF÷vâbb7—7FVÒæ7W'&VçEF–ÖTÖ–ÆÆ—2‚’Â&VF—5V&Æ—6†W%&WG'”gFW"’’° —&WGW&âfÇ6S° —Р —G'’„¦VF—2¦VF—2ÒV&Æ—6†W%ööÂævWE&W6÷W&6R‚’’° •7G&–ær6†ææVÂÒvWD6öæf–r‚’ævWE&VF—5&Vf—‚‚’²%f÷F–æuÇVv–åò"²6W'fW#° –Æöær7V'67&–&W'2Ò¦VF—2çV&Æ—6‚†6†ææVÂÀ ”§6öäVçfVÆ÷T6öFV2æVæ6öFR…f÷F–æuÇVv–åv—&Rçv—F…&VF—4FVÆ—fW'”–B†VçfVÆ÷R’’“° —&VF—5V&Æ—6†W%&WG'”gFW"Òð —&WGW&â7V'67&–&W'2â° —Ò6F6‚„W†6WF–öâR’° ––b‡W6U&WG'”6ööÆF÷vâ’° ’òò7FæFÆöæR'&öF67G2&VÖ–âVWVVBÂ6òF†V—"&WG&–W26â&RF‡&÷GFÆVB6fVÇ’à —&VF—5V&Æ—6†W%&WG'”gFW"Ò7—7FVÒæ7W'&VçEF–ÖTÖ–ÆÆ—2‚’²#ð —Р–FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р—Р —V&Æ–2&ööÆVâ6VæD×GDVçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ––b†×GD†æFÆW"ÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р—G'’° –×GD†æFÆW"çV&Æ—6„VçfVÆ÷R†vWD6öæf–r‚’ævWD×GE&Vf—‚‚’²'f÷F–æwÇVv–â÷6W'fW'2ò"²6W'fW"ÂVçfVÆ÷R“° —&WGW&âG'VS° —Ò6F6‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—&WGW&âfÇ6S° —Р—Р —V&Æ–2&ööÆVâ6VæE6ö6¶WDVçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ”ÖÅ7G&–ærÂö&¦V7Câ6öæf–wW&F–öâÒvWD6öæf–r‚’ævWE7–v÷E6W'fW$6öæf–wW&F–öâ‡6W'fW"“° ––b†6öæf–wW&F–öâÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р•7G&–ær†÷7BÒ6öæf–wW&F–öâævWB‚$†÷7B"’–ç7Fæ6Vöb7G&–ærò…7G&–ær’6öæf–wW&F–öâævWB‚$†÷7B"’¢"#° ––çB÷'BÒ6öæf–wW&F–öâævWB‚%÷'B"’–ç7Fæ6VöbçVÖ&W"ò‚„çVÖ&W"’6öæf–wW&F–öâævWB‚%÷'B"’’æ–çEfÇVR‚’¢#“ƒ° ––b††÷7Bæ—4V×G’‚’’° —&WGW&âfÇ6S° —Р •7G&–ær–ÆöBÒ§6öäVçfVÆ÷T6öFV2æVæ6öFR†VçfVÆ÷R“° •7G&–ærVæ6öFVBÒVæ7'—F–öä†æFÆW"ÒçVÆÂòVæ7'—F–öä†æFÆW"æVæ7'—B‡–ÆöB’¢–ÆöC° —G'’…6ö6¶WB6ö6¶WBÒæWr6ö6¶WB‚’’° —6ö6¶WBæ6öææV7B†æWr–æWE6ö6¶WDFG&W72††÷7BÂ÷'B’Â#“° —G'’„FF÷WGWE7G&VÒ÷WGWBÒæWrFF÷WGWE7G&VÒ‡6ö6¶WBævWD÷WGWE7G&VÒ‚’’’° –÷WGWBçw&—FUUDb†Væ6öFVB“° –÷WGWBæfÇW6‚‚“° —Р—&WGW&âG'VS° —Ò6F6‚„W†6WF–öâR’° –FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р—Р —V&Æ–2fö–B6VæE6W'fW$æÖTÖW76vR‚’° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° —6VæEÇVv–äÖW76vU6W'fW"‡2ÂÂf÷F–æuÇVv–åv—&Rç6W'fW$æÖR‡2’“° —Р—Р —V&Æ–2fö–B6VæEf÷FU'G’…7G&–ær6W'fW"’° ––b†—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡6W'fW"’’° –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡6W'fW"ÂÂf÷F–æuÇVv–åv—&Rçf÷FU'G”'VævVR‚’“° —Р—Р —V&Æ–2fö–B6WD7W'&VçEf÷FU'G•f÷FW2†–çBÖ÷VçB’° —f÷FU'G•f÷FW2ÒÖ÷VçC° —6WEf÷FT66†Uf÷FU'G”7W'&VçEf÷FW2†Ö÷VçB“° –FV'Vr‚$7W'&VçBf÷FR'G’F÷Fâ"²f÷FU'G•f÷FW2“° —Р —V&Æ–2'7G&7Bfö–B6WEf÷FT66†TÆ7EWFFVB‚“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†U&WdF’†–çBF’“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†U&WdÖöçF‚…7G&–ærFW‡B“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†U&WevVV²†–çBvVV²“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†Uf÷FT66†T–væ÷&UF–ÖR†&ööÆVâ–væ÷&R“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†Uf÷FU'G”7W'&VçEf÷FW2†–çBf÷FW2“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB†–çBf÷FW2“°  —V&Æ–2fö–B7FGW2‚’° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° ––b‚—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡2’’° –Æör‚$æòÆ–W'2öâ6W'fW""²2²"Fò6VæBFW7B7FGW2ÖW76vRÂÆV6R&WFW7Bv—F‚6öÖVöæRöæÆ–æR"“° —ÒVÇ6R° –Æör‚%6VæF–ær&WVW7Bf÷"7FGW2ÖW76vRöâ"²2“° –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡2ÂÂf÷F–æuÇVv–åv—&Rç7FGW2‡2’“° —Р—Р—Р ’ò¢¢'Vç26÷'&VÆFVBÂæöâ×f÷FR&÷VæBG&—÷fW"F†R7F—fR&6¶VæBG&ç7÷'Bâ¢ð —V&Æ–26ö×ÆWF&ÆTgWGW&SÄ6öÖ×Væ–6F–öåFW7E&W7VÇCâFW7D&6¶VæD6öÖ×Væ–6F–öâ…7G&–ær&WVW7FVE6W'fW"À –ÆöærF–ÖV÷WDÖ–ÆÆ—2’° •7G&–ær6W'fW"Ò&WVW7FVE6W'fW"ÓÒçVÆÂò""¢&WVW7FVE6W'fW"çG&–Ò‚“° ”'VævVTÖWF†öB7F—fTÖWF†öBÒÖWF†öC° ––b‡6W'fW"æ—4V×G’‚’ÇÂvWDÆÄf–Æ&ÆU6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%Tä´äõtåô$4´TäB"Â%F†R&6¶VæB—2æ÷B6öæf–wW&VBöâF†—2&÷‡’"’“° —Р––b†7F—fTÖWF†öBÓÒçVÆÂÇÂvÆö&ÄÖW76vU&÷‡”†æFÆW"ÓÒçVÆÂ’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%E$å5õ%EõTäd”Ä$ÄR"Â%F†R&÷‡’6öÖ×Væ–6F–öâG&ç7÷'B—2æ÷B'Vææ–ær"’“° —Р––b†7F—fTÖWF†öBÓÒ'VævVTÖWF†öBåÅTt”äÔU54t”ärbb—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡6W'fW"’’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%Ä”U%õ$UT•$TB"Â%ÇVv–âÖW76v–ær&WV—&W2âöæÆ–æRÆ–W"öâF†R6VÆV7FVB&6¶VæB"’“° —Р•66†VGVÆVDW†V7WF÷%6W'f–6R66†VGVÆW"ÒvWE66†VGVÆW"‚“° ––b‡66†VGVÆW"ÓÒçVÆÂ’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%E$å5õ%EõTäd”Ä$ÄR"Â%F†R&÷‡’66†VGVÆW"—2æ÷B'Vææ–ær"’“° —Р–Æöær&÷VæFVEF–ÖV÷WBÒÖF‚æÖ‚ƒSÂÂÖF‚æÖ–â‡F–ÖV÷WDÖ–ÆÆ—2Â3Â’“° •UT”B&WVW7D–BÒUT”Bç&æFöÕUT”B‚“° ”6ö×ÆWF&ÆTgWGW&SÄ6öÖ×Væ–6F–öåFW7E&W7VÇCâ&W7VÇBÒæWr6ö×ÆWF&ÆTgWGW&SÃâ‚“° •VæF–æt6öÖ×Væ–6F–öåFW7BVæF–ærÒæWrVæF–æt6öÖ×Væ–6F–öåFW7B‡6W'fW"Â7F—fTÖWF†öBÂ7—7FVÒæææõF–ÖR‚’Â&W7VÇB“° —VæF–æt6öÖ×Væ–6F–öåFW7G2çWB‡&WVW7D–BÂVæF–ær“° —&W7VÇBçv†Vä6ö×ÆWFR‚†–væ÷&VBÂf–ÇW&R’ÓâVæF–æt6öÖ×Væ–6F–öåFW7G2ç&VÖ÷fR‡&WVW7D–BÂVæF–ær’“° —G'’° ––b‚6VæD6öÖ×Væ–6F–öåFW7DVçfVÆ÷Tæ÷r‡6W'fW"Âf÷F–æuÇVv–åv—&Rç7FGW2‡6W'fW"Â&WVW7D–B’’’° —&W7VÇBæ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÂ%E$å5õ%EõTäd”Ä$ÄR"À ’%F†R7F—fRG&ç7÷'B6÷VÆBæ÷B66WBF†R6öÖ×Væ–6F–öâFW7B"’“° —&WGW&â&W7VÇC° —Р—66†VGVÆW"ç66†VGVÆR‚‚’Óâ&W7VÇBæ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%D”ÔTõUB"Â$æò6÷'&VÆFVB&WÇ’'&—fVB&Vf÷&RF†RF–ÖV÷WB"’’Â&÷VæFVEF–ÖV÷WBÂF–ÖUVæ—BäÔ”ÄÄ•4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâf–ÇW&R’° —&W7VÇBæ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÂ%4TäEôd”ÄTB"À ’%F†R&÷‡’6÷VÆBæ÷B6VæBF†R6öÖ×Væ–6F–öâFW7B"’“° —Р—&WGW&â&W7VÇC° —Р ’ò¢¢6VæG2F–væ÷7F–2–ÖÖVF–FVÇ’æB&W÷'G2v†WF†W"F†R7F—fRG&ç7÷'B66WFVB—Bâ¢ð —&÷FV7FVB&ööÆVâ6VæD6öÖ×Væ–6F–öåFW7DVçfVÆ÷Tæ÷r…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° —&WGW&â6VæE&÷‡”'&öF67DVçfVÆ÷Tæ÷r‡6W'fW"ÂVçfVÆ÷R“° —Р —&÷FV7FVBfö–B†æFÆU7FGW4ö¶’„§6öäVçfVÆ÷RÖW76vR’° •7G&–ær6W'fW"ÒÖW76vRævWDf–VÆG2‚’ævWD÷$FVfVÇB…f÷F–æuÇVv–åv—&Räµõ4U%dU"Â""“° •7G&–ær&WVW7BÒÖW76vRævWDf–VÆG2‚’ævWD÷$FVfVÇB…f÷F–æuÇVv–åv—&Räµõ$UTU5Eô”BÂ""“° ––b‡&WVW7Bæ—4V×G’‚’’° –Æör‚%7FGW2ö¶’f÷""²6W'fW"“° —&WGW&ã° —Р•UT”B&WVW7D–C° —G'’° —&WVW7D–BÒUT”Bæg&öÕ7G&–ær‡&WVW7B“° —Ò6F6‚„–ÆÆVvÄ&wVÖVçDW†6WF–öâ–væ÷&VB’° –FV'Vr‚$–væ÷&VB7FGW2&WÇ’v—F‚â–çfÆ–B&WVW7B”Bg&öÒ"²6W'fW"“° —&WGW&ã° —Р•VæF–æt6öÖ×Væ–6F–öåFW7BVæF–ærÒVæF–æt6öÖ×Væ–6F–öåFW7G2ævWB‡&WVW7D–B“° ––b‡VæF–ærÓÒçVÆÂÇÂVæF–ærç6W'fW"‚’æWVÇ2‡6W'fW"’’° –FV'Vr‚$–væ÷&VBVæW‡V7FVB7FGW2&WÇ’g&öÒ"²6W'fW"“° —&WGW&ã° —Р–Æöær&÷VæEG&—Ö–ÆÆ—2ÒÖF‚æÖ‚ƒÂÀ •F–ÖUVæ—Bäääõ4T4ôäE2çFôÖ–ÆÆ—2…7—7FVÒæææõF–ÖR‚’ÒVæF–ærç7F'FVDDææ÷2‚’’“° —VæF–ærç&W7VÇB‚’æ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBç7V66W72‡6W'fW"ÂVæF–æræÖWF†öB‚’Â&÷VæEG&—Ö–ÆÆ—2’“° —Р —&—fFRfö–B6æ6VÄ6öÖ×Væ–6F–öåFW7G2…7G&–ærÖW76vR’° —VæF–æt6öÖ×Væ–6F–öåFW7G2æf÷$V6‚‚‡&WVW7D–BÂVæF–ær’ÓâVæF–ærç&W7VÇB‚’æ6ö×ÆWFR€ ”6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡VæF–ærç6W'fW"‚’ÂVæF–æræÖWF†öB‚’Â%E$å5õ%Eõ5DõTB"ÂÖW76vR’’“° —VæF–æt6öÖ×Væ–6F–öåFW7G2æ6ÆV"‚“° —Р —V&Æ–2&V6÷&B6öÖ×Væ–6F–öåFW7E&W7VÇB†&ööÆVâ7V66W72Â7G&–ær6öFRÂ7G&–ærÖW76vRÂ7G&–ær6W'fW"À •7G&–ærÖWF†öBÂÆöær&÷VæEG&—Ö–ÆÆ—2’° —&—fFR7FF–26öÖ×Væ–6F–öåFW7E&W7VÇB7V66W72…7G&–ær6W'fW"Â'VævVTÖWF†öBÖWF†öBÂÆöær&÷VæEG&—Ö–ÆÆ—2’° —&WGW&âæWr6öÖ×Væ–6F–öåFW7E&W7VÇB‡G'VRÂ$ô²"Â$&6¶VæB&WÆ–VB÷fW"F†R7F—fRG&ç7÷'B"Â6W'fW"À –ÖWF†öBÓÒçVÆÂò""¢ÖWF†öBææÖR‚’Â&÷VæEG&—Ö–ÆÆ—2“° —Р —&—fFR7FF–26öÖ×Væ–6F–öåFW7E&W7VÇBf–ÇW&R…7G&–ær6W'fW"Â'VævVTÖWF†öBÖWF†öBÂ7G&–ær6öFRÂ7G&–ærÖW76vR’° —&WGW&âæWr6öÖ×Væ–6F–öåFW7E&W7VÇB†fÇ6RÂ6öFRÂÖW76vRÂ6W'fW"À –ÖWF†öBÓÒçVÆÂò""¢ÖWF†öBææÖR‚’ÂÓ“° —Р—Р —&—fFR&V6÷&BVæF–æt6öÖ×Væ–6F–öåFW7B…7G&–ær6W'fW"Â'VævVTÖWF†öBÖWF†öBÂÆöær7F'FVDDææ÷2À ”6ö×ÆWF&ÆTgWGW&SÄ6öÖ×Væ–6F–öåFW7E&W7VÇCâ&W7VÇB’²Р —&—fFRfö–B6VæEf÷FTFVÆ•&V¦V7FVB…7G&–ærÆ–W"Â7G&–ærWV–BÂ7G&–ær6W'f–6RÂ&ööÆVâÆ–W$öæÆ–æRÀ •7G&–ærÆ–W%6W'fW"’° ––b‚Æ–W$öæÆ–æRÇÂÆ–W%6W'fW"ÓÒçVÆÂÇÂvWDÆÄf–Æ&ÆU6W'fW'2‚’æ6öçF–ç2‡Æ–W%6W'fW"’’° –FV'Vr‚$æ÷B6VæF–ærf÷FRFVÆ’&V¦V7F–öâf÷""²Æ–W"²"&V6W6RF†RÆ–W"—2öffÆ–æR"“° —&WGW&ã° —Р –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡Æ–W%6W'fW"ÂÀ •f÷F–æuÇVv–åv—&Rçf÷FTFVÆ•&V¦V7FVB‡Æ–W"ÂWV–BÂ6W'f–6RÂG'VR’“° —Р —V&Æ–27G&–ærvWEv—EVçF–ÄFVÆ•6—FTg&öÕ6W'f–6R…7G&–ær6W'f–6R’° –f÷"…7G&–ær6—FR¢vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•6—FW2‚’’° ––b†vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•6W'f–6R‡6—FR’æWVÇ4–væ÷&T66R‡6W'f–6R’’° —&WGW&â6—FS° —Р—Р—&WGW&â"#° —Р —&—fFRÆöærvWDÆ7Ef÷FW5F–ÖR…7G&–ærWV–BÂ'&”Æ—7CÄ6öÇVÖãâ6öÇ2Â7G&–ær6—FRÂ7G&–ær6W'f–6RÂ7G&–ærÆ–W"À –&ööÆVâ–æ6ÇVFUF–ÖT6†ævUVWVR’° –ÆöærÖ÷7E&V6VçEF–ÖRÒ°  ––b†vWEf÷FT66†T†æFÆW"‚’æ†4öæÆ–æUf÷FW2‡WV–B’’° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâöæÆ–æUf÷FW2ÒvWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2‡WV–B“° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢öæÆ–æUf÷FW2’° ––b‡f÷FRævWE6W'f–6R‚’æWVÇ4–væ÷&T66R‡6W'f–6R’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂf÷FRævWEF–ÖR‚’“° —Р—Р—Р –f÷"…7G&–ær6W'fW"¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢vWEf÷FT66†T†æFÆW"‚’ævWEf÷FW2‡6W'fW"’’° ––b‡f÷FRævWEWV–B‚’æWVÇ2‡WV–B’bbf÷FRævWE6W'f–6R‚’æWVÇ4–væ÷&T66R‡6W'f–6R’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂf÷FRævWEF–ÖR‚’“° —Р—Р—Р ––b†–æ6ÇVFUF–ÖT6†ævUVWVRbbÆ–W"ÒçVÆÂ’° –f÷"…f÷FUF–ÖUVWVRVWVVEf÷FR¢vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’° ––b‡VWVVEf÷FRævWDæÖR‚’æWVÇ4–væ÷&T66R‡Æ–W" ’bbVWVVEf÷FRævWE6W'f–6R‚’æWVÇ4–væ÷&T66R‡6W'f–6R’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂVWVVEf÷FRævWEF–ÖR‚’“° —Р—Р—Р –f÷"„6öÇVÖâB¢6öÇ2’° ––b†BævWDæÖR‚’æWVÇ4–væ÷&T66R‚$Æ7Ef÷FW2"’’° ”FFfÇVRfÇVRÒBævWEfÇVR‚“° •7G&–æuµÒÆ—7BÒfÇVRævWE7G&–ær‚’ç7Æ—B‚"VÆ–æRR"“° –f÷"…7G&–ær7G"¢Æ—7B’° •7G&–æuµÒFFÒ7G"ç7Æ—B‚"òò"“° ––b†FF³ÒæWVÇ4–væ÷&T66R‡6—FR’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂÆöærçfÇVTöb†FF³Ò’“° —Р—Р—Р—Р—&WGW&âÖ÷7E&V6VçEF–ÖS° —Р —V&Æ–2&ööÆVâ6†V6µf÷FTFVÆ’…7G&–ærWV–BÂ7G&–ær6W'f–6RÂ'&”Æ—7CÄ6öÇVÖãâFF’° —&WGW&â6†V6µf÷FTFVÆ’‡WV–BÂçVÆÂÂ6W'f–6RÂFFÂfÇ6R“° —Р ’ò¢  ’¢6†V6·2F†R6öæf–wW&VBf÷FRFVÆ’Â÷F–öæÆÇ’–æ6ÇVF–ær66WFVBf÷FW2v—F–æp ’¢f÷"vÆö&ÄFFF–ÖR6†ævRFòf–æ—6‚à ’  ’¢&ÒWV–BÆ–W"UT”@ ’¢&ÒÆ–W"Æ–W"æÖRW6VB'’F†RF–ÖRÖ6†ævRVWVP ’¢&Ò6W'f–6Rf÷FR6W'f–6P ’¢&ÒFF7W'&VçBÆ–W"FF ’¢&Ò–æ6ÇVFUF–ÖT6†ævUVWVRv†WF†W"VWVVBf÷FW2&W6W'fRF†V—"FVÆ’6Æ÷@ ’¢&WGW&âG'VRv†VâF†Rf÷FRÖ’&R66WFV@ ’¢ð —V&Æ–2&ööÆVâ6†V6µf÷FTFVÆ’…7G&–ærWV–BÂ7G&–ærÆ–W"Â7G&–ær6W'f–6RÂ'&”Æ—7CÄ6öÇVÖãâFFÀ –&ööÆVâ–æ6ÇVFUF–ÖT6†ævUVWVR’° •7G&–ær6—FRÒvWEv—EVçF–ÄFVÆ•6—FTg&öÕ6W'f–6R‡6W'f–6R“° ––b‡6—FRæ—4V×G’‚’’° –FV'Vr‚$æò6W'f–6R6—FR6WBf÷""²6W'f–6R²"Â6¶—–ærf÷FRFVÆ’6†V6²"“° —&WGW&âG'VS° —Р ––çBf÷FTFVÆ’ÒvWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVÆ’‡6—FR“° ––çBf÷FTFVƔ֖âÒvWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVƔ֖â‡6—FR“°  –ÆöærÆ7Ef÷FRÒvWDÆ7Ef÷FW5F–ÖR‡WV–BÂFFÂ6—FRÂ6W'f–6RÂÆ–W"–æ6ÇVFUF–ÖT6†ævUVWVR“° ––b†Æ7Ef÷FRÓÒ’° –FV'Vr‚$æòÆ7Bf÷FRF–ÖRf÷VæBf÷""²WV–B²"ò"²6W'f–6R²"Â6¶—–ærf÷FRFVÆ’6†V6²"“° —&WGW&âG'VS° —Р —G'’° ”Æö6ÄFFUF–ÖRæ÷rÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚“° ”Æö6ÄFFUF–ÖRÆ7Ef÷FUF–ÖRÒÆö6ÄFFUF–ÖRæöd–ç7FçB„–ç7FçBæödWö6„Ö–ÆÆ’†Æ7Ef÷FR’¦öæT–Bç7—7FVÔFVfVÇB‚’ ’çÇW4†÷W'2†vWD6öæf–r‚’ævWEF–ÖT†÷W$öfe6WB‚’“°  ––b‚vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVÆ”F–Ç’‡6—FR’’° ––b‡f÷FTFVÆ’ÓÒbbf÷FTFVƔ֖âÓÒ’° –FV'Vr‚%f÷FRFVÆ’—2f÷""²6—FR²"Â6¶—–ærf÷FRFVÆ’6†V6²"“° —&WGW&âG'VS° —Р ”Æö6ÄFFUF–ÖRæW‡Gf÷FRÒÆ7Ef÷FUF–ÖRçÇW4†÷W'2‚†Æöær’f÷FTFVÆ’’çÇW4Ö–çWFW2‚†Æöær’f÷FTFVƔ֖⓰ —&WGW&âæ÷ræ—4gFW"†æW‡Gf÷FR“° —Р”Æö6ÄFFUF–ÖR&W6WEF–ÖRÒÆ7Ef÷FUF–ÖRçv—F„†÷W"†vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVÆ”†÷W"‡6—FR’ ’çv—F„Ö–çWFRƒ’çv—F…6V6öæBƒ“° ”Æö6ÄFFUF–ÖR&W6WEF–ÖUFöÖ÷'&÷rÒ&W6WEF–ÖRçÇW4†÷W'2ƒ#B“°  ––b†Æ7Ef÷FUF–ÖRæ—4&Vf÷&R‡&W6WEF–ÖR’’° ––b†æ÷ræ—4gFW"‡&W6WEF–ÖR’’° –FV'Vr‚%f÷FRFVÆ’—2ÖWBf÷""²WV–B²"ò"²6W'f–6R²"Âf÷FR6â&R&ö6W76VB"“° —&WGW&âG'VS° —Р—ÒVÇ6R° ––b†æ÷ræ—4gFW"‡&W6WEF–ÖUFöÖ÷'&÷r’’° –FV'Vr‚%f÷FRFVÆ’—2ÖWBf÷""²WV–B²"ò"²6W'f–6R²"Âf÷FR6â&R&ö6W76VB"“° —&WGW&âG'VS° —Р—Р—Ò6F6‚„W†6WF–öâR’° –Rç&–çE7F6µG&6R‚“° —Р –FV'Vr‚%f÷FRFVÆ’—2æ÷BÖWBf÷""²WV–B²"ò"²6W'f–6R²"Â6¶—–ærf÷FR"“° —&WGW&âfÇ6S° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–Bf÷FR…7G&–ærÆ–W"Â7G&–ær6W'f–6RÂ&ööÆVâ&VÅf÷FRÂ&ööÆVâF–ÖUVWVRÂÆöærVWVUF–ÖRÀ •f÷FUF÷FÇ56æ6†÷BFW‡BÂ7G&–ærWV–B’° —f÷FR‡Æ–W"Â6W'f–6RÂ&VÅf÷FRÂF–ÖUVWVRÂVWVUF–ÖRÂFW‡BÂWV–BÂçVÆ“° —Р —&—fFRVçVÒVWVVEf÷FU&W7VÇB° •5T44U52Â$UE%’ÂDU$Ô”äÀ —Р —&—fFR7–æ6‡&öæ—¦VBVWVVEf÷FU&W7VÇBf÷FR…7G&–ærÆ–W"Â7G&–ær6W'f–6RÂ&ööÆVâ&VÅf÷FRÂ&ööÆVâF–ÖUVWVRÂÆöærVWVUF–ÖRÀ •f÷FUF÷FÇ56æ6†÷BFW‡BÂ7G&–ærWV–BÂf÷FUF–ÖUVWVRVWVVEf÷FR’° —G'’° ––b‚6W'f–6U6—FUfÆ–FF÷"æ—5fÆ–B‡6W'f–6R’’° —v&â‚%&V¦V7FVBf÷FRv—F‚–çfÆ–B6W'f–6R6—FRr"²6W'f–6U6—FUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡6W'f–6R’²"r"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р––b‚Ö–æV7&gEW6W&æÖUfÆ–FF÷"æ—5fÆ–B‡Æ–W"ÂvWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’’’° —v&â‚%&V¦V7FVBf÷FRv—F‚–çfÆ–BÖ–æV7&gBW6W&æÖRr  ’²Ö–æV7&gEW6W&æÖUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡Æ–W"’²"rg&öÒ6W'f–6Rr  ’²Ö–æV7&gEW6W&æÖUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡6W'f–6R’²"r"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р •UT”Bf÷FT–BÒVWVVEf÷FRÓÒçVÆÂòçVÆ¢VWVVEf÷FRævWEf÷FT–B‚“° ––b‡f÷FT–BÓÒçVÆÂ’° —f÷FT–BÒUT”Bç&æFöÕUT”B‚“° —Р ’òòUT”B&W6öÇWF–öà ––b‚vWD6öæf–r‚’ævWDöæÆ–æTÖöFR‚’’° —WV–BÒvWEUT”B‡Æ–W"“° —Р ––b‡WV–BÓÒçVÆÂÇÂWV–Bæ—4V×G’‚’’° —WV–BÒvWEUT”B‡Æ–W"“°  ’òò&VG&ö6²&Vf—‚WFòÖFWFV7@ ––b‡WV–Bæ—4V×G’‚’bbvWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’æ—4V×G’‚ ’bbÆ–W"ç7F'G5v—F‚†vWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’’’° •7G&–ærWV–CÒvWEUT”B†vWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’²Æ–W"“° ––b‚WV–Cæ—4V×G’‚’’° –FV'Vr‚$FWFV7FVB&VG&ö6²Æ–W"v—F†÷WB&Vf—‚ÂF§W7F–ærâââ"“° —Æ–W"ÒvWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’²Æ–W#° —WV–BÒWV–C° —Р—Р—Р ––b‡WV–Bæ—4V×G’‚’’° ––b‡Æ–W"ç7F'G5v—F‚†vWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’’’° –Æör‚$–væ÷&–ærf÷FR6–æ6RVæ&ÆRFòvWBUT”Böb&VG&ö6²Æ–W""“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р––b‚vWD6öæf–r‚’ævWDÆÆ÷uVä¦ö–æVB‚’’° –Æör‚$–væ÷&–ærf÷FRg&öÒ"²Æ–W"²"6–æ6RÆ–W"†6âwB¦ö–æVB&Vf÷&R"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р––b‚vWD6öæf–r‚’ævWEUT”DÆöö·W‚’’° –Æör‚$f–ÆVBFòvWBWV–Bf÷""²Æ–W"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р –FV'Vr‚$fWF6†–ærUT”BöæÆ–æRÂ6–æ6RÆÆ÷wVæ¦ö–æVB—2Væ&ÆVB"“° •UT”BRÒçVÆð —G'’° ––b†vWD6öæf–r‚’ævWDöæÆ–æTÖöFR‚’’° —RÒfWF6…UT”B‡Æ–W"“° —Р—Ò6F6‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р––b‡RÓÒçVÆÂ’° –FV'Vr‚$f–ÆVBFòvWBWV–Bf÷""²Æ–W"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р—WV–BÒRçFõ7G&–ær‚“° —Р ’òòæ÷&ÖÆ—¦RUT”B7G&–ær–b÷76–&ÆP —G'’° ––b‡WV–BÒçVÆÂbbWV–Bæ—4V×G’‚’bbWV–BæWVÇ4–væ÷&T66R‚&çVÆÂ"’’° —WV–BÒUT”Bæg&öÕ7G&–ær‡WV–BçG&–Ò‚’’çFõ7G&–ær‚“° —Р—Ò6F6‚„W†6WF–öâ–væ÷&VB’° ’òò–væ÷&P —Р —Æ–W"ÒvWE&÷W$æÖR‡WV–BÂÆ–W"“°  ’òò66†RöæÆ–æR7FFR÷6W'fW"öæ6R„”Õõ%DåBf÷"'&öF67BÆöv–26÷'&V7FæW72 –f–æÂ&ööÆVâÆ–W$öæÆ–æRÒ—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær‡Æ–W"“° –f–æÂ7G&–ærÆ–W%6W'fW"ÒÆ–W$öæÆ–æRòvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær‡Æ–W"’¢çVÆð –ÆöærF–ÖRÒVWVUF–ÖRÒòVWVUF–ÖP “¢Æö6ÄFFUF–ÖRææ÷r‚’æE¦öæR…¦öæT–Bç7—7FVÔFVfVÇB‚’’çFô–ç7FçB‚’çFôWö6„Ö–ÆÆ’‚“°  •6WCÅ7G&–æsâ'&öF67EF&vWG2ÒVWVVEf÷FRÓÒçVÆÂòæWrÆ–æ¶VD†6…6WCÃâ‚ “¢æWrÆ–æ¶VD†6…6WCÃâ‡VWVVEf÷FRævWD'&öF67EF&vWG2‚’“° •6WCÅ7G&–æsâ'&öF67Df÷'v&FVE6W'fW'2ÒVWVVEf÷FRÓÒçVÆÂòæWrÆ–æ¶VD†6…6WCÃâ‚ “¢æWrÆ–æ¶VD†6…6WCÃâ‡VWVVEf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° –&ööÆVâ&÷‡”'&öF67D†æFÆVBÒVWVVEf÷FRÒçVÆÂbbVWVVEf÷FRæ—5&÷‡”'&öF67D†æFÆVB‚“° –&ööÆVâ&ö6W76W5F÷FÇ2ÒvWD6öæf–r‚’ævWE&–Ö'•6W'fW"‚’ÇÂvWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚“° –&ööÆVâÖævW5F÷FÇ2Ò&ö6W76W5F÷FÇ2bbvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚“° –&ööÆVâ6åfÆ–FFU7FæFÆöæT'&öF67BÒ6äf÷'v&E7FæFÆöæT'&öF67B†ÖævW5F÷FÇ2“° ”'&”Æ—7CÄ6öÇVÖãâFFÒçVÆð –&ööÆVâVWVTf÷%F–ÖT6†ævRÒfÇ6S°  ’òò6ö×ÆWF–öâ6ÆÆ&6²6âv—RF÷FÇ2æB&WÆ’öÆFW"VWVVBf÷FW2â'Vâ—@ ’òò&Vf÷&RÆöF–ærF†—2f÷FRw2FF&6R6æ6†÷B6òF†R6Æ7VÆF–öç2&VÆ÷rW6P ’òòF†R÷7B×&öÆÆ÷fW"7FFRà ––b†vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’bbvWDvÆö&ÄFF†æFÆW"‚’æ—5F–ÖT6†ævVD†VæVB‚’’° –vWDvÆö&ÄFF†æFÆW"‚’æ6†V6´f÷$f–æ—6†VEF–ÖT6†ævW2‚“° —VWVTf÷%F–ÖT6†ævRÒF–ÖUVWVRbbvWDvÆö&ÄFF†æFÆW"‚’æ—5F–ÖT6†ævVD†VæVB‚“° —Р ’òòfÆ–FFRF†Rf÷FR&Vf÷&Rç’–ÖÖVF–FRææ÷Væ6VÖVçBâF†—2¶VW2GWÆ–6FP ’òòf÷FW2&V¦V7FVB'’F†RFVÆ’6†V6²÷WBöbF†RvÆö&ÄFF&öÆÆ÷fW"VWVRæ@ ’òò&WfVçG2ææ÷Væ6–ærf÷FRF†Bv–ÆÂæ÷B&R&ö6W76VBà ––b†ÖævW5F÷FÇ2’° ––b†vWE&÷‡”ו5‚’ÓÒçVÆÂ’° –Æöu6WfW&R‚$×—7—2æ÷BÆöFVB6÷'&V7FÇ’Â7F÷–ærf÷FR&ö6W76–ær"“° —&WGW&âVWVVEf÷FU&W7VÇBå$UE%“° —Р ––b‚vWE&÷‡”ו5‚’æ6öçF–ç4¶W•VW'’‡WV–B’’° –vWE&÷‡”ו5‚’çWFFR‡WV–BÂ%Æ–W$æÖR"ÂæWrFFfÇVU7G&–ær‡Æ–W"’“° –vWE&÷‡”ו5‚’ævWEWV–G2‚’æFB‡WV–B“° —Р –FFÒvWE&÷‡”ו5‚’ævWDW†7EVW'’†æWr6öÇVÖâ‚'WV–B"ÂæWrFFfÇVU7G&–ær‡WV–B’’“° ––b‚6†V6µf÷FTFVÆ’‡WV–BÂÆ–W"Â6W'f–6RÂFFÂVWVVEf÷FRÓÒçVÆÂ’’° –Æör‚%f÷FRFVÆ’—2æ÷BÖWBf÷""²Æ–W"²"ò"²6W'f–6R²"Â6¶—–ærf÷FR"“° —6VæEf÷FTFVÆ•&V¦V7FVB‡Æ–W"ÂWV–BÂ6W'f–6RÂÆ–W$öæÆ–æRÂÆ–W%6W'fW"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р—Р ’òòf÷'v&Bâ66WFVBöffÆ–æR'&öF67B&Vf÷&RF†R7F–ÆÂÖ7F—fRvÆö&ÄFF ’òò6†ævRVWVW2F†R&Wv&B÷F÷FÇ2v÷&²âF†RVWVVBFVÆ—fW'’7FFR&WfVçG0 ’òò&WÆ––ær'&öF67G2F†BÇ&VG’&V6†VB&6¶VæBà ––b‡VWVTf÷%F–ÖT6†ævR’° •f÷FUF÷FÇ56æ6†÷B&ö¦V7FVEF÷FÇ2ÒÖævW5F÷FÇ2òvWE&ö¦V7FVE&öÆÆ÷fW%F÷FÇ2†FFÂÆ–W"’¢FW‡C° ––b†6åfÆ–FFU7FæFÆöæT'&öF67Bbb&÷‡”'&öF67DFV6–FW"çW6W4–ÖÖVF–FTf÷'v&F–ær‡Æ–W$öæÆ–æR’’° –'&öF67EF&vWG2æFDƇ&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2†fÇ6RÂçVÆÂ’“° —&÷‡”'&öF67D†æFÆVBÒG'VS° —Р•f÷FUF–ÖUVWVRFVÆ–VEf÷FRÒæWrf÷FUF–ÖUVWVR‡f÷FT–BÂÆ–W"Â6W'f–6RÂF–ÖRÀ —&÷‡”'&öF67D†æFÆVBÂ'&öF67EF&vWG2Â'&öF67Df÷'v&FVE6W'fW'2À —&ö¦V7FVEF÷FÇ2ÓÒçVÆÂò""¢&ö¦V7FVEF÷FÇ2çFõ7G&–ær‚’ÂfÇ6RÂWV–B“° ––b‚vWEf÷FT66†T†æFÆW"‚’æFEF–ÖUf÷FUFô66†R†FVÆ–VEf÷FR’’° –Æöu6WfW&R‚%Væ&ÆRFòW'6—7BVWVVB&öÆÆ÷fW"f÷FRf÷""²Æ–W"²"ò"²6W'f–6P ’²#²6¶—–ær&÷‡’'&öF67B"“° —&WGW&âVWVVEf÷FU&W7VÇBå$UE%“° —Р––b‡&÷‡”'&öF67D†æFÆVB’° –f÷"…7G&–ærF&vWB¢'&öF67EF&vWG2’° •6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡F&vWB’ÂWV–BÂÆ–W"À —6W'f–6RÂF–ÖRÂ&ö¦V7FVEF÷FÇ2ÓÒçVÆÂò""¢&ö¦V7FVEF÷FÇ2çFõ7G&–ær‚’ÂfÇ6R“° ––b†FVÆ–VEf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –'&öF67Df÷'v&FVE6W'fW'2æFDƆf÷'v&FVB“° —W'6—7EF–ÖUf÷FTFVÆ—fW'’†FVÆ–VEf÷FR“° —Р—Р—Р–Æör‚$66†–ærf÷FRg&öÒ"²Æ–W"²"ò"²6W'f–6P ’²"&V6W6RF–ÖR6†ævR—2†Væ–ær&–v‡Bæ÷r"“° —&WGW&âVWVVEf÷FU&W7VÇBå5T44U53° —Р –FEf÷FU'G’‚“°  ’òòF÷FÇ2&ö6W76–ær‡&–Ö'’6W'fW"õ"æò×VÇF—&÷‡’ ––b‡&ö6W76W5F÷FÇ2’° ––b†ÖævW5F÷FÇ2’° ––çBÆÅF–ÖUF÷FÂÒvWEfÇVR†FFÂ$ÆÅF–ÖUF÷FÂ"“° ––çBÖöçF…F÷FÂÒvWEfÇVR†FFÂ$ÖöçF…F÷FÂ"“°  ––çBFFTÖöçF…F÷FÂÒÓ° ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° ––b†vWD6öæf–r‚’ævWEW6TÖöçF„FFUF÷FÇ45&–Ö'•F÷F‚’’° –FFTÖöçF…F÷FÂÒvWEfÇVR†FFÂvWDÖöçF…F÷FÇ5v—F„FFUF‚‚’“° —ÒVÇ6R° –FFTÖöçF…F÷FÂÒÖöçF…F÷Fð —Р—Р ––çBvVV¶Ç•F÷FÂÒvWEfÇVR†FFÂ%vVV¶Ç•F÷FÂ"“° ––çBF–Ç•F÷FÂÒvWEfÇVR†FFÂ$F–Ç•F÷FÂ"“° ––çBö–çG2ÒvWEfÇVR†FFÂ%ö–çG2"ÂvWD6öæf–r‚’ævWEö–çG4öåf÷FR‚’“°  ––çBÖ…f÷FW2ÒvWD6öæf–r‚’ævWDÖ„Ö÷VçDöef÷FW5W$F’‚“° ––b†Ö…f÷FW2â’° ”Æö6ÄFFUF–ÖR5F–ÖRÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚“° ––çBF—2Ò5F–ÖRævWDF”ödÖöçF‚‚“° ––b†ÖöçF…F÷FÂâF—2¢Ö…f÷FW2’° –ÖöçF…F÷FÂÒF—2¢Ö…f÷FW3° —Р—Р ––b†vWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’âbbö–çG2âvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’’° —ö–çG2ÒvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚“° —Р —FW‡BÒæWrf÷FUF÷FÇ56æ6†÷B†ÆÅF–ÖUF÷FÂÂÖöçF…F÷FÂÂvVV¶Ç•F÷FÂÂF–Ç•F÷FÂÂö–çG2À —f÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VBÂFFTÖöçF…F÷F“°  ”'&”Æ—7CÄ6öÇVÖãâWFFRÒæWr'&”Æ—7CÃâ‚“° —WFFRæFB†æWr6öÇVÖâ‚$ÆÅF–ÖUF÷FÂ"ÂæWrFFfÇVT–çB†ÆÅF–ÖUF÷FÂ’’“° —WFFRæFB†æWr6öÇVÖâ‚$ÖöçF…F÷FÂ"ÂæWrFFfÇVT–çB†ÖöçF…F÷FÂ’’“° ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° —WFFRæFB†æWr6öÇVÖâ†vWDÖöçF…F÷FÇ5v—F„FFUF‚‚’ÂæWrFFfÇVT–çB†FFTÖöçF…F÷FÂ’’“° —Р—WFFRæFB†æWr6öÇVÖâ‚%vVV¶Ç•F÷FÂ"ÂæWrFFfÇVT–çB‡vVV¶Ç•F÷FÂ’’“° —WFFRæFB†æWr6öÇVÖâ‚$F–Ç•F÷FÂ"ÂæWrFFfÇVT–çB†F–Ç•F÷FÂ’’“° —WFFRæFB†æWr6öÇVÖâ‚%ö–çG2"ÂæWrFFfÇVT–çB‡ö–çG2’’“°  –FV'Vr‚%6WGF–ærF÷FÇ2"²FW‡BçFõ7G&–ær‚’²"Âf÷FT–CÒ"²f÷FT–B²"f÷""²Æ–W"²"ò  ’²6W'f–6R“° –vWE&÷‡”ו5‚’çWFFR‡WV–BÂWFFR“° —ÒVÇ6R° —FW‡BÒæWrf÷FUF÷FÇ56æ6†÷BƒÂÂÂÂÂf÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VB“° —Р—Р––b‡FW‡BÓÒçVÆÂ’° —FW‡BÒæWrf÷FUF÷FÇ56æ6†÷BƒÂÂÂÂÂf÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VB“° —Р •f÷FTÆöu7FGW2f÷FU7FGW2Òf÷FTÆöu7FGW2ä”ÔÔTD”DS° –&ööÆVâ7FæFÆöæU&÷‡”'&öF67BÒ6åfÆ–FFU7FæFÆöæT'&öF67Bbb‡&÷‡”'&öF67D†æFÆV@ —ÇÂ&÷‡”'&öF67DFV6–FW"çW6W4–ÖÖVF–FTf÷'v&F–ær‡Æ–W$öæÆ–æR’“° •6WCÅ7G&–æsâ&÷‡”'&öF67EF&vWG2Ò6öÆÆV7F–öç2æV×G•6WB‚“° ––b‡7FæFÆöæU&÷‡”'&öF67B’° ’òò†æFÆVBVWVVB'&öF67Bv2æV6W76&–Ç’6×ÆVBv†–ÆRF†RÆ–W"v0 ’òòöffÆ–æRâ&WG'’öæÇ’F&vWG2F†BF–Bæ÷B&Wf–÷W6Ç’66WBFVÆ—fW'’à —&÷‡”'&öF67EF&vWG2Ò&÷‡”'&öF67D†æFÆVBòæWrÆ–æ¶VD†6…6WCÃâ†'&öF67EF&vWG2 “¢&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2†fÇ6RÂçVÆ“° •6WCÅ7G&–æsâ&VÖ–æ–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ‡&÷‡”'&öF67EF&vWG2“° —&VÖ–æ–æuF&vWG2ç&VÖ÷fTƆ'&öF67Df÷'v&FVE6W'fW'2“° –'&öF67Df÷'v&FVE6W'fW'2æFDƇ6VæE&÷‡”'&öF67B‡&VÖ–æ–æuF&vWG2ÂWV–BÂÆ–W"Â6W'f–6RÂF–ÖRÀ —FW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’ÂfÇ6R’“° —Р ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР’òò6VæBf÷FR‡2’Fò&6¶VæB‡2 ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР––b†vWD6öæf–r‚’ævWE6VæEf÷FW5FôÆÅ6W'fW'2‚’’° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’°  –&ööÆVâf÷&6T66†RÒvWD6öæf–r‚’ævWEv—Df÷%W6W$öæÆ–æR‚ ’bb‚Æ–W$öæÆ–æRÇÂÆ–W%6W'fW"ÓÒçVÆÂÇÂÆ–W%6W'fW"æWVÇ4–væ÷&T66R‡2’“°  ––b†f÷&6T66†R’° –FV'Vr‚$f÷&6–ærf÷FRFò66†Rf÷"6W'fW""²2“° —Р ––b‚‚—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡2’bbÖWF†öBç&WV—&W5Æ–W$öæÆ–æR‚’’ÇÂf÷&6T66†R’° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFE6W'fW%f÷FR‡2À –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÀ —FW‡BçFõ7G&–ær‚’Â'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÀ —&÷‡”'&öF67EF&vWG2Â'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–ærf÷FRf÷""²Æ–W"²"öâ"²6W'f–6R²"f÷""²2“° —ÒVÇ6R° –&ööÆVâ'&öF67D†W&RÒ'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç2‡2“° ––b†'&öF67D†W&RbbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° •6WCÅ7G&–æsâF&vWG2Ò7FæFÆöæU&÷‡”'&öF67Bò&÷‡”'&öF67EF&vWG0 “¢&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡Æ–W$öæÆ–æRÂÆ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡2ÂF&vWG2“° —Р ––b‚6VæEf÷FTVçfVÆ÷T66WFVB‡2Â"À •f÷F–æuÇVv–åv—&Rçf÷FR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂG'VRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À —f÷FT–BÂvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’Â'&öF67D†W&RÂÂ’’’° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFE6W'fW%f÷FR‡2À –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÀ —FW‡BçFõ7G&–ær‚’Â'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÀ —&÷‡”'&öF67EF&vWG2Â'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–ærf÷FRgFW"F†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²2“° —Р—Р—Р—ÒVÇ6R° ’òò6–ævÆR×6W'fW"ÖöFS¢öæÆ–æRvöW2FòÆ–W"6W'fW#²÷F†W'v—6RVWVR2&öæÆ–æP ’òòf÷FR  ––b‡Æ–W$öæÆ–æRbbÆ–W%6W'fW"ÒçVÆÂbbvWDÆÄf–Æ&ÆU6W'fW'2‚’æ6öçF–ç2‡Æ–W%6W'fW"’’° •7G&–ær6W'fW"ÒÆ–W%6W'fW#°  –&ööÆVâ'&öF67D†W&RÒ'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç2‡6W'fW"“° ––b†'&öF67D†W&RbbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° •6WCÅ7G&–æsâF&vWG2Ò7FæFÆöæU&÷‡”'&öF67Bò&÷‡”'&öF67EF&vWG0 “¢&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡G'VRÂÆ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡6W'fW"ÂF&vWG2“° —Р –&ööÆVâ&Wv&D66WFVBÒ6VæEf÷FTVçfVÆ÷T66WFVB‡6W'fW"ÂÀ •f÷F–æuÇVv–åv—&Rçf÷FTöæÆ–æR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂG'VRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À —f÷FT–BÂvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’Â'&öF67D†W&RÂÂ’“° ––b‚&Wv&D66WFVB’° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFDöæÆ–æUf÷FR‡WV–BÀ –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À –'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÂ&÷‡”'&öF67EF&vWG2À –'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–æröæÆ–æRf÷FRgFW"F†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²6W'fW"“° —Р ––b‡&Wv&D66WFVBbb6åfÆ–FFU7FæFÆöæT'&öF67BbbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚ ’bb7FæFÆöæU&÷‡”'&öF67B’° •6WCÅ7G&–æsâF&vWG2Ò&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡G'VRÂÆ–W%6W'fW"“°  ––çB$FVÆ’Ò#° –f÷"…7G&–ærF&vWE6W'fW"¢F&vWG2’° ’òòfö–BF÷V&ÆRÖ'&öF67BöâF†R6ÖR6W'fW"F†BÇ&VG’v÷BF†Rf÷FTöæÆ–æP ––b‡F&vWE6W'fW"æWVÇ4–væ÷&T66R‡6W'fW"’’° –6öçF–çVS° —Р––b†vWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡F&vWE6W'fW"’’° –6öçF–çVS° —Р –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡F&vWE6W'fW"Â$FVÆ’À •f÷F–æuÇVv–åv—&Rçf÷FT'&öF67B‡WV–BÂÆ–W"Â6W'f–6RÂF–ÖRÀ —FW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’ÂG'VR’“° –$FVÆ’²³° —Р—Р ’òò×VÇF—&÷‡“¢VçfVÆ÷RÖöæÇ’6ÆV"f÷FP ––b‡&Wv&D66WFVBbbvWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚’bbvWD6öæf–r‚’ævWD×VÇF•&÷‡”öæTvÆö&Å&Wv&B‚’’° –×VÇF•&÷‡”†æFÆW"ç6VæD6ÆV%f÷FR‡WV–BÂÆ–W"“° —Р—ÒVÇ6R° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFDöæÆ–æUf÷FR‡WV–BÀ –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À –'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÂ&÷‡”'&öF67EF&vWG2À –'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–æröæÆ–æRf÷FRf÷""²Æ–W"²"öâ"²6W'f–6R“° —Р ––çBFVÆ’Ò#° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡2ÂFVÆ’²Âf÷F–æuÇVv–åv—&Rçf÷FUWFFR‡WV–BÀ —f÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VBÂ6W'f–6RÂF–ÖRÂFW‡BçFõ7G&–ær‚’’“° –FVÆ’³Ò#° —Р—Р ’òòf÷FRÆövv–æp ––b‡f÷FTÆöt×—7ÅF&ÆRÒçVÆÂbbvWD6öæf–r‚’ævWEf÷FTÆövv–ætVæ&ÆVB‚’’° —f÷FTÆöt×—7ÅF&ÆRæÆöuf÷FR‡f÷FT–BÂf÷FU7FGW2Â6W'f–6RÂWV–BÂÆ–W"ÂF–ÖRÀ –vWEf÷FT66†T†æFÆW"‚’ævWE&÷‡”66†VEF÷F‡WV–B’“° —Р ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР’òò×VÇF—&÷‡’f÷'v&F–æp ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР––b†vWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚’bbvWD6öæf–r‚’ævWE&–Ö'•6W'fW"‚’’° ––b‚vWD6öæf–r‚’ævWD×VÇF•&÷‡”öæTvÆö&Å&Wv&B‚’’° –FV'Vr‚%6VæF–ærvÆö&Â&÷‡’f÷FRVçfVÆ÷R"“° –×VÇF•&÷‡”†æFÆW"ç6VæD×VÇF•&÷‡”VçfVÆ÷R…f÷F–æuÇVv–åv—&Rçf÷FR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂfÇ6RÀ —&VÅf÷FRÂFW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’Âf÷FT–BÂfÇ6RÂfÇ6RÂÂ’“° —ÒVÇ6R° ’òòöæÇ’6VæBFò÷F†W"&÷†–W2–bF†RÆ–W"D”BäõBÇ&VG’&V6V—fR&Wv&Böâ ’òò&6¶Væ@ –&ööÆVâ6†÷VÆE6VæBÒG'VS° ––b‡Æ–W$öæÆ–æRbbÆ–W%6W'fW"ÒçVÆÂ’° ––b‚vWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡Æ–W%6W'fW"’’° —6†÷VÆE6VæBÒfÇ6S° —Р—Р ––b‡6†÷VÆE6VæB’° –FV'Vr‚%6VæF–ærvÆö&Â&÷‡’f÷FVöæÆ–æRVçfVÆ÷R"“° –×VÇF•&÷‡”†æFÆW  ’ç6VæD×VÇF•&÷‡”VçfVÆ÷R…f÷F–æuÇVv–åv—&Rçf÷FTöæÆ–æR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂfÇ6RÀ —&VÅf÷FRÂFW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’Âf÷FT–BÂfÇ6RÂfÇ6RÂÂ’“° —ÒVÇ6R° –FV'Vr‚$æ÷B6VæF–ærvÆö&Â&÷‡’ÖW76vRf÷"f÷FVöæÆ–æRÂÆ–W"Ç&VG’v÷B&Wv&B"“° —Р—Р—Р––b‡VWVVEf÷FRÒçVÆÂ’° —VWVVEf÷FRç6WE&ö6W76VB‡G'VR“° ––b‚vWEf÷FT66†T†æFÆW"‚’çWFFUF–ÖUf÷FR‡VWVVEf÷FR’’° —v&â‚%Væ&ÆRFòW'6—7B6ö×ÆWFVB&öÆÆ÷fW"f÷FR"²VWVVEf÷FRævWEf÷FT–B‚ ’²#²GFV×F–ærGW&&ÆR&VÖ÷f–ÖÖVF–FVÇ’"“° —Р—Р—&WGW&âVWVVEf÷FU&W7VÇBå5T44U53° —Ò6F6‚„W†6WF–öâR’° –Rç&–çE7F6µG&6R‚“° —&WGW&âVWVVEf÷FU&W7VÇBå$UE%“° —Р—Р —&—fFR7FF–2f–æÂ6Æ72VæF–æu&W6Væ6T†æFöfb° —&—fFRUT”B&WVW7D–C° —&—fFRf–æÂUT”BÆ–W%WV–C° —&—fFRf–æÂ7G&–ærÆ–W$æÖS° —&—fFRf–æÂ7G&–ærWV–C° —&—fFRf–æÂ7G&–ær6W'fW#° —&—fFRf–æÂUT”B6öææV7F–öä–C° —&—fFRf–æÂUT”B&6¶VæD–æ6&æF–öä–C° —&—fFRf–æÂÆöær&6¶VæE7F'FVDC° —&—fFRf–æÂÆöær6öæfÆ–7E6WVVæ6S° —&—fFRf–æÂÆöær7&VFVDC°  —&—fFRVæF–æu&W6Væ6T†æFöfb…7G&–ærÆ–W$æÖRÂ7G&–ærWV–BÂ7G&–ær6W'fW"ÂUT”B6öææV7F–öä–BÀ •UT”B&6¶VæD–æ6&æF–öä–BÂÆöær&6¶VæE7F'FVDBÂÆöær6öæfÆ–7E6WVVæ6RÂÆöær7&VFVDB’° —F†—2çÆ–W%WV–BÒ'6UÆ–W%WV–B‡WV–B“° —F†—2çÆ–W$æÖRÒÆ–W$æÖS° —F†—2çWV–BÒWV–C° —F†—2ç6W'fW"Ò6W'fW#° —F†—2æ6öææV7F–öä–BÒ6öææV7F–öä–C° —F†—2æ&6¶VæD–æ6&æF–öä–BÒ&6¶VæD–æ6&æF–öä–C° —F†—2æ&6¶VæE7F'FVDBÒ&6¶VæE7F'FVDC° —F†—2æ6öæfÆ–7E6WVVæ6RÒ6öæfÆ–7E6WVVæ6S° —F†—2æ7&VFVDBÒ7&VFVDC° —Р —&—fFR7FF–2UT”B'6UÆ–W%WV–B…7G&–ærWV–B’° —G'’° —&WGW&âUT”Bæg&öÕ7G&–ær‡WV–BçG&–Ò‚’“° —Ò6F6‚„W†6WF–öâ–væ÷&VB’° —&WGW&âçVÆð —Р—Р—Р —V&Æ–2'7G&7Bfö–Bv&â…7G&–ærÖW76vR“°  —V&Æ–2'7G&7B66†VGVÆVDW†V7WF÷%6W'f–6RvWE66†VGVÆW"‚“°§Ð  \ No newline at end of file diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 094d752c9..4c006c02d 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -80,6 +80,19 @@ void inboundDeliveryFenceRejectsCorruptionAndPathReplacement() throws Exception assertThrows(java.io.IOException.class, () -> store.reserve(java.util.UUID.randomUUID().toString())); } + @Test + void sealedInboundStoreCannotChangeAfterOwnershipHandoff() throws Exception { + Path credentials = directory.resolve("sealed-client"); + Files.createDirectories(credentials); + String id = java.util.UUID.randomUUID().toString(); + HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(credentials); + store.reserve(id); + store.markRunning(id); + store.seal(); + assertThrows(java.io.IOException.class, () -> store.markCompleted(id)); + assertEquals(HttpInboundDeliveryStore.State.RUNNING, new HttpInboundDeliveryStore(credentials).state(id)); + } + @Test void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { HttpTlsIdentity created = HttpTlsIdentity.loadOrCreate(directory, "localhost"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index d11aea72f..dd2fa418a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -89,6 +90,44 @@ void closeNeverWaitsForSetupOnTheCallingThread() throws Exception { finally { release.countDown(); blocked.join(TimeUnit.SECONDS.toMillis(1)); } } + @Test + @SuppressWarnings("unchecked") + void validationWaitsForThePreviousDirectoryOwner() throws Exception { + Path credentials = directory.resolve("http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy-owner"), "proxy.example.test"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://proxy.example.test:1297/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "C".repeat(43)); + HttpClientCredentialStore.saveEnrolled(credentials, code, identity.issueClientCertificate("lobby-1")); + VotingPluginMain plugin = mock(VotingPluginMain.class); + BungeeSettings settings = mock(BungeeSettings.class); + when(plugin.getDataFolder()).thenReturn(directory.toFile()); + when(plugin.getBungeeSettings()).thenReturn(settings); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + when(settings.getServer()).thenReturn("lobby-1"); + when(settings.getHttpConnectionCode()).thenReturn(""); + java.lang.reflect.Field ownersField = HttpBackendProxyTransport.class.getDeclaredField("DIRECTORY_OWNERS"); + ownersField.setAccessible(true); + var owners = (java.util.concurrent.ConcurrentHashMap) ownersField.get(null); + java.util.concurrent.Semaphore predecessor = new java.util.concurrent.Semaphore(0); + owners.put(credentials.toAbsolutePath().normalize(), predecessor); + HttpBackendProxyTransport transport = new HttpBackendProxyTransport(plugin); + transport.start(mock(GlobalMessageHandler.class)); + java.util.concurrent.atomic.AtomicReference failure = new java.util.concurrent.atomic.AtomicReference<>(); + CountDownLatch finished = new CountDownLatch(1); + Thread validation = new Thread(() -> { + try { transport.validate(); } + catch (Throwable thrown) { failure.set(thrown); } + finally { finished.countDown(); } + }); + validation.start(); + assertFalse(finished.await(150, TimeUnit.MILLISECONDS), "validation published before journal handoff"); + predecessor.release(); + assertTrue(finished.await(3, TimeUnit.SECONDS)); + validation.join(TimeUnit.SECONDS.toMillis(1)); + assertNull(failure.get()); + transport.close(); + } + private static HttpConnectionCode code(String serverId, Instant expiry) { return new HttpConnectionCode(serverId, URI.create("https://proxy.example.test:1297/"), "a".repeat(64), "b".repeat(64), expiry, "A".repeat(43)); From dd1ef83f2c3f1b646158679a6e2a0a2c895bb49d Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 21:16:23 -0600 Subject: [PATCH 22/36] fix(ci): restore complete proxy source --- .../votingplugin/proxy/VotingPluginProxy.java | 3684 ++++++++++++++++- 1 file changed, 3683 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 36b185f2d..5608a5ce8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1 +1,3683 @@ -YªçŠx-®éÜj×¢ëiºÚ+Š§j[h‘éÜ¢éíã}å:-jZ.¶›­–)Þ³W6¶vR6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡“° ¦–×÷'B¦fæ–òä'—FT'&”÷WGWE7G&VÓ°¦–×÷'B¦fæ–òäFF–çWE7G&VÓ°¦–×÷'B¦fæ–òäFF÷WGWE7G&VÓ°¦–×÷'B¦fæ–òäf–ÆS°¦–×÷'B¦fæ–òä”ôW†6WF–öã°¦–×÷'B¦fææWBä–æWE6ö6¶WDFG&W73°¦–×÷'B¦fææWBå6ö6¶WC°¦–×÷'B¦fææWBåU$“°¦–×÷'B¦fææWBæ‡GGä‡GG6Æ–VçC°¦–×÷'B¦fææWBæ‡GGä‡GG&WVW7C°¦–×÷'B¦fææWBæ‡GGä‡GG&W7öç6S°¦–×÷'B¦fç7Âå5ÄW†6WF–öã°¦–×÷'B¦fçF–ÖRäGW&F–öã°¦–×÷'B¦fçF–ÖRä–ç7FçC°¦–×÷'B¦fçF–ÖRäÆö6ÄFFUF–ÖS°¦–×÷'B¦fçF–ÖRå¦öæT–C°¦–×÷'B¦fçF–ÖRå¦öæTöfg6WC°¦–×÷'B¦fçWF–Âä'&”Æ—7C°¦–×÷'B¦fçWF–Âä6öÆÆV7F–öã°¦–×÷'B¦fçWF–Âä6öÆÆV7F–öç3°¦–×÷'B¦fçWF–Âä†6„Ö°¦–×÷'B¦fçWF–ÂäÆ–æ¶VD†6…6WC°¦–×÷'B¦fçWF–ÂäÆ—7C°¦–×÷'B¦fçWF–ÂäÖ°¦–×÷'B¦fçWF–Âå6WC°¦–×÷'B¦fçWF–ÂåUT”C°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBä6ö×ÆWF&ÆTgWGW&S°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBä6öæ7W'&VçD†6„Ö°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBäW†V7WF÷%6W'f–6S°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBäW†V7WF÷'3°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBå66†VGVÆVDW†V7WF÷%6W'f–6S°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBåF–ÖUVæ—C°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBæFöÖ–2äFöÖ–4&ööÆVã°¦–×÷'B¦fçWF–Âæ6öæ7W'&VçBæFöÖ–2äFöÖ–4Æöæs° ¦–×÷'B¦f‚ææWBç76Âå54Å&ÖWFW'3° ¦–×÷'B÷&ræV6Æ—6Rç†òæ6Æ–VçBæ×GGc2ä×GDW†6WF–öã° ¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ’çF–ÖRåF–ÖUG—S°¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ'VævVV’ævÆö&ÆFFävÆö&ÄFF†æFÆW%&÷‡“°¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ'VævVV’ævÆö&ÆFFävÆö&Äו5ð¦–×÷'B6öÒæ&Væ6öFW¢æGfæ6VF6÷&Ræ'VævVV’çF–ÖRä'VævVUF–ÖT6†V6¶W#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’æVæ7'—F–öâäVæ7'—F–öä†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’æ§6öâä§6öå'6W#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ6öFV2ä§6öäVçfVÆ÷S°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ6öFV2ä§6öäVçfVÆ÷T6öFV3°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒævÆö&ÂävÆö&ÄÖW76vTÆ—7FVæW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒævÆö&ÂävÆö&ÄÖW76vU&÷‡”†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ×GBä×GD†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ×GBä×GE6W'fW$6öÖÓ°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒæ×—7Âäו7ÄÖW76VævW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç&VF—2å&VF—4†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç&VF—2å&VF—4Æ—7FVæW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç6ö6¶WG2ä6Æ–VçD†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç6ö6¶WG2å6ö6¶WD†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç6W'fW&6öÖÒç6ö6¶WG2å6ö6¶WE&V6V—fW#°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7Âä6öÇVÖã°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂäFFG—S°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVS°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVT&ööÆVã°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVT–çC°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7ÂæFFäFFfÇVU7G&–æs°¦–×÷'B6öÒæ&Væ6öFW¢ç6–×ÆV’ç7Âæ×—7Âæ6öæf–rä×—7Ä6öæf–s°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âæ&6¶VæG&÷‡’æ‡GGä‡GGVç&öÆÆÖVçDWF†÷&—G“°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âæ&6¶VæG&÷‡’æ‡GGä‡GG&÷‡•G&ç7÷'E6W'fW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âæ&6¶VæG&÷‡’æ‡GGä‡GGFÇ4–FVçF—G“°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ'&öF67Bå&÷‡”'&öF67DFV6–FW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Rä•f÷FT66†S°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Råf÷FT66†T†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Rææöçf÷FVBä”æöåf÷FVEÆ–W'57F÷&vS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ66†Rææöçf÷FVBäæöåf÷FVEÆ–W'466†S°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ6öçG&öÂä6öçG&öÄ6öææV7F÷#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ6öçG&öÂä†÷7FVD6öçG&öÄÖævW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡”†æFÆW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡”ÖWF†öC°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡•6W'fW%6ö6¶WD6öæf–wW&F–öã°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’æ×VÇF—&÷‡’ä×VÇF•&÷‡•6W'fW%6ö6¶WD6öæf–wW&F–öä'VævVS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’ç&W6Væ6Rä&6¶VæEÆ–W%&W6Væ6UG&6¶W#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’ç&W6Væ6RåÆ–W%&W6Væ6S°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçF–ÖWVWVRåf÷FUF–ÖUVWVS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçF÷f÷FW"åF÷f÷FW#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçWF–ÂäÖ–æV7&gEW6W&æÖUfÆ–FF÷#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçWF–Âå6W'f–6U6—FUfÆ–FF÷#°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçf÷FVÆöråf÷FTÆöt×—7ÅF&ÆS°¦–×÷'B6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âçf÷FVÆöråf÷FTÆöt×—7ÅF&ÆRåf÷FTÆöu7FGW3°¦–×÷'B6öÒævöövÆRæw6öâä§6öäVÆVÖVçC°¦–×÷'B6öÒævöövÆRæw6öâä§6öäö&¦V7C° ¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2äFVfVÇD¦VF—46Æ–VçD6öæf–s°¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2ä†÷7DæE÷'C°¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2ä¦VF—3°¦–×÷'B&VF—2æ6Æ–VçG2æ¦VF—2ä¦VF—5ööð ¦–×÷'BÆöÖ&ö²ävWGFW#°¦–×÷'BÆöÖ&ö²å6WGFW#° §V&Æ–2'7G&7B6Æ72f÷F–æuÇVv–å&÷‡’° —&—fFR7FF–2f–æÂÆöær$U4Tä4Uô„äDôdeõD”ÔTõUEôÔ”ÄÄ•2ÒF–ÖUVæ—BäÔ”åUDU2çFôÖ–ÆÆ—2ƒ"“° —&—fFR7FF–2f–æÂÆöær$U4Tä4Uõ5D%EUõ$U5”ä5ôDTÄ•õ4T4ôäE2ÒTð —&—fFR7FF–2f–æÂÆöær$U4Tä4UôÔ”åDTää4Uô”åDU%dÅõ4T4ôäE2Ò3ð —&—fFR7FF–2f–æÂÆöær$U4Tä4Uô$4´TäEõD”ÔTõUEôÔ”ÄÄ•2ÒF–ÖUVæ—Bå4T4ôäE2çFôÖ–ÆÆ—2ƒ““° —&—fFR7FF–2f–æÂÆöær4ôåE$ôÅôTå$ôÄÄÔTåEôÔ”åô”åDU%dÅôääõ2ÒF–ÖUVæ—Bå4T4ôäE2çFôææ÷2ƒ“°  ”vWGFW  ”6WGFW  —&—fFR–çBf÷FU'G•f÷FW2Ò°  ”vWGFW  ”6WGFW  —&—fFR–çB7W'&VçEf÷FU'G•f÷FW5&WV—&VBÒ°  ”vWGFW  ”6WGFW  —&—fFR&÷‡”×—7ÅW6W%F&ÆR&÷‡”ו5ð  —&—fFRVæ7'—F–öä†æFÆW"Væ7'—F–öä†æFÆW#°  —&—fFR†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â6Æ–VçD†æFÆW3°  —&—fFR6ö6¶WD†æFÆW"6ö6¶WD†æFÆW#° —&—fFR‡GG&÷‡•G&ç7÷'E6W'fW"‡GGG&ç7÷'E6W'fW#° —&—fFR‡GGVç&öÆÆÖVçDWF†÷&—G’‡GGVç&öÆÆÖVçDWF†÷&—G“°  ”vWGFW  ”6WGFW  —&—fFR&ööÆVâf÷F–f–W$Væ&ÆVBÒG'VS°  ”vWGFW  —&—fFR6öæ7W'&VçD†6„ÖÅUT”BÂ7G&–æsâWV–EÆ–W$æÖT66†RÒæWr6öæ7W'&VçD†6„ÖÃâ‚“°  ”vWGFW  ”6WGFW  —&—fFRvÆö&ÄFF†æFÆW%&÷‡’vÆö&ÄFF†æFÆW#°  ”vWGFW  —&—fFR&VF—4†æFÆW"&VF—4†æFÆW#° —&—fFR¦VF—5ööÂ&VF—5V&Æ—6†W%ööð —&—fFRföÆF–ÆRÆöær&VF—5V&Æ—6†W%&WG'”gFW#° —&—fFR&ööÆVâF–ÖUf÷FU&WG'•66†VGVÆVC° —&—fFR&ööÆVâF–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVC° —&—fFR&ööÆVâ66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVC°  —&—fFR&ööÆVâVæ&ÆVC°  ”vWGFW  ”6WGFW  —&—fFR×VÇF•&÷‡”†æFÆW"×VÇF•&÷‡”†æFÆW#°  ”vWGFW  —&—fFR'VævVUF–ÖT6†V6¶W"'VævVUF–ÖT6†V6¶W#°  ”vWGFW  ”6WGFW  —&—fFR'VævVTÖWF†öBÖWF†öC°  ”vWGFW  —&—fFR×GD†æFÆW"×GD†æFÆW#°  ”vWGFW  —&—fFRvÆö&ÄÖW76vU&÷‡”†æFÆW"vÆö&ÄÖW76vU&÷‡”†æFÆW#°  ”vWGFW  ”6WGFW  —&—fFRו7ÄÖW76VævW"&÷‡”×—7ÄÖW76VævW#°  ”vWGFW  —&—fFRf÷FT66†T†æFÆW"f÷FT66†T†æFÆW#°  ”vWGFW  —&—fFRæöåf÷FVEÆ–W'466†Ræöåf÷FVEÆ–W'466†S°  ”vWGFW  —&—fFRf–æÂ&6¶VæEÆ–W%&W6Væ6UG&6¶W"&6¶VæEÆ–W%&W6Væ6UG&6¶W"ÒæWr&6¶VæEÆ–W%&W6Væ6UG&6¶W"‚“° —&—fFRf–æÂÖÅUT”BÂVæF–æu&W6Væ6T†æFöfcâVæF–æu&W6Væ6T†æFöfg2ÒæWr†6„ÖÃâ‚“° —&—fFRf–æÂ6WCÅ7G&–æsâVæF–æt&6¶VæE&V6÷fW'•6æ6†÷G2Ò6öæ7W'&VçD†6„ÖææWt¶W•6WB‚“° —&—fFRf–æÂÖÅ7G&–ærÂÆöæsâ6öçG&öÄVç&öÆÆÖVçDæW‡DÆÆ÷vVBÒæWr6öæ7W'&VçD†6„ÖÃâ‚“° —&—fFRf–æÂÖÅUT”BÂVæF–æt6öÖ×Væ–6F–öåFW7CâVæF–æt6öÖ×Væ–6F–öåFW7G2ÒæWr6öæ7W'&VçD†6„ÖÃâ‚“° —&—fFRföÆF–ÆR6öçG&öÄ6öææV7F÷"6öçG&öÄ6öææV7F÷#° —&—fFRföÆF–ÆR†÷7FVD6öçG&öÄÖævW"†÷7FVD6öçG&öÄÖævW#° —&—fFRf–æÂö&¦V7B6öçG&öÄÆ–fV7–6ÆTÆö6²ÒæWrö&¦V7B‚“° —&—fFRf–æÂFöÖ–4Æöær6öçG&öÅ6W'f–6W4vVæW&F–öâÒæWrFöÖ–4Æöær‚“° —&—fFRf–æÂW†V7WF÷%6W'f–6R6öçG&öÄÆ–fV7–6ÆTW†V7WF÷"ÒW†V7WF÷'2ææWu6–ævÆUF‡&VDW†V7WF÷"‡F6²Óâ° •F‡&VBF‡&VBÒæWrF‡&VB‡F6²Â'f÷F–æwÇVv–âÖ6öçG&öÂÖÆ–fV7–6ÆR"“° —F‡&VBç6WDFVÖöâ‡G'VR“° —&WGW&âF‡&VC° —Ò“°  —V&Æ–2f÷F–æuÇVv–å&÷‡’‚’° –Væ&ÆVBÒG'VS°  –'VævVUF–ÖT6†V6¶W"ÒæWr'VævVUF–ÖT6†V6¶W"†vWD6öæf–r‚’ævWEF–ÖU¦öæR‚’ÂvWD6öæf–r‚’ævWEF–ÖT†÷W$öfe6WB‚’À –vWD6öæf–r‚’ævWEF–ÖUvVV´öfe6WB‚’’°  ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…7G&–ærFW‡B’° –FV'Vs"‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2ÆöærvWDÆ7EWFFVB‚’° —&WGW&âvWEf÷FT66†TÆ7EWFFVB‚“° —Р ”÷fW'&–FP —V&Æ–2–çBvWE&WdF’‚’° —&WGW&âvWEf÷FT66†U&WdF’‚“° —Р ”÷fW'&–FP —V&Æ–27G&–ærvWE&WdÖöçF‚‚’° —&WGW&âvWEf÷FT66†U&WdÖöçF‚‚“° —Р ”÷fW'&–FP —V&Æ–2–çBvWE&WevVV²‚’° —&WGW&âvWEf÷FT66†U&WevVV²‚“° —Р ”÷fW'&–FP —V&Æ–2fö–B–æfò…7G&–ærFW‡B’° –Æör‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2&ööÆVâ—4Væ&ÆVB‚’° —&WGW&âVæ&ÆVC° —Р ”÷fW'&–FP —V&Æ–2&ööÆVâ—4–væ÷&UF–ÖR‚’° —&WGW&â—5f÷FT66†T–væ÷&UF–ÖR‚“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WD–væ÷&UF–ÖR†&ööÆVâ–væ÷&R’° —6WEf÷FT66†Uf÷FT66†T–væ÷&UF–ÖR†–væ÷&R“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WDÆ7EWFFVB‚’° —6WEf÷FT66†TÆ7EWFFVB‚“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WE&WdF’†–çBF’’° —6WEf÷FT66†U&WdF’†F’“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WE&WdÖöçF‚…7G&–ærFW‡B’° —6WEf÷FT66†U&WdÖöçF‚‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–B6WE&WevVV²†–çBvVV²’° —6WEf÷FT66†U&WevVV²‡vVV²“° —Р ”÷fW'&–FP —V&Æ–2fö–BF–ÖT6†ævVB…F–ÖUG—RG—RÂ&ööÆVâf¶RÂ&ööÆVâ&RÂ&ööÆVâ÷7B’° ––b†vWD6öæf–r‚’ævWEf÷FT66†UF–ÖR‚’â’° –vWEf÷FT66†T†æFÆW"‚’æ6†V6µf÷FT66†UF–ÖR†vWD6öæf–r‚’ævWEf÷FT66†UF–ÖR‚’“° —Р––b‚vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’’° —v&â‚$vÆö&ÂFFæ÷BVæ&ÆVB–væ÷&–ærF–ÖR6†ævRWfVçB"“° —&WGW&ã° —Р––çBFVÆ’Ò° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° ––b†vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æ6öçF–ç4¶W’‡2’’° •7G&–ærÆ7DöæÆ–æU7G"ÒvWDvÆö&ÄFF†æFÆW"‚’ævWE7G&–ær‡2Â$Æ7DöæÆ–æR"“° –ÆöærÆ7DöæÆ–æRÒ° —G'’° –Æ7DöæÆ–æRÒÆöærçfÇVTöb†Æ7DöæÆ–æU7G"“° —Ò6F6‚„çVÖ&W$f÷&ÖDW†6WF–öâR’° ’òò–væ÷&P —Р ––b„Æö6ÄFFUF–ÖRææ÷r‚’æE¦öæR…¦öæTöfg6WBåUD2’çFô–ç7FçB‚’çFôWö6„Ö–ÆÆ’‚’ÒÆ7DöæÆ–æR ’¢c¢c¢"’° ”†6„ÖÅ7G&–ærÂFFfÇVSâFFFõ6WBÒæWr†6„ÖÃâ‚“° –FFFõ6WBçWB‚$Æ7EWFFVB"ÂæWrFFfÇVU7G&–ær€ ’""²Æö6ÄFFUF–ÖRææ÷r‚’æE¦öæR…¦öæTöfg6WBåUD2’çFô–ç7FçB‚’çFôWö6„Ö–ÆÆ’‚’’“° –FFFõ6WBçWB‚$f–æ—6†VE&ö6W76–ær"ÂæWrFFfÇVT&ööÆVâ†fÇ6R’“° –FFFõ6WBçWB‡G—RçFõ7G&–ær‚’ÂæWrFFfÇVT&ööÆVâ‡G'VR’“° –vWDvÆö&ÄFF†æFÆW"‚’ç6WDFF‡2ÂFFFõ6WB“°  –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡2ÂFVÆ’Âf÷F–æuÇVv–åv—&Ræ'VævVUF–ÖT6†ævR‚’“° –FVÆ’²³° —ÒVÇ6R° —v&â‚%6W'fW""²2²"†6âwB&VVâöæÆ–æR&V6VçFÇ’"“° —Р—ÒVÇ6R° —v&â‚%6W'fW""²2²"vÆö&ÂFF†æFÆW"F—6&ÆVCò"“° —Р—Р–vÆö&ÄFF†æFÆW"æöåF–ÖT6†ævR‡G—R“° —Р ”÷fW'&–FP —V&Æ–2fö–Bv&æ–ær…7G&–ærFW‡B’° —v&â‡FW‡B“° —Р—Ó° —Р —V&Æ–2fö–BöåF–ÖT6†ævVDf–ÆVB…7G&–ær7'bÂF–ÖUG—RG—R’° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡7'bÂG—RçFõ7G&–ær‚’ÂfÇ6R“° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡7'bÂ$f–æ—6†VE&ö6W76–ær"ÂG'VR“° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡7'bÂ%&ö6W76–ær"ÂfÇ6R“° —Р —V&Æ–2fö–BöåF–ÖT6†ævVDf–æ—6†VB…F–ÖUG—RG—R’° ––b‡G—RæWVÇ2…F–ÖUG—RäÔôåD‚’’° –vWE&÷‡”ו5‚’æ6÷”6öÇVÖäFF…F÷f÷FW"äÖöçF†Ç’ævWD6öÇVÖäæÖR‚’Â$Æ7DÖöçF…F÷FÂ"“° —Р–vWE&÷‡”ו5‚’çv—T6öÇVÖäFF…F÷f÷FW"æöb‡G—R’ævWD6öÇVÖäæÖR‚’ÂFFG—Rä”åDTtU"“°  ––b‚vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’’° —&WGW&ã° —Р–f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° –vWDvÆö&ÄFF†æFÆW"‚’ç6WD&ööÆVâ‡2Â$f÷&6UWFFR"ÂG'VR“° –vWDvÆö&ÄÖW76vU&÷‡”†æFÆW"‚’ç6VæDÖW76vR‡2ÂÂf÷F–æuÇVv–åv—&Ræ'VævVUF–ÖT6†ævR‚’“° —Р—&ö6W75VWVR‚“° —Р ’ò¢  ’¢ÆöBו5²vÆö&ÂFF†æFÆW"à ’¢ð —V&Æ–2fö–BÆöD×—7„ח7Ä6öæf–r×—7Ä6öæf–rÂ×—7Ä6öæf–rvÆö&ÄFF×—7Ä6öæf–r’° ––b†×—7Ä6öæf–rævWD†÷7DæÖR‚’æ—4V×G’‚’ÇÂ×—7Ä6öæf–rævWDFF&6R‚’æ—4V×G’‚’’° –Æöu6WfW&R‚$ו5—2æ÷B6öæf–wW&VB6÷'&V7FÇ’â"²$Ö—76–ær†÷7BöFF&6Râ†÷7CÒ"²×—7Ä6öæf–rævWD†÷7DæÖR‚ ’²"F#Ò"²×—7Ä6öæf–rævWDFF&6R‚’“° —6WE&÷‡”ו5†çVÆ“° —&WGW&ã° —Р —6WE&÷‡”ו5†æWr&÷‡”×—7ÅW6W%F&ÆR‚%f÷F–æuÇVv–åõW6W'2"Â×—7Ä6öæf–rÂvWD6öæf–r‚’ævWDFV'Vr‚’’°  ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…5ÄW†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&R…7G&–ær7G&–ær’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöu6WfW&R‡7G&–ær“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöt–æfò…7G&–ær7G&–ær’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöt–æfò‡7G&–ær“° —Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…F‡&÷v&ÆRB’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° —Bç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vr…7G&–ær7G"’° –FV'Vs"‡7G"“° —Р—Ò“°  ”'&”Æ—7CÅ7G&–æsâ6W'fW'2ÒæWr'&”Æ—7CÅ7G&–æsâ†vWDÆÄf–Æ&ÆU6W'fW'2‚’“°  ––b†vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’’° ––b†vWD6öæf–r‚’ævWDvÆö&ÄFFW6TÖ–äו5‚’’° —6WDvÆö&ÄFF†æFÆW"†æWrvÆö&ÄFF†æFÆW%&÷‡’€ –æWrvÆö&Äו5‚%f÷F–æuÇVv–åôvÆö&ÄFF"ÂvWE&÷‡”ו5‚’ævWD×—7‚’’°  ”÷fW'&–FP —V&Æ–2fö–BFV'VtW‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'VtÆör…7G&–ærFW‡B’° –FV'Vr‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–B–æfò…7G&–ærFW‡B’° –Æöt–æfò‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&R…7G&–ærFW‡B’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöu6WfW&R‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–Bv&æ–ær…7G&–ærFW‡B’° —v&â‡FW‡B“° —Р—ÒÂ6W'fW'2’°  ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–ÆVB…7G&–ær7'bÂF–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–ÆVB‡7'bÂG—R“° —Р ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–æ—6†VB…F–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–æ—6†VB‡G—R“° —Р—Ò“° —ÒVÇ6R° —6WDvÆö&ÄFF†æFÆW"€ –æWrvÆö&ÄFF†æFÆW%&÷‡’†æWrvÆö&Äו5‚%f÷F–æuÇVv–åôvÆö&ÄFF"ÂvÆö&ÄFF×—7Ä6öæf–r’°  ”÷fW'&–FP —V&Æ–2fö–BFV'VtW‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'VtÆör…7G&–ærFW‡B’° –FV'Vr‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–B–æfò…7G&–ærFW‡B’° –Æöt–æfò‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&R…7G&–ærFW‡B’° •f÷F–æuÇVv–å&÷‡’çF†—2æÆöu6WfW&R‡FW‡B“° —Р ”÷fW'&–FP —V&Æ–2fö–Bv&æ–ær…7G&–ærFW‡B’° —v&â‡FW‡B“° —Р—ÒÂ6W'fW'2’°  ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–ÆVB…7G&–ær7'bÂF–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–ÆVB‡7'bÂG—R“° —Р ”÷fW'&–FP —V&Æ–2fö–BöåF–ÖT6†ævVDf–æ—6†VB…F–ÖUG—RG—R’° •f÷F–æuÇVv–å&÷‡’çF†—2æöåF–ÖT6†ævVDf–æ—6†VB‡G—R“° —Р—Ò“° —Р ’òòWFFRvÆö&Â66†VÖ6öÇVÖç2‡Væ6†ævVBg&öÒ÷&–v–æ –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$–væ÷&UF–ÖR"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$ÔôåD‚"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚%tTT²"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$D’"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$f–æ—6†VE&ö6W76–ær"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚%&ö6W76–ær"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$f÷&6UWFFR"Â%d$4„"ƒR’"“° –vWDvÆö&ÄFF†æFÆW"‚’ævWDvÆö&Ä×—7‚’æÇFW$6öÇVÖåG—R‚$Æ7EWFFVB"Â$ÔTD•TÕDU…B"“° —Р ’òò6öÇVÖâG—W2‡Væ6†ævVBg&öÒ÷&–v–æ –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%F÷f÷FW$–væ÷&R"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$6†V6µv÷&ÆB"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%&VÖ–æFVB"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F—6&ÆT'&öF67B"Â%d$4„"ƒR’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7DöæÆ–æR"Â%d$4„"ƒ#’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%Æ–W$æÖR"Â%d$4„"ƒ3’"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F–Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%vVV¶Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F•f÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$&W7DF•f÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%vVVµf÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$&W7EvVVµf÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%f÷FU'G•f÷FW2"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$ÖöçF…f÷FU7G&V²"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚%ö–çG2"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$†–v†W7DF–Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$ÆÅF–ÖUF÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$†–v†W7DÖöçF†Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$ÖöçF…F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$†–v†W7EvVV¶Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7DÖöçF…F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7EvVV¶Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$Æ7DF–Ç•F÷FÂ"Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$öffÆ–æU&Wv&G2"Â$ÔTD•TÕDU…B"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R‚$F•f÷FU7G&V´Æ7EWFFR"Â$ÔTD•TÕDU…B"“°  ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R†vWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖRææ÷r‚’’Â$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R†vWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖRææ÷r‚’çÇW4ÖöçF‡2ƒ’’À ’$”åBDTdTÅBsr"“° –vWE&÷‡”ו5‚’æÇFW$6öÇVÖåG—R†vWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖRææ÷r‚’çÇW4ÖöçF‡2ƒ"’’À ’$”åBDTdTÅBsr"“° —Р—Р —V&Æ–2fö–BFD7W'&VçEf÷FU'G•f÷FW2†–çBÖ÷VçB’° —f÷FU'G•f÷FW2³ÒÖ÷VçC° —6WEf÷FT66†Uf÷FU'G”7W'&VçEf÷FW2‡f÷FU'G•f÷FW2“° –FV'Vr‚$7W'&VçBf÷FR'G’F÷Fâ"²f÷FU'G•f÷FW2“° —Р —V&Æ–2fö–BFDæöåf÷FVEÆ–W"…7G&–ærWV–BÂ7G&–ærÆ–W$æÖR’° –æöåf÷FVEÆ–W'466†RæFEÆ–W"‡WV–BÂÆ–W$æÖR“° —Р —V&Æ–2fö–BFEf÷FU'G’‚’° ––b†vWD6öæf–r‚’ævWEf÷FU'G”Væ&ÆVB‚’’° –FD7W'&VçEf÷FU'G•f÷FW2ƒ“° –6†V6µf÷FU'G’‚“° —Р—Р —V&Æ–2'7G&7Bfö–B'&öF67B…7G&–ærÖW76vR“°  —&—fFR6WCÅ7G&–æsâ6VæE&÷‡”'&öF67B…6WCÅ7G&–æsâF&vWG2Â7G&–ærWV–BÂ7G&–ærÆ–W"Â7G&–ær6W'f–6RÂÆöærF–ÖRÀ •7G&–ærFW‡BÂ&ööÆVâv4öæÆ–æR’° •6WCÅ7G&–æsâf÷'v&FVBÒæWrÆ–æ¶VD†6…6WCÃâ‚“° –f÷"…7G&–ærF&vWE6W'fW"¢F&vWG2’° ”§6öäVçfVÆ÷RVçfVÆ÷RÒf÷F–æuÇVv–åv—&Rçf÷FT'&öF67B‡WV–BÂÆ–W"Â6W'f–6RÂF–ÖRÂFW‡BÂv4öæÆ–æR“° ––b‡6VæE&÷‡”'&öF67DVçfVÆ÷Tæ÷r‡F&vWE6W'fW"ÂVçfVÆ÷R’’° –f÷'v&FVBæFB‡F&vWE6W'fW"“° —Р—Р—&WGW&âf÷'v&FVC° —Р ’ò¢  ’¢6VæG27FæFÆöæR&÷‡’'&öF67BF‡&÷Vv‚F†R6VÆV7FVBG&ç7÷'BæB&W÷'G0 ’¢v†WF†W"F†BG&ç7÷'B66WFVBF†RÖW76vRà ’  ’¢&Ò6W'fW"F&vWB&6¶VæB6W'fW  ’¢&ÒVçfVÆ÷R7FæFÆöæR'&öF67BVçfVÆ÷P ’¢&WGW&âG'VRöæÇ’v†VâF†RG&ç7÷'B66WFVBF†RÖW76vP ’¢ð —&÷FV7FVB&ööÆVâ6VæE&÷‡”'&öF67DVçfVÆ÷Tæ÷r…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° —7v—F6‚†ÖWF†öB’° –66RÕEC  —&WGW&â6VæD×GDVçfVÆ÷U6W'fW"‡6W'fW"ÂVçfVÆ÷R“° –66RÕ•5à ––b‡&÷‡”×—7ÄÖW76VævW"ÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р—G'’° —&÷‡”×—7ÄÖW76VævW"ç6VæEFô&6¶VæB‡6W'fW"ÂVçfVÆ÷R“° —&WGW&âG'VS° —Ò6F6‚…5ÄW†6WF–öâR’° –FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р–66RÅTt”äÔU54t”äs  —&WGW&â6VæEÇVv–äÖW76vU6W'fW$æ÷r‡6W'fW"ÂVçfVÆ÷R“° –66R$TD•3  —&WGW&â6VæE&VF—4VçfVÆ÷U6W'fW"‡6W'fW"ÂVçfVÆ÷RÂG'VR“° –66R4ô4´UE3  ’òò7FæFÆöæR'&öF67G2W6RF†R6ÖR–æ—F–Æ—¦VB6Æ–VçB2æ÷&ÖÀ ’òòVçfVÆ÷W2âF†—2&W6W'fW2F†R6ö6¶WB6öææV7F–öâæB—G2FVÆ—fW' ’òò6¶æ÷vÆVFvVÖVçB–ç7FVBöb7&VF–ær6V6öæB6†÷'BÖÆ—fVB6ö6¶WBà —&WGW&â6VæE6ö6¶WDVçfVÆ÷R‡6W'fW"ÂVçfVÆ÷R“° –66R…EE  —&WGW&â6VæD‡GGVçfVÆ÷R‡6W'fW"ÂVçfVÆ÷R“° –FVfVÇC  —&WGW&âfÇ6S° —Р—Р ’ò¢  ’¢6VæG2&Wv&BÖ&V&–ærf÷FRVçfVÆ÷RæB&W÷'G2v†WF†W"F†R6VÆV7FV@ ’¢G&ç7÷'B66WFVB—BâÆVv7’G&ç7÷'G2&WF–âF†V—"W†—7F–ær7–æ6‡&öæ÷W0 ’¢6VÖçF–73²…EEW‡÷6W2—G2&÷VæFVB×VWVR&W7VÇB6òf÷FR—2æWfW"F—66&FV@ ’¢v†VâF†RVWVR—2gVÆÂà ’¢ð —&÷FV7FVB&ööÆVâ6VæEf÷FTVçfVÆ÷T66WFVB…7G&–ær6W'fW"–çBFVĤ6öäVçfVÆ÷RVçfVÆ÷R’° ––b†ÖWF†öBÓÒ'VævVTÖWF†öBä…EE’° —&WGW&â6VæD‡GGVçfVÆ÷R‡6W'fW"ÂVçfVÆ÷R“° —Р”vÆö&ÄÖW76vU&÷‡”†æFÆW"†æFÆW"ÒvÆö&ÄÖW76vU&÷‡”†æFÆW#° ––b††æFÆW"ÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р–†æFÆW"ç6VæDÖW76vR‡6W'fW"ÂFVÆ’ÂVçfVÆ÷R“° —&WGW&âG'VS° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–B6†V6´66†VEf÷FW2…7G&–ær6W'fW"’° ––çBFVÆ’Ò° ––b†—56W'fW%fÆ–B‡6W'fW"’’° ––b†—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡6W'fW"’’° ––b†vWEf÷FT66†T†æFÆW"‚’æ†5f÷FW2‡6W'fW"’bbvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ2ÒvWEf÷FT66†T†æFÆW"‚’ævWEf÷FW2‡6W'fW"“° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ&VÖ÷fVBÒæWr'&”Æ—7CÃâ‚“° ––b‚2æ—4V×G’‚’’° ––çBçVÒÒ° ––çBçVÖ&W$öef÷FW2Ò2ç6—¦R‚“° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢2’° ––b†66†Ræ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7E6W'fW%f÷FTFVÆ—fW'’‡6W'fW"Â66†R’’° –6öçF–çVS° —Р––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb66†RææVVG4'&öF67Döâ‡6W'fW"’’° •6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡6W'fW"’À –66†RævWEWV–B‚’Â66†RævWEÆ–W$æÖR‚’Â66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’À –66†RævWEFW‡B‚’ÂfÇ6R“° ––b†66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° ––b‚W'6—7E6W'fW%f÷FTFVÆ—fW'’‡6W'fW"Â66†R’’° –6öçF–çVS° —Р—Р—Р –&ööÆVâFõ6VæBÒG'VS° ––b†vWD6öæf–r‚’ævWEv—Df÷%W6W$öæÆ–æR‚’’° ––b‚—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’’’° —Fõ6VæBÒfÇ6S° —ÒVÇ6R–b†—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’ ’bbvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’’æWVÇ2‡6W'fW"’’° —Fõ6VæBÒfÇ6S° —Р—Р––b‡Fõ6VæB’° –&ööÆVâ'&öF67D†W&RÒ66†RææVVG4'&öF67Döâ‡6W'fW"“° ––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb'&öF67D†W&P ’bbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° –&ööÆVâÆ–W$öæÆ–æRÒ—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’“° •7G&–ærÆ–W%6W'fW"ÒÆ–W$öæÆ–æRòvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær†66†RævWEÆ–W$æÖR‚’ “¢çVÆð  •6WCÅ7G&–æsâF&vWG2Ò&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡Æ–W$öæÆ–æRÀ —Æ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡6W'fW"ÂF&vWG2“° —Р ––b‚6VæEf÷FTVçfVÆ÷T66WFVB‡6W'fW"ÂFVÆ’À •f÷F–æuÇVv–åv—&Rçf÷FR†66†RævWEÆ–W$æÖR‚’Â66†RævWEWV–B‚’À –66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’ÂfÇ6RÂ66†Ræ—5&VÅf÷FR‚’À –66†RævWEFW‡B‚’Â66†RævWEf÷FT–B‚’ÂvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’À –'&öF67D†W&RÂçVÒÂçVÖ&W$öef÷FW2’’’° –FV'Vr‚%&WF–æ–ær66†VBf÷FR&V6W6RF†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²6W'fW"“° –6öçF–çVS° —Р–FVÆ’²³° –çVÒ²³° —&VÖ÷fVBæFB†66†R“° —ÒVÇ6R° –FV'Vr‚$æ÷B6VæF–ærf÷FR&V6W6RW6W"—6âwBöâ6W'fW""²6W'fW"²#¢  ’²66†RçFõ7G&–ær‚’“° —Р—Р–vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fU6W'fW%f÷FW2‡6W'fW"Â&VÖ÷fVB“° —ÒVÇ6R° –FV'Vr‚$æò66†VBf÷FW2f÷"6W'fW#¢"²6W'fW"“° —Р—ÒVÇ6R° –FV'Vr‚$æò66†VBf÷FW2f÷"6W'fW#¢"²6W'fW"“° —Р—Р—ÒVÇ6R° –FV'Vr‚%6W'fW"æ÷BfÆ–C¢"²6W'fW"“° —Р—Р —V&Æ–27–æ6‡&öæ—¦VBfö–B6†V6´öæÆ–æUf÷FW2…7G&–ærÆ–W"Â7G&–ærWV–BÂ7G&–ær6W'fW"’° ––çBFVÆ’Ò° ––b†—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær‡Æ–W"’bbvWEf÷FT66†T†æFÆW"‚’æ†4öæÆ–æUf÷FW2‡WV–B’’° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ2ÒvWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2‡WV–B“° ––b‚2æ—4V×G’‚’’° ––b‡6W'fW"ÓÒçVÆÂ’° —6W'fW"ÒvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær‡Æ–W"“° —Р––b‚vWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° ––çBçVÒÒ° ––çBçVÖ&W$öef÷FW2Ò†–çB’2ç7G&VÒ‚’æf–ÇFW"‡f÷FRÓâf÷FRæ—5&Wv&DFVÆ—fW&VB‚’’æ6÷VçB‚“° –&ööÆVâFVÆ—fW&VE&Wv&BÒfÇ6S° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâ&WF–æVBÒæWr'&”Æ—7CÃâ‚“° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢2’° ––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’’° •6WCÅ7G&–æsâVæF–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ†66†RævWD'&öF67EF&vWG2‚’“° —VæF–æuF&vWG2ç&VÖ÷fTƆ66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b†&Æö6¶VE6W'fW'2ÒçVÆÂ’° —VæF–æuF&vWG2ç&VÖ÷fTƆ&Æö6¶VE6W'fW'2“° —Р–66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƇ6VæE&÷‡”'&öF67B‡VæF–æuF&vWG2À –66†RævWEWV–B‚’Â66†RævWEÆ–W$æÖR‚’Â66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’À –66†RævWEFW‡B‚’ÂfÇ6R’“° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° —Р–&ööÆVâ'&öF67D†W&RÒ66†RææVVG4'&öF67Döâ‡6W'fW"“° ––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb'&öF67D†W&P ’bbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° •7G&–ærÆ–W%6W'fW"Ò‡6W'fW"ÒçVÆÂ’ò6W'fW"¢vWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær‡Æ–W"“°  •6WCÅ7G&–æsâF&vWG2Ò&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡G'VRÂÆ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡6W'fW"ÂF&vWG2“° —Р ––b‚66†Ræ—5&Wv&DFVÆ—fW&VB‚’’° ––b‚6VæEf÷FTVçfVÆ÷T66WFVB‡6W'fW"ÂFVÆ’À •f÷F–æuÇVv–åv—&Rçf÷FTöæÆ–æR†66†RævWEÆ–W$æÖR‚’Â66†RævWEWV–B‚’Â66†RævWE6W'f–6R‚’À –66†RævWEF–ÖR‚’ÂfÇ6RÂ66†Ræ—5&VÅf÷FR‚’Â66†RævWEFW‡B‚’Â66†RævWEf÷FT–B‚’À –vWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’Â'&öF67D†W&RÂçVÒÂçVÖ&W$öef÷FW2’’’° –FV'Vr‚%&WF–æ–æröæÆ–æRf÷FR&V6W6RF†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²6W'fW"“° —&WF–æVBæFB†66†R“° –6öçF–çVS° —Р’òòF†Ræ÷&ÖÂVçfVÆ÷R—2Ç6òfÆ–B'&öF67BFVÆ—fW'’f÷"F†P ’òò7W'&VçBF&vWBâ&V6÷&B—B6ò&Wf–÷W6Ç’VæF–ær7FæFÆöæP ’òò&WG'’6ææ÷Bææ÷Væ6RF†R6ÖRf÷FRv–âÆFW"à ––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb'&öF67D†W&R’° –66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFB‡6W'fW"“° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° —Р–66†Rç6WE&Wv&DFVÆ—fW&VB‡G'VR“° –FVÆ—fW&VE&Wv&BÒG'VS° –FVÆ’²³° –çVÒ²³° —Р ––b†66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’bb66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° —&WF–æVBæFB†66†R“° —Р—Р–vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fTöæÆ–æUf÷FW2‡WV–B“° –f÷"„öffÆ–æT'VævVUf÷FRVæF–ær¢&WF–æVB’° –vWEf÷FT66†T†æFÆW"‚’æFDöæÆ–æUf÷FR‡WV–BÂVæF–ær“° —Р ’òò×VÇF—&÷‡“¢VçfVÆ÷RÖöæÇ ––b†FVÆ—fW&VE&Wv&BbbvWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚ ’bbvWD6öæf–r‚’ævWD×VÇF•&÷‡”öæTvÆö&Å&Wv&B‚’’° –×VÇF•&÷‡”†æFÆW"ç6VæD6ÆV%f÷FR‡WV–BÂÆ–W"“° —Р—Р—Р—Р—Р ’ò¢  ’¢&WG&–W2f÷FW"Ö¶W–VB7FæFÆöæR'&öF67G2v†Vâç’Æ–W"Ö¶W2F&vW@ ’¢&6¶VæBf–Æ&ÆR2ÇVv–âÖÖW76vR6'&–W"à ’  ’¢&Ò6W'fW"&6¶VæB6W'fW"F†Bv–æVB6'&–W  ’¢ð —&÷FV7FVB7–æ6‡&öæ—¦VBfö–B&WG'•VæF–ætöæÆ–æT'&öF67G2…7G&–ær6W'fW"’° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b‡6W'fW"ÓÒçVÆÂdž&Æö6¶VE6W'fW'2ÒçVÆÂbb&Æö6¶VE6W'fW'2æ6öçF–ç2‡6W'fW"’’’° —&WGW&ã° —Р–f÷"…7G&–ær66†VEWV–B¢vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FUUT”G2‚’’° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2†66†VEWV–B’’’° ––b†66†Ræ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R’’° –6öçF–çVS° —Р––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂ66†RææVVG4'&öF67Döâ‡6W'fW"’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡6W'fW"’Â66†RævWEWV–B‚’À –66†RævWEÆ–W$æÖR‚’Â66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’Â66†RævWEFW‡B‚’ÂfÇ6R“° ––b†66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° ––b†66†Ræ—5&Wv&DFVÆ—fW&VB‚’bb66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° –vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fTöæÆ–æUf÷FR†66†VEWV–BÂ66†R“° —ÒVÇ6R° —W'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R“° —Р—Р—Р—Р—Р —&÷FV7FVB7–æ6‡&öæ—¦VBfö–B&WG'•VæF–æuF–ÖT'&öF67G2…7G&–ær6W'fW"’° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b‡6W'fW"ÓÒçVÆÂdž&Æö6¶VE6W'fW'2ÒçVÆÂbb&Æö6¶VE6W'fW'2æ6öçF–ç2‡6W'fW"’’’° —&WGW&ã° —Р––b†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’ÓÒçVÆÂ’° —&WGW&ã° —Р–f÷"…f÷FUF–ÖUVWVRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR’’° –6öçF–çVS° —Р––b‚f÷FRæ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂf÷FRævWEWV–B‚’æ—4V×G’‚’ÇÂf÷FRævWD'&öF67EF&vWG2‚’æ6öçF–ç2‡6W'fW" —ÇÂf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡6W'fW"’Âf÷FRævWEWV–B‚’Âf÷FRævWDæÖR‚’À —f÷FRævWE6W'f–6R‚’Âf÷FRævWEF–ÖR‚’Âf÷FRævWEF÷FÇ2‚’ÂfÇ6R“° ––b‡f÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° —W'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR“° —Р—Р—Р ’ò¢  ’¢W&–öF–6ÆÇ’&WG&–W2WfW'’VæF–ærf÷FW"Ö¶W–VB7FæFÆöæR'&öF67BâF†—2—0 ’¢&WV—&VBf÷"'&ö¶W"G&ç7÷'G2v†÷6R&V6÷fW'’FöW2æ÷B&öGV6RÆ–W"ÖÆöv–à ’¢6'&–W"WfVçBà ’¢ð —V&Æ–27–æ6‡&öæ—¦VBfö–B&WG'•VæF–ætöæÆ–æT'&öF67G2‚’° –f÷"…7G&–ær66†VEWV–B¢æWrÆ–æ¶VD†6…6WCÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FUUT”G2‚’’’° –f÷"„öffÆ–æT'VævVUf÷FR66†R¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2†66†VEWV–B’’’° ––b†66†Ræ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R’’° –6öçF–çVS° —Р––b‚66†Ræ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂ66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâVæF–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ†66†RævWD'&öF67EF&vWG2‚’“° —VæF–æuF&vWG2ç&VÖ÷fTƆ66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b†&Æö6¶VE6W'fW'2ÒçVÆÂ’° —VæF–æuF&vWG2ç&VÖ÷fTƆ&Æö6¶VE6W'fW'2“° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B‡VæF–æuF&vWG2Â66†RævWEWV–B‚’Â66†RævWEÆ–W$æÖR‚’À –66†RævWE6W'f–6R‚’Â66†RævWEF–ÖR‚’Â66†RævWEFW‡B‚’ÂfÇ6R“° ––b†66†RævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –66†Rç6WD'&öF67Df÷'v&FVB†66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’“° ––b†66†Ræ—5&Wv&DFVÆ—fW&VB‚’bb66†Ræ—5&÷‡”'&öF67D6ö×ÆWFR‚’’° –vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fTöæÆ–æUf÷FR†66†VEWV–BÂ66†R“° —ÒVÇ6R° —W'6—7DöæÆ–æUf÷FTFVÆ—fW'’†66†VEWV–BÂ66†R“° —Р—Р—Р—Р—&WG'•VæF–æuF–ÖT'&öF67G2‚“° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–B&WG'•VæF–æuF–ÖT'&öF67G2‚’° ––b†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’ÓÒçVÆÂ’° —&WGW&ã° —Р–f÷"…f÷FUF–ÖUVWVRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’bbW'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR’’° –6öçF–çVS° —Р––b‚f÷FRæ—5&÷‡”'&öF67D†æFÆVB‚’ÇÂf÷FRævWEWV–B‚’æ—4V×G’‚’’° –6öçF–çVS° —Р•6WCÅ7G&–æsâVæF–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ‡f÷FRævWD'&öF67EF&vWG2‚’“° —VæF–æuF&vWG2ç&VÖ÷fTƇf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° ”Æ—7CÅ7G&–æsâ&Æö6¶VE6W'fW'2ÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° ––b†&Æö6¶VE6W'fW'2ÒçVÆÂ’° —VæF–æuF&vWG2ç&VÖ÷fTƆ&Æö6¶VE6W'fW'2“° —Р•6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B‡VæF–æuF&vWG2Âf÷FRævWEWV–B‚’Âf÷FRævWDæÖR‚’Âf÷FRævWE6W'f–6R‚’À —f÷FRævWEF–ÖR‚’Âf÷FRævWEF÷FÇ2‚’ÂfÇ6R“° ––b‡f÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° —W'6—7EF–ÖUf÷FTFVÆ—fW'’‡f÷FR“° —Р—Р—Р —&÷FV7FVB7–æ6‡&öæ—¦VB&ööÆVâW'6—7EF–ÖUf÷FTFVÆ—fW'’…f÷FUF–ÖUVWVRf÷FR’° ––b†vWEf÷FT66†T†æFÆW"‚’çWFFUF–ÖUf÷FR‡f÷FR’’° —f÷FRç6WDFVÆ—fW'•7FFTF—'G’†fÇ6R“° —&WGW&âG'VS° —Р—f÷FRç6WDFVÆ—fW'•7FFTF—'G’‡G'VR“° —66†VGVÆUF–ÖUf÷FTFVÆ—fW'•&WG'’‚“° —&WGW&âfÇ6S° —Р —&—fFRfö–B66†VGVÆUF–ÖUf÷FTFVÆ—fW'•&WG'’‚’° ––b‡F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÇÂvWE66†VGVÆW"‚’ÓÒçVÆÂ’° —&WGW&ã° —Р—F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒG'VS° —G'’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ° —7–æ6‡&öæ—¦VB…f÷F–æuÇVv–å&÷‡’çF†—2’° —F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° —Р—&WG'•VæF–æuF–ÖT'&öF67G2‚“° —ÒÂRÂF–ÖUVæ—Bå4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° —F–ÖUf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° –FV'Vr‚%Væ&ÆRFò66†VGVÆRF–ÖVB'&öF67B7FFR&WG'“¢"²RævWDÖW76vR‚’“° —Р—Р —&÷FV7FVB7–æ6‡&öæ—¦VB&ööÆVâW'6—7E6W'fW%f÷FTFVÆ—fW'’…7G&–ær6W'fW"ÂöffÆ–æT'VævVUf÷FRf÷FR’° ––b†vWEf÷FT66†T†æFÆW"‚’çWFFU6W'fW%f÷FR‡6W'fW"Âf÷FR’’° —f÷FRç6WDFVÆ—fW'•7FFTF—'G’†fÇ6R“° —&WGW&âG'VS° —Р—f÷FRç6WDFVÆ—fW'•7FFTF—'G’‡G'VR“° —66†VGVÆT66†VEf÷FTFVÆ—fW'•&WG'’‚“° —&WGW&âfÇ6S° —Р —&÷FV7FVB7–æ6‡&öæ—¦VB&ööÆVâW'6—7DöæÆ–æUf÷FTFVÆ—fW'’…7G&–ærWV–BÂöffÆ–æT'VævVUf÷FRf÷FR’° ––b†vWEf÷FT66†T†æFÆW"‚’çWFFTöæÆ–æUf÷FR‡WV–BÂf÷FR’’° —f÷FRç6WDFVÆ—fW'•7FFTF—'G’†fÇ6R“° —&WGW&âG'VS° —Р—f÷FRç6WDFVÆ—fW'•7FFTF—'G’‡G'VR“° —66†VGVÆT66†VEf÷FTFVÆ—fW'•&WG'’‚“° —&WGW&âfÇ6S° —Р —&—fFRfö–B66†VGVÆT66†VEf÷FTFVÆ—fW'•&WG'’‚’° ––b†66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÇÂvWE66†VGVÆW"‚’ÓÒçVÆÂ’° —&WGW&ã° —Р–66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒG'VS° —G'’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ° —7–æ6‡&öæ—¦VB…f÷F–æuÇVv–å&÷‡’çF†—2’° –66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° —Р—&WG'”66†VEf÷FTFVÆ—fW'•W'6—7FVæ6R‚“° —ÒÂRÂF–ÖUVæ—Bå4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° –66†VEf÷FTFVÆ—fW'•&WG'•66†VGVÆVBÒfÇ6S° –FV'Vr‚%Væ&ÆRFò66†VGVÆR66†VB'&öF67B7FFR&WG'“¢"²RævWDÖW76vR‚’“° —Р—Р —&—fFR7–æ6‡&öæ—¦VBfö–B&WG'”66†VEf÷FTFVÆ—fW'•W'6—7FVæ6R‚’° –f÷"…7G&–ær6W'fW"¢vWEf÷FT66†T†æFÆW"‚’ævWD66†VEf÷FW56W'fW'2‚’’° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWEf÷FW2‡6W'fW"’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’’° —W'6—7E6W'fW%f÷FTFVÆ—fW'’‡6W'fW"Âf÷FR“° —Р—Р—Р–f÷"…7G&–ærWV–B¢æWrÆ–æ¶VD†6…6WCÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FUUT”G2‚’’’° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢æWr'&”Æ—7CÃâ†vWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2‡WV–B’’’° ––b‡f÷FRæ—4FVÆ—fW'•7FFTF—'G’‚’’° —W'6—7DöæÆ–æUf÷FTFVÆ—fW'’‡WV–BÂf÷FR“° —Р—Р—Р—Р —V&Æ–2fö–B6†V6µf÷FU'G’‚’° ––b†vWD6öæf–r‚’ævWEf÷FU'G”Væ&ÆVB‚’’° ––b‡f÷FU'G•f÷FW2ãÒ7W'&VçEf÷FU'G•f÷FW5&WV—&VB’° –FV'Vr‚%f÷FR'G’&V6†VB"“° –FD7W'&VçEf÷FU'G•f÷FW2‚Ö7W'&VçEf÷FU'G•f÷FW5&WV—&VB“°  –7W'&VçEf÷FU'G•f÷FW5&WV—&VB³ÒvWD6öæf–r‚’ævWEf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚“° —6WEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB€ –vWEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚’²vWD6öæf–r‚’ævWEf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚’“°  ––b‚vWD6öæf–r‚’ævWEf÷FU'G”'&öF67B‚’æ—4V×G’‚’’° –'&öF67B†vWD6öæf–r‚’ævWEf÷FU'G”'&öF67B‚’“° —Р –f÷"…7G&–ær6öÖÖæB¢vWD6öæf–r‚’ævWEf÷FU'G”'VævVT6öÖÖæG2‚’’° —'Vä6öç6öÆT6öÖÖæB†6öÖÖæB“° —Р ––b†vWD6öæf–r‚’ævWEf÷FU'G•6VæEFôÆÅ6W'fW'2‚’’° –f÷"…7G&–ær6W'fW"¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° —6VæEf÷FU'G’‡6W'fW"“° —Р—ÒVÇ6R° –f÷"…7G&–ær6W'fW"¢vWD6öæf–r‚’ævWEf÷FU'G•6W'fW'5Fõ6VæB‚’’° —6VæEf÷FU'G’‡6W'fW"“° —Р—Р—Р—6fUf÷FT66†Tf–ÆR‚“° —Р—Р —V&Æ–2'7G&7Bfö–BFV'Vr…7G&–ær7G"“°  —&—fFRfö–BFV'Vs"…7G&–ærÖW76vR’° –FV'Vr†ÖW76vR“° —Р ’ò¢  ’¢…EE6Æ–VçBW6VBf÷"Öö¦ær’&WVW7G2à ’¢ð —&—fFRf–æ‡GG6Æ–VçB‡GG6Æ–VçBÒ‡GG6Æ–VçBææWt'V–ÆFW"‚’æ6öææV7EF–ÖV÷WB„GW&F–öâæöe6V6öæG2ƒR’’æ'V–ÆB‚“°  ’ò¢  ’¢fWF6†W2Æ–W"w2UT”Bg&öÒF†RÖö¦ær’à ’  ’¢&ÒÆ–W$æÖRÆ–W"æÖP ’¢&WGW&âÆ–W"UT”BÂ÷"´6öFRçVÆÇÒ–bæ÷Bf÷Væ@ ’¢F‡&÷w2”ôW†6WF–öâ–bF†R&WVW7Bf–Ç0 ’¢F‡&÷w2–çFW''WFVDW†6WF–öâ–b–çFW''WFVBv†–ÆRv—F–ærf÷"F†R&W7öç6P ’¢ð —V&Æ–2UT”BfWF6…UT”B…7G&–ærÆ–W$æÖR’F‡&÷w2”ôW†6WF–öâ–çFW''WFVDW†6WF–öâ° ––b‡Æ–W$æÖRÓÒçVÆÂÇÂÆ–W$æÖRæWVÇ4–væ÷&T66R‚&çVÆÂ"’’° —&WGW&âçVÆð —Р ”‡GG&WVW7B&WVW7BÒ‡GG&WVW7BææWt'V–ÆFW"‚ ’çW&’…U$’æ7&VFR‚&‡GG3¢òö’æÖö¦æræ6öÒ÷W6W'2÷&öf–ÆW2öÖ–æV7&gBò"²Æ–W$æÖR’’ätUB‚ ’çF–ÖV÷WB„GW&F–öâæöe6V6öæG2ƒR’’æ'V–ÆB‚“°  ”‡GG&W7öç6SÅ7G&–æsâ&W7öç6RÒ‡GG6Æ–VçBç6VæB‡&WVW7B‡GG&W7öç6Rä&öG”†æFÆW'2æöe7G&–ær‚’“°  ––b‡&W7öç6Rç7FGW46öFR‚’ÓÒCÇÂ&W7öç6Rç7FGW46öFR‚’ÓÒCB’° –Æör‚%F†W&R—2æòÆ–W"v—F‚F†RæÖRÂ""²Æ–W$æÖR²%Â""“° —&WGW&âçVÆð —Р ––b‡&W7öç6Rç7FGW46öFR‚’Â#ÇÂ&W7öç6Rç7FGW46öFR‚’ãÒ3’° —F‡&÷ræWr”ôW†6WF–öâ‚$f–ÆVBFòfWF6‚UT”Bf÷""²Æ–W$æÖR²"Â…EE"²&W7öç6Rç7FGW46öFR‚’“° —Р ”§6öäVÆVÖVçBVÆVÖVçBÒ§6öå'6W"ç'6U7G&–ær‡&W7öç6Ræ&öG’‚’“° ––b†VÆVÖVçBÓÒçVÆÂÇÂVÆVÖVçBæ—4§6öäö&¦V7B‚’’° —&WGW&âçVÆð —Р ”§6öäö&¦V7Bö&¦V7BÒVÆVÖVçBævWD4§6öäö&¦V7B‚“° ––b‚ö&¦V7Bæ†2‚&–B"’ÇÂö&¦V7BævWB‚&–B"’æ—4§6öäçVÆ‚’’° —&WGW&âçVÆð —Р •7G&–ærWV–D57G&–ærÒö&¦V7BævWB‚&–B"’ævWD57G&–ær‚“° —&WGW&â'6UUT”Dg&öÕ7G&–ær‡WV–D57G&–ær“° —Р —V&Æ–2'7G&7B6WCÅ7G&–æsâvWDÆÄf–Æ&ÆU6W'fW'2‚“°  ’ò¢¢6ö×ÆWFRÆFf÷&Ò6W'fW"6WB&Vf÷&Rv†—FVÆ—7Bö&Æö6¶VB&÷WF–ærf–ÇFW'2â¢ð —V&Æ–2'7G&7B6WCÅ7G&–æsâvWDÆÄ6öæf–wW&VE6W'fW'2‚“°  —V&Æ–2'7G&7Bf÷F–æuÇVv–å&÷‡”6öæf–rvWD6öæf–r‚“°  —V&Æ–2'7G&7B7G&–ærvWD7W'&VçEÆ–W%6W'fW"…7G&–ærÆ–W"“°  ’ò¢  ’¢&W6öÇfW2Æ–W"w26W'fW"f÷"f÷FR&÷WF–ærâFVF–6FVBf÷F–ær&÷‡’†2æð ’¢Æö6ÂÆ–W'2Â6ò—BW6W2F†R&6¶VæB&W6Væ6RG&6¶W"–ç7FVBà ’¢ð —&÷FV7FVB7G&–ærvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær…7G&–ærÆ–W"’° ––b†—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’’° —&WGW&â&6¶VæEÆ–W%&W6Væ6UG&6¶W"ævWEÆ–W"‡Æ–W"’æÖ‡&W6Væ6RÓâ&W6Væ6RævWE6W'fW"‚’’æ÷$VÇ6R†çVÆ“° —Р—&WGW&âvWD7W'&VçEÆ–W%6W'fW"‡Æ–W"“° —Р ’ò¢  ’¢FVF–6FVB&÷WF–ær—2–çFVçF–öæÆÇ’Væf–Æ&ÆRöâÇVv–âÖW76v–æs¢F†@ ’¢G&ç7÷'B—2GF6†VBFòÆ–W"Öf6–ær&÷‡’æBFöW2æ÷B6''’&6¶Væ@ ’¢&W6Væ6R6æ6†÷G2à ’¢ð —&÷FV7FVB&ööÆVâ—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’° —&WGW&âvWD6öæf–r‚’ævWDFVF–6FVEf÷F–æu&÷‡’‚’bbÖWF†öBÒçVÆÂbbÖWF†öBç7W÷'G4&6¶VæE&W6Væ6R‚“° —Р —V&Æ–2'7G&7Bf–ÆRvWDFFföÆFW%ÇVv–â‚“°  —V&Æ–27G&–ærvWDÖöçF…F÷FÇ5v—F„FFUF‚‚’° ”Æö6ÄFFUF–ÖR5F–ÖRÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚“° —&WGW&âvWDÖöçF…F÷FÇ5v—F„FFUF‚†5F–ÖR“° —Р —V&Æ–27G&–ærvWDÖöçF…F÷FÇ5v—F„FFUF‚„Æö6ÄFFUF–ÖR5F–ÖR’° —&WGW&â$ÖöçF…F÷FÂÒ"²5F–ÖRævWDÖöçF‚‚’çFõ7G&–ær‚’²"Ò"²5F–ÖRævWE–V"‚“° —Р —V&Æ–2'7G&7B7G&–ærvWE&÷W$æÖR…7G&–ærWV–BÂ7G&–ærÆ–W$æÖR“°  —V&Æ–2'7G&7B7G&–ærvWEUT”B…7G&–ærÆ–W$æÖR“°  —&—fFR–çBvWEfÇVR„'&”Æ—7CÄ6öÇVÖãâ6öÇ2Â7G&–ær6öÇVÖâ–çBFôFB’° –f÷"„6öÇVÖâB¢6öÇ2’° ––b†BævWDæÖR‚’æWVÇ4–væ÷&T66R†6öÇVÖâ’’° ”FFfÇVRfÇVRÒBævWEfÇVR‚“° ––çBçVÒÒ° ––b‡fÇVRÓÒçVÆÂ’° —&WGW&âFôFC° —Р––b‡fÇVRæ—4–çB‚’’° –çVÒÒfÇVRævWD–çB‚“° —ÒVÇ6R–b‡fÇVRæ—57G&–ær‚’’° —G'’° –çVÒÒ–çFVvW"ç'6T–çB‡fÇVRævWE7G&–ær‚’“° —Ò6F6‚„W†6WF–öâR’° ’òò–væ÷&P —Р—Р—&WGW&âçVÒ²FôFC° —Р—Р—&WGW&âFôFC° —Р —&—fFRf÷FUF÷FÇ56æ6†÷BvWE&ö¦V7FVE&öÆÆ÷fW%F÷FÇ2„'&”Æ—7CÄ6öÇVÖãâFFÂ7G&–ærÆ–W"’° ”Æ—7CÅF–ÖUG—SâF–ÖT6†ævW2ÒvWDvÆö&ÄFF†æFÆW"‚’ævWEF–ÖT6†ævW2‚“° –&ööÆVâ&W6WDÖöçF‚ÒF–ÖT6†ævW2æ6öçF–ç2…F–ÖUG—RäÔôåD‚“° –&ööÆVâ&W6WEvVV²ÒF–ÖT6†ævW2æ6öçF–ç2…F–ÖUG—RåtTT²“° –&ööÆVâ&W6WDF’ÒF–ÖT6†ævW2æ6öçF–ç2…F–ÖUG—RäD’“° ––çB66WFVEVWVVEf÷FW2Ò° ––çB66WFVDvÆö&ÅVWVVEf÷FW2Ò° –f÷"…f÷FUF–ÖUVWVRVWVVB¢vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’° ––b‚VWVVBæ—5&ö6W76VB‚’’° –66WFVDvÆö&ÅVWVVEf÷FW2²³° —Р––b‚VWVVBæ—5&ö6W76VB‚’bbVWVVBævWDæÖR‚’ÒçVÆÂbbVWVVBævWDæÖR‚’æWVÇ4–væ÷&T66R‡Æ–W"’’° –66WFVEVWVVEf÷FW2²³° —Р—Р––çBf÷FT–æ7&VÖVçBÒ66WFVEVWVVEf÷FW2²°  ––çBÆÅF–ÖUF÷FÂÒvWEfÇVR†FFÂ$ÆÅF–ÖUF÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBÖöçF…F÷FÂÒ&W6WDÖöçF‚òf÷FT–æ7&VÖVçB¢vWEfÇVR†FFÂ$ÖöçF…F÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBvVV¶Ç•F÷FÂÒ&W6WEvVV²òf÷FT–æ7&VÖVçB¢vWEfÇVR†FFÂ%vVV¶Ç•F÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBF–Ç•F÷FÂÒ&W6WDF’òf÷FT–æ7&VÖVçB¢vWEfÇVR†FFÂ$F–Ç•F÷FÂ"Âf÷FT–æ7&VÖVçB“° ––çBö–çG2ÒvWEfÇVR†FFÂ%ö–çG2"Âf÷FT–æ7&VÖVçB¢vWD6öæf–r‚’ævWEö–çG4öåf÷FR‚’“°  ––çBÖ…f÷FW2ÒvWD6öæf–r‚’ævWDÖ„Ö÷VçDöef÷FW5W$F’‚“° ––b†Ö…f÷FW2â’° ––çBF—2ÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚’ævWDF”ödÖöçF‚‚“° ––b†ÖöçF…F÷FÂâF—2¢Ö…f÷FW2’° –ÖöçF…F÷FÂÒF—2¢Ö…f÷FW3° —Р—Р––b†vWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’âbbö–çG2âvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’’° —ö–çG2ÒvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚“° —Р ––çBFFTÖöçF…F÷FÂÒÓ° ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° ––b†vWD6öæf–r‚’ævWEW6TÖöçF„FFUF÷FÇ45&–Ö'•F÷F‚’’° –FFTÖöçF…F÷FÂÒ&W6WDÖöçF‚òf÷FT–æ7&VÖVç@ “¢vWEfÇVR†FFÂvWDÖöçF…F÷FÇ5v—F„FFUF‚‚’Âf÷FT–æ7&VÖVçB“° —ÒVÇ6R° –FFTÖöçF…F÷FÂÒÖöçF…F÷Fð —Р—Р ––çEµÒ&ö¦V7FVEf÷FU'G’ÒvWE&ö¦V7FVEf÷FU'G•7FFR†66WFVDvÆö&ÅVWVVEf÷FW2²“° —&WGW&âæWrf÷FUF÷FÇ56æ6†÷B†ÆÅF–ÖUF÷FÂÂÖöçF…F÷FÂÂvVV¶Ç•F÷FÂÂF–Ç•F÷FÂÂö–çG2À —&ö¦V7FVEf÷FU'G•³ÒÂ&ö¦V7FVEf÷FU'G•³ÒÂFFTÖöçF…F÷F“° —Р —&÷FV7FVB&ööÆVâ6äf÷'v&E7FæFÆöæT'&öF67B†&ööÆVâÖævW5F÷FÇ2’° —&WGW&âÖævW5F÷FÇ3° —Р —&÷FV7FVB–çEµÒvWE&ö¦V7FVEf÷FU'G•7FFR†–çB66WFVEf÷FW2’° ––çB7W'&VçBÒf÷FU'G•f÷FW3° ––çB&WV—&VBÒ7W'&VçEf÷FU'G•f÷FW5&WV—&VC° ––b‚vWD6öæf–r‚’ævWEf÷FU'G”Væ&ÆVB‚’’° —&WGW&âæWr–çEµÒ²7W'&VçBÂ&WV—&VBÓ° —Р ––çB–æ7&V6RÒvWD6öæf–r‚’ævWEf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚“° –f÷"†–çB’Ò²’Â66WFVEf÷FW3²’²²’° –7W'&VçB²³° ––b†7W'&VçBãÒ&WV—&VB’° –7W'&VçBÓÒ&WV—&VC° —&WV—&VB³Ò–æ7&V6S° —Р—Р—&WGW&âæWr–çEµÒ²7W'&VçBÂ&WV—&VBÓ° —Р —V&Æ–2'7G&7B7G&–ærvWEÇVv–åfW'6–öâ‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†T7W'&VçEf÷FU'G•f÷FW2‚“°  —V&Æ–2'7G&7BÆöærvWEf÷FT66†TÆ7EWFFVB‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†U&WdF’‚“°  —V&Æ–2'7G&7B7G&–ærvWEf÷FT66†U&WdÖöçF‚‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†U&WevVV²‚“°  —V&Æ–2'7G&7B–çBvWEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚“°  —V&Æ–2'7G&7B&ööÆVâ—5Æ–W$öæÆ–æR…7G&–ærÆ–W$æÖR“°  ’ò¢  ’¢6†V6·2öæÆ–æR7FFRf÷"f÷FR&÷WF–ærÂW6–ær&6¶VæB&W6Væ6RöæÇ’v†VâF†—0 ’¢&÷‡’—2W‡Æ–6—FÇ’6öæf–wW&VB2F†RFVF–6FVBf÷F–ær&÷‡’à ’¢ð —&÷FV7FVB&ööÆVâ—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær…7G&–ærÆ–W$æÖR’° —&WGW&â—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’ò&6¶VæEÆ–W%&W6Væ6UG&6¶W"ævWEÆ–W"‡Æ–W$æÖR’æ—5&W6VçB‚ “¢—5Æ–W$öæÆ–æR‡Æ–W$æÖR“° —Р —V&Æ–2'7G&7B&ööÆVâ—56W'fW%fÆ–B…7G&–ær6W'fW"“°  —V&Æ–2'7G&7B&ööÆVâ—56öÖVöæTöæÆ–æU6W'fW"…7G&–ær6W'fW"“°  —&÷FV7FVB&ööÆVâ—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær…7G&–ær6W'fW"’° ––b‚—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’’° —&WGW&â—56öÖVöæTöæÆ–æU6W'fW"‡6W'fW"“° —Р–6öÒæ&Væ6öFW¢çf÷F–æwÇVv–âç&÷‡’ç&W6Væ6Rä&6¶VæE&W6Væ6U7FGW27FGW2Ò&6¶VæEÆ–W%&W6Væ6UG&6¶W  ’ævWD&6¶VæE7FGW2‡6W'fW"“° —&WGW&â7FGW2ÒçVÆÂbb7FGW2æ—4f–Æ&ÆR‚’bb7FGW2ævWEÆ–W$6÷VçB‚’â° —Р —V&Æ–2'7G&7B&ööÆVâ—5f÷FT66†T–væ÷&UF–ÖR‚“°  —V&Æ–2'7G&7B×—7Ä6öæf–rvWEf÷FT66†Tו5Ä6öæf–r‚“°  —V&Æ–2'7G&7B×—7Ä6öæf–rvWDæöåf÷FVD66†Tו5Ä6öæf–r‚“°  —V&Æ–2'7G&7B×—7Ä6öæf–rvWEf÷FTÆövv–ætו5Ä6öæf–r‚“°  ’ò¢  ’¢6‡WFF÷vâו5Â×&VÆFVB&W6÷W&6W26fVÇ’à ’¢ð —V&Æ–2fö–B6‡WFF÷väו7‚’° ––b†vWE&÷‡”×—7ÄÖW76VævW"‚’ÒçVÆÂ’° –vWE&÷‡”×—7ÄÖW76VævW"‚’ç6‡WFF÷vâ‚“° —6WE&÷‡”×—7ÄÖW76VævW"†çVÆ“° —Р ––b†vWE&÷‡”ו5‚’ÒçVÆÂ’° –vWE&÷‡”ו5‚’ç6‡WFF÷vâ‚“° —6WE&÷‡”ו5†çVÆ“° —Р—Р —V&Æ–2fö–BÆöB„•f÷FT66†R§6öå7F÷&vR”æöåf÷FVEÆ–W'57F÷&vRæöåf÷FVD66†T§6öâ’° –ÖWF†öBÒ'VævVTÖWF†öBævWD'”æÖR†vWD6öæf–r‚’ævWD'VævVTÖWF†öB‚’“° ––b†vWDÖWF†öB‚’ÓÒçVÆÂ’° –ÖWF†öBÒ'VævVTÖWF†öBåÅTt”äÔU54t”äs° —Р—v&åVç7W÷'FVDFVF–6FVEf÷F–æu&÷‡”ÖöFR‚“° —WV–EÆ–W$æÖT66†RÒvWE&÷‡”ו5‚’ævWE&÷w5UT”DæÖUVW'’‚“°  –'VævVUF–ÖT6†V6¶W"ç6WEF–ÖT6†ævTf–Å6fT'—72†vWD6öæf–r‚’ævWEF–ÖT6†ævTf–Å6fT'—72‚’“° –'VævVUF–ÖT6†V6¶W"æÆöEF–ÖW"‚“°  —f÷FT66†T†æFÆW"ÒæWrf÷FT66†T†æFÆW"†vWEf÷FT66†Tו5Ä6öæf–r‚’ÂvWD6öæf–r‚’ævWEf÷FT66†UW6Tו5‚’À –vWD6öæf–r‚’ævWEf÷FT66†UW6TÖ–äו5‚’ÂvWE&÷‡”ו5‚’ævWD×—7‚’ÂvWD6öæf–r‚’ævWDFV'Vr‚’À –§6öå7F÷&vR’°  ”÷fW'&–FP —V&Æ–2fö–BÆöt–æfó…7G&–ær×6r’° –Æöt–æfò†×6r“° —Р ”÷fW'&–FP —V&Æ–2fö–BÆöu6WfW&S…7G&–ær×6r’° –Æöu6WfW&R†×6r“° —Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vs„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vs…7G&–ær×6r’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –FV'Vr†×6r“° —Р—Р ”÷fW'&–FP —V&Æ–2fö–BFV'Vs…F‡&÷v&ÆRR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р—Ó° —f÷FT66†T†æFÆW"æÆöB‚“°  –æöåf÷FVEÆ–W'466†RÒæWræöåf÷FVEÆ–W'466†R†vWDæöåf÷FVD66†Tו5Ä6öæf–r‚’À –vWD6öæf–r‚’ævWDæöåf÷FVD66†UW6Tו5‚’ÂvWD6öæf–r‚’ævWDæöåf÷FVD66†UW6TÖ–äו5‚’À –vWE&÷‡”ו5‚’ævWD×—7‚’Âæöåf÷FVD66†T§6öâÂvWD6öæf–r‚’ævWDFV'Vr‚’’°  ”÷fW'&–FP —V&Íx÷Þm¢G§²ÚîÆ­yÔ”B†Ö÷7E6–t&—G2ÂÆV7E6–t&—G2“° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–B&ö6W75VWVR‚’° —v†–ÆR†vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’ç6—¦R‚’â’° •f÷FUF–ÖUVWVRf÷FRÒvWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’æVÆVÖVçB‚“° ––b‚f÷FRæ—5&ö6W76VB‚’’° •f÷FUF÷FÇ56æ6†÷BVWVVEF÷FÇ2Òf÷FRævWEF÷FÇ2‚’ÓÒçVÆÂÇÂf÷FRævWEF÷FÇ2‚’æ—4V×G’‚’òçVÆÀ “¢f÷FUF÷FÇ56æ6†÷Bç'6U7F÷&vR‡f÷FRævWEF÷FÇ2‚’“° •VWVVEf÷FU&W7VÇB&W7VÇBÒf÷FR‡f÷FRævWDæÖR‚’Âf÷FRævWE6W'f–6R‚’ÂG'VRÂfÇ6RÂf÷FRævWEF–ÖR‚’ÂVWVVEF÷FÇ2À —f÷FRævWEWV–B‚’Âf÷FR“° ––b‡&W7VÇBÓÒVWVVEf÷FU&W7VÇBå$UE%’’° —66†VGVÆUF–ÖUf÷FU&WG'’‚“° —&WGW&ã° —Р––b‡&W7VÇBÓÒVWVVEf÷FU&W7VÇBåDU$Ô”äÂ’° —v&â‚%&VÖ÷f–ærFW&Ö–æÂ&öÆÆ÷fW"f÷FR"²f÷FRævWEf÷FT–B‚’²"f÷""²f÷FRævWDæÖR‚’²"ò  ’²6W'f–6U6—FUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡f÷FRævWE6W'f–6R‚’’“° —Р—Р––b‚vWEf÷FT66†T†æFÆW"‚’ç&VÖ÷fUF–ÖUf÷FR‡f÷FR’’° —66†VGVÆUF–ÖUf÷FU&WG'’‚“° —&WGW&ã° —Р—Р—Р —&—fFRfö–B66†VGVÆUF–ÖUf÷FU&WG'’‚’° ––b‡F–ÖUf÷FU&WG'•66†VGVÆVBÇÂvWE66†VGVÆW"‚’ÓÒçVÆÂ’° —&WGW&ã° —Р—F–ÖUf÷FU&WG'•66†VGVÆVBÒG'VS° —G'’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ° —7–æ6‡&öæ—¦VB…f÷F–æuÇVv–å&÷‡’çF†—2’° —F–ÖUf÷FU&WG'•66†VGVÆVBÒfÇ6S° —Р—&ö6W75VWVR‚“° —ÒÂRÂF–ÖUVæ—Bå4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° —F–ÖUf÷FU&WG'•66†VGVÆVBÒfÇ6S° –FV'Vr‚%Væ&ÆRFò66†VGVÆR&öÆÆ÷fW"f÷FR&WG'“¢"²RævWDÖW76vR‚’“° —Р—Р —V&Æ–2fö–B&VÆöB‚’° —&VÆöE'VçF–ÖR‡G'VR“° —Р ’ò¢¢Æ–W26öçG&öÂÖ÷&–v–æFVB6öæf–wW&F–öâ&VÆöBv—F†÷WB7F÷–ær—G26öææV7F÷"÷"†÷7FVB6W'f–6Râ¢ð —V&Æ–2fö–B&VÆöDg&öÔ6öçG&ö‚’° —&VÆöE'VçF–ÖR†fÇ6R“° —Р —&—fFRfö–B&VÆöE'VçF–ÖR†&ööÆVâ&W7F'D6öçG&öÅ6W'f–6W2’° –ÖWF†öBÒ'VævVTÖWF†öBævWD'”æÖR†vWD6öæf–r‚’ævWD'VævVTÖWF†öB‚’“° ––b†vWDÖWF†öB‚’ÓÒçVÆÂ’° –ÖWF†öBÒ'VævVTÖWF†öBåÅTt”äÔU54t”äs° —Р—v&åVç7W÷'FVDFVF–6FVEf÷F–æu&÷‡”ÖöFR‚“° ––b‚&W7F'D6öçG&öÅ6W'f–6W2bbÖWF†öBÓÒ'VævVTÖWF†öBå4ô4´UE2’° —&V'V–ÆE6ö6¶WD6Æ–VçG2‚“° —Р —6WD7W'&VçEf÷FU'G•f÷FW5&WV—&VB€ –vWD6öæf–r‚’ævWEf÷FU'G•f÷FW5&WV—&VB‚’²vWEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB‚’“° ––b‡&W7F'D6öçG&öÅ6W'f–6W2’° –ÆöD×VÇF•&÷‡•7W÷'B‚“° —&W7F'D6öçG&öÅ6W'f–6W47–æ2‚“° —Р—Р —&—fFR7–æ6‡&öæ—¦VBfö–B&V'V–ÆE6ö6¶WD6Æ–VçG2‚’° ”†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â&V'V–ÇBÒæWr†6„ÖÃâ‚“° —G'’° ”Æ—7CÅ7G&–æsâ&Æö6¶VBÒvWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚“° –f÷"…7G&–ær6W'fW"¢vWD6öæf–r‚’ævWE7–v÷E6W'fW'2‚’’° ––b†&Æö6¶VBæ6öçF–ç2‡6W'fW"’’6öçF–çVS° ”ÖÅ7G&–ærÂö&¦V7CâFFÒvWD6öæf–r‚’ævWE7–v÷E6W'fW$6öæf–wW&F–öâ‡6W'fW"“° •7G&–ær†÷7BÒFFæ6öçF–ç4¶W’‚$†÷7B"’ò…7G&–ær’FFævWB‚$†÷7B"’¢"#° ––çB÷'BÒFFæ6öçF–ç4¶W’‚%÷'B"’ò†–çB’FFævWB‚%÷'B"’¢#“ƒ° —&V'V–ÇBçWB‡6W'fW"ÂæWr6Æ–VçD†æFÆW"††÷7BÂ÷'BÂVæ7'—F–öä†æFÆW"ÂvWD6öæf–r‚’ævWDFV'Vr‚’’“° —Р—Ò6F6‚…'VçF–ÖTW†6WF–öâf–ÇW&R’° —7F÷6ö6¶WD6Æ–VçG2‡&V'V–ÇB“° —F‡&÷rf–ÇW&S° —Р”†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â&Wf–÷W2Ò6Æ–VçD†æFÆW3° –6Æ–VçD†æFÆW2Ò&V'V–ÇC° —7F÷6ö6¶WD6Æ–VçG2‡&Wf–÷W2“° —Р —&—fFR7–æ6‡&öæ—¦VB&ööÆVâ6VæE6ö6¶WDVçfVÆ÷R…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ”6Æ–VçD†æFÆW"6ö6¶WD6Æ–VçBÒ6Æ–VçD†æFÆW2ÓÒçVÆÂòçVÆ¢6Æ–VçD†æFÆW2ævWB‡6W'fW"“° ––b‡6ö6¶WD6Æ–VçBÓÒçVÆÂ’&WGW&âfÇ6S° —G'’° —6ö6¶WD6Æ–VçBç6VæDVçfVÆ÷R†VçfVÆ÷R“° —&WGW&âG'VS° —Ò6F6‚…'VçF–ÖTW†6WF–öâR’° –FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р—Р —&—fFR7–æ6‡&öæ—¦VB&ööÆVâ6VæD‡GGVçfVÆ÷R…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ”‡GG&÷‡•G&ç7÷'E6W'fW"G&ç7÷'BÒ‡GGG&ç7÷'E6W'fW#° —&WGW&âG&ç7÷'BÒçVÆÂbbG&ç7÷'Bç6VæB‡6W'fW"ÂVçfVÆ÷R“° —Р —&—fFRfö–B7F'D‡GGG&ç7÷'B‚’° —G'’° •U$’VæGö–çBÒU$’æ7&VFR†vWD6öæf–r‚’ævWD‡GGV&Æ–4VæGö–çB‚’“° ––b‚&‡GG2"æWVÇ4–væ÷&T66R†VæGö–çBævWE66†VÖR‚’’ÇÂVæGö–çBævWD†÷7B‚’ÓÒçVÆÀ —ÇÂVæGö–çBævWE÷'B‚’ÓÒÇÂVæGö–çBævWE÷'B‚’âcSS3P —ÇÂVæGö–çBævWEW6W$–æfò‚’ÒçVÆÂÇÂVæGö–çBævWEVW'’‚’ÒçVÆÂÇÂVæGö–çBævWDg&vÖVçB‚’ÒçVÆÀ —džVæGö–çBævWEF‚‚’ÒçVÆÂbbVæGö–çBævWEF‚‚’æ—4V×G’‚’bb"ò"æWVÇ2†VæGö–çBævWEF‚‚’’’’° —F‡&÷ræWr–ÆÆVvÄ&wVÖVçDW†6WF–öâ‚$…EEåV&Æ–4VæGö–çB×W7B&Râ…EE2÷&–v–â"“° —Р”f–ÆRF—&V7F÷'’ÒæWrf–ÆR†vWDFFföÆFW%ÇVv–â‚’Â&‡GG"“° ”‡GGFÇ4–FVçF—G’–FVçF—G’Ò‡GGFÇ4–FVçF—G’æÆöD÷$7&VFR†F—&V7F÷'’çFõF‚‚’ÂVæGö–çBævWD†÷7B‚’“° –‡GGVç&öÆÆÖVçDWF†÷&—G’ÒæWr‡GGVç&öÆÆÖVçDWF†÷&—G’†–FVçF—G’ÂF—&V7F÷'’çFõF‚‚’“° –‡GGG&ç7÷'E6W'fW"ÒæWr‡GG&÷‡•G&ç7÷'E6W'fW"€ –æWr–æWE6ö6¶WDFG&W72†vWD6öæf–r‚’ævWD‡GG†÷7B‚’ÂvWD6öæf–r‚’ævWD‡GG÷'B‚’’–FVçF—G’À –‡GGVç&öÆÆÖVçDWF†÷&—G’ÂF—&V7F÷'’çFõF‚‚’ç&W6öÇfR‚&÷WFvö–ær×c"’ÂF†—3£¦†æFÆT‡GGG&ç7÷'DVçfVÆ÷R“° –‡GGG&ç7÷'E6W'fW"ç7F'B‚“° –Æöt–æfò‚$…EEG&ç7÷'BÆ—7FVæ–ær6V7W&VÇ’öâ"²vWD6öæf–r‚’ævWD‡GG†÷7B‚’²#¢  ’²‡GGG&ç7÷'E6W'fW"ç÷'B‚’²#²W6R÷f÷F–æwÇVv–æ'VævVR‡GG6öFRÇ6W'fW#âf÷"V6‚&6¶VæB"“° —Ò6F6‚„W†6WF–öâf–ÇW&R’° –6Æ÷6T‡GGG&ç7÷'B‚“° —F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚$…EEG&ç7÷'B6÷VÆBæ÷B7F'B6V7W&VÇ’"Âf–ÇW&R“° —Р—Р ’ò¢¢¶VW2F†RWF†VçF–6FVBÕDÅ2&6¶VæB–FVçF—G’GF6†VBFò6V7W&—G’×6Vç6—F—fR&÷‡’&÷WF–ærâ¢ð —&÷FV7FVBfö–B†æFÆT‡GGG&ç7÷'DVçfVÆ÷R„‡GG&÷‡•G&ç7÷'E6W'fW"å&V6V—fVDVçfVÆ÷R&V6V—fVB’° ––b‚—4WF†VçF–6FVD‡GGVçfVÆ÷TÆÆ÷vVB‡&V6V—fVB’’° –FV'Vr‚$–væ÷&VB…EEVçfVÆ÷Rv†÷6RÆ–W"×&W6Væ6R6Æ–ÒF–Bæ÷BÖF6‚—G2WF†VçF–6FVB&6¶VæB"“° —&WGW&ã° —Р”vÆö&ÄÖW76vU&÷‡”†æFÆW"†æFÆW"ÒvÆö&ÄÖW76vU&÷‡”†æFÆW#° ––b††æFÆW"ÓÒçVÆÂ’F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚$…EEÖW76vR&÷WFW"—2æ÷B&VG’"“° –†æFÆW"æöäÖW76vR‡&V6V—fVBæVçfVÆ÷R‚’“° —Р —&—fFR&ööÆVâ—4WF†VçF–6FVD‡GGVçfVÆ÷TÆÆ÷vVB„‡GG&÷‡•G&ç7÷'E6W'fW"å&V6V—fVDVçfVÆ÷R&V6V—fVB’° ––b‡&V6V—fVBÓÒçVÆÂÇÂ&V6V—fVBæVçfVÆ÷R‚’ÓÒçVÆÂÇÂ&V6V—fVBç6W'fW$–B‚’ÓÒçVÆÂ’&WGW&âfÇ6S° •7G&–ær7F×VE6W'fW"Ò&V6V—fVBæVçfVÆ÷R‚’ævWDf–VÆG2‚’ævWD÷$FVfVÇB…f÷F–æuÇVv–åv—&Räµõ4U%dU"Â""“° ––b‚&V6V—fVBç6W'fW$–B‚’æWVÇ4–væ÷&T66R‡7F×VE6W'fW"’’&WGW&âfÇ6S° ––b‚f÷F–æuÇVv–åv—&Rå5T%ôÄôt”âæWVÇ2‡&V6V—fVBæVçfVÆ÷R‚’ævWE7V$6†ææV‚’’’&WGW&âG'VS° •f÷F–æuÇVv–åv—&RåÆ–W%&W6Væ6TWfVçBWfVçBÒf÷F–æuÇVv–åv—&Rç&VEÆ–W%&W6Væ6TWfVçB‡&V6V—fVBæVçfVÆ÷R‚’“° –&ööÆVâÖöFW&âÒWfVçBæ6öææV7F–öä–BÒçVÆÂÇÂWfVçBæ&6¶VæD–æ6&æF–öä–BÒçVÆÀ —ÇÂWfVçBæ&6¶VæE7F'FVDBÒÂÇÂWfVçBç&W6Væ6UF–ÖW7F×Òð ––b‚ÖöFW&âÇ—4FVF–6FVEf÷F–æu&÷‡”Væ&ÆVB‚’’&WGW&âG'VS° ’òòÆ–W"Öf6–ær&÷‡’†27G&öævW"WF†÷&—G’F†âç’&6¶VæC¢—G2Æ—fP ’òòÆ–W"6öææV7F–öâ7WÆ–W2&÷F‚F†R7W'&VçB&÷WFRæB†–âöæÆ–æRÖöFR’UT”Bà —&WGW&â—4ÆVv7”Æöv–äFW7F–æF–öäWF†÷&—FF—fR†WfVçBçÆ–W"ÂWfVçBçWV–BÂ&V6V—fVBç6W'fW$–B‚’“° —Р —&—fFR7–æ6‡&öæ—¦VBfö–B6Æ÷6T‡GGG&ç7÷'B‚’° ”‡GG&÷‡•G&ç7÷'E6W'fW"G&ç7÷'BÒ‡GGG&ç7÷'E6W'fW#° –‡GGG&ç7÷'E6W'fW"ÒçVÆð –‡GGVç&öÆÆÖVçDWF†÷&—G’ÒçVÆð ––b‡G&ç7÷'BÒçVÆÂ’G&ç7÷'Bæ6Æ÷6R‚“° —Р —V&Æ–27G&–ær7&VFT‡GG6öææV7F–öä6öFR…7G&–ær6W'fW$–B’° ”‡GGVç&öÆÆÖVçDWF†÷&—G’WF†÷&—G’Ò‡GGVç&öÆÆÖVçDWF†÷&—G“° ––b†ÖWF†öBÒ'VævVTÖWF†öBä…EEÇÂWF†÷&—G’ÓÒçVÆÂ’° —F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚%F†R…EEG&ç7÷'B—2æ÷B'Vææ–ær"“° —Р—&WGW&âWF†÷&—G’æ7&VFT6öææV7F–öä6öFR‡6W'fW$–BÂU$’æ7&VFR†vWD6öæf–r‚’ævWD‡GGV&Æ–4VæGö–çB‚’’ÂGW&F–öâæödÖ–çWFW2ƒR’ ’æVæ6öFR‚“° —Р —V&Æ–2fö–B&Wfö¶T‡GG&6¶VæB…7G&–ær6W'fW$–B’° ”‡GGVç&öÆÆÖVçDWF†÷&—G’WF†÷&—G’Ò‡GGVç&öÆÆÖVçDWF†÷&—G“° ––b†ÖWF†öBÒ'VævVTÖWF†öBä…EEÇÂWF†÷&—G’ÓÒçVÆÂ’F‡&÷ræWr–ÆÆVvÅ7FFTW†6WF–öâ‚%F†R…EEG&ç7÷'B—2æ÷B'Vææ–ær"“° –WF†÷&—G’ç&Wfö¶R„‡GGFÇ4–FVçF—G’æ6æöæ–6Å6W'fW$–B‡6W'fW$–B’“° —Р —&—fFR7–æ6‡&öæ—¦VBfö–B6Æ÷6U6ö6¶WD6Æ–VçG2‚’° ”†6„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â6Æ–VçG2Ò6Æ–VçD†æFÆW3° –6Æ–VçD†æFÆW2ÒçVÆð —7F÷6ö6¶WD6Æ–VçG2†6Æ–VçG2“° —Р —7FF–2fö–B7F÷6ö6¶WD6Æ–VçG2„ÖÅ7G&–ærÂ6Æ–VçD†æFÆW#â6Æ–VçG2’° ––b†6Æ–VçG2ÓÒçVÆÂ’&WGW&ã° –f÷"„6Æ–VçD†æFÆW"6Æ–VçB¢6Æ–VçG2çfÇVW2‚’’° ––b†6Æ–VçBÓÒçVÆÂ’6öçF–çVS° —G'’° –6Æ–VçBç7F÷6öææV7F–öâ‚“° —Ò6F6‚…'VçF–ÖTW†6WF–öâ–væ÷&VB’° ’òò&W7BVff÷'C¢öæR'&ö¶Vâ6Æ–VçB×W7Bæ÷B&WfVçBF†R&VÖ–æ–ær6ö6¶WG2g&öÒ6Æ÷6–ærࠗР—Р—Р —&—fFRfö–Bv&åVç7W÷'FVDFVF–6FVEf÷F–æu&÷‡”ÖöFR‚’° ––b†vWD6öæf–r‚’ævWDFVF–6FVEf÷F–æu&÷‡’‚’bb†ÖWF†öBÓÒçVÆÂÇÂÖWF†öBç7W÷'G4&6¶VæE&W6Væ6R‚’’’° –Æöu6WfW&R‚$FVF–6FVEf÷F–æu&÷‡’&WV—&W2Õ•5ÂÂ$TD•2ÂÕEBÂ4ô4´UE2Â÷"…EE²ÅTt”äÔU54t”är—2F—6&ÆVBf÷"  ’²&FVF–6FVB×&÷‡’&÷WF–ærâfÆÆ–ær&6²Fòæ÷&ÖÂ&÷‡’&÷WF–ærâ"“° —Р—Р —V&Æ–2'7G&7Bfö–B'Vä7–æ2…'Vææ&ÆR'Vâ“°  ’ò¢¢ÆFf÷&ÒæÖRW6VBöæÇ’f÷"F†RG&ç7÷'BÖæWWG&Â6öçG&öÂF—66÷fW'’6öçG&7Bâ¢ð —V&Æ–2'7G&7B7G&–ærvWE&÷‡•ÆFf÷&Ò‚“°  —V&Æ–2'7G&7Bfö–B'Vä6öç6öÆT6öÖÖæB…7G&–ær6öÖÖæB“°  —V&Æ–2'7G&7Bfö–B6fUf÷FT66†Tf–ÆR‚“°  —V&Æ–2'7G&7Bfö–B&VÆöD6÷&R†&ööÆVâ×—7“°  ’ò¢¢7G&–7B6öçG&öÂ&VÆöBFƒ²f–ÇW&W2&÷vFR6òF†R6ÆÆW"6â&W7F÷&R—G2&6·Wâ¢ð —V&Æ–2'7G&7Bfö–B&VÆöD6öçG&öÄ6öæf–wW&F–öâ‚’F‡&÷w2W†6WF–öã°  —V&Æ–2'7G&7B&ööÆVâ6VæEÇVv–äÖW76vTFF…7G&–ær6W'fW"Â7G&–ær6†ææVÂÂ'—FUµÒFFÂ&ööÆVâVWVR“°  —&—fFR7FF–2f–æ–çBÅTt”åôÔU54tUô„$EôĔԕBÒ3#scs° —&—fFR7FF–2f–æ–çBÅTt”åôÔU54tUõ4ôeEôĔԕBÒ3°  —V&Æ–2fö–B6VæEÇVv–äÖW76vU6W'fW"…7G&–ær6W'fW"–çBFVĤ6öäVçfVÆ÷RVçfVÆ÷R’° –vWE66†VGVÆW"‚’ç66†VGVÆR‚‚’Óâ6VæEÇVv–äÖW76vU6W'fW$æ÷r‡6W'fW"ÂVçfVÆ÷R’ÂFVÆ’¢TÂÂF–ÖUVæ—BäÔ”ÄÄ•4T4ôäE2“° —Р ’ò¢  ’¢6VæG2ÇVv–âÖÖW76vRVçfVÆ÷R–ÖÖVF–FVÇ’æB&W÷'G2v†WF†W"F†R&÷‡ ’¢66WFVB—Bf÷"FVÆ—fW'’à ’  ’¢&Ò6W'fW"F&vWB&6¶VæB6W'fW  ’¢&ÒVçfVÆ÷RVçfVÆ÷RFò6Væ@ ’¢&WGW&âG'VRv†VâF†R&÷‡’66WFVBF†RÖW76vRf÷"FVÆ—fW' ’¢ð —&÷FV7FVB&ööÆVâ6VæEÇVv–äÖW76vU6W'fW$æ÷r…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° –f–æÂ7G&–ær7V$6†ææVÂÒVçfVÆ÷RævWE7V$6†ææV‚“° –f–æÂ7G&–ær–ÆöBÒ§6öäVçfVÆ÷T6öFV2æVæ6öFR†VçfVÆ÷R“°  –f–æÂ'—FUµÒ7V$6†ææVÄ'—FW2Ò7V$6†ææVÂævWD'—FW2†¦fææ–òæ6†'6WBå7FæF&D6†'6WG2åUDeó‚“° –f–æÂ'—FUµÒ–ÆöD'—FW2Ò–ÆöBævWD'—FW2†¦fææ–òæ6†'6WBå7FæF&D6†'6WG2åUDeó‚“°  ’òòW7F–ÖFR'—FW2w&—GFVã  ’òòÒw&—FUUDbFG2"Ö'—FRÆVæwF‚&Vf—‚²UDbÓ‚'—FW0 ’òòÒw&—FT–çB—2B'—FW0 ––çBW7F–ÖFVE6—¦RÒ"²7V$6†ææVÄ'—FW2æÆVæwF‚²òò7V$6†ææVÂUDb†ÆVâ&Vf—‚²'—FW2 “B²òò–ÆöBÆVæwF‚–ç@ “"²–ÆöD'—FW2æÆVæwFƒ²òò–ÆöBUDb†ÆVâ&Vf—‚²'—FW2  ––b†W7F–ÖFVE6—¦RâÅTt”åôÔU54tUõ4ôeEôĔԕB’° –FV'Vr‚%µÇVv–äÖW76vUÒ–ÆöBæV&–ærÆ–Ö—B‚"²W7F–ÖFVE6—¦R²"'—FW2’6W'fW#Ò"²6W'fW  ’²"7V$6†ææVÃÒ"²7V$6†ææV²"( B6öç6–FW"&VF—2–ç7FVB"“° —Р ––b†W7F–ÖFVE6—¦RâÅTt”åôÔU54tUô„$EôĔԕB’° –FV'Vr‚%µÇVv–äÖW76vUÒ–ÆöBDôòÄ$tR‚"²W7F–ÖFVE6—¦R²"'—FW2ÂÖƒÒ"²ÅTt”åôÔU54tUô„$EôĔԕ@ ’²"’6W'fW#Ò"²6W'fW"²"7V$6†ææVÃÒ"²7V$6†ææV²"( BäõB6VçB"“° —&WGW&âfÇ6S° —Р —G'’„'—FT'&”÷WGWE7G&VÒ'—FT÷WE7G&VÒÒæWr'—FT'&”÷WGWE7G&VÒ‚“° ”FF÷WGWE7G&VÒ÷WBÒæWrFF÷WGWE7G&VÒ†'—FT÷WE7G&VÒ’’° ––b†vWD6öæf–r‚’ævWEÇVv–äÖW76vTVæ7'—F–öâ‚’bbVæ7'—F–öä†æFÆW"ÒçVÆÂ’° –÷WBçw&—FUUDb†Væ7'—F–öä†æFÆW"æVæ7'—B‡7V$6†ææVÂ’“° —ÒVÇ6R° –÷WBçw&—FUUDb‡7V$6†ææV“° —Р ’òò6æ—G’öæÇ“¢ÕU5B&R'—FW2Âæ÷B6†'0 –÷WBçw&—FT–çB‡–ÆöD'—FW2æÆVæwF‚“°  ––b†vWD6öæf–r‚’ævWEÇVv–äÖW76vTVæ7'—F–öâ‚’bbVæ7'—F–öä†æFÆW"ÒçVÆÂ’° –÷WBçw&—FUUDb†Væ7'—F–öä†æFÆW"æVæ7'—B‡–ÆöB’“° —ÒVÇ6R° –÷WBçw&—FUUDb‡–ÆöB“° —Р–÷WBæfÇW6‚‚“°  –&ööÆVâ6VçBÒ6VæEÇVv–äÖW76vTFF‡6W'fW"ÂvWD6öæf–r‚’ævWEÇVv–äÖW76vT6†ææV‚’çFôÆ÷vW$66R‚’À –'—FT÷WE7G&VÒçFô'—FT'&’‚’ÂfÇ6R“° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –FV'Vr‚‡6VçBò%6VçB"¢$6÷VÆBæ÷B6VæB"’²"ÇVv–âVçfVÆ÷R‚"²W7F–ÖFVE6—¦R²"'—FW2’"²6W'fW  ’²""²7V$6†ææV²""²VçfVÆ÷RævWDf–VÆG2‚’“° —Р—&WGW&â6VçC° —Ò6F6‚„W†6WF–öâR’° –Rç&–çE7F6µG&6R‚“° —&WGW&âfÇ6S° —Р—Р —7FF–2FVfVÇD¦VF—46Æ–VçD6öæf–r'V–ÆE&VF—46Æ–VçD6öæf–r…f÷F–æuÇVv–å&÷‡”6öæf–r6öæf–u6÷W&6R’° ”FVfVÇD¦VF—46Æ–VçD6öæf–rä'V–ÆFW"6öæf–rÒFVfVÇD¦VF—46Æ–VçD6öæf–ræ'V–ÆFW"‚ ’æFF&6R†6öæf–u6÷W&6RævWE&VF—4F$–æFW‚‚’’ç76†6öæf–u6÷W&6RævWE&VF—576‚’’æ6öææV7F–öåF–ÖV÷WDÖ–ÆÆ—2ƒ# ’ç6ö6¶WEF–ÖV÷WDÖ–ÆÆ—2ƒ#“° ––b†6öæf–u6÷W&6RævWE&VF—576‚’’° •54Å&ÖWFW'276Å&ÖWFW'2ÒæWr54Å&ÖWFW'2‚“° —76Å&ÖWFW'2ç6WDVæGö–çD–FVçF–f–6F–öäÆv÷&—F†Ò‚$…EE2"“° –6öæf–rç76Å&ÖWFW'2‡76Å&ÖWFW'2“° —Р––b†6öæf–u6÷W&6RævWE&VF—5W6W&æÖR‚’ÒçVÆÂbb6öæf–u6÷W&6RævWE&VF—5W6W&æÖR‚’æ—4V×G’‚’’° –6öæf–rçW6W"†6öæf–u6÷W&6RævWE&VF—5W6W&æÖR‚’“° —Р––b†6öæf–u6÷W&6RævWE&VF—577v÷&B‚’ÒçVÆÂbb6öæf–u6÷W&6RævWE&VF—577v÷&B‚’æ—4V×G’‚’’° –6öæf–rç77v÷&B†6öæf–u6÷W&6RævWE&VF—577v÷&B‚’“° —Р—&WGW&â6öæf–ræ'V–ÆB‚“° —Р —V&Æ–2&ööÆVâ6VæE&VF—4VçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° —&WGW&â6VæE&VF—4VçfVÆ÷U6W'fW"‡6W'fW"ÂVçfVÆ÷RÂfÇ6R“° —Р —&—fFR&ööÆVâ6VæE&VF—4VçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷RÂ&ööÆVâW6U&WG'”6ööÆF÷vâ’° ”¦VF—5ööÂV&Æ—6†W%ööÂÒ&VF—5V&Æ—6†W%ööð ––b‡V&Æ—6†W%ööÂÓÒçVÆÂLJW6U&WG'”6ööÆF÷vâbb7—7FVÒæ7W'&VçEF–ÖTÖ–ÆÆ—2‚’Â&VF—5V&Æ—6†W%&WG'”gFW"’’° —&WGW&âfÇ6S° —Р —G'’„¦VF—2¦VF—2ÒV&Æ—6†W%ööÂævWE&W6÷W&6R‚’’° •7G&–ær6†ææVÂÒvWD6öæf–r‚’ævWE&VF—5&Vf—‚‚’²%f÷F–æuÇVv–åò"²6W'fW#° –Æöær7V'67&–&W'2Ò¦VF—2çV&Æ—6‚†6†ææVÂÀ ”§6öäVçfVÆ÷T6öFV2æVæ6öFR…f÷F–æuÇVv–åv—&Rçv—F…&VF—4FVÆ—fW'”–B†VçfVÆ÷R’’“° —&VF—5V&Æ—6†W%&WG'”gFW"Òð —&WGW&â7V'67&–&W'2â° —Ò6F6‚„W†6WF–öâR’° ––b‡W6U&WG'”6ööÆF÷vâ’° ’òò7FæFÆöæR'&öF67G2&VÖ–âVWVVBÂ6òF†V—"&WG&–W26â&RF‡&÷GFÆVB6fVÇ’à —&VF—5V&Æ—6†W%&WG'”gFW"Ò7—7FVÒæ7W'&VçEF–ÖTÖ–ÆÆ—2‚’²#ð —Р–FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р—Р —V&Æ–2&ööÆVâ6VæD×GDVçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ––b†×GD†æFÆW"ÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р—G'’° –×GD†æFÆW"çV&Æ—6„VçfVÆ÷R†vWD6öæf–r‚’ævWD×GE&Vf—‚‚’²'f÷F–æwÇVv–â÷6W'fW'2ò"²6W'fW"ÂVçfVÆ÷R“° —&WGW&âG'VS° —Ò6F6‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—&WGW&âfÇ6S° —Р—Р —V&Æ–2&ööÆVâ6VæE6ö6¶WDVçfVÆ÷U6W'fW"…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° ”ÖÅ7G&–ærÂö&¦V7Câ6öæf–wW&F–öâÒvWD6öæf–r‚’ævWE7–v÷E6W'fW$6öæf–wW&F–öâ‡6W'fW"“° ––b†6öæf–wW&F–öâÓÒçVÆÂ’° —&WGW&âfÇ6S° —Р•7G&–ær†÷7BÒ6öæf–wW&F–öâævWB‚$†÷7B"’–ç7Fæ6Vöb7G&–ærò…7G&–ær’6öæf–wW&F–öâævWB‚$†÷7B"’¢"#° ––çB÷'BÒ6öæf–wW&F–öâævWB‚%÷'B"’–ç7Fæ6VöbçVÖ&W"ò‚„çVÖ&W"’6öæf–wW&F–öâævWB‚%÷'B"’’æ–çEfÇVR‚’¢#“ƒ° ––b††÷7Bæ—4V×G’‚’’° —&WGW&âfÇ6S° —Р •7G&–ær–ÆöBÒ§6öäVçfVÆ÷T6öFV2æVæ6öFR†VçfVÆ÷R“° •7G&–ærVæ6öFVBÒVæ7'—F–öä†æFÆW"ÒçVÆÂòVæ7'—F–öä†æFÆW"æVæ7'—B‡–ÆöB’¢–ÆöC° —G'’…6ö6¶WB6ö6¶WBÒæWr6ö6¶WB‚’’° —6ö6¶WBæ6öææV7B†æWr–æWE6ö6¶WDFG&W72††÷7BÂ÷'B’Â#“° —G'’„FF÷WGWE7G&VÒ÷WGWBÒæWrFF÷WGWE7G&VÒ‡6ö6¶WBævWD÷WGWE7G&VÒ‚’’’° –÷WGWBçw&—FUUDb†Væ6öFVB“° –÷WGWBæfÇW6‚‚“° —Р—&WGW&âG'VS° —Ò6F6‚„W†6WF–öâR’° –FV'Vr†RævWDÖW76vR‚’“° —&WGW&âfÇ6S° —Р—Р —V&Æ–2fö–B6VæE6W'fW$æÖTÖW76vR‚’° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° —6VæEÇVv–äÖW76vU6W'fW"‡2ÂÂf÷F–æuÇVv–åv—&Rç6W'fW$æÖR‡2’“° —Р—Р —V&Æ–2fö–B6VæEf÷FU'G’…7G&–ær6W'fW"’° ––b†—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡6W'fW"’’° –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡6W'fW"ÂÂf÷F–æuÇVv–åv—&Rçf÷FU'G”'VævVR‚’“° —Р—Р —V&Æ–2fö–B6WD7W'&VçEf÷FU'G•f÷FW2†–çBÖ÷VçB’° —f÷FU'G•f÷FW2ÒÖ÷VçC° —6WEf÷FT66†Uf÷FU'G”7W'&VçEf÷FW2†Ö÷VçB“° –FV'Vr‚$7W'&VçBf÷FR'G’F÷Fâ"²f÷FU'G•f÷FW2“° —Р —V&Æ–2'7G&7Bfö–B6WEf÷FT66†TÆ7EWFFVB‚“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†U&WdF’†–çBF’“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†U&WdÖöçF‚…7G&–ærFW‡B“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†U&WevVV²†–çBvVV²“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†Uf÷FT66†T–væ÷&UF–ÖR†&ööÆVâ–væ÷&R“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†Uf÷FU'G”7W'&VçEf÷FW2†–çBf÷FW2“°  —V&Æ–2'7G&7Bfö–B6WEf÷FT66†Uf÷FU'G”–æ7&V6Uf÷FW5&WV—&VB†–çBf÷FW2“°  —V&Æ–2fö–B7FGW2‚’° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° ––b‚—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡2’’° –Æör‚$æòÆ–W'2öâ6W'fW""²2²"Fò6VæBFW7B7FGW2ÖW76vRÂÆV6R&WFW7Bv—F‚6öÖVöæRöæÆ–æR"“° —ÒVÇ6R° –Æör‚%6VæF–ær&WVW7Bf÷"7FGW2ÖW76vRöâ"²2“° –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡2ÂÂf÷F–æuÇVv–åv—&Rç7FGW2‡2’“° —Р—Р—Р ’ò¢¢'Vç26÷'&VÆFVBÂæöâ×f÷FR&÷VæBG&—÷fW"F†R7F—fR&6¶VæBG&ç7÷'Bâ¢ð —V&Æ–26ö×ÆWF&ÆTgWGW&SÄ6öÖ×Væ–6F–öåFW7E&W7VÇCâFW7D&6¶VæD6öÖ×Væ–6F–öâ…7G&–ær&WVW7FVE6W'fW"À –ÆöærF–ÖV÷WDÖ–ÆÆ—2’° •7G&–ær6W'fW"Ò&WVW7FVE6W'fW"ÓÒçVÆÂò""¢&WVW7FVE6W'fW"çG&–Ò‚“° ”'VævVTÖWF†öB7F—fTÖWF†öBÒÖWF†öC° ––b‡6W'fW"æ—4V×G’‚’ÇÂvWDÆÄf–Æ&ÆU6W'fW'2‚’æ6öçF–ç2‡6W'fW"’’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%Tä´äõtåô$4´TäB"Â%F†R&6¶VæB—2æ÷B6öæf–wW&VBöâF†—2&÷‡’"’“° —Р––b†7F—fTÖWF†öBÓÒçVÆÂÇÂvÆö&ÄÖW76vU&÷‡”†æFÆW"ÓÒçVÆÂ’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%E$å5õ%EõTäd”Ä$ÄR"Â%F†R&÷‡’6öÖ×Væ–6F–öâG&ç7÷'B—2æ÷B'Vææ–ær"’“° —Р––b†7F—fTÖWF†öBÓÒ'VævVTÖWF†öBåÅTt”äÔU54t”ärbb—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡6W'fW"’’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%Ä”U%õ$UT•$TB"Â%ÇVv–âÖW76v–ær&WV—&W2âöæÆ–æRÆ–W"öâF†R6VÆV7FVB&6¶VæB"’“° —Р•66†VGVÆVDW†V7WF÷%6W'f–6R66†VGVÆW"ÒvWE66†VGVÆW"‚“° ––b‡66†VGVÆW"ÓÒçVÆÂ’° —&WGW&â6ö×ÆWF&ÆTgWGW&Ræ6ö×ÆWFVDgWGW&R„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%E$å5õ%EõTäd”Ä$ÄR"Â%F†R&÷‡’66†VGVÆW"—2æ÷B'Vææ–ær"’“° —Р–Æöær&÷VæFVEF–ÖV÷WBÒÖF‚æÖ‚ƒSÂÂÖF‚æÖ–â‡F–ÖV÷WDÖ–ÆÆ—2Â3Â’“° •UT”B&WVW7D–BÒUT”Bç&æFöÕUT”B‚“° ”6ö×ÆWF&ÆTgWGW&SÄ6öÖ×Væ–6F–öåFW7E&W7VÇCâ&W7VÇBÒæWr6ö×ÆWF&ÆTgWGW&SÃâ‚“° •VæF–æt6öÖ×Væ–6F–öåFW7BVæF–ærÒæWrVæF–æt6öÖ×Væ–6F–öåFW7B‡6W'fW"Â7F—fTÖWF†öBÂ7—7FVÒæææõF–ÖR‚’Â&W7VÇB“° —VæF–æt6öÖ×Væ–6F–öåFW7G2çWB‡&WVW7D–BÂVæF–ær“° —&W7VÇBçv†Vä6ö×ÆWFR‚†–væ÷&VBÂf–ÇW&R’ÓâVæF–æt6öÖ×Væ–6F–öåFW7G2ç&VÖ÷fR‡&WVW7D–BÂVæF–ær’“° —G'’° ––b‚6VæD6öÖ×Væ–6F–öåFW7DVçfVÆ÷Tæ÷r‡6W'fW"Âf÷F–æuÇVv–åv—&Rç7FGW2‡6W'fW"Â&WVW7D–B’’’° —&W7VÇBæ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÂ%E$å5õ%EõTäd”Ä$ÄR"À ’%F†R7F—fRG&ç7÷'B6÷VÆBæ÷B66WBF†R6öÖ×Væ–6F–öâFW7B"’“° —&WGW&â&W7VÇC° —Р—66†VGVÆW"ç66†VGVÆR‚‚’Óâ&W7VÇBæ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÀ ’%D”ÔTõUB"Â$æò6÷'&VÆFVB&WÇ’'&—fVB&Vf÷&RF†RF–ÖV÷WB"’’Â&÷VæFVEF–ÖV÷WBÂF–ÖUVæ—BäÔ”ÄÄ•4T4ôäE2“° —Ò6F6‚…'VçF–ÖTW†6WF–öâf–ÇW&R’° —&W7VÇBæ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡6W'fW"Â7F—fTÖWF†öBÂ%4TäEôd”ÄTB"À ’%F†R&÷‡’6÷VÆBæ÷B6VæBF†R6öÖ×Væ–6F–öâFW7B"’“° —Р—&WGW&â&W7VÇC° —Р ’ò¢¢6VæG2F–væ÷7F–2–ÖÖVF–FVÇ’æB&W÷'G2v†WF†W"F†R7F—fRG&ç7÷'B66WFVB—Bâ¢ð —&÷FV7FVB&ööÆVâ6VæD6öÖ×Væ–6F–öåFW7DVçfVÆ÷Tæ÷r…7G&–ær6W'fW"§6öäVçfVÆ÷RVçfVÆ÷R’° —&WGW&â6VæE&÷‡”'&öF67DVçfVÆ÷Tæ÷r‡6W'fW"ÂVçfVÆ÷R“° —Р —&÷FV7FVBfö–B†æFÆU7FGW4ö¶’„§6öäVçfVÆ÷RÖW76vR’° •7G&–ær6W'fW"ÒÖW76vRævWDf–VÆG2‚’ævWD÷$FVfVÇB…f÷F–æuÇVv–åv—&Räµõ4U%dU"Â""“° •7G&–ær&WVW7BÒÖW76vRævWDf–VÆG2‚’ævWD÷$FVfVÇB…f÷F–æuÇVv–åv—&Räµõ$UTU5Eô”BÂ""“° ––b‡&WVW7Bæ—4V×G’‚’’° –Æör‚%7FGW2ö¶’f÷""²6W'fW"“° —&WGW&ã° —Р•UT”B&WVW7D–C° —G'’° —&WVW7D–BÒUT”Bæg&öÕ7G&–ær‡&WVW7B“° —Ò6F6‚„–ÆÆVvÄ&wVÖVçDW†6WF–öâ–væ÷&VB’° –FV'Vr‚$–væ÷&VB7FGW2&WÇ’v—F‚â–çfÆ–B&WVW7B”Bg&öÒ"²6W'fW"“° —&WGW&ã° —Р•VæF–æt6öÖ×Væ–6F–öåFW7BVæF–ærÒVæF–æt6öÖ×Væ–6F–öåFW7G2ævWB‡&WVW7D–B“° ––b‡VæF–ærÓÒçVÆÂÇÂVæF–ærç6W'fW"‚’æWVÇ2‡6W'fW"’’° –FV'Vr‚$–væ÷&VBVæW‡V7FVB7FGW2&WÇ’g&öÒ"²6W'fW"“° —&WGW&ã° —Р–Æöær&÷VæEG&—Ö–ÆÆ—2ÒÖF‚æÖ‚ƒÂÀ •F–ÖUVæ—Bäääõ4T4ôäE2çFôÖ–ÆÆ—2…7—7FVÒæææõF–ÖR‚’ÒVæF–ærç7F'FVDDææ÷2‚’’“° —VæF–ærç&W7VÇB‚’æ6ö×ÆWFR„6öÖ×Væ–6F–öåFW7E&W7VÇBç7V66W72‡6W'fW"ÂVæF–æræÖWF†öB‚’Â&÷VæEG&—Ö–ÆÆ—2’“° —Р —&—fFRfö–B6æ6VÄ6öÖ×Væ–6F–öåFW7G2…7G&–ærÖW76vR’° —VæF–æt6öÖ×Væ–6F–öåFW7G2æf÷$V6‚‚‡&WVW7D–BÂVæF–ær’ÓâVæF–ærç&W7VÇB‚’æ6ö×ÆWFR€ ”6öÖ×Væ–6F–öåFW7E&W7VÇBæf–ÇW&R‡VæF–ærç6W'fW"‚’ÂVæF–æræÖWF†öB‚’Â%E$å5õ%Eõ5DõTB"ÂÖW76vR’’“° —VæF–æt6öÖ×Væ–6F–öåFW7G2æ6ÆV"‚“° —Р —V&Æ–2&V6÷&B6öÖ×Væ–6F–öåFW7E&W7VÇB†&ööÆVâ7V66W72Â7G&–ær6öFRÂ7G&–ærÖW76vRÂ7G&–ær6W'fW"À •7G&–ærÖWF†öBÂÆöær&÷VæEG&—Ö–ÆÆ—2’° —&—fFR7FF–26öÖ×Væ–6F–öåFW7E&W7VÇB7V66W72…7G&–ær6W'fW"Â'VævVTÖWF†öBÖWF†öBÂÆöær&÷VæEG&—Ö–ÆÆ—2’° —&WGW&âæWr6öÖ×Væ–6F–öåFW7E&W7VÇB‡G'VRÂ$ô²"Â$&6¶VæB&WÆ–VB÷fW"F†R7F—fRG&ç7÷'B"Â6W'fW"À –ÖWF†öBÓÒçVÆÂò""¢ÖWF†öBææÖR‚’Â&÷VæEG&—Ö–ÆÆ—2“° —Р —&—fFR7FF–26öÖ×Væ–6F–öåFW7E&W7VÇBf–ÇW&R…7G&–ær6W'fW"Â'VævVTÖWF†öBÖWF†öBÂ7G&–ær6öFRÂ7G&–ærÖW76vR’° —&WGW&âæWr6öÖ×Væ–6F–öåFW7E&W7VÇB†fÇ6RÂ6öFRÂÖW76vRÂ6W'fW"À –ÖWF†öBÓÒçVÆÂò""¢ÖWF†öBææÖR‚’ÂÓ“° —Р—Р —&—fFR&V6÷&BVæF–æt6öÖ×Væ–6F–öåFW7B…7G&–ær6W'fW"Â'VævVTÖWF†öBÖWF†öBÂÆöær7F'FVDDææ÷2À ”6ö×ÆWF&ÆTgWGW&SÄ6öÖ×Væ–6F–öåFW7E&W7VÇCâ&W7VÇB’²Р —&—fFRfö–B6VæEf÷FTFVÆ•&V¦V7FVB…7G&–ærÆ–W"Â7G&–ærWV–BÂ7G&–ær6W'f–6RÂ&ööÆVâÆ–W$öæÆ–æRÀ •7G&–ærÆ–W%6W'fW"’° ––b‚Æ–W$öæÆ–æRÇÂÆ–W%6W'fW"ÓÒçVÆÂÇÂvWDÆÄf–Æ&ÆU6W'fW'2‚’æ6öçF–ç2‡Æ–W%6W'fW"’’° –FV'Vr‚$æ÷B6VæF–ærf÷FRFVÆ’&V¦V7F–öâf÷""²Æ–W"²"&V6W6RF†RÆ–W"—2öffÆ–æR"“° —&WGW&ã° —Р –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡Æ–W%6W'fW"ÂÀ •f÷F–æuÇVv–åv—&Rçf÷FTFVÆ•&V¦V7FVB‡Æ–W"ÂWV–BÂ6W'f–6RÂG'VR’“° —Р —V&Æ–27G&–ærvWEv—EVçF–ÄFVÆ•6—FTg&öÕ6W'f–6R…7G&–ær6W'f–6R’° –f÷"…7G&–ær6—FR¢vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•6—FW2‚’’° ––b†vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•6W'f–6R‡6—FR’æWVÇ4–væ÷&T66R‡6W'f–6R’’° —&WGW&â6—FS° —Р—Р—&WGW&â"#° —Р —&—fFRÆöærvWDÆ7Ef÷FW5F–ÖR…7G&–ærWV–BÂ'&”Æ—7CÄ6öÇVÖãâ6öÇ2Â7G&–ær6—FRÂ7G&–ær6W'f–6RÂ7G&–ærÆ–W"À –&ööÆVâ–æ6ÇVFUF–ÖT6†ævUVWVR’° –ÆöærÖ÷7E&V6VçEF–ÖRÒ°  ––b†vWEf÷FT66†T†æFÆW"‚’æ†4öæÆ–æUf÷FW2‡WV–B’’° ”'&”Æ—7CÄöffÆ–æT'VævVUf÷FSâöæÆ–æUf÷FW2ÒvWEf÷FT66†T†æFÆW"‚’ævWDöæÆ–æUf÷FW2‡WV–B“° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢öæÆ–æUf÷FW2’° ––b‡f÷FRævWE6W'f–6R‚’æWVÇ4–væ÷&T66R‡6W'f–6R’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂf÷FRævWEF–ÖR‚’“° —Р—Р—Р –f÷"…7G&–ær6W'fW"¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° –f÷"„öffÆ–æT'VævVUf÷FRf÷FR¢vWEf÷FT66†T†æFÆW"‚’ævWEf÷FW2‡6W'fW"’’° ––b‡f÷FRævWEWV–B‚’æWVÇ2‡WV–B’bbf÷FRævWE6W'f–6R‚’æWVÇ4–væ÷&T66R‡6W'f–6R’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂf÷FRævWEF–ÖR‚’“° —Р—Р—Р ––b†–æ6ÇVFUF–ÖT6†ævUVWVRbbÆ–W"ÒçVÆÂ’° –f÷"…f÷FUF–ÖUVWVRVWVVEf÷FR¢vWEf÷FT66†T†æFÆW"‚’ævWEF–ÖT6†ævUVWVR‚’’° ––b‡VWVVEf÷FRævWDæÖR‚’æWVÇ4–væ÷&T66R‡Æ–W" ’bbVWVVEf÷FRævWE6W'f–6R‚’æWVÇ4–væ÷&T66R‡6W'f–6R’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂVWVVEf÷FRævWEF–ÖR‚’“° —Р—Р—Р –f÷"„6öÇVÖâB¢6öÇ2’° ––b†BævWDæÖR‚’æWVÇ4–væ÷&T66R‚$Æ7Ef÷FW2"’’° ”FFfÇVRfÇVRÒBævWEfÇVR‚“° •7G&–æuµÒÆ—7BÒfÇVRævWE7G&–ær‚’ç7Æ—B‚"VÆ–æRR"“° –f÷"…7G&–ær7G"¢Æ—7B’° •7G&–æuµÒFFÒ7G"ç7Æ—B‚"òò"“° ––b†FF³ÒæWVÇ4–væ÷&T66R‡6—FR’’° –Ö÷7E&V6VçEF–ÖRÒÖF‚æÖ‚†Ö÷7E&V6VçEF–ÖRÂÆöærçfÇVTöb†FF³Ò’“° —Р—Р—Р—Р—&WGW&âÖ÷7E&V6VçEF–ÖS° —Р —V&Æ–2&ööÆVâ6†V6µf÷FTFVÆ’…7G&–ærWV–BÂ7G&–ær6W'f–6RÂ'&”Æ—7CÄ6öÇVÖãâFF’° —&WGW&â6†V6µf÷FTFVÆ’‡WV–BÂçVÆÂÂ6W'f–6RÂFFÂfÇ6R“° —Р ’ò¢  ’¢6†V6·2F†R6öæf–wW&VBf÷FRFVÆ’Â÷F–öæÆÇ’–æ6ÇVF–ær66WFVBf÷FW2v—F–æp ’¢f÷"vÆö&ÄFFF–ÖR6†ævRFòf–æ—6‚à ’  ’¢&ÒWV–BÆ–W"UT”@ ’¢&ÒÆ–W"Æ–W"æÖRW6VB'’F†RF–ÖRÖ6†ævRVWVP ’¢&Ò6W'f–6Rf÷FR6W'f–6P ’¢&ÒFF7W'&VçBÆ–W"FF ’¢&Ò–æ6ÇVFUF–ÖT6†ævUVWVRv†WF†W"VWVVBf÷FW2&W6W'fRF†V—"FVÆ’6Æ÷@ ’¢&WGW&âG'VRv†VâF†Rf÷FRÖ’&R66WFV@ ’¢ð —V&Æ–2&ööÆVâ6†V6µf÷FTFVÆ’…7G&–ærWV–BÂ7G&–ærÆ–W"Â7G&–ær6W'f–6RÂ'&”Æ—7CÄ6öÇVÖãâFFÀ –&ööÆVâ–æ6ÇVFUF–ÖT6†ævUVWVR’° •7G&–ær6—FRÒvWEv—EVçF–ÄFVÆ•6—FTg&öÕ6W'f–6R‡6W'f–6R“° ––b‡6—FRæ—4V×G’‚’’° –FV'Vr‚$æò6W'f–6R6—FR6WBf÷""²6W'f–6R²"Â6¶—–ærf÷FRFVÆ’6†V6²"“° —&WGW&âG'VS° —Р ––çBf÷FTFVÆ’ÒvWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVÆ’‡6—FR“° ––çBf÷FTFVƔ֖âÒvWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVƔ֖â‡6—FR“°  –ÆöærÆ7Ef÷FRÒvWDÆ7Ef÷FW5F–ÖR‡WV–BÂFFÂ6—FRÂ6W'f–6RÂÆ–W"–æ6ÇVFUF–ÖT6†ævUVWVR“° ––b†Æ7Ef÷FRÓÒ’° –FV'Vr‚$æòÆ7Bf÷FRF–ÖRf÷VæBf÷""²WV–B²"ò"²6W'f–6R²"Â6¶—–ærf÷FRFVÆ’6†V6²"“° —&WGW&âG'VS° —Р —G'’° ”Æö6ÄFFUF–ÖRæ÷rÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚“° ”Æö6ÄFFUF–ÖRÆ7Ef÷FUF–ÖRÒÆö6ÄFFUF–ÖRæöd–ç7FçB„–ç7FçBæödWö6„Ö–ÆÆ’†Æ7Ef÷FR’¦öæT–Bç7—7FVÔFVfVÇB‚’ ’çÇW4†÷W'2†vWD6öæf–r‚’ævWEF–ÖT†÷W$öfe6WB‚’“°  ––b‚vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVÆ”F–Ç’‡6—FR’’° ––b‡f÷FTFVÆ’ÓÒbbf÷FTFVƔ֖âÓÒ’° –FV'Vr‚%f÷FRFVÆ’—2f÷""²6—FR²"Â6¶—–ærf÷FRFVÆ’6†V6²"“° —&WGW&âG'VS° —Р ”Æö6ÄFFUF–ÖRæW‡Gf÷FRÒÆ7Ef÷FUF–ÖRçÇW4†÷W'2‚†Æöær’f÷FTFVÆ’’çÇW4Ö–çWFW2‚†Æöær’f÷FTFVƔ֖⓰ —&WGW&âæ÷ræ—4gFW"†æW‡Gf÷FR“° —Р”Æö6ÄFFUF–ÖR&W6WEF–ÖRÒÆ7Ef÷FUF–ÖRçv—F„†÷W"†vWD6öæf–r‚’ævWEv—EVçF–Åf÷FTFVÆ•f÷FTFVÆ”†÷W"‡6—FR’ ’çv—F„Ö–çWFRƒ’çv—F…6V6öæBƒ“° ”Æö6ÄFFUF–ÖR&W6WEF–ÖUFöÖ÷'&÷rÒ&W6WEF–ÖRçÇW4†÷W'2ƒ#B“°  ––b†Æ7Ef÷FUF–ÖRæ—4&Vf÷&R‡&W6WEF–ÖR’’° ––b†æ÷ræ—4gFW"‡&W6WEF–ÖR’’° –FV'Vr‚%f÷FRFVÆ’—2ÖWBf÷""²WV–B²"ò"²6W'f–6R²"Âf÷FR6â&R&ö6W76VB"“° —&WGW&âG'VS° —Р—ÒVÇ6R° ––b†æ÷ræ—4gFW"‡&W6WEF–ÖUFöÖ÷'&÷r’’° –FV'Vr‚%f÷FRFVÆ’—2ÖWBf÷""²WV–B²"ò"²6W'f–6R²"Âf÷FR6â&R&ö6W76VB"“° —&WGW&âG'VS° —Р—Р—Ò6F6‚„W†6WF–öâR’° –Rç&–çE7F6µG&6R‚“° —Р –FV'Vr‚%f÷FRFVÆ’—2æ÷BÖWBf÷""²WV–B²"ò"²6W'f–6R²"Â6¶—–ærf÷FR"“° —&WGW&âfÇ6S° —Р —V&Æ–27–æ6‡&öæ—¦VBfö–Bf÷FR…7G&–ærÆ–W"Â7G&–ær6W'f–6RÂ&ööÆVâ&VÅf÷FRÂ&ööÆVâF–ÖUVWVRÂÆöærVWVUF–ÖRÀ •f÷FUF÷FÇ56æ6†÷BFW‡BÂ7G&–ærWV–B’° —f÷FR‡Æ–W"Â6W'f–6RÂ&VÅf÷FRÂF–ÖUVWVRÂVWVUF–ÖRÂFW‡BÂWV–BÂçVÆ“° —Р —&—fFRVçVÒVWVVEf÷FU&W7VÇB° •5T44U52Â$UE%’ÂDU$Ô”äÀ —Р —&—fFR7–æ6‡&öæ—¦VBVWVVEf÷FU&W7VÇBf÷FR…7G&–ærÆ–W"Â7G&–ær6W'f–6RÂ&ööÆVâ&VÅf÷FRÂ&ööÆVâF–ÖUVWVRÂÆöærVWVUF–ÖRÀ •f÷FUF÷FÇ56æ6†÷BFW‡BÂ7G&–ærWV–BÂf÷FUF–ÖUVWVRVWVVEf÷FR’° —G'’° ––b‚6W'f–6U6—FUfÆ–FF÷"æ—5fÆ–B‡6W'f–6R’’° —v&â‚%&V¦V7FVBf÷FRv—F‚–çfÆ–B6W'f–6R6—FRr"²6W'f–6U6—FUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡6W'f–6R’²"r"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р––b‚Ö–æV7&gEW6W&æÖUfÆ–FF÷"æ—5fÆ–B‡Æ–W"ÂvWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’’’° —v&â‚%&V¦V7FVBf÷FRv—F‚–çfÆ–BÖ–æV7&gBW6W&æÖRr  ’²Ö–æV7&gEW6W&æÖUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡Æ–W"’²"rg&öÒ6W'f–6Rr  ’²Ö–æV7&gEW6W&æÖUfÆ–FF÷"ç6æ—F—¦Tf÷$Æör‡6W'f–6R’²"r"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р •UT”Bf÷FT–BÒVWVVEf÷FRÓÒçVÆÂòçVÆ¢VWVVEf÷FRævWEf÷FT–B‚“° ––b‡f÷FT–BÓÒçVÆÂ’° —f÷FT–BÒUT”Bç&æFöÕUT”B‚“° —Р ’òòUT”B&W6öÇWF–öà ––b‚vWD6öæf–r‚’ævWDöæÆ–æTÖöFR‚’’° —WV–BÒvWEUT”B‡Æ–W"“° —Р ––b‡WV–BÓÒçVÆÂÇÂWV–Bæ—4V×G’‚’’° —WV–BÒvWEUT”B‡Æ–W"“°  ’òò&VG&ö6²&Vf—‚WFòÖFWFV7@ ––b‡WV–Bæ—4V×G’‚’bbvWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’æ—4V×G’‚ ’bbÆ–W"ç7F'G5v—F‚†vWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’’’° •7G&–ærWV–CÒvWEUT”B†vWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’²Æ–W"“° ––b‚WV–Cæ—4V×G’‚’’° –FV'Vr‚$FWFV7FVB&VG&ö6²Æ–W"v—F†÷WB&Vf—‚ÂF§W7F–ærâââ"“° —Æ–W"ÒvWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’²Æ–W#° —WV–BÒWV–C° —Р—Р—Р ––b‡WV–Bæ—4V×G’‚’’° ––b‡Æ–W"ç7F'G5v—F‚†vWD6öæf–r‚’ævWD&VG&ö6µÆ–W%&Vf—‚‚’’’° –Æör‚$–væ÷&–ærf÷FR6–æ6RVæ&ÆRFòvWBUT”Böb&VG&ö6²Æ–W""“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р––b‚vWD6öæf–r‚’ævWDÆÆ÷uVä¦ö–æVB‚’’° –Æör‚$–væ÷&–ærf÷FRg&öÒ"²Æ–W"²"6–æ6RÆ–W"†6âwB¦ö–æVB&Vf÷&R"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р––b‚vWD6öæf–r‚’ævWEUT”DÆöö·W‚’’° –Æör‚$f–ÆVBFòvWBWV–Bf÷""²Æ–W"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р –FV'Vr‚$fWF6†–ærUT”BöæÆ–æRÂ6–æ6RÆÆ÷wVæ¦ö–æVB—2Væ&ÆVB"“° •UT”BRÒçVÆð —G'’° ––b†vWD6öæf–r‚’ævWDöæÆ–æTÖöFR‚’’° —RÒfWF6…UT”B‡Æ–W"“° —Р—Ò6F6‚„W†6WF–öâR’° ––b†vWD6öæf–r‚’ævWDFV'Vr‚’’° –Rç&–çE7F6µG&6R‚“° —Р—Р––b‡RÓÒçVÆÂ’° –FV'Vr‚$f–ÆVBFòvWBWV–Bf÷""²Æ–W"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р—WV–BÒRçFõ7G&–ær‚“° —Р ’òòæ÷&ÖÆ—¦RUT”B7G&–ær–b÷76–&ÆP —G'’° ––b‡WV–BÒçVÆÂbbWV–Bæ—4V×G’‚’bbWV–BæWVÇ4–væ÷&T66R‚&çVÆÂ"’’° —WV–BÒUT”Bæg&öÕ7G&–ær‡WV–BçG&–Ò‚’’çFõ7G&–ær‚“° —Р—Ò6F6‚„W†6WF–öâ–væ÷&VB’° ’òò–væ÷&P —Р —Æ–W"ÒvWE&÷W$æÖR‡WV–BÂÆ–W"“°  ’òò66†RöæÆ–æR7FFR÷6W'fW"öæ6R„”Õõ%DåBf÷"'&öF67BÆöv–26÷'&V7FæW72 –f–æÂ&ööÆVâÆ–W$öæÆ–æRÒ—5Æ–W$öæÆ–æTf÷%f÷FU&÷WF–ær‡Æ–W"“° –f–æÂ7G&–ærÆ–W%6W'fW"ÒÆ–W$öæÆ–æRòvWD7W'&VçEÆ–W%6W'fW$f÷%f÷FU&÷WF–ær‡Æ–W"’¢çVÆð –ÆöærF–ÖRÒVWVUF–ÖRÒòVWVUF–ÖP “¢Æö6ÄFFUF–ÖRææ÷r‚’æE¦öæR…¦öæT–Bç7—7FVÔFVfVÇB‚’’çFô–ç7FçB‚’çFôWö6„Ö–ÆÆ’‚“°  •6WCÅ7G&–æsâ'&öF67EF&vWG2ÒVWVVEf÷FRÓÒçVÆÂòæWrÆ–æ¶VD†6…6WCÃâ‚ “¢æWrÆ–æ¶VD†6…6WCÃâ‡VWVVEf÷FRævWD'&öF67EF&vWG2‚’“° •6WCÅ7G&–æsâ'&öF67Df÷'v&FVE6W'fW'2ÒVWVVEf÷FRÓÒçVÆÂòæWrÆ–æ¶VD†6…6WCÃâ‚ “¢æWrÆ–æ¶VD†6…6WCÃâ‡VWVVEf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’“° –&ööÆVâ&÷‡”'&öF67D†æFÆVBÒVWVVEf÷FRÒçVÆÂbbVWVVEf÷FRæ—5&÷‡”'&öF67D†æFÆVB‚“° –&ööÆVâ&ö6W76W5F÷FÇ2ÒvWD6öæf–r‚’ævWE&–Ö'•6W'fW"‚’ÇÂvWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚“° –&ööÆVâÖævW5F÷FÇ2Ò&ö6W76W5F÷FÇ2bbvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚“° –&ööÆVâ6åfÆ–FFU7FæFÆöæT'&öF67BÒ6äf÷'v&E7FæFÆöæT'&öF67B†ÖævW5F÷FÇ2“° ”'&”Æ—7CÄ6öÇVÖãâFFÒçVÆð –&ööÆVâVWVTf÷%F–ÖT6†ævRÒfÇ6S°  ’òò6ö×ÆWF–öâ6ÆÆ&6²6âv—RF÷FÇ2æB&WÆ’öÆFW"VWVVBf÷FW2â'Vâ—@ ’òò&Vf÷&RÆöF–ærF†—2f÷FRw2FF&6R6æ6†÷B6òF†R6Æ7VÆF–öç2&VÆ÷rW6P ’òòF†R÷7B×&öÆÆ÷fW"7FFRà ––b†vWD6öæf–r‚’ævWDvÆö&ÄFFVæ&ÆVB‚’bbvWDvÆö&ÄFF†æFÆW"‚’æ—5F–ÖT6†ævVD†VæVB‚’’° –vWDvÆö&ÄFF†æFÆW"‚’æ6†V6´f÷$f–æ—6†VEF–ÖT6†ævW2‚“° —VWVTf÷%F–ÖT6†ævRÒF–ÖUVWVRbbvWDvÆö&ÄFF†æFÆW"‚’æ—5F–ÖT6†ævVD†VæVB‚“° —Р ’òòfÆ–FFRF†Rf÷FR&Vf÷&Rç’–ÖÖVF–FRææ÷Væ6VÖVçBâF†—2¶VW2GWÆ–6FP ’òòf÷FW2&V¦V7FVB'’F†RFVÆ’6†V6²÷WBöbF†RvÆö&ÄFF&öÆÆ÷fW"VWVRæ@ ’òò&WfVçG2ææ÷Væ6–ærf÷FRF†Bv–ÆÂæ÷B&R&ö6W76VBà ––b†ÖævW5F÷FÇ2’° ––b†vWE&÷‡”ו5‚’ÓÒçVÆÂ’° –Æöu6WfW&R‚$×—7—2æ÷BÆöFVB6÷'&V7FÇ’Â7F÷–ærf÷FR&ö6W76–ær"“° —&WGW&âVWVVEf÷FU&W7VÇBå$UE%“° —Р ––b‚vWE&÷‡”ו5‚’æ6öçF–ç4¶W•VW'’‡WV–B’’° –vWE&÷‡”ו5‚’çWFFR‡WV–BÂ%Æ–W$æÖR"ÂæWrFFfÇVU7G&–ær‡Æ–W"’“° –vWE&÷‡”ו5‚’ævWEWV–G2‚’æFB‡WV–B“° —Р –FFÒvWE&÷‡”ו5‚’ævWDW†7EVW'’†æWr6öÇVÖâ‚'WV–B"ÂæWrFFfÇVU7G&–ær‡WV–B’’“° ––b‚6†V6µf÷FTFVÆ’‡WV–BÂÆ–W"Â6W'f–6RÂFFÂVWVVEf÷FRÓÒçVÆÂ’’° –Æör‚%f÷FRFVÆ’—2æ÷BÖWBf÷""²Æ–W"²"ò"²6W'f–6R²"Â6¶—–ærf÷FR"“° —6VæEf÷FTFVÆ•&V¦V7FVB‡Æ–W"ÂWV–BÂ6W'f–6RÂÆ–W$öæÆ–æRÂÆ–W%6W'fW"“° —&WGW&âVWVVEf÷FU&W7VÇBåDU$Ô”äð —Р—Р ’òòf÷'v&Bâ66WFVBöffÆ–æR'&öF67B&Vf÷&RF†R7F–ÆÂÖ7F—fRvÆö&ÄFF ’òò6†ævRVWVW2F†R&Wv&B÷F÷FÇ2v÷&²âF†RVWVVBFVÆ—fW'’7FFR&WfVçG0 ’òò&WÆ––ær'&öF67G2F†BÇ&VG’&V6†VB&6¶VæBà ––b‡VWVTf÷%F–ÖT6†ævR’° •f÷FUF÷FÇ56æ6†÷B&ö¦V7FVEF÷FÇ2ÒÖævW5F÷FÇ2òvWE&ö¦V7FVE&öÆÆ÷fW%F÷FÇ2†FFÂÆ–W"’¢FW‡C° ––b†6åfÆ–FFU7FæFÆöæT'&öF67Bbb&÷‡”'&öF67DFV6–FW"çW6W4–ÖÖVF–FTf÷'v&F–ær‡Æ–W$öæÆ–æR’’° –'&öF67EF&vWG2æFDƇ&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2†fÇ6RÂçVÆÂ’“° —&÷‡”'&öF67D†æFÆVBÒG'VS° —Р•f÷FUF–ÖUVWVRFVÆ–VEf÷FRÒæWrf÷FUF–ÖUVWVR‡f÷FT–BÂÆ–W"Â6W'f–6RÂF–ÖRÀ —&÷‡”'&öF67D†æFÆVBÂ'&öF67EF&vWG2Â'&öF67Df÷'v&FVE6W'fW'2À —&ö¦V7FVEF÷FÇ2ÓÒçVÆÂò""¢&ö¦V7FVEF÷FÇ2çFõ7G&–ær‚’ÂfÇ6RÂWV–B“° ––b‚vWEf÷FT66†T†æFÆW"‚’æFEF–ÖUf÷FUFô66†R†FVÆ–VEf÷FR’’° –Æöu6WfW&R‚%Væ&ÆRFòW'6—7BVWVVB&öÆÆ÷fW"f÷FRf÷""²Æ–W"²"ò"²6W'f–6P ’²#²6¶—–ær&÷‡’'&öF67B"“° —&WGW&âVWVVEf÷FU&W7VÇBå$UE%“° —Р––b‡&÷‡”'&öF67D†æFÆVB’° –f÷"…7G&–ærF&vWB¢'&öF67EF&vWG2’° •6WCÅ7G&–æsâf÷'v&FVBÒ6VæE&÷‡”'&öF67B„6öÆÆV7F–öç2ç6–ævÆWFöâ‡F&vWB’ÂWV–BÂÆ–W"À —6W'f–6RÂF–ÖRÂ&ö¦V7FVEF÷FÇ2ÓÒçVÆÂò""¢&ö¦V7FVEF÷FÇ2çFõ7G&–ær‚’ÂfÇ6R“° ––b†FVÆ–VEf÷FRævWD'&öF67Df÷'v&FVE6W'fW'2‚’æFDƆf÷'v&FVB’’° –'&öF67Df÷'v&FVE6W'fW'2æFDƆf÷'v&FVB“° —W'6—7EF–ÖUf÷FTFVÆ—fW'’†FVÆ–VEf÷FR“° —Р—Р—Р–Æör‚$66†–ærf÷FRg&öÒ"²Æ–W"²"ò"²6W'f–6P ’²"&V6W6RF–ÖR6†ævR—2†Væ–ær&–v‡Bæ÷r"“° —&WGW&âVWVVEf÷FU&W7VÇBå5T44U53° —Р –FEf÷FU'G’‚“°  ’òòF÷FÇ2&ö6W76–ær‡&–Ö'’6W'fW"õ"æò×VÇF—&÷‡’ ––b‡&ö6W76W5F÷FÇ2’° ––b†ÖævW5F÷FÇ2’° ––çBÆÅF–ÖUF÷FÂÒvWEfÇVR†FFÂ$ÆÅF–ÖUF÷FÂ"“° ––çBÖöçF…F÷FÂÒvWEfÇVR†FFÂ$ÖöçF…F÷FÂ"“°  ––çBFFTÖöçF…F÷FÂÒÓ° ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° ––b†vWD6öæf–r‚’ævWEW6TÖöçF„FFUF÷FÇ45&–Ö'•F÷F‚’’° –FFTÖöçF…F÷FÂÒvWEfÇVR†FFÂvWDÖöçF…F÷FÇ5v—F„FFUF‚‚’“° —ÒVÇ6R° –FFTÖöçF…F÷FÂÒÖöçF…F÷Fð —Р—Р ––çBvVV¶Ç•F÷FÂÒvWEfÇVR†FFÂ%vVV¶Ç•F÷FÂ"“° ––çBF–Ç•F÷FÂÒvWEfÇVR†FFÂ$F–Ç•F÷FÂ"“° ––çBö–çG2ÒvWEfÇVR†FFÂ%ö–çG2"ÂvWD6öæf–r‚’ævWEö–çG4öåf÷FR‚’“°  ––çBÖ…f÷FW2ÒvWD6öæf–r‚’ævWDÖ„Ö÷VçDöef÷FW5W$F’‚“° ––b†Ö…f÷FW2â’° ”Æö6ÄFFUF–ÖR5F–ÖRÒvWD'VævVUF–ÖT6†V6¶W"‚’ævWEF–ÖR‚“° ––çBF—2Ò5F–ÖRævWDF”ödÖöçF‚‚“° ––b†ÖöçF…F÷FÂâF—2¢Ö…f÷FW2’° –ÖöçF…F÷FÂÒF—2¢Ö…f÷FW3° —Р—Р ––b†vWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’âbbö–çG2âvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚’’° —ö–çG2ÒvWD6öæf–r‚’ævWDÆ–Ö—Ef÷FUö–çG2‚“° —Р —FW‡BÒæWrf÷FUF÷FÇ56æ6†÷B†ÆÅF–ÖUF÷FÂÂÖöçF…F÷FÂÂvVV¶Ç•F÷FÂÂF–Ç•F÷FÂÂö–çG2À —f÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VBÂFFTÖöçF…F÷F“°  ”'&”Æ—7CÄ6öÇVÖãâWFFRÒæWr'&”Æ—7CÃâ‚“° —WFFRæFB†æWr6öÇVÖâ‚$ÆÅF–ÖUF÷FÂ"ÂæWrFFfÇVT–çB†ÆÅF–ÖUF÷FÂ’’“° —WFFRæFB†æWr6öÇVÖâ‚$ÖöçF…F÷FÂ"ÂæWrFFfÇVT–çB†ÖöçF…F÷FÂ’’“° ––b†vWD6öæf–r‚’ævWE7F÷&TÖöçF…F÷FÇ5v—F„FFR‚’’° —WFFRæFB†æWr6öÇVÖâ†vWDÖöçF…F÷FÇ5v—F„FFUF‚‚’ÂæWrFFfÇVT–çB†FFTÖöçF…F÷FÂ’’“° —Р—WFFRæFB†æWr6öÇVÖâ‚%vVV¶Ç•F÷FÂ"ÂæWrFFfÇVT–çB‡vVV¶Ç•F÷FÂ’’“° —WFFRæFB†æWr6öÇVÖâ‚$F–Ç•F÷FÂ"ÂæWrFFfÇVT–çB†F–Ç•F÷FÂ’’“° —WFFRæFB†æWr6öÇVÖâ‚%ö–çG2"ÂæWrFFfÇVT–çB‡ö–çG2’’“°  –FV'Vr‚%6WGF–ærF÷FÇ2"²FW‡BçFõ7G&–ær‚’²"Âf÷FT–CÒ"²f÷FT–B²"f÷""²Æ–W"²"ò  ’²6W'f–6R“° –vWE&÷‡”ו5‚’çWFFR‡WV–BÂWFFR“° —ÒVÇ6R° —FW‡BÒæWrf÷FUF÷FÇ56æ6†÷BƒÂÂÂÂÂf÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VB“° —Р—Р––b‡FW‡BÓÒçVÆÂ’° —FW‡BÒæWrf÷FUF÷FÇ56æ6†÷BƒÂÂÂÂÂf÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VB“° —Р •f÷FTÆöu7FGW2f÷FU7FGW2Òf÷FTÆöu7FGW2ä”ÔÔTD”DS° –&ööÆVâ7FæFÆöæU&÷‡”'&öF67BÒ6åfÆ–FFU7FæFÆöæT'&öF67Bbb‡&÷‡”'&öF67D†æFÆV@ —ÇÂ&÷‡”'&öF67DFV6–FW"çW6W4–ÖÖVF–FTf÷'v&F–ær‡Æ–W$öæÆ–æR’“° •6WCÅ7G&–æsâ&÷‡”'&öF67EF&vWG2Ò6öÆÆV7F–öç2æV×G•6WB‚“° ––b‡7FæFÆöæU&÷‡”'&öF67B’° ’òò†æFÆVBVWVVB'&öF67Bv2æV6W76&–Ç’6×ÆVBv†–ÆRF†RÆ–W"v0 ’òòöffÆ–æRâ&WG'’öæÇ’F&vWG2F†BF–Bæ÷B&Wf–÷W6Ç’66WBFVÆ—fW'’à —&÷‡”'&öF67EF&vWG2Ò&÷‡”'&öF67D†æFÆVBòæWrÆ–æ¶VD†6…6WCÃâ†'&öF67EF&vWG2 “¢&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2†fÇ6RÂçVÆ“° •6WCÅ7G&–æsâ&VÖ–æ–æuF&vWG2ÒæWrÆ–æ¶VD†6…6WCÃâ‡&÷‡”'&öF67EF&vWG2“° —&VÖ–æ–æuF&vWG2ç&VÖ÷fTƆ'&öF67Df÷'v&FVE6W'fW'2“° –'&öF67Df÷'v&FVE6W'fW'2æFDƇ6VæE&÷‡”'&öF67B‡&VÖ–æ–æuF&vWG2ÂWV–BÂÆ–W"Â6W'f–6RÂF–ÖRÀ —FW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’ÂfÇ6R’“° —Р ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР’òò6VæBf÷FR‡2’Fò&6¶VæB‡2 ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР––b†vWD6öæf–r‚’ævWE6VæEf÷FW5FôÆÅ6W'fW'2‚’’° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’°  –&ööÆVâf÷&6T66†RÒvWD6öæf–r‚’ævWEv—Df÷%W6W$öæÆ–æR‚ ’bb‚Æ–W$öæÆ–æRÇÂÆ–W%6W'fW"ÓÒçVÆÂÇÂÆ–W%6W'fW"æWVÇ4–væ÷&T66R‡2’“°  ––b†f÷&6T66†R’° –FV'Vr‚$f÷&6–ærf÷FRFò66†Rf÷"6W'fW""²2“° —Р ––b‚‚—56öÖVöæTöæÆ–æU6W'fW$f÷%f÷FU&÷WF–ær‡2’bbÖWF†öBç&WV—&W5Æ–W$öæÆ–æR‚’’ÇÂf÷&6T66†R’° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFE6W'fW%f÷FR‡2À –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÀ —FW‡BçFõ7G&–ær‚’Â'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÀ —&÷‡”'&öF67EF&vWG2Â'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–ærf÷FRf÷""²Æ–W"²"öâ"²6W'f–6R²"f÷""²2“° —ÒVÇ6R° –&ööÆVâ'&öF67D†W&RÒ'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç2‡2“° ––b†'&öF67D†W&RbbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° •6WCÅ7G&–æsâF&vWG2Ò7FæFÆöæU&÷‡”'&öF67Bò&÷‡”'&öF67EF&vWG0 “¢&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡Æ–W$öæÆ–æRÂÆ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡2ÂF&vWG2“° —Р ––b‚6VæEf÷FTVçfVÆ÷T66WFVB‡2Â"À •f÷F–æuÇVv–åv—&Rçf÷FR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂG'VRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À —f÷FT–BÂvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’Â'&öF67D†W&RÂÂ’’’° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFE6W'fW%f÷FR‡2À –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÀ —FW‡BçFõ7G&–ær‚’Â'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÀ —&÷‡”'&öF67EF&vWG2Â'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–ærf÷FRgFW"F†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²2“° —Р—Р—Р—ÒVÇ6R° ’òò6–ævÆR×6W'fW"ÖöFS¢öæÆ–æRvöW2FòÆ–W"6W'fW#²÷F†W'v—6RVWVR2&öæÆ–æP ’òòf÷FR  ––b‡Æ–W$öæÆ–æRbbÆ–W%6W'fW"ÒçVÆÂbbvWDÆÄf–Æ&ÆU6W'fW'2‚’æ6öçF–ç2‡Æ–W%6W'fW"’’° •7G&–ær6W'fW"ÒÆ–W%6W'fW#°  –&ööÆVâ'&öF67D†W&RÒ'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç2‡6W'fW"“° ––b†'&öF67D†W&RbbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚’’° •6WCÅ7G&–æsâF&vWG2Ò7FæFÆöæU&÷‡”'&öF67Bò&÷‡”'&öF67EF&vWG0 “¢&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡G'VRÂÆ–W%6W'fW"“° –'&öF67D†W&RÒ&÷‡”'&öF67DFV6–FW"ç6†÷VÆD'&öF67B‡6W'fW"ÂF&vWG2“° —Р –&ööÆVâ&Wv&D66WFVBÒ6VæEf÷FTVçfVÆ÷T66WFVB‡6W'fW"ÂÀ •f÷F–æuÇVv–åv—&Rçf÷FTöæÆ–æR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂG'VRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À —f÷FT–BÂvWD6öæf–r‚’ævWD'VævVTÖævUF÷FÇ2‚’Â'&öF67D†W&RÂÂ’“° ––b‚&Wv&D66WFVB’° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFDöæÆ–æUf÷FR‡WV–BÀ –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À –'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÂ&÷‡”'&öF67EF&vWG2À –'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–æröæÆ–æRf÷FRgFW"F†RG&ç7÷'B&V¦V7FVBFVÆ—fW'’f÷""²6W'fW"“° —Р ––b‡&Wv&D66WFVBbb6åfÆ–FFU7FæFÆöæT'&öF67BbbvWD6öæf–r‚’ævWE&÷‡”'&öF67DVæ&ÆVB‚ ’bb7FæFÆöæU&÷‡”'&öF67B’° •6WCÅ7G&–æsâF&vWG2Ò&÷‡”'&öF67DFV6–FW"ç&W6öÇfUF&vWG2‡G'VRÂÆ–W%6W'fW"“°  ––çB$FVÆ’Ò#° –f÷"…7G&–ærF&vWE6W'fW"¢F&vWG2’° ’òòfö–BF÷V&ÆRÖ'&öF67BöâF†R6ÖR6W'fW"F†BÇ&VG’v÷BF†Rf÷FTöæÆ–æP ––b‡F&vWE6W'fW"æWVÇ4–væ÷&T66R‡6W'fW"’’° –6öçF–çVS° —Р––b†vWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡F&vWE6W'fW"’’° –6öçF–çVS° —Р –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡F&vWE6W'fW"Â$FVÆ’À •f÷F–æuÇVv–åv—&Rçf÷FT'&öF67B‡WV–BÂÆ–W"Â6W'f–6RÂF–ÖRÀ —FW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’ÂG'VR’“° –$FVÆ’²³° —Р—Р ’òò×VÇF—&÷‡“¢VçfVÆ÷RÖöæÇ’6ÆV"f÷FP ––b‡&Wv&D66WFVBbbvWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚’bbvWD6öæf–r‚’ævWD×VÇF•&÷‡”öæTvÆö&Å&Wv&B‚’’° –×VÇF•&÷‡”†æFÆW"ç6VæD6ÆV%f÷FR‡WV–BÂÆ–W"“° —Р—ÒVÇ6R° —f÷FU7FGW2Òf÷FTÆöu7FGW2ä44„TC° –&ööÆVâ'&öF67Df÷'v&FVBÒ7FæFÆöæU&÷‡”'&öF67@ ’bb'&öF67Df÷'v&FVE6W'fW'2æ6öçF–ç4Ƈ&÷‡”'&öF67EF&vWG2“° –vWEf÷FT66†T†æFÆW"‚’æFDöæÆ–æUf÷FR‡WV–BÀ –æWröffÆ–æT'VævVUf÷FR‡f÷FT–BÂÆ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂ&VÅf÷FRÂFW‡BçFõ7G&–ær‚’À –'&öF67Df÷'v&FVBÂ7FæFÆöæU&÷‡”'&öF67BÂ&÷‡”'&öF67EF&vWG2À –'&öF67Df÷'v&FVE6W'fW'2ÂfÇ6R’“° –FV'Vr‚$66†–æröæÆ–æRf÷FRf÷""²Æ–W"²"öâ"²6W'f–6R“° —Р ––çBFVÆ’Ò#° –f÷"…7G&–ær2¢vWDÆÄf–Æ&ÆU6W'fW'2‚’’° –vÆö&ÄÖW76vU&÷‡”†æFÆW"ç6VæDÖW76vR‡2ÂFVÆ’²Âf÷F–æuÇVv–åv—&Rçf÷FUWFFR‡WV–BÀ —f÷FU'G•f÷FW2Â7W'&VçEf÷FU'G•f÷FW5&WV—&VBÂ6W'f–6RÂF–ÖRÂFW‡BçFõ7G&–ær‚’’“° –FVÆ’³Ò#° —Р—Р ’òòf÷FRÆövv–æp ––b‡f÷FTÆöt×—7ÅF&ÆRÒçVÆÂbbvWD6öæf–r‚’ævWEf÷FTÆövv–ætVæ&ÆVB‚’’° —f÷FTÆöt×—7ÅF&ÆRæÆöuf÷FR‡f÷FT–BÂf÷FU7FGW2Â6W'f–6RÂWV–BÂÆ–W"ÂF–ÖRÀ –vWEf÷FT66†T†æFÆW"‚’ævWE&÷‡”66†VEF÷F‡WV–B’“° —Р ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР’òò×VÇF—&÷‡’f÷'v&F–æp ’òòÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓР––b†vWD6öæf–r‚’ævWD×VÇF•&÷‡•7W÷'B‚’bbvWD6öæf–r‚’ævWE&–Ö'•6W'fW"‚’’° ––b‚vWD6öæf–r‚’ævWD×VÇF•&÷‡”öæTvÆö&Å&Wv&B‚’’° –FV'Vr‚%6VæF–ærvÆö&Â&÷‡’f÷FRVçfVÆ÷R"“° –×VÇF•&÷‡”†æFÆW"ç6VæD×VÇF•&÷‡”VçfVÆ÷R…f÷F–æuÇVv–åv—&Rçf÷FR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂfÇ6RÀ —&VÅf÷FRÂFW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’Âf÷FT–BÂfÇ6RÂfÇ6RÂÂ’“° —ÒVÇ6R° ’òòöæÇ’6VæBFò÷F†W"&÷†–W2–bF†RÆ–W"D”BäõBÇ&VG’&V6V—fR&Wv&Böâ ’òò&6¶Væ@ –&ööÆVâ6†÷VÆE6VæBÒG'VS° ––b‡Æ–W$öæÆ–æRbbÆ–W%6W'fW"ÒçVÆÂ’° ––b‚vWD6öæf–r‚’ævWD&Æö6¶VE6W'fW'2‚’æ6öçF–ç2‡Æ–W%6W'fW"’’° —6†÷VÆE6VæBÒfÇ6S° —Р—Р ––b‡6†÷VÆE6VæB’° –FV'Vr‚%6VæF–ærvÆö&Â&÷‡’f÷FVöæÆ–æRVçfVÆ÷R"“° –×VÇF•&÷‡”†æFÆW  ’ç6VæD×VÇF•&÷‡”VçfVÆ÷R…f÷F–æuÇVv–åv—&Rçf÷FTöæÆ–æR‡Æ–W"ÂWV–BÂ6W'f–6RÂF–ÖRÂfÇ6RÀ —&VÅf÷FRÂFW‡BÓÒçVÆÂò""¢FW‡BçFõ7G&–ær‚’Âf÷FT–BÂfÇ6RÂfÇ6RÂÂ’“° —ÒVÇ6R° –FV'Vr‚$æ÷B6VæF–ærvÆö&Â&÷‡’ÖW76vRf÷"f÷FVöæÆ–æRÂÆ–W"Ç&VG’v÷B&Wv&B"“° —Р—Р—Р––b‡VWVVEf÷FRÒçVÆÂ’° —VWVVEf÷FRç6WE&ö6W76VB‡G'VR“° ––b‚vWEf÷FT66†T†æFÆW"‚’çWFFUF–ÖUf÷FR‡VWVVEf÷FR’’° —v&â‚%Væ&ÆRFòW'6—7B6ö×ÆWFVB&öÆÆ÷fW"f÷FR"²VWVVEf÷FRævWEf÷FT–B‚ ’²#²GFV×F–ærGW&&ÆR&VÖ÷f–ÖÖVF–FVÇ’"“° —Р—Р—&WGW&âVWVVEf÷FU&W7VÇBå5T44U53° —Ò6F6‚„W†6WF–öâR’° –Rç&–çE7F6µG&6R‚“° —&WGW&âVWVVEf÷FU&W7VÇBå$UE%“° —Р—Р —&—fFR7FF–2f–æÂ6Æ72VæF–æu&W6Væ6T†æFöfb° —&—fFRUT”B&WVW7D–C° —&—fFRf–æÂUT”BÆ–W%WV–C° —&—fFRf–æÂ7G&–ærÆ–W$æÖS° —&—fFRf–æÂ7G&–ærWV–C° —&—fFRf–æÂ7G&–ær6W'fW#° —&—fFRf–æÂUT”B6öææV7F–öä–C° —&—fFRf–æÂUT”B&6¶VæD–æ6&æF–öä–C° —&—fFRf–æÂÆöær&6¶VæE7F'FVDC° —&—fFRf–æÂÆöær6öæfÆ–7E6WVVæ6S° —&—fFRf–æÂÆöær7&VFVDC°  —&—fFRVæF–æu&W6Væ6T†æFöfb…7G&–ærÆ–W$æÖRÂ7G&–ærWV–BÂ7G&–ær6W'fW"ÂUT”B6öææV7F–öä–BÀ •UT”B&6¶VæD–æ6&æF–öä–BÂÆöær&6¶VæE7F'FVDBÂÆöær6öæfÆ–7E6WVVæ6RÂÆöær7&VFVDB’° —F†—2çÆ–W%WV–BÒ'6UÆ–W%WV–B‡WV–B“° —F†—2çÆ–W$æÖRÒÆ–W$æÖS° —F†—2çWV–BÒWV–C° —F†—2ç6W'fW"Ò6W'fW#° —F†—2æ6öææV7F–öä–BÒ6öææV7F–öä–C° —F†—2æ&6¶VæD–æ6&æF–öä–BÒ&6¶VæD–æ6&æF–öä–C° —F†—2æ&6¶VæE7F'FVDBÒ&6¶VæE7F'FVDC° —F†—2æ6öæfÆ–7E6WVVæ6RÒ6öæfÆ–7E6WVVæ6S° —F†—2æ7&VFVDBÒ7&VFVDC° —Р —&—fFR7FF–2UT”B'6UÆ–W%WV–B…7G&–ærWV–B’° —G'’° —&WGW&âUT”Bæg&öÕ7G&–ær‡WV–BçG&–Ò‚’“° —Ò6F6‚„W†6WF–öâ–væ÷&VB’° —&WGW&âçVÆð —Р—Р—Р —V&Æ–2'7G&7Bfö–Bv&â…7G&–ærÖW76vR“°  —V&Æ–2'7G&7B66†VGVÆVDW†V7WF÷%6W'f–6RvWE66†VGVÆW"‚“°§Ð  \ No newline at end of file +package com.bencodez.votingplugin.proxy; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import javax.net.ssl.SSLParameters; + +import org.eclipse.paho.client.mqttv3.MqttException; + +import com.bencodez.advancedcore.api.time.TimeType; +import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalDataHandlerProxy; +import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalMySQL; +import com.bencodez.advancedcore.bungeeapi.time.BungeeTimeChecker; +import com.bencodez.simpleapi.encryption.EncryptionHandler; +import com.bencodez.simpleapi.json.JsonParser; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageListener; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler; +import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; +import com.bencodez.simpleapi.servercomm.mqtt.MqttServerComm; +import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; +import com.bencodez.simpleapi.servercomm.redis.RedisHandler; +import com.bencodez.simpleapi.servercomm.redis.RedisListener; +import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; +import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; +import com.bencodez.simpleapi.servercomm.sockets.SocketReceiver; +import com.bencodez.simpleapi.sql.Column; +import com.bencodez.simpleapi.sql.DataType; +import com.bencodez.simpleapi.sql.data.DataValue; +import com.bencodez.simpleapi.sql.data.DataValueBoolean; +import com.bencodez.simpleapi.sql.data.DataValueInt; +import com.bencodez.simpleapi.sql.data.DataValueString; +import com.bencodez.simpleapi.sql.mysql.config.MysqlConfig; +import com.bencodez.votingplugin.backendproxy.http.HttpEnrollmentAuthority; +import com.bencodez.votingplugin.backendproxy.http.HttpProxyTransportServer; +import com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity; +import com.bencodez.votingplugin.proxy.broadcast.ProxyBroadcastDecider; +import com.bencodez.votingplugin.proxy.cache.IVoteCache; +import com.bencodez.votingplugin.proxy.cache.VoteCacheHandler; +import com.bencodez.votingplugin.proxy.cache.nonvoted.INonVotedPlayersStorage; +import com.bencodez.votingplugin.proxy.cache.nonvoted.NonVotedPlayersCache; +import com.bencodez.votingplugin.proxy.control.ControlConnector; +import com.bencodez.votingplugin.proxy.control.HostedControlManager; +import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyHandler; +import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyMethod; +import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfiguration; +import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfigurationBungee; +import com.bencodez.votingplugin.proxy.presence.BackendPlayerPresenceTracker; +import com.bencodez.votingplugin.proxy.presence.PlayerPresence; +import com.bencodez.votingplugin.timequeue.VoteTimeQueue; +import com.bencodez.votingplugin.topvoter.TopVoter; +import com.bencodez.votingplugin.util.MinecraftUsernameValidator; +import com.bencodez.votingplugin.util.ServiceSiteValidator; +import com.bencodez.votingplugin.votelog.VoteLogMysqlTable; +import com.bencodez.votingplugin.votelog.VoteLogMysqlTable.VoteLogStatus; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import redis.clients.jedis.DefaultJedisClientConfig; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import lombok.Getter; +import lombok.Setter; + +public abstract class VotingPluginProxy { + private static final long PRESENCE_HANDOFF_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(2); + private static final long PRESENCE_STARTUP_RESYNC_DELAY_SECONDS = 5L; + private static final long PRESENCE_MAINTENANCE_INTERVAL_SECONDS = 30L; + private static final long PRESENCE_BACKEND_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(90); + private static final long CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10); + + @Getter + @Setter + private int votePartyVotes = 0; + + @Getter + @Setter + private int currentVotePartyVotesRequired = 0; + + @Getter + @Setter + private ProxyMysqlUserTable proxyMySQL; + + private EncryptionHandler encryptionHandler; + + private HashMap clientHandles; + + private SocketHandler socketHandler; + private HttpProxyTransportServer httpTransportServer; + private HttpEnrollmentAuthority httpEnrollmentAuthority; + + @Getter + @Setter + private boolean votifierEnabled = true; + + @Getter + private ConcurrentHashMap uuidPlayerNameCache = new ConcurrentHashMap<>(); + + @Getter + @Setter + private GlobalDataHandlerProxy globalDataHandler; + + @Getter + private RedisHandler redisHandler; + private JedisPool redisPublisherPool; + private volatile long redisPublisherRetryAfter; + private boolean timeVoteRetryScheduled; + private boolean timeVoteDeliveryRetryScheduled; + private boolean cachedVoteDeliveryRetryScheduled; + + private boolean enabled; + + @Getter + @Setter + private MultiProxyHandler multiProxyHandler; + + @Getter + private BungeeTimeChecker bungeeTimeChecker; + + @Getter + @Setter + private BungeeMethod method; + + @Getter + private MqttHandler mqttHandler; + + @Getter + private GlobalMessageProxyHandler globalMessageProxyHandler; + + @Getter + @Setter + private MySqlMessenger proxyMysqlMessenger; + + @Getter + private VoteCacheHandler voteCacheHandler; + + @Getter + private NonVotedPlayersCache nonVotedPlayersCache; + + @Getter + private final BackendPlayerPresenceTracker backendPlayerPresenceTracker = new BackendPlayerPresenceTracker(); + private final Map pendingPresenceHandoffs = new HashMap<>(); + private final Set pendingBackendRecoverySnapshots = ConcurrentHashMap.newKeySet(); + private final Map controlEnrollmentNextAllowed = new ConcurrentHashMap<>(); + private final Map pendingCommunicationTests = new ConcurrentHashMap<>(); + private volatile ControlConnector controlConnector; + private volatile HostedControlManager hostedControlManager; + private final Object controlLifecycleLock = new Object(); + private final AtomicLong controlServicesGeneration = new AtomicLong(); + private final ExecutorService controlLifecycleExecutor = Executors.newSingleThreadExecutor(task -> { + Thread thread = new Thread(task, "votingplugin-control-lifecycle"); + thread.setDaemon(true); + return thread; + }); + + public VotingPluginProxy() { + enabled = true; + + bungeeTimeChecker = new BungeeTimeChecker(getConfig().getTimeZone(), getConfig().getTimeHourOffSet(), + getConfig().getTimeWeekOffSet()) { + + @Override + public void debug(String text) { + debug2(text); + } + + @Override + public long getLastUpdated() { + return getVoteCacheLastUpdated(); + } + + @Override + public int getPrevDay() { + return getVoteCachePrevDay(); + } + + @Override + public String getPrevMonth() { + return getVoteCachePrevMonth(); + } + + @Override + public int getPrevWeek() { + return getVoteCachePrevWeek(); + } + + @Override + public void info(String text) { + log(text); + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + public boolean isIgnoreTime() { + return isVoteCacheIgnoreTime(); + } + + @Override + public void setIgnoreTime(boolean ignore) { + setVoteCacheVoteCacheIgnoreTime(ignore); + } + + @Override + public void setLastUpdated() { + setVoteCacheLastUpdated(); + } + + @Override + public void setPrevDay(int day) { + setVoteCachePrevDay(day); + } + + @Override + public void setPrevMonth(String text) { + setVoteCachePrevMonth(text); + } + + @Override + public void setPrevWeek(int week) { + setVoteCachePrevWeek(week); + } + + @Override + public void timeChanged(TimeType type, boolean fake, boolean pre, boolean post) { + if (getConfig().getVoteCacheTime() > 0) { + getVoteCacheHandler().checkVoteCacheTime(getConfig().getVoteCacheTime()); + } + if (!getConfig().getGlobalDataEnabled()) { + warn("Global data not enabled, ignoring time change event"); + return; + } + int delay = 1; + for (String s : getAllAvailableServers()) { + if (getGlobalDataHandler().getGlobalMysql().containsKey(s)) { + String lastOnlineStr = getGlobalDataHandler().getString(s, "LastOnline"); + long lastOnline = 0; + try { + lastOnline = Long.valueOf(lastOnlineStr); + } catch (NumberFormatException e) { + // ignore + } + + if (LocalDateTime.now().atZone(ZoneOffset.UTC).toInstant().toEpochMilli() - lastOnline < 1000 + * 60 * 60 * 12) { + HashMap dataToSet = new HashMap<>(); + dataToSet.put("LastUpdated", new DataValueString( + "" + LocalDateTime.now().atZone(ZoneOffset.UTC).toInstant().toEpochMilli())); + dataToSet.put("FinishedProcessing", new DataValueBoolean(false)); + dataToSet.put(type.toString(), new DataValueBoolean(true)); + getGlobalDataHandler().setData(s, dataToSet); + + globalMessageProxyHandler.sendMessage(s, delay, VotingPluginWire.bungeeTimeChange()); + delay++; + } else { + warn("Server " + s + " hasn't been online recently"); + } + } else { + warn("Server " + s + " global data handler disabled?"); + } + } + globalDataHandler.onTimeChange(type); + } + + @Override + public void warning(String text) { + warn(text); + } + }; + } + + public void onTimeChangedFailed(String srv, TimeType type) { + getGlobalDataHandler().setBoolean(srv, type.toString(), false); + getGlobalDataHandler().setBoolean(srv, "FinishedProcessing", true); + getGlobalDataHandler().setBoolean(srv, "Processing", false); + } + + public void onTimeChangedFinished(TimeType type) { + if (type.equals(TimeType.MONTH)) { + getProxyMySQL().copyColumnData(TopVoter.Monthly.getColumnName(), "LastMonthTotal"); + } + getProxyMySQL().wipeColumnData(TopVoter.of(type).getColumnName(), DataType.INTEGER); + + if (!getConfig().getGlobalDataEnabled()) { + return; + } + for (String s : getAllAvailableServers()) { + getGlobalDataHandler().setBoolean(s, "ForceUpdate", true); + getGlobalMessageProxyHandler().sendMessage(s, 1, VotingPluginWire.bungeeTimeChange()); + } + processQueue(); + } + + /** + * Load MySQL + global data handler. + */ + public void loadMysql(MysqlConfig mysqlConfig, MysqlConfig globalDataMysqlConfig) { + if (mysqlConfig.getHostName().isEmpty() || mysqlConfig.getDatabase().isEmpty()) { + logSevere("MySQL is not configured correctly. " + "Missing host/database. host=" + mysqlConfig.getHostName() + + " db=" + mysqlConfig.getDatabase()); + setProxyMySQL(null); + return; + } + + setProxyMySQL(new ProxyMysqlUserTable("VotingPlugin_Users", mysqlConfig, getConfig().getDebug()) { + + @Override + public void debug(SQLException e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void logSevere(String string) { + VotingPluginProxy.this.logSevere(string); + } + + @Override + public void logInfo(String string) { + VotingPluginProxy.this.logInfo(string); + } + + @Override + public void debug(Throwable t) { + if (getConfig().getDebug()) { + t.printStackTrace(); + } + } + + @Override + public void debug(String str) { + debug2(str); + } + }); + + ArrayList servers = new ArrayList(getAllAvailableServers()); + + if (getConfig().getGlobalDataEnabled()) { + if (getConfig().getGlobalDataUseMainMySQL()) { + setGlobalDataHandler(new GlobalDataHandlerProxy( + new GlobalMySQL("VotingPlugin_GlobalData", getProxyMySQL().getMysql()) { + + @Override + public void debugEx(Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void debugLog(String text) { + debug(text); + } + + @Override + public void info(String text) { + logInfo(text); + } + + @Override + public void logSevere(String text) { + VotingPluginProxy.this.logSevere(text); + } + + @Override + public void warning(String text) { + warn(text); + } + }, servers) { + + @Override + public void onTimeChangedFailed(String srv, TimeType type) { + VotingPluginProxy.this.onTimeChangedFailed(srv, type); + } + + @Override + public void onTimeChangedFinished(TimeType type) { + VotingPluginProxy.this.onTimeChangedFinished(type); + } + }); + } else { + setGlobalDataHandler( + new GlobalDataHandlerProxy(new GlobalMySQL("VotingPlugin_GlobalData", globalDataMysqlConfig) { + + @Override + public void debugEx(Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void debugLog(String text) { + debug(text); + } + + @Override + public void info(String text) { + logInfo(text); + } + + @Override + public void logSevere(String text) { + VotingPluginProxy.this.logSevere(text); + } + + @Override + public void warning(String text) { + warn(text); + } + }, servers) { + + @Override + public void onTimeChangedFailed(String srv, TimeType type) { + VotingPluginProxy.this.onTimeChangedFailed(srv, type); + } + + @Override + public void onTimeChangedFinished(TimeType type) { + VotingPluginProxy.this.onTimeChangedFinished(type); + } + }); + } + + // update global schema columns (unchanged from original) + getGlobalDataHandler().getGlobalMysql().alterColumnType("IgnoreTime", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("MONTH", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("WEEK", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("DAY", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("FinishedProcessing", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("Processing", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("ForceUpdate", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("LastUpdated", "MEDIUMTEXT"); + } + + // column types (unchanged from original) + getProxyMySQL().alterColumnType("TopVoterIgnore", "VARCHAR(5)"); + getProxyMySQL().alterColumnType("CheckWorld", "VARCHAR(5)"); + getProxyMySQL().alterColumnType("Reminded", "VARCHAR(5)"); + getProxyMySQL().alterColumnType("DisableBroadcast", "VARCHAR(5)"); + getProxyMySQL().alterColumnType("LastOnline", "VARCHAR(20)"); + getProxyMySQL().alterColumnType("PlayerName", "VARCHAR(30)"); + getProxyMySQL().alterColumnType("DailyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("WeeklyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("DayVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("BestDayVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("WeekVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("BestWeekVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("VotePartyVotes", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("MonthVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("Points", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("HighestDailyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("AllTimeTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("HighestMonthlyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("MonthTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("HighestWeeklyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("LastMonthTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("LastWeeklyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("LastDailyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("OfflineRewards", "MEDIUMTEXT"); + getProxyMySQL().alterColumnType("DayVoteStreakLastUpdate", "MEDIUMTEXT"); + + if (getConfig().getStoreMonthTotalsWithDate()) { + getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now()), "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now().plusMonths(1)), + "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now().plusMonths(2)), + "INT DEFAULT '0'"); + } + } + + public void addCurrentVotePartyVotes(int amount) { + votePartyVotes += amount; + setVoteCacheVotePartyCurrentVotes(votePartyVotes); + debug("Current vote party total: " + votePartyVotes); + } + + public void addNonVotedPlayer(String uuid, String playerName) { + nonVotedPlayersCache.addPlayer(uuid, playerName); + } + + public void addVoteParty() { + if (getConfig().getVotePartyEnabled()) { + addCurrentVotePartyVotes(1); + checkVoteParty(); + } + } + + public abstract void broadcast(String message); + + private Set sendProxyBroadcast(Set targets, String uuid, String player, String service, long time, + String text, boolean wasOnline) { + Set forwarded = new LinkedHashSet<>(); + for (String targetServer : targets) { + JsonEnvelope envelope = VotingPluginWire.voteBroadcast(uuid, player, service, time, text, wasOnline); + if (sendProxyBroadcastEnvelopeNow(targetServer, envelope)) { + forwarded.add(targetServer); + } + } + return forwarded; + } + + /** + * Sends a standalone proxy broadcast through the selected transport and reports + * whether that transport accepted the message. + * + * @param server target backend server + * @param envelope standalone broadcast envelope + * @return true only when the transport accepted the message + */ + protected boolean sendProxyBroadcastEnvelopeNow(String server, JsonEnvelope envelope) { + switch (method) { + case MQTT: + return sendMqttEnvelopeServer(server, envelope); + case MYSQL: + if (proxyMysqlMessenger == null) { + return false; + } + try { + proxyMysqlMessenger.sendToBackend(server, envelope); + return true; + } catch (SQLException e) { + debug(e.getMessage()); + return false; + } + case PLUGINMESSAGING: + return sendPluginMessageServerNow(server, envelope); + case REDIS: + return sendRedisEnvelopeServer(server, envelope, true); + case SOCKETS: + // Standalone broadcasts use the same initialized client as normal + // envelopes. This preserves the socket connection and its delivery + // acknowledgement instead of creating a second short-lived socket. + return sendSocketEnvelope(server, envelope); + case HTTP: + return sendHttpEnvelope(server, envelope); + default: + return false; + } + } + + /** + * Sends a reward-bearing vote envelope and reports whether the selected + * transport accepted it. Legacy transports retain their existing asynchronous + * semantics; HTTP exposes its bounded-queue result so a vote is never discarded + * when the queue is full. + */ + protected boolean sendVoteEnvelopeAccepted(String server, int delay, JsonEnvelope envelope) { + if (method == BungeeMethod.HTTP) { + return sendHttpEnvelope(server, envelope); + } + GlobalMessageProxyHandler handler = globalMessageProxyHandler; + if (handler == null) { + return false; + } + handler.sendMessage(server, delay, envelope); + return true; + } + + public synchronized void checkCachedVotes(String server) { + int delay = 1; + if (isServerValid(server)) { + if (isSomeoneOnlineServerForVoteRouting(server)) { + if (getVoteCacheHandler().hasVotes(server) && !getConfig().getBlockedServers().contains(server)) { + ArrayList c = getVoteCacheHandler().getVotes(server); + ArrayList removed = new ArrayList<>(); + if (!c.isEmpty()) { + int num = 1; + int numberOfVotes = c.size(); + for (OfflineBungeeVote cache : c) { + if (cache.isDeliveryStateDirty() && !persistServerVoteDelivery(server, cache)) { + continue; + } + if (cache.isProxyBroadcastHandled() && cache.needsBroadcastOn(server)) { + Set forwarded = sendProxyBroadcast(Collections.singleton(server), + cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), + cache.getText(), false); + if (cache.getBroadcastForwardedServers().addAll(forwarded)) { + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + if (!persistServerVoteDelivery(server, cache)) { + continue; + } + } + } + + boolean toSend = true; + if (getConfig().getWaitForUserOnline()) { + if (!isPlayerOnlineForVoteRouting(cache.getPlayerName())) { + toSend = false; + } else if (isPlayerOnlineForVoteRouting(cache.getPlayerName()) + && !getCurrentPlayerServerForVoteRouting(cache.getPlayerName()).equals(server)) { + toSend = false; + } + } + if (toSend) { + boolean broadcastHere = cache.needsBroadcastOn(server); + if (!cache.isProxyBroadcastHandled() && broadcastHere + && getConfig().getProxyBroadcastEnabled()) { + boolean playerOnline = isPlayerOnlineForVoteRouting(cache.getPlayerName()); + String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(cache.getPlayerName()) + : null; + + Set targets = proxyBroadcastDecider.resolveTargets(playerOnline, + playerServer); + broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); + } + + if (!sendVoteEnvelopeAccepted(server, delay, + VotingPluginWire.vote(cache.getPlayerName(), cache.getUuid(), + cache.getService(), cache.getTime(), false, cache.isRealVote(), + cache.getText(), cache.getVoteId(), getConfig().getBungeeManageTotals(), + broadcastHere, num, numberOfVotes))) { + debug("Retaining cached vote because the transport rejected delivery for " + server); + continue; + } + delay++; + num++; + removed.add(cache); + } else { + debug("Not sending vote because user isn't on server " + server + ": " + + cache.toString()); + } + } + getVoteCacheHandler().removeServerVotes(server, removed); + } else { + debug("No cached votes for server: " + server); + } + } else { + debug("No cached votes for server: " + server); + } + } + } else { + debug("Server not valid: " + server); + } + } + + public synchronized void checkOnlineVotes(String player, String uuid, String server) { + int delay = 1; + if (isPlayerOnlineForVoteRouting(player) && getVoteCacheHandler().hasOnlineVotes(uuid)) { + ArrayList c = getVoteCacheHandler().getOnlineVotes(uuid); + if (!c.isEmpty()) { + if (server == null) { + server = getCurrentPlayerServerForVoteRouting(player); + } + if (!getConfig().getBlockedServers().contains(server)) { + int num = 1; + int numberOfVotes = (int) c.stream().filter(vote -> !vote.isRewardDelivered()).count(); + boolean deliveredReward = false; + ArrayList retained = new ArrayList<>(); + for (OfflineBungeeVote cache : c) { + if (cache.isProxyBroadcastHandled()) { + Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); + pendingTargets.removeAll(cache.getBroadcastForwardedServers()); + List blockedServers = getConfig().getBlockedServers(); + if (blockedServers != null) { + pendingTargets.removeAll(blockedServers); + } + cache.getBroadcastForwardedServers().addAll(sendProxyBroadcast(pendingTargets, + cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), + cache.getText(), false)); + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + } + boolean broadcastHere = cache.needsBroadcastOn(server); + if (!cache.isProxyBroadcastHandled() && broadcastHere + && getConfig().getProxyBroadcastEnabled()) { + String playerServer = (server != null) ? server : getCurrentPlayerServerForVoteRouting(player); + + Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); + broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); + } + + if (!cache.isRewardDelivered()) { + if (!sendVoteEnvelopeAccepted(server, delay, + VotingPluginWire.voteOnline(cache.getPlayerName(), cache.getUuid(), cache.getService(), + cache.getTime(), false, cache.isRealVote(), cache.getText(), cache.getVoteId(), + getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes))) { + debug("Retaining online vote because the transport rejected delivery for " + server); + retained.add(cache); + continue; + } + // The normal envelope is also a valid broadcast delivery for the + // current target. Record it so a previously pending standalone + // retry cannot announce the same vote again later. + if (cache.isProxyBroadcastHandled() && broadcastHere) { + cache.getBroadcastForwardedServers().add(server); + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + } + cache.setRewardDelivered(true); + deliveredReward = true; + delay++; + num++; + } + + if (cache.isProxyBroadcastHandled() && !cache.isProxyBroadcastComplete()) { + retained.add(cache); + } + } + getVoteCacheHandler().removeOnlineVotes(uuid); + for (OfflineBungeeVote pending : retained) { + getVoteCacheHandler().addOnlineVote(uuid, pending); + } + + // multiproxy: envelope-only + if (deliveredReward && getConfig().getMultiProxySupport() + && getConfig().getMultiProxyOneGlobalReward()) { + multiProxyHandler.sendClearVote(uuid, player); + } + } + } + } + } + + /** + * Retries voter-keyed standalone broadcasts when any player makes a target + * backend available as a plugin-message carrier. + * + * @param server backend server that gained a carrier + */ + protected synchronized void retryPendingOnlineBroadcasts(String server) { + List blockedServers = getConfig().getBlockedServers(); + if (server == null || (blockedServers != null && blockedServers.contains(server))) { + return; + } + for (String cachedUuid : getVoteCacheHandler().getOnlineVoteUUIDs()) { + for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { + if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { + continue; + } + if (!cache.isProxyBroadcastHandled() || !cache.needsBroadcastOn(server)) { + continue; + } + Set forwarded = sendProxyBroadcast(Collections.singleton(server), cache.getUuid(), + cache.getPlayerName(), cache.getService(), cache.getTime(), cache.getText(), false); + if (cache.getBroadcastForwardedServers().addAll(forwarded)) { + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { + getVoteCacheHandler().removeOnlineVote(cachedUuid, cache); + } else { + persistOnlineVoteDelivery(cachedUuid, cache); + } + } + } + } + } + + protected synchronized void retryPendingTimeBroadcasts(String server) { + List blockedServers = getConfig().getBlockedServers(); + if (server == null || (blockedServers != null && blockedServers.contains(server))) { + return; + } + if (getVoteCacheHandler().getTimeChangeQueue() == null) { + return; + } + for (VoteTimeQueue vote : new ArrayList<>(getVoteCacheHandler().getTimeChangeQueue())) { + if (vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { + continue; + } + if (!vote.isProxyBroadcastHandled() || vote.getUuid().isEmpty() || !vote.getBroadcastTargets().contains(server) + || vote.getBroadcastForwardedServers().contains(server)) { + continue; + } + Set forwarded = sendProxyBroadcast(Collections.singleton(server), vote.getUuid(), vote.getName(), + vote.getService(), vote.getTime(), vote.getTotals(), false); + if (vote.getBroadcastForwardedServers().addAll(forwarded)) { + persistTimeVoteDelivery(vote); + } + } + } + + /** + * Periodically retries every pending voter-keyed standalone broadcast. This is + * required for broker transports whose recovery does not produce a player-login + * carrier event. + */ + public synchronized void retryPendingOnlineBroadcasts() { + for (String cachedUuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { + for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { + if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { + continue; + } + if (!cache.isProxyBroadcastHandled() || cache.isProxyBroadcastComplete()) { + continue; + } + Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); + pendingTargets.removeAll(cache.getBroadcastForwardedServers()); + List blockedServers = getConfig().getBlockedServers(); + if (blockedServers != null) { + pendingTargets.removeAll(blockedServers); + } + Set forwarded = sendProxyBroadcast(pendingTargets, cache.getUuid(), cache.getPlayerName(), + cache.getService(), cache.getTime(), cache.getText(), false); + if (cache.getBroadcastForwardedServers().addAll(forwarded)) { + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { + getVoteCacheHandler().removeOnlineVote(cachedUuid, cache); + } else { + persistOnlineVoteDelivery(cachedUuid, cache); + } + } + } + } + retryPendingTimeBroadcasts(); + } + + public synchronized void retryPendingTimeBroadcasts() { + if (getVoteCacheHandler().getTimeChangeQueue() == null) { + return; + } + for (VoteTimeQueue vote : new ArrayList<>(getVoteCacheHandler().getTimeChangeQueue())) { + if (vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { + continue; + } + if (!vote.isProxyBroadcastHandled() || vote.getUuid().isEmpty()) { + continue; + } + Set pendingTargets = new LinkedHashSet<>(vote.getBroadcastTargets()); + pendingTargets.removeAll(vote.getBroadcastForwardedServers()); + List blockedServers = getConfig().getBlockedServers(); + if (blockedServers != null) { + pendingTargets.removeAll(blockedServers); + } + Set forwarded = sendProxyBroadcast(pendingTargets, vote.getUuid(), vote.getName(), vote.getService(), + vote.getTime(), vote.getTotals(), false); + if (vote.getBroadcastForwardedServers().addAll(forwarded)) { + persistTimeVoteDelivery(vote); + } + } + } + + protected synchronized boolean persistTimeVoteDelivery(VoteTimeQueue vote) { + if (getVoteCacheHandler().updateTimeVote(vote)) { + vote.setDeliveryStateDirty(false); + return true; + } + vote.setDeliveryStateDirty(true); + scheduleTimeVoteDeliveryRetry(); + return false; + } + + private void scheduleTimeVoteDeliveryRetry() { + if (timeVoteDeliveryRetryScheduled || getScheduler() == null) { + return; + } + timeVoteDeliveryRetryScheduled = true; + try { + getScheduler().schedule(() -> { + synchronized (VotingPluginProxy.this) { + timeVoteDeliveryRetryScheduled = false; + } + retryPendingTimeBroadcasts(); + }, 5, TimeUnit.SECONDS); + } catch (RuntimeException e) { + timeVoteDeliveryRetryScheduled = false; + debug("Unable to schedule timed broadcast state retry: " + e.getMessage()); + } + } + + protected synchronized boolean persistServerVoteDelivery(String server, OfflineBungeeVote vote) { + if (getVoteCacheHandler().updateServerVote(server, vote)) { + vote.setDeliveryStateDirty(false); + return true; + } + vote.setDeliveryStateDirty(true); + scheduleCachedVoteDeliveryRetry(); + return false; + } + + protected synchronized boolean persistOnlineVoteDelivery(String uuid, OfflineBungeeVote vote) { + if (getVoteCacheHandler().updateOnlineVote(uuid, vote)) { + vote.setDeliveryStateDirty(false); + return true; + } + vote.setDeliveryStateDirty(true); + scheduleCachedVoteDeliveryRetry(); + return false; + } + + private void scheduleCachedVoteDeliveryRetry() { + if (cachedVoteDeliveryRetryScheduled || getScheduler() == null) { + return; + } + cachedVoteDeliveryRetryScheduled = true; + try { + getScheduler().schedule(() -> { + synchronized (VotingPluginProxy.this) { + cachedVoteDeliveryRetryScheduled = false; + } + retryCachedVoteDeliveryPersistence(); + }, 5, TimeUnit.SECONDS); + } catch (RuntimeException e) { + cachedVoteDeliveryRetryScheduled = false; + debug("Unable to schedule cached broadcast state retry: " + e.getMessage()); + } + } + + private synchronized void retryCachedVoteDeliveryPersistence() { + for (String server : getVoteCacheHandler().getCachedVotesServers()) { + for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getVotes(server))) { + if (vote.isDeliveryStateDirty()) { + persistServerVoteDelivery(server, vote); + } + } + } + for (String uuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { + for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(uuid))) { + if (vote.isDeliveryStateDirty()) { + persistOnlineVoteDelivery(uuid, vote); + } + } + } + } + + public void checkVoteParty() { + if (getConfig().getVotePartyEnabled()) { + if (votePartyVotes >= currentVotePartyVotesRequired) { + debug("Vote party reached"); + addCurrentVotePartyVotes(-currentVotePartyVotesRequired); + + currentVotePartyVotesRequired += getConfig().getVotePartyIncreaseVotesRequired(); + setVoteCacheVotePartyIncreaseVotesRequired( + getVoteCacheVotePartyIncreaseVotesRequired() + getConfig().getVotePartyIncreaseVotesRequired()); + + if (!getConfig().getVotePartyBroadcast().isEmpty()) { + broadcast(getConfig().getVotePartyBroadcast()); + } + + for (String command : getConfig().getVotePartyBungeeCommands()) { + runConsoleCommand(command); + } + + if (getConfig().getVotePartySendToAllServers()) { + for (String server : getAllAvailableServers()) { + sendVoteParty(server); + } + } else { + for (String server : getConfig().getVotePartyServersToSend()) { + sendVoteParty(server); + } + } + } + saveVoteCacheFile(); + } + } + + public abstract void debug(String str); + + private void debug2(String message) { + debug(message); + } + + /** + * HTTP client used for Mojang API requests. + */ + private final HttpClient httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); + + /** + * Fetches a player's UUID from the Mojang API. + * + * @param playerName player name + * @return player UUID, or {@code null} if not found + * @throws IOException if the request fails + * @throws InterruptedException if interrupted while waiting for the response + */ + public UUID fetchUUID(String playerName) throws IOException, InterruptedException { + if (playerName == null || playerName.equalsIgnoreCase("null")) { + return null; + } + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create("https://api.mojang.com/users/profiles/minecraft/" + playerName)).GET() + .timeout(Duration.ofSeconds(5)).build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() == 400 || response.statusCode() == 404) { + log("There is no player with the name \"" + playerName + "\"!"); + return null; + } + + if (response.statusCode() < 200 || response.statusCode() >= 300) { + throw new IOException("Failed to fetch UUID for " + playerName + ", HTTP " + response.statusCode()); + } + + JsonElement element = JsonParser.parseString(response.body()); + if (element == null || !element.isJsonObject()) { + return null; + } + + JsonObject object = element.getAsJsonObject(); + if (!object.has("id") || object.get("id").isJsonNull()) { + return null; + } + + String uuidAsString = object.get("id").getAsString(); + return parseUUIDFromString(uuidAsString); + } + + public abstract Set getAllAvailableServers(); + + /** Complete platform server set before whitelist/blocked routing filters. */ + public abstract Set getAllConfiguredServers(); + + public abstract VotingPluginProxyConfig getConfig(); + + public abstract String getCurrentPlayerServer(String player); + + /** + * Resolves a player's server for vote routing. A dedicated voting proxy has no + * local players, so it uses the backend presence tracker instead. + */ + protected String getCurrentPlayerServerForVoteRouting(String player) { + if (isDedicatedVotingProxyEnabled()) { + return backendPlayerPresenceTracker.getPlayer(player).map(presence -> presence.getServer()).orElse(null); + } + return getCurrentPlayerServer(player); + } + + /** + * Dedicated routing is intentionally unavailable on plugin messaging: that + * transport is attached to a player-facing proxy and does not carry backend + * presence snapshots. + */ + protected boolean isDedicatedVotingProxyEnabled() { + return getConfig().getDedicatedVotingProxy() && method != null && method.supportsBackendPresence(); + } + + public abstract File getDataFolderPlugin(); + + public String getMonthTotalsWithDatePath() { + LocalDateTime cTime = getBungeeTimeChecker().getTime(); + return getMonthTotalsWithDatePath(cTime); + } + + public String getMonthTotalsWithDatePath(LocalDateTime cTime) { + return "MonthTotal-" + cTime.getMonth().toString() + "-" + cTime.getYear(); + } + + public abstract String getProperName(String uuid, String playerName); + + public abstract String getUUID(String playerName); + + private int getValue(ArrayList cols, String column, int toAdd) { + for (Column d : cols) { + if (d.getName().equalsIgnoreCase(column)) { + DataValue value = d.getValue(); + int num = 0; + if (value == null) { + return toAdd; + } + if (value.isInt()) { + num = value.getInt(); + } else if (value.isString()) { + try { + num = Integer.parseInt(value.getString()); + } catch (Exception e) { + // ignore + } + } + return num + toAdd; + } + } + return toAdd; + } + + private VoteTotalsSnapshot getProjectedRolloverTotals(ArrayList data, String player) { + List timeChanges = getGlobalDataHandler().getTimeChanges(); + boolean resetMonth = timeChanges.contains(TimeType.MONTH); + boolean resetWeek = timeChanges.contains(TimeType.WEEK); + boolean resetDay = timeChanges.contains(TimeType.DAY); + int acceptedQueuedVotes = 0; + int acceptedGlobalQueuedVotes = 0; + for (VoteTimeQueue queued : getVoteCacheHandler().getTimeChangeQueue()) { + if (!queued.isProcessed()) { + acceptedGlobalQueuedVotes++; + } + if (!queued.isProcessed() && queued.getName() != null && queued.getName().equalsIgnoreCase(player)) { + acceptedQueuedVotes++; + } + } + int voteIncrement = acceptedQueuedVotes + 1; + + int allTimeTotal = getValue(data, "AllTimeTotal", voteIncrement); + int monthTotal = resetMonth ? voteIncrement : getValue(data, "MonthTotal", voteIncrement); + int weeklyTotal = resetWeek ? voteIncrement : getValue(data, "WeeklyTotal", voteIncrement); + int dailyTotal = resetDay ? voteIncrement : getValue(data, "DailyTotal", voteIncrement); + int points = getValue(data, "Points", voteIncrement * getConfig().getPointsOnVote()); + + int maxVotes = getConfig().getMaxAmountOfVotesPerDay(); + if (maxVotes > 0) { + int days = getBungeeTimeChecker().getTime().getDayOfMonth(); + if (monthTotal > days * maxVotes) { + monthTotal = days * maxVotes; + } + } + if (getConfig().getLimitVotePoints() > 0 && points > getConfig().getLimitVotePoints()) { + points = getConfig().getLimitVotePoints(); + } + + int dateMonthTotal = -1; + if (getConfig().getStoreMonthTotalsWithDate()) { + if (getConfig().getUseMonthDateTotalsAsPrimaryTotal()) { + dateMonthTotal = resetMonth ? voteIncrement + : getValue(data, getMonthTotalsWithDatePath(), voteIncrement); + } else { + dateMonthTotal = monthTotal; + } + } + + int[] projectedVoteParty = getProjectedVotePartyState(acceptedGlobalQueuedVotes + 1); + return new VoteTotalsSnapshot(allTimeTotal, monthTotal, weeklyTotal, dailyTotal, points, + projectedVoteParty[0], projectedVoteParty[1], dateMonthTotal); + } + + protected boolean canForwardStandaloneBroadcast(boolean managesTotals) { + return managesTotals; + } + + protected int[] getProjectedVotePartyState(int acceptedVotes) { + int current = votePartyVotes; + int required = currentVotePartyVotesRequired; + if (!getConfig().getVotePartyEnabled()) { + return new int[] { current, required }; + } + + int increase = getConfig().getVotePartyIncreaseVotesRequired(); + for (int i = 0; i < acceptedVotes; i++) { + current++; + if (current >= required) { + current -= required; + required += increase; + } + } + return new int[] { current, required }; + } + + public abstract String getPluginVersion(); + + public abstract int getVoteCacheCurrentVotePartyVotes(); + + public abstract long getVoteCacheLastUpdated(); + + public abstract int getVoteCachePrevDay(); + + public abstract String getVoteCachePrevMonth(); + + public abstract int getVoteCachePrevWeek(); + + public abstract int getVoteCacheVotePartyIncreaseVotesRequired(); + + public abstract boolean isPlayerOnline(String playerName); + + /** + * Checks online state for vote routing, using backend presence only when this + * proxy is explicitly configured as the dedicated voting proxy. + */ + protected boolean isPlayerOnlineForVoteRouting(String playerName) { + return isDedicatedVotingProxyEnabled() ? backendPlayerPresenceTracker.getPlayer(playerName).isPresent() + : isPlayerOnline(playerName); + } + + public abstract boolean isServerValid(String server); + + public abstract boolean isSomeoneOnlineServer(String server); + + protected boolean isSomeoneOnlineServerForVoteRouting(String server) { + if (!isDedicatedVotingProxyEnabled()) { + return isSomeoneOnlineServer(server); + } + com.bencodez.votingplugin.proxy.presence.BackendPresenceStatus status = backendPlayerPresenceTracker + .getBackendStatus(server); + return status != null && status.isAvailable() && status.getPlayerCount() > 0; + } + + public abstract boolean isVoteCacheIgnoreTime(); + + public abstract MysqlConfig getVoteCacheMySQLConfig(); + + public abstract MysqlConfig getNonVotedCacheMySQLConfig(); + + public abstract MysqlConfig getVoteLoggingMySQLConfig(); + + /** + * Shutdown MySQL-related resources safely. + */ + public void shutdownMySql() { + if (getProxyMysqlMessenger() != null) { + getProxyMysqlMessenger().shutdown(); + setProxyMysqlMessenger(null); + } + + if (getProxyMySQL() != null) { + getProxyMySQL().shutdown(); + setProxyMySQL(null); + } + } + + public void load(IVoteCache jsonStorage, INonVotedPlayersStorage nonVotedCacheJson) { + method = BungeeMethod.getByName(getConfig().getBungeeMethod()); + if (getMethod() == null) { + method = BungeeMethod.PLUGINMESSAGING; + } + warnUnsupportedDedicatedVotingProxyMode(); + uuidPlayerNameCache = getProxyMySQL().getRowsUUIDNameQuery(); + + bungeeTimeChecker.setTimeChangeFailSafeBypass(getConfig().getTimeChangeFailSafeBypass()); + bungeeTimeChecker.loadTimer(); + + voteCacheHandler = new VoteCacheHandler(getVoteCacheMySQLConfig(), getConfig().getVoteCacheUseMySQL(), + getConfig().getVoteCacheUseMainMySQL(), getProxyMySQL().getMysql(), getConfig().getDebug(), + jsonStorage) { + + @Override + public void logInfo1(String msg) { + logInfo(msg); + } + + @Override + public void logSevere1(String msg) { + logSevere(msg); + } + + @Override + public void debug1(Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void debug1(String msg) { + if (getConfig().getDebug()) { + debug(msg); + } + } + + @Override + public void debug1(Throwable e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + }; + voteCacheHandler.load(); + + nonVotedPlayersCache = new NonVotedPlayersCache(getNonVotedCacheMySQLConfig(), + getConfig().getNonVotedCacheUseMySQL(), getConfig().getNonVotedCacheUseMainMySQL(), + getProxyMySQL().getMysql(), nonVotedCacheJson, getConfig().getDebug()) { + + @Override + public boolean userExists(String uuid) { + return getProxyMySQL().containsKeyQuery(uuid); + } + + @Override + public void logInfo1(String msg) { + logInfo(msg); + } + + @Override + public void logSevere1(String msg) { + logSevere(msg); + } + + @Override + public void debug1(Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void debug1(String msg) { + if (getConfig().getDebug()) { + debug(msg); + } + } + + @Override + public Set getAllUUIDs() { + return getProxyMySQL().getUuids(); + } + }; + + if (method.equals(BungeeMethod.MYSQL)) { + try { + proxyMysqlMessenger = new MySqlMessenger("VotingPlugin", + getProxyMySQL().getMysql().getConnectionManager().getDataSource(), MySqlMessenger.Mode.PROXY, + null, // no serverId in PROXY mode + msg -> { + if (getConfig().getDebug()) { + debug("Got from " + msg.source + ": " + msg.envelope.getSubChannel() + " " + + msg.envelope.getFields()); + } + globalMessageProxyHandler.onMessage(msg.envelope); + }); + } catch (SQLException e) { + e.printStackTrace(); + } + } else if (method.equals(BungeeMethod.PLUGINMESSAGING)) { + if (getConfig().getPluginMessageEncryption()) { + encryptionHandler = new EncryptionHandler("VotingPlugin", + new File(getDataFolderPlugin(), "secretkey.key")); + } + } else if (method.equals(BungeeMethod.SOCKETS)) { + encryptionHandler = new EncryptionHandler("VotingPlugin", new File(getDataFolderPlugin(), "secretkey.key")); + + socketHandler = new SocketHandler(getPluginVersion(), getConfig().getBungeeHost(), + getConfig().getBungeePort(), encryptionHandler, getConfig().getDebug()) { + + @Override + public void log(String str) { + logInfo(str); + } + }; + + socketHandler.add(new SocketReceiver() { + @Override + public void onReceiveEnvelope(JsonEnvelope envelope) { + globalMessageProxyHandler.onMessage(envelope); + } + }); + + rebuildSocketClients(); + } else if (method.equals(BungeeMethod.REDIS)) { + redisHandler = new RedisHandler(getConfig().getRedisHost(), getConfig().getRedisPort(), + getConfig().getRedisUsername(), getConfig().getRedisPassword(), getConfig().getRedisDbIndex(), + getConfig().getRedisSsl()) { + + @Override + public void debug(String message) { + debug2(message); + } + }; + redisPublisherPool = new JedisPool(new HostAndPort(getConfig().getRedisHost(), getConfig().getRedisPort()), + buildRedisClientConfig(getConfig())); + + runAsync(() -> { + RedisListener listener = redisHandler.createEnvelopeListener( + getConfig().getRedisPrefix() + "VotingPlugin", + (ch, env) -> globalMessageProxyHandler.onMessage(env)); + redisHandler.loadListener(listener); + }); + + } else if (method.equals(BungeeMethod.MQTT)) { + try { + mqttHandler = new MqttHandler(new MqttServerComm(getConfig().getMqttClientID(), + getConfig().getMqttBrokerURL(), getConfig().getMqttUsername(), getConfig().getMqttPassword()), + 2); + + mqttHandler.subscribeEnvelopes(getConfig().getMqttPrefix() + "votingplugin/servers/proxy", + (topic, env) -> globalMessageProxyHandler.onMessage(env)); + + } catch (MqttException e) { + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + currentVotePartyVotesRequired = getConfig().getVotePartyVotesRequired() + + getVoteCacheVotePartyIncreaseVotesRequired(); + votePartyVotes = getVoteCacheCurrentVotePartyVotes(); + + globalMessageProxyHandler = new GlobalMessageProxyHandler() { + @Override + public void sendMessage(String server, int delay, JsonEnvelope envelope) { + switch (method) { + case MQTT: + sendMqttEnvelopeServer(server, envelope); + break; + case MYSQL: + try { + proxyMysqlMessenger.sendToBackend(server, envelope); + } catch (SQLException e) { + e.printStackTrace(); + } + break; + case PLUGINMESSAGING: + sendPluginMessageServer(server, delay, envelope); + break; + case REDIS: + sendRedisEnvelopeServer(server, envelope); + break; + case SOCKETS: + sendSocketEnvelope(server, envelope); + break; + case HTTP: + sendHttpEnvelope(server, envelope); + break; + default: + break; + } + } + }; + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_LOGIN) { + @Override + public void onReceive(JsonEnvelope message) { + handleLoginMessage(message); + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_LOGOUT) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(message); + if (!isPresenceServerValid(event.server, VotingPluginWire.SUB_LOGOUT) + || !isPresenceGenerationValid(event.backendIncarnationId, event.backendStartedAt, + event.presenceTimestamp, + VotingPluginWire.SUB_LOGOUT)) { + return; + } + if (!backendPlayerPresenceTracker.playerOffline(event.uuid, event.server, event.connectionId, + event.backendIncarnationId, event.backendStartedAt, event.presenceTimestamp, + System.currentTimeMillis())) { + debug("Ignored invalid or stale logout envelope: " + message.getFields()); + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_STARTED) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); + long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); + long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); + if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_STARTED) + && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, + VotingPluginWire.SUB_BACKEND_STARTED)) { + if (backendPlayerPresenceTracker.backendStarted(server, backendIncarnationId, backendStartedAt, + presenceTimestamp, System.currentTimeMillis())) { + discardPendingPresenceHandoffs(server); + pendingBackendRecoverySnapshots.add(presenceServerKey(server)); + requestBackendPresenceSnapshot(server); + } + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_STOPPED) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); + long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); + long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); + if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_STOPPED) + && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, + VotingPluginWire.SUB_BACKEND_STOPPED)) { + if (backendPlayerPresenceTracker.backendStopped(server, backendIncarnationId, backendStartedAt, + presenceTimestamp, System.currentTimeMillis())) { + discardPendingPresenceHandoffs(server); + pendingBackendRecoverySnapshots.remove(presenceServerKey(server)); + } + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_HEARTBEAT) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); + long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); + long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); + if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_HEARTBEAT) + && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, + VotingPluginWire.SUB_BACKEND_HEARTBEAT)) { + backendPlayerPresenceTracker.heartbeat(server, backendIncarnationId, backendStartedAt, + presenceTimestamp, System.currentTimeMillis()); + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_PRESENCE_SNAPSHOT) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + if (!isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_SNAPSHOT)) { + return; + } + VotingPluginWire.PresenceSnapshot snapshot = VotingPluginWire.readPresenceSnapshot(message); + long now = System.currentTimeMillis(); + boolean accepted = snapshot.valid + && isPresenceGenerationValid(snapshot.backendIncarnationId, snapshot.backendStartedAt, + snapshot.presenceTimestamp, + VotingPluginWire.SUB_PRESENCE_SNAPSHOT) + && backendPlayerPresenceTracker.applySnapshotChunk(snapshot.server, + snapshot.requestId, snapshot.chunkIndex, snapshot.chunkCount, snapshot.players, + snapshot.backendIncarnationId, snapshot.backendStartedAt, + snapshot.presenceTimestamp, now); + if (!accepted) { + debug("Ignored invalid or unexpected presence snapshot from " + snapshot.server); + if (backendPlayerPresenceTracker.getPendingSnapshotRequestId(snapshot.server, now) == null) { + discardPendingPresenceHandoffs(snapshot.requestId); + } + } else if (backendPlayerPresenceTracker.getPendingSnapshotRequestId(snapshot.server, now) == null) { + pendingBackendRecoverySnapshots.remove(presenceServerKey(snapshot.server)); + Set handoffPlayers = completePendingPresenceHandoffs(snapshot.requestId, snapshot.server, + snapshot.backendIncarnationId, snapshot.backendStartedAt, now); + processDedicatedSnapshotLogins(snapshot.server, handoffPlayers); + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_STATUS_OKAY) { + @Override + public void onReceive(JsonEnvelope message) { + handleStatusOkay(message); + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener("voteupdate") { + @Override + public void onReceive(JsonEnvelope message) { + int delay = 1; + for (String send : getAllAvailableServers()) { + globalMessageProxyHandler.sendMessage(send, delay, message); + delay++; + } + } + }); + + // Do not accept backend traffic until the router and all of its ordered + // presence/message listeners are installed. + if (method.equals(BungeeMethod.HTTP)) startHttpTransport(); + + proxyBroadcastDecider = new ProxyBroadcastDecider(() -> getConfig(), () -> getAllAvailableServers(), + s -> isServerValid(s), + s -> getConfig().getBlockedServers() != null && getConfig().getBlockedServers().contains(s)); + + loadMultiProxySupport(); + loadVoteLoggingMySQL(); + if (method.supportsBackendPresence()) { + scheduleBackendPresenceStartupResync(); + loadTaskTimer(this::maintainBackendPresence, PRESENCE_MAINTENANCE_INTERVAL_SECONDS, + PRESENCE_MAINTENANCE_INTERVAL_SECONDS); + } + startControlServices(); + + debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); + } + + private void startControlServices() { + synchronized (controlLifecycleLock) { + ControlConnector predecessor = controlConnector; + if (predecessor != null && predecessor.deferReplacementUntilSafe(this::restartControlServicesAsync)) { + log("[Control] service restart deferred until the current result is acknowledged"); + return; + } + stopControlServicesLocked(true); + startControlServicesLocked(); + } + } + + /** Keeps potentially long hosted-Control handoffs off proxy command/event threads. */ + private void restartControlServicesAsync() { + final long generation = controlServicesGeneration.incrementAndGet(); + try { + controlLifecycleExecutor.execute(() -> { + try { + synchronized (controlLifecycleLock) { + if (!enabled || generation != controlServicesGeneration.get()) return; + ControlConnector predecessor = controlConnector; + if (predecessor != null + && predecessor.deferReplacementUntilSafe(this::restartControlServicesAsync)) { + log("[Control] service restart deferred until the current result is acknowledged"); + return; + } + stopControlServicesLocked(true); + startControlServicesLocked(); + } + } catch (RuntimeException failure) { + if (generation == controlServicesGeneration.get()) { + logSevere("[Control] asynchronous service restart failed: " + failure.getMessage()); + } + } + }); + } catch (RuntimeException failure) { + logSevere("[Control] services were not restarted because async scheduling failed"); + } + } + + /** Rebuilds a recovery connector from current settings after its durable result is acknowledged. */ + public final void restartControlServicesAfterRecovery() { + restartControlServicesAsync(); + } + + private void stopControlServices(boolean waitForHosted) { + synchronized (controlLifecycleLock) { + stopControlServicesLocked(waitForHosted); + } + } + + private void startControlServicesLocked() { + if (getConfig().getControlHostedEnabled()) { + try { + hostedControlManager = HostedControlManager.create(this); + if (hostedControlManager != null) hostedControlManager.start(); + } catch (IOException | IllegalArgumentException e) { + hostedControlManager = null; + logSevere("[Control Host] configuration or automatic enrollment is invalid; VotingPlugin remains unaffected"); + } + } + try { + controlConnector = ControlConnector.create(this); + if (controlConnector != null) controlConnector.start(); + } catch (IOException | IllegalArgumentException e) { + controlConnector = null; + logSevere("[Control] connector configuration or credential is invalid; voting remains unaffected"); + } + } + + private void stopControlServicesLocked(boolean waitForHosted) { + ControlConnector connector = controlConnector; + if (connector != null) { + try { + connector.close(); + if (controlConnector == connector) controlConnector = null; + } catch (RuntimeException failure) { + if (waitForHosted) throw failure; + if (controlConnector == connector) controlConnector = null; + logSevere("[Control] connector did not stop cleanly; proxy cleanup will continue"); + } + } + HostedControlManager manager = hostedControlManager; + if (manager != null) { + try { + if (waitForHosted) { + manager.closeAndWait(); + } else { + manager.close(); + } + if (hostedControlManager == manager) hostedControlManager = null; + } catch (RuntimeException failure) { + if (waitForHosted) throw failure; + if (hostedControlManager == manager) hostedControlManager = null; + logSevere("[Control Host] manager did not stop cleanly; proxy cleanup will continue"); + } + } + } + + public String getControlConnectorStatus() { + ControlConnector connector = controlConnector; + return connector == null ? "DISABLED" : connector.status().name(); + } + + public String getHostedControlStatus() { + HostedControlManager manager = hostedControlManager; + return manager == null ? "DISABLED" : manager.status().name(); + } + + /** + * Handles both the original login notification and extended presence logins. + * Kept protected so transport-policy behavior can be regression tested without + * initializing a live proxy transport. + * + * @param message login envelope + */ + protected void handleLoginMessage(JsonEnvelope message) { + VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(message); + String player = event.player; + String uuid = event.uuid; + String server = event.server; + + if (player.isEmpty() || uuid.isEmpty()) { + logSevere("Invalid login envelope received: " + message.getFields()); + return; + } + boolean legacy = event.connectionId == null && event.backendIncarnationId == null + && event.backendStartedAt == 0L && event.presenceTimestamp == 0L; + boolean accepted = false; + String deliveryServer = server; + if (legacy) { + if (method == BungeeMethod.PLUGINMESSAGING) { + String proxyServer = getCurrentPlayerServer(player); + accepted = isLegacyLoginDestinationAuthoritative(player, uuid, proxyServer); + if (accepted) { + deliveryServer = proxyServer; + } + } else if (method != null && method.supportsBackendPresence() + && isPresenceServerValid(server, VotingPluginWire.SUB_LOGIN)) { + accepted = isLegacyLoginDestinationAuthoritative(player, uuid, server); + } + } else if (method != null && method.supportsBackendPresence() && event.connectionId != null + && isPresenceServerValid(server, VotingPluginWire.SUB_LOGIN) + && isPresenceGenerationValid(event.backendIncarnationId, event.backendStartedAt, + event.presenceTimestamp, VotingPluginWire.SUB_LOGIN)) { + BackendPlayerPresenceTracker.PlayerOnlineResult result = backendPlayerPresenceTracker.playerOnlineResult( + player, uuid, server, event.connectionId, + event.backendIncarnationId, event.backendStartedAt, event.presenceTimestamp, + System.currentTimeMillis()); + accepted = result.isAccepted(); + if (result.isConflictingPresence()) { + requestBackendPresenceSnapshot(server, + new PendingPresenceHandoff(player, uuid, server, event.connectionId, + event.backendIncarnationId, event.backendStartedAt, + result.getConflictSequence(), System.currentTimeMillis())); + } + } + + debug("Login: " + player + "/" + uuid + " " + server); + if (accepted) { + discardPendingPresenceHandoff(uuid); + login(player, uuid, deliveryServer); + } else { + debug("Ignored invalid or stale login envelope: " + message.getFields()); + } + } + + /** + * Validates a legacy login against an authority independent of the envelope. + * Player-facing proxies use their native live route and UUID. A dedicated + * voting proxy has no native player session, so it requires an exact modern + * presence match for the claimed destination. + */ + private boolean isLegacyLoginDestinationAuthoritative(String player, String uuid, String server) { + if (server == null || server.isBlank()) { + return false; + } + + UUID claimedUuid; + try { + claimedUuid = UUID.fromString(uuid.trim()); + } catch (RuntimeException e) { + return false; + } + + if (isDedicatedVotingProxyEnabled()) { + PlayerPresence presence = backendPlayerPresenceTracker.getPlayer(player).orElse(null); + return presence != null && presence.getServer().equalsIgnoreCase(server) + && (!getConfig().getOnlineMode() || presence.getUuid().equals(claimedUuid)); + } + + if (!isPlayerOnline(player)) { + return false; + } + String proxyServer = getCurrentPlayerServer(player); + if (proxyServer == null || !proxyServer.equalsIgnoreCase(server)) { + return false; + } + if (!getConfig().getOnlineMode()) { + return true; + } + + String authoritativeUuid = getUUID(player); + if (authoritativeUuid == null || authoritativeUuid.isBlank()) { + return false; + } + try { + return claimedUuid.equals(UUID.fromString(authoritativeUuid.trim())); + } catch (IllegalArgumentException e) { + return false; + } + } + + private VoteLogMysqlTable voteLogMysqlTable; + + @Getter + private ProxyBroadcastDecider proxyBroadcastDecider; + + public void loadVoteLoggingMySQL() { + if (getConfig().getVoteLoggingEnabled()) { + if (getConfig().getVoteLoggingUseMainMySQL()) { + voteLogMysqlTable = new VoteLogMysqlTable("votingplugin_votelog", getProxyMySQL().getMysql(), + getVoteLoggingMySQLConfig(), getConfig().getDebug()) { + + @Override + public void logSevere(String string) { + VotingPluginProxy.this.logSevere(string); + } + + @Override + public void logInfo(String string) { + VotingPluginProxy.this.logInfo(string); + } + + @Override + public void debug(Throwable e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public String getServerName() { + return "Proxy"; + } + }; + } else { + voteLogMysqlTable = new VoteLogMysqlTable("votingplugin_votelog", getVoteLoggingMySQLConfig(), + getConfig().getDebug()) { + + @Override + public void logSevere(String string) { + VotingPluginProxy.this.logSevere(string); + } + + @Override + public void logInfo(String string) { + VotingPluginProxy.this.logInfo(string); + } + + @Override + public void debug(Throwable e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public String getServerName() { + return "Proxy"; + } + }; + } + + if (getConfig().getVoteLoggingPurgeDays() > 0) { + loadTaskTimer(() -> voteLogMysqlTable.purgeOlderThanDays(getConfig().getVoteLoggingPurgeDays(), 100), + 60, 60 * 60); + } + + debug("Vote logging MySQL enabled"); + } else { + debug("Vote logging MySQL disabled"); + } + } + + public abstract void loadTaskTimer(Runnable runnable, long delaySeconds, long repeatSeconds); + + public void loadMultiProxySupport() { + if (multiProxyHandler != null) { + multiProxyHandler.close(); + } + multiProxyHandler = new MultiProxyHandler() { + + @Override + public void addNonVotedPlayerCache(String uuid, String player) { + addNonVotedPlayer(uuid, player); + } + + @Override + public void clearVote(String uuid) { + getVoteCacheHandler().clearOnlineVoteRewards(uuid); + } + + @Override + public boolean getDebug() { + return getConfig().getDebug(); + } + + @Override + public EncryptionHandler getEncryptionHandler() { + return encryptionHandler; + } + + @Override + public MultiProxyMethod getMultiProxyMethod() { + return MultiProxyMethod.getByName(getConfig().getMultiProxyMethod()); + } + + @Override + public String getMultiProxyPassword() { + return getConfig().getMultiProxyRedisPassword(); + } + + @Override + public String getMultiProxyRedisHost() { + return getConfig().getMultiProxyRedisHost(); + } + + @Override + public int getMultiProxyRedisPort() { + return getConfig().getMultiProxyRedisPort(); + } + + @Override + public boolean getMultiProxyRedisSsl() { + return getConfig().getMultiProxyRedisSsl(); + } + + @Override + public int getMultiProxyRedisDbIndex() { + return getConfig().getMultiProxyRedisDbIndex(); + } + + @Override + public boolean getMultiProxyRedisUseExistingConnection() { + return getConfig().getMultiProxyRedisUseExistingConnection(); + } + + @Override + public String getMultiProxyServerName() { + return getConfig().getProxyServerName(); + } + + @Override + public Collection getMultiProxyServers() { + return getConfig().getMultiProxyServers(); + } + + @Override + public MultiProxyServerSocketConfiguration getMultiProxyServersConfiguration(String s) { + return new MultiProxyServerSocketConfigurationBungee(s, + getConfig().getMultiProxyServersConfiguration(s)); + } + + @Override + public String getMultiProxySocketHostHost() { + return getConfig().getMultiProxySocketHostHost(); + } + + @Override + public int getMultiProxySocketHostPort() { + return getConfig().getMultiProxySocketHostPort(); + } + + @Override + public boolean getMultiProxySupportEnabled() { + return getConfig().getMultiProxySupport(); + } + + @Override + public String getMultiProxyUsername() { + return getConfig().getMultiProxyRedisUsername(); + } + + @Override + public File getPluginDataFolder() { + return getDataFolderPlugin(); + } + + @Override + public boolean getPrimaryServer() { + return getConfig().getPrimaryServer(); + } + + @Override + public List getProxyServers() { + return getConfig().getProxyServers(); + } + + @Override + public RedisHandler getRedisHandler() { + return redisHandler; + } + + @Override + public String getVersion() { + return getPluginVersion(); + } + + @Override + public void logInfo(String msg) { + log(msg); + } + + @Override + public void runAsnc(Runnable runnable) { + runAsync(runnable); + } + + @Override + public void setEncryptionHandler(EncryptionHandler encryptionHandler1) { + encryptionHandler = encryptionHandler1; + } + + @Override + public void triggerVote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid) { + vote(player, service, realVote, timeQueue, queueTime, text, uuid); + } + }; + multiProxyHandler.loadMultiProxySupport(); + } + + public abstract void log(String message); + + /** + * Requests a complete player-presence snapshot from one backend server. + * + * @param server configured backend server name + * @return new or already-active request identifier, or null when the server is + * invalid or is inside the snapshot-request cooldown + */ + public UUID requestBackendPresenceSnapshot(String server) { + return requestBackendPresenceSnapshot(server, null); + } + + private UUID requestBackendPresenceSnapshot(String server, PendingPresenceHandoff handoff) { + return requestBackendPresenceSnapshot(server, handoff, System.currentTimeMillis(), false); + } + + private UUID requestBackendPresenceSnapshot(String server, PendingPresenceHandoff handoff, long now, + boolean handoffAlreadyQueued) { + if (method == null || !method.supportsBackendPresence() || globalMessageProxyHandler == null + || !isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_SNAPSHOT_REQUEST)) { + return null; + } + long backendStartedAt = backendPlayerPresenceTracker.getBackendStartedAt(server); + UUID backendIncarnationId = backendPlayerPresenceTracker.getBackendIncarnationId(server); + if (backendStartedAt <= 0L || backendIncarnationId == null) { + return null; + } + if (handoff != null && (!server.equalsIgnoreCase(handoff.server) + || !backendIncarnationId.equals(handoff.backendIncarnationId) + || backendStartedAt != handoff.backendStartedAt)) { + return null; + } + if (handoff != null && (handoffAlreadyQueued ? !isPendingPresenceHandoff(handoff, now) + : !queuePendingPresenceHandoff(handoff, now))) { + return null; + } + UUID requestId = handoff == null + ? backendPlayerPresenceTracker.beginSnapshot(server, UUID.randomUUID(), backendIncarnationId, + backendStartedAt, now) + : backendPlayerPresenceTracker.beginSnapshotForDestinationClaim(server, UUID.randomUUID(), + backendIncarnationId, backendStartedAt, handoff.playerUuid, handoff.conflictSequence, now); + boolean created = requestId != null; + if (!created) { + requestId = handoff == null ? backendPlayerPresenceTracker.getPendingSnapshotRequestId(server, now) + : backendPlayerPresenceTracker.getPendingSnapshotRequestIdForDestinationClaim(server, + handoff.playerUuid, handoff.conflictSequence, now); + } + if (requestId == null) { + if (handoff != null && !backendPlayerPresenceTracker.isCurrentDestinationClaim(handoff.playerUuid, + handoff.server, handoff.conflictSequence)) { + discardPendingPresenceHandoff(handoff); + } + // A handoff stays unassigned while the destination is inside its snapshot + // cooldown. Presence maintenance will attach it to the next allowed snapshot. + return null; + } + if (handoff != null) { + assignPendingPresenceHandoff(handoff, requestId, now); + } + if (created) { + JsonEnvelope request = VotingPluginWire.presenceSnapshotRequest(server, requestId, backendIncarnationId, + backendStartedAt, now); + globalMessageProxyHandler.sendMessage(server, 1, request); + } + return requestId; + } + + private boolean queuePendingPresenceHandoff(PendingPresenceHandoff handoff, long now) { + if (!isPresenceHandoffValid(handoff, now)) { + return false; + } + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + PendingPresenceHandoff current = pendingPresenceHandoffs.get(handoff.playerUuid); + if (current != null && current.conflictSequence > handoff.conflictSequence) { + return false; + } + handoff.requestId = null; + pendingPresenceHandoffs.put(handoff.playerUuid, handoff); + return true; + } + } + + private boolean isPendingPresenceHandoff(PendingPresenceHandoff handoff, long now) { + if (!isPresenceHandoffValid(handoff, now)) { + return false; + } + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + return pendingPresenceHandoffs.get(handoff.playerUuid) == handoff; + } + } + + private void assignPendingPresenceHandoff(PendingPresenceHandoff handoff, UUID requestId, long now) { + if (requestId == null || !isPresenceHandoffValid(handoff, now)) { + return; + } + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + if (pendingPresenceHandoffs.get(handoff.playerUuid) == handoff) { + handoff.requestId = requestId; + } + } + } + + private boolean isPresenceHandoffValid(PendingPresenceHandoff handoff, long now) { + return handoff != null && handoff.playerUuid != null && handoff.connectionId != null + && handoff.conflictSequence > 0L + && now >= handoff.createdAt && now - handoff.createdAt <= PRESENCE_HANDOFF_TIMEOUT_MILLIS; + } + + private Set completePendingPresenceHandoffs(UUID requestId, String server, UUID backendIncarnationId, + long backendStartedAt, long now) { + List completed = new ArrayList<>(); + Set completedPlayers = new LinkedHashSet<>(); + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + pendingPresenceHandoffs.entrySet().removeIf(entry -> { + PendingPresenceHandoff handoff = entry.getValue(); + if (!requestId.equals(handoff.requestId)) { + return false; + } + if (handoff.server.equalsIgnoreCase(server) + && handoff.backendIncarnationId.equals(backendIncarnationId) + && handoff.backendStartedAt == backendStartedAt) { + completed.add(handoff); + } + return true; + }); + } + for (PendingPresenceHandoff handoff : completed) { + PlayerPresence presence = backendPlayerPresenceTracker.getPlayer(handoff.playerUuid).orElse(null); + if (presence != null && presence.getServer().equalsIgnoreCase(handoff.server) + && presence.getConnectionId().equals(handoff.connectionId)) { + login(handoff.playerName, handoff.uuid, handoff.server); + completedPlayers.add(handoff.playerUuid); + } + releaseDestinationClaim(handoff); + } + return completedPlayers; + } + + /** + * Drains voter-keyed cached rewards when a complete recovery snapshot first + * confirms a player on a dedicated voting proxy. Cross-backend handoffs are + * already processed by their token-bound completion path and are excluded to + * avoid a second login callback. + */ + protected void processDedicatedSnapshotLogins(String server, Set handoffPlayers) { + if (!isDedicatedVotingProxyEnabled() || server == null || server.isBlank()) { + return; + } + Set excluded = handoffPlayers == null ? Collections.emptySet() : handoffPlayers; + for (PlayerPresence presence : backendPlayerPresenceTracker.getOnlinePlayers()) { + if (presence.getServer().equalsIgnoreCase(server) && !excluded.contains(presence.getUuid())) { + login(presence.getPlayerName(), presence.getUuid().toString(), presence.getServer()); + } + } + } + + private void discardPendingPresenceHandoff(String uuid) { + try { + UUID playerUuid = UUID.fromString(uuid.trim()); + PendingPresenceHandoff removed; + synchronized (pendingPresenceHandoffs) { + removed = pendingPresenceHandoffs.remove(playerUuid); + } + releaseDestinationClaim(removed); + } catch (Exception ignored) { + // Invalid identities are rejected by the presence tracker. + } + } + + private void discardPendingPresenceHandoff(PendingPresenceHandoff handoff) { + boolean removed = false; + synchronized (pendingPresenceHandoffs) { + if (handoff != null && pendingPresenceHandoffs.get(handoff.playerUuid) == handoff) { + pendingPresenceHandoffs.remove(handoff.playerUuid); + removed = true; + } + } + if (removed) { + releaseDestinationClaim(handoff); + } + } + + private void discardPendingPresenceHandoffs(String server) { + synchronized (pendingPresenceHandoffs) { + pendingPresenceHandoffs.entrySet().removeIf(entry -> { + if (!entry.getValue().server.equalsIgnoreCase(server)) { + return false; + } + releaseDestinationClaim(entry.getValue()); + return true; + }); + } + } + + private void discardPendingPresenceHandoffs(UUID requestId) { + if (requestId == null) { + return; + } + synchronized (pendingPresenceHandoffs) { + pendingPresenceHandoffs.entrySet().removeIf(entry -> { + if (!requestId.equals(entry.getValue().requestId)) { + return false; + } + releaseDestinationClaim(entry.getValue()); + return true; + }); + } + } + + private void prunePendingPresenceHandoffs(long now) { + pendingPresenceHandoffs.entrySet().removeIf(entry -> { + PendingPresenceHandoff handoff = entry.getValue(); + if (now >= handoff.createdAt && now - handoff.createdAt <= PRESENCE_HANDOFF_TIMEOUT_MILLIS) { + return false; + } + releaseDestinationClaim(handoff); + return true; + }); + } + + private void releaseDestinationClaim(PendingPresenceHandoff handoff) { + if (handoff != null) { + backendPlayerPresenceTracker.releaseDestinationClaim(handoff.playerUuid, handoff.server, + handoff.conflictSequence); + } + } + + protected void retryPendingPresenceHandoffs(long now) { + List retry = new ArrayList<>(); + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + for (PendingPresenceHandoff handoff : pendingPresenceHandoffs.values()) { + UUID activeRequestId = backendPlayerPresenceTracker.getPendingSnapshotRequestId(handoff.server, now); + if (handoff.requestId != null && !handoff.requestId.equals(activeRequestId)) { + handoff.requestId = null; + } + if (handoff.requestId == null) { + retry.add(handoff); + } + } + } + for (PendingPresenceHandoff handoff : retry) { + requestBackendPresenceSnapshot(handoff.server, handoff, now, true); + } + } + + protected int getPendingPresenceHandoffCount() { + synchronized (pendingPresenceHandoffs) { + return pendingPresenceHandoffs.size(); + } + } + + protected void scheduleBackendPresenceStartupResync() { + ScheduledExecutorService scheduler = getScheduler(); + if (method == null || !method.supportsBackendPresence() || scheduler == null) { + return; + } + scheduler.schedule(this::requestBackendPresenceStartupResync, + PRESENCE_STARTUP_RESYNC_DELAY_SECONDS, TimeUnit.SECONDS); + } + + protected void requestBackendPresenceStartupResync() { + if (!enabled || method == null || !method.supportsBackendPresence() || globalMessageProxyHandler == null) { + return; + } + long requestedAt = System.currentTimeMillis(); + int delay = 1; + for (String server : getAllAvailableServers()) { + if (!isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_RESYNC_REQUEST)) { + continue; + } + globalMessageProxyHandler.sendMessage(server, delay++, + VotingPluginWire.presenceResyncRequest(server, UUID.randomUUID(), requestedAt)); + } + } + + private void maintainBackendPresence() { + if (!enabled || method == null || !method.supportsBackendPresence()) { + return; + } + expireBackendPresence(PRESENCE_BACKEND_TIMEOUT_MILLIS); + for (String server : getAllAvailableServers()) { + if (pendingBackendRecoverySnapshots.contains(presenceServerKey(server))) { + requestBackendPresenceSnapshot(server); + } + } + retryPendingPresenceHandoffs(System.currentTimeMillis()); + } + + private String presenceServerKey(String server) { + return server == null ? "" : server.trim().toLowerCase(java.util.Locale.ROOT); + } + + private boolean isPresenceServerValid(String server, String subChannel) { + // The presence protocol's trust boundary is the configured backend set. The + // selected transport must only be accessible to backend servers trusted not to + // impersonate one another. + if (server == null || server.isBlank() || !isServerValid(server)) { + debug("Ignored " + subChannel + " presence envelope for an unconfigured server"); + return false; + } + return true; + } + + private boolean isPresenceGenerationValid(UUID backendIncarnationId, long backendStartedAt, + long presenceTimestamp, String subChannel) { + if (backendIncarnationId == null || backendStartedAt <= 0L || presenceTimestamp < backendStartedAt) { + debug("Ignored " + subChannel + " presence envelope with an invalid backend generation"); + return false; + } + return true; + } + + /** + * Removes presence owned by backends that have stopped reporting heartbeats. + * Scheduling and timeout configuration are intentionally left to dedicated + * proxy mode. + * + * @param timeoutMillis maximum backend silence before expiry + * @return expired backend server names + */ + public Set expireBackendPresence(long timeoutMillis) { + if (method == null || !method.supportsBackendPresence()) { + return Collections.emptySet(); + } + long now = System.currentTimeMillis(); + Set expired = backendPlayerPresenceTracker.expireBackends(now, timeoutMillis); + for (String server : expired) { + discardPendingPresenceHandoffs(server); + // Keep recovery pending while this generation is unavailable. If the same + // backend process resumes, its heartbeat can mark it available again and the + // maintenance task will request a fresh snapshot of players who stayed online. + pendingBackendRecoverySnapshots.add(presenceServerKey(server)); + } + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + } + return expired; + } + + public void login(String playerName, String uuid, String serverName) { + if (!getConfig().getOnlineMode()) { + uuid = getUUID(playerName); + } + + try { + if (uuid != null && !uuid.isEmpty() && !uuid.equalsIgnoreCase("null")) { + uuid = UUID.fromString(uuid.trim()).toString(); + } + } catch (Exception ignored) { + // ignore + } + + if (getConfig().getOnlineMode()) { + addNonVotedPlayer(uuid, playerName); + } + if (isPlayerOnlineForVoteRouting(playerName)) { + if (getConfig().getGlobalDataEnabled()) { + if (getGlobalDataHandler().isTimeChangedHappened()) { + getGlobalDataHandler().checkForFinishedTimeChanges(); + } + } + + checkCachedVotes(serverName); + retryPendingOnlineBroadcasts(serverName); + retryPendingTimeBroadcasts(serverName); + checkOnlineVotes(playerName, uuid, serverName); + multiProxyHandler.login(uuid, playerName); + } + } + + private void logInfo(String msg) { + log(msg); + } + + public abstract void logSevere(String message); + + public void onDisable() { + onDisable(false); + } + + /** Full runtime replacement waits for hosted workers; final proxy stop remains non-blocking. */ + public void onDisable(boolean waitForHosted) { + if (waitForHosted) { + prepareForRuntimeReplacement(); + } else { + controlServicesGeneration.incrementAndGet(); + controlLifecycleExecutor.shutdownNow(); + stopControlServices(false); + } + completeRuntimeReplacementShutdown(); + } + + /** Fail-closed gate that must complete before a replacement proxy runtime is created. */ + public void prepareForRuntimeReplacement() { + controlServicesGeneration.incrementAndGet(); + synchronized (controlLifecycleLock) { + ControlConnector connector = controlConnector; + if (connector != null && !connector.reserveRuntimeReplacement()) { + throw new IllegalStateException("Control result must be acknowledged before proxy runtime replacement"); + } + controlLifecycleExecutor.shutdown(); + stopControlServicesLocked(true); + } + } + + /** Best-effort remainder of runtime teardown after the Control overlap gate has succeeded. */ + public void completeRuntimeReplacementShutdown() { + cancelCommunicationTests("Proxy runtime stopped before the backend replied"); + runCleanup("vote cache", () -> getVoteCacheHandler().saveVoteCache()); + runCleanup("proxy MySQL messenger", () -> { + if (getProxyMysqlMessenger() != null) getProxyMysqlMessenger().shutdown(); + }); + runCleanup("proxy MySQL", () -> { + if (getProxyMySQL() != null) getProxyMySQL().shutdown(); + }); + runCleanup("multi-proxy handler", () -> { + if (multiProxyHandler != null) multiProxyHandler.close(); + }); + runCleanup("socket listener", () -> { + if (socketHandler != null) socketHandler.closeConnection(); + }); + runCleanup("socket clients", this::closeSocketClients); + runCleanup("HTTP transport", this::closeHttpTransport); + runCleanup("Redis subscriber", () -> { + if (redisHandler != null) redisHandler.close(); + }); + runCleanup("Redis publisher", () -> { + JedisPool pool = redisPublisherPool; + try { + if (pool != null) pool.close(); + } finally { + if (redisPublisherPool == pool) redisPublisherPool = null; + } + }); + runCleanup("MQTT transport", () -> { + if (mqttHandler != null) mqttHandler.disconnect(); + }); + runCleanup("time checker", () -> bungeeTimeChecker.shutdown()); + runCleanup("global data", () -> { + if (getGlobalDataHandler() != null) getGlobalDataHandler().shutdown(); + }); + enabled = false; + } + + private void runCleanup(String service, CleanupAction cleanup) { + try { + cleanup.run(); + } catch (Exception failure) { + logSevere("Unable to stop " + service + "; remaining proxy cleanup will continue"); + } + } + + @FunctionalInterface + private interface CleanupAction { void run() throws Exception; } + + public void onPluginMessageReceived(DataInputStream in) { + onPluginMessageReceived(in, null); + } + + /** Receives a plugin message bound to the backend server connection that sent it. */ + public void onPluginMessageReceived(DataInputStream in, String sourceServer) { + runAsync(() -> { + try { + final String headerSub; + if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { + headerSub = encryptionHandler.decrypt(in.readUTF()); + } else { + headerSub = in.readUTF(); + } + + int size = in.readInt(); // sanity only + + if (getConfig().getDebug()) { + debug("Received plugin message header=" + headerSub + " size=" + size); + } + + String payload = ""; + if (size > 0) { + if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { + payload = encryptionHandler.decrypt(in.readUTF()); + } else { + payload = in.readUTF(); + } + } + + JsonEnvelope envelope = JsonEnvelopeCodec.decode(payload); + + if (!headerSub.equalsIgnoreCase(envelope.getSubChannel())) { + if (getConfig().getDebug()) { + warn("PluginMessage subChannel mismatch: header=" + headerSub + " env=" + + envelope.getSubChannel()); + } + return; + } + + if (VotingPluginWire.SUB_CONTROL_ENROLLMENT_REQUEST.equals(envelope.getSubChannel())) { + handleControlEnrollmentRequest(sourceServer, envelope); + return; + } + + globalMessageProxyHandler.onMessage(envelope); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + + private void handleControlEnrollmentRequest(String sourceServer, JsonEnvelope envelope) { + VotingPluginWire.ControlEnrollmentRequest request = VotingPluginWire.readControlEnrollmentRequest(envelope); + if (!request.valid || sourceServer == null || sourceServer.isBlank()) return; + if (!sourceServer.equals(request.nodeId)) { + sendPluginMessageServer(sourceServer, 0, + VotingPluginWire.controlEnrollmentResult(sourceServer, request.requestId, false)); + return; + } + long now = System.nanoTime(); + AtomicBoolean allowed = new AtomicBoolean(); + controlEnrollmentNextAllowed.compute(sourceServer, (ignored, nextAllowed) -> { + if (nextAllowed == null || now >= nextAllowed) { + allowed.set(true); + return now + CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS; + } + return nextAllowed; + }); + if (!allowed.get()) return; + HostedControlManager manager = hostedControlManager; + if (manager == null) { + sendPluginMessageServer(sourceServer, 0, + VotingPluginWire.controlEnrollmentResult(sourceServer, request.requestId, false)); + return; + } + manager.installNodeVerifier(sourceServer, request.verifier, request.endpoint).whenComplete((installed, failure) -> { + boolean success = failure == null && Boolean.TRUE.equals(installed); + sendPluginMessageServer(sourceServer, 0, + VotingPluginWire.controlEnrollmentResult(sourceServer, request.requestId, success)); + if (success) log("[Control] automatically enrolled backend node " + sourceServer); + }); + } + + private UUID parseUUIDFromString(String uuidAsString) { + String[] parts = { "0x" + uuidAsString.substring(0, 8), "0x" + uuidAsString.substring(8, 12), + "0x" + uuidAsString.substring(12, 16), "0x" + uuidAsString.substring(16, 20), + "0x" + uuidAsString.substring(20, 32) }; + + long mostSigBits = Long.decode(parts[0]).longValue(); + mostSigBits <<= 16; + mostSigBits |= Long.decode(parts[1]).longValue(); + mostSigBits <<= 16; + mostSigBits |= Long.decode(parts[2]).longValue(); + + long leastSigBits = Long.decode(parts[3]).longValue(); + leastSigBits <<= 48; + leastSigBits |= Long.decode(parts[4]).longValue(); + + return new UUID(mostSigBits, leastSigBits); + } + + public synchronized void processQueue() { + while (getVoteCacheHandler().getTimeChangeQueue().size() > 0) { + VoteTimeQueue vote = getVoteCacheHandler().getTimeChangeQueue().element(); + if (!vote.isProcessed()) { + VoteTotalsSnapshot queuedTotals = vote.getTotals() == null || vote.getTotals().isEmpty() ? null + : VoteTotalsSnapshot.parseStorage(vote.getTotals()); + QueuedVoteResult result = vote(vote.getName(), vote.getService(), true, false, vote.getTime(), queuedTotals, + vote.getUuid(), vote); + if (result == QueuedVoteResult.RETRY) { + scheduleTimeVoteRetry(); + return; + } + if (result == QueuedVoteResult.TERMINAL) { + warn("Removing terminal rollover vote " + vote.getVoteId() + " for " + vote.getName() + "/" + + ServiceSiteValidator.sanitizeForLog(vote.getService())); + } + } + if (!getVoteCacheHandler().removeTimeVote(vote)) { + scheduleTimeVoteRetry(); + return; + } + } + } + + private void scheduleTimeVoteRetry() { + if (timeVoteRetryScheduled || getScheduler() == null) { + return; + } + timeVoteRetryScheduled = true; + try { + getScheduler().schedule(() -> { + synchronized (VotingPluginProxy.this) { + timeVoteRetryScheduled = false; + } + processQueue(); + }, 5, TimeUnit.SECONDS); + } catch (RuntimeException e) { + timeVoteRetryScheduled = false; + debug("Unable to schedule rollover vote retry: " + e.getMessage()); + } + } + + public void reload() { + reloadRuntime(true); + } + + /** Applies a Control-originated configuration reload without stopping its connector or hosted service. */ + public void reloadFromControl() { + reloadRuntime(false); + } + + private void reloadRuntime(boolean restartControlServices) { + method = BungeeMethod.getByName(getConfig().getBungeeMethod()); + if (getMethod() == null) { + method = BungeeMethod.PLUGINMESSAGING; + } + warnUnsupportedDedicatedVotingProxyMode(); + if (!restartControlServices && method == BungeeMethod.SOCKETS) { + rebuildSocketClients(); + } + + setCurrentVotePartyVotesRequired( + getConfig().getVotePartyVotesRequired() + getVoteCacheVotePartyIncreaseVotesRequired()); + if (restartControlServices) { + loadMultiProxySupport(); + restartControlServicesAsync(); + } + } + + private synchronized void rebuildSocketClients() { + HashMap rebuilt = new HashMap<>(); + try { + List blocked = getConfig().getBlockedServers(); + for (String server : getConfig().getSpigotServers()) { + if (blocked.contains(server)) continue; + Map data = getConfig().getSpigotServerConfiguration(server); + String host = data.containsKey("Host") ? (String) data.get("Host") : ""; + int port = data.containsKey("Port") ? (int) data.get("Port") : 1298; + rebuilt.put(server, new ClientHandler(host, port, encryptionHandler, getConfig().getDebug())); + } + } catch (RuntimeException failure) { + stopSocketClients(rebuilt); + throw failure; + } + HashMap previous = clientHandles; + clientHandles = rebuilt; + stopSocketClients(previous); + } + + private synchronized boolean sendSocketEnvelope(String server, JsonEnvelope envelope) { + ClientHandler socketClient = clientHandles == null ? null : clientHandles.get(server); + if (socketClient == null) return false; + try { + socketClient.sendEnvelope(envelope); + return true; + } catch (RuntimeException e) { + debug(e.getMessage()); + return false; + } + } + + private synchronized boolean sendHttpEnvelope(String server, JsonEnvelope envelope) { + HttpProxyTransportServer transport = httpTransportServer; + return transport != null && transport.send(server, envelope); + } + + private void startHttpTransport() { + try { + URI endpoint = URI.create(getConfig().getHttpPublicEndpoint()); + if (!"https".equalsIgnoreCase(endpoint.getScheme()) || endpoint.getHost() == null + || endpoint.getPort() == 0 || endpoint.getPort() > 65535 + || endpoint.getUserInfo() != null || endpoint.getQuery() != null || endpoint.getFragment() != null + || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) { + throw new IllegalArgumentException("HTTP.PublicEndpoint must be an HTTPS origin"); + } + File directory = new File(getDataFolderPlugin(), "http"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.toPath(), endpoint.getHost()); + httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); + httpTransportServer = new HttpProxyTransportServer( + new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, + httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope); + httpTransportServer.start(); + logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" + + httpTransportServer.port() + "; use /votingpluginbungee httpcode for each backend"); + } catch (Exception failure) { + closeHttpTransport(); + throw new IllegalStateException("HTTP transport could not start securely", failure); + } + } + + /** Keeps the authenticated mTLS backend identity attached to security-sensitive proxy routing. */ + protected void handleHttpTransportEnvelope(HttpProxyTransportServer.ReceivedEnvelope received) { + if (!isAuthenticatedHttpEnvelopeAllowed(received)) { + debug("Ignored HTTP envelope whose player-presence claim did not match its authenticated backend"); + return; + } + GlobalMessageProxyHandler handler = globalMessageProxyHandler; + if (handler == null) throw new IllegalStateException("HTTP message router is not ready"); + handler.onMessage(received.envelope()); + } + + private boolean isAuthenticatedHttpEnvelopeAllowed(HttpProxyTransportServer.ReceivedEnvelope received) { + if (received == null || received.envelope() == null || received.serverId() == null) return false; + String stampedServer = received.envelope().getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + if (!received.serverId().equalsIgnoreCase(stampedServer)) return false; + if (!VotingPluginWire.SUB_LOGIN.equals(received.envelope().getSubChannel())) return true; + VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(received.envelope()); + boolean modern = event.connectionId != null || event.backendIncarnationId != null + || event.backendStartedAt != 0L || event.presenceTimestamp != 0L; + if (!modern || isDedicatedVotingProxyEnabled()) return true; + // A player-facing proxy has a stronger authority than any backend: its live + // player connection supplies both the current route and (in online mode) UUID. + return isLegacyLoginDestinationAuthoritative(event.player, event.uuid, received.serverId()); + } + + private synchronized void closeHttpTransport() { + HttpProxyTransportServer transport = httpTransportServer; + httpTransportServer = null; + httpEnrollmentAuthority = null; + if (transport != null) transport.close(); + } + + public String createHttpConnectionCode(String serverId) { + HttpEnrollmentAuthority authority = httpEnrollmentAuthority; + if (method != BungeeMethod.HTTP || authority == null) { + throw new IllegalStateException("The HTTP transport is not running"); + } + return authority.createConnectionCode(serverId, URI.create(getConfig().getHttpPublicEndpoint()), Duration.ofMinutes(15)) + .encode(); + } + + public void revokeHttpBackend(String serverId) { + HttpEnrollmentAuthority authority = httpEnrollmentAuthority; + if (method != BungeeMethod.HTTP || authority == null) throw new IllegalStateException("The HTTP transport is not running"); + authority.revoke(HttpTlsIdentity.canonicalServerId(serverId)); + } + + private synchronized void closeSocketClients() { + HashMap clients = clientHandles; + clientHandles = null; + stopSocketClients(clients); + } + + static void stopSocketClients(Map clients) { + if (clients == null) return; + for (ClientHandler client : clients.values()) { + if (client == null) continue; + try { + client.stopConnection(); + } catch (RuntimeException ignored) { + // Best effort: one broken client must not prevent the remaining sockets from closing. + } + } + } + + private void warnUnsupportedDedicatedVotingProxyMode() { + if (getConfig().getDedicatedVotingProxy() && (method == null || !method.supportsBackendPresence())) { + logSevere("DedicatedVotingProxy requires MYSQL, REDIS, MQTT, SOCKETS, or HTTP; PLUGINMESSAGING is disabled for " + + "dedicated-proxy routing. Falling back to normal proxy routing."); + } + } + + public abstract void runAsync(Runnable run); + + /** Platform name used only for the transport-neutral Control discovery contract. */ + public abstract String getProxyPlatform(); + + public abstract void runConsoleCommand(String command); + + public abstract void saveVoteCacheFile(); + + public abstract void reloadCore(boolean mysql); + + /** Strict Control reload path; failures propagate so the caller can restore its backup. */ + public abstract void reloadControlConfiguration() throws Exception; + + public abstract boolean sendPluginMessageData(String server, String channel, byte[] data, boolean queue); + + private static final int PLUGIN_MESSAGE_HARD_LIMIT = 32767; + private static final int PLUGIN_MESSAGE_SOFT_LIMIT = 30000; + + public void sendPluginMessageServer(String server, int delay, JsonEnvelope envelope) { + getScheduler().schedule(() -> sendPluginMessageServerNow(server, envelope), delay * 5L, TimeUnit.MILLISECONDS); + } + + /** + * Sends a plugin-message envelope immediately and reports whether the proxy + * accepted it for delivery. + * + * @param server target backend server + * @param envelope envelope to send + * @return true when the proxy accepted the message for delivery + */ + protected boolean sendPluginMessageServerNow(String server, JsonEnvelope envelope) { + final String subChannel = envelope.getSubChannel(); + final String payload = JsonEnvelopeCodec.encode(envelope); + + final byte[] subChannelBytes = subChannel.getBytes(java.nio.charset.StandardCharsets.UTF_8); + final byte[] payloadBytes = payload.getBytes(java.nio.charset.StandardCharsets.UTF_8); + + // Estimate bytes written: + // - writeUTF adds 2-byte length prefix + UTF-8 bytes + // - writeInt is 4 bytes + int estimatedSize = 2 + subChannelBytes.length + // subChannel UTF (len prefix + bytes) + 4 + // payload length int + 2 + payloadBytes.length; // payload UTF (len prefix + bytes) + + if (estimatedSize > PLUGIN_MESSAGE_SOFT_LIMIT) { + debug("[PluginMessage] Payload nearing limit (" + estimatedSize + " bytes) server=" + server + + " subChannel=" + subChannel + " — consider Redis instead"); + } + + if (estimatedSize > PLUGIN_MESSAGE_HARD_LIMIT) { + debug("[PluginMessage] Payload TOO LARGE (" + estimatedSize + " bytes, max=" + PLUGIN_MESSAGE_HARD_LIMIT + + ") server=" + server + " subChannel=" + subChannel + " — NOT sent"); + return false; + } + + try (ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(byteOutStream)) { + if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { + out.writeUTF(encryptionHandler.encrypt(subChannel)); + } else { + out.writeUTF(subChannel); + } + + // sanity only: MUST be bytes, not chars + out.writeInt(payloadBytes.length); + + if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { + out.writeUTF(encryptionHandler.encrypt(payload)); + } else { + out.writeUTF(payload); + } + out.flush(); + + boolean sent = sendPluginMessageData(server, getConfig().getPluginMessageChannel().toLowerCase(), + byteOutStream.toByteArray(), false); + if (getConfig().getDebug()) { + debug((sent ? "Sent" : "Could not send") + " plugin envelope (" + estimatedSize + " bytes) " + server + + " " + subChannel + " " + envelope.getFields()); + } + return sent; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + static DefaultJedisClientConfig buildRedisClientConfig(VotingPluginProxyConfig configSource) { + DefaultJedisClientConfig.Builder config = DefaultJedisClientConfig.builder() + .database(configSource.getRedisDbIndex()).ssl(configSource.getRedisSsl()).connectionTimeoutMillis(2000) + .socketTimeoutMillis(2000); + if (configSource.getRedisSsl()) { + SSLParameters sslParameters = new SSLParameters(); + sslParameters.setEndpointIdentificationAlgorithm("HTTPS"); + config.sslParameters(sslParameters); + } + if (configSource.getRedisUsername() != null && !configSource.getRedisUsername().isEmpty()) { + config.user(configSource.getRedisUsername()); + } + if (configSource.getRedisPassword() != null && !configSource.getRedisPassword().isEmpty()) { + config.password(configSource.getRedisPassword()); + } + return config.build(); + } + + public boolean sendRedisEnvelopeServer(String server, JsonEnvelope envelope) { + return sendRedisEnvelopeServer(server, envelope, false); + } + + private boolean sendRedisEnvelopeServer(String server, JsonEnvelope envelope, boolean useRetryCooldown) { + JedisPool publisherPool = redisPublisherPool; + if (publisherPool == null || (useRetryCooldown && System.currentTimeMillis() < redisPublisherRetryAfter)) { + return false; + } + + try (Jedis jedis = publisherPool.getResource()) { + String channel = getConfig().getRedisPrefix() + "VotingPlugin_" + server; + long subscribers = jedis.publish(channel, + JsonEnvelopeCodec.encode(VotingPluginWire.withRedisDeliveryId(envelope))); + redisPublisherRetryAfter = 0L; + return subscribers > 0; + } catch (Exception e) { + if (useRetryCooldown) { + // Standalone broadcasts remain queued, so their retries can be throttled safely. + redisPublisherRetryAfter = System.currentTimeMillis() + 2000L; + } + debug(e.getMessage()); + return false; + } + } + + public boolean sendMqttEnvelopeServer(String server, JsonEnvelope envelope) { + if (mqttHandler == null) { + return false; + } + try { + mqttHandler.publishEnvelope(getConfig().getMqttPrefix() + "votingplugin/servers/" + server, envelope); + return true; + } catch (Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + return false; + } + } + + public boolean sendSocketEnvelopeServer(String server, JsonEnvelope envelope) { + Map configuration = getConfig().getSpigotServerConfiguration(server); + if (configuration == null) { + return false; + } + String host = configuration.get("Host") instanceof String ? (String) configuration.get("Host") : ""; + int port = configuration.get("Port") instanceof Number ? ((Number) configuration.get("Port")).intValue() : 1298; + if (host.isEmpty()) { + return false; + } + + String payload = JsonEnvelopeCodec.encode(envelope); + String encoded = encryptionHandler != null ? encryptionHandler.encrypt(payload) : payload; + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(host, port), 2000); + try (DataOutputStream output = new DataOutputStream(socket.getOutputStream())) { + output.writeUTF(encoded); + output.flush(); + } + return true; + } catch (Exception e) { + debug(e.getMessage()); + return false; + } + } + + public void sendServerNameMessage() { + for (String s : getAllAvailableServers()) { + sendPluginMessageServer(s, 1, VotingPluginWire.serverName(s)); + } + } + + public void sendVoteParty(String server) { + if (isSomeoneOnlineServerForVoteRouting(server)) { + globalMessageProxyHandler.sendMessage(server, 1, VotingPluginWire.votePartyBungee()); + } + } + + public void setCurrentVotePartyVotes(int amount) { + votePartyVotes = amount; + setVoteCacheVotePartyCurrentVotes(amount); + debug("Current vote party total: " + votePartyVotes); + } + + public abstract void setVoteCacheLastUpdated(); + + public abstract void setVoteCachePrevDay(int day); + + public abstract void setVoteCachePrevMonth(String text); + + public abstract void setVoteCachePrevWeek(int week); + + public abstract void setVoteCacheVoteCacheIgnoreTime(boolean ignore); + + public abstract void setVoteCacheVotePartyCurrentVotes(int votes); + + public abstract void setVoteCacheVotePartyIncreaseVotesRequired(int votes); + + public void status() { + for (String s : getAllAvailableServers()) { + if (!isSomeoneOnlineServerForVoteRouting(s)) { + log("No players on server " + s + " to send test status message, please retest with someone online"); + } else { + log("Sending request for status message on " + s); + globalMessageProxyHandler.sendMessage(s, 1, VotingPluginWire.status(s)); + } + } + } + + /** Runs a correlated, non-vote round trip over the active backend transport. */ + public CompletableFuture testBackendCommunication(String requestedServer, + long timeoutMillis) { + String server = requestedServer == null ? "" : requestedServer.trim(); + BungeeMethod activeMethod = method; + if (server.isEmpty() || !getAllAvailableServers().contains(server)) { + return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, + "UNKNOWN_BACKEND", "The backend is not configured on this proxy")); + } + if (activeMethod == null || globalMessageProxyHandler == null) { + return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, + "TRANSPORT_UNAVAILABLE", "The proxy communication transport is not running")); + } + if (activeMethod == BungeeMethod.PLUGINMESSAGING && !isSomeoneOnlineServerForVoteRouting(server)) { + return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, + "PLAYER_REQUIRED", "Plugin messaging requires an online player on the selected backend")); + } + ScheduledExecutorService scheduler = getScheduler(); + if (scheduler == null) { + return CompletableFuture.completedFuture(CommunicationTestResult.failure(server, activeMethod, + "TRANSPORT_UNAVAILABLE", "The proxy scheduler is not running")); + } + long boundedTimeout = Math.max(500L, Math.min(timeoutMillis, 30000L)); + UUID requestId = UUID.randomUUID(); + CompletableFuture result = new CompletableFuture<>(); + PendingCommunicationTest pending = new PendingCommunicationTest(server, activeMethod, System.nanoTime(), result); + pendingCommunicationTests.put(requestId, pending); + result.whenComplete((ignored, failure) -> pendingCommunicationTests.remove(requestId, pending)); + try { + if (!sendCommunicationTestEnvelopeNow(server, VotingPluginWire.status(server, requestId))) { + result.complete(CommunicationTestResult.failure(server, activeMethod, "TRANSPORT_UNAVAILABLE", + "The active transport could not accept the communication test")); + return result; + } + scheduler.schedule(() -> result.complete(CommunicationTestResult.failure(server, activeMethod, + "TIMEOUT", "No correlated reply arrived before the timeout")), boundedTimeout, TimeUnit.MILLISECONDS); + } catch (RuntimeException failure) { + result.complete(CommunicationTestResult.failure(server, activeMethod, "SEND_FAILED", + "The proxy could not send the communication test")); + } + return result; + } + + /** Sends a diagnostic immediately and reports whether the active transport accepted it. */ + protected boolean sendCommunicationTestEnvelopeNow(String server, JsonEnvelope envelope) { + return sendProxyBroadcastEnvelopeNow(server, envelope); + } + + protected void handleStatusOkay(JsonEnvelope message) { + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + String request = message.getFields().getOrDefault(VotingPluginWire.K_REQUEST_ID, ""); + if (request.isEmpty()) { + log("Status okay for " + server); + return; + } + UUID requestId; + try { + requestId = UUID.fromString(request); + } catch (IllegalArgumentException ignored) { + debug("Ignored status reply with an invalid request ID from " + server); + return; + } + PendingCommunicationTest pending = pendingCommunicationTests.get(requestId); + if (pending == null || !pending.server().equals(server)) { + debug("Ignored unexpected status reply from " + server); + return; + } + long roundTripMillis = Math.max(0L, + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - pending.startedAtNanos())); + pending.result().complete(CommunicationTestResult.success(server, pending.method(), roundTripMillis)); + } + + private void cancelCommunicationTests(String message) { + pendingCommunicationTests.forEach((requestId, pending) -> pending.result().complete( + CommunicationTestResult.failure(pending.server(), pending.method(), "TRANSPORT_STOPPED", message))); + pendingCommunicationTests.clear(); + } + + public record CommunicationTestResult(boolean success, String code, String message, String server, + String method, long roundTripMillis) { + private static CommunicationTestResult success(String server, BungeeMethod method, long roundTripMillis) { + return new CommunicationTestResult(true, "OK", "Backend replied over the active transport", server, + method == null ? "" : method.name(), roundTripMillis); + } + + private static CommunicationTestResult failure(String server, BungeeMethod method, String code, String message) { + return new CommunicationTestResult(false, code, message, server, + method == null ? "" : method.name(), -1L); + } + } + + private record PendingCommunicationTest(String server, BungeeMethod method, long startedAtNanos, + CompletableFuture result) { } + + private void sendVoteDelayRejected(String player, String uuid, String service, boolean playerOnline, + String playerServer) { + if (!playerOnline || playerServer == null || !getAllAvailableServers().contains(playerServer)) { + debug("Not sending vote delay rejection for " + player + " because the player is offline"); + return; + } + + globalMessageProxyHandler.sendMessage(playerServer, 1, + VotingPluginWire.voteDelayRejected(player, uuid, service, true)); + } + + public String getWaitUntilDelaySiteFromService(String service) { + for (String site : getConfig().getWaitUntilVoteDelaySites()) { + if (getConfig().getWaitUntilVoteDelayService(site).equalsIgnoreCase(service)) { + return site; + } + } + return ""; + } + + private long getLastVotesTime(String uuid, ArrayList cols, String site, String service, String player, + boolean includeTimeChangeQueue) { + long mostRecentTime = 0; + + if (getVoteCacheHandler().hasOnlineVotes(uuid)) { + ArrayList onlineVotes = getVoteCacheHandler().getOnlineVotes(uuid); + for (OfflineBungeeVote vote : onlineVotes) { + if (vote.getService().equalsIgnoreCase(service)) { + mostRecentTime = Math.max(mostRecentTime, vote.getTime()); + } + } + } + + for (String server : getAllAvailableServers()) { + for (OfflineBungeeVote vote : getVoteCacheHandler().getVotes(server)) { + if (vote.getUuid().equals(uuid) && vote.getService().equalsIgnoreCase(service)) { + mostRecentTime = Math.max(mostRecentTime, vote.getTime()); + } + } + } + + if (includeTimeChangeQueue && player != null) { + for (VoteTimeQueue queuedVote : getVoteCacheHandler().getTimeChangeQueue()) { + if (queuedVote.getName().equalsIgnoreCase(player) + && queuedVote.getService().equalsIgnoreCase(service)) { + mostRecentTime = Math.max(mostRecentTime, queuedVote.getTime()); + } + } + } + + for (Column d : cols) { + if (d.getName().equalsIgnoreCase("LastVotes")) { + DataValue value = d.getValue(); + String[] list = value.getString().split("%line%"); + for (String str : list) { + String[] data = str.split("//"); + if (data[0].equalsIgnoreCase(site)) { + mostRecentTime = Math.max(mostRecentTime, Long.valueOf(data[1])); + } + } + } + } + return mostRecentTime; + } + + public boolean checkVoteDelay(String uuid, String service, ArrayList data) { + return checkVoteDelay(uuid, null, service, data, false); + } + + /** + * Checks the configured vote delay, optionally including accepted votes waiting + * for a GlobalData time change to finish. + * + * @param uuid player UUID + * @param player player name used by the time-change queue + * @param service vote service + * @param data current player data + * @param includeTimeChangeQueue whether queued votes reserve their delay slot + * @return true when the vote may be accepted + */ + public boolean checkVoteDelay(String uuid, String player, String service, ArrayList data, + boolean includeTimeChangeQueue) { + String site = getWaitUntilDelaySiteFromService(service); + if (site.isEmpty()) { + debug("No service site set for " + service + ", skipping vote delay check"); + return true; + } + + int voteDelay = getConfig().getWaitUntilVoteDelayVoteDelay(site); + int voteDelayMin = getConfig().getWaitUntilVoteDelayVoteDelayMin(site); + + long lastVote = getLastVotesTime(uuid, data, site, service, player, includeTimeChangeQueue); + if (lastVote == 0) { + debug("No last vote time found for " + uuid + "/" + service + ", skipping vote delay check"); + return true; + } + + try { + LocalDateTime now = getBungeeTimeChecker().getTime(); + LocalDateTime lastVoteTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastVote), ZoneId.systemDefault()) + .plusHours(getConfig().getTimeHourOffSet()); + + if (!getConfig().getWaitUntilVoteDelayVoteDelayDaily(site)) { + if (voteDelay == 0 && voteDelayMin == 0) { + debug("Vote delay is 0 for " + site + ", skipping vote delay check"); + return true; + } + + LocalDateTime nextvote = lastVoteTime.plusHours((long) voteDelay).plusMinutes((long) voteDelayMin); + return now.isAfter(nextvote); + } + LocalDateTime resetTime = lastVoteTime.withHour(getConfig().getWaitUntilVoteDelayVoteDelayHour(site)) + .withMinute(0).withSecond(0); + LocalDateTime resetTimeTomorrow = resetTime.plusHours(24); + + if (lastVoteTime.isBefore(resetTime)) { + if (now.isAfter(resetTime)) { + debug("Vote delay is met for " + uuid + "/" + service + ", vote can be processed"); + return true; + } + } else { + if (now.isAfter(resetTimeTomorrow)) { + debug("Vote delay is met for " + uuid + "/" + service + ", vote can be processed"); + return true; + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + debug("Vote delay is not met for " + uuid + "/" + service + ", skipping vote"); + return false; + } + + public synchronized void vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid) { + vote(player, service, realVote, timeQueue, queueTime, text, uuid, null); + } + + private enum QueuedVoteResult { + SUCCESS, RETRY, TERMINAL + } + + private synchronized QueuedVoteResult vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid, VoteTimeQueue queuedVote) { + try { + if (!ServiceSiteValidator.isValid(service)) { + warn("Rejected vote with invalid service site '" + ServiceSiteValidator.sanitizeForLog(service) + "'"); + return QueuedVoteResult.TERMINAL; + } + if (!MinecraftUsernameValidator.isValid(player, getConfig().getBedrockPlayerPrefix())) { + warn("Rejected vote with invalid Minecraft username '" + + MinecraftUsernameValidator.sanitizeForLog(player) + "' from service '" + + MinecraftUsernameValidator.sanitizeForLog(service) + "'"); + return QueuedVoteResult.TERMINAL; + } + + UUID voteId = queuedVote == null ? null : queuedVote.getVoteId(); + if (voteId == null) { + voteId = UUID.randomUUID(); + } + + // UUID resolution + if (!getConfig().getOnlineMode()) { + uuid = getUUID(player); + } + + if (uuid == null || uuid.isEmpty()) { + uuid = getUUID(player); + + // Bedrock prefix auto-detect + if (uuid.isEmpty() && !getConfig().getBedrockPlayerPrefix().isEmpty() + && !player.startsWith(getConfig().getBedrockPlayerPrefix())) { + String uuid1 = getUUID(getConfig().getBedrockPlayerPrefix() + player); + if (!uuid1.isEmpty()) { + debug("Detected bedrock player without prefix, adjusting..."); + player = getConfig().getBedrockPlayerPrefix() + player; + uuid = uuid1; + } + } + } + + if (uuid.isEmpty()) { + if (player.startsWith(getConfig().getBedrockPlayerPrefix())) { + log("Ignoring vote since unable to get UUID of bedrock player"); + return QueuedVoteResult.TERMINAL; + } + if (!getConfig().getAllowUnJoined()) { + log("Ignoring vote from " + player + " since player hasn't joined before"); + return QueuedVoteResult.TERMINAL; + } + if (!getConfig().getUUIDLookup()) { + log("Failed to get uuid for " + player); + return QueuedVoteResult.TERMINAL; + } + + debug("Fetching UUID online, since allowunjoined is enabled"); + UUID u = null; + try { + if (getConfig().getOnlineMode()) { + u = fetchUUID(player); + } + } catch (Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + if (u == null) { + debug("Failed to get uuid for " + player); + return QueuedVoteResult.TERMINAL; + } + uuid = u.toString(); + } + + // Normalize UUID string if possible + try { + if (uuid != null && !uuid.isEmpty() && !uuid.equalsIgnoreCase("null")) { + uuid = UUID.fromString(uuid.trim()).toString(); + } + } catch (Exception ignored) { + // ignore + } + + player = getProperName(uuid, player); + + // Cache online state/server once (IMPORTANT for broadcast logic correctness) + final boolean playerOnline = isPlayerOnlineForVoteRouting(player); + final String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(player) : null; + long time = queueTime != 0 ? queueTime + : LocalDateTime.now().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + + Set broadcastTargets = queuedVote == null ? new LinkedHashSet<>() + : new LinkedHashSet<>(queuedVote.getBroadcastTargets()); + Set broadcastForwardedServers = queuedVote == null ? new LinkedHashSet<>() + : new LinkedHashSet<>(queuedVote.getBroadcastForwardedServers()); + boolean proxyBroadcastHandled = queuedVote != null && queuedVote.isProxyBroadcastHandled(); + boolean processesTotals = getConfig().getPrimaryServer() || !getConfig().getMultiProxySupport(); + boolean managesTotals = processesTotals && getConfig().getBungeeManageTotals(); + boolean canValidateStandaloneBroadcast = canForwardStandaloneBroadcast(managesTotals); + ArrayList data = null; + boolean queueForTimeChange = false; + + // A completion callback can wipe totals and replay older queued votes. Run it + // before loading this vote's database snapshot so the calculations below use + // the post-rollover state. + if (getConfig().getGlobalDataEnabled() && getGlobalDataHandler().isTimeChangedHappened()) { + getGlobalDataHandler().checkForFinishedTimeChanges(); + queueForTimeChange = timeQueue && getGlobalDataHandler().isTimeChangedHappened(); + } + + // Validate the vote before any immediate announcement. This keeps duplicate + // votes rejected by the delay check out of the GlobalData rollover queue and + // prevents announcing a vote that will not be processed. + if (managesTotals) { + if (getProxyMySQL() == null) { + logSevere("Mysql is not loaded correctly, stopping vote processing"); + return QueuedVoteResult.RETRY; + } + + if (!getProxyMySQL().containsKeyQuery(uuid)) { + getProxyMySQL().update(uuid, "PlayerName", new DataValueString(player)); + getProxyMySQL().getUuids().add(uuid); + } + + data = getProxyMySQL().getExactQuery(new Column("uuid", new DataValueString(uuid))); + if (!checkVoteDelay(uuid, player, service, data, queuedVote == null)) { + log("Vote delay is not met for " + player + "/" + service + ", skipping vote"); + sendVoteDelayRejected(player, uuid, service, playerOnline, playerServer); + return QueuedVoteResult.TERMINAL; + } + } + + // Forward an accepted offline broadcast before the still-active GlobalData + // change queues the reward/totals work. The queued delivery state prevents + // replaying broadcasts that already reached a backend. + if (queueForTimeChange) { + VoteTotalsSnapshot projectedTotals = managesTotals ? getProjectedRolloverTotals(data, player) : text; + if (canValidateStandaloneBroadcast && proxyBroadcastDecider.usesImmediateForwarding(playerOnline)) { + broadcastTargets.addAll(proxyBroadcastDecider.resolveTargets(false, null)); + proxyBroadcastHandled = true; + } + VoteTimeQueue delayedVote = new VoteTimeQueue(voteId, player, service, time, + proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, + projectedTotals == null ? "" : projectedTotals.toString(), false, uuid); + if (!getVoteCacheHandler().addTimeVoteToCache(delayedVote)) { + logSevere("Unable to persist queued rollover vote for " + player + "/" + service + + "; skipping proxy broadcast"); + return QueuedVoteResult.RETRY; + } + if (proxyBroadcastHandled) { + for (String target : broadcastTargets) { + Set forwarded = sendProxyBroadcast(Collections.singleton(target), uuid, player, + service, time, projectedTotals == null ? "" : projectedTotals.toString(), false); + if (delayedVote.getBroadcastForwardedServers().addAll(forwarded)) { + broadcastForwardedServers.addAll(forwarded); + persistTimeVoteDelivery(delayedVote); + } + } + } + log("Caching vote from " + player + "/" + service + + " because time change is happening right now"); + return QueuedVoteResult.SUCCESS; + } + + addVoteParty(); + + // Totals processing (primary server OR no multiproxy) + if (processesTotals) { + if (managesTotals) { + int allTimeTotal = getValue(data, "AllTimeTotal", 1); + int monthTotal = getValue(data, "MonthTotal", 1); + + int dateMonthTotal = -1; + if (getConfig().getStoreMonthTotalsWithDate()) { + if (getConfig().getUseMonthDateTotalsAsPrimaryTotal()) { + dateMonthTotal = getValue(data, getMonthTotalsWithDatePath(), 1); + } else { + dateMonthTotal = monthTotal; + } + } + + int weeklyTotal = getValue(data, "WeeklyTotal", 1); + int dailyTotal = getValue(data, "DailyTotal", 1); + int points = getValue(data, "Points", getConfig().getPointsOnVote()); + + int maxVotes = getConfig().getMaxAmountOfVotesPerDay(); + if (maxVotes > 0) { + LocalDateTime cTime = getBungeeTimeChecker().getTime(); + int days = cTime.getDayOfMonth(); + if (monthTotal > days * maxVotes) { + monthTotal = days * maxVotes; + } + } + + if (getConfig().getLimitVotePoints() > 0 && points > getConfig().getLimitVotePoints()) { + points = getConfig().getLimitVotePoints(); + } + + text = new VoteTotalsSnapshot(allTimeTotal, monthTotal, weeklyTotal, dailyTotal, points, + votePartyVotes, currentVotePartyVotesRequired, dateMonthTotal); + + ArrayList update = new ArrayList<>(); + update.add(new Column("AllTimeTotal", new DataValueInt(allTimeTotal))); + update.add(new Column("MonthTotal", new DataValueInt(monthTotal))); + if (getConfig().getStoreMonthTotalsWithDate()) { + update.add(new Column(getMonthTotalsWithDatePath(), new DataValueInt(dateMonthTotal))); + } + update.add(new Column("WeeklyTotal", new DataValueInt(weeklyTotal))); + update.add(new Column("DailyTotal", new DataValueInt(dailyTotal))); + update.add(new Column("Points", new DataValueInt(points))); + + debug("Setting totals " + text.toString() + ", voteId=" + voteId + " for " + player + "/" + + service); + getProxyMySQL().update(uuid, update); + } else { + text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); + } + } + if (text == null) { + text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); + } + + VoteLogStatus voteStatus = VoteLogStatus.IMMEDIATE; + boolean standaloneProxyBroadcast = canValidateStandaloneBroadcast && (proxyBroadcastHandled + || proxyBroadcastDecider.usesImmediateForwarding(playerOnline)); + Set proxyBroadcastTargets = Collections.emptySet(); + if (standaloneProxyBroadcast) { + // A handled queued broadcast was necessarily sampled while the player was + // offline. Retry only targets that did not previously accept delivery. + proxyBroadcastTargets = proxyBroadcastHandled ? new LinkedHashSet<>(broadcastTargets) + : proxyBroadcastDecider.resolveTargets(false, null); + Set remainingTargets = new LinkedHashSet<>(proxyBroadcastTargets); + remainingTargets.removeAll(broadcastForwardedServers); + broadcastForwardedServers.addAll(sendProxyBroadcast(remainingTargets, uuid, player, service, time, + text == null ? "" : text.toString(), false)); + } + + // =========================== + // Send vote(s) to backend(s) + // =========================== + if (getConfig().getSendVotesToAllServers()) { + for (String s : getAllAvailableServers()) { + + boolean forceCache = getConfig().getWaitForUserOnline() + && (!playerOnline || playerServer == null || !playerServer.equalsIgnoreCase(s)); + + if (forceCache) { + debug("Forcing vote to cache for server " + s); + } + + if ((!isSomeoneOnlineServerForVoteRouting(s) && method.requiresPlayerOnline()) || forceCache) { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addServerVote(s, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, + text.toString(), broadcastForwarded, standaloneProxyBroadcast, + proxyBroadcastTargets, broadcastForwardedServers, false)); + debug("Caching vote for " + player + " on " + service + " for " + s); + } else { + boolean broadcastHere = !broadcastForwardedServers.contains(s); + if (broadcastHere && getConfig().getProxyBroadcastEnabled()) { + Set targets = standaloneProxyBroadcast ? proxyBroadcastTargets + : proxyBroadcastDecider.resolveTargets(playerOnline, playerServer); + broadcastHere = proxyBroadcastDecider.shouldBroadcast(s, targets); + } + + if (!sendVoteEnvelopeAccepted(s, 2, + VotingPluginWire.vote(player, uuid, service, time, true, realVote, text.toString(), + voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1))) { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addServerVote(s, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, + text.toString(), broadcastForwarded, standaloneProxyBroadcast, + proxyBroadcastTargets, broadcastForwardedServers, false)); + debug("Caching vote after the transport rejected delivery for " + s); + } + } + } + } else { + // Single-server mode: online goes to player server; otherwise queue as "online + // vote" + if (playerOnline && playerServer != null && getAllAvailableServers().contains(playerServer)) { + String server = playerServer; + + boolean broadcastHere = !broadcastForwardedServers.contains(server); + if (broadcastHere && getConfig().getProxyBroadcastEnabled()) { + Set targets = standaloneProxyBroadcast ? proxyBroadcastTargets + : proxyBroadcastDecider.resolveTargets(true, playerServer); + broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); + } + + boolean rewardAccepted = sendVoteEnvelopeAccepted(server, 1, + VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(), + voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1)); + if (!rewardAccepted) { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addOnlineVote(uuid, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), + broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, + broadcastForwardedServers, false)); + debug("Caching online vote after the transport rejected delivery for " + server); + } + + if (rewardAccepted && canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled() + && !standaloneProxyBroadcast) { + Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); + + int bDelay = 2; + for (String targetServer : targets) { + // avoid double-broadcast on the same server that already got the voteOnline + if (targetServer.equalsIgnoreCase(server)) { + continue; + } + if (getConfig().getBlockedServers().contains(targetServer)) { + continue; + } + + globalMessageProxyHandler.sendMessage(targetServer, bDelay, + VotingPluginWire.voteBroadcast(uuid, player, service, time, + text == null ? "" : text.toString(), true)); + bDelay++; + } + } + + // multiproxy: envelope-only clear vote + if (rewardAccepted && getConfig().getMultiProxySupport() && getConfig().getMultiProxyOneGlobalReward()) { + multiProxyHandler.sendClearVote(uuid, player); + } + } else { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addOnlineVote(uuid, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), + broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, + broadcastForwardedServers, false)); + debug("Caching online vote for " + player + " on " + service); + } + + int delay = 2; + for (String s : getAllAvailableServers()) { + globalMessageProxyHandler.sendMessage(s, delay + 1, VotingPluginWire.voteUpdate(uuid, + votePartyVotes, currentVotePartyVotesRequired, service, time, text.toString())); + delay += 2; + } + } + + // Vote logging + if (voteLogMysqlTable != null && getConfig().getVoteLoggingEnabled()) { + voteLogMysqlTable.logVote(voteId, voteStatus, service, uuid, player, time, + getVoteCacheHandler().getProxyCachedTotal(uuid)); + } + + // =========================== + // Multiproxy forwarding + // =========================== + if (getConfig().getMultiProxySupport() && getConfig().getPrimaryServer()) { + if (!getConfig().getMultiProxyOneGlobalReward()) { + debug("Sending global proxy vote envelope"); + multiProxyHandler.sendMultiProxyEnvelope(VotingPluginWire.vote(player, uuid, service, time, false, + realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); + } else { + // Only send to other proxies if the player DID NOT already receive reward on a + // backend + boolean shouldSend = true; + if (playerOnline && playerServer != null) { + if (!getConfig().getBlockedServers().contains(playerServer)) { + shouldSend = false; + } + } + + if (shouldSend) { + debug("Sending global proxy voteonline envelope"); + multiProxyHandler + .sendMultiProxyEnvelope(VotingPluginWire.voteOnline(player, uuid, service, time, false, + realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); + } else { + debug("Not sending global proxy message for voteonline, player already got reward"); + } + } + } + if (queuedVote != null) { + queuedVote.setProcessed(true); + if (!getVoteCacheHandler().updateTimeVote(queuedVote)) { + warn("Unable to persist completed rollover vote " + queuedVote.getVoteId() + + "; attempting durable removal immediately"); + } + } + return QueuedVoteResult.SUCCESS; + } catch (Exception e) { + e.printStackTrace(); + return QueuedVoteResult.RETRY; + } + } + + private static final class PendingPresenceHandoff { + private UUID requestId; + private final UUID playerUuid; + private final String playerName; + private final String uuid; + private final String server; + private final UUID connectionId; + private final UUID backendIncarnationId; + private final long backendStartedAt; + private final long conflictSequence; + private final long createdAt; + + private PendingPresenceHandoff(String playerName, String uuid, String server, UUID connectionId, + UUID backendIncarnationId, long backendStartedAt, long conflictSequence, long createdAt) { + this.playerUuid = parsePlayerUuid(uuid); + this.playerName = playerName; + this.uuid = uuid; + this.server = server; + this.connectionId = connectionId; + this.backendIncarnationId = backendIncarnationId; + this.backendStartedAt = backendStartedAt; + this.conflictSequence = conflictSequence; + this.createdAt = createdAt; + } + + private static UUID parsePlayerUuid(String uuid) { + try { + return UUID.fromString(uuid.trim()); + } catch (Exception ignored) { + return null; + } + } + } + + public abstract void warn(String message); + + public abstract ScheduledExecutorService getScheduler(); +} From 0e9cf5ac05a35eae30c929653f3a1d320aeb04b8 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 21:18:00 -0600 Subject: [PATCH 23/36] fix(test): disambiguate transport validation assertion --- .../backendproxy/transport/HttpBackendProxyTransportTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index dd2fa418a..5c1a68650 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -47,7 +47,7 @@ void validatesInitialConnectionCodeSynchronously() { "invalid configuration must fail before the enrollment worker starts"); when(settings.getHttpConnectionCode()).thenReturn(code("lobby-1", Instant.now().plusSeconds(60)).encode()); - assertDoesNotThrow(transport::validate); + assertDoesNotThrow(() -> transport.validate()); } @Test From f81aa90ba66e32f508e2dd8f5c342c563004c54d Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 21:30:53 -0600 Subject: [PATCH 24/36] fix(http): validate readiness off the server thread --- .../votingplugin/VotingPluginMain.java | 70 +++++++++++++++---- .../http/HttpBackendTransportConnector.java | 9 +++ .../http/HttpClientCredentialStore.java | 7 +- .../backendproxy/http/HttpTlsIdentity.java | 8 ++- .../http/HttpTransportProtocol.java | 11 ++- .../transport/HttpBackendProxyTransport.java | 6 +- .../control/BackendControlConnector.java | 46 ++++++++++-- .../http/HttpTransportRuntimeTest.java | 27 +++++++ .../HttpBackendProxyTransportTest.java | 6 +- 9 files changed, 163 insertions(+), 27 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index a2d243fa5..26d26f3ef 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1214,34 +1214,74 @@ public String getBackendHostedControlStatus() { } /** Recreates proxy transports after Control applies BungeeSettings.yml. */ - public synchronized void restartBackendProxyHandler() { + public void restartBackendProxyHandler() { restartBackendProxyHandler(System.nanoTime() + TimeUnit.SECONDS.toNanos(25)); } /** Recreates proxy transports while preserving the caller's end-to-end validation deadline. */ - public synchronized void restartBackendProxyHandler(long validationDeadlineNanos) { + public void restartBackendProxyHandler(long validationDeadlineNanos) { + BackendProxyRestart restart = prepareBackendProxyHandlerRestart(); + try { + validateBackendProxyHandlerRestart(restart, validationDeadlineNanos); + completeBackendProxyHandlerRestart(restart); + } catch (RuntimeException failure) { + abortBackendProxyHandlerRestart(restart); + throw failure; + } + } + + /** Prepared on the Bukkit thread, validated off-thread, then atomically published on Bukkit. */ + public static final class BackendProxyRestart { + private final BackendProxyHandler previous; + private final BackendProxyHandler replacement; + private final boolean disabled; + private boolean finished; + + private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler replacement, boolean disabled) { + this.previous = previous; + this.replacement = replacement; + this.disabled = disabled; + } + } + + public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { BackendProxyHandler previous = backendProxyHandler; if (!bungeeSettings.isUseBungeecoord()) { - backendProxyHandler = null; - if (previous != null) previous.close(); - BackendControlAutoEnrollment enrollment = backendControlAutoEnrollment; - backendControlAutoEnrollment = null; - if (enrollment != null) enrollment.close(); - return; + return new BackendProxyRestart(previous, null, true); } BungeeMethod replacementMethod = BungeeMethod.getByName(bungeeSettings.getBungeeMethod()); if (previous != null) previous.prepareForReplacement(replacementMethod); BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); try { replacement.load(); - replacement.validateTransport(validationDeadlineNanos); - if (previous != null) previous.completeRedisHandoff(replacement); } catch (RuntimeException failure) { replacement.close(); throw failure; } - backendProxyHandler = replacement; - if (previous != null) previous.close(); + return new BackendProxyRestart(previous, replacement, false); + } + + public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long validationDeadlineNanos) { + if (restart == null) throw new IllegalArgumentException("Backend proxy restart is required"); + if (restart.replacement != null) restart.replacement.validateTransport(validationDeadlineNanos); + } + + public synchronized void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { + if (restart == null || restart.finished) throw new IllegalStateException("Backend proxy restart is no longer active"); + if (backendProxyHandler != restart.previous) throw new IllegalStateException("Backend proxy handler changed during restart"); + if (restart.disabled) { + backendProxyHandler = null; + if (restart.previous != null) restart.previous.close(); + BackendControlAutoEnrollment enrollment = backendControlAutoEnrollment; + backendControlAutoEnrollment = null; + if (enrollment != null) enrollment.close(); + restart.finished = true; + return; + } + if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); + backendProxyHandler = restart.replacement; + if (restart.previous != null) restart.previous.close(); + restart.finished = true; try { refreshBackendControlAutoEnrollment(); } catch (IOException e) { @@ -1249,6 +1289,12 @@ public synchronized void restartBackendProxyHandler(long validationDeadlineNanos } } + public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart restart) { + if (restart == null || restart.finished) return; + if (restart.replacement != null) restart.replacement.close(); + restart.finished = true; + } + /** Keeps one plugin-message listener for the plugin lifetime and atomically swaps its active backend handler. */ public synchronized void activateBackendPluginMessageHandler(GlobalMessageHandler target) { backendPluginMessageTarget.set(target); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 820d4ab5e..f05f5eb61 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -21,6 +21,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadFactory; @@ -45,6 +46,7 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private final URI transportEndpoint; private final ThreadPoolExecutor callbackExecutor; private final AtomicBoolean running = new AtomicBoolean(); + private final CountDownLatch firstResponse = new CountDownLatch(1); private final Object state = new Object(); private final Object renewal = new Object(); private final LinkedHashMap outgoing = new LinkedHashMap<>(); @@ -132,6 +134,11 @@ public void start() { if (!running.compareAndSet(false, true)) return; poller = new Thread(this::pollLoop, "VotingPlugin-HTTP-poll"); poller.setDaemon(true); poller.start(); } + /** Waits for one authenticated, protocol-valid transport response. */ + public boolean awaitFirstResponse(long deadlineNanos) throws InterruptedException { + long remaining = deadlineNanos - System.nanoTime(); + return remaining > 0L && firstResponse.await(remaining, TimeUnit.NANOSECONDS) && running.get(); + } /** * Inserts an in-memory at-least-once delivery. It survives retry/lost responses while this process remains alive; * callers needing restart durability must retain the application operation independently. @@ -167,12 +174,14 @@ public synchronized boolean pollOnce() { acknowledgementsConfirmed = true; synchronized (state) { for (String ack : packet.acks()) outgoing.remove(ack); } for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); + firstResponse.countDown(); return true; } catch (Exception failure) { return false; } finally { if (!acknowledgementsConfirmed) requeueAcknowledgements(acks); } } @Override public void close() { running.getAndSet(false); + firstResponse.countDown(); // Revoke this connector's journal writer before a replacement snapshots it. // In-flight transitions serialize with seal(): either COMPLETED is already // durable, or the delivery remains durably RUNNING and fail-closed. diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index b100cc5a7..518cf4ee7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -97,8 +97,13 @@ static StagedCredential stageReplacement(Path directory, HttpTlsIdentity.IssuedC private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClientCertificate issued, HttpClientProfile profile, String connectionCodeDigest) throws Exception { if (directory == null || issued == null) throw new IllegalArgumentException("Credential replacement is required"); - Path generations = directory.toAbsolutePath().normalize().resolve(GENERATIONS_DIRECTORY); + Path credentialDirectory = directory.toAbsolutePath().normalize(); + boolean created = !Files.exists(credentialDirectory, LinkOption.NOFOLLOW_LINKS); + Path generations = credentialDirectory.resolve(GENERATIONS_DIRECTORY); Files.createDirectories(generations); + // Credential files cannot make the newly created credential-root entry durable. + // Persist its parent before an enrolled transport can activate this root. + if (created) DurableFiles.forceDirectory(credentialDirectory.getParent()); if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP credential generation directory is unsafe"); String name = java.util.UUID.randomUUID().toString(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java index 41522eacc..5a61daefc 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -86,7 +86,13 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock if (advertisedHost == null || advertisedHost.isBlank() || advertisedHost.length() > 253) throw new IllegalArgumentException("Advertised HTTPS host is invalid"); if (clock == null) throw new IllegalArgumentException("Clock is required"); - Files.createDirectories(directory); + Path identityDirectory = directory.toAbsolutePath().normalize(); + boolean created = !Files.exists(identityDirectory, LinkOption.NOFOLLOW_LINKS); + Files.createDirectories(identityDirectory); + // The identity files cannot make the newly created directory entry durable. + // Persist its parent before the TLS identity is returned for listener use. + if (created) DurableFiles.forceDirectory(identityDirectory.getParent()); + directory = identityDirectory; Path caFile = safe(directory.resolve(CA_FILE)); Path serverFile = safe(directory.resolve(SERVER_FILE)); Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java index 3fa01f878..b3c825cdd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java @@ -214,8 +214,15 @@ private static long integer(JsonObject object, String name) { } catch (RuntimeException failure) { throw bad(); } } private static long nonNegative(JsonObject object, String name) { long n = integer(object, name); if (n < 0) throw bad(); return n; } - private static String uuid(JsonObject object, String name) { String value = string(object, name, 64); try { return UUID.fromString(value).toString(); } catch (IllegalArgumentException invalid) { throw bad(); } } - private static void validId(String id) { if (id == null || id.length() > 64) throw bad(); try { UUID.fromString(id); } catch (IllegalArgumentException invalid) { throw bad(); } } + private static String uuid(JsonObject object, String name) { return canonicalUuid(string(object, name, 64)); } + private static void validId(String id) { if (id == null || id.length() > 64) throw bad(); canonicalUuid(id); } + private static String canonicalUuid(String value) { + try { + UUID parsed = UUID.fromString(value); + if (!parsed.toString().equals(value)) throw bad(); + return value; + } catch (IllegalArgumentException invalid) { throw bad(); } + } private static IllegalArgumentException bad() { return new IllegalArgumentException("Invalid HTTP transport message"); } record Delivery(String id, JsonEnvelope envelope) { } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 944557439..2f5566382 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -65,12 +65,16 @@ private void initialize(Path directory, String serverId, String configuredCode, if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalStateException("Persisted HTTP identity belongs to a different backend Server name"); replacement = new HttpBackendTransportConnector(directory, messageHandler::onMessage); + replacement.start(); + if (!replacement.awaitFirstResponse(System.nanoTime() + + TimeUnit.SECONDS.toNanos(DEFAULT_STARTUP_VALIDATION_SECONDS))) { + throw new IllegalStateException("HTTP backend could not authenticate with the proxy"); + } boolean discard = false; synchronized (lifecycle) { if (closed) { discard = true; } else { - replacement.start(); while (!startupQueue.isEmpty()) { if (!replacement.send(startupQueue.removeFirst())) { throw new IllegalStateException("HTTP startup queue could not be transferred"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 00899595c..f2f14567f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -111,26 +111,58 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set } private void reloadConfiguration(String fileName) throws Exception { - Future reload; long validationDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(29); + Future preparation; synchronized (operationLifecycle) { if (closed) throw new IllegalStateException("Bukkit Control connector is stopping"); - reload = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { + preparation = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { plugin.reloadFromControl(); - if ("BungeeSettings.yml".equals(fileName)) plugin.restartBackendProxyHandler(validationDeadline); - return null; + return "BungeeSettings.yml".equals(fileName) ? plugin.prepareBackendProxyHandlerRestart() : null; }); - activeReload = reload; + activeReload = preparation; } + VotingPluginMain.BackendProxyRestart restart = null; try { - reload.get(30, TimeUnit.SECONDS); + restart = preparation.get(remaining(validationDeadline), TimeUnit.NANOSECONDS); + if (restart == null) return; + // Network enrollment/readiness is deliberately awaited on this Control worker, + // never on Bukkit's primary thread. + plugin.validateBackendProxyHandlerRestart(restart, validationDeadline); + VotingPluginMain.BackendProxyRestart prepared = restart; + Future publication = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { + plugin.completeBackendProxyHandlerRestart(prepared); + return null; + }); + synchronized (operationLifecycle) { + if (closed) publication.cancel(true); + activeReload = publication; + } + publication.get(remaining(validationDeadline), TimeUnit.NANOSECONDS); + } catch (Exception failure) { + if (restart != null) { + VotingPluginMain.BackendProxyRestart prepared = restart; + try { + Future abort = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { + plugin.abortBackendProxyHandlerRestart(prepared); + return null; + }); + abort.get(5, TimeUnit.SECONDS); + } catch (Exception cleanupFailure) { failure.addSuppressed(cleanupFailure); } + } + throw failure; } finally { synchronized (operationLifecycle) { - if (activeReload == reload) activeReload = null; + activeReload = null; } } } + private static long remaining(long deadlineNanos) throws java.util.concurrent.TimeoutException { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) throw new java.util.concurrent.TimeoutException("Bukkit configuration reload timed out"); + return remaining; + } + public static BackendControlConnector create(VotingPluginMain plugin) throws IOException { Path root = plugin.getDataFolder().toPath().toAbsolutePath().normalize(); BackendControlResultStore.State recovered = BackendControlResultStore.load(root); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index fbd30301f..3c2658a54 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -38,6 +38,8 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(directory.resolve("client"), envelope -> backendReceived.countDown())) { connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8)), + "an authenticated transport response must make the connector ready"); assertTrue(connector.send(JsonEnvelope.builder("to-proxy").put("server", "forged").build())); assertTrue(proxyReceived.await(8, TimeUnit.SECONDS)); assertEquals("lobby-1", received.get().serverId()); @@ -185,6 +187,31 @@ void packetNumbersMustUseCanonicalJsonIntegerTokens() { } } + @Test + void packetParsingRejectsNoncanonicalUuidForms() { + String deliveryId = java.util.UUID.randomUUID().toString(); + com.google.gson.JsonObject packet = com.google.gson.JsonParser.parseString(new String(HttpTransportProtocol.request( + "lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(deliveryId), + java.util.List.of(new HttpTransportProtocol.Delivery(deliveryId, JsonEnvelope.builder("payload").build()))), + java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); + String abbreviated = "1-1-1-1-1"; + + com.google.gson.JsonObject invalidSession = packet.deepCopy(); + invalidSession.addProperty("session", abbreviated); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidSession.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + com.google.gson.JsonObject invalidAck = packet.deepCopy(); + invalidAck.getAsJsonArray("acks").set(0, new com.google.gson.JsonPrimitive(abbreviated)); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidAck.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + com.google.gson.JsonObject invalidMessage = packet.deepCopy(); + invalidMessage.getAsJsonArray("messages").get(0).getAsJsonObject().addProperty("id", abbreviated); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidMessage.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + @Test void persistedProfileStartsAfterTheEnrollmentCodeExpires() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java index 5c1a68650..1b8491efd 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransportTest.java @@ -92,7 +92,7 @@ void closeNeverWaitsForSetupOnTheCallingThread() throws Exception { @Test @SuppressWarnings("unchecked") - void validationWaitsForThePreviousDirectoryOwner() throws Exception { + void validationWaitsForThePreviousDirectoryOwnerBeforeReadinessFailure() throws Exception { Path credentials = directory.resolve("http"); HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy-owner"), "proxy.example.test"); HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://proxy.example.test:1297/"), @@ -115,7 +115,7 @@ void validationWaitsForThePreviousDirectoryOwner() throws Exception { java.util.concurrent.atomic.AtomicReference failure = new java.util.concurrent.atomic.AtomicReference<>(); CountDownLatch finished = new CountDownLatch(1); Thread validation = new Thread(() -> { - try { transport.validate(); } + try { transport.validate(System.nanoTime() + TimeUnit.SECONDS.toNanos(1)); } catch (Throwable thrown) { failure.set(thrown); } finally { finished.countDown(); } }); @@ -124,7 +124,7 @@ void validationWaitsForThePreviousDirectoryOwner() throws Exception { predecessor.release(); assertTrue(finished.await(3, TimeUnit.SECONDS)); validation.join(TimeUnit.SECONDS.toMillis(1)); - assertNull(failure.get()); + assertTrue(failure.get() instanceof IllegalStateException); transport.close(); } From b7503b9eb877bc3af8693fca7aff6a4ac1c25840 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 21:41:49 -0600 Subject: [PATCH 25/36] fix(control): cancel stale transport publication --- .../votingplugin/control/BackendControlConnector.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index f2f14567f..1bb3edd2e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -122,6 +122,7 @@ private void reloadConfiguration(String fileName) throws Exception { activeReload = preparation; } VotingPluginMain.BackendProxyRestart restart = null; + Future publication = null; try { restart = preparation.get(remaining(validationDeadline), TimeUnit.NANOSECONDS); if (restart == null) return; @@ -129,7 +130,7 @@ private void reloadConfiguration(String fileName) throws Exception { // never on Bukkit's primary thread. plugin.validateBackendProxyHandlerRestart(restart, validationDeadline); VotingPluginMain.BackendProxyRestart prepared = restart; - Future publication = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { + publication = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { plugin.completeBackendProxyHandlerRestart(prepared); return null; }); @@ -139,6 +140,8 @@ private void reloadConfiguration(String fileName) throws Exception { } publication.get(remaining(validationDeadline), TimeUnit.NANOSECONDS); } catch (Exception failure) { + // A timed-out Bukkit publication must not remain queued ahead of rollback. + if (publication != null) publication.cancel(false); if (restart != null) { VotingPluginMain.BackendProxyRestart prepared = restart; try { From 67e9d1262ad5264c2602800274d5297c1ccc9f4f Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 31 Aug 2026 21:49:09 -0600 Subject: [PATCH 26/36] fix(http): open proxy listener after runtime setup --- .../com/bencodez/votingplugin/proxy/VotingPluginProxy.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 5608a5ce8..1a23fed4a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1576,10 +1576,6 @@ public void onReceive(JsonEnvelope message) { } }); - // Do not accept backend traffic until the router and all of its ordered - // presence/message listeners are installed. - if (method.equals(BungeeMethod.HTTP)) startHttpTransport(); - proxyBroadcastDecider = new ProxyBroadcastDecider(() -> getConfig(), () -> getAllAvailableServers(), s -> isServerValid(s), s -> getConfig().getBlockedServers() != null && getConfig().getBlockedServers().contains(s)); @@ -1592,6 +1588,9 @@ public void onReceive(JsonEnvelope message) { PRESENCE_MAINTENANCE_INTERVAL_SECONDS); } startControlServices(); + // Open the listener last: backend callbacks can immediately reach routing, + // presence, vote-log, multi-proxy, and Control-adjacent runtime helpers. + if (method.equals(BungeeMethod.HTTP)) startHttpTransport(); debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); } From 425664be7cf08f51b693d1f4c68534a04148a6af Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 04:48:52 -0600 Subject: [PATCH 27/36] Restore HTTP transport after failed control reload --- .../votingplugin/VotingPluginMain.java | 15 +++++++++++---- .../backendproxy/BackendProxyHandler.java | 9 ++++++++- .../BackendProxyTransportManager.java | 16 ++++++++++++++++ .../transport/HttpBackendProxyTransport.java | 19 +++++++++++++++++++ .../control/BackendControlConnector.java | 16 ++++++++++++++-- 5 files changed, 68 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 26d26f3ef..867137251 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1235,30 +1235,34 @@ public static final class BackendProxyRestart { private final BackendProxyHandler previous; private final BackendProxyHandler replacement; private final boolean disabled; + private final boolean previousPrepared; private boolean finished; - private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler replacement, boolean disabled) { + private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler replacement, boolean disabled, + boolean previousPrepared) { this.previous = previous; this.replacement = replacement; this.disabled = disabled; + this.previousPrepared = previousPrepared; } } public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { BackendProxyHandler previous = backendProxyHandler; if (!bungeeSettings.isUseBungeecoord()) { - return new BackendProxyRestart(previous, null, true); + return new BackendProxyRestart(previous, null, true, false); } BungeeMethod replacementMethod = BungeeMethod.getByName(bungeeSettings.getBungeeMethod()); - if (previous != null) previous.prepareForReplacement(replacementMethod); + boolean previousPrepared = previous != null && previous.prepareForReplacement(replacementMethod); BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); try { replacement.load(); } catch (RuntimeException failure) { replacement.close(); + if (previousPrepared) previous.restoreAfterFailedReplacement(); throw failure; } - return new BackendProxyRestart(previous, replacement, false); + return new BackendProxyRestart(previous, replacement, false, previousPrepared); } public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long validationDeadlineNanos) { @@ -1292,6 +1296,9 @@ public synchronized void completeBackendProxyHandlerRestart(BackendProxyRestart public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart restart) { if (restart == null || restart.finished) return; if (restart.replacement != null) restart.replacement.close(); + if (restart.previousPrepared && backendProxyHandler == restart.previous) { + restart.previous.restoreAfterFailedReplacement(); + } restart.finished = true; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index f485374bb..10062b023 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -102,10 +102,17 @@ public void close() { } /** Releases a same-method subscriber/listener before its replacement starts. */ - public void prepareForReplacement(BungeeMethod replacementMethod) { + public boolean prepareForReplacement(BungeeMethod replacementMethod) { if (method == replacementMethod && method != BungeeMethod.PLUGINMESSAGING && method != BungeeMethod.REDIS) { transportManager.prepareForReplacement(); + return method == BungeeMethod.HTTP; } + return false; + } + + /** Restores a prepared HTTP transport when its replacement fails validation. */ + public void restoreAfterFailedReplacement() { + transportManager.restorePreparedTransport(); } /** Fails a configuration apply when its selected transport did not initialize. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index fecb46302..6446a23d0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -19,6 +19,7 @@ public class BackendProxyTransportManager { private final VotingPluginMain plugin; private final ProcessedVoteCache processedVoteCache; private BackendProxyTransport transport; + private BackendProxyTransport preparedTransport; public BackendProxyTransportManager(VotingPluginMain plugin) { this(plugin, new ProcessedVoteCache()); @@ -67,6 +68,10 @@ public void close() { transport.close(); transport = null; } + if (preparedTransport != null) { + preparedTransport.close(); + preparedTransport = null; + } } public void validate() { @@ -82,10 +87,21 @@ public void validate(long deadlineNanos) { public void prepareForReplacement() { if (transport != null) { transport.prepareForReplacement(); + preparedTransport = transport; transport = null; } } + public void restorePreparedTransport() { + if (transport != null || preparedTransport == null) return; + if (preparedTransport instanceof HttpBackendProxyTransport http) { + transport = http.recreatePrepared(); + } else { + throw new IllegalStateException("Prepared backend proxy transport cannot be restored"); + } + preparedTransport = null; + } + public void closeRedisForHandoff() { if (!(transport instanceof RedisBackendProxyTransport)) { throw new IllegalStateException("Redis backend proxy transport is unavailable"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 2f5566382..e1dada297 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -29,6 +29,10 @@ public final class HttpBackendProxyTransport implements BackendProxyTransport { private volatile RuntimeException startupFailure; private volatile boolean started; private volatile boolean closed; + private Path configuredDirectory; + private String configuredServerId; + private String configuredConnectionCode; + private GlobalMessageHandler configuredMessageHandler; private Semaphore directoryOwner; private final java.util.concurrent.atomic.AtomicBoolean queueWarning = new java.util.concurrent.atomic.AtomicBoolean(); @@ -41,7 +45,16 @@ public void start(GlobalMessageHandler messageHandler) { Path directory = plugin.getDataFolder().toPath().resolve("http"); String serverId = plugin.getBungeeSettings().getServer(); String connectionCode = plugin.getBungeeSettings().getHttpConnectionCode(); + start(directory, serverId, connectionCode, messageHandler); + } + + private void start(Path directory, String serverId, String connectionCode, + GlobalMessageHandler messageHandler) { validateConfiguration(directory, serverId, connectionCode); + configuredDirectory = directory; + configuredServerId = serverId; + configuredConnectionCode = connectionCode; + configuredMessageHandler = messageHandler; started = true; worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler), "VotingPlugin-HTTP-Backend-Setup"); @@ -49,6 +62,12 @@ public void start(GlobalMessageHandler messageHandler) { worker.start(); } + HttpBackendProxyTransport recreatePrepared() { + HttpBackendProxyTransport restored = new HttpBackendProxyTransport(plugin); + restored.start(configuredDirectory, configuredServerId, configuredConnectionCode, configuredMessageHandler); + return restored; + } + private void initialize(Path directory, String serverId, String configuredCode, GlobalMessageHandler messageHandler) { Path ownerKey = directory.toAbsolutePath().normalize(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 1bb3edd2e..804b0312b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -22,6 +22,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.regex.Pattern; @@ -112,12 +113,20 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set private void reloadConfiguration(String fileName) throws Exception { long validationDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(29); + AtomicBoolean preparationAbandoned = new AtomicBoolean(); + AtomicReference preparedRestart = new AtomicReference<>(); Future preparation; synchronized (operationLifecycle) { if (closed) throw new IllegalStateException("Bukkit Control connector is stopping"); preparation = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { plugin.reloadFromControl(); - return "BungeeSettings.yml".equals(fileName) ? plugin.prepareBackendProxyHandlerRestart() : null; + VotingPluginMain.BackendProxyRestart prepared = "BungeeSettings.yml".equals(fileName) + ? plugin.prepareBackendProxyHandlerRestart() : null; + preparedRestart.set(prepared); + if (prepared != null && preparationAbandoned.get()) { + plugin.abortBackendProxyHandlerRestart(prepared); + } + return prepared; }); activeReload = preparation; } @@ -140,8 +149,11 @@ private void reloadConfiguration(String fileName) throws Exception { } publication.get(remaining(validationDeadline), TimeUnit.NANOSECONDS); } catch (Exception failure) { - // A timed-out Bukkit publication must not remain queued ahead of rollback. + // Timed-out Bukkit work must not remain queued ahead of configuration rollback. + preparationAbandoned.set(true); + preparation.cancel(false); if (publication != null) publication.cancel(false); + if (restart == null) restart = preparedRestart.get(); if (restart != null) { VotingPluginMain.BackendProxyRestart prepared = restart; try { From 57517d7593234c05c7ef7e4718c7a98c7d444f02 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 17:32:42 -0600 Subject: [PATCH 28/36] fix(http): restore credentials after failed reload --- .../votingplugin/VotingPluginMain.java | 69 +++++++++++--- .../backendproxy/BackendProxyHandler.java | 4 + .../http/HttpClientCredentialStore.java | 38 ++++++++ .../BackendProxyTransportManager.java | 4 + .../transport/HttpBackendProxyTransport.java | 62 +++++++++++- .../control/BackendControlConnector.java | 94 +++++++++++++++++-- .../http/HttpTransportSecurityTest.java | 24 +++++ 7 files changed, 270 insertions(+), 25 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 867137251..259f3fa0c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1237,6 +1237,8 @@ public static final class BackendProxyRestart { private final boolean disabled; private final boolean previousPrepared; private boolean finished; + private boolean abandonmentRequested; + private volatile boolean published; private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler replacement, boolean disabled, boolean previousPrepared) { @@ -1247,6 +1249,18 @@ private BackendProxyRestart(BackendProxyHandler previous, BackendProxyHandler re } } + public static final class BackendProxyRestartPreparationException extends RuntimeException { + private static final long serialVersionUID = 1L; + private final BackendProxyRestart restart; + + private BackendProxyRestartPreparationException(BackendProxyRestart restart, RuntimeException cause) { + super(cause); + this.restart = restart; + } + + public BackendProxyRestart restart() { return restart; } + } + public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { BackendProxyHandler previous = backendProxyHandler; if (!bungeeSettings.isUseBungeecoord()) { @@ -1259,7 +1273,12 @@ public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { replacement.load(); } catch (RuntimeException failure) { replacement.close(); - if (previousPrepared) previous.restoreAfterFailedReplacement(); + if (previousPrepared) { + previous.restoreAfterFailedReplacement(); + BackendProxyRestart failed = new BackendProxyRestart(previous, null, false, true); + failed.finished = true; + throw new BackendProxyRestartPreparationException(failed, failure); + } throw failure; } return new BackendProxyRestart(previous, replacement, false, previousPrepared); @@ -1270,22 +1289,30 @@ public void validateBackendProxyHandlerRestart(BackendProxyRestart restart, long if (restart.replacement != null) restart.replacement.validateTransport(validationDeadlineNanos); } - public synchronized void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { - if (restart == null || restart.finished) throw new IllegalStateException("Backend proxy restart is no longer active"); - if (backendProxyHandler != restart.previous) throw new IllegalStateException("Backend proxy handler changed during restart"); - if (restart.disabled) { - backendProxyHandler = null; + public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { + synchronized (this) { + if (restart == null || restart.finished) throw new IllegalStateException("Backend proxy restart is no longer active"); + if (restart.abandonmentRequested) { + abortBackendProxyHandlerRestart(restart); + return; + } + if (backendProxyHandler != restart.previous) throw new IllegalStateException("Backend proxy handler changed during restart"); + if (restart.disabled) { + backendProxyHandler = null; + if (restart.previous != null) restart.previous.close(); + BackendControlAutoEnrollment enrollment = backendControlAutoEnrollment; + backendControlAutoEnrollment = null; + if (enrollment != null) enrollment.close(); + restart.finished = true; + restart.published = true; + return; + } + if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); + backendProxyHandler = restart.replacement; if (restart.previous != null) restart.previous.close(); - BackendControlAutoEnrollment enrollment = backendControlAutoEnrollment; - backendControlAutoEnrollment = null; - if (enrollment != null) enrollment.close(); restart.finished = true; - return; + restart.published = true; } - if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); - backendProxyHandler = restart.replacement; - if (restart.previous != null) restart.previous.close(); - restart.finished = true; try { refreshBackendControlAutoEnrollment(); } catch (IOException e) { @@ -1293,6 +1320,14 @@ public synchronized void completeBackendProxyHandlerRestart(BackendProxyRestart } } + /** Returns false once publication committed and can no longer be rolled back as a failed apply. */ + public synchronized boolean requestBackendProxyHandlerRestartAbandonment(BackendProxyRestart restart) { + if (restart == null) return true; + if (restart.published) return false; + restart.abandonmentRequested = true; + return true; + } + public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart restart) { if (restart == null || restart.finished) return; if (restart.replacement != null) restart.replacement.close(); @@ -1302,6 +1337,12 @@ public synchronized void abortBackendProxyHandlerRestart(BackendProxyRestart res restart.finished = true; } + public void awaitBackendProxyHandlerRollback(BackendProxyRestart restart, long deadlineNanos) { + if (restart != null && restart.previousPrepared) { + restart.previous.awaitRestoreAfterFailedReplacement(deadlineNanos); + } + } + /** Keeps one plugin-message listener for the plugin lifetime and atomically swaps its active backend handler. */ public synchronized void activateBackendPluginMessageHandler(GlobalMessageHandler target) { backendPluginMessageTarget.set(target); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 10062b023..ec7998b54 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -115,6 +115,10 @@ public void restoreAfterFailedReplacement() { transportManager.restorePreparedTransport(); } + public void awaitRestoreAfterFailedReplacement(long deadlineNanos) { + transportManager.awaitPreparedTransportRestoration(deadlineNanos); + } + /** Fails a configuration apply when its selected transport did not initialize. */ public void validateTransport() { validateTransport(System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(25)); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index 518cf4ee7..b9b835ef3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -183,12 +183,43 @@ public static boolean matchesEnrollmentCode(Path directory, HttpConnectionCode c /** Loads and cross-checks the persisted client key material and bound normal-transport profile. */ public static EnrolledClient loadEnrolled(Path directory) throws Exception { Path active = activeDirectory(directory); + return loadEnrolledDirectory(active); + } + + private static EnrolledClient loadEnrolledDirectory(Path active) throws Exception { ClientCredential credential = loadCredential(active); HttpClientProfile profile = loadProfileFile(active); if (!matchesProfile(credential, profile)) throw new IOException("HTTP client certificate does not match its profile"); return new EnrolledClient(profile, credential); } + /** Captures the exact credential generation used by a transport before a staged replacement starts. */ + public static ActiveCredentialGeneration snapshotActiveGeneration(Path directory) throws Exception { + Path root = directory.toAbsolutePath().normalize(); + Path active = activeDirectory(root); + loadEnrolledDirectory(active); + return new ActiveCredentialGeneration(active.equals(root) ? "" : active.getFileName().toString()); + } + + /** Atomically restores a previously validated credential generation after replacement rollback. */ + public static void restoreActiveGeneration(Path directory, ActiveCredentialGeneration snapshot) throws Exception { + if (directory == null || snapshot == null) throw new IllegalArgumentException("Credential rollback is required"); + Path root = directory.toAbsolutePath().normalize(); + Path generations = root.resolve(GENERATIONS_DIRECTORY); + Path target = snapshot.name().isEmpty() ? root : generations.resolve(snapshot.name()).normalize(); + if (!snapshot.name().isEmpty() && (!target.getParent().equals(generations) + || Files.isSymbolicLink(target) || !Files.isDirectory(target, LinkOption.NOFOLLOW_LINKS))) + throw new IOException("HTTP client credential generation is invalid"); + loadEnrolledDirectory(target); + Path current = safe(root.resolve(CURRENT_FILE)); + if (snapshot.name().isEmpty()) { + Files.deleteIfExists(current); + DurableFiles.forceDirectory(root); + } else { + writePrivate(current, snapshot.name().getBytes(StandardCharsets.US_ASCII)); + } + } + public record ClientCredential(PrivateKey privateKey, X509Certificate certificate, X509Certificate caCertificate, char[] password) { public ClientCredential { password = password.clone(); } @Override public char[] password() { return password.clone(); } @@ -207,6 +238,13 @@ public record HttpClientProfile(String serverId, URI endpoint, String serverCert public record EnrolledClient(HttpClientProfile profile, ClientCredential credential) { } + public record ActiveCredentialGeneration(String name) { + public ActiveCredentialGeneration { + if (name == null || (!name.isEmpty() && !name.matches("[0-9a-f-]{36}"))) + throw new IllegalArgumentException("HTTP client credential generation is invalid"); + } + } + private static boolean matchesProfile(ClientCredential credential, HttpClientProfile profile) { try { String authorityPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 6446a23d0..309ab7bda 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -102,6 +102,10 @@ public void restorePreparedTransport() { preparedTransport = null; } + public void awaitPreparedTransportRestoration(long deadlineNanos) { + if (transport instanceof HttpBackendProxyTransport http) http.awaitCredentialRestoration(deadlineNanos); + } + public void closeRedisForHandoff() { if (!(transport instanceof RedisBackendProxyTransport)) { throw new IllegalStateException("Redis backend proxy transport is unavailable"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index e1dada297..9651c4203 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -14,6 +14,7 @@ import com.bencodez.votingplugin.backendproxy.http.HttpBackendTransportConnector; import com.bencodez.votingplugin.backendproxy.http.HttpClientCredentialStore; import com.bencodez.votingplugin.backendproxy.http.HttpConnectionCode; +import com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity; /** Backend adapter for the secure outbound-only HTTP proxy transport. */ public final class HttpBackendProxyTransport implements BackendProxyTransport { @@ -23,16 +24,20 @@ public final class HttpBackendProxyTransport implements BackendProxyTransport { private final VotingPluginMain plugin; private final Object lifecycle = new Object(); private final CountDownLatch startupComplete = new CountDownLatch(1); + private final CountDownLatch credentialRestoreComplete = new CountDownLatch(1); private final ArrayDeque startupQueue = new ArrayDeque<>(); private volatile HttpBackendTransportConnector connector; private volatile Thread worker; private volatile RuntimeException startupFailure; + private volatile RuntimeException credentialRestoreFailure; private volatile boolean started; private volatile boolean closed; private Path configuredDirectory; private String configuredServerId; private String configuredConnectionCode; private GlobalMessageHandler configuredMessageHandler; + private HttpClientCredentialStore.ActiveCredentialGeneration configuredCredentialGeneration; + private HttpClientCredentialStore.ActiveCredentialGeneration credentialGenerationToRestore; private Semaphore directoryOwner; private final java.util.concurrent.atomic.AtomicBoolean queueWarning = new java.util.concurrent.atomic.AtomicBoolean(); @@ -50,11 +55,19 @@ public void start(GlobalMessageHandler messageHandler) { private void start(Path directory, String serverId, String connectionCode, GlobalMessageHandler messageHandler) { - validateConfiguration(directory, serverId, connectionCode); + start(directory, serverId, connectionCode, messageHandler, null); + } + + private void start(Path directory, String serverId, String connectionCode, + GlobalMessageHandler messageHandler, + HttpClientCredentialStore.ActiveCredentialGeneration generationToRestore) { + if (generationToRestore == null) validateConfiguration(directory, serverId, connectionCode); + else HttpTlsIdentity.canonicalServerId(serverId); configuredDirectory = directory; configuredServerId = serverId; configuredConnectionCode = connectionCode; configuredMessageHandler = messageHandler; + credentialGenerationToRestore = generationToRestore; started = true; worker = new Thread(() -> initialize(directory, serverId, connectionCode, messageHandler), "VotingPlugin-HTTP-Backend-Setup"); @@ -64,10 +77,21 @@ private void start(Path directory, String serverId, String connectionCode, HttpBackendProxyTransport recreatePrepared() { HttpBackendProxyTransport restored = new HttpBackendProxyTransport(plugin); - restored.start(configuredDirectory, configuredServerId, configuredConnectionCode, configuredMessageHandler); + restored.start(configuredDirectory, configuredServerId, configuredConnectionCode, configuredMessageHandler, + configuredCredentialGeneration); return restored; } + @Override + public void prepareForReplacement() { + try { + configuredCredentialGeneration = HttpClientCredentialStore.snapshotActiveGeneration(configuredDirectory); + } catch (Exception failure) { + throw new IllegalStateException("Could not preserve the active HTTP client credential", failure); + } + close(); + } + private void initialize(Path directory, String serverId, String configuredCode, GlobalMessageHandler messageHandler) { Path ownerKey = directory.toAbsolutePath().normalize(); @@ -77,7 +101,24 @@ private void initialize(Path directory, String serverId, String configuredCode, try { owner.acquire(); acquired = true; - synchronized (lifecycle) { if (closed) return; } + synchronized (lifecycle) { + if (closed) { + if (credentialGenerationToRestore != null) + credentialRestoreFailure = new IllegalStateException( + "Previous HTTP client credential restoration was cancelled"); + return; + } + } + if (credentialGenerationToRestore != null) { + try { + HttpClientCredentialStore.restoreActiveGeneration(directory, credentialGenerationToRestore); + } catch (Exception failure) { + credentialRestoreFailure = new IllegalStateException( + "Could not restore the previous HTTP client credential", failure); + throw failure; + } + } + credentialRestoreComplete.countDown(); HttpConnectionCode code = enrollmentCode(directory, serverId, configuredCode); if (code != null) HttpBackendTransportConnector.enroll(code, serverId, directory); HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); @@ -109,6 +150,7 @@ private void initialize(Path directory, String serverId, String configuredCode, startupFailure = new IllegalStateException("Secure HTTP backend enrollment or connection failed", failure); plugin.getLogger().severe("Secure HTTP backend transport is unavailable; check the connection code and proxy endpoint"); } finally { + credentialRestoreComplete.countDown(); if (!installed) { if (replacement != null) replacement.close(); if (acquired) owner.release(); @@ -117,6 +159,20 @@ private void initialize(Path directory, String serverId, String configuredCode, } } + void awaitCredentialRestoration(long deadlineNanos) { + if (credentialGenerationToRestore == null) return; + try { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L || !credentialRestoreComplete.await(remaining, TimeUnit.NANOSECONDS)) + throw new IllegalStateException("Previous HTTP client credential restoration timed out"); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Previous HTTP client credential restoration was interrupted", interrupted); + } + RuntimeException failure = credentialRestoreFailure; + if (failure != null) throw failure; + } + @Override public void send(JsonEnvelope envelope) { synchronized (lifecycle) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 804b0312b..e2c4c4009 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -15,6 +15,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; @@ -22,6 +23,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.regex.Pattern; @@ -68,6 +70,7 @@ public final class BackendControlConnector implements AutoCloseable { private final AtomicBoolean running = new AtomicBoolean(); private final AtomicBoolean inspecting = new AtomicBoolean(); private final Object operationLifecycle = new Object(); + private final AtomicReference pendingBackendProxyRollback = new AtomicReference<>(); private final Object journalLifecycle = new Object(); private volatile boolean closed; private volatile boolean registered; @@ -112,21 +115,43 @@ private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Set } private void reloadConfiguration(String fileName) throws Exception { + finishPendingBackendProxyRollback(); long validationDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(29); AtomicBoolean preparationAbandoned = new AtomicBoolean(); + AtomicInteger preparationState = new AtomicInteger(); + CountDownLatch preparationSettled = new CountDownLatch(1); + CountDownLatch rollbackAbortSettled = new CountDownLatch(1); AtomicReference preparedRestart = new AtomicReference<>(); Future preparation; synchronized (operationLifecycle) { if (closed) throw new IllegalStateException("Bukkit Control connector is stopping"); preparation = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { - plugin.reloadFromControl(); - VotingPluginMain.BackendProxyRestart prepared = "BungeeSettings.yml".equals(fileName) - ? plugin.prepareBackendProxyHandlerRestart() : null; - preparedRestart.set(prepared); - if (prepared != null && preparationAbandoned.get()) { - plugin.abortBackendProxyHandlerRestart(prepared); + try { + if (!preparationState.compareAndSet(0, 1)) return null; + plugin.reloadFromControl(); + VotingPluginMain.BackendProxyRestart prepared; + try { + prepared = "BungeeSettings.yml".equals(fileName) + ? plugin.prepareBackendProxyHandlerRestart() : null; + } catch (VotingPluginMain.BackendProxyRestartPreparationException failure) { + preparedRestart.set(failure.restart()); + throw failure; + } + preparedRestart.set(prepared); + return prepared; + } finally { + try { + VotingPluginMain.BackendProxyRestart prepared = preparedRestart.get(); + if (preparationAbandoned.get()) { + try { + if (prepared != null) plugin.abortBackendProxyHandlerRestart(prepared); + } finally { rollbackAbortSettled.countDown(); } + } + } finally { + preparationState.set(2); + preparationSettled.countDown(); + } } - return prepared; }); activeReload = preparation; } @@ -151,19 +176,49 @@ private void reloadConfiguration(String fileName) throws Exception { } catch (Exception failure) { // Timed-out Bukkit work must not remain queued ahead of configuration rollback. preparationAbandoned.set(true); + boolean abandonedBeforePreparation = preparationState.compareAndSet(0, 3); preparation.cancel(false); + boolean publicationCommitted = publication != null && restart != null + && !plugin.requestBackendProxyHandlerRestartAbandonment(restart); if (publication != null) publication.cancel(false); + // A publication that won the synchronized commit race is the successful + // runtime state; rolling its YAML back would create a split-brain config. + if (publicationCommitted) return; + PendingBackendProxyRollback pendingRollback = null; + if (restart == null && !abandonedBeforePreparation) { + pendingRollback = new PendingBackendProxyRollback(preparationSettled, rollbackAbortSettled, preparedRestart); + pendingBackendProxyRollback.set(pendingRollback); + try { + if (!preparationSettled.await(40, TimeUnit.SECONDS)) + throw new java.util.concurrent.TimeoutException( + "Bukkit configuration preparation did not settle during rollback"); + } catch (Exception preparationFailure) { + failure.addSuppressed(preparationFailure); + throw failure; + } + } if (restart == null) restart = preparedRestart.get(); if (restart != null) { + if (pendingRollback == null) { + pendingRollback = new PendingBackendProxyRollback( + preparationSettled, rollbackAbortSettled, preparedRestart); + pendingBackendProxyRollback.set(pendingRollback); + } VotingPluginMain.BackendProxyRestart prepared = restart; try { Future abort = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { - plugin.abortBackendProxyHandlerRestart(prepared); + try { plugin.abortBackendProxyHandlerRestart(prepared); } + finally { rollbackAbortSettled.countDown(); } return null; }); abort.get(5, TimeUnit.SECONDS); + finishPendingBackendProxyRollback(); } catch (Exception cleanupFailure) { failure.addSuppressed(cleanupFailure); } } + if (pendingRollback != null && restart == null) { + rollbackAbortSettled.countDown(); + pendingBackendProxyRollback.compareAndSet(pendingRollback, null); + } throw failure; } finally { synchronized (operationLifecycle) { @@ -172,6 +227,29 @@ private void reloadConfiguration(String fileName) throws Exception { } } + private void finishPendingBackendProxyRollback() throws Exception { + PendingBackendProxyRollback pending = pendingBackendProxyRollback.get(); + if (pending == null) return; + if (!pending.preparationSettled().await(40, TimeUnit.SECONDS)) + throw new java.util.concurrent.TimeoutException( + "Previous Bukkit configuration preparation is still rolling back"); + VotingPluginMain.BackendProxyRestart restart = pending.preparedRestart().get(); + if (restart != null) { + if (!pending.rollbackAbortSettled().await(40, TimeUnit.SECONDS)) + throw new java.util.concurrent.TimeoutException( + "Previous Bukkit configuration abort is still pending"); + // Keep credential-journal I/O off Bukkit's primary thread and finish it + // before the configuration service starts its automatic backup reload. + plugin.awaitBackendProxyHandlerRollback(restart, + System.nanoTime() + TimeUnit.SECONDS.toNanos(40)); + } + pendingBackendProxyRollback.compareAndSet(pending, null); + } + + private record PendingBackendProxyRollback(CountDownLatch preparationSettled, + CountDownLatch rollbackAbortSettled, + AtomicReference preparedRestart) { } + private static long remaining(long deadlineNanos) throws java.util.concurrent.TimeoutException { long remaining = deadlineNanos - System.nanoTime(); if (remaining <= 0L) throw new java.util.concurrent.TimeoutException("Bukkit configuration reload timed out"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 4c006c02d..5b6a20f17 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -284,5 +284,29 @@ void stagedCredentialDoesNotReplaceActiveGenerationUntilAtomicActivation() throw HttpTransportSecrets.certificatePin(HttpClientCredentialStore.loadEnrolled(client).credential().certificate())); } + @Test + void restoresCredentialGenerationAfterFailedReenrollment() throws Exception { + Path client = directory.resolve("client-rollback"); + HttpTlsIdentity oldIdentity = HttpTlsIdentity.loadOrCreate(directory.resolve("old-proxy"), "old.example.test"); + HttpConnectionCode oldCode = new HttpConnectionCode("lobby-1", URI.create("https://old.example.test:1297/"), + oldIdentity.serverCertificatePin(), oldIdentity.caCertificatePin(), Instant.now().plusSeconds(60), + "R".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, oldCode, oldIdentity.issueClientCertificate("lobby-1")); + HttpClientCredentialStore.ActiveCredentialGeneration previous = + HttpClientCredentialStore.snapshotActiveGeneration(client); + + HttpTlsIdentity replacementIdentity = HttpTlsIdentity.loadOrCreate(directory.resolve("new-proxy"), "new.example.test"); + HttpConnectionCode replacementCode = new HttpConnectionCode("lobby-1", URI.create("https://new.example.test:1297/"), + replacementIdentity.serverCertificatePin(), replacementIdentity.caCertificatePin(), + Instant.now().plusSeconds(60), "S".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, replacementCode, + replacementIdentity.issueClientCertificate("lobby-1")); + assertEquals(replacementCode.endpoint(), HttpClientCredentialStore.loadProfile(client).endpoint()); + + HttpClientCredentialStore.restoreActiveGeneration(client, previous); + assertEquals(oldCode.endpoint(), HttpClientCredentialStore.loadProfile(client).endpoint()); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, oldCode)); + } + private static String pin(char character) { return String.valueOf(character).repeat(64); } } From 9b9232b11033e014a20fcaf4c74c8faf2c65dce4 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 18:43:24 -0600 Subject: [PATCH 29/36] fix(http): preserve independent delivery retries --- .../http/HttpBackendTransportConnector.java | 12 +++ .../http/HttpClientCredentialStore.java | 44 ++++++++++- .../http/HttpProxyTransportServer.java | 57 +++++++++---- .../transport/HttpBackendProxyTransport.java | 9 ++- .../http/HttpTransportRuntimeTest.java | 79 +++++++++++++++++++ .../http/HttpTransportSecurityTest.java | 26 ++++++ 6 files changed, 207 insertions(+), 20 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index f05f5eb61..4370d732d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -189,6 +189,18 @@ public synchronized boolean pollOnce() { Thread current = poller; if (current != null) current.interrupt(); callbackExecutor.shutdown(); try { if (!callbackExecutor.awaitTermination(5, TimeUnit.SECONDS)) callbackExecutor.shutdownNow(); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); callbackExecutor.shutdownNow(); } + // The owning transport may release the credential-directory semaphore as soon + // as close returns. Wait for the interrupted poller so an in-flight renewal + // cannot activate an old credential generation after that ownership handoff. + joinPoller(current); + } + boolean pollerAlive() { Thread current = poller; return current != null && current.isAlive(); } + private static void joinPoller(Thread poller) { + if (poller == null || poller == Thread.currentThread()) return; + boolean interrupted = false; + while (poller.isAlive()) try { poller.join(); } + catch (InterruptedException stopRequested) { interrupted = true; poller.interrupt(); } + if (interrupted) Thread.currentThread().interrupt(); } private void pollLoop() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index b9b835ef3..dc96b9310 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -197,8 +197,33 @@ private static EnrolledClient loadEnrolledDirectory(Path active) throws Exceptio public static ActiveCredentialGeneration snapshotActiveGeneration(Path directory) throws Exception { Path root = directory.toAbsolutePath().normalize(); Path active = activeDirectory(root); - loadEnrolledDirectory(active); - return new ActiveCredentialGeneration(active.equals(root) ? "" : active.getFileName().toString()); + EnrolledClient enrolled = loadEnrolledDirectory(active); + return new ActiveCredentialGeneration(active.equals(root) ? "" : active.getFileName().toString(), + enrolled.profile(), readConnectionCodeDigest(active)); + } + + /** + * Restores a pre-replacement generation unless the replacement already activated a + * newer credential for the same backend endpoint. A successful renewal or same-endpoint + * re-enrollment may revoke the snapshotted certificate at the proxy, so that newer + * generation is the only safe rollback identity. + */ + public static void restoreActiveGenerationAfterReplacement(Path directory, + ActiveCredentialGeneration snapshot) throws Exception { + if (directory == null || snapshot == null) throw new IllegalArgumentException("Credential rollback is required"); + HttpClientProfile previous = snapshot.profile(); + Path active = null; + HttpClientProfile current = null; + if (previous != null) try { + active = activeDirectory(directory); + current = loadEnrolledDirectory(active).profile(); + } catch (Exception unavailable) { active = null; current = null; } + if (active != null && previous.serverId().equals(current.serverId()) + && previous.endpoint().equals(current.endpoint())) { + restoreConnectionCodeDigest(active, snapshot.connectionCodeDigest()); + return; + } + restoreActiveGeneration(directory, snapshot); } /** Atomically restores a previously validated credential generation after replacement rollback. */ @@ -238,10 +263,13 @@ public record HttpClientProfile(String serverId, URI endpoint, String serverCert public record EnrolledClient(HttpClientProfile profile, ClientCredential credential) { } - public record ActiveCredentialGeneration(String name) { + public record ActiveCredentialGeneration(String name, HttpClientProfile profile, String connectionCodeDigest) { + public ActiveCredentialGeneration(String name) { this(name, null, null); } public ActiveCredentialGeneration { if (name == null || (!name.isEmpty() && !name.matches("[0-9a-f-]{36}"))) throw new IllegalArgumentException("HTTP client credential generation is invalid"); + if (connectionCodeDigest != null && !connectionCodeDigest.matches("[0-9a-f]{64}")) + throw new IllegalArgumentException("HTTP connection-code marker is invalid"); } } @@ -299,6 +327,16 @@ private static String connectionCodeDigest(HttpConnectionCode code) { return HttpTransportSecrets.sha256Hex(code.encode().getBytes(StandardCharsets.US_ASCII)); } + private static void restoreConnectionCodeDigest(Path directory, String digest) throws IOException { + Path marker = safe(directory.resolve(CONNECTION_CODE_DIGEST_FILE)); + if (digest == null) { + Files.deleteIfExists(marker); + DurableFiles.forceDirectory(directory); + } else { + writePrivate(marker, digest.getBytes(StandardCharsets.US_ASCII)); + } + } + private static void writePrivate(Path file, byte[] contents) throws IOException { Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index 8e1ecc925..766a741ed 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -37,6 +37,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.function.LongSupplier; import javax.net.ssl.SSLParameters; import javax.net.ssl.SSLPeerUnverifiedException; @@ -236,18 +237,22 @@ static record Response(Collection acks, Collection outgoing = new LinkedHashMap<>(); private final Set seen = new LinkedHashSet<>(); private final Set processing = new LinkedHashSet<>(); private final ArrayDeque acknowledgements = new ArrayDeque<>(); - private final Set delivered = new LinkedHashSet<>(); - private long lastDeliveryNanos; + private final Map deliveredAtNanos = new HashMap<>(); private double requestTokens = 24.0d; private long lastTokenNanos = System.nanoTime(); private boolean activePoll; - BackendState() { this(null, null); } + BackendState() { this(null, null, System::nanoTime); } + BackendState(LongSupplier nanoTime) { this(null, null, nanoTime); } private BackendState(String serverId, DurableOutgoingQueue durableOutgoing) { - this.serverId = serverId; this.durableOutgoing = durableOutgoing; + this(serverId, durableOutgoing, System::nanoTime); + } + private BackendState(String serverId, DurableOutgoingQueue durableOutgoing, LongSupplier nanoTime) { + this.serverId = serverId; this.durableOutgoing = durableOutgoing; this.nanoTime = nanoTime; } private synchronized void restore(Collection deliveries) { for (HttpTransportProtocol.Delivery delivery : deliveries) outgoing.put(delivery.id(), delivery); @@ -259,7 +264,7 @@ private boolean allowRequest() { lastTokenNanos = now; if (requestTokens < 1.0d) return false; requestTokens -= 1.0d; return true; } boolean acceptSession(String requested, long requestedSequence) { - if (!requested.equals(session)) { session = requested; sequence = -1L; delivered.clear(); lastDeliveryNanos = 0L; } + if (!requested.equals(session)) { session = requested; sequence = -1L; deliveredAtNanos.clear(); } // The connector allocates a fresh monotonic sequence for every attempt. Rejecting equality // prevents a captured request from being replayed with altered ACKs or a new payload. if (requestedSequence <= sequence) return false; sequence = requestedSequence; return true; @@ -276,7 +281,7 @@ private void acknowledge(Collection acks) throws IOException { // A 200 response is the backend's proof that its durable replay fence may // be deleted. Never return success while the proxy delivery still exists. if (durableOutgoing != null) durableOutgoing.remove(serverId, id); - outgoing.remove(id); delivered.remove(id); + outgoing.remove(id); deliveredAtNanos.remove(id); } } List acceptIncoming(List received) { @@ -293,24 +298,46 @@ List acceptIncoming(List acks = new java.util.ArrayList<>(); while (!acknowledgements.isEmpty() && acks.size() < HttpTransportProtocol.MAX_BATCH) acks.add(acknowledgements.remove()); List candidates = new java.util.ArrayList<>(); - if (hasUndelivered() || redeliveryDue()) for (HttpTransportProtocol.Delivery delivery : outgoing.values()) { - if (!delivered.contains(delivery.id()) || redeliveryDue()) candidates.add(delivery); + long now = nanoTime.getAsLong(); + if (hasUndelivered() || redeliveryDue(now)) for (HttpTransportProtocol.Delivery delivery : outgoing.values()) { + if (!deliveredAtNanos.containsKey(delivery.id()) || redeliveryDue(delivery.id(), now)) candidates.add(delivery); if (candidates.size() == HttpTransportProtocol.MAX_BATCH) break; } List messages = HttpTransportProtocol.fittingMessages(serverId, requestedSession, requestedSequence, acks, candidates); - for (HttpTransportProtocol.Delivery delivery : messages) delivered.add(delivery.id()); - if (!messages.isEmpty()) lastDeliveryNanos = System.nanoTime(); + long deliveredAt = nanoTime.getAsLong(); + for (HttpTransportProtocol.Delivery delivery : messages) deliveredAtNanos.put(delivery.id(), deliveredAt); return new Response(acks, messages); } - private boolean hasUndelivered() { for (String id : outgoing.keySet()) if (!delivered.contains(id)) return true; return false; } - private boolean redeliveryDue() { return !outgoing.isEmpty() && lastDeliveryNanos > 0L && System.nanoTime() - lastDeliveryNanos >= LONG_POLL.toNanos(); } + private boolean hasUndelivered() { for (String id : outgoing.keySet()) if (!deliveredAtNanos.containsKey(id)) return true; return false; } + private boolean redeliveryDue(long now) { + for (String id : outgoing.keySet()) if (redeliveryDue(id, now)) return true; + return false; + } + private boolean redeliveryDue(String id, long now) { + Long deliveredAt = deliveredAtNanos.get(id); + return deliveredAt != null && now - deliveredAt >= LONG_POLL.toNanos(); + } + private long nanosUntilRedelivery(long now) { + long remaining = Long.MAX_VALUE; + for (String id : outgoing.keySet()) { + Long deliveredAt = deliveredAtNanos.get(id); + if (deliveredAt == null) continue; + long candidate = LONG_POLL.toNanos() - (now - deliveredAt); + if (candidate <= 0L) return 0L; + remaining = Math.min(remaining, candidate); + } + return remaining; + } private synchronized void signal() { notifyAll(); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java index 9651c4203..8598431c1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/HttpBackendProxyTransport.java @@ -111,7 +111,8 @@ private void initialize(Path directory, String serverId, String configuredCode, } if (credentialGenerationToRestore != null) { try { - HttpClientCredentialStore.restoreActiveGeneration(directory, credentialGenerationToRestore); + HttpClientCredentialStore.restoreActiveGenerationAfterReplacement(directory, + credentialGenerationToRestore); } catch (Exception failure) { credentialRestoreFailure = new IllegalStateException( "Could not restore the previous HTTP client credential", failure); @@ -119,7 +120,11 @@ private void initialize(Path directory, String serverId, String configuredCode, } } credentialRestoreComplete.countDown(); - HttpConnectionCode code = enrollmentCode(directory, serverId, configuredCode); + // A rollback always resumes a validated enrolled generation. Replaying the + // previous temporary code is both unnecessary and unsafe after a same-endpoint + // renewal/re-enrollment retained the newer active generation. + HttpConnectionCode code = enrollmentCode(directory, serverId, + credentialGenerationToRestore == null ? configuredCode : null); if (code != null) HttpBackendTransportConnector.enroll(code, serverId, directory); HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(directory); if (!enrolled.profile().serverId().equals(com.bencodez.votingplugin.backendproxy.http.HttpTlsIdentity.canonicalServerId(serverId))) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 3c2658a54..0e6a26bd0 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -17,6 +17,7 @@ import java.time.Instant; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -57,6 +58,27 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti } } + @Test + void closeWaitsForTheCredentialOwningPollerToStop() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("close-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("close-authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> { })) { + server.start(); + Path clientDirectory = directory.resolve("close-client"); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(code, "lobby-1", clientDirectory); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { + connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + connector.close(); + assertFalse(connector.pollerAlive(), + "credential-directory ownership must outlive every poller filesystem mutation"); + } + } + } + @Test void normalTransportRejectsAClientWithoutCertificate() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); @@ -315,6 +337,63 @@ void duplicateInboundDeliveryIsReAcknowledgedWithoutSecondDispatch() { assertEquals(java.util.List.of(id), state.await("lobby-1", replacementSession, 0).acks()); } + @Test + void newerDeliveriesDoNotPostponeRetryOfOlderUnacknowledgedDelivery() { + AtomicLong nanoTime = new AtomicLong(1L); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(nanoTime::get); + String session = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery first = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("first").build()); + HttpTransportProtocol.Delivery second = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("second").build()); + + assertTrue(state.acceptSession(session, 0)); + assertTrue(state.enqueue(first)); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 0).messages()); + + nanoTime.addAndGet(TimeUnit.SECONDS.toNanos(1)); + assertTrue(state.acceptSession(session, 1)); + assertTrue(state.enqueue(second)); + assertEquals(java.util.List.of(second), state.await("lobby-1", session, 1).messages()); + + nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(1100)); + assertTrue(state.acceptSession(session, 2)); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 2).messages(), + "sending newer traffic must not reset an older delivery's retry age"); + } + + @Test + void longPollWakesAtTheOldestDeliveryRetryDeadline() throws Exception { + AtomicLong nanoTime = new AtomicLong(1L); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(nanoTime::get); + String session = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery first = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("first").build()); + HttpTransportProtocol.Delivery second = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("second").build()); + assertTrue(state.acceptSession(session, 0)); + assertTrue(state.enqueue(first)); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 0).messages()); + + nanoTime.addAndGet(HttpProxyTransportServer.LONG_POLL.minusMillis(100).toNanos()); + assertTrue(state.acceptSession(session, 1)); + assertTrue(state.enqueue(second)); + assertEquals(java.util.List.of(second), state.await("lobby-1", session, 1).messages()); + assertTrue(state.acceptSession(session, 2)); + Thread clock = new Thread(() -> { + try { Thread.sleep(50L); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(100)); + }, "HTTP-retry-test-clock"); + clock.setDaemon(true); + long started = System.nanoTime(); + clock.start(); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 2).messages()); + clock.join(); + assertTrue(System.nanoTime() - started < TimeUnit.SECONDS.toNanos(1), + "the poll must wake at the oldest delivery deadline, not a fresh long-poll deadline"); + } + @Test void proxyDedupWindowEvictsOldestCompletedDeliveryAtCapacity() { HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 5b6a20f17..662e9e84a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -308,5 +308,31 @@ void restoresCredentialGenerationAfterFailedReenrollment() throws Exception { assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, oldCode)); } + @Test + void rollbackRetainsNewerCredentialForTheSameEndpoint() throws Exception { + Path client = directory.resolve("client-renewal-rollback"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("renewal-proxy"), "renew.example.test"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://renew.example.test:1297/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), + "T".repeat(43)); + HttpTlsIdentity.IssuedClientCertificate original = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, code, original); + HttpClientCredentialStore.ActiveCredentialGeneration previous = + HttpClientCredentialStore.snapshotActiveGeneration(client); + + HttpConnectionCode replacementCode = new HttpConnectionCode("lobby-1", code.endpoint(), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), + "U".repeat(43)); + HttpTlsIdentity.IssuedClientCertificate renewed = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, replacementCode, renewed); + HttpClientCredentialStore.restoreActiveGenerationAfterReplacement(client, previous); + + assertEquals(HttpTransportSecrets.certificatePin(renewed.certificate()), + HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate()), + "rollback must not reactivate a same-endpoint certificate that renewal may have revoked"); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, code), + "the retained credential must recognize the connection code restored in YAML"); + } + private static String pin(char character) { return String.valueOf(character).repeat(64); } } From 59b08052fedcad8437e1ba152f79b6545d747967 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 19:45:39 -0600 Subject: [PATCH 30/36] fix(http): retain rejected vote-party rewards --- .../http/HttpClientCredentialStore.java | 13 +- .../backendproxy/http/HttpConnectionCode.java | 22 +- .../http/HttpProxyTransportServer.java | 69 ++++-- .../http/HttpTransportProtocol.java | 2 +- .../votingplugin/proxy/VotingPluginProxy.java | 203 +++++++++++++++--- .../proxy/bungee/BungeeJsonVoteCache.java | 38 ++++ .../proxy/bungee/VotingPluginBungee.java | 22 ++ .../votingplugin/proxy/cache/IVoteCache.java | 7 + .../proxy/cache/VotePartyCacheDurability.java | 76 +++++++ .../proxy/velocity/VelocityJsonVoteCache.java | 37 ++++ .../proxy/velocity/VotingPluginVelocity.java | 22 ++ .../http/HttpTransportRuntimeTest.java | 29 +++ .../http/HttpTransportSecurityTest.java | 21 +- .../cache/VotePartyCacheDurabilityTest.java | 29 +++ .../tests/VotingPluginProxyTest.java | 49 +++++ .../tests/VotingPluginProxyTestImpl.java | 58 ++++- 16 files changed, 641 insertions(+), 56 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurabilityTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java index dc96b9310..9bc247ded 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpClientCredentialStore.java @@ -176,8 +176,11 @@ public static boolean matchesEnrollmentCode(Path directory, HttpConnectionCode c if (code == null) throw new IllegalArgumentException("Connection code is required"); String stored = readConnectionCodeDigest(activeDirectory(directory)); if (stored == null) return false; - return HttpTransportSecrets.constantTimeEquals(stored.getBytes(StandardCharsets.US_ASCII), - connectionCodeDigest(code).getBytes(StandardCharsets.US_ASCII)); + byte[] storedBytes = stored.getBytes(StandardCharsets.US_ASCII); + return HttpTransportSecrets.constantTimeEquals(storedBytes, + connectionCodeDigest(code.encode()).getBytes(StandardCharsets.US_ASCII)) + || HttpTransportSecrets.constantTimeEquals(storedBytes, + connectionCodeDigest(code.encodeLegacy()).getBytes(StandardCharsets.US_ASCII)); } /** Loads and cross-checks the persisted client key material and bound normal-transport profile. */ @@ -324,7 +327,11 @@ private static String readConnectionCodeDigest(Path directory) throws IOExceptio } private static String connectionCodeDigest(HttpConnectionCode code) { - return HttpTransportSecrets.sha256Hex(code.encode().getBytes(StandardCharsets.US_ASCII)); + return connectionCodeDigest(code.encode()); + } + + private static String connectionCodeDigest(String encoded) { + return HttpTransportSecrets.sha256Hex(encoded.getBytes(StandardCharsets.US_ASCII)); } private static void restoreConnectionCodeDigest(Path directory, String digest) throws IOException { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java index 5700147fb..62bf1ff62 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpConnectionCode.java @@ -16,7 +16,8 @@ */ public record HttpConnectionCode(String serverId, URI endpoint, String serverCertificatePin, String caCertificatePin, Instant expiresAt, String enrollmentToken) { - private static final String VERSION = "VPH1"; + private static final String LEGACY_VERSION = "VPH1"; + private static final String VERSION = "VPH2"; private static final int MAX_CODE_LENGTH = 4096; public HttpConnectionCode { @@ -29,9 +30,19 @@ public record HttpConnectionCode(String serverId, URI endpoint, String serverCer } public String encode() { + String serverPart = Base64.getUrlEncoder().withoutPadding() + .encodeToString(serverId.getBytes(StandardCharsets.UTF_8)); + return encode(VERSION, serverPart); + } + + String encodeLegacy() { + return encode(LEGACY_VERSION, serverId); + } + + private String encode(String version, String serverPart) { String endpointPart = Base64.getUrlEncoder().withoutPadding() .encodeToString(endpoint.toASCIIString().getBytes(StandardCharsets.UTF_8)); - String unsigned = String.join(".", VERSION, serverId, endpointPart, serverCertificatePin, caCertificatePin, + String unsigned = String.join(".", version, serverPart, endpointPart, serverCertificatePin, caCertificatePin, Long.toString(expiresAt.getEpochSecond()), enrollmentToken); byte[] token = Base64.getUrlDecoder().decode(enrollmentToken); return unsigned + "." + HttpTransportSecrets.hmacSha256Url(token, unsigned); @@ -49,15 +60,18 @@ public static HttpConnectionCode parse(String code) { if (code == null || code.length() > MAX_CODE_LENGTH || code.indexOf('\n') >= 0 || code.indexOf('\r') >= 0) throw new IllegalArgumentException("Connection code is invalid"); String[] parts = code.split("\\.", -1); - if (parts.length != 8 || !VERSION.equals(parts[0])) throw new IllegalArgumentException("Connection code is invalid"); + if (parts.length != 8 || (!VERSION.equals(parts[0]) && !LEGACY_VERSION.equals(parts[0]))) + throw new IllegalArgumentException("Connection code is invalid"); try { + String serverId = LEGACY_VERSION.equals(parts[0]) ? parts[1] + : new String(Base64.getUrlDecoder().decode(parts[1]), StandardCharsets.UTF_8); String endpoint = new String(Base64.getUrlDecoder().decode(parts[2]), StandardCharsets.UTF_8); String unsigned = String.join(".", parts[0], parts[1], parts[2], parts[3], parts[4], parts[5], parts[6]); byte[] token = Base64.getUrlDecoder().decode(parts[6]); String expected = HttpTransportSecrets.hmacSha256Url(token, unsigned); if (!HttpTransportSecrets.constantTimeEquals(expected.getBytes(StandardCharsets.US_ASCII), parts[7].getBytes(StandardCharsets.US_ASCII))) throw new IllegalArgumentException("Connection code is invalid"); - return new HttpConnectionCode(parts[1], new URI(endpoint), parts[3], parts[4], Instant.ofEpochSecond(Long.parseLong(parts[5])), + return new HttpConnectionCode(serverId, new URI(endpoint), parts[3], parts[4], Instant.ofEpochSecond(Long.parseLong(parts[5])), parts[6]); } catch (IllegalArgumentException | URISyntaxException failure) { throw new IllegalArgumentException("Connection code is invalid", failure); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java index 766a741ed..4ba34e528 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpProxyTransportServer.java @@ -22,6 +22,7 @@ import java.security.cert.X509Certificate; import java.time.Duration; import java.util.ArrayDeque; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; @@ -67,21 +68,30 @@ public final class HttpProxyTransportServer implements AutoCloseable { private final Map backends = new HashMap<>(); private final DurableOutgoingQueue durableOutgoing; private final Consumer onEnvelope; + private final DeliveryAcknowledgement onAcknowledged; private volatile boolean closed; public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, Consumer onEnvelope) throws Exception { - this(bind, identity, authority, null, onEnvelope); + this(bind, identity, authority, null, onEnvelope, (serverId, deliveryId) -> { }); } public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, Path outgoingDirectory, Consumer onEnvelope) throws Exception { - if (bind == null || identity == null || authority == null || onEnvelope == null) throw new IllegalArgumentException("HTTP transport configuration is required"); + this(bind, identity, authority, outgoingDirectory, onEnvelope, (serverId, deliveryId) -> { }); + } + + public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Path outgoingDirectory, Consumer onEnvelope, + DeliveryAcknowledgement onAcknowledged) throws Exception { + if (bind == null || identity == null || authority == null || onEnvelope == null || onAcknowledged == null) + throw new IllegalArgumentException("HTTP transport configuration is required"); this.identity = identity; this.authority = authority; this.onEnvelope = onEnvelope; + this.onAcknowledged = onAcknowledged; durableOutgoing = outgoingDirectory == null ? null : new DurableOutgoingQueue(outgoingDirectory); if (durableOutgoing != null) for (Map.Entry> pending : durableOutgoing.load().entrySet()) { - BackendState state = new BackendState(pending.getKey(), durableOutgoing); + BackendState state = new BackendState(pending.getKey(), durableOutgoing, onAcknowledged); state.restore(pending.getValue()); backends.put(pending.getKey(), state); } @@ -126,14 +136,23 @@ private void renew(HttpsExchange exchange) throws IOException { /** Queues a proxy-origin envelope durably before reporting acceptance. */ public boolean send(String serverId, JsonEnvelope envelope) { + return send(serverId, UUID.randomUUID().toString(), envelope); + } + + /** Queues a proxy-origin envelope with a stable, caller-persisted delivery ID. */ + public boolean send(String serverId, String deliveryId, JsonEnvelope envelope) { if (closed || serverId == null || envelope == null) return false; - try { serverId = HttpTlsIdentity.canonicalServerId(serverId); HttpTransportProtocol.validateEnvelope(envelope); } + try { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + HttpTransportProtocol.validId(deliveryId); + HttpTransportProtocol.validateEnvelope(envelope); + } catch (IllegalArgumentException invalid) { return false; } BackendState backend; final String canonicalServerId = serverId; synchronized (backends) { backend = backends.computeIfAbsent(serverId, - ignored -> new BackendState(canonicalServerId, durableOutgoing)); } - return backend.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), envelope)); + ignored -> new BackendState(canonicalServerId, durableOutgoing, onAcknowledged)); } + return backend.enqueue(new HttpTransportProtocol.Delivery(deliveryId, envelope)); } @Override public void close() { @@ -166,7 +185,7 @@ private void transport(HttpsExchange exchange) throws IOException { if (certificate == null || !authority.authenticate(packet.server(), certificate)) { reply(exchange, 401, new byte[0]); return; } BackendState backend; synchronized (backends) { backend = backends.computeIfAbsent(packet.server(), - ignored -> new BackendState(packet.server(), durableOutgoing)); } + ignored -> new BackendState(packet.server(), durableOutgoing, onAcknowledged)); } if (!backend.beginPoll(packet.session())) { reply(exchange, 409, new byte[0]); return; } try { handlePacket(packet, backend); @@ -183,8 +202,9 @@ private void handlePacket(HttpTransportProtocol.Packet packet, BackendState back synchronized (backend) { if (!backend.allowRequest()) throw new IllegalArgumentException("transport rate limited"); if (!backend.acceptSession(packet.session(), packet.sequence())) throw new IllegalArgumentException("stale session request"); - backend.acknowledge(packet.acks()); accepted = backend.acceptIncoming(packet.messages()); } + backend.acknowledge(packet.acks()); + synchronized (backend) { accepted = backend.acceptIncoming(packet.messages()); } for (HttpTransportProtocol.Delivery delivery : accepted) dispatch(packet.server(), backend, delivery); } private void dispatch(String serverId, BackendState backend, HttpTransportProtocol.Delivery delivery) { @@ -233,10 +253,16 @@ private static ThreadPoolExecutor executor(String name, int threads, int queue) private static void shutdown(ExecutorService executor) { executor.shutdown(); try { if (!executor.awaitTermination(5, TimeUnit.SECONDS)) executor.shutdownNow(); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); executor.shutdownNow(); } } public record ReceivedEnvelope(String serverId, String messageId, JsonEnvelope envelope) { } + + @FunctionalInterface + public interface DeliveryAcknowledgement { + void confirm(String serverId, String deliveryId) throws IOException; + } static record Response(Collection acks, Collection messages) { } static final class BackendState { private final String serverId; private final DurableOutgoingQueue durableOutgoing; + private final DeliveryAcknowledgement onAcknowledged; private final LongSupplier nanoTime; private String session; private long sequence = -1L; private final LinkedHashMap outgoing = new LinkedHashMap<>(); @@ -246,13 +272,19 @@ static final class BackendState { private double requestTokens = 24.0d; private long lastTokenNanos = System.nanoTime(); private boolean activePoll; - BackendState() { this(null, null, System::nanoTime); } - BackendState(LongSupplier nanoTime) { this(null, null, nanoTime); } + BackendState() { this(null, null, (serverId, deliveryId) -> { }, System::nanoTime); } + BackendState(LongSupplier nanoTime) { this(null, null, (serverId, deliveryId) -> { }, nanoTime); } private BackendState(String serverId, DurableOutgoingQueue durableOutgoing) { - this(serverId, durableOutgoing, System::nanoTime); + this(serverId, durableOutgoing, (ignoredServer, ignoredDelivery) -> { }, System::nanoTime); } - private BackendState(String serverId, DurableOutgoingQueue durableOutgoing, LongSupplier nanoTime) { - this.serverId = serverId; this.durableOutgoing = durableOutgoing; this.nanoTime = nanoTime; + BackendState(String serverId, DurableOutgoingQueue durableOutgoing, + DeliveryAcknowledgement onAcknowledged) { + this(serverId, durableOutgoing, onAcknowledged, System::nanoTime); + } + private BackendState(String serverId, DurableOutgoingQueue durableOutgoing, + DeliveryAcknowledgement onAcknowledged, LongSupplier nanoTime) { + this.serverId = serverId; this.durableOutgoing = durableOutgoing; + this.onAcknowledged = onAcknowledged; this.nanoTime = nanoTime; } private synchronized void restore(Collection deliveries) { for (HttpTransportProtocol.Delivery delivery : deliveries) outgoing.put(delivery.id(), delivery); @@ -270,18 +302,25 @@ boolean acceptSession(String requested, long requestedSequence) { if (requestedSequence <= sequence) return false; sequence = requestedSequence; return true; } synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { + HttpTransportProtocol.Delivery existing = outgoing.get(delivery.id()); + if (existing != null) return Arrays.equals(HttpTransportProtocol.storedDelivery(existing), + HttpTransportProtocol.storedDelivery(delivery)); if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; if (durableOutgoing != null) try { durableOutgoing.persist(serverId, delivery); } catch (IOException failure) { return false; } outgoing.put(delivery.id(), delivery); signal(); return true; } - private void acknowledge(Collection acks) throws IOException { + void acknowledge(Collection acks) throws IOException { for (String id : acks) { - if (!outgoing.containsKey(id)) continue; + synchronized (this) { if (!outgoing.containsKey(id)) continue; } + onAcknowledged.confirm(serverId, id); + synchronized (this) { + if (!outgoing.containsKey(id)) continue; // A 200 response is the backend's proof that its durable replay fence may // be deleted. Never return success while the proxy delivery still exists. if (durableOutgoing != null) durableOutgoing.remove(serverId, id); outgoing.remove(id); deliveredAtNanos.remove(id); + } } } List acceptIncoming(List received) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java index b3c825cdd..2aafffb33 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportProtocol.java @@ -215,7 +215,7 @@ private static long integer(JsonObject object, String name) { } private static long nonNegative(JsonObject object, String name) { long n = integer(object, name); if (n < 0) throw bad(); return n; } private static String uuid(JsonObject object, String name) { return canonicalUuid(string(object, name, 64)); } - private static void validId(String id) { if (id == null || id.length() > 64) throw bad(); canonicalUuid(id); } + static void validId(String id) { if (id == null || id.length() > 64) throw bad(); canonicalUuid(id); } private static String canonicalUuid(String value) { try { UUID parsed = UUID.fromString(value); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 1a23fed4a..75e81d3c7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -103,6 +103,7 @@ public abstract class VotingPluginProxy { private static final long PRESENCE_MAINTENANCE_INTERVAL_SECONDS = 30L; private static final long PRESENCE_BACKEND_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(90); private static final long CONTROL_ENROLLMENT_MIN_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(10); + private static final int MAX_PENDING_VOTE_PARTY_REWARDS = 1024; @Getter @Setter @@ -142,6 +143,7 @@ public abstract class VotingPluginProxy { private boolean timeVoteRetryScheduled; private boolean timeVoteDeliveryRetryScheduled; private boolean cachedVoteDeliveryRetryScheduled; + private boolean votePartyDeliveryRetryScheduled; private boolean enabled; @@ -949,36 +951,77 @@ private synchronized void retryCachedVoteDeliveryPersistence() { } } - public void checkVoteParty() { - if (getConfig().getVotePartyEnabled()) { - if (votePartyVotes >= currentVotePartyVotesRequired) { - debug("Vote party reached"); - addCurrentVotePartyVotes(-currentVotePartyVotesRequired); - - currentVotePartyVotesRequired += getConfig().getVotePartyIncreaseVotesRequired(); - setVoteCacheVotePartyIncreaseVotesRequired( - getVoteCacheVotePartyIncreaseVotesRequired() + getConfig().getVotePartyIncreaseVotesRequired()); - - if (!getConfig().getVotePartyBroadcast().isEmpty()) { - broadcast(getConfig().getVotePartyBroadcast()); - } - - for (String command : getConfig().getVotePartyBungeeCommands()) { - runConsoleCommand(command); - } + public synchronized void checkVoteParty() { + if (!getConfig().getVotePartyEnabled()) return; + if (votePartyVotes < currentVotePartyVotesRequired) { + saveVoteCacheFile(); + return; + } + Collection targets = getConfig().getVotePartySendToAllServers() + ? getAllAvailableServers() : getConfig().getVotePartyServersToSend(); + Map onlineTargets = onlineVotePartyTargets(targets); + if (method == BungeeMethod.HTTP && !canQueueVotePartyRewards(onlineTargets)) { + try { + saveVotePartyStateDurably(); + } catch (IOException failure) { + throw new IllegalStateException("Unable to retain the full HTTP vote-party backlog", failure); + } + return; + } - if (getConfig().getVotePartySendToAllServers()) { - for (String server : getAllAvailableServers()) { - sendVoteParty(server); - } - } else { - for (String server : getConfig().getVotePartyServersToSend()) { - sendVoteParty(server); - } - } + Map stagedRewards = new LinkedHashMap<>(); + if (method == BungeeMethod.HTTP) { + for (String canonicalServer : onlineTargets.keySet()) { + String deliveryId = UUID.randomUUID().toString(); + setVoteCachePendingVotePartyReward(canonicalServer, deliveryId, true); + stagedRewards.put(canonicalServer, deliveryId); + } + } + int previousVotes = votePartyVotes; + int previousRequired = currentVotePartyVotesRequired; + int previousIncrease = getVoteCacheVotePartyIncreaseVotesRequired(); + debug("Vote party reached"); + addCurrentVotePartyVotes(-currentVotePartyVotesRequired); + currentVotePartyVotesRequired += getConfig().getVotePartyIncreaseVotesRequired(); + setVoteCacheVotePartyIncreaseVotesRequired( + previousIncrease + getConfig().getVotePartyIncreaseVotesRequired()); + try { + if (method == BungeeMethod.HTTP) saveVotePartyStateDurably(); + else saveVoteCacheFile(); + } catch (IOException | RuntimeException failure) { + votePartyVotes = previousVotes; + setVoteCacheVotePartyCurrentVotes(previousVotes); + currentVotePartyVotesRequired = previousRequired; + setVoteCacheVotePartyIncreaseVotesRequired(previousIncrease); + for (Map.Entry staged : stagedRewards.entrySet()) + setVoteCachePendingVotePartyReward(staged.getKey(), staged.getValue(), false); + throw failure instanceof RuntimeException runtime ? runtime + : new IllegalStateException("Unable to persist HTTP vote-party rewards", failure); + } + + if (!getConfig().getVotePartyBroadcast().isEmpty()) broadcast(getConfig().getVotePartyBroadcast()); + for (String command : getConfig().getVotePartyBungeeCommands()) runConsoleCommand(command); + if (method == BungeeMethod.HTTP) retryPendingVotePartyRewards(); + else for (String server : targets) sendVoteParty(server); + } + + private Map onlineVotePartyTargets(Collection targets) { + Map online = new LinkedHashMap<>(); + for (String server : targets) if (isSomeoneOnlineServerForVoteRouting(server)) + online.putIfAbsent(server.toLowerCase(Locale.ROOT), server); + return online; + } + + private boolean canQueueVotePartyRewards(Map targets) { + for (String server : targets.keySet()) { + Collection pending = getVoteCachePendingVotePartyRewardIds(server); + if (pending != null && pending.size() >= MAX_PENDING_VOTE_PARTY_REWARDS) { + logSevere("HTTP vote-party reward backlog is full for " + targets.get(server) + + "; retaining the vote-party threshold"); + return false; } - saveVoteCacheFile(); } + return true; } public abstract void debug(String str); @@ -1186,6 +1229,12 @@ protected int[] getProjectedVotePartyState(int acceptedVotes) { public abstract int getVoteCacheVotePartyIncreaseVotesRequired(); + public abstract Collection getVoteCachePendingVotePartyServers(); + + public abstract Collection getVoteCachePendingVotePartyRewardIds(String server); + + public abstract void saveVotePartyStateDurably() throws IOException; + public abstract boolean isPlayerOnline(String playerName); /** @@ -1590,7 +1639,10 @@ public void onReceive(JsonEnvelope message) { startControlServices(); // Open the listener last: backend callbacks can immediately reach routing, // presence, vote-log, multi-proxy, and Control-adjacent runtime helpers. - if (method.equals(BungeeMethod.HTTP)) startHttpTransport(); + if (method.equals(BungeeMethod.HTTP)) { + startHttpTransport(); + scheduleVotePartyDeliveryRetry(); + } debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); } @@ -2697,6 +2749,11 @@ private synchronized boolean sendHttpEnvelope(String server, JsonEnvelope envelo return transport != null && transport.send(server, envelope); } + protected synchronized boolean sendHttpEnvelope(String server, String deliveryId, JsonEnvelope envelope) { + HttpProxyTransportServer transport = httpTransportServer; + return transport != null && transport.send(server, deliveryId, envelope); + } + private void startHttpTransport() { try { URI endpoint = URI.create(getConfig().getHttpPublicEndpoint()); @@ -2711,7 +2768,8 @@ private void startHttpTransport() { httpEnrollmentAuthority = new HttpEnrollmentAuthority(identity, directory.toPath()); httpTransportServer = new HttpProxyTransportServer( new InetSocketAddress(getConfig().getHttpHost(), getConfig().getHttpPort()), identity, - httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope); + httpEnrollmentAuthority, directory.toPath().resolve("outgoing-v1"), this::handleHttpTransportEnvelope, + this::acknowledgeVotePartyDelivery); httpTransportServer.start(); logInfo("HTTP transport listening securely on " + getConfig().getHttpHost() + ":" + httpTransportServer.port() + "; use /votingpluginbungee httpcode for each backend"); @@ -2971,9 +3029,90 @@ public void sendServerNameMessage() { } } - public void sendVoteParty(String server) { - if (isSomeoneOnlineServerForVoteRouting(server)) { + public synchronized void sendVoteParty(String server) { + if (!isSomeoneOnlineServerForVoteRouting(server)) return; + if (method != BungeeMethod.HTTP) { globalMessageProxyHandler.sendMessage(server, 1, VotingPluginWire.votePartyBungee()); + return; + } + Collection pending = getVoteCachePendingVotePartyRewardIds(server); + if (pending != null && pending.size() >= MAX_PENDING_VOTE_PARTY_REWARDS) { + logSevere("HTTP vote-party reward backlog is full for " + server); + return; + } + String deliveryId = UUID.randomUUID().toString(); + // Persist intent before the bounded HTTP queue is attempted. A rejection or + // restart therefore leaves a retryable reward instead of silently losing it. + setVoteCachePendingVotePartyReward(server, deliveryId, true); + try { + saveVotePartyStateDurably(); + } catch (IOException failure) { + setVoteCachePendingVotePartyReward(server, deliveryId, false); + throw new IllegalStateException("Unable to persist HTTP vote-party reward", failure); + } + retryPendingVotePartyRewards(); + } + + protected synchronized void retryPendingVotePartyRewards() { + if (!enabled || method != BungeeMethod.HTTP) return; + boolean retryRequired = false; + Collection servers = getVoteCachePendingVotePartyServers(); + if (servers == null) return; + for (String server : new ArrayList<>(servers)) { + Collection pendingIds = getVoteCachePendingVotePartyRewardIds(server); + if (pendingIds == null || pendingIds.isEmpty()) continue; + String routingServer = resolveVotePartyRoutingServer(server); + for (String deliveryId : new ArrayList<>(pendingIds)) { + if (!isSomeoneOnlineServerForVoteRouting(routingServer) + || !sendHttpEnvelope(routingServer, deliveryId, VotingPluginWire.votePartyBungee())) { + retryRequired = true; + break; + } + retryRequired = true; + } + } + if (retryRequired) scheduleVotePartyDeliveryRetry(); + } + + private String resolveVotePartyRoutingServer(String canonicalServer) { + for (String configuredServer : getAllAvailableServers()) { + if (configuredServer.equalsIgnoreCase(canonicalServer)) return configuredServer; + } + return canonicalServer; + } + + protected synchronized void acknowledgeVotePartyDelivery(String server, String deliveryId) throws IOException { + Collection pendingServers = getVoteCachePendingVotePartyServers(); + if (pendingServers == null) return; + for (String pendingServer : new ArrayList<>(pendingServers)) { + if (!pendingServer.equalsIgnoreCase(server)) continue; + Collection pending = getVoteCachePendingVotePartyRewardIds(pendingServer); + if (pending == null || !pending.contains(deliveryId)) return; + setVoteCachePendingVotePartyReward(pendingServer, deliveryId, false); + try { + saveVotePartyStateDurably(); + } catch (IOException | RuntimeException failure) { + setVoteCachePendingVotePartyReward(pendingServer, deliveryId, true); + if (failure instanceof IOException ioFailure) throw ioFailure; + throw (RuntimeException) failure; + } + return; + } + } + + private void scheduleVotePartyDeliveryRetry() { + if (!enabled || votePartyDeliveryRetryScheduled || method != BungeeMethod.HTTP || getScheduler() == null) return; + Collection pendingServers = getVoteCachePendingVotePartyServers(); + if (pendingServers == null || pendingServers.isEmpty()) return; + votePartyDeliveryRetryScheduled = true; + try { + getScheduler().schedule(() -> { + synchronized (VotingPluginProxy.this) { votePartyDeliveryRetryScheduled = false; } + retryPendingVotePartyRewards(); + }, 5, TimeUnit.SECONDS); + } catch (RuntimeException failure) { + votePartyDeliveryRetryScheduled = false; + debug("Unable to schedule HTTP vote-party reward retry: " + failure.getMessage()); } } @@ -2997,6 +3136,8 @@ public void setCurrentVotePartyVotes(int amount) { public abstract void setVoteCacheVotePartyIncreaseVotesRequired(int votes); + public abstract void setVoteCachePendingVotePartyReward(String server, String deliveryId, boolean pending); + public void status() { for (String s : getAllAvailableServers()) { if (!isSomeoneOnlineServerForVoteRouting(s)) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java index 0291a0a48..9fd7db463 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeJsonVoteCache.java @@ -1,7 +1,11 @@ package com.bencodez.votingplugin.proxy.bungee; import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; import java.util.Collection; +import java.util.Locale; import com.bencodez.simpleapi.file.BungeeJsonFile; import com.bencodez.votingplugin.proxy.OfflineBungeeVote; @@ -128,6 +132,19 @@ public void removeTimedVotes() { public int getVotePartyCache(String server) { return getInt("VoteParty.Cache." + server, 0); } + + @Override + public Collection getPendingVotePartyRewardServers() { + Collection encoded = getKeys("VoteParty.PendingRewards"); + Collection servers = new ArrayList<>(); + if (encoded != null) for (String key : encoded) servers.add(decodeServerKey(key)); + return servers; + } + + @Override + public Collection getPendingVotePartyRewardIds(String server) { + return getKeys("VoteParty.PendingRewards." + encodeServerKey(server)); + } public int getVotePartyCurrentVotes() { return getInt("VoteParty.CurrentVotes", 0); @@ -140,6 +157,27 @@ public int getVotePartyInreaseVotesRequired() { public void setVotePartyCache(String server, int amount) { setInt("VoteParty.Cache." + server, amount); } + + @Override + public void setPendingVotePartyReward(String server, String deliveryId, boolean pending) { + String serverPath = "VoteParty.PendingRewards." + encodeServerKey(server); + String path = serverPath + "." + deliveryId; + if (pending) setBoolean(path, true); + else { + setString(path, null); + Collection remaining = getKeys(serverPath); + if (remaining == null || remaining.isEmpty()) setString(serverPath, null); + } + } + + private static String encodeServerKey(String server) { + return Base64.getUrlEncoder().withoutPadding() + .encodeToString(server.toLowerCase(Locale.ROOT).getBytes(StandardCharsets.UTF_8)); + } + + private static String decodeServerKey(String server) { + return new String(Base64.getUrlDecoder().decode(server), StandardCharsets.UTF_8); + } public void setVotePartyCurrentVotes(int amount) { setInt("VoteParty.CurrentVotes", amount); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java index bd644c474..629fe1642 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java @@ -7,6 +7,7 @@ import java.io.Reader; import java.net.URL; import java.security.CodeSource; +import java.util.Collection; import java.util.HashSet; import java.util.Locale; import java.util.Map.Entry; @@ -623,6 +624,16 @@ public int getVoteCachePrevWeek() { public int getVoteCacheVotePartyIncreaseVotesRequired() { return voteCacheFile.getVotePartyInreaseVotesRequired(); } + + @Override + public Collection getVoteCachePendingVotePartyServers() { + return voteCacheFile.getPendingVotePartyRewardServers(); + } + + @Override + public Collection getVoteCachePendingVotePartyRewardIds(String server) { + return voteCacheFile.getPendingVotePartyRewardIds(server); + } @Override public boolean isPlayerOnline(String playerName) { @@ -670,6 +681,12 @@ public void runConsoleCommand(String command) { public void saveVoteCacheFile() { voteCacheFile.save(); } + + @Override + public void saveVotePartyStateDurably() throws java.io.IOException { + com.bencodez.votingplugin.proxy.cache.VotePartyCacheDurability.saveAndVerify( + new File(getDataFolder(), "votecache.json").toPath(), voteCacheFile); + } @Override public boolean sendPluginMessageData(String server, String channel, byte[] data, boolean queue) { @@ -718,6 +735,11 @@ public void setVoteCacheVotePartyCurrentVotes(int votes) { public void setVoteCacheVotePartyIncreaseVotesRequired(int votes) { voteCacheFile.setVotePartyInreaseVotesRequired(votes); } + + @Override + public void setVoteCachePendingVotePartyReward(String server, String deliveryId, boolean pending) { + voteCacheFile.setPendingVotePartyReward(server, deliveryId, pending); + } @Override public void warn(String message) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java index 03b6e4977..eb9905725 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/IVoteCache.java @@ -117,6 +117,11 @@ public interface IVoteCache { */ int getVotePartyCache(String server); + /** Returns backend IDs with persisted, undelivered vote-party rewards. */ + Collection getPendingVotePartyRewardServers(); + + Collection getPendingVotePartyRewardIds(String server); + /** * Gets the current vote party votes. * @@ -139,6 +144,8 @@ public interface IVoteCache { */ void setVotePartyCache(String server, int amount); + void setPendingVotePartyReward(String server, String deliveryId, boolean pending); + /** * Sets the current vote party votes. * diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java new file mode 100644 index 000000000..bfcdf3d76 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurability.java @@ -0,0 +1,76 @@ +package com.bencodez.votingplugin.proxy.cache; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import com.bencodez.votingplugin.util.DurableFiles; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +/** Verifies that the cache library actually persisted the vote-party transaction. */ +public final class VotePartyCacheDurability { + private VotePartyCacheDurability() { } + + public static void saveAndVerify(Path file, IVoteCache cache) throws IOException { + Map> expectedPending = pending(cache); + int expectedVotes = cache.getVotePartyCurrentVotes(); + int expectedIncrease = cache.getVotePartyInreaseVotesRequired(); + cache.save(); + DurableFiles.forceFile(file); + DurableFiles.forceDirectory(file.toAbsolutePath().normalize().getParent()); + try { + JsonObject root = JsonParser.parseString(Files.readString(file, StandardCharsets.UTF_8)).getAsJsonObject(); + JsonObject voteParty = object(root.get("VoteParty")); + if (integer(voteParty, "CurrentVotes") != expectedVotes + || integer(voteParty, "IncreaseVotes") != expectedIncrease + || !pending(voteParty).equals(expectedPending)) + throw new IOException("Vote-party cache state was not persisted"); + } catch (IOException failure) { + throw failure; + } catch (RuntimeException invalid) { + throw new IOException("Vote-party cache state is unreadable", invalid); + } + } + + private static Map> pending(IVoteCache cache) { + Map> pending = new HashMap<>(); + var servers = cache.getPendingVotePartyRewardServers(); + if (servers == null) return pending; + for (String server : servers) { + var ids = cache.getPendingVotePartyRewardIds(server); + if (ids != null && !ids.isEmpty()) pending.put(server, new HashSet<>(ids)); + } + return pending; + } + + private static Map> pending(JsonObject voteParty) { + Map> pending = new HashMap<>(); + JsonObject encodedServers = object(voteParty.get("PendingRewards")); + for (Map.Entry server : encodedServers.entrySet()) { + String serverId = new String(Base64.getUrlDecoder().decode(server.getKey()), StandardCharsets.UTF_8); + JsonObject rewards = object(server.getValue()); + Set ids = new HashSet<>(); + for (Map.Entry reward : rewards.entrySet()) + if (reward.getValue().isJsonPrimitive() && reward.getValue().getAsBoolean()) ids.add(reward.getKey()); + if (!ids.isEmpty()) pending.put(serverId, ids); + } + return pending; + } + + private static JsonObject object(JsonElement element) { + return element == null || element.isJsonNull() ? new JsonObject() : element.getAsJsonObject(); + } + + private static int integer(JsonObject object, String name) { + JsonElement value = object.get(name); + return value == null || value.isJsonNull() ? 0 : value.getAsInt(); + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java index 0569de173..b811ff764 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityJsonVoteCache.java @@ -1,7 +1,11 @@ package com.bencodez.votingplugin.proxy.velocity; import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; import java.util.Collection; +import java.util.Locale; import com.bencodez.simpleapi.file.velocity.VelocityJSONFile; import com.bencodez.votingplugin.proxy.OfflineBungeeVote; @@ -134,6 +138,19 @@ public int getVotePartyCache(String server) { return getNode("VoteParty", "Cache", server).getInt(0); } + @Override + public Collection getPendingVotePartyRewardServers() { + Collection encoded = getKeys(getNode("VoteParty", "PendingRewards")); + Collection servers = new ArrayList<>(); + if (encoded != null) for (String key : encoded) servers.add(decodeServerKey(key)); + return servers; + } + + @Override + public Collection getPendingVotePartyRewardIds(String server) { + return getKeys(getNode("VoteParty", "PendingRewards", encodeServerKey(server))); + } + @Override public int getVotePartyCurrentVotes() { return getInt(getNode("VoteParty", "CurrentVotes"), 0); @@ -149,6 +166,26 @@ public void setVotePartyCache(String server, int amount) { setPath(amount, "VoteParty", "Cache", server); } + @Override + public void setPendingVotePartyReward(String server, String deliveryId, boolean pending) { + String serverKey = encodeServerKey(server); + if (pending) setPath(true, "VoteParty", "PendingRewards", serverKey, deliveryId); + else { + remove("VoteParty", "PendingRewards", serverKey, deliveryId); + Collection remaining = getKeys(getNode("VoteParty", "PendingRewards", serverKey)); + if (remaining == null || remaining.isEmpty()) remove("VoteParty", "PendingRewards", serverKey); + } + } + + private static String encodeServerKey(String server) { + return Base64.getUrlEncoder().withoutPadding() + .encodeToString(server.toLowerCase(Locale.ROOT).getBytes(StandardCharsets.UTF_8)); + } + + private static String decodeServerKey(String server) { + return new String(Base64.getUrlDecoder().decode(server), StandardCharsets.UTF_8); + } + @Override public void setVotePartyCurrentVotes(int amount) { setPath(amount, "VoteParty", "CurrentVotes"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java index 176c045c6..9202c866e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java @@ -13,6 +13,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.security.CodeSource; +import java.util.Collection; import java.util.HashSet; import java.util.Locale; import java.util.Map.Entry; @@ -714,6 +715,16 @@ public int getVoteCacheVotePartyIncreaseVotesRequired() { return voteCacheFile.getVotePartyInreaseVotesRequired(); } + @Override + public Collection getVoteCachePendingVotePartyServers() { + return voteCacheFile.getPendingVotePartyRewardServers(); + } + + @Override + public Collection getVoteCachePendingVotePartyRewardIds(String server) { + return voteCacheFile.getPendingVotePartyRewardIds(server); + } + @Override public boolean isVoteCacheIgnoreTime() { return voteCacheFile.getNode("Time", "IgnoreTime").getBoolean(); @@ -759,6 +770,11 @@ public void setVoteCacheVotePartyIncreaseVotesRequired(int votes) { voteCacheFile.setVotePartyInreaseVotesRequired(votes); } + @Override + public void setVoteCachePendingVotePartyReward(String server, String deliveryId, boolean pending) { + voteCacheFile.setPendingVotePartyReward(server, deliveryId, pending); + } + @Override public boolean isPlayerOnline(String playerName) { if (playerName == null) { @@ -805,6 +821,12 @@ public void saveVoteCacheFile() { voteCacheFile.save(); } + @Override + public void saveVotePartyStateDurably() throws java.io.IOException { + com.bencodez.votingplugin.proxy.cache.VotePartyCacheDurability.saveAndVerify( + dataDirectory.resolve("votecache.json"), voteCacheFile); + } + @Override public boolean sendPluginMessageData(String serverName, String channelName, byte[] data, boolean queue) { if (!server.getServer(serverName).isPresent()) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 0e6a26bd0..61b8b785b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -58,6 +58,35 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti } } + @Test + void stableProxyDeliveryIdsAreIdempotentAndAcknowledgedBeforeRemoval() throws Exception { + AtomicLong acknowledged = new AtomicLong(); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", null, + (server, deliveryId) -> acknowledged.incrementAndGet()); + String deliveryId = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("vote-party").build()); + assertTrue(state.enqueue(delivery)); + assertTrue(state.enqueue(delivery)); + assertFalse(state.enqueue(new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("different").build()))); + state.acknowledge(java.util.List.of(deliveryId)); + assertEquals(1L, acknowledged.get()); + assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); + } + + @Test + void failedAcknowledgementCallbackRetainsProxyDelivery() throws Exception { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", null, + (server, deliveryId) -> { throw new java.io.IOException("cache save failed"); }); + String deliveryId = java.util.UUID.randomUUID().toString(); + assertTrue(state.enqueue(new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("vote-party").build()))); + assertThrows(java.io.IOException.class, () -> state.acknowledge(java.util.List.of(deliveryId))); + assertEquals(deliveryId, state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0) + .messages().iterator().next().id()); + } + @Test void closeWaitsForTheCredentialOwningPollerToStop() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("close-proxy"), "localhost"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 662e9e84a..fa00d1342 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -41,10 +41,11 @@ void backendResponseReaderRejectsBodiesBeyondTheWireLimit() throws Exception { @Test void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { - HttpConnectionCode original = new HttpConnectionCode("lobby", URI.create("https://Proxy.Example.test:8443/http"), pin('a'), pin('b'), + HttpConnectionCode original = new HttpConnectionCode("lobby.eu", URI.create("https://Proxy.Example.test:8443/http"), pin('a'), pin('b'), Instant.parse("2030-01-01T00:00:00Z"), HttpTransportSecrets.randomToken()); String encoded = original.encode(); HttpConnectionCode parsed = HttpConnectionCode.parse(encoded); + assertEquals("lobby.eu", parsed.serverId()); assertEquals(URI.create("https://proxy.example.test:8443/http/"), parsed.endpoint()); assertEquals(original.serverCertificatePin(), parsed.serverCertificatePin()); char last = encoded.charAt(encoded.length() - 1); @@ -53,6 +54,24 @@ void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { assertThrows(IllegalArgumentException.class, () -> HttpConnectionCode.parse("http://not-a-code")); } + @Test + void legacyConnectionCodesAndConsumedMarkersRemainCompatible() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("legacy-code-proxy"), "proxy.example.test"); + HttpConnectionCode legacy = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), + HttpTransportSecrets.randomToken()); + assertEquals(legacy, HttpConnectionCode.parse(legacy.encodeLegacy())); + + Path client = directory.resolve("legacy-code-client"); + HttpClientCredentialStore.saveEnrolled(client, legacy, identity.issueClientCertificate("lobby")); + Path active = client.resolve("http-transport-client-generations") + .resolve(Files.readString(client.resolve("http-transport-client-current"))); + Files.writeString(active.resolve("http-transport-connection-code.sha256"), + HttpTransportSecrets.sha256Hex(legacy.encodeLegacy().getBytes(java.nio.charset.StandardCharsets.US_ASCII))); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, + HttpConnectionCode.parse(legacy.encodeLegacy()))); + } + @Test void expiredCodesAreNotActive() { HttpConnectionCode code = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test/"), pin('a'), pin('b'), diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurabilityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurabilityTest.java new file mode 100644 index 000000000..c78157c1a --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/cache/VotePartyCacheDurabilityTest.java @@ -0,0 +1,29 @@ +package com.bencodez.votingplugin.proxy.cache; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Set; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class VotePartyCacheDurabilityTest { + @TempDir Path directory; + + @Test + void detectsWhenCacheSaveReturnsWithoutUpdatingTheFile() throws Exception { + IVoteCache cache = mock(IVoteCache.class); + when(cache.getPendingVotePartyRewardServers()).thenReturn(Set.of("lobby")); + when(cache.getPendingVotePartyRewardIds("lobby")).thenReturn(Set.of("delivery-id")); + when(cache.getVotePartyCurrentVotes()).thenReturn(3); + when(cache.getVotePartyInreaseVotesRequired()).thenReturn(2); + Path file = directory.resolve("votecache.json"); + Files.writeString(file, "{\"VoteParty\":{\"CurrentVotes\":3,\"IncreaseVotes\":2}}"); + + assertThrows(java.io.IOException.class, () -> VotePartyCacheDurability.saveAndVerify(file, cache)); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java index 6b3fb402d..12b4faede 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.never; @@ -80,6 +81,54 @@ void testAddCurrentVotePartyVotes() { assertEquals(5, votingPluginProxy.getVotePartyVotes()); } + @Test + void rejectedHttpVotePartyRewardRemainsPendingUntilAccepted() throws Exception { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + + votingPluginProxy.sendVoteParty("Server1"); + assertEquals(1, votingPluginProxy.getVoteCachePendingVotePartyRewardIds("Server1").size()); + String deliveryId = votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(0); + + votingPluginProxy.setVoteEnvelopeDeliveryResult(true); + votingPluginProxy.retryPendingVotePartyRewardsForTest(); + assertEquals(deliveryId, votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(1)); + assertEquals(1, votingPluginProxy.getVoteCachePendingVotePartyRewardIds("Server1").size()); + votingPluginProxy.acknowledgeVotePartyDeliveryForTest("server1", deliveryId); + assertEquals(0, votingPluginProxy.getVoteCachePendingVotePartyRewardIds("Server1").size()); + } + + @Test + void failedVotePartyAcknowledgementSaveRestoresPendingMarker() { + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + votingPluginProxy.sendVoteParty("Server1"); + String deliveryId = votingPluginProxy.getAttemptedVotePartyDeliveryIds().get(0); + + votingPluginProxy.failNextVoteCacheSave(); + assertThrows(IllegalStateException.class, + () -> votingPluginProxy.acknowledgeVotePartyDeliveryForTest("server1", deliveryId)); + assertTrue(votingPluginProxy.getVoteCachePendingVotePartyRewardIds("Server1").contains(deliveryId)); + } + + @Test + void httpVotePartyStagesEveryTargetBeforeDelivery() { + Mockito.when(votingPluginProxy.getConfig().getVotePartyEnabled()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getVotePartySendToAllServers()).thenReturn(true); + Mockito.when(votingPluginProxy.getConfig().getVotePartyBroadcast()).thenReturn(""); + Mockito.when(votingPluginProxy.getConfig().getVotePartyBungeeCommands()).thenReturn(java.util.List.of()); + votingPluginProxy.setMethod(BungeeMethod.HTTP); + votingPluginProxy.setVoteEnvelopeDeliveryResult(false); + votingPluginProxy.setVotePartyVotes(1); + votingPluginProxy.setCurrentVotePartyVotesRequired(1); + + votingPluginProxy.checkVoteParty(); + + assertEquals(1, votingPluginProxy.getVoteCachePendingVotePartyRewardIds("server1").size()); + assertEquals(1, votingPluginProxy.getVoteCachePendingVotePartyRewardIds("SERVER2").size()); + assertEquals(0, votingPluginProxy.getVotePartyVotes()); + } + @Test void rolloverProjectionIncludesQueuedVotesAndVotePartyThresholds() { Mockito.when(votingPluginProxy.getConfig().getVotePartyEnabled()).thenReturn(true); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index a2d18b1c1..15ea91aef 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -4,6 +4,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.HashMap; +import java.util.Collection; import java.util.List; import java.util.Set; import java.util.UUID; @@ -29,6 +31,9 @@ public class VotingPluginProxyTestImpl extends VotingPluginProxy { private JsonEnvelope lastCommunicationTestEnvelope; private boolean playerOnline = true; private ScheduledExecutorService scheduler; + private boolean failNextVoteCacheSave; + private final java.util.Map> pendingVotePartyRewards = new HashMap<>(); + private final List attemptedVotePartyDeliveryIds = new ArrayList<>(); public List getWarnings() { return warnings; @@ -125,6 +130,17 @@ public int getVoteCacheVotePartyIncreaseVotesRequired() { return 10; } + @Override + public Collection getVoteCachePendingVotePartyServers() { + return new HashSet<>(pendingVotePartyRewards.keySet()); + } + + @Override + public Collection getVoteCachePendingVotePartyRewardIds(String server) { + return new HashSet<>(pendingVotePartyRewards.getOrDefault(server.toLowerCase(java.util.Locale.ROOT), + java.util.Set.of())); + } + @Override public boolean isPlayerOnline(String playerName) { return playerOnline; @@ -161,7 +177,19 @@ public void runConsoleCommand(String command) { @Override public void saveVoteCacheFile() { - // Mocked for testing + if (failNextVoteCacheSave) { + failNextVoteCacheSave = false; + throw new IllegalStateException("vote cache save failed"); + } + } + + @Override + public void saveVotePartyStateDurably() { + saveVoteCacheFile(); + } + + public void failNextVoteCacheSave() { + failNextVoteCacheSave = true; } @Override @@ -226,6 +254,20 @@ public void setVoteEnvelopeDeliveryResult(boolean voteEnvelopeDeliveryResult) { this.voteEnvelopeDeliveryResult = voteEnvelopeDeliveryResult; } + @Override + protected boolean sendHttpEnvelope(String server, String deliveryId, JsonEnvelope envelope) { + attemptedVotePartyDeliveryIds.add(deliveryId); + return voteEnvelopeDeliveryResult; + } + + public List getAttemptedVotePartyDeliveryIds() { + return attemptedVotePartyDeliveryIds; + } + + public void acknowledgeVotePartyDeliveryForTest(String server, String deliveryId) throws java.io.IOException { + acknowledgeVotePartyDelivery(server, deliveryId); + } + @Override protected boolean sendCommunicationTestEnvelopeNow(String server, JsonEnvelope envelope) { lastCommunicationTestEnvelope = envelope; @@ -286,6 +328,10 @@ public void retryPendingTimeBroadcastsForTest(String server) { retryPendingTimeBroadcasts(server); } + public void retryPendingVotePartyRewardsForTest() { + retryPendingVotePartyRewards(); + } + public int[] getProjectedVotePartyStateForTest(int acceptedVotes) { return getProjectedVotePartyState(acceptedVotes); } @@ -364,6 +410,16 @@ public void setVoteCacheVotePartyIncreaseVotesRequired(int votes) { } + @Override + public void setVoteCachePendingVotePartyReward(String server, String deliveryId, boolean pending) { + server = server.toLowerCase(java.util.Locale.ROOT); + if (pending) pendingVotePartyRewards.computeIfAbsent(server, ignored -> new HashSet<>()).add(deliveryId); + else { + java.util.Set rewards = pendingVotePartyRewards.get(server); + if (rewards != null && rewards.remove(deliveryId) && rewards.isEmpty()) pendingVotePartyRewards.remove(server); + } + } + @Override public ScheduledExecutorService getScheduler() { return scheduler; From c10236a620198c9b4fda5690ca7298eeb82404bd Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 19:47:56 -0600 Subject: [PATCH 31/36] fix(http): import vote-party collection helpers --- .../java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 75e81d3c7..f9b2ffeee 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -21,8 +21,10 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.UUID; From 7fec7976a4f84fc384f028bad711230f39b6d1c5 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 19:51:05 -0600 Subject: [PATCH 32/36] test(http): normalize legacy code expiry precision --- .../backendproxy/http/HttpTransportSecurityTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index fa00d1342..02024377b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -58,7 +58,8 @@ void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { void legacyConnectionCodesAndConsumedMarkersRemainCompatible() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("legacy-code-proxy"), "proxy.example.test"); HttpConnectionCode legacy = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test:8443/"), - identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), + identity.serverCertificatePin(), identity.caCertificatePin(), + Instant.now().plusSeconds(60).truncatedTo(java.time.temporal.ChronoUnit.SECONDS), HttpTransportSecrets.randomToken()); assertEquals(legacy, HttpConnectionCode.parse(legacy.encodeLegacy())); From 01dea25564db4f3c980d0780785289f0a5e67ff8 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 20:01:20 -0600 Subject: [PATCH 33/36] fix(http): bound streamed response reads --- .../http/HttpBackendTransportConnector.java | 76 +++++++++++++++++-- .../http/HttpTransportRuntimeTest.java | 28 +++++++ 2 files changed, 96 insertions(+), 8 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 4370d732d..1f5f83990 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -1,12 +1,14 @@ package com.bencodez.votingplugin.backendproxy.http; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.security.KeyStore; @@ -21,8 +23,11 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Flow; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; @@ -335,13 +340,68 @@ private static HttpClient client(HttpClientCredentialStore.HttpClientProfile pro return HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); } - private static LimitedResponse sendLimited(HttpClient client, HttpRequest request) throws IOException, InterruptedException { - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); - try (InputStream body = response.body()) { - long declaredLength = response.headers().firstValueAsLong("Content-Length").orElse(-1L); - if (declaredLength > HttpTransportProtocol.MAX_BODY_BYTES) - throw new IOException("HTTP transport response exceeds its limit"); - return new LimitedResponse(response.statusCode(), readLimited(body)); + static LimitedResponse sendLimited(HttpClient client, HttpRequest request) throws IOException, InterruptedException { + HttpResponse response = client.send(request, + ignored -> new LimitedBodySubscriber(HttpTransportProtocol.MAX_BODY_BYTES)); + long declaredLength = response.headers().firstValueAsLong("Content-Length").orElse(-1L); + if (declaredLength > HttpTransportProtocol.MAX_BODY_BYTES) + throw new IOException("HTTP transport response exceeds its limit"); + return new LimitedResponse(response.statusCode(), response.body()); + } + + private static final class LimitedBodySubscriber implements HttpResponse.BodySubscriber { + private final int maximum; + private final ByteArrayOutputStream body = new ByteArrayOutputStream(); + private final CompletableFuture result = new CompletableFuture<>(); + private Flow.Subscription subscription; + + private LimitedBodySubscriber(int maximum) { + this.maximum = maximum; + } + + @Override + public CompletionStage getBody() { + return result; + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + if (this.subscription != null) { + subscription.cancel(); + return; + } + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(List buffers) { + try { + for (ByteBuffer buffer : buffers) { + if (buffer.remaining() > maximum - body.size()) { + subscription.cancel(); + result.completeExceptionally(new IOException("HTTP transport response exceeds its limit")); + return; + } + byte[] chunk = new byte[buffer.remaining()]; + buffer.get(chunk); + body.writeBytes(chunk); + } + subscription.request(1); + } catch (RuntimeException failure) { + subscription.cancel(); + result.completeExceptionally(failure); + } + } + + @Override + public void onError(Throwable failure) { + result.completeExceptionally(failure); + } + + @Override + public void onComplete() { + result.complete(body.toByteArray()); } } static byte[] readLimited(InputStream body) throws IOException { @@ -350,7 +410,7 @@ static byte[] readLimited(InputStream body) throws IOException { throw new IOException("HTTP transport response exceeds its limit"); return bytes; } - private record LimitedResponse(int statusCode, byte[] body) { } + record LimitedResponse(int statusCode, byte[] body) { } private static SSLContext clientContext(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { String caPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java index 61b8b785b..96a826c62 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportRuntimeTest.java @@ -4,10 +4,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; import static org.junit.jupiter.api.Assertions.assertTrue; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import java.net.InetSocketAddress; +import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; @@ -58,6 +60,32 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti } } + @Test + void responseBodyConsumptionRemainsBoundedByRequestTimeout() throws Exception { + com.sun.net.httpserver.HttpServer server = com.sun.net.httpserver.HttpServer.create( + new InetSocketAddress("localhost", 0), 1); + CountDownLatch release = new CountDownLatch(1); + server.createContext("/stall", exchange -> { + exchange.sendResponseHeaders(200, 8); + try (var output = exchange.getResponseBody()) { + output.write(1); + output.flush(); + try { release.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } + }); + server.start(); + try { + HttpRequest request = HttpRequest.newBuilder(URI.create("http://localhost:" + server.getAddress().getPort() + + "/stall")).timeout(Duration.ofMillis(250)).GET().build(); + assertTimeoutPreemptively(Duration.ofSeconds(2), () -> assertThrows(java.io.IOException.class, + () -> HttpBackendTransportConnector.sendLimited(HttpClient.newHttpClient(), request))); + } finally { + release.countDown(); + server.stop(0); + } + } + @Test void stableProxyDeliveryIdsAreIdempotentAndAcknowledgedBeforeRemoval() throws Exception { AtomicLong acknowledged = new AtomicLong(); From f8113897b202aeb8dbc18c391fabf05fe785e97d Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 1 Sep 2026 20:04:31 -0600 Subject: [PATCH 34/36] fix(http): cancel stalled body reads at deadline --- .../http/HttpBackendTransportConnector.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java index 1f5f83990..f10132e50 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpBackendTransportConnector.java @@ -8,6 +8,7 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.net.http.HttpTimeoutException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Path; @@ -27,11 +28,13 @@ import java.util.concurrent.CompletionStage; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Flow; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import javax.net.ssl.KeyManagerFactory; @@ -341,8 +344,23 @@ private static HttpClient client(HttpClientCredentialStore.HttpClientProfile pro .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); } static LimitedResponse sendLimited(HttpClient client, HttpRequest request) throws IOException, InterruptedException { - HttpResponse response = client.send(request, + CompletableFuture> exchange = client.sendAsync(request, ignored -> new LimitedBodySubscriber(HttpTransportProtocol.MAX_BODY_BYTES)); + HttpResponse response; + try { + Duration timeout = request.timeout().orElse(CLIENT_TIMEOUT); + response = exchange.get(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (TimeoutException timeout) { + exchange.cancel(true); + throw new HttpTimeoutException("HTTP transport response timed out"); + } catch (InterruptedException interrupted) { + exchange.cancel(true); + throw interrupted; + } catch (ExecutionException failure) { + Throwable cause = failure.getCause(); + if (cause instanceof IOException ioFailure) throw ioFailure; + throw new IOException("HTTP transport request failed", cause); + } long declaredLength = response.headers().firstValueAsLong("Content-Length").orElse(-1L); if (declaredLength > HttpTransportProtocol.MAX_BODY_BYTES) throw new IOException("HTTP transport response exceeds its limit"); From 7495519d61b42f69feccbbd53cf6ffda762a6abb Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:26:29 -0600 Subject: [PATCH 35/36] fix(http): recover incomplete TLS provisioning --- .../backendproxy/http/HttpTlsIdentity.java | 45 +++++++++++++++++-- .../http/HttpTransportSecurityTest.java | 43 ++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java index 5a61daefc..9aead7e8c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/http/HttpTlsIdentity.java @@ -54,6 +54,9 @@ public final class HttpTlsIdentity { private static final String CA_FILE = "http-transport-ca.p12"; private static final String SERVER_FILE = "http-transport-server.p12"; private static final String PASSWORD_FILE = "http-transport-password"; + private static final String INITIALIZING_FILE = "http-transport-initializing"; + private static final String ENROLLMENT_STATE_FILE = "http-transport-clients.properties"; + private static final String OUTGOING_DIRECTORY = "outgoing-v1"; private static final char[] EMPTY_PASSWORD = new char[0]; static final Duration RENEW_BEFORE = Duration.ofDays(30); static final Duration CA_RENEW_BEFORE = Duration.ofDays(365); @@ -96,11 +99,26 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock Path caFile = safe(directory.resolve(CA_FILE)); Path serverFile = safe(directory.resolve(SERVER_FILE)); Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); + Path initializingFile = safe(directory.resolve(INITIALIZING_FILE)); boolean caExists = Files.exists(caFile, LinkOption.NOFOLLOW_LINKS); boolean serverExists = Files.exists(serverFile, LinkOption.NOFOLLOW_LINKS); boolean passwordExists = Files.exists(passwordFile, LinkOption.NOFOLLOW_LINKS); + boolean initializing = Files.exists(initializingFile, LinkOption.NOFOLLOW_LINKS); + boolean anyIdentityFile = caExists || serverExists || passwordExists; + boolean completeIdentity = caExists && serverExists && passwordExists; + boolean persistentTransportState = hasPersistentTransportState(directory); + if (initializing || (anyIdentityFile && !completeIdentity)) { + if (persistentTransportState) + throw new IOException("HTTP TLS identity files are incomplete"); + if (!initializing) writeInitializationMarker(initializingFile); + discardUncommittedIdentity(caFile, serverFile, passwordFile); + caExists = false; + serverExists = false; + passwordExists = false; + initializing = true; + } if (caExists || serverExists || passwordExists) { - if (!(caExists && serverExists && passwordExists)) throw new IOException("HTTP TLS identity files are incomplete"); + if (!completeIdentity) throw new IOException("HTTP TLS identity files are incomplete"); char[] password = readPassword(passwordFile); try { KeyStore ca = load(caFile, password); @@ -137,6 +155,9 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock advertisedHost); } finally { Arrays.fill(password, '\0'); } } + if (persistentTransportState) + throw new IOException("HTTP TLS identity files are missing"); + if (!initializing) writeInitializationMarker(initializingFile); ensureBouncyCastle(); char[] password = HttpTransportSecrets.randomToken().toCharArray(); try { @@ -154,9 +175,10 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock server.setKeyEntry("server", serverPair.getPrivate(), password, new Certificate[] { serverCertificate, caCertificate }); writeStore(caFile, ca, password); writeStore(serverFile, server, password); - byte[] passwordBytes = asciiBytes(password); - try { writePrivate(passwordFile, passwordBytes); } - finally { Arrays.fill(passwordBytes, (byte) 0); } + byte[] passwordBytes = asciiBytes(password); + try { writePrivate(passwordFile, passwordBytes); } + finally { Arrays.fill(passwordBytes, (byte) 0); } + DurableFiles.deleteIfExists(initializingFile); return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password, caFile, serverFile, advertisedHost); } finally { Arrays.fill(password, '\0'); } @@ -391,6 +413,21 @@ private static void writePrivate(Path file, byte[] contents) throws IOException } finally { Files.deleteIfExists(temporary); } } + private static void writeInitializationMarker(Path file) throws IOException { + writePrivate(file, "initializing\n".getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + } + + private static void discardUncommittedIdentity(Path caFile, Path serverFile, Path passwordFile) throws IOException { + DurableFiles.deleteIfExists(caFile); + DurableFiles.deleteIfExists(serverFile); + DurableFiles.deleteIfExists(passwordFile); + } + + private static boolean hasPersistentTransportState(Path directory) { + return Files.exists(directory.resolve(ENROLLMENT_STATE_FILE), LinkOption.NOFOLLOW_LINKS) + || Files.exists(directory.resolve(OUTGOING_DIRECTORY), LinkOption.NOFOLLOW_LINKS); + } + private static byte[] asciiBytes(char[] characters) { byte[] output = new byte[characters.length]; for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java index 02024377b..f67173889 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/http/HttpTransportSecurityTest.java @@ -131,6 +131,49 @@ void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { assertNotEquals(created.serverCertificatePin(), rotated.serverCertificatePin()); } + @Test + void incompleteFirstRunTlsProvisioningRecoversWithoutManualCleanup() throws Exception { + Path source = directory.resolve("complete-identity"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(source, "localhost"); + Path interrupted = directory.resolve("interrupted-identity"); + Files.createDirectories(interrupted); + Files.copy(source.resolve("http-transport-ca.p12"), interrupted.resolve("http-transport-ca.p12")); + Files.copy(source.resolve("http-transport-server.p12"), interrupted.resolve("http-transport-server.p12")); + + HttpTlsIdentity recovered = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); + + assertNotEquals(original.caCertificatePin(), recovered.caCertificatePin()); + assertTrue(Files.exists(interrupted.resolve("http-transport-ca.p12"))); + assertTrue(Files.exists(interrupted.resolve("http-transport-server.p12"))); + assertTrue(Files.exists(interrupted.resolve("http-transport-password"))); + assertFalse(Files.exists(interrupted.resolve("http-transport-initializing"))); + } + + @Test + void completedFirstRunFilesRecoverWhenInitializationMarkerSurvives() throws Exception { + Path interrupted = directory.resolve("marked-identity"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); + String originalCaPin = original.caCertificatePin(); + Files.writeString(interrupted.resolve("http-transport-initializing"), "initializing\n"); + + HttpTlsIdentity recovered = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); + + assertNotEquals(originalCaPin, recovered.caCertificatePin()); + assertFalse(Files.exists(interrupted.resolve("http-transport-initializing"))); + } + + @Test + void incompleteEstablishedTlsIdentityFailsClosed() throws Exception { + Path established = directory.resolve("established-identity"); + HttpTlsIdentity.loadOrCreate(established, "localhost"); + Files.writeString(established.resolve("http-transport-clients.properties"), "version=2\n"); + Files.delete(established.resolve("http-transport-server.p12")); + + assertThrows(java.io.IOException.class, () -> HttpTlsIdentity.loadOrCreate(established, "localhost")); + assertTrue(Files.exists(established.resolve("http-transport-ca.p12"))); + assertTrue(Files.exists(established.resolve("http-transport-password"))); + } + @Test void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory, "localhost"); From e23a9691d0525575057de5c1e953853ac27e130f Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:44:24 -0600 Subject: [PATCH 36/36] fix(http): defer replacement presence publication --- .../votingplugin/VotingPluginMain.java | 3 +- .../backendproxy/BackendProxyHandler.java | 28 +++++++++++-- .../presence/BackendPresenceManager.java | 28 ++++++++----- .../BackendProxyHandlerLifecycleTest.java | 39 +++++++++++++++++++ 4 files changed, 83 insertions(+), 15 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 259f3fa0c..201b84640 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1270,7 +1270,7 @@ public synchronized BackendProxyRestart prepareBackendProxyHandlerRestart() { boolean previousPrepared = previous != null && previous.prepareForReplacement(replacementMethod); BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); try { - replacement.load(); + replacement.loadForReplacement(); } catch (RuntimeException failure) { replacement.close(); if (previousPrepared) { @@ -1308,6 +1308,7 @@ public void completeBackendProxyHandlerRestart(BackendProxyRestart restart) { return; } if (restart.previous != null) restart.previous.completeRedisHandoff(restart.replacement); + restart.replacement.activatePresenceReporting(); backendProxyHandler = restart.replacement; if (restart.previous != null) restart.previous.close(); restart.finished = true; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index ec7998b54..eb269f08b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -39,6 +39,7 @@ public class BackendProxyHandler implements Listener { private final BackendGlobalDataSync globalDataSync; private BackendPresenceManager presenceManager; + private boolean presenceReportingActivated; private BackendVotePartySync votePartySync; private BackendProxyMessageRouter messageRouter; @@ -62,6 +63,15 @@ public BackendProxyHandler(VotingPluginMain plugin, ProcessedVoteCache processed * Loads the configured backend/proxy communication components. */ public void load() { + load(true); + } + + /** Loads a replacement without announcing a new presence generation before publication. */ + public void loadForReplacement() { + load(false); + } + + private void load(boolean activatePresenceReporting) { plugin.debug("Loading backend proxy handler"); method = BungeeMethod.getByName(plugin.getBungeeSettings().getBungeeMethod()); plugin.getLogger().info("Using BungeeMethod: " + method.toString()); @@ -84,15 +94,24 @@ public void sendMessage(JsonEnvelope envelope) { if (plugin.getOptions().getServer().equalsIgnoreCase("pleaseset")) { plugin.getLogger().warning("Server name for bungee voting is not set, please set it"); } - presenceManager.start(); + if (activatePresenceReporting) activatePresenceReporting(); + } + + /** Starts presence only after a staged handler reaches the atomic publication boundary. */ + public void activatePresenceReporting() { + if (presenceManager != null && !presenceReportingActivated) { + presenceManager.start(); + presenceReportingActivated = true; + } } /** * Closes backend/proxy components and persists cached proxy state. */ public void close() { - if (presenceManager != null) { + if (presenceManager != null && presenceReportingActivated) { presenceManager.stop(); + presenceReportingActivated = false; } transportManager.close(); if (votePartySync != null) { @@ -152,14 +171,15 @@ public void playerOffline(String playerName) { } public void reloadPresenceReporting() { - if (presenceManager != null) { + if (presenceManager != null && presenceReportingActivated) { presenceManager.reload(); } } public void disablePresenceReporting() { - if (presenceManager != null) { + if (presenceManager != null && presenceReportingActivated) { presenceManager.stop(); + presenceReportingActivated = false; } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java index a82c5551a..9b560a3c3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/presence/BackendPresenceManager.java @@ -67,20 +67,28 @@ public void start() { lastResyncRequestAtNanos = 0L; lastSnapshotRequestId = null; lastSnapshotRequestAtNanos = 0L; - send(VotingPluginWire.backendStarted(server, incarnationId, startedAt, now)); - send(VotingPluginWire.backendHeartbeat(server, incarnationId, startedAt, nextTimestamp())); - if (heartbeatTask != null) { heartbeatTask.cancel(false); } - heartbeatTask = plugin.getTimer().scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - sendHeartbeat(); - } - }, HEARTBEAT_SECONDS, HEARTBEAT_SECONDS, TimeUnit.SECONDS); + try { + heartbeatTask = plugin.getTimer().scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + sendHeartbeat(); + } + }, HEARTBEAT_SECONDS, HEARTBEAT_SECONDS, TimeUnit.SECONDS); + seedOnlinePlayers(); + } catch (RuntimeException failure) { + if (heartbeatTask != null) heartbeatTask.cancel(false); + heartbeatTask = null; + reporting = false; + server = null; + incarnationId = null; + throw failure; + } + send(VotingPluginWire.backendStarted(server, incarnationId, startedAt, now)); + send(VotingPluginWire.backendHeartbeat(server, incarnationId, startedAt, nextTimestamp())); } - seedOnlinePlayers(); } public void stop() { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index b436fadd5..f26bad9de 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -2,9 +2,12 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; import java.lang.reflect.Field; import java.util.concurrent.ScheduledExecutorService; @@ -12,11 +15,14 @@ import org.junit.jupiter.api.Test; import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; import com.bencodez.simpleapi.servercomm.redis.RedisHandler; import com.bencodez.votingplugin.backendproxy.global.BackendGlobalDataSync; import com.bencodez.votingplugin.backendproxy.cache.ProcessedVoteCache; +import com.bencodez.votingplugin.backendproxy.presence.BackendPresenceManager; +import com.bencodez.votingplugin.config.BungeeSettings; import com.bencodez.votingplugin.backendproxy.transport.MqttBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.MysqlBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.BackendProxyTransport; @@ -26,6 +32,39 @@ import com.bencodez.votingplugin.proxy.BungeeMethod; class BackendProxyHandlerLifecycleTest { + @Test + void failedPresenceActivationDoesNotAnnounceReplacementGeneration() { + com.bencodez.votingplugin.VotingPluginMain plugin = mock(com.bencodez.votingplugin.VotingPluginMain.class); + BungeeSettings settings = mock(BungeeSettings.class); + ScheduledExecutorService timer = mock(ScheduledExecutorService.class); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + when(plugin.getBungeeSettings()).thenReturn(settings); + when(settings.getServer()).thenReturn("lobby"); + when(plugin.getTimer()).thenReturn(timer); + when(timer.scheduleAtFixedRate(org.mockito.ArgumentMatchers.any(Runnable.class), + org.mockito.ArgumentMatchers.anyLong(), org.mockito.ArgumentMatchers.anyLong(), + org.mockito.ArgumentMatchers.any())).thenThrow(new java.util.concurrent.RejectedExecutionException()); + + BackendPresenceManager presence = new BackendPresenceManager(plugin, BungeeMethod.HTTP, messages); + assertThrows(java.util.concurrent.RejectedExecutionException.class, presence::start); + + verifyNoInteractions(messages); + } + + @Test + void stagedPresenceStartsOnlyAtExplicitPublication() throws Exception { + BackendProxyHandler handler = new BackendProxyHandler(null); + BackendPresenceManager presence = mock(BackendPresenceManager.class); + Field field = BackendProxyHandler.class.getDeclaredField("presenceManager"); + field.setAccessible(true); + field.set(handler, presence); + + verifyNoInteractions(presence); + handler.activatePresenceReporting(); + handler.activatePresenceReporting(); + + verify(presence, times(1)).start(); + } @Test void sharesVoteDeduplicationAcrossHandlerReplacement() {