From 4cd5d568bc185bcb9e5d1581d98d2dac6ab311ae Mon Sep 17 00:00:00 2001 From: Doksanbir Date: Thu, 3 Sep 2026 12:18:29 +0300 Subject: [PATCH 1/2] feat: add Microservices Bulkhead pattern (#3228) --- microservices-bulkhead/README.md | 320 ++++++++++++++++++ .../etc/microservices-bulkhead.urm.puml | 53 +++ microservices-bulkhead/pom.xml | 70 ++++ .../main/java/com/iluwatar/bulkhead/App.java | 132 ++++++++ .../java/com/iluwatar/bulkhead/Bulkhead.java | 146 ++++++++ .../bulkhead/BulkheadFullException.java | 46 +++ .../iluwatar/bulkhead/InventoryService.java | 41 +++ .../com/iluwatar/bulkhead/PaymentService.java | 60 ++++ .../com/iluwatar/bulkhead/RemoteService.java | 42 +++ .../java/com/iluwatar/bulkhead/AppTest.java | 151 +++++++++ .../com/iluwatar/bulkhead/BulkheadTest.java | 156 +++++++++ .../bulkhead/InventoryServiceTest.java | 39 +++ .../iluwatar/bulkhead/PaymentServiceTest.java | 54 +++ pom.xml | 1 + 14 files changed, 1311 insertions(+) create mode 100644 microservices-bulkhead/README.md create mode 100644 microservices-bulkhead/etc/microservices-bulkhead.urm.puml create mode 100644 microservices-bulkhead/pom.xml create mode 100644 microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/App.java create mode 100644 microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/Bulkhead.java create mode 100644 microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/BulkheadFullException.java create mode 100644 microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/InventoryService.java create mode 100644 microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/PaymentService.java create mode 100644 microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/RemoteService.java create mode 100644 microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/AppTest.java create mode 100644 microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/BulkheadTest.java create mode 100644 microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/InventoryServiceTest.java create mode 100644 microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/PaymentServiceTest.java diff --git a/microservices-bulkhead/README.md b/microservices-bulkhead/README.md new file mode 100644 index 000000000000..46e5cd990592 --- /dev/null +++ b/microservices-bulkhead/README.md @@ -0,0 +1,320 @@ +--- +title: "Bulkhead Pattern in Java: Isolating Failures in Microservices" +shortTitle: Bulkhead +description: "Learn the Bulkhead pattern in Java. Isolate each downstream dependency in its own thread pool so that a slow or failing service cannot exhaust the resources of the whole application. Includes a working example, class diagram, and trade-offs." +category: Resilience +language: en +tag: + - Concurrency + - Fault tolerance + - Isolation + - Microservices + - Resource management +--- + +## Also known as + +* Compartmentalization +* Resource isolation + +## Intent of Bulkhead Design Pattern + +Partition the resources of a service, typically its threads and connections, into isolated compartments so that a failure or an overload in one downstream dependency cannot consume the resources needed by the others. The compartment that is full rejects new calls immediately instead of letting them pile up. + +## Detailed Explanation of Bulkhead Pattern with Real-World Examples + +Real-world example + +> The hull of a ship is divided into watertight compartments called bulkheads. If the hull is breached, only the flooded compartment fills with water and the ship stays afloat. In an order service, the calls to a payment provider and the calls to an inventory system are placed in separate compartments. When the payment provider becomes slow and its compartment fills up, the extra payment requests are turned away at once, while inventory lookups keep flowing through their own compartment as if nothing happened. + +In plain words + +> Give every downstream dependency its own bounded pool of threads, so one misbehaving dependency can only exhaust its own pool. + +Microservices.io says + +> Bulkhead is a pattern that isolates the resources used by a service so that a failure in one part of the system does not cascade to other parts. + +Sequence diagram + +```mermaid +sequenceDiagram + participant Caller as Order service + participant PB as Bulkhead payment (2 threads, queue 2) + participant IB as Bulkhead inventory (2 threads, queue 2) + participant Pay as Payment provider (slow) + participant Inv as Inventory system (healthy) + + Caller->>PB: submit payment call 1..4 + PB->>Pay: run 2 calls, queue 2 calls + Caller->>PB: submit payment call 5 + PB-->>Caller: BulkheadFullException (fail fast) + Caller->>IB: submit inventory call + IB->>Inv: run call on a free thread + Inv-->>IB: Inventory reserved + IB-->>Caller: response without waiting for payment +``` + +## Programmatic Example of Bulkhead Pattern in Java + +Our order service depends on two remote systems. Both implement the same `RemoteService` contract. + +```java +@FunctionalInterface +public interface RemoteService { + String call(String request); +} +``` + +The payment provider has become slow: every call takes the configured latency. The inventory system is healthy and answers immediately. + +```java +@Slf4j +public class PaymentService implements RemoteService { + + private final Duration latency; + + public PaymentService(Duration latency) { + this.latency = latency; + } + + @Override + public String call(String request) { + LOGGER.info("Payment provider received '{}', it will take {} ms", request, latency.toMillis()); + try { + Thread.sleep(latency); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Payment for '" + request + "' was interrupted", e); + } + return "Payment approved for " + request; + } +} + +@Slf4j +public class InventoryService implements RemoteService { + + @Override + public String call(String request) { + LOGGER.info("Inventory system received '{}'", request); + return "Inventory reserved for " + request; + } +} +``` + +The `Bulkhead` is the compartment. It owns a `ThreadPoolExecutor` with a fixed number of worker threads and a bounded queue. The `AbortPolicy` makes the executor throw when both are full, and the bulkhead translates that into a `BulkheadFullException` so the caller fails fast. It also keeps a counter of rejected calls for monitoring. + +```java +@Slf4j +public class Bulkhead implements AutoCloseable { + + @Getter private final String name; + @Getter private final int maxConcurrentCalls; + @Getter private final int maxQueueSize; + private final ThreadPoolExecutor executor; + private final AtomicLong rejectedCalls = new AtomicLong(); + + public Bulkhead(String name, int maxConcurrentCalls, int maxQueueSize) { + // argument validation omitted + this.name = name; + this.maxConcurrentCalls = maxConcurrentCalls; + this.maxQueueSize = maxQueueSize; + BlockingQueue queue = + maxQueueSize == 0 ? new SynchronousQueue<>() : new ArrayBlockingQueue<>(maxQueueSize); + var threadCounter = new AtomicInteger(); + this.executor = + new ThreadPoolExecutor( + maxConcurrentCalls, + maxConcurrentCalls, + 0L, + TimeUnit.MILLISECONDS, + queue, + runnable -> + new Thread(runnable, "bulkhead-" + name + "-" + threadCounter.incrementAndGet()), + new ThreadPoolExecutor.AbortPolicy()); + } + + public Future submit(Callable task) { + if (executor.isShutdown()) { + throw new IllegalStateException("Bulkhead '" + name + "' is shut down"); + } + try { + return executor.submit(task); + } catch (RejectedExecutionException e) { + rejectedCalls.incrementAndGet(); + LOGGER.warn( + "Bulkhead '{}' is full ({} active, {} queued), rejecting call", + name, + executor.getActiveCount(), + executor.getQueue().size()); + throw new BulkheadFullException(name); + } + } + + public int getActiveCalls() { + return executor.getActiveCount(); + } + + public int getQueuedCalls() { + return executor.getQueue().size(); + } + + public long getRejectedCalls() { + return rejectedCalls.get(); + } + + public void shutdown() { + executor.shutdownNow(); + } + + @Override + public void close() { + shutdown(); + } +} +``` + +`BulkheadFullException` carries the name of the compartment that turned the call away. + +```java +public class BulkheadFullException extends RuntimeException { + + private final String bulkheadName; + + public BulkheadFullException(String bulkheadName) { + super("Bulkhead '" + bulkheadName + "' is full, call rejected"); + this.bulkheadName = bulkheadName; + } + + public String getBulkheadName() { + return bulkheadName; + } +} +``` + +The application runs two scenarios. In the first one every downstream call goes through one shared pool of two threads with a queue of two. Four slow payment calls fill the pool, and the next inventory call is rejected although the inventory system is healthy. In the second scenario each dependency gets its own bulkhead. The payment compartment still saturates and rejects the excess calls immediately, but the inventory compartment keeps answering because payment calls can no longer take its threads. + +```java +public static void main(String[] args) { + var payment = new PaymentService(PAYMENT_LATENCY); + var inventory = new InventoryService(); + + LOGGER.info("--- Scenario 1: one shared thread pool for every downstream call ---"); + try (var sharedPool = new Bulkhead("shared-pool", 2, 2)) { + var paymentFutures = flood(sharedPool, payment, "order", 4); + callInventory(sharedPool, inventory, "order-5"); + awaitAll(paymentFutures); + } + + LOGGER.info("--- Scenario 2: a dedicated bulkhead for each downstream dependency ---"); + try (var paymentBulkhead = new Bulkhead("payment", 2, 2); + var inventoryBulkhead = new Bulkhead("inventory", 2, 2)) { + var paymentFutures = flood(paymentBulkhead, payment, "order", 10); + for (var i = 1; i <= 3; i++) { + callInventory(inventoryBulkhead, inventory, "order-" + i); + } + awaitAll(paymentFutures); + LOGGER.info( + "Bulkhead '{}' rejected {} of 10 calls, bulkhead '{}' rejected {} of 3 calls", + paymentBulkhead.getName(), + paymentBulkhead.getRejectedCalls(), + inventoryBulkhead.getName(), + inventoryBulkhead.getRejectedCalls()); + } +} +``` + +`flood` submits a burst of calls and logs the ones that are rejected, `callInventory` submits one inventory call and reports whether it was served or rejected, and `awaitAll` waits for the accepted payment calls. + +```java +private static List> flood( + Bulkhead bulkhead, RemoteService service, String requestPrefix, int calls) { + var accepted = new ArrayList>(); + for (var i = 1; i <= calls; i++) { + var request = requestPrefix + "-" + i; + try { + accepted.add(bulkhead.submit(() -> service.call(request))); + } catch (BulkheadFullException e) { + LOGGER.info("Request '{}' rejected immediately: {}", request, e.getMessage()); + } + } + return accepted; +} +``` + +Running the application produces output similar to the following. + +``` +--- Scenario 1: one shared thread pool for every downstream call --- +Payment provider received 'order-1', it will take 300 ms +Payment provider received 'order-2', it will take 300 ms +Bulkhead 'shared-pool' is full (2 active, 2 queued), rejecting call +Inventory check for 'order-5' rejected although the inventory system is healthy: Bulkhead 'shared-pool' is full, call rejected +Payment response: Payment approved for order-1 +... +--- Scenario 2: a dedicated bulkhead for each downstream dependency --- +Payment provider received 'order-1', it will take 300 ms +Payment provider received 'order-2', it will take 300 ms +Bulkhead 'payment' is full (2 active, 2 queued), rejecting call +Request 'order-5' rejected immediately: Bulkhead 'payment' is full, call rejected +... +Inventory system received 'order-1' +Inventory response: Inventory reserved for order-1 +Inventory system received 'order-2' +Inventory response: Inventory reserved for order-2 +Inventory system received 'order-3' +Inventory response: Inventory reserved for order-3 +Payment response: Payment approved for order-1 +... +Bulkhead 'payment' rejected 6 of 10 calls, bulkhead 'inventory' rejected 0 of 3 calls +``` + +## Class diagram + +See [microservices-bulkhead.urm.puml](./etc/microservices-bulkhead.urm.puml) for the PlantUML class diagram. + +## When to Use the Bulkhead Pattern in Java + +* A service calls several downstream dependencies and a slowdown in one of them must not degrade the others. +* Requests have different importance and the critical ones need guaranteed capacity. +* Threads, connections, or memory are shared and an overloaded consumer could starve the rest of the application. +* You prefer rejecting excess load quickly over queueing it indefinitely and timing out later. + +## Real-World Applications of Bulkhead Pattern in Java + +* [Resilience4j Bulkhead](https://resilience4j.readme.io/docs/bulkhead) offers a semaphore based and a thread pool based bulkhead. +* [Netflix Hystrix](https://github.com/Netflix/Hystrix/wiki/How-it-Works#isolation) isolated every command in its own thread pool. +* Separate connection pools per database or per tenant in JDBC and HTTP client configurations. +* Kubernetes resource limits and separate node pools that keep noisy workloads apart. + +## Benefits and Trade-offs of Bulkhead Pattern + +Benefits: + +* Contains failures: an overloaded dependency can only exhaust its own compartment. +* Fails fast: callers learn immediately that a compartment is full and can degrade gracefully. +* Predictable capacity: every dependency has a known, bounded share of the resources. +* Easy to observe: active, queued, and rejected calls per compartment are natural metrics. + +Trade-offs: + +* Resources sit idle in one compartment while another is saturated, so overall utilisation can drop. +* Every compartment needs sizing and tuning, which adds configuration and operational overhead. +* Thread pool bulkheads add a thread hop and a small latency cost for every call. +* Rejected calls still need a strategy, such as a fallback or a retry, to give the user a sensible result. + +## Related Java Design Patterns + +* [Circuit Breaker](../circuit-breaker): stops calling a dependency that keeps failing, while a bulkhead limits how much of the caller a dependency can occupy. +* [Fallback](../fallback): supplies a degraded response when a bulkhead rejects a call. +* [Retry](../retry): retries a call that was rejected once the compartment has free capacity again. +* [Throttling](../throttling) and [Rate Limiting](../rate-limiting-pattern): limit how many calls a client may make over time, whereas a bulkhead limits how many calls may run at once. +* [Health Check](../health-check): reports the state of dependencies that bulkheads protect. + +## References and Credits + +* [Release It!: Design and Deploy Production-Ready Software](https://amzn.to/3Uul4kF) +* [Microservices Patterns: With examples in Java](https://amzn.to/3UyWD5O) +* [Bulkhead pattern (microservices.io)](https://microservices.io/patterns/reliability/bulkhead.html) +* [Bulkhead pattern (Azure Architecture Center)](https://learn.microsoft.com/en-us/azure/architecture/patterns/bulkhead) +* [Resilience4j Bulkhead](https://resilience4j.readme.io/docs/bulkhead) diff --git a/microservices-bulkhead/etc/microservices-bulkhead.urm.puml b/microservices-bulkhead/etc/microservices-bulkhead.urm.puml new file mode 100644 index 000000000000..038e37526989 --- /dev/null +++ b/microservices-bulkhead/etc/microservices-bulkhead.urm.puml @@ -0,0 +1,53 @@ +@startuml +package com.iluwatar.bulkhead { + interface RemoteService { + + call(request : String) : String {abstract} + } + class Bulkhead { + - name : String + - maxConcurrentCalls : int + - maxQueueSize : int + - executor : ThreadPoolExecutor + - rejectedCalls : AtomicLong + + Bulkhead(name : String, maxConcurrentCalls : int, maxQueueSize : int) + + submit(task : Callable) : Future + + getName() : String + + getMaxConcurrentCalls() : int + + getMaxQueueSize() : int + + getActiveCalls() : int + + getQueuedCalls() : int + + getRejectedCalls() : long + + shutdown() : void + + close() : void + } + class BulkheadFullException { + - bulkheadName : String + + BulkheadFullException(bulkheadName : String) + + getBulkheadName() : String + } + class PaymentService { + - latency : Duration + + PaymentService(latency : Duration) + + call(request : String) : String + } + class InventoryService { + + InventoryService() + + call(request : String) : String + } + class App { + - PAYMENT_LATENCY : Duration {static} + - WAIT_TIMEOUT : Duration {static} + + App() + + main(args : String[]) : void {static} + } +} +Bulkhead ..|> AutoCloseable +BulkheadFullException --|> RuntimeException +PaymentService ..|> RemoteService +InventoryService ..|> RemoteService +Bulkhead ..> BulkheadFullException +App ..> Bulkhead +App ..> PaymentService +App ..> InventoryService +App ..> BulkheadFullException +@enduml diff --git a/microservices-bulkhead/pom.xml b/microservices-bulkhead/pom.xml new file mode 100644 index 000000000000..a261642c597d --- /dev/null +++ b/microservices-bulkhead/pom.xml @@ -0,0 +1,70 @@ + + + + 4.0.0 + + com.iluwatar + java-design-patterns + 1.26.0-SNAPSHOT + + microservices-bulkhead + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + + + org.junit.jupiter + junit-jupiter-engine + test + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + + + + com.iluwatar.bulkhead.App + + + + + + + + + diff --git a/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/App.java b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/App.java new file mode 100644 index 000000000000..eddb17b27c32 --- /dev/null +++ b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/App.java @@ -0,0 +1,132 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.iluwatar.bulkhead; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import lombok.extern.slf4j.Slf4j; + +/** + * The Bulkhead pattern partitions the resources of a service so that a failure or an overload in + * one dependency cannot sink the whole service, just like the watertight compartments in a ship's + * hull keep a single leak from flooding the entire vessel. + * + *

In this example an order service calls two downstream dependencies: a payment provider that + * has become very slow and an inventory system that is perfectly healthy. The demo first sends both + * kinds of calls through one shared thread pool. The slow payment calls fill every thread and the + * queue, so a request for the healthy inventory system is rejected although nothing is wrong with + * it. The demo then gives each dependency its own {@link Bulkhead}. The payment compartment still + * saturates and rejects the excess calls fast, but the inventory compartment keeps answering + * immediately because the payment calls can no longer consume its threads. + */ +@Slf4j +public class App { + + private static final Duration PAYMENT_LATENCY = Duration.ofMillis(300); + private static final Duration WAIT_TIMEOUT = Duration.ofSeconds(5); + + /** + * Program entry point. + * + * @param args command line arguments, not used + */ + public static void main(String[] args) { + var payment = new PaymentService(PAYMENT_LATENCY); + var inventory = new InventoryService(); + + LOGGER.info("--- Scenario 1: one shared thread pool for every downstream call ---"); + try (var sharedPool = new Bulkhead("shared-pool", 2, 2)) { + var paymentFutures = flood(sharedPool, payment, "order", 4); + callInventory(sharedPool, inventory, "order-5"); + awaitAll(paymentFutures); + } + + LOGGER.info("--- Scenario 2: a dedicated bulkhead for each downstream dependency ---"); + try (var paymentBulkhead = new Bulkhead("payment", 2, 2); + var inventoryBulkhead = new Bulkhead("inventory", 2, 2)) { + var paymentFutures = flood(paymentBulkhead, payment, "order", 10); + for (var i = 1; i <= 3; i++) { + callInventory(inventoryBulkhead, inventory, "order-" + i); + } + awaitAll(paymentFutures); + LOGGER.info( + "Bulkhead '{}' rejected {} of 10 calls, bulkhead '{}' rejected {} of 3 calls", + paymentBulkhead.getName(), + paymentBulkhead.getRejectedCalls(), + inventoryBulkhead.getName(), + inventoryBulkhead.getRejectedCalls()); + } + } + + static List> flood( + Bulkhead bulkhead, RemoteService service, String requestPrefix, int calls) { + var accepted = new ArrayList>(); + for (var i = 1; i <= calls; i++) { + var request = requestPrefix + "-" + i; + try { + accepted.add(bulkhead.submit(() -> service.call(request))); + } catch (BulkheadFullException e) { + LOGGER.info("Request '{}' rejected immediately: {}", request, e.getMessage()); + } + } + return accepted; + } + + static void callInventory(Bulkhead bulkhead, RemoteService inventory, String request) { + try { + var response = bulkhead.submit(() -> inventory.call(request)); + LOGGER.info( + "Inventory response: {}", response.get(WAIT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)); + } catch (BulkheadFullException e) { + LOGGER.error( + "Inventory check for '{}' rejected although the inventory system is healthy: {}", + request, + e.getMessage()); + } catch (ExecutionException | TimeoutException e) { + LOGGER.error("Inventory check for '{}' failed", request, e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + static void awaitAll(List> futures) { + for (var future : futures) { + try { + LOGGER.info( + "Payment response: {}", future.get(WAIT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)); + } catch (ExecutionException | TimeoutException e) { + LOGGER.error("Payment call failed", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } +} diff --git a/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/Bulkhead.java b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/Bulkhead.java new file mode 100644 index 000000000000..430b935a9a8c --- /dev/null +++ b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/Bulkhead.java @@ -0,0 +1,146 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.iluwatar.bulkhead; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +/** + * Isolates the calls made to one downstream dependency inside a dedicated, bounded thread pool. + * + *

The pool has a fixed number of worker threads and a bounded waiting queue. When both are full + * the bulkhead does not block the caller and it cannot borrow threads from anywhere else: the call + * fails fast with a {@link BulkheadFullException}. Each dependency gets its own instance, so a slow + * or hanging dependency can only exhaust its own compartment while the rest of the system keeps + * serving requests. + */ +@Slf4j +public class Bulkhead implements AutoCloseable { + + @Getter private final String name; + @Getter private final int maxConcurrentCalls; + @Getter private final int maxQueueSize; + private final ThreadPoolExecutor executor; + private final AtomicLong rejectedCalls = new AtomicLong(); + + /** + * Creates a bulkhead with its own thread pool. + * + * @param name name of the compartment, used in logs and worker thread names + * @param maxConcurrentCalls number of calls that may run at the same time + * @param maxQueueSize number of calls that may wait for a free thread; zero disables queueing + */ + public Bulkhead(String name, int maxConcurrentCalls, int maxQueueSize) { + if (maxConcurrentCalls < 1) { + throw new IllegalArgumentException("maxConcurrentCalls must be at least 1"); + } + if (maxQueueSize < 0) { + throw new IllegalArgumentException("maxQueueSize must not be negative"); + } + this.name = name; + this.maxConcurrentCalls = maxConcurrentCalls; + this.maxQueueSize = maxQueueSize; + BlockingQueue queue = + maxQueueSize == 0 ? new SynchronousQueue<>() : new ArrayBlockingQueue<>(maxQueueSize); + var threadCounter = new AtomicInteger(); + this.executor = + new ThreadPoolExecutor( + maxConcurrentCalls, + maxConcurrentCalls, + 0L, + TimeUnit.MILLISECONDS, + queue, + runnable -> + new Thread(runnable, "bulkhead-" + name + "-" + threadCounter.incrementAndGet()), + new ThreadPoolExecutor.AbortPolicy()); + } + + /** + * Submits a call to this compartment. + * + * @param task the call to execute + * @param type of the result + * @return a future that completes with the result of the call + * @throws BulkheadFullException if every thread is busy and the queue is full + * @throws IllegalStateException if the bulkhead has been shut down + */ + public Future submit(Callable task) { + if (executor.isShutdown()) { + throw new IllegalStateException("Bulkhead '" + name + "' is shut down"); + } + try { + var future = executor.submit(task); + LOGGER.debug( + "Bulkhead '{}' accepted call ({} active, {} queued)", + name, + executor.getActiveCount(), + executor.getQueue().size()); + return future; + } catch (RejectedExecutionException e) { + rejectedCalls.incrementAndGet(); + LOGGER.warn( + "Bulkhead '{}' is full ({} active, {} queued), rejecting call", + name, + executor.getActiveCount(), + executor.getQueue().size()); + throw new BulkheadFullException(name); + } + } + + /** Number of calls currently running. */ + public int getActiveCalls() { + return executor.getActiveCount(); + } + + /** Number of calls waiting for a free thread. */ + public int getQueuedCalls() { + return executor.getQueue().size(); + } + + /** Number of calls rejected since the bulkhead was created. */ + public long getRejectedCalls() { + return rejectedCalls.get(); + } + + /** Stops the compartment, interrupting calls that are still running. */ + public void shutdown() { + executor.shutdownNow(); + } + + @Override + public void close() { + shutdown(); + } +} diff --git a/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/BulkheadFullException.java b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/BulkheadFullException.java new file mode 100644 index 000000000000..de4c0bb516bb --- /dev/null +++ b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/BulkheadFullException.java @@ -0,0 +1,46 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.iluwatar.bulkhead; + +/** + * Thrown by a {@link Bulkhead} when a call cannot be accepted because every worker thread is busy + * and the waiting queue is full. The caller receives this exception immediately instead of + * blocking, which is the fail-fast behaviour the pattern relies on: the caller can degrade + * gracefully while the overloaded dependency keeps consuming only the capacity reserved for it. + */ +public class BulkheadFullException extends RuntimeException { + + private final String bulkheadName; + + public BulkheadFullException(String bulkheadName) { + super("Bulkhead '" + bulkheadName + "' is full, call rejected"); + this.bulkheadName = bulkheadName; + } + + /** Name of the bulkhead that rejected the call. */ + public String getBulkheadName() { + return bulkheadName; + } +} diff --git a/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/InventoryService.java b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/InventoryService.java new file mode 100644 index 000000000000..a1648738a613 --- /dev/null +++ b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/InventoryService.java @@ -0,0 +1,41 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.iluwatar.bulkhead; + +import lombok.extern.slf4j.Slf4j; + +/** + * Simulates a healthy inventory system that answers immediately. It represents the dependency that + * should keep working even while another dependency is overloaded. + */ +@Slf4j +public class InventoryService implements RemoteService { + + @Override + public String call(String request) { + LOGGER.info("Inventory system received '{}'", request); + return "Inventory reserved for " + request; + } +} diff --git a/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/PaymentService.java b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/PaymentService.java new file mode 100644 index 000000000000..e4c1599a4a3d --- /dev/null +++ b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/PaymentService.java @@ -0,0 +1,60 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.iluwatar.bulkhead; + +import java.time.Duration; +import lombok.extern.slf4j.Slf4j; + +/** + * Simulates a payment provider that has become slow. Every call takes the configured latency, so a + * burst of payment requests keeps the threads that serve it busy for a long time. Without a + * bulkhead these calls would also occupy the threads needed by healthy dependencies. + */ +@Slf4j +public class PaymentService implements RemoteService { + + private final Duration latency; + + /** + * Creates a payment service. + * + * @param latency time every call takes to complete + */ + public PaymentService(Duration latency) { + this.latency = latency; + } + + @Override + public String call(String request) { + LOGGER.info("Payment provider received '{}', it will take {} ms", request, latency.toMillis()); + try { + Thread.sleep(latency); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Payment for '" + request + "' was interrupted", e); + } + return "Payment approved for " + request; + } +} diff --git a/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/RemoteService.java b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/RemoteService.java new file mode 100644 index 000000000000..be4f21941415 --- /dev/null +++ b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/RemoteService.java @@ -0,0 +1,42 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.iluwatar.bulkhead; + +/** + * A remote dependency that a service calls over the network, such as a payment provider or an + * inventory system. Every call is routed through a {@link Bulkhead} so that the caller never lets + * one dependency monopolise its threads. + */ +@FunctionalInterface +public interface RemoteService { + + /** + * Performs the remote call. + * + * @param request identifier of the request, used in the response and in the logs + * @return the response of the dependency + */ + String call(String request); +} diff --git a/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/AppTest.java b/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/AppTest.java new file mode 100644 index 000000000000..35c85bf9c77d --- /dev/null +++ b/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/AppTest.java @@ -0,0 +1,151 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.iluwatar.bulkhead; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +class AppTest { + + private static final long TIMEOUT_SECONDS = 5; + + @AfterEach + void clearInterruptFlag() { + Thread.interrupted(); + } + + @Test + void shouldLaunchApp() { + assertDoesNotThrow(() -> App.main(new String[] {})); + } + + @Test + void shouldBeInstantiable() { + assertNotNull(new App(), "App should be instantiable"); + } + + @Test + void callInventoryShouldReportFailureOfTheRemoteCall() { + try (var bulkhead = new Bulkhead("inventory", 1, 1)) { + RemoteService failing = + request -> { + throw new IllegalStateException("inventory system down"); + }; + + assertDoesNotThrow(() -> App.callInventory(bulkhead, failing, "order-1")); + } + } + + @Test + void callInventoryShouldReportRejectionWhenBulkheadIsFull() throws Exception { + var started = new CountDownLatch(1); + var gate = new CountDownLatch(1); + try (var bulkhead = new Bulkhead("inventory", 1, 0)) { + var blocking = bulkhead.submit(blockOn(started, gate)); + assertTrue(started.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + + assertDoesNotThrow(() -> App.callInventory(bulkhead, new InventoryService(), "order-1")); + + gate.countDown(); + blocking.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + } + + @Test + void callInventoryShouldKeepInterruptFlagWhenWaitingIsInterrupted() throws Exception { + var started = new CountDownLatch(1); + var gate = new CountDownLatch(1); + try (var bulkhead = new Bulkhead("inventory", 1, 1)) { + RemoteService slow = + request -> { + started.countDown(); + awaitQuietly(gate); + return "late"; + }; + Thread.currentThread().interrupt(); + + assertDoesNotThrow(() -> App.callInventory(bulkhead, slow, "order-1")); + + assertTrue(Thread.interrupted()); + gate.countDown(); + assertTrue(started.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + } + + @Test + void awaitAllShouldReportFailedCalls() { + try (var bulkhead = new Bulkhead("payment", 1, 1)) { + Future failed = + bulkhead.submit( + () -> { + throw new IllegalStateException("payment provider down"); + }); + + assertDoesNotThrow(() -> App.awaitAll(List.of(failed))); + } + } + + @Test + void awaitAllShouldStopAndKeepInterruptFlagWhenWaitingIsInterrupted() throws Exception { + var started = new CountDownLatch(1); + var gate = new CountDownLatch(1); + try (var bulkhead = new Bulkhead("payment", 1, 1)) { + var blocking = bulkhead.submit(blockOn(started, gate)); + assertTrue(started.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + Thread.currentThread().interrupt(); + + assertDoesNotThrow(() -> App.awaitAll(List.of(blocking))); + + assertTrue(Thread.interrupted()); + gate.countDown(); + blocking.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + } + + private static Callable blockOn(CountDownLatch started, CountDownLatch gate) { + return () -> { + started.countDown(); + awaitQuietly(gate); + return "done"; + }; + } + + private static void awaitQuietly(CountDownLatch gate) { + try { + gate.await(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } +} diff --git a/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/BulkheadTest.java b/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/BulkheadTest.java new file mode 100644 index 000000000000..1ba207328a74 --- /dev/null +++ b/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/BulkheadTest.java @@ -0,0 +1,156 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.iluwatar.bulkhead; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; + +class BulkheadTest { + + private static final long TIMEOUT_SECONDS = 5; + + @Test + void shouldExecuteCallWhenCapacityIsAvailable() throws Exception { + try (var bulkhead = new Bulkhead("test", 1, 1)) { + var future = bulkhead.submit(() -> "ok"); + + assertEquals("ok", future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + assertEquals(0, bulkhead.getRejectedCalls()); + } + } + + @Test + void shouldRejectCallWhenThreadsAndQueueAreFull() throws Exception { + var started = new CountDownLatch(1); + var gate = new CountDownLatch(1); + try (var bulkhead = new Bulkhead("payment", 1, 1)) { + var running = bulkhead.submit(blockOn(started, gate)); + assertTrue(started.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + var queued = bulkhead.submit(() -> "queued"); + + var exception = + assertThrows(BulkheadFullException.class, () -> bulkhead.submit(() -> "rejected")); + + assertEquals("payment", exception.getBulkheadName()); + assertEquals("Bulkhead 'payment' is full, call rejected", exception.getMessage()); + assertEquals(1, bulkhead.getActiveCalls()); + assertEquals(1, bulkhead.getQueuedCalls()); + assertEquals(1, bulkhead.getRejectedCalls()); + + gate.countDown(); + assertEquals("running", running.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + assertEquals("queued", queued.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + } + + @Test + void shouldAcceptCallsAgainAfterCapacityIsReleased() throws Exception { + var started = new CountDownLatch(1); + var gate = new CountDownLatch(1); + try (var bulkhead = new Bulkhead("payment", 1, 1)) { + var running = bulkhead.submit(blockOn(started, gate)); + assertTrue(started.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + var queued = bulkhead.submit(() -> "queued"); + assertThrows(BulkheadFullException.class, () -> bulkhead.submit(() -> "rejected")); + + gate.countDown(); + assertEquals("running", running.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + assertEquals("queued", queued.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + + // The queue is empty again, so the next call is accepted even if the worker thread has not + // yet returned to polling the queue. A bulkhead without a queue would race here, because the + // hand-off to the single thread only succeeds once that thread is idle. + var afterRelease = bulkhead.submit(() -> "accepted"); + assertEquals("accepted", afterRelease.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + assertEquals(1, bulkhead.getRejectedCalls()); + } + } + + @Test + void shouldPropagateFailureOfTheCallThroughTheFuture() { + try (var bulkhead = new Bulkhead("test", 1, 1)) { + var future = + bulkhead.submit( + () -> { + throw new IllegalStateException("downstream failure"); + }); + + var exception = + assertThrows( + ExecutionException.class, () -> future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + + assertInstanceOf(IllegalStateException.class, exception.getCause()); + assertEquals(0, bulkhead.getRejectedCalls()); + } + } + + @Test + void shouldRunCallsOnThreadsNamedAfterTheBulkhead() throws Exception { + try (var bulkhead = new Bulkhead("payment", 1, 1)) { + var threadName = bulkhead.submit(() -> Thread.currentThread().getName()); + + assertTrue(threadName.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).startsWith("bulkhead-payment-")); + } + } + + @Test + void shouldRejectSubmissionAfterShutdown() { + var bulkhead = new Bulkhead("test", 1, 1); + bulkhead.shutdown(); + + assertThrows(IllegalStateException.class, () -> bulkhead.submit(() -> "late")); + } + + @Test + void shouldRejectInvalidConfiguration() { + assertThrows(IllegalArgumentException.class, () -> new Bulkhead("test", 0, 1)); + assertThrows(IllegalArgumentException.class, () -> new Bulkhead("test", 1, -1)); + } + + @Test + void shouldExposeConfiguration() { + try (var bulkhead = new Bulkhead("inventory", 3, 4)) { + assertEquals("inventory", bulkhead.getName()); + assertEquals(3, bulkhead.getMaxConcurrentCalls()); + assertEquals(4, bulkhead.getMaxQueueSize()); + } + } + + private static Callable blockOn(CountDownLatch started, CountDownLatch gate) { + return () -> { + started.countDown(); + gate.await(TIMEOUT_SECONDS, TimeUnit.SECONDS); + return "running"; + }; + } +} diff --git a/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/InventoryServiceTest.java b/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/InventoryServiceTest.java new file mode 100644 index 000000000000..96d4e2a38732 --- /dev/null +++ b/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/InventoryServiceTest.java @@ -0,0 +1,39 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.iluwatar.bulkhead; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class InventoryServiceTest { + + @Test + void shouldReserveInventoryImmediately() { + var service = new InventoryService(); + + assertEquals("Inventory reserved for order-1", service.call("order-1")); + } +} diff --git a/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/PaymentServiceTest.java b/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/PaymentServiceTest.java new file mode 100644 index 000000000000..7b1c685f3093 --- /dev/null +++ b/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/PaymentServiceTest.java @@ -0,0 +1,54 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.iluwatar.bulkhead; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import org.junit.jupiter.api.Test; + +class PaymentServiceTest { + + @Test + void shouldApprovePaymentAfterLatency() { + var service = new PaymentService(Duration.ofMillis(10)); + + assertEquals("Payment approved for order-1", service.call("order-1")); + } + + @Test + void shouldFailAndKeepInterruptFlagWhenInterrupted() { + var service = new PaymentService(Duration.ofSeconds(10)); + Thread.currentThread().interrupt(); + try { + assertThrows(IllegalStateException.class, () -> service.call("order-1")); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + assertTrue(Thread.interrupted()); + } + } +} diff --git a/pom.xml b/pom.xml index a71630d289d3..072bd46ac88f 100644 --- a/pom.xml +++ b/pom.xml @@ -260,6 +260,7 @@ rate-limiting-pattern fallback onion-architecture + microservices-bulkhead From 0165376b39226dec90a007a1b30ae08239f22e17 Mon Sep 17 00:00:00 2001 From: Doksanbir Date: Mon, 7 Sep 2026 11:04:21 +0300 Subject: [PATCH 2/2] fix: cancel queued calls on bulkhead shutdown Shutdown now cancels the queued futures so callers do not hang, a shutdown racing a submit is no longer counted as a rejection, and interrupted inventory checks are logged. The class diagram is a rendered PNG. --- microservices-bulkhead/README.md | 17 ++++++++++------ .../etc/microservices-bulkhead.urm.png | Bin 0 -> 46872 bytes .../main/java/com/iluwatar/bulkhead/App.java | 2 ++ .../java/com/iluwatar/bulkhead/Bulkhead.java | 15 ++++++++++++-- .../com/iluwatar/bulkhead/BulkheadTest.java | 19 ++++++++++++++++++ .../iluwatar/bulkhead/PaymentServiceTest.java | 16 +++++++++------ 6 files changed, 55 insertions(+), 14 deletions(-) create mode 100644 microservices-bulkhead/etc/microservices-bulkhead.urm.png diff --git a/microservices-bulkhead/README.md b/microservices-bulkhead/README.md index 46e5cd990592..1f26db89103e 100644 --- a/microservices-bulkhead/README.md +++ b/microservices-bulkhead/README.md @@ -55,6 +55,8 @@ sequenceDiagram IB-->>Caller: response without waiting for payment ``` +![Bulkhead class diagram](./etc/microservices-bulkhead.urm.png) + ## Programmatic Example of Bulkhead Pattern in Java Our order service depends on two remote systems. Both implement the same `RemoteService` contract. @@ -102,7 +104,7 @@ public class InventoryService implements RemoteService { } ``` -The `Bulkhead` is the compartment. It owns a `ThreadPoolExecutor` with a fixed number of worker threads and a bounded queue. The `AbortPolicy` makes the executor throw when both are full, and the bulkhead translates that into a `BulkheadFullException` so the caller fails fast. It also keeps a counter of rejected calls for monitoring. +The `Bulkhead` is the compartment. It owns a `ThreadPoolExecutor` with a fixed number of worker threads and a bounded queue. The `AbortPolicy` makes the executor throw when both are full, and the bulkhead translates that into a `BulkheadFullException` so the caller fails fast. It also keeps a counter of rejected calls for monitoring, and on shutdown it cancels the futures of the calls that were still queued so their callers are released instead of waiting forever. ```java @Slf4j @@ -141,6 +143,9 @@ public class Bulkhead implements AutoCloseable { try { return executor.submit(task); } catch (RejectedExecutionException e) { + if (executor.isShutdown()) { + throw new IllegalStateException("Bulkhead '" + name + "' is shut down"); + } rejectedCalls.incrementAndGet(); LOGGER.warn( "Bulkhead '{}' is full ({} active, {} queued), rejecting call", @@ -164,7 +169,11 @@ public class Bulkhead implements AutoCloseable { } public void shutdown() { - executor.shutdownNow(); + for (var pending : executor.shutdownNow()) { + if (pending instanceof Future future) { + future.cancel(false); + } + } } @Override @@ -269,10 +278,6 @@ Payment response: Payment approved for order-1 Bulkhead 'payment' rejected 6 of 10 calls, bulkhead 'inventory' rejected 0 of 3 calls ``` -## Class diagram - -See [microservices-bulkhead.urm.puml](./etc/microservices-bulkhead.urm.puml) for the PlantUML class diagram. - ## When to Use the Bulkhead Pattern in Java * A service calls several downstream dependencies and a slowdown in one of them must not degrade the others. diff --git a/microservices-bulkhead/etc/microservices-bulkhead.urm.png b/microservices-bulkhead/etc/microservices-bulkhead.urm.png new file mode 100644 index 0000000000000000000000000000000000000000..ddaba0a76528cbf544d5e30f579df90698bde1e1 GIT binary patch literal 46872 zcmbq)V{~L~6JTtc6HYX7CZ2F&+qRvKZBA_4ww+9D+s4G+p7;H}vwP0&k3GA$e{}cl z=hjoMdaA0!Q4r&Xs}M$Nkdu>>{nDU})%o`A8v`9H z9UDCZ3nK$3BUP*}eS$F~Jsa?1W#RqK&C33RgE7(cd$J`5J0Hh)QEqlgK0ZEfPW~Sp zQhfZtmmohszp#MfPiY}hSusI5AyH*2AnPHW)Nwup{2UkXi7{mfiIpjFHA%_M88Q6@i9@9+z^f`fvo0s4 zB_p#lH)XgYZKNu5v^GCwsvu=LH#ax0vLL6cAh)luV5qccq%41;xoEPjENi8_yu7TT zqPo7OY@)h+s-|qJt8%8Ue7d`#_@J(?uA#NDX0EZjySt~SXM~z@Vq#)`etvm*`S9@Y z3V2;#-Q3)Ketzo0nPmaxP2(i2=45DV=VoPW;shdUY-8-8?__L5Y~V(0=Hz7O$i={5 zXQglBeYn0Rba4S5kBO?|l$ZARO0>Or;r{RYnZ2Db**YP&8|=lp=FM zY23sZIK!`VO45qP8Qj-)R)<=13Pasf(P;58#L^!8q>mXMm%k@!Z5;~mq$&LAjPcvR z`joKj^|MML^9oVZS*H^Szyw=_runGB|A3?ls`+l(zwR>%z68pX2qJbP?u+rpKw8kN z@_haxIA#`=#{FU)@X91`o3XxVio#4BXR_Tssv)8)BEg)`Dn_s(&}yvqlkzD76de8o z4iQAh7guQ&E55Nw@lYbnUY(W`&l~io*tZoN<)M{MnvloN{+gZh)4bAhB)(3CZ|q}{ zhN0iL)OuKF3Vnb5o~ApU%$d2&C9eiC1sE+`R^@{fRqjAmy5EQISvSn2kZkn^E#@(F zg7D6>tMu$PRC-!7Do@pwTw%zTyTT;?>|BoS$wYnFV_YUxojuBh+o?#5@OCj>UYX&3 z?ylUKILP`jal%`lZXN8&4f&k`I zlxNfC-D6^J_{25o$Hwq)S2p*GX9{Q!QDW2KxM#X7a@pTTTpw?&tzu`~_O>+nTER-y z4p)1xzi43YyA16R-kec{eS*oGj-2-W_hiJUGvj-E(1;?1viz!{Ot-4#EY~#!EL#aP z($e)^mcg0_xlZAwp}RbJo4!sbbc3e0y>hzZYj?MLV|RYSD~8NE+ReB7gMj#ghzkiQ zxvrjfB5NrbzdZUd_4goTtUO{Ejn8GC2vh(CA{G_X92x~eppPK39>$;ekH0;Z0E&0Urt^ zwaU2g=d|Zpidd_=E8Z-~6h?u76<9vf+qnGpu-Nf)RTV-YMsDENGoT{>i;y9N0LlR9 z)ZY1bOL%irR0sh*gXS(2k<`vt)rOVcmDB!5Z&-Sc z1ytwRWR9rovK_QZYa~tD`a^o~M89C;@0Oec_5csEf}Y3ubPFl#XTIbf_6s*G%d;}1 z8Wq_&&;be+j2Hm601Ej&%AopBGK`JUj*CT}rUrmF0{>R!M0zzJr^)-uo?!{x+}Okn zFA)rIsUYh8R2GCX^J`IntFMa1b8E>OKRc8+Xyi+;+EZLZJCG+NAyktJwco?t6zk#H zT2v}bdZ7PwnWouG$Ytie<#WQQ9yCQ;!hkJic<&=R`cJqwvuh>Ys=J7;6aCf8ABa)l3FoB7={Xg(yIr3@v85yVNoMXZaa@xYS+ z;sagt0H6e9he?LNOMo>Hk8DXn#{rBiz7~ zm9n<_Ab91zg*YRg!bkxw+%v;gk9p}XHFREdb?@(I}pCj}Q4e7bi-^qQxY=I+$26goZvRV`$os~8(55Ub9|HT(WcC}t+pmcRY>9|t&ngk&* zj_OI;2EC{9cX3e^>LS&ykE{#36a)LK69Q177<-#>QWjkgYbd%ll(^Lp47g^qv<{!B9TS;fn3M-9LvrGfj*mK+4j+$0=vz#Xwh6I)-e=~Gl%lz{}0i|8J#e; zGqz;8o7cxb8>QSWhPDVH(jO}|YZ?Y52&sCt9v%&W51UES9j78158(vlT_lC3 z4!R$SCZ`BT!WMlIB2RIX(kl+A4G>{3Wbdo3$Ijx~^u-RP>!1-da8fh!cZ*bwW>H7` zhwO4eKzI3UhuNFppZs30uz_GfQ(}?QgI9@FeS;A6ODy}VKQRoaKO@@z}-1XS0B7`&M3 zU$F(U|2(oEQ#cf3H4U}LiQ`W!MlAQr1I>>6uaE!#v8Ml96ZFhn8ehls4G;|~_|641 zZq+9oU?2f^xC=}+6xRg-@*(wx?jN&*?dj!atALsIZ2RjS2Aet7ARTwX?56`L> zo2hM3-k7FyQQ~#Db%6f;KuBrxq*peQ1|jsul{dg}q@GDm(Q%R4A`GfW;id7?yJ**z zMh2P83lSJm_XoNktZwAz?!n;ywYm)zo8K%lg1qNmXn)BJ_pyIy>hziJ5iZ`H#6}#na2I$C*_gIq%J9#eIq{zrBJl%zGKa4 zTd%6$wD~&+0@5nEo8HEl3K+nEOpJVbC!ocWk}=jOF*Y{YB@U-+eSZ*Fj2n+A+p%%4 zL|2mc)SmD~7Sja>ii`7xVG=MKwU9gCC+sSz^n(HNkMJI{0_c}i9uI(4rMX?NRt?xX z=pw#d+8S5o!u}vC&{72vKVJ}==-x&k)MHe->t)MJIBX{R&f2XzVeJE^=<(AP9`$In zo*XfIo;6uvm#~pI2dxPaMVZBQA%R;yUsv;RKQz{t5#p)E=smTHX?^MZVj*nUn|v65Gxsya9;>X3aQ{mVzOd7_i0N#SbAQ`=SscJmbW`gTAqb=qZY zV@Y1iR~x>x`1CD)ObrNu^*!qX1-K7h=)JzewM%Hk+igk0L57t++P{YUFAl^!mq~&= zdEpK4KsEW3>&pSpf&ev3|96{Uv|Xz@I7Ks52v}Xu3?c+PQHbfVf1;-s2?~Tf9Mi`^ zIt}vH~JY!03=im&Ua)NVE#{F5BPzgrtxAPCJW&2}JQmy@HvRNZ*fIotG(?1)p z0Q$qukO|$y=c6df`>e5L55znfh5WAPfp#|yUV3U;cOZoC94u6mNYsmNih8o{ET45k zmX_k;t3MSXFfpO8y&NsltbhF!3Q|9Am=*54tVb?ap0**spa3q2tPho3_49^Uw1?Qf zOf_t&^N@nFPo%;0KdR2L43d`rpL9<@a}~4 z^{#SGJ%i_9(OCAzsAI*8VmnB{)!<|ET7N-Nrl2K;=?l03;vdg8TaD~F^R7HTUEi3# z?5BrV9(Zw8Y>xH*QVyr#zT%x%IU-DYd>7FBHCswhJpS-4QM7U6yEEa z5v8`fuUEG1W%wfr9V{{mr<$Mjdq8f0;h4sLNm*?(#iIKfKQLKh=_N|Gai6yL56^+H zd2%#33R=Q+XEN5}I#6tJAa3x0GULe|{Y}HmaoRMHUJb}7SL6w~4Il|c24d|_;hmX# z$dDYh>YcB;tnl@uWPY+W3E zp20kpIE6@;U}#=YP2dlK6Is6#p;En+?9GH*rnHD$GJ?g|0_ZBruMPcBw#@-6YRsw} zJlJGws@2&1J+C`KPHj#<8C~MP*Y#Bn4LJ=K{XE+ENWtPXkn?$Au7O_b<4zFCB8Y4U z8c-O7%C08#)uj*fm~r2nH{a*1I+&-%hLMm0x@QLlzZcMzeJlc82MOkJ zf7hEDg5+i+cwp^S^NzSs!?<}rf53ym2Bp_3Oj@n(Ei^Pv96-6PmNyX@6gQFc)X073A;!=ON1LkQ2`J!!*`Sf$;ek%o@k+WI>p2i7T|!MzB& zPm9~2_QRN}@K-zCqj?F57ai&V5l}lBf&<>MDtzY_Bk3D2sTR&5E@0HgZPH+fbQ_RN zvCtpRS6C9)_**Ef>IlbD@pm7reYo&6)3=r>s`LY4K~4u8=XZU{ zJH%t&y4M z4|doIJUl<9a>lHIx_BHgXvohRsLc$JW5mx-7S}YkP!eCUUshEHkxY};Ml2^pXi#1S zn3bjKAl>3J3L@{DU{# z?b^qUKNYIUrD>-=^}WIX?J0D2BJ!C;suMy}SUQAD{mQ?Y95?Jl6ws+eX5wlNjRBEur>V8sHRrO#@YFQ z^cplg*+K}AVpvTDQ!i2u9f9pTpM#csTO$h{u42RDh8tg9ER7=0l`aR{I|camo_z-y zgPMM>9~<)8Em0Wr2djr61qDJBc3y)m!(2tuX1ac6L>VybRo7ee))q|1ii}tg2JhOa zd%8E%_$D$70}VG$8VUh4K1{D5&Q%aQwuRLO2#6asJFFCEEtkb2y5`G8!P6oCN0Zj| zFla~<)B)V{Y(&+%?XBw|D5Rgs%3JZS6=ajE4M{H;Tvim6-n$Nz+(GIde(EavSdb>M zVnC8{^owsR^99et?gm)osv}}CemVQqvsq&sF>3~EPE-NvR8tz!Q;ac9+R!a* zGk&8F7-G9J4_WMX{kmV=2{{=I%v9>YGBS?ZK0&F(|Gmqqp}*+sE$F|T zd*stqShlZ!lzm-uwg`lSVpUhxV|vRgeg~2vaNetHr60kCT4HjH(y~?vQwg8ZU=CmT z4I!!NpsctA$I5F(ZyA=15P*vDmu!KI-UllV{puT@4wLUa$BOX-qzzeW_VY#3i=7|i zq`sgGax;#{38%t2?h1P~5jl)!w2Wg^BY8cvXfrtucq0uQb% zK2$>-07SqOY_EcuDV^S>E|&H5BYpl!S1?8nim!Jx-{Y}xkrzC^wf6JW`$PRFN+2-L zO7#Ej^zpV&v|40OHHPlAJlX+-J*+a@cjsvzXyQ-?y^KUcMGTMF ze4B|#*vo?|1$DlZuZ=sd!|TVQqq9G&1uTaGE6|Cr3jIZ^wW8G(PMO zzytzPJhgmI+I4x-d#asr{z&qsm-=rM-FR?`&?&EB(ahzMwC_rwoxtxCbrN|m&pr>@ z%)9bqAP=eIo_HQjzP_5}`2#x6J`{Pf4xrsb4NaWl}?DW?F*POfv+Hkm8GijN@Ii8oMfH zjB5X168dX+cr2HJE3ugu*8~(|n^BD0U(nThqu6-}jFawDmeS3LOQz6Fx?_|rsLj7^ z`-{Dr((+RnAiV|JO(yD-btx`#JA@w7rB@*#c7PZ&nFK_4YQX=LWujTtCviUAxyW_F z;&pI)Z4u`R^7%%(gE*uv%ZQKH#k*iRLI9wf2TS-c9fpKC>QbkCvF5rNDsA{A@PFhI zgzI?xu`|9S-&&iCr32}ZfklciF%r8$n2mHUy^2h30OT)T8D_aDUwZ$3G6k`J?>B3a z2=U4XV!p;pi1F7;^e1iAjHBNbSJARbk5c*WeB*S^hR9%OPK=WhbN9;q;y6Xf&E(_K zMS?RLqB^PAuWYup{$=EwjhxZHR_g_lm11ufP@CbJ`#u&J;r)d+b1HU9(4!%MrR-U? zwl8x*JB{M#)oM$T_j#O_8!O$6r{ESL!AS(-j$r*1KW76lWA3+XFm2|DpE?YltO%vv zh`7K)L`ahuBfom`o2@D1tfzV3Q&Fc#xXPNH^(ACu0riWPbM`d` zkP3F24(1qE^sQfBOGqWHFW;TprMQeTxKWm03>=&O#X?A9@CR#0;lg zv<`%{Rjsq;<(yo9=`=bv$bBIPE3b53%ZS^=^G1TU@(#+?ehxJQ@(o^#hGs$jPIr{&1jm zEqIFjLZoNe=}Ha^Fjjk6o zX{Id`_7r_OE8MvcQMvLKmsRKj0hE~uR=i8ml4i#E0J&Ulszs{=$!co9`VIgAqj@TT zS59~S(ChpytmfR z*shO0J5fHjaVnbS2oy_mNe zXxq34D1}NTf!4@kGyCw>I2qMC90jqdDce1~#r>j!9G#fSqjqP7R-&rZ%j`^vX!Li& zHNbo8^@*5Zg5ZDal#{wg)~yaqL5>fT@-BN-RAxKs!x7V3f5J!2f8HLF(tv4HPHy#3 zz+YxOFh;n-`705J}oUPo+GM{KFjrL-Sa;`zq& zP_35@jDJ-uGvoT-=&0^NfgK)j#vs5`I1{ z8cH7%jx$!Y2Q6=$_tia%r_5?lPK?ZXbC>{nG!X{5S>WCU`Xib-vcI(6yhD4MQ9S$` z4c(9r83bpTc4H_X)wkjyqBic&j;sb`66wbxk`H|Q-WafmPZ1dAKZ$PHv}+;#4yZ)1 z@edWx?PWy?H^?ynvVxV&uUq^jo5)GL!Y^-cKLQ|==7`2AE+EnJ0>v|c?a?+IpT%e4 zDDlZsE#~ej*!xt2`5mq^CQ?5ASzAUA03SlV_syJccEZ9(yw7{s9W@jj_~8tR4%)1J z)9aruyjk8r*d%A+KS-&H*p=`X^cwid@ZmCmcmNnIwP5pBk3(%4HJt~sIeQ<(y-cep z#qfxCpq~AiWU!0GkhCbM@PMHp+hp49kbbfiNGu;ruw+W%G+j-GwNj9aM++`lNUhUG4N~68W@KH~zrgt5Y`KxpEy73HMdiI0ga78ZXHbZwD_UQu=Uu-QW&j*r0k6)U`}#PXW2O#k?C z@7mk5&N1svEb(*49xLAp2W?WE>@=2)fjye!0y6JA#;Wc0oF&IN6s+`%eu^-Gyn)A} zgXd?D+s^v20I8o%M-b|lgTa-m&+dnhyy>h)WC0L3=*iwS9DRA6Za+7p-~FyWP<{?^ zvP06ZV%c5i7|{%_kft_5I0;KxQ9pZIbL0$(pRJ2p(SrB-&cpB?W)BE!(Bjchg&R%f zuNYGxJP1MO3(?e6;W_J_FkH$)V8VA(X=9(P4A}L0{$lR|`dp_91=^vE7^vI+rNd%48_|ys1_e=i7&EpgLHby*w%_YrGRC>XJE3F4 zNmuJ(&>qZ8rk)ZSkYT7phtzMgi9;0?Nnqr^qt46wFaKnm1o(V>R3OYA{Q_1++4o;e zWe?w591(bNVYn8O#XaLw+XrXLvR^RByn;=I@oBmw2Cp7gLCBnFD%ZC#Y)pOTGcBM&bfF3xGZsCm|A#a)L zrM1;r$7;={hOjTNel7DZ@Z#U6EV)>jd`m#YF+ml|*@Cyl+}UT_k(FDkPrE1(mPHj( zAK<;6dm(X5OoH%-NXcn3Wd*Jr@Y`gFT9sUd7BXS1{wCN`aR+Pi=p6Upi$jJT^8gZy z8(AMKpgMyvOW&gZk|vhIfDk6-k8Ub0%#lhbkHLX79=n_v%T7NI3JCh+0=@NX-BWqTFVH3)i+YO(l5IqaWB>4nMbq8@PtbFuL&Z{1-)^K|l}wgQ zUg|u$+b~)+1uc+6Q?U)4vX)Yq?tRwFoynZalfLeCY`1&ccm*&J5;62itY$dl#MMJRw4OlZfU)dI7)r^ZxrR@Y_-OQjBM1etO1ymv> z$9s$`K&baleuy}C6fvk!)yGOnqs-h`<^UtU(R?^eJxb<5C6Y32usUjqE6U3V)=Ne^ zWVL=ulw4kI_XyEupqA$>>h88HBZNE~JXGb-jwcCH6Vv;E-0gvJe$&>m)V#Nlk%7b^ zOZ`v`0PF@jvfg_}-U^SEhqr6O&ZeA#A*+F1ECsE({_Yl&1BpE_mTd9^gNS59d=b2H zE@T#7JSJo1uCJwZ86KVA#!3^L5r1?91s8!qO*~uKt|qCm;tNeh;enSX z7767S0t0)k6qt->>H=8LjK+#B6cXJ|XsK?)skeb$vZwbF#01*yCmpT~a_s}FWN%+OwdXA;ZH zDU8yoe?Rt=qcZ+HiGmI(s0OxSYiDT7d!h)32Bir1{%^}~-DQj?GtVs-u?_MkUUDpe zrC^5cIko|QbXOJg6^QVYV9s8Jg=N8aehU#MTOv&o>yOU*+hxF6jI0oA1NtH&WA1(& zAHvyDca$BeJE7H22PDf4b`q^xf1 zxWuD2RLh2n>m2efO;OvBsIuJ-P2;i+{_a+XEj^}5_rLuE{*~CEUrI~h1ZUkhmtGW9 z<%I1YC?|?PF~&y_r?zbiV!-z*s&Q;hHn{}8N*K1UP?gQ_c7Hc6oAVByP`sN@|K}hC zez;l@nKAGS#KAEhTOhubjfeeNc(yp03U^~N^mzh@S4hIvVteKzWuW8Ts&Tce!h1g4 zg52hLzdve}RN#%&*SffBp+J_RB=RH4tsQdPc7DxRCGz6zxrX} z(;PjW!$V}9$kNCO9`~?E-mq0bbh{4B^jGx2uZ^uueoZdKwi|l{*W2y zM1R4meV#`>y2SA&6UTq{WjZNkx{Zm>BYP2Uu)jMlaLqhauD3T06!39N+2o?I#^gUoG2>dwSCou6MN11Zs1ap9yV7U0_PWAzmd)^ylOc8 zWN*tvH}!9~Rg6w;3vWiX&Sc7sgQn>jZ$d>$-%!P0Kb1nt#DV{IHzK(+Lkx#RIEQz^ z`lm-cZ(`)`Lny*5x)$>J|+=yz>USC1Y z4kULMRw(1*oWcuBOsEE9I7r;y@GCEpA{S_A{UIyXXB2U$CXm;|_WF|p)*3;6DmxH4 zK24rgAtJ!?aW0BMn=7fiuUB$-3sA^!84UzT+g0 z{i#G@j+_vN^s0yvwPpW+arg^3;&d0d^zERjnqE&F$YQ+12LDx|SP zK1j$mP5g93y(+@3)2cd`FR)o;%*k2MDd;aYpLArWm4eZktrdbT68m%yUcoZy0}V)b z`0KO$gUeV>apOo1q zvjYr@hmWKZcDzv?motp0_Wtfot1ev;&ruTu*g}0FZ*!OF$Ru&R2p#-A72jDFy1vhr zLHQSSyqj+Wd?<_e9sZkBzAo_(oR|R!kF%5hEta$QMifXEC4OO9GEWugrAQ?1k|kCQ z?23a%w35?@;7CRijP^{_>QI^|c5`eDIHVLsQtyG43qK6DntvOi#k&nRO8pC^O1eqy z-P4I*Sr40pz|wrfd`UvjpT3yNevroWL19lFo=t2M=a&YZ&BAIj!8U*;$UA6O9M*hE z1NTe?4yJhyJrXVZWD<$-#civtsc`7);_zH570&bwjGZxY%AA&*EC-G^>~eNBn|jGs)Jifh)hNau&sthA8(IdNnhhw%)B1+PiRlyh621jW z#=X~oIi)^=NmXcYw!~D*bi;Aei*b05g$rx&$Am(Pj#I;feIj`X7;uG_Z9=KpL@NY{ z_g7e!I;%;&mtr$CTq{7;o77(Y_K?P+2(<{8wos$Q*+KM6D_cS6htZNEUzTG>AQgCA z?E~3wA_M)&iF}y9pmgl?AiH;-8*E!{LEc&I*?1Xc-R4&?1szM&&n1PGNPZf`%3+)m zCnuAEh(mpSnOa+@?`jZA)@IHS4-m*5u%YfjZGj_2Xt%e+s~gcD4?CqU@RFmj=Kxa` z*Q*Cwg2S)v7Dn@6V|`6s&L=F>M@WVrno$(xQcb74rk^p>U(58MEyyL}IYSztLw#l6 zp^~b>v0m2ov}BRR#HeBi7e#|%$Fng?2)>Y-H3`K61WF-{)PA6$c!t;O7A!ywbMFZ% zd#4qCRC}kHoBh67R7sor2xabl`zrMOgl5{l#Q>v_LFX6eIxTgrD)BG}&$+oeYQIVZ zoStX*46AoW-}71?g9Nl~W-PlbvEbJCw8?++@9BMrExu>tqOlJ~%y%FYbW~|pY;96v zkAF)p0AqNP?vy#0?|p8IBz;0lr_)i$7*?AdsJ=DWC00Q)#T*QH&jyaXYh9B}0=nD# ztJOnWjb|RZIe8T zj_THU4~?dGy*uaE+wWMbW&(T}HP9V9#nvf%a)!EgrX*yZB&4bvBowwY>$Km)v{3_R zpg>AS(ChZO{$z#O<)c)r{nP+`isO*mKGGBQ$h3{EY9@Z*aqdYvJncBX?TOf$+1+?a z&Q&Tn%&g&{CKI=r=I-tbC;jTR2&7}6y=KHkq?*-qesI# z_C9X_<8n_i2R`QVQ3|QHZ4Fy@ZPZN}3|sNBKC#vw4b$!hu%=4NdSyQeSXL$TOiFf% znxxx4WnQ*|72Qd-2Z(1Hfd#qx2>s5W`g-A~kZMpyasifCV-L3&wYGa{nABDpD(*t} z-A)rB(zRJ-U3VtnT7PxD)$JL{m`$@O-%g5tHfi_hdjW1~DbOt`sDGRo18Xs(;r0Ji zQp2%cn|POew50JDd@B#;8DIDbeG6IDOUfW^RBn)b4Ifm@ob5U{ZLTX(g_;8MVaTtf zs;_QBicwLF>z`mm(9QGU!tKbR?J+P-A#1O|x4LT>?W4s1JI^2x@6a&W0{;cOOr*HE zdv#_V?C(qiSA+1yd@rt|JXkm-@%)uPTI}XT zq+igum=(CT5IUA0L&M*2mYeJp^=GSC(?QWfz$-R2;L0F>H=RnlEG(6OsQxk3#Xbg9 zV4O2ncW|}HCCfu#TdjzP&-v&s@$b5;IQ~RIZ&pMyeW0|7w z`0o(*Z++Iu1FZ6*kQW|@@BLncGUu9F2I2CKqAF|bUFxdjg;|wYA#D^Unau~T808Ba zOHtyLCBZ%=^6V5a(9hIg`o)*G-GokCwWwX!vMRf2%8>{!qP^B54|$Zhn^aoBO37Ig z4VC7XR~&wXk6NP$uXrEKr8j`X?84cmn-dwuKHvcDq2Zc&@*^RTjvF|WS`GZ%h3+IR z88I^_`kn+rvp_25Iuq58DYmQqS@CQX!%7!bz6J_aeOrLK(J=^D7U%ldS4%~QKjcL7 zVh$saC*!<}z_))*Ke;G_T;(z}c$x>gWuyMq=D}Xc{faH(mD7YBohYOQo@m@3*=#dx zMk;NyBZl!o5$y5~WjO4nykR?9S@=6`KC0+(UA5C7rFd>~Rv>z$}ie~mH`vbBje#GA`jTejwmpuA+lomG~YU*~O)(l## z$I}uTXlcfR@#P`EH4n!C1XBkvtY`)h38G( z**ff#d|p0JwMPAa2B5-pe8_vu*f6(o%*|h--cuUjQQ^b%Wps|G>guWkdMM_NvF42l z7uk!yA2!)h7A|ewb7F|DEhNSz2^+6m=RXFNbRPOs$GFfqR_anaUD#-Vs{Aq0)LE z^Uu5y=ljZQyFo;}r4eg}T^A;J@_~}u4oMr@_cjU*68Wu{FNTqXj#>A=>GNj{iiN&W(?yKd z0cVb*u`gC+lOc~&vd*SXb$r}Wl|MX2B?Ziq6qB8`LXf}lC%Z8iP8or?k5##P<3|+n+%pCxm1ViSdoC-JJvWa$`JrDnqChRb8SK? z2hFWR8vt_hlg;S8zJjpyCtzKT6lE%ZyjOf6BT`p6bOz-Hiop0Zj(RJb@A#E`u-B(m>kjP=35ow}&+1WPwu&?q8yI_4&EVK}AX)^5_BimpzL~4?6kIws zD{org*!qxTj6kTY;^Esy-)W+Q$@cZZ3r9iUhP1A0!G(nd!(N`M^wU!PfG>O6l2CdN zUCmwC1vHY*65Zmh!)66(v-78E@PPD9gUTbY+seWu6LZxxBPi*5 z>+I^05!eDOte9M5UIl(5PQk#kx^~~=+~f+P3aP)IiUn30NeOqkwkj~#tIz3106*1* zSEDJR;Qm5&QE%iWO2RYCysSH?HTiTc7XQ^{O5X9z)xjep*iivEch|2xAlqvfI0ph&NsW;!NO(Nzx z=L}J5t0K4#;c~l07aRayv}MUcT_+t|6R!aT2)>PhW+nC69|LLF2Kun49dV3_h)RZeIPOmmQDN>^qGoP20p`ZmT&FMtQ9R zxk@br_>L@F>BD{l?LL%GWG`q9Yf||^bVv8TV>KHs1eSG=pR%8FlH`}PyLhH(ZhU3W zYTe{aXV+Phk-?rnvJrA=QQtuEs`t6OVq7mGkc@;BU{=QzD=xm4q3Q>KgM~X~s{S#P z{1T?mCANw#jMk{nlc}dAx%w-#r=H{Ws(qh}mtkDh>>w=S%Jn5!c5zPSaciK;T1+paAO=pT0e0pd=w-llJv1{DqLmZlzm@kGxVbL#hz z4CVfD#?C=7eW~8=L;G#8(_i%g%cwp1vaDM5z1m1%GF^UOr?U}52|jM{blcyP{Z~zx z_V)P=+HQGrV)>teR6T!jZ@-m^ngn-hiQUdWi_`Paqp+VKTQV@jCvKktBjl^zH$xlaLa;Q6_W zJL`9D9iViuGw+`YBzYXuyr&w)f%rmIAi^*2=UW1fUm5 zDW+UYIKA_}_9}n`OjAV>4oq>$4m|9#KXbJAKKJu>m<{-i)A@Q^@6BpeybrV zJ_cyyrdjU#>q?#+CrK@;U8{3t@&i=QT{Fw0{^F_x%!wk2J)y-{Ew57E(9*pF@I{1@w0En8h%UJ3#N0WN+P1P+#5p zkleke9Cql>;(w+`G`+9f_lCZKuL!4qzu+!IHMHc~(?>Ims>}Fi6y#-@rK^G?fx}1k z*`Y zF=ZYOFIa({>wBDEXaDYRu_U>ncn%My9yg*$&b93|h00cd1QfwbC@#Yg!<Boc zDTb!>cnyNe!-p2P!w@#oC8is(M&5ZQ%!+^B)`{OOQEZpph&Ie?KH=S4<5T17Rb^_$ z(w7f9(l5<64Iq`nB0}lhUGd1<`k2Q3xz;JaK9Em-ppX?qzc;eQs@ak7k72@{)`AO+ z9Gf^BTaPa{?}s^s91j0?nk|im0xL2KID-;;giN-WjJdE!Zk>vD@ZA~(AF7{c-f|e# zw_S0{-T}(R+n}b<<(}iJ2>!UZXfg90KA+bPbA7ghDMX0=%YeF~?(@d>z4qW(X9f+- zX>ILvPuN$xwmLQno3gUl|rxw5wex?(Xicg~8p5JH-kV8QfiqI}~?!cXtZK z-QC^Y;hXlHd!PILUz23AvUl{&Mvvm}4nc550Y-jaTew2j+L*J`gb!X!TQsGw@7_|C z;8)N1_5y)_{Vxa;ek~(+1DbX8SH2nW+?jGV-1LSywd;*0mFI07^x@{veK(AE!it_6pih1DT^LGmy9Z zlD`GB3W!(D&&Lap)a!N0-Cc)mG*ASIxo-Z}9;J|?mPsO^9|DCdxh)`Ey)>?Bv+vAJ z>n;QR+{;TpC4Nb3#Z$ilsYKhRUbE6cq2JYP`F{t|tsxAb3~R6Jl;rPJkM7COg2z0S#I3+og&EFCBXw=IFzE#g; z8H;JVd1;_9I%riz^wI0l6c13HY1QG9*l}I9*5cL^K3|;RN$E^*0^Q7(A5@aNUtD`) zVEJMGOw+HjYCbJ_9=OA!MQzmnKph7!n=H^SD=+82o`DTG~p$Qr@e9l8Qlq~aby-2rpdjq9YC@-U3Z2ammIpA4;!&V zXh-4NMc+%do_v3)o1+V(77Xh`U2+4o92ByCMv{G>mZpXdBMO9zHVjPRGn+lZ3xl68 zT+Wr$rd>vnM&s+#;~VNT=Qj?EeCz58K-Y^Aq)Q(xn;4f_oKWU@?+Ie z%WR;HJl)#+YUdmNbM5L|;5~j(QrbFk{{}&a`*I4(1uphVS zWij0Kj70I}(9Ex9ZW_GKOzZl?hJ$v`TMLYzeA1w8pRhO~GI7pSqkXJ-@a7w^xh&BI zql~vwi&$YRknjiMv707#9)1vI@?6NqMzsP7B&1!HE9v@hIyCa$H{sVrvv zvo0CD--riF_z$nd{w^JOl^jVdMT-`z1DXy^>2dxbkkq%k)eKd1>wQMzlw$qikl5E( zfZgXPpT;3UKnINMpNh3S#dEW=%f&cv^U`IDT9)XU$Adacin%(Gq1@o2S7*x0A(ZDi zgV{fd!6`XUU$*4lmW=cpevypEYuLldrzfN3J_E>rT-9Vj;;faqbCQV3%8RC z>@YqLO}src=^7KOYHi?AFZmAg1J>fm<~(gZL!=j`Om!?VHi?o-G0+t$B&`st8l&sl zd6fT-LS`Ver(rw%7@@ZNF9!(BS->oQ;w~76Mpj`}UiOEWm1T*+SO{dw<5|cH*9unk z?b1HyG%y^6l*avy-Mh;JX^l3eFKX=lsp7GOfA+l93hAhfjGQQPd2=R!T7zY%yAeXa zGF`p}UAM*i@cXcy(Zj`13t&PmxrNv;|63j{9?mm8FTmm|&^<_-M4I_;c!D z=!+8m7oAj=Bm_$ow4eP)zd2b8-nF!s?M(ME;`+SOtun-xB_Xap&5gcDBmHl;r>_%b z=*GW=xPdfC0V+}YlQW76+H{mCYf&=A`HrRJH@NR)22vZnS;(ra!?Y8U_rYyhD@yo3 z7LD|eP7R=0DM9oP_b|}={kh7L*S#R@M60^x-%v{<1V~xUcUGpD%F)U@qop~oqQ!Fei`fwagMWlE`UPO5$YQ1aE{^9P>>>vCK%L4ARi=2V~LF_m^e0Uo$w5`zZ z?#j;lEO^B5gf7j2jj~4(H~D0~6-U19fXDnqT{jzG6X6^8bM6h8tDqttxF@Wu8=LlOT@q)VfH`u#(M{fRYWzBPlZP2x&v?>FbBKN;zv-uVn` zH~R=C8qjPLuI5X`CSj6Rr_pK0S;C2z^1O+t*LI0yC2EK(FS~wjduUs~@9pNCdF~ro zM5~|drcZa}XPX>A{q<}MwL)?Do%+BEWd{LPPC=h6HUE>bsEK_ETXym$ZMhMvAep4sR9;?JdyY=3on!LW0 z1-6&1U7kPqF^un)uJibS?^_suo^JGmVpq;K@&@qM_m~EajuEO@CE+Wt-Y+3nTN8;# zr4HdWc?s@&+NVgnYFryV30b zX(O0&G2fO=@gD01DaRB9^nb;5w&J<}R=};*!&=F-6FHW6X!*@silA7*o(s8Vp?9hI zKH24o45o9J{W zF}dSGs0*-L$vku4DfvWff6^gpKy&IDs4cg8JrsM&N3qYeck=&9~f^hgv{~P);<+TUj%SB zW}2NmfY09_ZoUh@3Ip6~KO}w$31x3=Zq$EU$?k$bX-$tG0}>GViRo1@_9Ho`e-Uhn z3$%zC^6xQoPxjXtW=&%(s!QRs zlr(F#QITJ?tYI4Hh25tCe%T&~j(UeV$+egR6;prTi96*wg($o)|01=+5%xp&<$5V| zK1O%XsVX)iNtX@;^*lUq2rEH^`0l^5Ue^YK+pS0wcjZ`q!!E5^X*7fzb_pa6ADBKt%mNf%X_tMn^h7dMA`XU zR6tEvu;ZR!L7Q(99)id9V@zFhNZuMjjnelW|Lwja0O~dKE5EwG9n0|^6&{|8Qyq-G z+97#<^!D~~P$^<|$C{{U{5GG=C#YaQM}zX8*=b+VJ2SPm2i1AdQ^G*}f)X zhSxh@(`b-!4iQpU@(4mTGOnt^d~KVzgpwjait_IXv|E22vcpf-7}(k^2l#pcq)X~Aw#hW@H@Zl|4>FdL+kDhJhLooB^!c)y@yl@=YVQSkpC39&V zJFl|N*Q+N#?57@r<=I(ShwHMUmTSKn9P=5?z&+QfY?_`pisR^83(zL5o%}}SJ*7Q` zuZaV`O$1KtT*U)5aOxnF2ehDfNv}CkRxPIPzN!!Do<;XZO+dwsOf8J1g~#^PZ_1UN zjSsMR}f?H>F zsarFb=CNbDw2wMD%tnH_NNo@X232@o+w29;_wgob6V^s8LF~~vua{cMCWZf+9Ffml zQ~s4gPXSr?G~$eva#Y^j4%FUe$K`95=h#{@cylX!b4NxUDzh)xY~;u17x-3r%S91^ zPdz4gZiLU7!J~HD&7d_MHj85M4)*Jom0tUt+b)2p>T#Z8NPpWi-SnNOl#o}k1zdqD zZ=3F_ksx~8ySVued^AVuZUtc*eqq?~(`T*--38a(wg7hOrWY2fXQ&q}3;L~CpQU)6 zfpSAmVeZI`rm(Z6sHO!?LKtjeDL9jxq%#Kl8BJEViAD@_2v`d1opKWjGi(ousL_P@ zM3)A`1LYW_O0u94H-8PHff3%Y=_VgUbOW0ZID0UxIlz(At*2U4+&AMIbfv2sb=Jc( zULCbo=PUD)`E{~LQ#1x^YX;3Kcr>>F9THfmUd5NmHlCb=8tuN&z9`KB9X~cfn*<6# ztT?^wd##~)E#t+K8XKG6W8k(xk~KS30VGiOe7}63uJSl#x`ME{6iUfP-~p+%I&x7EJnzOXu-=r4a)I?ESw?Yxzv2f9ntM+*hT~A1sV}sh zbECg_YsiF0;|PyBO|l?<7${LmdU4bL7V~QPJ2Og2X3qJ!q)l#w+ih;R-qblX z35YpVYaiRth#*}j!=bT4=kcXbvAh&+zk}l_H>(&C^C-OKl}cDmO=@;`bHdkYQrJP= za<##VV%|EE2(Bhw=G}O|%s#y@EG;AqWK+giS?{fLCX(7BxUOkggTv!wx_t_Ml&S18 zH33CL&u-I5gAp%W+VwL@p-3Lv&oUct_8fvl5pik_LlfRzylpw}HgRXSD$KZCHQ%&s z6Lq)uG_7jO(~TGx*=TgjTHwl7r;8iiGhQ4$9RX-}oIt^WOKYsLkgqNE`cgmH)h{oo)=kF8x$Qvm^aeSkg<3hM6R37>>KKA< z&(5okSNJ+ldsuQ?<)!qVbT_12TG~o@n`fgj$(`12=hBFp)$LY>BEBOqet#5p*ByoJ z2|yKlu3H}}h4ZTuLCUZ0$}?iT&E8+GQ?**%Udvjw&r)HjpYO^uFbT3#SDSOs)|kL^ z0pbh)y8P{xF85sHqL2Jk1}_T61al;tUS66Za)KM#XJg8<6oZ-05#~VB@2WI!=2&GKm{`Q#UGkfHov;tSdtRD!P+ni3%9ST zN#>P}YEx5UrKP123;XA~WLUvxuBBXUmTc&7kZNj@dum$*I2vahK&w9<^^qh3)jShb zYiS3}=^tXj;gM-@86!N29F_=Y}54R-xwIVMi{QRf33dz88Jrw5~eNLryC#Ijs8O*TwYq5%2IiMk&BS3aN+Z_ zOVzKmlu;>8oD2GoF!Y#a%oUNPB#S``_A%kLNlYsNzu|viq#7GHW@xb@&DPyPCK?n(kkJ<(_FK&IP8@Hyzu4c_=OY#0cUN z5c~>;qe1uu$j?et_*gE8Z}33AVMM7l%28toBmmpSn zn3Xz9e3&qK!W+v7F{YNL*xPft*S&uE}l4mN15o^NhM4i9LoP>gckszpH*Q8q||1t8D9U}C&W;cY~}fH-{|XDaUS zHX2xX5qg;P6CW6P0O(T+fCf>M7kZ7InDZ3rE+2!%Y!Lf_jm)HH7iR! zBI9{ZC<_6u=^TAH6h=)Q^3Y!3B@FT7cdPfR8sJt_b#5GaiKp-tSIb;D00a_pRKO^K~_MdSfjnzn!JQAz1WF1^}& zu33u?rCYp0R%0ydX7|EJE$*lj1fJ<>(l&2!)d;$3y)bG(V?ryuT16dV?zlyVp27`ajit*@*>*GN_OIUg6V z{3MZmXF?m&Q4;O9j%R*XguEMPQ;bLWQ%_j;KR*UPuA}AB&?y>Cp#aC*eRvbp3K$csg`l;OkZK#+(+29To#Qexqcpzne=sz;|WfGR#trAQT&fL!T}6K(7Js zAd%$B+iJ%A%Tcv%S2UepK zf4Xt6{>7pNeFcHMeJb}6I|KF?2~eCW;5#z5v)hio0%-4DcW)ph1EP7J+;sTOmfweA~33)kx=5i+%eZ*5NU&xTn`**pIPj(!U8qWjLLrg!i0F zx7V^=s?j|b__QzOnHK5WslyrBB=?oSS(&_zix*&yS)uK@@6RcrW~gx-V2&**MVIrr zUw@AiKWT=y3DVq}oqal6=CH`@cx5|!A-{I5K z-+vK8*3xReejT^%=c142f^Pei$*K(S_sT4uPl{K>JD1X_EfIyIYl_>1itJc?5u&s2 zfqZ5~&%2c0@BjtJ?pH$a;(vso+l^bH7nf_-oIJ9bX|7yvo zspZ0yiSSpG7&RPQzYYPM#iCXHaB9Gg^6px-f4Vw~P>Mn=&VugY@*(pwCQ=0HkA zg1gDZP){4kw^vdk;{WD~CJcnSr|#{l>n^;hfD$kpC0%2t?Z|v*or@y_P(L%q3=R~0 zmj1y@mMe6swn-Xu{)S-#t zI7#h(o%-7I?&HiqB6i-*cb|DlF%IS0yQ0^i(sUl!c`E&HW?7tUrMt;2$y4ql*B<52 zWQ$H4cgki+On`?FB_cNXAUcF5R!|#7kF9QhGkAd0gx!Eb zm0pJ#^(~tH15Tu6>Y?#%nZFPKUL$2_$02cTe+CRv1_?My$ofbf<=PFJ1>#8GxbN!5gO@6VcdU`M+{5Too z6inXjw+K~jnxo^W;RP!Ajy3vs&!dg5)n4E?=Ks!3le$DF+!RhB3&3-VTl22jRI+6Z zP?NTe6@0EuOn&2Xvw0yWlkR&z*RL6>t$WHOabjQG3l-RskuLuBDYL~bU0Jau!xdl} z&^G=roIjjT%jj+E5N{1oFlb0gpvRtLrCt1-ejOAjYXmqtOdeP#-*xJwO$A_6t~gf9 z-FxvMfv*u*?(N5FFi8(uI~@ItN=%Omfq?CD>U>JeHYl1^w3LK&dj1&S_*}U+mlCgN zz?(RuHjd_y9ceImF; zpdVK8w_@Lb)#oe^s-(geIg?nvjftY6QI9n0l%qJ8b+`2mi-l4-?sbg}8%~_$a1Q3l zD&YGz)+h8n-l#~3eZzq#96e5nMMclBsOz_2j;-(RYN||3i!5GZOn2kElaYIt=mB{* zjH#LQ_0fhvi&{Bnlz}Lo4KkJW$4!7F$En)0!4sffe#$}Rt90>TX>dH*KcY+f+eqcLwXn9#hf=9TsxVEX} zGyj_43I|=X&M1cwD=7vZAiK@Kr~k)Bj&sr*~j`8Cz;6*obk`v!FEF-+WDT={1;@+6WQBRCf^H*Uc|&VTd7Jv$NXt!6CT&**3~ZQ zRDB-W=RK@yYzcd^jdWTXUYk=9Wh!?df~zRy>H?HFGVJ{VeKX{q1c@ z*J8=_Gsu2$Eacy&uJ#<`nXqBoZSRr#SI0;~u8bYzRYiL3!7aB=qNWbJ2R@dA^KPaN zaR$zjCSEP4v$9Y|J}o@geU*c@-%O=FqOy(6{*%N;?dKH8IpA5#>@r}Y9kDJ?tZKph zX<2OGRa2J{C^$2Zh3zre`v-lQS#82167DmMW;iOVEeyh5_V_So+srWr&ag2HwX$t& zdm^nfv+XH?5*~%`P9?t8O1Z!(0jFw``=$a2jl#H`o1OLUFkF*LXem_Pn#}M(tu0?O zC*20Fm7n;7tSojsd}f+kmIiD=fYL(d7vzb-r!Uuj^-&xtRM@=gyA~1PP(9>h@H5>W$kmcW%Jf7sjL zbTjr(EP?hq|8B+L5M@ospjL!2LYzSX3hc-zyHo2cTY+yRzY#h4q-Y~5O{_hZXR-)e zUVhG}dIu~U<$+_C6;^zmQjWa?EV6M@DtLG)f;mnS-Tkm_N$;p&99R_P%$!_PzH@dj zlKQ#Gvip5&Zi|XV)%8ET&zwN2a4iBE=Rk<^6Lb*Nm*<5C)7y|R!zBInii}=U(7!Fs z_m7O|qh{V*lK4KetBo*QqoArw*YkL2)H#UO1S=DROZZgaGv*?)u zQW2I+Pr5p`Z_eqHi9)^%Up%dZuVJ4a#p7B>I9?0d;RJY*VA-5A-t zeNlVDh(93^_K)?|pqJ2(UF{F+da?B7%b*5Fq;PP|c{fHw9lc^#gI)Y4d~;B^eRD!2 zrE&zdz0`f~;qUhgbl;Xpv-O#5)Tfn5_W?Ya1P1yv4qBm|l1K|ve*LE{2NM7f zGArsKip@b<7@Svq6`-j1S_>bjZ99cB&E$PJ!A3oMg|<2S z!&k6kc7>QIM+DGf|Ce2cLM)K2KT;Tj>weP%EfDz}XUm5ijPl1F5NLx6?72s}1M1RF z(s9?pO6Dy>D5y#=iRfq}Np4Ad)T;7V)-VutLI?2}ab;0V^zxFEu*^CZ4}M6XxVWAI zFS@Wc_i!|o@g9naQyF&Nv$kN?u z7{kC&F~x&J;Glw@`_3F@@;OOQuO2lXm{5Z@P$6Q{8QFd@BD;Xj4ihyMG|mO}Z<8=jN(D>3T!ZWYomIJM0Kr3#_>*}q(hGe+O$-)-!R@(5R{n} zmwtIAMy&+=zL_SckC{TNYi%Lm)f_pcqbW_l+z|kvb0z=s`#>O!bUH*(3U$wAFOb#& zuQ_A7G-6MZHsd01Cab*ImsR&m9QCKwI9rtLa%RF4-4%!aw>G&ca{6O~izmq*7})`y zHS?hGcy(;b<5AW9hZYDY&2GbBtsk`LCsJ;*z*T{L@f*Cd@d%F&FJ;^C?A%HA> z6{AY#k>Psd8m%dstw>iP;z)42@NXl*B7uHBK~$`haB@54kJ78y1))#EeTjt<*;j#y6n5S=X$ z@-ZoWd<)O*Q~K4mni1;t9)FEMq8AV27cAEIJK(?tV&8=O>oTI?>kq=F%H{{2jnp&d zagrH54W4kG8}JPs@Up@3Y5%Xts(2cvwXL2kViL#}P?A2Rn;7}DCs zy-EFtI>RP`&aJ~*dyNZZg4*1i!h2-h>0{D_K<>4e06oS!S+STHss46-hi`wJNrk)HqKsy- zaNi-c&L~XEklB6sirK;;AodphmD7F((AguZaPl9BxX&kWmK$(m65;EOp84bx%YWEq z4>O>VNE`?GYE!gMc?fXG9Upo7Hp=p7jjO~QWp2RYgpUlruXrDChBXTfcct%+p zMM-Ks%c(-JY~}#lN5JaVXy6P4>y7a(H%7Gqz~QVnNdJ{v7*s8h#VMAH#X*7b95(m4EH>(Il)f+>M)BFVFO(!ud;n755kIC^Ffm_=AP|p2@bMiW| ztZlzxb`&h-rkXi-Ei^`=(Wr2OCWIY0q>*3b17Yh%;*;B88_Wofadesq3K1PNi&!vSSZr^bz<` zy2S2L_qZBs(jw)N8}+&ZwLrQ$nC%EQ33GTdMB+@I{q_qzTbb9^ge<3x$%HkB0>PD7 zkpVFRnW%kljhxU_D||3(Iy3Cp*#@`CqbloSlqe@&KUfWtc~ zjTALRHH_MPjL8!)X!*d^>%BPidL&Za&=0tHYS@kz`hg3 zlbo5i;4^Yl>_+iZWB{O0RQw&_=NcJ}6jzKfLZap55!^tbnb6vM3$QVF{&(7i9=!n;f@^ov`da^Fu!?(iE_yms zzs6U*!H2b2j^|2|L-Qoenp?g0`gJ^foV8oX0ZEBB`zq0!N;7*D3#A9GCE2leLt4d5$amzsfR{{|#Iu3@^T^>I zYQ+z5+S1r4S<+7t)X=dnJU_Kiu&NcRyJJ+{U45mBQ->c<|9Z(f|9uzOil>aF zm|cLmH+d%N%tiCX2e?!|Y0ujANY?t~hhj%Oe<%I_@qiQ$6M#gO=oU-JQ}mF(5%yT2 z);2{~PsVCXS`S_-hV%|m)}*kh|DTdL2U{P6CIdX2QnNgr=j0OT&}T!?nle`*oYw*hLa=?5o6>4J(XHzxoYt4km@#(+oPu4qI-nwV(bKIOrOjh3uI^-QN$zE;SR~3Ac z+MM;-T9FWQp>$yvnY&4X1xR9?ec~sNQ>Ztzp~+LB(srgoR)1gtZq1375^xR2DRp_t z9CGy39S+3~S8Z;!ez8^0$l)Z+3rTpT6o4h{xpkoUy%!3W=)E7m&yV;z1flX4Xa>0` z5Pu3!&7<8>6tdCyU#AEMB^8j^XmoM#9583sT{d;y2>FywLMnQYoW(0+^%)KP5qE*3 zY6*HScOG%WMQ=4>&8|RDrXX!Y>9#g>{JV9HRUF5kzrODup$3Y76@u@PiA{*Y%La|# zqG^6r_f8Bc)O0p4{q<)Zo*7GaC4%s_hFi1r;TX=jb*qgnbOOw*>@l=BLa189XlE8e zZWij1iFF_XqBCcHvs5IcKOt@~IE|E_>&gJryi9jMun=zz_xmP=y+*`e#uoHH$?EGV z#d_T~=S;a*Q1W%J6U&plx@YYSxyC(^Xelo1HlhQNZh=Gi&#Skw{!2(DtBtxW!rfw~ zclDIVCyf|sRxr~@Q*Xz6ox6#~8aW^<;6r;{YGb;;5b{YRyUAPYQd&{AqMWs*31$Wf zxm>AO0V$&r#u)flMque_Q5_LMcrKwbdEmR{o#xvT^P&(Vs4#?4MT4dFq*@I-Nyo0W zJ~;eF)_6|)cH&cECgifO&O z;GH1>^=&n(1HDDHm7y*gc+4_{80Z6bbyzbO z^`PBFQiE)kZL?S;*Hk2c{)x^V7j)}@M$IQ{z#$_KGek3wHL`Gf?nHw&D54xp{rR|b z$l(pRnhHKMQ6o6YuC(U#tjQIXY-{bFxku@f!D^>!fD9>z9Tg=)HA!emymAh7niB0R zjw)!Isn7fpL?STBE%9B3tEOH}$_lKPWF`2h(zspss==Dgm1Hd=j{Htiq&I0PEx)lF z(`g~U@sq(XPtdA2Z~W|-srmK)IGxNK<96;R>+k+;n>m)nK5?sypV7Iwt)*L5G+RBK zCUk!8tEIuDxjztWu7JMck?Yne=WS4$sLPd3ik^Qs@!i{Cm_R_p;He+84m##SBe4Lv zc=EGOK17|Cb;MC_3Llts8IhL!gM|!z{S%7}vfaxk0m~=iYHz+zObgR$<%sb#5pauk zQ1Y$371Q9Cm0JCC0V7G>^F)3Si2~OSb+)$UyK6K7!hch_B~U8YKHq)LdcD%#$O+B3 zGdALKgSVQeB{#29YTj%jcTmbQye2=rwq%PoO6^0hcu|;Y022BV8q#Q2wEMOEN=`w> znqXOP#%J;5bX^6Jy{sfhO3e6|!lt~x%7VUgcnjaxAvJ^ewv7C?IoLg0)vKRE z9omybegT3sB;>^4`%bz1NTjbhfT!s+?HE3fgx380jM#QhKv|C!xY}#Xb-m+b&OU4B zFSOL+rb4mc;VuhMF(G-=V!0uh1XZXIHST+?7A)zB*cgqmNY1)5ENYm|kOuJR{BJZVu80nCMk@7qQ;!fF zL7ve4w6w^}pYHW?UDUK0%$!u~5%Jm>d3zJNr}~i?d$qsUW(Crd+_U4L<=5&88c}+e zwqNJ!cl;%x4H_5wAQg2R_E6%+IKyn3?m5^rx~VdGpM9(-HF0LH@v?rjl#Rfq{oG== zlet$J!Htwn?n1*t;|;r}LC3zQPx&rmX6xclK^XyM9?DudTz_MPy-2n8hFn=dfK!Gt zn;&)q60}hmd@gu#_;vM%{Y_s%T7Z$aQ76P1nq}ZWi07Wj52X zNmS04fD4U}Ozhz07RX+}NvZ<_$hZ)4&#*&&H1ai!AYM>Vi)o})P4^8b)m2d$(3H0^ z|ClPRE))s0#zGAT+gM8kJ(Zad(`P8kZ@ETgwRYkPV(wU$G0WglE18j7q{85BE-+{` z?yHHR0zX!i5z1mP?XT*rZ-GjBSg@G;YR?-IB*mS~%b>CBZc=Q)6A2%zJL4Ed(tFer z#_XEYC*~44q0?XCSx%piN611nvVTUb!qjJV5Iz?4D(Ybgw}U|%8A)dQJjB~zxj!6k zv|i|!QAMAJ+mqF$Q&rrZjL$9N0ZTb{5?MeRiFB?eIBEzRAaNT3zq9JpH38`jA$@ z2^ND-cKdGM)-us`LbiAdoa>G*g3JG5Ca1+wZ{D!1I-4Xe0{7!u?>{FWHPwh8o|rYV z=Ca=W2u;*X>f3Y5ih#CD`9E;ww2bFd?EO25Kw6#@dEq!FC%czInIP!0-UF@y9E~4L ztSU;BUIjD@iX@z6goMBLXBGzp>_N!-*Rn}z@-Zr4Q%@$nC5Ok2Ey2H~S77kWh^Eg?A9HjpB(6HNdcYeT5Z`u=Q+APT9x350cH5N1+%Z zErLItijf>U{JAz~Jc-vks#w3MK)FS;cpMZ~aigK#WY@LyXi|J>5xnbA(Nq}Lx)jKk zy4N#ax=95&3}4B7`rcPe`LDvRxgMkSI_@VMRe^m~?lk`3Y;5BBGa|2V0+F-Ch%xA5 zS=gh8pT+*v|BdEIm{85Q*h}>@RJ5=WHlh?qbq@jaV_HVF`$9Omt0CZQ*J9-Z5=z4f%OQQQ6JWi%Bia)e&3H=D{eBYhMiUz-4l2wNzu<%0Ilu5+RVjUM93)!Ate0XJ8Lp=xXobfO zC^ED|;M+hx(nE~|E9tZB=Qp>K3KZYsp|(QEE8C*9{P%9qI>oHJEaLOHWj{s9-_F1> zE8#P+Fj+-nMbM*KE-ZK=*!9>25{~Im!<+~&2R6yIxL8=24+m~=5HIGe7TsXTUOP2(}n9-fW?yl}r`)Bk!wV96OobG%I?O z^c!pp3OQY(6Yjtr7w>{0OTFTJ$QwP^?&Xz!lrpJ7K+U7)aJAk}yq7|Ezxi(Dp$q=g z&9rSj;#DB2=<;pCi$UnD^P~SzK+T*K%6x@Vk)+)qubrvA{Y3=EoK$Q>1C;89Ai(sh46|f0%*E3*K%xp~acmGj zE&qCIQ^c?IGJp3nct`%}`bon7aU{k7qzPP_ms_{xSCzpr1I5TU5lgSW73teuduUP2 zw>`_)Su8Q6VWLy^xiG^f5#v>d`orfC8Ks{}_xe8b3JiijxWq4?#ujWczDkcAlQs|% z3CX^8Mi)bKc_|b}7Ko7uU)L2b+p6VtWRQW%93duyC@kI{mu4krmk=>kuZ;lsg7*Na zV2Gg!I>At+nIpKn<}b-&V#xYxoeVGaUt>H}l=#u^ICd3&y;{!ZYCH!P<;o_%dj_C| z#glJvtYrw0kpUsHf67XslwWD?l|rVGHBI=2e8pI*v0_i}L}~U}|wH=u%MNNKHdbH#d>cg2e1d3kC$urO&yW z_0s!p22B{mISN9c-pYNFS&Aj*73~@qPBumo9>4UjI^bRGli}XhdFL>De+AX6kwVU= z{5{m`b$llHy`IHnY^Ma9vT1hnaesBA-pY*AgkRMP+41OtUeL_0bC&J4TXh%n&>MT= zKy_&E?~8NSX2KsNAqo&{6V@_FMaBNhu*%v$UHz45UGaE3EUK)#PFET zRhZVSNcXKVLIZ~;L(a`dXP7+I`=j4$W7_Mj_vDTj&RwPkA7DOc!QgD90TA9l_M-qm z>9?BC0Yt&T-;mpo(+(+P<)sy7Y5=+IjIaE=o^AFxkVYcVw9ke!&Uz>sG3>6pQdXbP zgmP8W50n{M(|pWJ+x(B#QrE*S2s84J=*lFU_aBsms2qRob^rGrvGf~i7L!PLdU7jV#WDv4F8Id zw8(!$zdPb2<({y#BHiJP-(75+DtIK_;bb5}z`#MW$o`Sw{X)quV7UraQSs@4mH(GE zvvQR#v7!5n;90>&Axqa00BPE$OU?I2l!ZPHkjkq5G0wrS7R&AD?Z*5`j-jm4kd+SR zbpx5~QI|YKLrgpl37%`9tUgUqW-{8<$Hc}V_j$Ft%Q?m(;N>4&KvP= zQQ4q^hvumD}Ih88Xwtf?SixjtkGn4BDQhzrxtLMTbiLW#;>0e3l z4uPl?`eIYa2 zO=~>paE=II;=NTj&8#{L2%;!SquL*(vsBGw%L2(~y;l|$6$?Hi5fa)Axj~qcaA?m< zFVe@?m-3LIBtP186S@R`+>-ZpXs2l*=z5sh@7gq#$Naz2zA`G#CE5~qC%C&qfZ!nw z0Rq9@gG+)t!QI{6-7UCFaHny1cW-Pux%bU_Z`RDbwPyH3_k#MWzOu8=u6<5TPGGSq zzhg$+#Sj|N0P1~iqm%~)lkLEi1x-JyFxqYISB|dDUafh!{?V~EbTR6BcC3wU zzmHtb%E=OIqb$@Fy1}fYl_+qdWg(gw^+crLx~>Xg-CsMQI~kL)&r&B%8f;j|X#{r2jDEz^>*7_`ObVHC;b_lYhJ1TRO& zwQud`ze>H29Ld&I=-`=^&#}i^_H*hI8reAZDY$g);D`Gw9 zYIJ8vy|vh^kq=#ZgGHI?6PKV6$qkw@s>{lpvN~9vye_*a(_Wb`j?!!NJwyaWmJKJiS~vr0drEhV2cJ5ZqdjHL+S<(@fc2Iqr3)kEJ`brR!rL@Ef_Xx zsnjIFOhUl2`QW^ESFtslj4}7EmM)dVj>*eYYIg#+qym%Z@rD0Wbw^q^FvDe>TDOJA z$BYkcoU(&G&ZF^XX>K2GytL4lkC}s0S?=e`MJk(ux@t}@8ruP{rYMv7OH}m~XH&k6 zQhSC(!6$sLzC|>;ys}A2A>PtLA)zE3bw*srCcU?p$>pHBz4QC|nzRVDT|rR+6E8aA zbg}2HIgrk(wc8%AwR07VF8*?Z6e|=5WKb(tdRd3Hx+1=lrSYz5f`^IkqHFu+GMNJl zu8a6o|JDZc7T%sSD@*uGb~oqZIov6Bz8WdB9At`6?e80r)n$Omkl@;&xzUE{EmtH% zyPHy#cmek)i^df<8&wbGrby&=E3F(=RjwacMzftqaW}i&O1Xh6Ia@S=javs)g&{tC zIZ?r9tw+?CxSpsbbqaA996vM%cIxzKOblmvgt`-axX6gY4U}o~A!@oE@MM)7X%TTn z$3e({dxx3=h8mI7q+b9`S_c%+u2Q#o;+uJvfN5yB;UD>4nUpV`7S#vOSB<;t#ph3! z_$z5*<6pzx_grXvZqiFh8kbX&VE?%#io$U6<)lnt&k}7F=eCW;wF~0?3wdiYJsCpK zgg1E(OwA>cPV>?0OeT=m1JHPT%|k>rFg9Hbs-_?)k{fZqiWTT-0F=Ew>_ovi-uoE# zY!pHxvKer*xMa@O&v)#H1ghL}p7aOA%3!+XP492Fv!tx%H}r`204I*PSI>AFqztYj zs?_?JgX5M6VQQ#cZ?_P`y=H&TWf=<~58PLxJq=bmg?MZsUf=JQPNwG)C3?<*)(84O zNmx?Tt|9BPK8^7blL#9Hn4M~pv_FT(YCsPOu3Wh!B$!iKOm;yUXRQ$AJfiyN!Ph2! zXuBcryx;KkN>DH0Mif3Fe^q~hY-bl*x78QKHhL1iB@>w$3hp{hV)BUJiV3v2se7v! zITmeSe|F})onw$|lrxLN=p!`{-bOnTtgGCvb#yq`j_ij$FyS*$6|;8V^aE2-o9y}r zrj7%d1)pYP9>YRane)f}w-y;QT9IahQUF%1-|jxNRE3k1V&6hp{GfWOqvL0F^*xzr zu=K&RDQt{5ppep$ayf1Sn*5Z+kDRa~l)5#!_Y}5V&lN}E7u5c>kf=Ke5&CN%Rv&@v zFlfeW#i>=fOSF`Gh&8~_c)+2toEt~tg@q!P^VCn~8sRo25;;@BZ@1o4PQtQkMIP2&RYS)A}M6*Fk4$X9MI?lJ)+8?U2BL+K?yT?EyR8S~7Nk9=-LRb%E??UK&JDxKRGzI*PoW$pzS6Ry zpxB1k*6ZYKCy<%Jm8Wl|e3ZLD007+Vm~WR*-iNxkCdo)w4K^^vFNk&(RyONp0%rC| z(NESz`PchDuNH?u%r!=7_DjnHPXXc|#pdc0=em@9;nErF@O6jN5$14~h>tt`z{$&4 zmQgG?QI|`IK8Dc)j5FNM(=7}6^j4pmKj}LXM{DqnQp5xs$~XSjP$5A@Pcd`O9?^9h(@>GNea2 zQPBs^w1`8--K|2N-Fgm@hg(xrXZfl|AqrxY&R+6`A1-K${d}`CRW71Z9U92kV@gbQ zV=7TbJvmVv_K+sP^y$l}x zH(vLIB{$oHFT)!eY8)C$?8<&+h;eQIQI7;V0+hhgV9j|vN8w13WPa7jT6KoH&5W10 zzeK%o;aYj`eR~qav{Q+Awhwwu{aovaHxRLoBGgc#&SfbRnpmC%>uUnh66>*hBe@(W z4^7i5MV;PAf)&H=I;9-T=_S?VrK~qB@3s&>|KYe-?qz$QSJQbfBwlUj*Du3|$Hg-q zdcg8Fq5JxB;?Lf*Ag#E#+P!qcBxRQ=gZhJwxwt^Mf#T~4-^8r6aBYqm(khmx${EUSQ3F>7F)z>1u{-k#ZMG!#ZuPbi0 zF%LB7@s=BGEGs|iYeZkbJA%!>20{X_7`}xuKSG5d{pEOo|H|qP8KUGfG$EZ5VE+-= z(H@gwedyiGP{LFePvj$2jyr2*&^W3>7VZgI<{xi%TKhFk6_c5b;pyo#%6D`-MbF#O zx>r@UI3V?g^=hS+dWY%SfOx+J|wk-N` z6Enu)J5ZY}I2-TY_>0jRe;FZa0exv8;AlPcJ`|eSHWKsTJRMhI7$`;6(vJn|;`#bx zd!-Irf2m@45;C*r?Xg}F!hYKIgRs@m?W!@>Cx?6_$lkW^ELM>UL>rm$NSacX5hj>chumM`*{9=IKcy=Kf#?8C9~#!<)sNF&5u~f`-)6!jtIx*lJPKOx z(+>8p43H zZ^2P+F&kGPbxSo=Iz9!X}qwo?c;QgXqwI5Ltz{uPkF&9Z&dRO zuEa|TW}|;amKeql-sGM(bkgU}`bjsaI{PYlxy6;*12r!y?F}gatuF9$%9KAJPc^1SVRpyOmCptQkGKdflMu`Bsz{5}Pxk`E9Lm z$q9wozSTliI^($!_Wn5;zB-a6=4o|EqsrW$D|3d>{|EPP}}jOOc()k6+n=xbdNyr$KuRJFfgFx4_xJ);2Un(42lW!}Za>RIz z&>VMr1bc6o9tEU|9IsAzNxfe*z-xkV%+@uxr)WKf?d`QORCE2cNfATIAyPABuX#-1 z8U#>ehsNSonyBfrn2!DZZYr9+I`{MC4sWn5qkwW>V$(M*=+ba)2z^v<<-y$40w<8E z%9t9;40vVymJ`{R+DlQIYrY?cJYJi)MJ(GMrF%l6>RM2)P1AA@ z{mDpFZcSDnPQf$}My02*8Jig`-F3;=W4nChYJlSpKw%o2kh1}4Mm>Y>N0J?}qO|&% zG@6s6c@1F&=4Vf}G^G4j(&$O5GahS&O1AaHtGP)hC?1?jcgiBEa;*3bJVWJQ`(h-! zk(f zneDRc4K-ITNj*#&@k~Li7oaKCUnU}P&6>7?R#rK)xFx)@=c8p z&fh!|no&mRWy6VyVCu?$)a}?C5?)!m^g>JUfv}_Q8O3jgMap?b6Exf@=bDlD9w}o# zxyugWk|#}_eh$l{oe%Y~-fa(v5&-$BDv-#CbFNOjQXaM?mWSvt0oS#IG-==4{e_ux zuNy6Ep^$@kp)=%JIU=A%0lWK0cAusuw46c#mZr{XkM^>~9QVutk7E>|q5ken%DP`n5CcU&GaK=KKv{?kII z?2p8pyYdgkUb*~QM`@}#7mF-r*{vuCrUCoJg%XE8p`l8;Q9dow`^w%YkO4FV_Jc&L z`bOyZ<$p0~__lD$NAaC)Q)^CZ_7hftKdrtfIC9>Pv$`9u)t_eqyXUC!@v0H{B2{;S zHJ44y!MU3V>G?&rkfOxm@)!F&gqvoBu ziiY93RWtqVyuUw;?Z_?8lP5>9xusOw*VoRp@o+L^y%e1W zd?9fV0%OLJpHqMti+ZYJuzzVu1FsQl!iw~owxXee2$Nv;&k_nn~a4gY}PMu4RLFOjr5_jwcX`X7M`vzIO9R5plo# zuXL#E)cCmOh!Y`HjgLX60T%;=$w_S}7=4@{?+~Ew0C=f zDl`K*<5M_rSgV~$b#*ww-xcYyKnHeGlIE_ZCG)+=iQy%sfp$WpY(WOI&$*^0s8%Qi zMTEH<46uFm0)d~IE15K}klg5|;5ek8|K<~ujG%N0cwC~_Qoc}q>i8L^L!*1(w@V!Il)i5BPrM#8Ap_Q#dgWlwNQk7tYqYf;dveY zEq%e!hm$5V1(bl;Wq$-3lqJ|Di>^7)hAt9|d4IiyzyQK2C63q&%?W-m8}%dllyj(Q zG=%41ThfhD!HvVpHJZtbAD(~w@w1~Dfk`PNHkWDcc3QFzp{aqg(ekY~xkPdF z+mOtaW6!qoxr#^?!2D%Id*HLSQElj5fPbiCh~%b$g?6d$X%b_y2aT`_9fGT$NRHj6 zvhi}3CI9;?`I(;*fN6C#6DbwE3C-s4H~iu9(yNmO^? zgEjOcQ@=@xzt6UlAw4|6q}S+80WK9&B*llYkk?JRLGgKhb#F`_8Y{-XNb(A{1QjFY zA3RS;!vP&Sm%$if5-$3Z;@Zw-=vH2)Z9U_$sa{y7jLfqKN!w^|S~04Jz$xEzj)5-0($s+NY17&5)P6t)u!f-ye6B+NM(%9Ia%Q zZ;avByqOsTv+DgeWbD}#x@7lNYhk~Qo~gE&_3kp;SfMo^Qzi0zdKgpKJA)REju!d@ z_T##p=^Q#r1avFZ35SmT0uUG?+<@pZiMRG5Fv7&aTIE&GS!HmJQSi7Mu8kBzKhY*B zB%1Pu#_u{Y0o|_z;A`KSvcWJM1{04UoXFqUC>QAOW@I#pI8yemDprpI8W0AZWiUxy z52^pZ8pKl(5^0+!JZh`1cAv-~l!OgrrHC*t5z;8YP3yJU&F z^<88=$;@i)lJW+n3Qr^O@j*w^*)u5A;`^NJ{WPi*u&T~CSDjaIxxhu2_!}-`XZq$K z(Hm>-8^jUZL)D21KkkcpM22Wl&X~*5!6rp%qn zm|Jd99UvQ&p~JRPmqOGZU-3jvXTs$xu;mgc4u(MoS0ay2)=@pQBNjE$;Z*8;i01AZ z<6z2HY1c6)gVq%!u$X@EEx;V)j=G3#f?{a%BlBu#8s%hCGC(X#_N{yKUb5{Y@?41= zfBNJ<-p`a1Usdaav&VOFkJ6D;5)~Bt_nvb~SZN6V3aqcxrhQ)0=+~Fki;1`}|AL&9 zZ10wr{otb7p|JX2aG6*iHq=voKPb{_I0taUmp`F3m|gwS7Fd?mif6({@$?y zv1gV7%Q=d8vETc^nBQdXZWeJr(5!K0?q1hXgKCWD+Z;B7$-tf%nkW0z^EuUg5I`up zk>2kB&>LAH`M&LbbYJ&n>S}#`RU|`d`)OTpa$kb=z*$a}!U7~O~$2Uw!!>A(Sc2UU^=iQ>~i>TXcfI6|I1J)q4!cI)7iQu6n%y3_huUgyC5*( zkT>-`gj7QT-E9IOsY)-!l)e;Zy>EBgFy%b&4rSQ#omS3!Bxw)|XURReps4mhM3bnv z^>b5C{Pk=LpA8mxXxFi+zq-b_|u)w>#P&C`@@b>4bP@^on;YzSwXM?EJzKFaXX! z5L2aKRLAzB(fGtPDfjlADWV*P;T%%3As=TMTggyPUmOC1`gmiig=c)%(WIyJv32>1 zIG?wIJxBLCIMKBcmo~LXHL}!|8XL_c9&tzRIy_ZL6ATns%QK|sU!l90SUo2-?MrTK zRgQ60c9^=s+Aa8y9inBlzISDId!)m2HysjWfujV;@X$r^%o}KrSwdAsU5VGT{P>M; zY*Z!A++iz>3Q@^ZZkf5D_%1@MibPAdv6i@IrdXF=$%P{3(VW^J0!dSr1G+IF(Lk=G zB-B%n!zlBVG9U-ufoY0Jke`d05=Z(ZeF@+?<&IEkcBqZ}apZ{S@ zqu2IMFo#={H#{r-DjD(<>m6x5IqQ}E3}%0EF=&?hFf+zj!?(n~ae+bbYcU+YY@!_R z1lcsyYL^%vTLl$rvhEG@g$9J+EC#F`1G((?2n++;Z&O&WNqKbli^(GFP8^ob*07=< z=TJ`3U-&K6#m>_?1!Vw2^2r@9D4?sTZcm6Kya%8V%fUFcg>RTgx6K~Tbi3vd^2YOf zGm4K|HY*;=UrXHtC$2rIdi(vpo zeFNQ}Pomm~lGd^=pHkD~x%P=$@`uB4TYVxwT$RpxJO2KyfS#3;P7e4`VrA-s|193-^~F#7&pI}y6FVk&X(jOF|OAdDrQj8Z2= zI&!|CXJO^EKUs1H)kPLCje+cJUKr{P{{-q*R}6r@{11WP6eEc6onp0YiXVT2n);IO z@mzU)fpEaVmW136d*oG3YJW0fk+hDh%lkf?Y=#p5rdNBeEy^mEm91QYG^H#A2L172 z@eFUs{L;W#Sdxu54hbB5!J2bi=JF`kAe&F9d04iqDAF;9@@1Nbw-4SKlt7x5Y4&ChzcBWd`J=MZf#2W?|ifL+@?tfT!0Z^~6;nZmCvZPkxVVpNdivKXqCLbB<#^Z@@ZjAsn4K9N2(4y;9LB}S@ zslTeE^(SMJ@4spA3F^ln#uKnu9~==iI}MI+{6*eFp&*kZ0+?dONEM=n9_r zkw=S+*rl!7lWGl->CsQKX3RXvWm;LAsP-Sj2)K~dtP5TDzeIaRMB3~nu#F#0khrES zKBB8HmX|?)qub70;$1d;&z~OzeshZ{qT`C4^V{W!G{$YjCP*$vwuYNsd$jp9M*Vj} z@6z?h%W77nJJZ&Rd5*b@@;Ue%4BelAz(GeG1y=f>Rx4vnd zCr-OFSrNL8S9NF4hVQ>;R7s?j1pWi;=>#(~U&Gvt$IOurNJ4+etHLpaaS=x&?xj68=7qTXY(1y%lMtFeL68e7SzYmB+!Y zWP|+u%cz=!Lr(lvoThsBxmtF)5e%{uY^I^CQHQ#t7BL3B;(bi|1;rhH~7GKMUe}8Nu3j&&tHBi+<%NqQGK+Qk7>_;u-t^ ze{$E3Q>E|hXL&(8`**@ghR%ZJ()u-}7N_qH=9O5caGC>|Ri9{&!y4(Cg)VlVYsM3I zYcpTyYKl)$?Q=cHN_YBu<`*I*Z!y>=ptm9==j}^UMg}gKNh^1(M;4U6c4!pI<}GcE zu=6EyHxz9<=rm)my%GSb-CFl9ilj1J3rTbtm1X*;F%`XPPRR?zv7o^QYPq+fNSXdY zE~oa?RXxR*-2Zf;13f_mG81Tv3%i+{&NjDLt36luM$PC9&ur~}k(QULW8ulUgF1TX!D7^qg zqQ7_q>8VF9C;o{uV%qtk-Irs-H7^Nx@7*!BbUe)?sBjOvZ(4*RVQtF{!Up(3ygjPv>SpxFH8K4-K>Z;me|192?_pE{Gg#JIWBi;J+V{I zKx1v%Y#|rW5OX2AWg~5CgK{M*N~QeFOl5PL_0VW~wGk z{lZMAJr=4{2Q|N;`J6U8Z)L<)wLCMww_D%LoOHGHE0%{X=~VH1+T7y!;-2DnBJd5d zkOYG8PgdjceCu`s={8(Uu472rt+;O{xAj#vsq?3Y=GRwC+KEgqmS2X_l17{ue#snL z_?hQeg0HxQ2NNb$IoMl@PGtU8-qXu;llC}H7=f=G=vX`@?;juh)Nfo;$@|9otQkDl7 zA!+zdu-RX;!`l&j$z=ykSM+A&CHGsSrylqG%Sc44zJ+*QgO=L%99uz4>22IqRi~eI z(p0XTVn-UVm-ZYRz9CprA^0v(kpaw$t}U@E8}QnlXkJePI=HE-RwFyOl_J3ZOp$b& z^LMNSuP*DH>!Z(Zw!V#`(P>*^x9@>|$!@Ld9GcHuEwSF#H=BjWHcB;HMEN&>{UhS& ziT1ORAgiuhHVs%~*``#yb`zM&Z=qgpc~vqa=B>c_y;63`wKnaB`OnMX_M;6J0f;)C zdeJ*K-o?9Yn#tazVm3Kpyfw_sH~i3PotZD(sdw?P{Lcebvak8iDDfS;EA`BBPniqM z8r<9CxC7@+MrOe=2z?{Q$Y_T#lTO(kIGw^7*3|yO8Rr{ah-%BCRi|yaPYn8>j?C+J z6$do5hm755!P!X>wu1l86BgLTPlLu%KiGW9uf9}zas&O#M8$ z6wD{%5d#0V2A?t__y!|`G{95ERv66WBT%w}WqkjuUx{~92kx6KUr$_N3{ma>U7?Q0 zMf#GlFBsI}HU244moimMglitEW+OD4i9k4V2^jfKB}^T2@uxl0kRJl2`3>4gA2`7g zM8cFl`7#SxGz9Fx1={*UtcxwD5*DFo1aeKdmIcV)^AjSXg*u((^TFa6qM3Sae{M|w z57+s>ZPNeKd;FgZZLLiN&Zov9`1*4vV%&Z`o7WEybk0gx`mA9qWd@Bw?OU>5A}?j> zC)ade*Y}G=MNx~M?%#Jz$(Pw&$O^y|iLg{mqo>kYVXcQn!0Hxdda)M4;;&7cl(5Vx z4^CKufD!(T9{y)!F#r1=|3A0%|1wPfZK34b?$7q9L*&!_tPTPho~x1Tc)o*(VWS@S z+opM&tEL4FqH8xJ?%vDKrJ8K@TnN4h#s9pvR_?2Gr^;FM5=KfMx%5imut5S?%_Qtk z-OhL((7fpQTO5U};j3k6c?~loWmAQtKX48_*~m4)=d>hUi{gV%`$DeMH-gC#Wj`_s z8eqXx3)D91M@@M^IJ z%lFxVU65!*;8n+UU@90$$lIk44I|tJmRawIv-SS{3=jTHoyoEN_ql$abN0E@PXAN_ z{({^;zxnqMRL+9zZ+sq9dNKq#9Tf7Gjs)aGjXvZO7l>|vr|vw+=irbMNb}+6Zfq=f zA_QLvp{6LJbhP}mfUaf@`A@F=2fA>+=5vJfSGVfXavq&W(ELz=s z#C0EMRs328ps8#^NWuHS*S-ghZ!w2zQ4U`%IxbILXIFSjD!{iK`vzkG%vaU8STmL+ zq}zC&(h@YbZQ5XjqxCQeUi&pcrz;MgY#wKg1*#=7pE0Nnf>8ji=X3G0H~nK?N(QSL zh1%N-om`mdU*$g_4SdIhK}5scGz}vybgOgg%+|41 z5P{1eW!02EefPmTcm2Vshzu;mY8t1{-d{hvIjD-Oi%@Z-n&O9wzy=S zm@$)kAFJvTbJpbj3F!dK!t_6KDS6Lf{`)n)Woi4t_$linQZEbh)73Dbp2t(M@hFUE zQ`dvyo@jOD8KU2&G)p9q$JY9Jt^ee=lUW&o6wy8(HhO>!c zsTomm>V1yP#nWz+wW?o%Xwq(F_7Dmn?RS`x&V)!}Q(zz(xMN9kK5f1hfRK!m^{UMc zWHx6fjKB^4m-Ij?F#Zvs&qR5)vPT?}BK==d8+ zjE5^L>F|7dSQ1G8QxbAVrt|N~QhZS4y7U~4SD~tmR$MO4OkHto!$~Eh_l+d-`8?|T zbmlCZwOzEK_&OH zcZ-n+6shhix=EliAq-I{${h2AJJgps)D7&b)6#&oCs*>cv_)s0Md(>b;LQ7-o9kmN?65Q=|bq=^nCT7ZnZxW)Mo9^lxm^Sg;D}+h;cND=-T5F7VFrSu(@TAJD z8WcJ!lK*N49tWhD$*h2;^|mze$EK;gv?_Dg7@|4nw_b%W{X=6d5;^j}IHx~j+$~BD zs3Jgtm#qY)zB{g-_DVzA%@+f}@Yvc4E{`m5`^R}42%8Flk}P;#65J;3+-F8n<6kEs z?O{?+SF7QseJ>tgW$v>l@7wcV%)r%Rfj1K&r)B#umi)l;Pi7v~RTICSf@`3#`3xgWP0o?&M*-dl;r|O-(Fc6eS_fys zI-L{;TVZe=`sb!ff?9)8P^sWV6T;CO-rL|U|Nn|z{xOO)C)+LW{{Wm2@NOM;2?jh8 zD3M^J@V|#gG%GLV3V$jL`3c^i3x~PNkyD=}Ex~4|n_^|V1&qI>EI$hvZP-O4U{UzD yX2FgjfPoDLbr0gbE!enVl>GmIt^Z=hpS^ZRbF8VN-#0*jf4+*zidKHs^Zg&L^imK2 literal 0 HcmV?d00001 diff --git a/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/App.java b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/App.java index eddb17b27c32..3074a2b32370 100644 --- a/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/App.java +++ b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/App.java @@ -113,6 +113,8 @@ static void callInventory(Bulkhead bulkhead, RemoteService inventory, String req LOGGER.error("Inventory check for '{}' failed", request, e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); + LOGGER.warn( + "Inventory check for '{}' was interrupted while waiting for the response", request); } } diff --git a/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/Bulkhead.java b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/Bulkhead.java index 430b935a9a8c..9b04849c053f 100644 --- a/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/Bulkhead.java +++ b/microservices-bulkhead/src/main/java/com/iluwatar/bulkhead/Bulkhead.java @@ -109,6 +109,9 @@ public Future submit(Callable task) { executor.getQueue().size()); return future; } catch (RejectedExecutionException e) { + if (executor.isShutdown()) { + throw new IllegalStateException("Bulkhead '" + name + "' is shut down"); + } rejectedCalls.incrementAndGet(); LOGGER.warn( "Bulkhead '{}' is full ({} active, {} queued), rejecting call", @@ -134,9 +137,17 @@ public long getRejectedCalls() { return rejectedCalls.get(); } - /** Stops the compartment, interrupting calls that are still running. */ + /** + * Stops the compartment, interrupting calls that are still running and cancelling the calls that + * were still waiting in the queue, so that callers blocked on their future are released instead + * of waiting for a result that will never be produced. + */ public void shutdown() { - executor.shutdownNow(); + for (var pending : executor.shutdownNow()) { + if (pending instanceof Future future) { + future.cancel(false); + } + } } @Override diff --git a/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/BulkheadTest.java b/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/BulkheadTest.java index 1ba207328a74..33df93981fae 100644 --- a/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/BulkheadTest.java +++ b/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/BulkheadTest.java @@ -30,8 +30,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; @@ -131,6 +133,23 @@ void shouldRejectSubmissionAfterShutdown() { assertThrows(IllegalStateException.class, () -> bulkhead.submit(() -> "late")); } + @Test + void shouldCancelQueuedCallsOnShutdown() throws Exception { + var started = new CountDownLatch(1); + var gate = new CountDownLatch(1); + Future queued; + try (var bulkhead = new Bulkhead("payment", 1, 1)) { + bulkhead.submit(blockOn(started, gate)); + assertTrue(started.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + queued = bulkhead.submit(() -> "queued"); + } + + // Without the cancellation the queued task would simply be dropped and this call would block + // until the timeout expires, because nobody ever completes its future. + assertTrue(queued.isCancelled()); + assertThrows(CancellationException.class, () -> queued.get(1, TimeUnit.SECONDS)); + } + @Test void shouldRejectInvalidConfiguration() { assertThrows(IllegalArgumentException.class, () -> new Bulkhead("test", 0, 1)); diff --git a/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/PaymentServiceTest.java b/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/PaymentServiceTest.java index 7b1c685f3093..5ff96742c0a7 100644 --- a/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/PaymentServiceTest.java +++ b/microservices-bulkhead/src/test/java/com/iluwatar/bulkhead/PaymentServiceTest.java @@ -29,10 +29,16 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.time.Duration; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; class PaymentServiceTest { + @AfterEach + void clearInterruptFlag() { + Thread.interrupted(); + } + @Test void shouldApprovePaymentAfterLatency() { var service = new PaymentService(Duration.ofMillis(10)); @@ -44,11 +50,9 @@ void shouldApprovePaymentAfterLatency() { void shouldFailAndKeepInterruptFlagWhenInterrupted() { var service = new PaymentService(Duration.ofSeconds(10)); Thread.currentThread().interrupt(); - try { - assertThrows(IllegalStateException.class, () -> service.call("order-1")); - assertTrue(Thread.currentThread().isInterrupted()); - } finally { - assertTrue(Thread.interrupted()); - } + + assertThrows(IllegalStateException.class, () -> service.call("order-1")); + + assertTrue(Thread.currentThread().isInterrupted()); } }