Drain Spring Boot workers during graceful shutdown

CHANGELOG.md

@@ -41,6 +41,9 @@ to docs, or any other relevant information.
   specific to the Temporal CLI dev server implementation and may no longer be valid if that implementation changes.
 
 ### Fixed
+- Spring Boot now drains root and non-root workers during application context shutdown, including
+  SIGINT and SIGTERM, within `spring.lifecycle.timeout-per-shutdown-phase`. Closing a child context
+  no longer shuts down its parent's non-root workers.
 - Test server now honors retry expiration deadlines that fall exactly on a whole second. Previously such deadlines were
   ignored and retries were scheduled past them instead of failing with `RETRY_STATE_TIMEOUT`.
 

temporal-sdk/src/main/java/io/temporal/worker/WorkerFactory.java

@@ -45,7 +45,12 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-/** Maintains worker creation and lifecycle. */
+/**
+ * Maintains worker creation and lifecycle. Applications are responsible for calling {@link
+ * #shutdown()} and {@link #awaitTermination(long, TimeUnit)} before exiting. For example, a JVM
+ * shutdown hook can drain workers on SIGINT and SIGTERM. Hosting integrations such as Temporal
+ * Spring Boot manage this lifecycle for the application.
+ */
 public final class WorkerFactory {
   private static final Logger log = LoggerFactory.getLogger(WorkerFactory.class);
 

temporal-spring-boot-autoconfigure/README.md

@@ -2,6 +2,22 @@
 
 For documentation on the Temporal Spring Boot Integration, please visit [https://docs.temporal.io/develop/java/spring-boot-integration](https://docs.temporal.io/develop/java/spring-boot-integration)
 
+## Worker shutdown
+
+Closing the Spring application context initiates graceful shutdown for workers in the root and
+non-root namespaces and waits for in-flight tasks before destroying their dependencies. With
+Spring Boot's JVM shutdown hook enabled, both SIGINT and SIGTERM follow this path on POSIX systems.
+Worker startup still follows `spring.temporal.start-workers` and its per-namespace overrides.
+
+The wait is bounded by `spring.lifecycle.timeout-per-shutdown-phase` (30 seconds by default).
+For example, set it to `60s` to allow more time for activities to finish. When the timeout expires,
+Spring continues closing the context; the integration does not interrupt activities with
+`shutdownNow()`. Allow enough process termination time for the configured grace period.
+
+Applications that disable Spring Boot's JVM shutdown hook must close the application context
+themselves. Standalone `WorkerFactory` applications manage their own shutdown; a JVM shutdown hook
+should call `shutdown()` followed by `awaitTermination(timeout, unit)` before closing service stubs.
+
 # Running Multiple Name Space (experimental)
 
 Along with the root namespace, you can configure multiple non-root namespaces in the application.yml file. Different namespaces can have different configurations including but not limited to different connection options, registered workflows/activities, data converters etc.
@@ -60,4 +76,4 @@ annotation with the bean name corresponding to the namespace alias + `WorkflowCl
 
     @Resource(name = "unassignWorkflowClient")
     private WorkflowClient unassignWorkflowClient;
-```
+```

temporal-spring-boot-autoconfigure/build.gradle

@@ -65,7 +65,8 @@ dependencies {
 
 tasks.test {
     useJUnitPlatform()
+    systemProperty 'temporal.test.classpath', sourceSets.test.runtimeClasspath.asPath
     testLogging {
         events("passed", "skipped", "failed")
     }
-}
+}

temporal-spring-boot-autoconfigure/src/main/java/io/temporal/spring/boot/autoconfigure/NonRootBeanPostProcessor.java

@@ -14,6 +14,7 @@
 import io.temporal.serviceclient.WorkflowServiceStubsOptions;
 import io.temporal.serviceclient.WorkflowServiceStubsPlugin;
 import io.temporal.spring.boot.TemporalOptionsCustomizer;
+import io.temporal.spring.boot.autoconfigure.internal.WorkerFactoryLifecycle;
 import io.temporal.spring.boot.autoconfigure.properties.ConnectionProperties;
 import io.temporal.spring.boot.autoconfigure.properties.NonRootNamespaceProperties;
 import io.temporal.spring.boot.autoconfigure.properties.TemporalProperties;
@@ -193,6 +194,9 @@ private void injectBeanByNonRootNamespace(NonRootNamespaceProperties ns) {
     beanFactory.registerSingleton(
         beanPrefix + ScheduleClient.class.getSimpleName(), scheduleClient);
     beanFactory.registerSingleton(beanPrefix + WorkerFactory.class.getSimpleName(), workerFactory);
+    beanFactory.registerSingleton(
+        beanPrefix + WorkerFactoryLifecycle.class.getSimpleName(),
+        new WorkerFactoryLifecycle(workerFactory));
   }
 
   @Override

temporal-spring-boot-autoconfigure/src/main/java/io/temporal/spring/boot/autoconfigure/NonRootNamespaceAutoConfiguration.java

@@ -23,7 +23,6 @@
 import org.springframework.context.annotation.Conditional;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.context.event.ApplicationContextEvent;
-import org.springframework.context.event.ContextClosedEvent;
 import org.springframework.context.event.ContextRefreshedEvent;
 
 @AutoConfiguration(
@@ -70,8 +69,6 @@ public void onApplicationEvent(ApplicationContextEvent event) {
         if (event instanceof ContextRefreshedEvent) {
           onStart();
         }
-      } else if (event instanceof ContextClosedEvent) {
-        onStop();
       }
     }
 
@@ -102,14 +99,6 @@ private void onStart() {
           });
     }
 
-    private void onStop() {
-      this.executeByNamespace(
-          (nonRootNamespaceProperties, workersTemplate) -> {
-            log.info("shutdown workers for non-root namespace");
-            workersTemplate.getWorkerFactory().shutdown();
-          });
-    }
-
     private void executeByNamespace(
         BiConsumer<NonRootNamespaceProperties, WorkersTemplate> consumer) {
       if (temporalProperties.getNamespaces() == null) {

temporal-spring-boot-autoconfigure/src/main/java/io/temporal/spring/boot/autoconfigure/RootNamespaceAutoConfiguration.java

@@ -14,6 +14,7 @@
 import io.temporal.serviceclient.WorkflowServiceStubs;
 import io.temporal.serviceclient.WorkflowServiceStubsPlugin;
 import io.temporal.spring.boot.TemporalOptionsCustomizer;
+import io.temporal.spring.boot.autoconfigure.internal.WorkerFactoryLifecycle;
 import io.temporal.spring.boot.autoconfigure.properties.TemporalProperties;
 import io.temporal.spring.boot.autoconfigure.template.ClientTemplate;
 import io.temporal.spring.boot.autoconfigure.template.NamespaceTemplate;
@@ -42,6 +43,7 @@
 import org.springframework.boot.context.event.ApplicationReadyEvent;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.ApplicationListener;
+import org.springframework.context.SmartLifecycle;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Conditional;
 import org.springframework.context.annotation.DependsOn;
@@ -202,13 +204,19 @@ public WorkersTemplate workersTemplate(
   }
 
   @Primary
-  @Bean(name = "temporalWorkerFactory", destroyMethod = "shutdown")
+  @Bean(name = "temporalWorkerFactory", destroyMethod = "")
   @Conditional(WorkersPresentCondition.class)
   public WorkerFactory workerFactory(
       @Qualifier("temporalWorkersTemplate") WorkersTemplate workersTemplate) {
     return workersTemplate.getWorkerFactory();
   }
 
+  @Bean
+  @Conditional(WorkersPresentCondition.class)
+  public SmartLifecycle temporalWorkerFactoryLifecycle(WorkerFactory workerFactory) {
+    return new WorkerFactoryLifecycle(workerFactory);
+  }
+
   @Primary
   @Bean(name = "temporalWorkers")
   @Conditional(WorkersPresentCondition.class)

temporal-spring-boot-autoconfigure/src/main/java/io/temporal/spring/boot/autoconfigure/internal/WorkerFactoryLifecycle.java

@@ -0,0 +1,67 @@
+package io.temporal.spring.boot.autoconfigure.internal;
+
+import io.temporal.worker.WorkerFactory;
+import java.util.concurrent.TimeUnit;
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.context.SmartLifecycle;
+
+/** Drains workers before Spring destroys their dependencies. */
+public final class WorkerFactoryLifecycle implements SmartLifecycle, DisposableBean {
+  private final WorkerFactory workerFactory;
+
+  public WorkerFactoryLifecycle(WorkerFactory workerFactory) {
+    this.workerFactory = workerFactory;
+  }
+
+  @Override
+  public boolean isAutoStartup() {
+    return false;
+  }
+
+  @Override
+  public void start() {
+    // Worker startup is controlled by the existing application event listeners.
+  }
+
+  @Override
+  public boolean isRunning() {
+    return !workerFactory.isTerminated();
+  }
+
+  @Override
+  public void stop() {
+    shutdown();
+    workerFactory.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+  }
+
+  @Override
+  public void stop(Runnable callback) {
+    shutdown();
+    Thread waiter =
+        new Thread(
+            () -> {
+              try {
+                workerFactory.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+              } finally {
+                callback.run();
+              }
+            },
+            "temporal-worker-shutdown-"
+                + workerFactory.getWorkflowClient().getOptions().getNamespace());
+    // Spring's lifecycle processor bounds the wait with its shutdown phase timeout.
+    waiter.setDaemon(true);
+    waiter.start();
+  }
+
+  @Override
+  public void destroy() {
+    // Bean destruction also runs when context initialization fails before lifecycle shutdown.
+    shutdown();
+  }
+
+  private void shutdown() {
+    if (!workerFactory.isShutdown()) {
+      workerFactory.shutdown();
+    }
+  }
+}

temporal-spring-boot-autoconfigure/src/test/java/io/temporal/spring/boot/autoconfigure/WorkerFactoryLifecycleTest.java

@@ -0,0 +1,107 @@
+package io.temporal.spring.boot.autoconfigure;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+import io.temporal.client.WorkflowClient;
+import io.temporal.client.WorkflowClientOptions;
+import io.temporal.spring.boot.autoconfigure.internal.WorkerFactoryLifecycle;
+import io.temporal.worker.WorkerFactory;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.springframework.beans.BeansException;
+import org.springframework.context.support.DefaultLifecycleProcessor;
+import org.springframework.context.support.GenericApplicationContext;
+
+@Timeout(10)
+class WorkerFactoryLifecycleTest {
+  @Test
+  void boundsContextCloseWithoutInterruptingTasksOrStartingWorkers() throws Exception {
+    WorkerFactory factory = mock(WorkerFactory.class);
+    WorkflowClient client = mock(WorkflowClient.class);
+    when(factory.getWorkflowClient()).thenReturn(client);
+    when(client.getOptions()).thenReturn(WorkflowClientOptions.getDefaultInstance());
+    CountDownLatch awaitingTermination = new CountDownLatch(1);
+    CountDownLatch release = new CountDownLatch(1);
+    CountDownLatch finished = new CountDownLatch(1);
+    AtomicBoolean interrupted = new AtomicBoolean();
+    doAnswer(
+            invocation -> {
+              awaitingTermination.countDown();
+              try {
+                release.await();
+              } catch (InterruptedException e) {
+                interrupted.set(true);
+              } finally {
+                finished.countDown();
+              }
+              return null;
+            })
+        .when(factory)
+        .awaitTermination(anyLong(), any());
+    try (GenericApplicationContext context = new GenericApplicationContext()) {
+      DefaultLifecycleProcessor processor = new DefaultLifecycleProcessor();
+      processor.setBeanFactory(context.getBeanFactory());
+      processor.setTimeoutPerShutdownPhase(100);
+      context.getBeanFactory().registerSingleton("lifecycleProcessor", processor);
+      context
+          .getBeanFactory()
+          .registerSingleton("workerLifecycle", new WorkerFactoryLifecycle(factory));
+      context.refresh();
+      verify(factory, never()).start();
+      context.close();
+      assertTrue(awaitingTermination.await(1, TimeUnit.SECONDS));
+      verify(factory).shutdown();
+      verify(factory, never()).shutdownNow();
+      assertEquals(1, finished.getCount());
+      assertFalse(interrupted.get());
+    } finally {
+      release.countDown();
+      if (awaitingTermination.getCount() == 0) {
+        assertTrue(finished.await(1, TimeUnit.SECONDS));
+      }
+    }
+  }
+
+  @Test
+  void awaitsShutdownAlreadyInitiatedByApplication() {
+    WorkerFactory factory = mock(WorkerFactory.class);
+    WorkflowClient client = mock(WorkflowClient.class);
+    when(factory.getWorkflowClient()).thenReturn(client);
+    when(client.getOptions()).thenReturn(WorkflowClientOptions.getDefaultInstance());
+    when(factory.isShutdown()).thenReturn(true);
+    try (GenericApplicationContext context = new GenericApplicationContext()) {
+      context
+          .getBeanFactory()
+          .registerSingleton("workerLifecycle", new WorkerFactoryLifecycle(factory));
+      context.refresh();
+      context.close();
+      verify(factory, never()).shutdown();
+      verify(factory).awaitTermination(anyLong(), any());
+    }
+  }
+
+  @Test
+  void shutsDownWorkersWhenContextInitializationFails() {
+    WorkerFactory factory = mock(WorkerFactory.class);
+    try (GenericApplicationContext context = new GenericApplicationContext()) {
+      context.registerBean(
+          "workerLifecycle",
+          WorkerFactoryLifecycle.class,
+          () -> new WorkerFactoryLifecycle(factory));
+      context.registerBean(
+          "broken",
+          Object.class,
+          () -> {
+            throw new IllegalStateException("Context initialization failed.");
+          },
+          definition -> definition.setDependsOn("workerLifecycle"));
+      assertThrows(BeansException.class, context::refresh);
+      verify(factory).shutdown();
+      verify(factory, never()).shutdownNow();
+    }
+  }
+}