From 9e5c6871091558f68104ea23d4a71abfc7533be5 Mon Sep 17 00:00:00 2001 From: Quinn Klassen Date: Thu, 22 Oct 2026 14:58:40 -0700 Subject: [PATCH] Fix skipped mutable side effect replay recovery Resolve pending mutable side effect invocations at the original workflow-task boundary without changing legacy replay timing. Gate recovery behind a first-release-disabled SDK flag and register opt-in use at the mutableSideEffect call site. Add cache-eviction, callback-time, and nondeterminism detection replay coverage for issue #3163. --- CHANGELOG.md | 3 + .../io/temporal/internal/common/SdkFlag.java | 16 + .../MutableSideEffectStateMachine.java | 24 +- .../statemachines/WorkflowStateMachines.java | 60 ++- .../MutableSideEffectStateMachineTest.java | 183 ++++++++ .../workflow/MutableSideEffectReplayTest.java | 297 +++++++++++++ .../mutableSideEffectCallbackTime.json | 27 ++ .../mutableSideEffectCommandlessTasks.json | 391 ++++++++++++++++++ 8 files changed, 989 insertions(+), 12 deletions(-) create mode 100644 temporal-sdk/src/test/java/io/temporal/workflow/MutableSideEffectReplayTest.java create mode 100644 temporal-sdk/src/test/resources/mutableSideEffectCallbackTime.json create mode 100644 temporal-sdk/src/test/resources/mutableSideEffectCommandlessTasks.json diff --git a/CHANGELOG.md b/CHANGELOG.md index d426a29297..808fa18aec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,9 @@ to docs, or any other relevant information. specific to the Temporal CLI dev server implementation and may no longer be valid if that implementation changes. ### Fixed +- Skipped `Workflow.mutableSideEffect` calls no longer leave invalid marker commands after cache + eviction or a worker restart when the new recovery flag is enabled. The flag remains disabled + by default in this release to preserve replay compatibility during SDK rollout. - `ActivityClient` now sends the `temporal-namespace` gRPC header like `WorkflowClient` and `ScheduleClient` do, so standalone activity calls are no longer rejected with `PERMISSION_DENIED` by namespace-scoped auth (e.g. Temporal Cloud API keys). diff --git a/temporal-sdk/src/main/java/io/temporal/internal/common/SdkFlag.java b/temporal-sdk/src/main/java/io/temporal/internal/common/SdkFlag.java index d7dbdd5780..b2a52648ec 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/common/SdkFlag.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/common/SdkFlag.java @@ -50,6 +50,22 @@ public enum SdkFlag { * Bug: https://github.com/temporalio/sdk-java/issues/2989 */ SCHEDULE_ASYNC_STUB_OPERATIONS(6), + /* + * Resolves skipped mutable-side-effect invocations after all command events in their replayed + * workflow task have been applied. An invocation still waiting for a marker at that point + * cancels its replay placeholder and returns the cached value. This lets its callback run at + * that task's workflow time, before the next task advances time or delivers signals. + * + * Also resolves pending replay placeholders at a live-task boundary, preventing empty + * RecordMarker commands from being sent to the server when no command event triggered cleanup. + * + * Introduced: 1.41.0 + * + * Enabled: (pending) + * + * Bug: https://github.com/temporalio/sdk-java/issues/3163 + */ + MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY(7), UNKNOWN(Integer.MAX_VALUE); private final int value; diff --git a/temporal-sdk/src/main/java/io/temporal/internal/statemachines/MutableSideEffectStateMachine.java b/temporal-sdk/src/main/java/io/temporal/internal/statemachines/MutableSideEffectStateMachine.java index 238924b27d..3c2c05fb9a 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/statemachines/MutableSideEffectStateMachine.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/statemachines/MutableSideEffectStateMachine.java @@ -27,6 +27,7 @@ final class MutableSideEffectStateMachine { private final String id; private UserMetadata metadata; private final Functions.Func replaying; + private final Functions.Func useReplayRecovery; private final Functions.Proc1 commandSink; private Optional result = Optional.empty(); @@ -130,6 +131,21 @@ State getExecutionState() { return replaying.apply() ? State.REPLAYING : State.EXECUTING; } + @Override + public void handleWorkflowTaskStarted() { + // Recover only when the flag is already enabled. Unflagged tasks retain the callback timing + // of the SDK that wrote them. + if (getState() == State.RESULT_NOTIFIED_REPLAYING && useReplayRecovery.apply()) { + notifySkippedReplaying(); + } + } + + void notifySkippedReplaying() { + if (getState() == State.RESULT_NOTIFIED_REPLAYING) { + explicitEvent(ExplicitEvent.NON_MATCHING_EVENT); + } + } + @Override public WorkflowStateMachines.HandleEventStatus handleEvent( HistoryEvent event, boolean hasNextEvent) { @@ -249,30 +265,34 @@ public static MutableSideEffectStateMachine newInstance( String id, UserMetadata metadata, Functions.Func replaying, + Functions.Func useReplayRecovery, Functions.Proc1 commandSink, Functions.Proc1 stateMachineSink) { return new MutableSideEffectStateMachine( - id, metadata, replaying, commandSink, stateMachineSink); + id, metadata, replaying, useReplayRecovery, commandSink, stateMachineSink); } private MutableSideEffectStateMachine( String id, UserMetadata metadata, Functions.Func replaying, + Functions.Func useReplayRecovery, Functions.Proc1 commandSink, Functions.Proc1 stateMachineSink) { this.id = Objects.requireNonNull(id); this.metadata = metadata; this.replaying = Objects.requireNonNull(replaying); + this.useReplayRecovery = Objects.requireNonNull(useReplayRecovery); this.commandSink = Objects.requireNonNull(commandSink); } - public void mutableSideEffect( + public InvocationStateMachine mutableSideEffect( Functions.Func1, Optional> func, Functions.Proc1> callback, Functions.Proc1 stateMachineSink) { InvocationStateMachine ism = new InvocationStateMachine(func, callback, stateMachineSink); ism.explicitEvent(ExplicitEvent.CHECK_EXECUTION_STATE); ism.explicitEvent(ExplicitEvent.SCHEDULE); + return ism; } } diff --git a/temporal-sdk/src/main/java/io/temporal/internal/statemachines/WorkflowStateMachines.java b/temporal-sdk/src/main/java/io/temporal/internal/statemachines/WorkflowStateMachines.java index 893dc3fe5a..5b7f48bc14 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/statemachines/WorkflowStateMachines.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/statemachines/WorkflowStateMachines.java @@ -163,6 +163,9 @@ enum HandleEventStatus { /** Key is mutable side effect id */ private final Map mutableSideEffects = new HashMap<>(); + private final Queue + replayingMutableSideEffects = new ArrayDeque<>(); + /** Key is changeId */ private final Map versions = new HashMap<>(); @@ -370,6 +373,27 @@ private void handleEventsBatch(WFTBuffer.EventBatch eventBatch, boolean hasNextB handleSingleMessage(msg); } } + + if (eventBatch.getWorkflowTaskCompletedEvent().isPresent()) { + completeMutableSideEffects(); + } + } + + private void completeMutableSideEffects() { + // Only mutable-side-effect invocations participate. A callback can create another invocation; + // process it in invocation order, at this task's time, before applying the next task. + while (!replayingMutableSideEffects.isEmpty()) { + MutableSideEffectStateMachine.InvocationStateMachine invocation = + replayingMutableSideEffects.peek(); + if (!invocation.isFinalState() + && !flags.checkSdkFlag(SdkFlag.MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY)) { + // Preserve the timing of an unflagged invocation until normal command matching or a live + // task recovers it and records the recovery flag. + return; + } + replayingMutableSideEffects.poll(); + invocation.notifySkippedReplaying(); + } } /** Handle an event when looking ahead at history during replay */ @@ -1225,20 +1249,35 @@ public void mutableSideEffect( Functions.Func1, Optional> func, Functions.Proc1> callback) { checkEventLoopExecuting(); + // Opt-in only for this flag's introductory release. Register its use at the API call, rather + // than waiting for a skipped invocation to need cleanup. + if (initialFlags.contains(SdkFlag.MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY)) { + flags.tryUseSdkFlag(SdkFlag.MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY); + } MutableSideEffectStateMachine stateMachine = mutableSideEffects.computeIfAbsent( id, (idKey) -> MutableSideEffectStateMachine.newInstance( - idKey, userMetadata, this::isReplaying, commandSink, stateMachineSink)); - stateMachine.mutableSideEffect( - func, - (r) -> { - callback.apply(r); - // callback unblocked mutableSideEffect call. Give workflow code chance to make progress. - eventLoop(); - }, - stateMachineSink); + idKey, + userMetadata, + this::isReplaying, + () -> flags.checkSdkFlag(SdkFlag.MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY), + commandSink, + stateMachineSink)); + MutableSideEffectStateMachine.InvocationStateMachine invocation = + stateMachine.mutableSideEffect( + func, + (r) -> { + callback.apply(r); + // callback unblocked mutableSideEffect call. Give workflow code chance to make + // progress. + eventLoop(); + }, + stateMachineSink); + if (isReplaying()) { + replayingMutableSideEffects.add(invocation); + } } public Integer getVersion( @@ -1524,7 +1563,8 @@ public void workflowTaskStarted( List suggestContinueAsNewReasons, boolean isTargetWorkerDeploymentVersionChanged) { setCurrentTimeMillis(currentTimeMillis); - for (CancellableCommand cancellableCommand : commands) { + // Notifying a skipped call can execute workflow code and append commands. + for (CancellableCommand cancellableCommand : new ArrayList<>(commands)) { cancellableCommand.handleWorkflowTaskStarted(); } // Give local activities a chance to recreate their requests if they were lost due diff --git a/temporal-sdk/src/test/java/io/temporal/internal/statemachines/MutableSideEffectStateMachineTest.java b/temporal-sdk/src/test/java/io/temporal/internal/statemachines/MutableSideEffectStateMachineTest.java index e65d900d09..40e4ef1ae7 100644 --- a/temporal-sdk/src/test/java/io/temporal/internal/statemachines/MutableSideEffectStateMachineTest.java +++ b/temporal-sdk/src/test/java/io/temporal/internal/statemachines/MutableSideEffectStateMachineTest.java @@ -6,6 +6,7 @@ import static io.temporal.internal.statemachines.MutableSideEffectStateMachine.MUTABLE_SIDE_EFFECT_MARKER_NAME; import static io.temporal.internal.statemachines.TestHistoryBuilder.assertCommand; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -18,9 +19,18 @@ import io.temporal.api.history.v1.HistoryEvent; import io.temporal.api.history.v1.MarkerRecordedEventAttributes; import io.temporal.api.history.v1.TimerFiredEventAttributes; +import io.temporal.api.history.v1.WorkflowExecutionSignaledEventAttributes; +import io.temporal.api.history.v1.WorkflowTaskCompletedEventAttributes; +import io.temporal.api.sdk.v1.WorkflowTaskCompletedMetadata; +import io.temporal.api.workflowservice.v1.GetSystemInfoResponse; import io.temporal.common.converter.DataConverter; import io.temporal.common.converter.DefaultDataConverter; +import io.temporal.internal.common.SdkFlag; +import io.temporal.worker.WorkflowImplementationOptions; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; import java.util.List; import java.util.Optional; import org.junit.AfterClass; @@ -196,6 +206,179 @@ protected void buildWorkflow(AsyncWorkflowBuilder builder) { } } + @Test + public void testSkippedInCommandlessTaskWithRecordedRecoveryFlag() { + List results = new ArrayList<>(); + boolean[] live = {true}; + class TestListener extends TestEntityManagerListenerBase { + @Override + protected void buildWorkflow(AsyncWorkflowBuilder builder) { + readValue(builder); + } + + @Override + protected void signal(HistoryEvent event, AsyncWorkflowBuilder builder) { + if (event.getWorkflowExecutionSignaledEventAttributes().getSignalName().equals("skip")) { + readValue(builder); + } else { + builder.add(v -> stateMachines.completeWorkflow(Optional.empty())); + } + } + + private void readValue(AsyncWorkflowBuilder builder) { + builder + .>add1( + (v, callback) -> + stateMachines.mutableSideEffect( + "id1", + null, + previous -> { + assertTrue( + "Replayed mutable side effect must not execute its function.", + live[0]); + return previous.isPresent() + ? Optional.empty() + : converter.toPayloads("value"); + }, + callback)) + .add( + result -> + results.add(converter.fromPayloads(0, result, String.class, String.class))); + } + } + TestHistoryBuilder history = + new TestHistoryBuilder() + .add(EventType.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED) + .addWorkflowTaskScheduledAndStarted() + .add( + EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED, + WorkflowTaskCompletedEventAttributes.newBuilder() + .setScheduledEventId(2) + .setStartedEventId(3) + .setSdkMetadata( + WorkflowTaskCompletedMetadata.newBuilder() + .addLangUsedFlags( + SdkFlag.MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY.getValue())) + .build()) + .add( + EventType.EVENT_TYPE_MARKER_RECORDED, + MarkerRecordedEventAttributes.newBuilder() + .setMarkerName(MUTABLE_SIDE_EFFECT_MARKER_NAME) + .putDetails(MARKER_ID_KEY, converter.toPayloads("id1").get()) + .putDetails(MARKER_DATA_KEY, converter.toPayloads("value").get()) + .putDetails(MARKER_SKIP_COUNT_KEY, converter.toPayloads(0).get()) + .build()) + .add( + EventType.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED, + WorkflowExecutionSignaledEventAttributes.newBuilder().setSignalName("skip").build()) + .addWorkflowTask() + .add( + EventType.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED, + WorkflowExecutionSignaledEventAttributes.newBuilder() + .setSignalName("finish") + .build()) + .addWorkflowTask() + .add(EventType.EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED); + + stateMachines = newStateMachines(new TestListener()); + assertCommand( + CommandType.COMMAND_TYPE_RECORD_MARKER, + history.handleWorkflowTaskTakeCommands(stateMachines, 1)); + assertEquals(Arrays.asList("value"), results); + assertTrue(history.handleWorkflowTaskTakeCommands(stateMachines, 2).isEmpty()); + assertEquals(Arrays.asList("value", "value"), results); + assertTrue( + "Reading a recorded flag must not activate a new flag", + stateMachines.takeNewSdkFlags().isEmpty()); + + // A fresh worker replays the commandless task before executing the final signal. + live[0] = false; + results.clear(); + stateMachines = newStateMachines(new TestListener()); + assertCommand( + CommandType.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + history.handleWorkflowTaskTakeCommands(stateMachines, 3)); + assertEquals(Arrays.asList("value", "value"), results); + + // The same history must also replay to completion without emitting commands. + results.clear(); + stateMachines = newStateMachines(new TestListener()); + assertTrue(history.handleWorkflowTaskTakeCommands(stateMachines).isEmpty()); + assertEquals(Arrays.asList("value", "value"), results); + } + + @Test + public void testRecoveryFlagIsRecordedAtMutableSideEffectCall() { + class TestListener extends TestEntityManagerListenerBase { + @Override + protected void buildWorkflow(AsyncWorkflowBuilder builder) {} + + @Override + protected void signal(HistoryEvent event, AsyncWorkflowBuilder builder) { + builder + .>add1( + (v, callback) -> + stateMachines.mutableSideEffect( + "id1", null, previous -> converter.toPayloads("value"), callback)) + .add(stateMachines::completeWorkflow); + } + } + TestHistoryBuilder history = + new TestHistoryBuilder() + .add(EventType.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED) + .addWorkflowTask() + .add( + EventType.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED, + WorkflowExecutionSignaledEventAttributes.newBuilder().setSignalName("use").build()) + .addWorkflowTaskScheduledAndStarted() + .add( + EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED, + WorkflowTaskCompletedEventAttributes.newBuilder() + .setScheduledEventId(6) + .setStartedEventId(7) + .setSdkMetadata( + WorkflowTaskCompletedMetadata.newBuilder() + .addLangUsedFlags( + SdkFlag.MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY.getValue())) + .build()) + .add( + EventType.EVENT_TYPE_MARKER_RECORDED, + MarkerRecordedEventAttributes.newBuilder() + .setMarkerName(MUTABLE_SIDE_EFFECT_MARKER_NAME) + .putDetails(MARKER_ID_KEY, converter.toPayloads("id1").get()) + .putDetails(MARKER_DATA_KEY, converter.toPayloads("value").get()) + .putDetails(MARKER_SKIP_COUNT_KEY, converter.toPayloads(0).get()) + .build()) + .add(EventType.EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED); + stateMachines = + new WorkflowStateMachines( + new TestListener(), + stateMachineList::add, + GetSystemInfoResponse.Capabilities.newBuilder().setSdkMetadata(true).build(), + WorkflowImplementationOptions.newBuilder().build()); + assertTrue(history.handleWorkflowTaskTakeCommands(stateMachines, 1).isEmpty()); + assertFalse( + stateMachines.takeNewSdkFlags().contains(SdkFlag.MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY)); + + List savedInitialFlags = WorkflowStateMachines.initialFlags; + List enabledFlags = new ArrayList<>(savedInitialFlags); + enabledFlags.add(SdkFlag.MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY); + WorkflowStateMachines.initialFlags = Collections.unmodifiableList(enabledFlags); + try { + List commands = history.handleWorkflowTaskTakeCommands(stateMachines, 2); + assertEquals(2, commands.size()); + assertEquals(CommandType.COMMAND_TYPE_RECORD_MARKER, commands.get(0).getCommandType()); + assertEquals( + CommandType.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, commands.get(1).getCommandType()); + assertEquals( + EnumSet.of(SdkFlag.MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY), stateMachines.takeNewSdkFlags()); + } finally { + WorkflowStateMachines.initialFlags = savedInitialFlags; + } + stateMachines = newStateMachines(new TestListener()); + assertTrue(history.handleWorkflowTaskTakeCommands(stateMachines).isEmpty()); + } + @Test public void testRecordAcrossMultipleWorkflowTasks() { class TestListener extends TestEntityManagerListenerBase { diff --git a/temporal-sdk/src/test/java/io/temporal/workflow/MutableSideEffectReplayTest.java b/temporal-sdk/src/test/java/io/temporal/workflow/MutableSideEffectReplayTest.java new file mode 100644 index 0000000000..87da43db67 --- /dev/null +++ b/temporal-sdk/src/test/java/io/temporal/workflow/MutableSideEffectReplayTest.java @@ -0,0 +1,297 @@ +package io.temporal.workflow; + +import static org.junit.Assert.*; + +import com.google.common.io.Resources; +import io.temporal.api.enums.v1.EventType; +import io.temporal.api.history.v1.History; +import io.temporal.api.history.v1.HistoryEvent; +import io.temporal.api.history.v1.WorkflowTaskCompletedEventAttributes; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowStub; +import io.temporal.common.WorkflowExecutionHistory; +import io.temporal.internal.common.SdkFlag; +import io.temporal.internal.statemachines.WorkflowStateMachines; +import io.temporal.testing.WorkflowReplayer; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.junit.Rule; +import org.junit.Test; + +public class MutableSideEffectReplayTest { + @Rule + public SDKTestWorkflowRule testWorkflowRule = + SDKTestWorkflowRule.newBuilder() + .setWorkflowTypes(SkippedMutableSideEffectWorkflowImpl.class) + .build(); + + @Test + public void skippedMutableSideEffectSurvivesFullReplay() throws Exception { + List savedInitialFlags = WorkflowStateMachines.initialFlags; + List enabledFlags = new ArrayList<>(savedInitialFlags); + enabledFlags.add(SdkFlag.MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY); + WorkflowStateMachines.initialFlags = Collections.unmodifiableList(enabledFlags); + try { + WorkflowReplayer.replayWorkflowExecution( + runWorkflow(true), SkippedMutableSideEffectWorkflowImpl.class); + } finally { + WorkflowStateMachines.initialFlags = savedInitialFlags; + } + } + + @Test + public void replayHistoryWrittenWithoutEviction() throws Exception { + assertFalse( + "The recovery flag must remain disabled by default in its first release", + WorkflowStateMachines.initialFlags.contains(SdkFlag.MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY)); + WorkflowExecutionHistory history = runWorkflow(false); + WorkflowReplayer.replayWorkflowExecution(history, SkippedMutableSideEffectWorkflowImpl.class); + testWorkflowRule.regenerateHistoryForReplay( + history.getWorkflowExecution().getWorkflowId(), "mutableSideEffectCommandlessTasks"); + } + + @Test + public void replayHistoryWrittenBeforeFix() throws Exception { + // Captured on the unmodified SDK, with both commandless tasks executed from the sticky cache. + WorkflowReplayer.replayWorkflowExecutionFromResource( + "mutableSideEffectCommandlessTasks.json", SkippedMutableSideEffectWorkflowImpl.class); + } + + @Test + public void replayPreservesClockAfterSkippedMutableSideEffect() throws Exception { + // This test-server history has controlled, monotonic task timestamps to distinguish callback + // timing without wall-clock waits. + // The recorded execution resumes the skipped call at base + 1000. Legacy replay resumes it + // at base + 3000, while unconditional task-start cleanup resumes it at base + 2000. + // Both the recorded execution and legacy replay emit two timers; the intermediate time + // changes the branch and omits one timer. No skip-count matching is involved. + WorkflowReplayer.replayWorkflowExecutionFromResource( + "mutableSideEffectCallbackTime.json", ClockSensitiveWorkflowImpl.class); + } + + @Test + public void replayUsesOriginalClockWhenRecoveryFlagIsAlreadyPresent() throws Exception { + // With the flag present from the start, the skipped continuation must run at its original + // task time (+1000), not the next task's time (+2000). + WorkflowReplayer.replayWorkflowExecution( + new WorkflowExecutionHistory(clockHistoryWithRecoveryFlag(4).build()), + ClockSensitiveWorkflowImpl.class); + } + + @Test + public void replayRepeatsRecordedRecoveryBoundary() throws Exception { + // Recovery was first recorded at +2000. Its live callback took the branch that omits the + // second timer, so persist that decision and verify replay makes the same choice. + History.Builder recovered = clockHistoryWithRecoveryFlag(16); + recovered.removeEvents(21); + recovered.setEvents(21, recovered.getEvents(21).toBuilder().setEventId(22)); + WorkflowReplayer.replayWorkflowExecution( + new WorkflowExecutionHistory(recovered.build()), ClockSensitiveWorkflowImpl.class); + } + + @Test + public void recoveryDoesNotHideAnExtraTimerCommand() throws Exception { + RuntimeException error = + assertThrows( + RuntimeException.class, + () -> + WorkflowReplayer.replayWorkflowExecution( + new WorkflowExecutionHistory(clockHistoryWithRecoveryFlag(4).build()), + ExtraTimerWorkflowImpl.class)); + assertTrue(error.getMessage(), error.getMessage().contains("[TMPRL1100]")); + } + + @Test + public void recoveryDoesNotIgnoreARecordedMutableSideEffectMarker() throws Exception { + RuntimeException error = + assertThrows( + RuntimeException.class, + () -> + WorkflowReplayer.replayWorkflowExecution( + new WorkflowExecutionHistory(clockHistoryWithRecoveryFlag(4).build()), + NoMutableSideEffectWorkflowImpl.class)); + assertTrue(error.getMessage(), error.getMessage().contains("[TMPRL1100]")); + } + + private History.Builder clockHistoryWithRecoveryFlag(int completedEventId) throws Exception { + WorkflowExecutionHistory history = + WorkflowExecutionHistory.fromJson( + Resources.toString( + Resources.getResource("mutableSideEffectCallbackTime.json"), + StandardCharsets.UTF_8)); + History.Builder flagged = history.getHistory().toBuilder(); + int completedIndex = completedEventId - 1; + HistoryEvent completed = flagged.getEvents(completedIndex); + assertEquals(EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED, completed.getEventType()); + WorkflowTaskCompletedEventAttributes attributes = + completed.getWorkflowTaskCompletedEventAttributes(); + flagged.setEvents( + completedIndex, + completed.toBuilder() + .setWorkflowTaskCompletedEventAttributes( + attributes.toBuilder() + .setSdkMetadata( + attributes.getSdkMetadata().toBuilder() + .addLangUsedFlags( + SdkFlag.MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY.getValue())))); + return flagged; + } + + private WorkflowExecutionHistory runWorkflow(boolean evict) throws Exception { + SkippedMutableSideEffectWorkflow workflow = + testWorkflowRule.newWorkflowStubTimeoutOptions(SkippedMutableSideEffectWorkflow.class); + WorkflowClient.start(workflow::run); + assertEquals("1", workflow.getValues()); + + workflow.setValue(1); + // A strongly consistent query waits for the preceding signal's task to complete, without + // adding a command event that could consume the skipped call's fake marker during replay. + assertEquals("1,1", workflow.getValues()); + if (evict) { + testWorkflowRule.invalidateWorkflowCache(); + } + + workflow.setValue(2); + assertEquals("1,1,2", workflow.getValues()); + + workflow.setValue(2); + assertEquals("1,1,2,2", workflow.getValues()); + if (evict) { + testWorkflowRule.invalidateWorkflowCache(); + } + + workflow.setValue(3); + assertEquals( + "1,1,2,2", WorkflowStub.fromTyped(workflow).getResult(10, TimeUnit.SECONDS, String.class)); + + WorkflowExecutionHistory history = + testWorkflowRule.getExecutionHistory( + WorkflowStub.fromTyped(workflow).getExecution().getWorkflowId()); + assertEquals( + 2, + history.getEvents().stream() + .filter(event -> event.hasMarkerRecordedEventAttributes()) + .count()); + assertEquals( + "The recovery flag must only be recorded after explicit test opt-in", + evict, + history.getEvents().stream() + .filter(HistoryEvent::hasWorkflowTaskCompletedEventAttributes) + .anyMatch( + event -> + event + .getWorkflowTaskCompletedEventAttributes() + .getSdkMetadata() + .getLangUsedFlagsList() + .contains(SdkFlag.MUTABLE_SIDE_EFFECT_REPLAY_RECOVERY.getValue()))); + SDKTestWorkflowRule.assertNoHistoryEvent( + history.getHistory(), EventType.EVENT_TYPE_WORKFLOW_TASK_FAILED); + SDKTestWorkflowRule.assertNoHistoryEvent( + history.getHistory(), EventType.EVENT_TYPE_WORKFLOW_TASK_TIMED_OUT); + return history; + } + + @WorkflowInterface + public interface SkippedMutableSideEffectWorkflow { + @WorkflowMethod + String run(); + + @SignalMethod + void setValue(int value); + + @QueryMethod + String getValues(); + } + + public static class SkippedMutableSideEffectWorkflowImpl + implements SkippedMutableSideEffectWorkflow { + private int value = 1; + private int signals; + private final StringBuilder result = new StringBuilder(); + + @Override + public String run() { + for (int i = 0; i < 4; i++) { + int current = + Workflow.mutableSideEffect("value", Integer.class, (a, b) -> !a.equals(b), () -> value); + if (i > 0) { + result.append(","); + } + result.append(current); + int iteration = i; + Workflow.await(() -> signals > iteration); + } + return result.toString(); + } + + @Override + public void setValue(int value) { + this.value = value; + signals++; + } + + @Override + public String getValues() { + return result.toString(); + } + } + + @WorkflowInterface + public interface ClockSensitiveWorkflow { + @WorkflowMethod(name = "Probe") + String run(); + + @SignalMethod + void poke(); + } + + public static class ClockSensitiveWorkflowImpl implements ClockSensitiveWorkflow { + private static final long INTERMEDIATE_TASK_TIME = 1800000002000L; + private int signals; + + @Override + public String run() { + Workflow.mutableSideEffect("key", Integer.class, (a, b) -> !a.equals(b), () -> 1); + Workflow.await(() -> signals >= 1); + Workflow.mutableSideEffect("key", Integer.class, (a, b) -> !a.equals(b), () -> 1); + long observed = Workflow.currentTimeMillis(); + Workflow.await(() -> signals >= 3); + if (observed != INTERMEDIATE_TASK_TIME) { + Workflow.newTimer(Duration.ofSeconds(10)); + } + return "done"; + } + + @Override + public void poke() { + signals++; + if (signals == 3) { + Workflow.newTimer(Duration.ofSeconds(30)); + } + } + } + + public static class ExtraTimerWorkflowImpl extends ClockSensitiveWorkflowImpl { + @Override + public String run() { + String result = super.run(); + Workflow.newTimer(Duration.ofSeconds(10)); + return result; + } + } + + public static class NoMutableSideEffectWorkflowImpl implements ClockSensitiveWorkflow { + @Override + public String run() { + return "done"; + } + + @Override + public void poke() {} + } +} diff --git a/temporal-sdk/src/test/resources/mutableSideEffectCallbackTime.json b/temporal-sdk/src/test/resources/mutableSideEffectCallbackTime.json new file mode 100644 index 0000000000..ac31f14bc2 --- /dev/null +++ b/temporal-sdk/src/test/resources/mutableSideEffectCallbackTime.json @@ -0,0 +1,27 @@ +{ + "events": [ + {"eventId":"1","eventTime":"2027-01-15T08:00:00.500Z","eventType":"EVENT_TYPE_WORKFLOW_EXECUTION_STARTED","workflowExecutionStartedEventAttributes":{"workflowType":{"name":"Probe"},"taskQueue":{"name":"callback-time-replay"},"input":{},"workflowExecutionTimeout":"315360000s","workflowRunTimeout":"200s","workflowTaskTimeout":"5s","originalExecutionRunId":"68fac5c9-f787-43f1-8ac9-939531b5ed97","firstExecutionRunId":"68fac5c9-f787-43f1-8ac9-939531b5ed97","attempt":1}}, + {"eventId":"2","eventTime":"2027-01-15T08:00:00.500Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_SCHEDULED","workflowTaskScheduledEventAttributes":{"taskQueue":{"name":"callback-time-replay"},"startToCloseTimeout":"5s","attempt":1}}, + {"eventId":"3","eventTime":"2027-01-15T08:00:00.500Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_STARTED","workflowTaskStartedEventAttributes":{"scheduledEventId":"2"}}, + {"eventId":"4","eventTime":"2027-01-15T08:00:00.500Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_COMPLETED","workflowTaskCompletedEventAttributes":{"scheduledEventId":"2","startedEventId":"3","sdkMetadata":{"langUsedFlags":[1,3],"sdkName":"temporal-java","sdkVersion":"1.41.0"}}}, + {"eventId":"5","eventTime":"2027-01-15T08:00:00.500Z","eventType":"EVENT_TYPE_MARKER_RECORDED","markerRecordedEventAttributes":{"markerName":"MutableSideEffect","details":{"data":{"payloads":[{"metadata":{"encoding":"anNvbi9wbGFpbg=="},"data":"MQ=="}]},"id":{"payloads":[{"metadata":{"encoding":"anNvbi9wbGFpbg=="},"data":"ImtleSI="}]},"skipCount":{"payloads":[{"metadata":{"encoding":"anNvbi9wbGFpbg=="},"data":"MA=="}]}},"workflowTaskCompletedEventId":"4"}}, + {"eventId":"6","eventTime":"2027-01-15T08:00:00.500Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_SCHEDULED","workflowTaskScheduledEventAttributes":{"taskQueue":{"name":"callback-time-replay"},"startToCloseTimeout":"5s","attempt":1}}, + {"eventId":"7","eventTime":"2027-01-15T08:00:00.500Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_STARTED","workflowTaskStartedEventAttributes":{"scheduledEventId":"6"}}, + {"eventId":"8","eventTime":"2027-01-15T08:00:00.500Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_COMPLETED","workflowTaskCompletedEventAttributes":{"scheduledEventId":"6","startedEventId":"7","sdkMetadata":{"sdkName":"temporal-java","sdkVersion":"1.41.0"}}}, + {"eventId":"9","eventTime":"2027-01-15T08:00:01.000Z","eventType":"EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED","workflowExecutionSignaledEventAttributes":{"signalName":"poke","input":{}}}, + {"eventId":"10","eventTime":"2027-01-15T08:00:01.000Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_SCHEDULED","workflowTaskScheduledEventAttributes":{"taskQueue":{"name":"callback-time-replay"},"startToCloseTimeout":"5s","attempt":1}}, + {"eventId":"11","eventTime":"2027-01-15T08:00:01.000Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_STARTED","workflowTaskStartedEventAttributes":{"scheduledEventId":"10"}}, + {"eventId":"12","eventTime":"2027-01-15T08:00:01.000Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_COMPLETED","workflowTaskCompletedEventAttributes":{"scheduledEventId":"10","startedEventId":"11","sdkMetadata":{"sdkName":"temporal-java","sdkVersion":"1.41.0"}}}, + {"eventId":"13","eventTime":"2027-01-15T08:00:02.000Z","eventType":"EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED","workflowExecutionSignaledEventAttributes":{"signalName":"poke","input":{}}}, + {"eventId":"14","eventTime":"2027-01-15T08:00:02.000Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_SCHEDULED","workflowTaskScheduledEventAttributes":{"taskQueue":{"name":"callback-time-replay"},"startToCloseTimeout":"5s","attempt":1}}, + {"eventId":"15","eventTime":"2027-01-15T08:00:02.000Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_STARTED","workflowTaskStartedEventAttributes":{"scheduledEventId":"14"}}, + {"eventId":"16","eventTime":"2027-01-15T08:00:02.000Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_COMPLETED","workflowTaskCompletedEventAttributes":{"scheduledEventId":"14","startedEventId":"15","sdkMetadata":{"sdkName":"temporal-java","sdkVersion":"1.41.0"}}}, + {"eventId":"17","eventTime":"2027-01-15T08:00:03.000Z","eventType":"EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED","workflowExecutionSignaledEventAttributes":{"signalName":"poke","input":{}}}, + {"eventId":"18","eventTime":"2027-01-15T08:00:03.000Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_SCHEDULED","workflowTaskScheduledEventAttributes":{"taskQueue":{"name":"callback-time-replay"},"startToCloseTimeout":"5s","attempt":1}}, + {"eventId":"19","eventTime":"2027-01-15T08:00:03.000Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_STARTED","workflowTaskStartedEventAttributes":{"scheduledEventId":"18"}}, + {"eventId":"20","eventTime":"2027-01-15T08:00:03.000Z","eventType":"EVENT_TYPE_WORKFLOW_TASK_COMPLETED","workflowTaskCompletedEventAttributes":{"scheduledEventId":"18","startedEventId":"19","sdkMetadata":{"sdkName":"temporal-java","sdkVersion":"1.41.0"}}}, + {"eventId":"21","eventTime":"2027-01-15T08:00:03.000Z","eventType":"EVENT_TYPE_TIMER_STARTED","timerStartedEventAttributes":{"timerId":"d63e6b51-e1a5-38a6-be64-4f97ff32c740","startToFireTimeout":"30s","workflowTaskCompletedEventId":"20"}}, + {"eventId":"22","eventTime":"2027-01-15T08:00:03.000Z","eventType":"EVENT_TYPE_TIMER_STARTED","timerStartedEventAttributes":{"timerId":"fa05f94e-315e-3119-8255-c9f48bbf2cd7","startToFireTimeout":"10s","workflowTaskCompletedEventId":"20"}}, + {"eventId":"23","eventTime":"2027-01-15T08:00:03.000Z","eventType":"EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED","workflowExecutionCompletedEventAttributes":{"result":{"payloads":[{"metadata":{"encoding":"anNvbi9wbGFpbg=="},"data":"ImRvbmUi"}]},"workflowTaskCompletedEventId":"20"}} + ] +} diff --git a/temporal-sdk/src/test/resources/mutableSideEffectCommandlessTasks.json b/temporal-sdk/src/test/resources/mutableSideEffectCommandlessTasks.json new file mode 100644 index 0000000000..884be692ca --- /dev/null +++ b/temporal-sdk/src/test/resources/mutableSideEffectCommandlessTasks.json @@ -0,0 +1,391 @@ +{ + "events": [ + { + "eventId": "1", + "eventTime": "2026-10-22T00:42:35.586Z", + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_STARTED", + "workflowExecutionStartedEventAttributes": { + "workflowType": { + "name": "SkippedMutableSideEffectWorkflow" + }, + "taskQueue": { + "name": "WorkflowTest-replayHistoryWrittenWithoutEviction-251bd7da-2289-4e0f-bca6-2ebc65bdf048" + }, + "input": {}, + "workflowExecutionTimeout": "315360000s", + "workflowRunTimeout": "200s", + "workflowTaskTimeout": "5s", + "originalExecutionRunId": "36afea4a-442a-483c-948e-c1d5b04f3b35", + "identity": "65249@MacBook-Pro.local", + "firstExecutionRunId": "36afea4a-442a-483c-948e-c1d5b04f3b35", + "attempt": 1, + "firstWorkflowTaskBackoff": "0s", + "header": {} + } + }, + { + "eventId": "2", + "eventTime": "2026-10-22T00:42:35.586Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "WorkflowTest-replayHistoryWrittenWithoutEviction-251bd7da-2289-4e0f-bca6-2ebc65bdf048" + }, + "startToCloseTimeout": "5s", + "attempt": 1 + } + }, + { + "eventId": "3", + "eventTime": "2026-10-22T00:42:35.586Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "2", + "identity": "65249@MacBook-Pro.local", + "historySizeBytes": "391" + } + }, + { + "eventId": "4", + "eventTime": "2026-10-22T00:42:35.588Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "2", + "identity": "65249@MacBook-Pro.local", + "sdkMetadata": { + "langUsedFlags": [ + 1, + 3 + ], + "sdkName": "temporal-java", + "sdkVersion": "1.41.0" + }, + "meteringMetadata": {} + } + }, + { + "eventId": "5", + "eventTime": "2026-10-22T00:42:35.588Z", + "eventType": "EVENT_TYPE_MARKER_RECORDED", + "markerRecordedEventAttributes": { + "markerName": "MutableSideEffect", + "details": { + "data": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg\u003d\u003d" + }, + "data": "MQ\u003d\u003d" + } + ] + }, + "id": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg\u003d\u003d" + }, + "data": "InZhbHVlIg\u003d\u003d" + } + ] + }, + "skipCount": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg\u003d\u003d" + }, + "data": "MA\u003d\u003d" + } + ] + } + }, + "workflowTaskCompletedEventId": "3" + } + }, + { + "eventId": "6", + "eventTime": "2026-10-22T00:42:35.588Z", + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED", + "workflowExecutionSignaledEventAttributes": { + "signalName": "setValue", + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg\u003d\u003d" + }, + "data": "MQ\u003d\u003d" + } + ] + }, + "identity": "65249@MacBook-Pro.local", + "header": {} + } + }, + { + "eventId": "7", + "eventTime": "2026-10-22T00:42:35.588Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "WorkflowTest-replayHistoryWrittenWithoutEviction-251bd7da-2289-4e0f-bca6-2ebc65bdf048" + }, + "startToCloseTimeout": "5s", + "attempt": 1 + } + }, + { + "eventId": "8", + "eventTime": "2026-10-22T00:42:35.588Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "7", + "identity": "65249@MacBook-Pro.local", + "historySizeBytes": "892" + } + }, + { + "eventId": "9", + "eventTime": "2026-10-22T00:42:35.589Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "7", + "identity": "65249@MacBook-Pro.local", + "sdkMetadata": { + "sdkName": "temporal-java", + "sdkVersion": "1.41.0" + }, + "meteringMetadata": {} + } + }, + { + "eventId": "10", + "eventTime": "2026-10-22T00:42:35.590Z", + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED", + "workflowExecutionSignaledEventAttributes": { + "signalName": "setValue", + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg\u003d\u003d" + }, + "data": "Mg\u003d\u003d" + } + ] + }, + "identity": "65249@MacBook-Pro.local", + "header": {} + } + }, + { + "eventId": "11", + "eventTime": "2026-10-22T00:42:35.590Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "WorkflowTest-replayHistoryWrittenWithoutEviction-251bd7da-2289-4e0f-bca6-2ebc65bdf048" + }, + "startToCloseTimeout": "5s", + "attempt": 1 + } + }, + { + "eventId": "12", + "eventTime": "2026-10-22T00:42:35.590Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "11", + "identity": "65249@MacBook-Pro.local", + "historySizeBytes": "1220" + } + }, + { + "eventId": "13", + "eventTime": "2026-10-22T00:42:35.591Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "11", + "identity": "65249@MacBook-Pro.local", + "sdkMetadata": { + "sdkName": "temporal-java", + "sdkVersion": "1.41.0" + }, + "meteringMetadata": {} + } + }, + { + "eventId": "14", + "eventTime": "2026-10-22T00:42:35.591Z", + "eventType": "EVENT_TYPE_MARKER_RECORDED", + "markerRecordedEventAttributes": { + "markerName": "MutableSideEffect", + "details": { + "data": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg\u003d\u003d" + }, + "data": "Mg\u003d\u003d" + } + ] + }, + "id": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg\u003d\u003d" + }, + "data": "InZhbHVlIg\u003d\u003d" + } + ] + }, + "skipCount": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg\u003d\u003d" + }, + "data": "MQ\u003d\u003d" + } + ] + } + }, + "workflowTaskCompletedEventId": "12" + } + }, + { + "eventId": "15", + "eventTime": "2026-10-22T00:42:35.591Z", + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED", + "workflowExecutionSignaledEventAttributes": { + "signalName": "setValue", + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg\u003d\u003d" + }, + "data": "Mg\u003d\u003d" + } + ] + }, + "identity": "65249@MacBook-Pro.local", + "header": {} + } + }, + { + "eventId": "16", + "eventTime": "2026-10-22T00:42:35.591Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "WorkflowTest-replayHistoryWrittenWithoutEviction-251bd7da-2289-4e0f-bca6-2ebc65bdf048" + }, + "startToCloseTimeout": "5s", + "attempt": 1 + } + }, + { + "eventId": "17", + "eventTime": "2026-10-22T00:42:35.592Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "16", + "identity": "65249@MacBook-Pro.local", + "historySizeBytes": "1717" + } + }, + { + "eventId": "18", + "eventTime": "2026-10-22T00:42:35.593Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "16", + "identity": "65249@MacBook-Pro.local", + "sdkMetadata": { + "sdkName": "temporal-java", + "sdkVersion": "1.41.0" + }, + "meteringMetadata": {} + } + }, + { + "eventId": "19", + "eventTime": "2026-10-22T00:42:35.593Z", + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED", + "workflowExecutionSignaledEventAttributes": { + "signalName": "setValue", + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg\u003d\u003d" + }, + "data": "Mw\u003d\u003d" + } + ] + }, + "identity": "65249@MacBook-Pro.local", + "header": {} + } + }, + { + "eventId": "20", + "eventTime": "2026-10-22T00:42:35.593Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "WorkflowTest-replayHistoryWrittenWithoutEviction-251bd7da-2289-4e0f-bca6-2ebc65bdf048" + }, + "startToCloseTimeout": "5s", + "attempt": 1 + } + }, + { + "eventId": "21", + "eventTime": "2026-10-22T00:42:35.593Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "20", + "identity": "65249@MacBook-Pro.local", + "historySizeBytes": "2045" + } + }, + { + "eventId": "22", + "eventTime": "2026-10-22T00:42:35.596Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "20", + "identity": "65249@MacBook-Pro.local", + "sdkMetadata": { + "sdkName": "temporal-java", + "sdkVersion": "1.41.0" + }, + "meteringMetadata": {} + } + }, + { + "eventId": "23", + "eventTime": "2026-10-22T00:42:35.596Z", + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED", + "workflowExecutionCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg\u003d\u003d" + }, + "data": "IjEsMSwyLDIi" + } + ] + }, + "workflowTaskCompletedEventId": "21" + } + } + ] +} \ No newline at end of file