From 8089ba3a597481b175579ca8ce4d596947fd998e Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 16 Oct 2026 08:00:00 +0200 Subject: [PATCH 01/50] feat(tracing): core span model records bus, handler and channel spans into an application-registered recorder Spans, their kind, status and events live in Ecotone\Api\Tracing with the W3C traceparent and tracestate headers as the only carrier; the tracer, interceptors and module live in Ecotone\Tracing and register only when a TracingConfiguration names a SpanRecorder or a tracing package hands over a Tracer, so an application without tracing pays nothing. The suite mirrors the OpenTelemetry package's tree, correlation and asynchronous scenarios against the in-memory recorder. --- .../Ecotone/Api/Messaging/MessageHeaders.php | 3 +- .../Api/Tracing/InMemorySpanRecorder.php | 32 ++ packages/Ecotone/Api/Tracing/Span.php | 37 ++ packages/Ecotone/Api/Tracing/SpanEvent.php | 23 + packages/Ecotone/Api/Tracing/SpanKind.php | 17 + packages/Ecotone/Api/Tracing/SpanRecorder.php | 13 + packages/Ecotone/Api/Tracing/SpanStatus.php | 15 + .../Api/Tracing/TracingConfiguration.php | 29 ++ .../src/Messaging/Config/ModuleClassList.php | 2 + .../Messaging/Support/FrameworkHeaders.php | 3 +- packages/Ecotone/src/Tracing/ActiveSpan.php | 24 + .../src/Tracing/CompositeSpanRecorder.php | 28 ++ .../src/Tracing/Config/TracerReference.php | 15 + .../TracingChannelInterceptorBuilder.php | 43 ++ .../src/Tracing/Config/TracingModule.php | 128 ++++++ .../Ecotone/src/Tracing/EcotoneActiveSpan.php | 85 ++++ .../Ecotone/src/Tracing/EcotoneTracer.php | 63 +++ .../src/Tracing/MessageSpanAttributes.php | 40 ++ packages/Ecotone/src/Tracing/SpanContext.php | 52 +++ packages/Ecotone/src/Tracing/Tracer.php | 20 + .../src/Tracing/TracingChannelInterceptor.php | 78 ++++ .../src/Tracing/TracingInterceptor.php | 112 +++++ .../tests/Tracing/AsynchronousTracingTest.php | 353 ++++++++++++++ .../Ecotone/tests/Tracing/TracingTestCase.php | 169 +++++++ .../Ecotone/tests/Tracing/TracingTreeTest.php | 432 ++++++++++++++++++ 25 files changed, 1814 insertions(+), 2 deletions(-) create mode 100644 packages/Ecotone/Api/Tracing/InMemorySpanRecorder.php create mode 100644 packages/Ecotone/Api/Tracing/Span.php create mode 100644 packages/Ecotone/Api/Tracing/SpanEvent.php create mode 100644 packages/Ecotone/Api/Tracing/SpanKind.php create mode 100644 packages/Ecotone/Api/Tracing/SpanRecorder.php create mode 100644 packages/Ecotone/Api/Tracing/SpanStatus.php create mode 100644 packages/Ecotone/Api/Tracing/TracingConfiguration.php create mode 100644 packages/Ecotone/src/Tracing/ActiveSpan.php create mode 100644 packages/Ecotone/src/Tracing/CompositeSpanRecorder.php create mode 100644 packages/Ecotone/src/Tracing/Config/TracerReference.php create mode 100644 packages/Ecotone/src/Tracing/Config/TracingChannelInterceptorBuilder.php create mode 100644 packages/Ecotone/src/Tracing/Config/TracingModule.php create mode 100644 packages/Ecotone/src/Tracing/EcotoneActiveSpan.php create mode 100644 packages/Ecotone/src/Tracing/EcotoneTracer.php create mode 100644 packages/Ecotone/src/Tracing/MessageSpanAttributes.php create mode 100644 packages/Ecotone/src/Tracing/SpanContext.php create mode 100644 packages/Ecotone/src/Tracing/Tracer.php create mode 100644 packages/Ecotone/src/Tracing/TracingChannelInterceptor.php create mode 100644 packages/Ecotone/src/Tracing/TracingInterceptor.php create mode 100644 packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php create mode 100644 packages/Ecotone/tests/Tracing/TracingTestCase.php create mode 100644 packages/Ecotone/tests/Tracing/TracingTreeTest.php diff --git a/packages/Ecotone/Api/Messaging/MessageHeaders.php b/packages/Ecotone/Api/Messaging/MessageHeaders.php index 01db0d45e..48d1b2761 100644 --- a/packages/Ecotone/Api/Messaging/MessageHeaders.php +++ b/packages/Ecotone/Api/Messaging/MessageHeaders.php @@ -72,7 +72,8 @@ final class MessageHeaders public const CHANNEL_SEND_RETRY_NUMBER = 'channelSendRetryNumber'; - public const TEMPORARY_SPAN_CONTEXT_HEADER = 'ecotone.temporarySpanContext'; + public const TRACEPARENT = 'traceparent'; + public const TRACESTATE = 'tracestate'; public const EVENT_AGGREGATE_TYPE = '_aggregate_type'; diff --git a/packages/Ecotone/Api/Tracing/InMemorySpanRecorder.php b/packages/Ecotone/Api/Tracing/InMemorySpanRecorder.php new file mode 100644 index 000000000..b52146cef --- /dev/null +++ b/packages/Ecotone/Api/Tracing/InMemorySpanRecorder.php @@ -0,0 +1,32 @@ +spans[] = $span; + } + + /** + * @return Span[] + */ + public function recordedSpans(): array + { + return $this->spans; + } + + public function clear(): void + { + $this->spans = []; + } +} diff --git a/packages/Ecotone/Api/Tracing/Span.php b/packages/Ecotone/Api/Tracing/Span.php new file mode 100644 index 000000000..3cece36d7 --- /dev/null +++ b/packages/Ecotone/Api/Tracing/Span.php @@ -0,0 +1,37 @@ + $attributes + * @param SpanEvent[] $events + */ + public function __construct( + public readonly string $traceId, + public readonly string $spanId, + public readonly ?string $parentSpanId, + public readonly string $name, + public readonly SpanKind $kind, + public readonly array $attributes, + public readonly SpanStatus $status, + public readonly ?string $statusDescription, + public readonly array $events, + public readonly DatePoint $startedAt, + public readonly DatePoint $endedAt, + ) { + } + + public function isRoot(): bool + { + return $this->parentSpanId === null; + } +} diff --git a/packages/Ecotone/Api/Tracing/SpanEvent.php b/packages/Ecotone/Api/Tracing/SpanEvent.php new file mode 100644 index 000000000..92e551e11 --- /dev/null +++ b/packages/Ecotone/Api/Tracing/SpanEvent.php @@ -0,0 +1,23 @@ + $attributes + */ + public function __construct( + public readonly string $name, + public readonly array $attributes, + public readonly DatePoint $at, + ) { + } +} diff --git a/packages/Ecotone/Api/Tracing/SpanKind.php b/packages/Ecotone/Api/Tracing/SpanKind.php new file mode 100644 index 000000000..e6feba422 --- /dev/null +++ b/packages/Ecotone/Api/Tracing/SpanKind.php @@ -0,0 +1,17 @@ + [...$this->spanRecorderReferenceNames, $referenceName]]); + } +} diff --git a/packages/Ecotone/src/Messaging/Config/ModuleClassList.php b/packages/Ecotone/src/Messaging/Config/ModuleClassList.php index 0c9c5d8b0..a58db0032 100644 --- a/packages/Ecotone/src/Messaging/Config/ModuleClassList.php +++ b/packages/Ecotone/src/Messaging/Config/ModuleClassList.php @@ -87,6 +87,7 @@ use Ecotone\Sqs\Configuration\SqsModule; use Ecotone\SymfonyBundle\Config\SymfonyConnectionModule; use Ecotone\Tempest\Config\TempestModule; +use Ecotone\Tracing\Config\TracingModule; /** * licence Apache-2.0 @@ -116,6 +117,7 @@ class ModuleClassList RouterModule::class, ScheduledModule::class, CollectorModule::class, + TracingModule::class, DeliveryConfirmationModule::class, ChannelSetupModule::class, SerializerModule::class, diff --git a/packages/Ecotone/src/Messaging/Support/FrameworkHeaders.php b/packages/Ecotone/src/Messaging/Support/FrameworkHeaders.php index 8d04d7b4d..0d9ea2765 100644 --- a/packages/Ecotone/src/Messaging/Support/FrameworkHeaders.php +++ b/packages/Ecotone/src/Messaging/Support/FrameworkHeaders.php @@ -66,7 +66,6 @@ public static function unsetEnqueueMetadata(array $metadata): array $metadata[MessageHeaders::INBOUND_REQUEST_CHANNEL], $metadata[MessageHeaders::CONSUMER_POLLING_METADATA], $metadata[MessageHeaders::REPLY_CHANNEL], - $metadata[MessageHeaders::TEMPORARY_SPAN_CONTEXT_HEADER], ); return $metadata; @@ -152,6 +151,8 @@ private static function getFrameworksHeaderNames(): array MessageHeaders::STREAM_BASED_SOURCED, MessagingEntrypointService::ENTRYPOINT, MessageHeaders::CHANNEL_SEND_RETRY_NUMBER, + MessageHeaders::TRACEPARENT, + MessageHeaders::TRACESTATE, ]; } } diff --git a/packages/Ecotone/src/Tracing/ActiveSpan.php b/packages/Ecotone/src/Tracing/ActiveSpan.php new file mode 100644 index 000000000..43cc2a08b --- /dev/null +++ b/packages/Ecotone/src/Tracing/ActiveSpan.php @@ -0,0 +1,24 @@ + $attributes + */ + public function addEvent(string $name, array $attributes = []): void; + + public function end(): void; + + public function fail(Throwable $failure): void; +} diff --git a/packages/Ecotone/src/Tracing/CompositeSpanRecorder.php b/packages/Ecotone/src/Tracing/CompositeSpanRecorder.php new file mode 100644 index 000000000..cded68f03 --- /dev/null +++ b/packages/Ecotone/src/Tracing/CompositeSpanRecorder.php @@ -0,0 +1,28 @@ +spanRecorders as $spanRecorder) { + $spanRecorder->record($span); + } + } +} diff --git a/packages/Ecotone/src/Tracing/Config/TracerReference.php b/packages/Ecotone/src/Tracing/Config/TracerReference.php new file mode 100644 index 000000000..18505acc9 --- /dev/null +++ b/packages/Ecotone/src/Tracing/Config/TracerReference.php @@ -0,0 +1,15 @@ +channelName; + } + + public function getPrecedence(): int + { + return Precedence::DEFAULT_PRECEDENCE; + } + + public function compile(MessagingContainerBuilder $builder): Definition + { + return new Definition(TracingChannelInterceptor::class, [ + $this->channelName, + new Reference(Tracer::class), + new Reference(MessageBuilder::class), + ]); + } +} diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php new file mode 100644 index 000000000..0d06810a7 --- /dev/null +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -0,0 +1,128 @@ + 1) { + throw ConfigurationException::create('Only one Tracer can be active, yet these were registered: ' . implode(', ', array_map(fn (TracerReference $reference): string => $reference->referenceName, $tracerReferences)) . '. Enable a single tracing package.'); + } + + if ($tracingConfiguration->spanRecorderReferenceNames === [] && $tracerReferences === []) { + return; + } + + $messagingConfiguration->registerServiceDefinition( + SpanRecorder::class, + new Definition(CompositeSpanRecorder::class, [ + array_map(fn (string $referenceName): Reference => new Reference($referenceName), $tracingConfiguration->spanRecorderReferenceNames), + ]) + ); + $messagingConfiguration->registerServiceDefinition( + Tracer::class, + $tracerReferences === [] + ? new Definition(EcotoneTracer::class, [new Reference(EcotoneClockInterface::class), new Reference(SpanRecorder::class)]) + : new Reference($tracerReferences[0]->referenceName) + ); + $messagingConfiguration->registerServiceDefinition( + TracingInterceptor::class, + new Definition(TracingInterceptor::class, [new Reference(Tracer::class), new Reference(MessageBuilder::class)]) + ); + + foreach ([ + 'traceCommandHandler' => CommandHandler::class, + 'traceQueryHandler' => QueryHandler::class, + 'traceEventHandler' => EventHandler::class, + 'traceMessageHandler' => InternalHandler::class . '&¬(' . MessageHeadersPropagatorInterceptor::class . ')', + 'traceCommandBus' => CommandBus::class, + 'traceQueryBus' => QueryBus::class, + 'traceEventBus' => EventBus::class, + 'traceAsynchronousEndpoint' => AsynchronousRunningEndpoint::class, + 'traceDistributedBus' => DistributedBus::class, + ] as $method => $pointcut) { + $messagingConfiguration->registerAroundMethodInterceptor( + AroundInterceptorBuilder::create( + TracingInterceptor::class, + $interfaceToCallRegistry->getFor(TracingInterceptor::class, $method), + Precedence::TRACING_PRECEDENCE, + $pointcut + ) + ); + } + + foreach (ExtensionObjectResolver::resolve(MessageChannelBuilder::class, $extensionObjects) as $messageChannelBuilder) { + if ($messageChannelBuilder->isPollable()) { + $messagingConfiguration->registerChannelInterceptor(new TracingChannelInterceptorBuilder($messageChannelBuilder->getMessageChannelName())); + } + } + + $messagingConfiguration->registerBeforeMethodInterceptor( + MethodInterceptorBuilder::create( + Reference::to(TracingInterceptor::class), + $interfaceToCallRegistry->getFor(TracingInterceptor::class, 'provideContextForDistributedBus'), + Precedence::DEFAULT_PRECEDENCE, + DistributedBus::class + ) + ); + } + + public function getModulePackageName(): string + { + return ModulePackageList::CORE_PACKAGE; + } +} diff --git a/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php b/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php new file mode 100644 index 000000000..3fb8cc4da --- /dev/null +++ b/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php @@ -0,0 +1,85 @@ + $attributes + */ + public function __construct( + private SpanContext $context, + private ?string $parentSpanId, + private string $name, + private SpanKind $kind, + private array $attributes, + private DatePoint $startedAt, + private EcotoneClockInterface $clock, + private EcotoneTracer $tracer, + ) { + } + + public function context(): SpanContext + { + return $this->context; + } + + public function addEvent(string $name, array $attributes = []): void + { + $this->events[] = new SpanEvent($name, $attributes, $this->clock->now()); + } + + public function end(): void + { + $this->finish(SpanStatus::Ok, null); + } + + public function fail(Throwable $failure): void + { + $this->addEvent('exception', [ + 'exception.type' => $failure::class, + 'exception.message' => $failure->getMessage(), + 'exception.stacktrace' => $failure->getTraceAsString(), + ]); + $this->finish(SpanStatus::Error, $failure->getMessage()); + } + + private function finish(SpanStatus $status, ?string $statusDescription): void + { + if ($this->ended) { + return; + } + $this->ended = true; + + $this->tracer->endSpan($this, new Span( + $this->context->traceId, + $this->context->spanId, + $this->parentSpanId, + $this->name, + $this->kind, + $this->attributes, + $status, + $statusDescription, + $this->events, + $this->startedAt, + $this->clock->now(), + )); + } +} diff --git a/packages/Ecotone/src/Tracing/EcotoneTracer.php b/packages/Ecotone/src/Tracing/EcotoneTracer.php new file mode 100644 index 000000000..55c008afe --- /dev/null +++ b/packages/Ecotone/src/Tracing/EcotoneTracer.php @@ -0,0 +1,63 @@ +currentSpan()?->context(); + + $span = new EcotoneActiveSpan( + $parent?->newChild() ?? SpanContext::newTrace(), + $parent?->spanId, + $name, + $kind, + $attributes, + $this->clock->now(), + $this->clock, + $this, + ); + $this->activeSpans[] = $span; + + return $span; + } + + public function currentSpan(): ?ActiveSpan + { + return $this->activeSpans[count($this->activeSpans) - 1] ?? null; + } + + public function endSpan(EcotoneActiveSpan $activeSpan, Span $span): void + { + $position = array_search($activeSpan, $this->activeSpans, true); + if ($position !== false) { + array_splice($this->activeSpans, $position, 1); + } + + $this->spanRecorder->record($span); + } +} diff --git a/packages/Ecotone/src/Tracing/MessageSpanAttributes.php b/packages/Ecotone/src/Tracing/MessageSpanAttributes.php new file mode 100644 index 000000000..938bbceba --- /dev/null +++ b/packages/Ecotone/src/Tracing/MessageSpanAttributes.php @@ -0,0 +1,40 @@ + + */ + public static function from(Message $message): array + { + $attributes = []; + foreach (FrameworkHeaders::unsetAllFrameworkHeaders($message->getHeaders()->headers()) as $key => $value) { + if (is_scalar($value)) { + $attributes[$key] = $value; + } + } + + $headers = $message->getHeaders(); + $attributes[MessageHeaders::MESSAGE_ID] = $headers->getMessageId(); + $attributes[MessageHeaders::MESSAGE_CORRELATION_ID] = $headers->getCorrelationId(); + if ($headers->getParentId() !== null) { + $attributes[MessageHeaders::PARENT_MESSAGE_ID] = $headers->getParentId(); + } + $attributes[MessageHeaders::TIMESTAMP] = $headers->getTimestamp(); + + return $attributes; + } +} diff --git a/packages/Ecotone/src/Tracing/SpanContext.php b/packages/Ecotone/src/Tracing/SpanContext.php new file mode 100644 index 000000000..0d69da0d9 --- /dev/null +++ b/packages/Ecotone/src/Tracing/SpanContext.php @@ -0,0 +1,52 @@ +traceId, bin2hex(random_bytes(8)), $this->sampled); + } + + public static function fromTraceparent(?string $traceparent): ?self + { + if ($traceparent === null || preg_match(self::TRACEPARENT_PATTERN, $traceparent, $matches) !== 1) { + return null; + } + if ($matches[1] === str_repeat('0', 32) || $matches[2] === str_repeat('0', 16)) { + return null; + } + + return new self($matches[1], $matches[2], (hexdec($matches[3]) & self::SAMPLED_FLAG) === self::SAMPLED_FLAG); + } + + public function toTraceparent(): string + { + return sprintf('00-%s-%s-%02x', $this->traceId, $this->spanId, $this->sampled ? self::SAMPLED_FLAG : 0); + } +} diff --git a/packages/Ecotone/src/Tracing/Tracer.php b/packages/Ecotone/src/Tracing/Tracer.php new file mode 100644 index 000000000..5b278ec71 --- /dev/null +++ b/packages/Ecotone/src/Tracing/Tracer.php @@ -0,0 +1,20 @@ + $attributes + */ + public function startSpan(string $name, SpanKind $kind, array $attributes = [], ?SpanContext $remoteParent = null): ActiveSpan; + + public function currentSpan(): ?ActiveSpan; +} diff --git a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php new file mode 100644 index 000000000..aa8025614 --- /dev/null +++ b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php @@ -0,0 +1,78 @@ +tracer->startSpan('Sending to Channel: ' . $this->channelName, SpanKind::Producer, MessageSpanAttributes::from($message)); + $this->openSendSpans[] = $span; + + return $this->messageBuilder->fromMessage($message) + ->setHeader(MessageHeaders::TRACEPARENT, $span->context()->toTraceparent()) + ->build(); + } + + public function postSend(Message $message, MessageChannel $messageChannel): void + { + } + + public function afterSendCompletion(Message $message, MessageChannel $messageChannel, ?Throwable $exception): bool + { + $span = array_pop($this->openSendSpans); + if ($exception !== null) { + $span?->fail($exception); + } else { + $span?->end(); + } + + return false; + } + + public function preReceive(MessageChannel $messageChannel): bool + { + return true; + } + + public function afterReceiveCompletion(?Message $message, MessageChannel $messageChannel, ?Throwable $exception): void + { + if ($exception === null || $message === null) { + return; + } + + $this->tracer + ->startSpan('Asynchronous Channel: ' . $this->channelName, SpanKind::Consumer, MessageSpanAttributes::from($message), TracingInterceptor::remoteParentOf($message)) + ->fail($exception); + } + + public function postReceive(Message $message, MessageChannel $messageChannel): ?Message + { + return $message; + } +} diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php new file mode 100644 index 000000000..3745982d7 --- /dev/null +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -0,0 +1,112 @@ +trace('Command Bus', $methodInvocation, $message); + } + + public function traceEventBus(MethodInvocation $methodInvocation, Message $message): mixed + { + return $this->trace('Event Bus', $methodInvocation, $message); + } + + public function traceQueryBus(MethodInvocation $methodInvocation, Message $message): mixed + { + return $this->trace('Query Bus', $methodInvocation, $message); + } + + public function traceCommandHandler(MethodInvocation $methodInvocation, Message $message): mixed + { + return $this->trace('Command Handler: ' . $methodInvocation->getName(), $methodInvocation, $message); + } + + public function traceQueryHandler(MethodInvocation $methodInvocation, Message $message): mixed + { + return $this->trace('Query Handler: ' . $methodInvocation->getName(), $methodInvocation, $message); + } + + public function traceEventHandler(MethodInvocation $methodInvocation, Message $message): mixed + { + return $this->trace('Event Handler: ' . $methodInvocation->getName(), $methodInvocation, $message); + } + + public function traceMessageHandler(MethodInvocation $methodInvocation, Message $message): mixed + { + return $this->trace('Message Handler: ' . $methodInvocation->getName(), $methodInvocation, $message); + } + + public function traceAsynchronousEndpoint(MethodInvocation $methodInvocation, Message $message): mixed + { + $headers = $message->getHeaders(); + $name = match (true) { + $headers->containsKey(MessageHeaders::POLLED_CHANNEL_NAME) => 'Receiving from channel: ' . $headers->get(MessageHeaders::POLLED_CHANNEL_NAME), + $headers->containsKey(MessageHeaders::INBOUND_REQUEST_CHANNEL) => 'Receiving from inbound channel adapter: ' . $headers->get(MessageHeaders::INBOUND_REQUEST_CHANNEL), + default => 'Endpoint: ' . $headers->get(MessageHeaders::CONSUMER_POLLING_METADATA)->getEndpointId() . ' produced Message', + }; + + return $this->trace($name, $methodInvocation, $message, SpanKind::Consumer, self::remoteParentOf($message)); + } + + public function traceDistributedBus(MethodInvocation $methodInvocation, Message $message): mixed + { + return $this->trace('Distributed Bus', $methodInvocation, $message, SpanKind::Producer); + } + + public function provideContextForDistributedBus(Message $message): Message + { + $currentSpan = $this->tracer->currentSpan(); + if ($currentSpan === null) { + return $message; + } + + return $this->messageBuilder->fromMessage($message) + ->setHeader(MessageHeaders::TRACEPARENT, $currentSpan->context()->toTraceparent()) + ->build(); + } + + public static function remoteParentOf(Message $message): ?SpanContext + { + $headers = $message->getHeaders(); + + return SpanContext::fromTraceparent($headers->containsKey(MessageHeaders::TRACEPARENT) ? $headers->get(MessageHeaders::TRACEPARENT) : null); + } + + private function trace(string $name, MethodInvocation $methodInvocation, Message $message, SpanKind $kind = SpanKind::Server, ?SpanContext $remoteParent = null): mixed + { + $span = $this->tracer->startSpan($name, $kind, MessageSpanAttributes::from($message), $remoteParent); + + try { + $result = $methodInvocation->proceed(); + } catch (Throwable $failure) { + $span->fail($failure); + + throw $failure; + } + + $span->end(); + + return $result; + } +} diff --git a/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php b/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php new file mode 100644 index 000000000..f4756d6ec --- /dev/null +++ b/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php @@ -0,0 +1,353 @@ +asynchronousUsers(); + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + ) + ->sendCommandWithRouting('user.register', '1') + ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::compareTreesByDetails( + [ + [ + 'details' => ['name' => 'Command Bus'], + 'children' => [ + [ + 'details' => ['name' => 'Sending to Channel: async_channel', 'kind' => SpanKind::Producer], + 'children' => [ + [ + 'details' => ['name' => 'Receiving from channel: async_channel', 'kind' => SpanKind::Consumer], + 'children' => [ + ['details' => ['name' => 'Command Handler: ' . $users::class . '::register'], 'children' => []], + ], + ], + ], + ], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + } + + public function test_the_consumer_continues_the_trace_of_the_sender_across_the_queue(): void + { + $recorder = new InMemorySpanRecorder(); + $users = $this->asynchronousUsers(); + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + ) + ->sendCommandWithRouting('user.register', '1') + ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $traceIds = array_unique(array_map(fn (Span $span): string => $span->traceId, $recorder->recordedSpans())); + $this->assertCount(1, $traceIds); + } + + public function test_tracing_same_message_for_asynchronous_scenario(): void + { + $recorder = new InMemorySpanRecorder(); + $users = $this->asynchronousUsers(); + $messageId = Uuid::v7()->toRfc4122(); + $correlationId = Uuid::v7()->toRfc4122(); + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + ) + ->sendCommandWithRouting('user.register', '1', metadata: [ + MessageHeaders::MESSAGE_ID => $messageId, + MessageHeaders::MESSAGE_CORRELATION_ID => $correlationId, + ]) + ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $node = self::getNodeAtTargetedSpan( + [ + 'details' => ['name' => 'Command Bus'], + 'child' => [ + 'details' => ['name' => 'Sending to Channel: async_channel'], + 'child' => ['details' => ['name' => 'Receiving from channel: async_channel']], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + + $this->assertSame($correlationId, $node['details']['attributes'][MessageHeaders::MESSAGE_CORRELATION_ID]); + $this->assertSame($messageId, $node['details']['attributes'][MessageHeaders::MESSAGE_ID]); + $child = $node['children'][array_key_first($node['children'])]; + $this->assertSame($correlationId, $child['details']['attributes'][MessageHeaders::MESSAGE_CORRELATION_ID]); + $this->assertSame($messageId, $child['details']['attributes'][MessageHeaders::MESSAGE_ID]); + } + + public function test_two_traces_with_asynchronous_handlers(): void + { + $recorder = new InMemorySpanRecorder(); + $users = $this->asynchronousUsers(); + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + ) + ->sendCommandWithRouting('user.register', '1', metadata: ['flowId' => '1']) + ->sendCommandWithRouting('user.register', '2', metadata: ['flowId' => '2']) + ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 2)); + + $expectedFlow = fn (string $flowId): array => [ + 'details' => ['name' => 'Command Bus', 'attributes' => ['flowId' => $flowId]], + 'children' => [ + [ + 'details' => ['name' => 'Sending to Channel: async_channel', 'attributes' => ['flowId' => $flowId]], + 'children' => [ + [ + 'details' => ['name' => 'Receiving from channel: async_channel', 'attributes' => ['flowId' => $flowId]], + 'children' => [ + ['details' => ['name' => 'Command Handler: ' . $users::class . '::register', 'attributes' => ['flowId' => $flowId]], 'children' => []], + ], + ], + ], + ], + ], + ]; + self::compareTreesByDetails([$expectedFlow('1'), $expectedFlow('2')], self::buildTree($recorder->recordedSpans())); + $this->assertCount(2, array_unique(array_map(fn (Span $span): string => $span->traceId, $recorder->recordedSpans()))); + } + + public function test_user_headers_travel_with_the_message_into_every_span(): void + { + $recorder = new InMemorySpanRecorder(); + $users = $this->asynchronousUsers(); + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + ) + ->sendCommandWithRouting('user.register', '1', metadata: ['user_id' => '123']) + ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + foreach (['Command Bus', 'Sending to Channel: async_channel', 'Receiving from channel: async_channel', 'Command Handler: ' . $users::class . '::register'] as $spanName) { + $spans = array_values(array_filter($recorder->recordedSpans(), fn (Span $span): bool => $span->name === $spanName)); + $this->assertCount(1, $spans, $spanName); + $this->assertSame('123', $spans[0]->attributes['user_id'], $spanName); + } + } + + public function test_the_trace_carrier_header_is_not_reported_as_a_user_attribute(): void + { + $recorder = new InMemorySpanRecorder(); + $users = $this->asynchronousUsers(); + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + ) + ->sendCommandWithRouting('user.register', '1') + ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + foreach ($recorder->recordedSpans() as $span) { + $this->assertArrayNotHasKey(MessageHeaders::TRACEPARENT, $span->attributes, $span->name); + } + } + + public function test_an_event_collected_during_the_asynchronous_handler_is_sent_under_the_consumer_span_in_the_same_trace(): void + { + $recorder = new InMemorySpanRecorder(); + $users = new class () { + #[Asynchronous('async_channel')] + #[CommandHandler('user.register', endpointId: 'user.register.endpoint')] + public function register(string $userId, EventBus $eventBus): void + { + $eventBus->publishWithRouting('user.registered', $userId); + } + }; + $notifier = new class () { + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.registered.endpoint')] + public function notify(string $userId): void + { + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$users::class, $notifier::class], + [$users, $notifier, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + ) + ->sendCommandWithRouting('user.register', '1') + ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 2)); + + self::getNodeAtTargetedSpan( + [ + 'details' => ['name' => 'Command Bus'], + 'child' => [ + 'details' => ['name' => 'Sending to Channel: async_channel'], + 'child' => [ + 'details' => ['name' => 'Receiving from channel: async_channel'], + 'child' => [ + 'details' => ['name' => 'Sending to Channel: async_channel'], + 'child' => [ + 'details' => ['name' => 'Receiving from channel: async_channel'], + 'child' => ['details' => ['name' => 'Event Handler: ' . $notifier::class . '::notify']], + ], + ], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + self::getNodeAtTargetedSpan( + [ + 'details' => ['name' => 'Command Bus'], + 'child' => [ + 'details' => ['name' => 'Sending to Channel: async_channel'], + 'child' => [ + 'details' => ['name' => 'Receiving from channel: async_channel'], + 'child' => [ + 'details' => ['name' => 'Command Handler: ' . $users::class . '::register'], + 'child' => ['details' => ['name' => 'Event Bus']], + ], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + $this->assertCount(1, array_unique(array_map(fn (Span $span): string => $span->traceId, $recorder->recordedSpans()))); + } + + public function test_tracing_scheduled_handler(): void + { + $recorder = new InMemorySpanRecorder(); + $scheduled = new class () { + #[Scheduled(endpointId: 'scheduled_handler')] + #[Poller(fixedRateInMilliseconds: 1)] + public function handle(): void + { + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$scheduled::class], + [$scheduled, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->run('scheduled_handler', ExecutionPollingMetadata::createWithDefaults()->withExecutionAmountLimit(2)); + + self::compareTreesByDetails( + [ + ['details' => ['name' => 'Receiving from inbound channel adapter: nullChannel', 'kind' => SpanKind::Consumer], 'children' => []], + ['details' => ['name' => 'Receiving from inbound channel adapter: nullChannel', 'kind' => SpanKind::Consumer], 'children' => []], + ], + self::buildTree($recorder->recordedSpans()) + ); + $this->assertCount(2, array_unique(array_map(fn (Span $span): string => $span->traceId, $recorder->recordedSpans()))); + } + + public function test_tracing_workflow_scheduled_handler(): void + { + $recorder = new InMemorySpanRecorder(); + $workflow = $this->workflowScheduledHandler(); + + EcotoneLite::bootstrapFlowTesting( + [$workflow::class], + [$workflow, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->run('scheduled_handler', ExecutionPollingMetadata::createWithDefaults()->withExecutionAmountLimit(2)); + + $expectedRun = [ + 'details' => ['name' => 'Receiving from inbound channel adapter: add'], + 'children' => [ + ['details' => ['name' => 'Message Handler: ' . $workflow::class . '::add'], 'children' => []], + ], + ]; + self::compareTreesByDetails([$expectedRun, $expectedRun], self::buildTree($recorder->recordedSpans())); + } + + public function test_no_tracing_when_there_is_nothing_to_trace(): void + { + $recorder = new InMemorySpanRecorder(); + $workflow = $this->workflowScheduledHandler(); + + EcotoneLite::bootstrapFlowTesting( + [$workflow::class], + [$workflow, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->run('scheduled_handler_without_result', ExecutionPollingMetadata::createWithDefaults()->withExecutionAmountLimit(1)); + + $this->assertSame([], $recorder->recordedSpans()); + } + + private function asynchronousUsers(): object + { + return new class () { + #[Asynchronous('async_channel')] + #[CommandHandler('user.register', endpointId: 'user.register.endpoint')] + public function register(string $userId): void + { + } + }; + } + + private function workflowScheduledHandler(): object + { + return new class () { + #[Scheduled(requestChannelName: 'add', endpointId: 'scheduled_handler')] + #[Poller(fixedRateInMilliseconds: 1)] + public function produce(): int + { + return 1; + } + + #[Scheduled(requestChannelName: 'add', endpointId: 'scheduled_handler_without_result')] + #[Poller(fixedRateInMilliseconds: 1)] + public function produceNothing(): ?int + { + return null; + } + + #[InternalHandler(inputChannelName: 'add')] + public function add(): void + { + } + }; + } +} diff --git a/packages/Ecotone/tests/Tracing/TracingTestCase.php b/packages/Ecotone/tests/Tracing/TracingTestCase.php new file mode 100644 index 000000000..929dde3a6 --- /dev/null +++ b/packages/Ecotone/tests/Tracing/TracingTestCase.php @@ -0,0 +1,169 @@ + $recorder]; + } + + protected static function tracingConfiguration(): TracingConfiguration + { + return TracingConfiguration::createWithDefaults()->withSpanRecorder(InMemorySpanRecorder::class); + } + + /** + * @param Span[] $spans + * @return string[] + */ + public static function spanNamesWithoutFrameworkNoise(array $spans): array + { + return array_values(array_filter( + array_map(fn (Span $span): string => $span->name, $spans), + fn (string $name): bool => ! str_starts_with($name, self::INTERNAL_NOISE_SPAN_NAME_PREFIX), + )); + } + + /** + * @param Span[] $spans + */ + public static function buildTree(array $spans): array + { + $tree = []; + foreach ($spans as $span) { + if (str_starts_with($span->name, self::INTERNAL_NOISE_SPAN_NAME_PREFIX)) { + continue; + } + + $preparedSpan = [ + 'details' => [ + 'name' => $span->name, + 'trace_id' => $span->traceId, + 'span_id' => $span->spanId, + 'parent_id' => $span->parentSpanId, + 'attributes' => $span->attributes, + 'kind' => $span->kind, + 'status' => $span->status, + 'events' => $span->events, + ], + 'children' => [], + ]; + + $tree = self::putInTree($preparedSpan, $tree, true); + } + + return $tree; + } + + public static function compareTreesByDetails(array $expectedTree, array $collectedTree): void + { + foreach ($expectedTree as $expectedNode) { + self::findAndCompareNode($expectedNode, $collectedTree); + } + } + + private static function putInTree(array $searchedSpan, array $tree, bool $isRoot): array + { + $searchedSpanId = $searchedSpan['details']['span_id']; + $searchedParentId = $searchedSpan['details']['parent_id']; + + foreach ($tree as $nodeId => $node) { + if ($node['details']['parent_id'] === $searchedSpanId) { + unset($tree[$nodeId]); + $searchedSpan['children'][$nodeId] = $node; + } + } + $changedStructure = $tree; + + foreach ($tree as $nodeId => $node) { + if ($nodeId === $searchedParentId) { + if (in_array($searchedSpanId, array_keys($node['children']))) { + continue; + } + + $changedStructure[$nodeId]['children'][$searchedSpanId] = $searchedSpan; + } elseif ($node['details']['parent_id'] === $searchedSpanId) { + unset($changedStructure[$nodeId]); + $searchedSpan['children'][$nodeId] = $node; + + $changedStructure = self::putInTree($searchedSpan, $changedStructure, true); + } else { + $changedStructure[$nodeId]['children'] = self::putInTree($searchedSpan, $node['children'], false); + } + } + + if ($isRoot && $changedStructure == $tree) { + $changedStructure[$searchedSpanId] = $searchedSpan; + } + + return $changedStructure; + } + + private static function findAndCompareNode(array $expectedTreeNode, array $collectedTree): void + { + foreach ($collectedTree as $collectedNode) { + if ($expectedTreeNode['details']['name'] !== $collectedNode['details']['name']) { + continue; + } + if (isset($expectedTreeNode['details']['attributes']['flowId'])) { + if (($collectedNode['details']['attributes']['flowId'] ?? null) != $expectedTreeNode['details']['attributes']['flowId']) { + continue; + } + } + + foreach ($expectedTreeNode['details'] as $expectedKey => $expectedValue) { + self::assertArrayHasKey($expectedKey, $collectedNode['details']); + if ($expectedKey === 'attributes') { + foreach ($expectedValue as $expectedAttributeKey => $expectedAttributeValue) { + self::assertArrayHasKey($expectedAttributeKey, $collectedNode['details']['attributes'], "Attribute {$expectedAttributeKey} missing on span {$collectedNode['details']['name']}"); + self::assertSame($expectedAttributeValue, $collectedNode['details']['attributes'][$expectedAttributeKey]); + } + } else { + self::assertSame($expectedValue, $collectedNode['details'][$expectedKey]); + } + } + self::compareTreesByDetails($expectedTreeNode['children'], $collectedNode['children']); + + return; + } + + self::fail("Could not find span named {$expectedTreeNode['details']['name']}. Spans at this level: " . json_encode(array_map(fn (array $node) => $node['details']['name'], array_values($collectedTree)))); + } + + public static function getNodeAtTargetedSpan(array $expectedTreeNode, array $collectedTree): array + { + foreach ($collectedTree as $collectedNode) { + if ($expectedTreeNode['details']['name'] !== $collectedNode['details']['name']) { + continue; + } + foreach ($expectedTreeNode['details'] as $expectedKey => $expectedValue) { + self::assertSame($expectedValue, $collectedNode['details'][$expectedKey]); + } + + if (! array_key_exists('child', $expectedTreeNode)) { + return $collectedNode; + } + + return self::getNodeAtTargetedSpan($expectedTreeNode['child'], $collectedNode['children']); + } + + self::fail("Could not find span named {$expectedTreeNode['details']['name']}. Spans at this level: " . json_encode(array_map(fn (array $node) => $node['details']['name'], array_values($collectedTree)))); + } +} diff --git a/packages/Ecotone/tests/Tracing/TracingTreeTest.php b/packages/Ecotone/tests/Tracing/TracingTreeTest.php new file mode 100644 index 000000000..ccee0be13 --- /dev/null +++ b/packages/Ecotone/tests/Tracing/TracingTreeTest.php @@ -0,0 +1,432 @@ +sendCommandWithRouting('user.register', '1'); + + self::compareTreesByDetails( + [ + [ + 'details' => ['name' => 'Command Bus', 'kind' => SpanKind::Server, 'parent_id' => null], + 'children' => [ + [ + 'details' => ['name' => 'Command Handler: ' . $users::class . '::register', 'kind' => SpanKind::Server], + 'children' => [], + ], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + } + + public function test_tracing_tree_with_two_levels_of_nesting_and_message_handler(): void + { + $recorder = new InMemorySpanRecorder(); + $handler = new class () { + #[CommandHandler('handleCommand', outputChannelName: 'handleMessage')] + public function handleCommand(Message $message): Message + { + return $message; + } + + #[InternalHandler('handleMessage')] + public function handle(Message $message): void + { + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$handler::class], + [$handler, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->sendCommandWithRouting('handleCommand'); + + self::compareTreesByDetails( + [ + [ + 'details' => ['name' => 'Command Bus'], + 'children' => [ + ['details' => ['name' => 'Command Handler: ' . $handler::class . '::handleCommand'], 'children' => []], + ['details' => ['name' => 'Message Handler: ' . $handler::class . '::handle'], 'children' => []], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + } + + public function test_tracing_tree_with_two_levels_of_nesting(): void + { + $recorder = new InMemorySpanRecorder(); + $users = $this->users(); + $subscriber = $this->merchantSubscriber(); + + EcotoneLite::bootstrapFlowTesting( + [$users::class, $subscriber::class], + [$users, $subscriber, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->publishEventWithRouting('merchant.created', '1'); + + self::compareTreesByDetails( + [ + [ + 'details' => ['name' => 'Event Bus'], + 'children' => [ + [ + 'details' => ['name' => 'Event Handler: ' . $subscriber::class . '::merchantToUser'], + 'children' => [ + [ + 'details' => ['name' => 'Command Bus'], + 'children' => [ + ['details' => ['name' => 'Command Handler: ' . $users::class . '::register'], 'children' => []], + ], + ], + ], + ], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + } + + public function test_tracing_tree_with_two_branches_on_same_level(): void + { + $recorder = new InMemorySpanRecorder(); + $users = $this->users(); + $subscriberOne = $this->merchantSubscriber(); + $subscriberTwo = new class () { + #[EventHandler('merchant.created')] + public function merchantToUser(string $merchantId, CommandBus $commandBus): void + { + $commandBus->sendWithRouting('user.register', $merchantId . '-second'); + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$users::class, $subscriberOne::class, $subscriberTwo::class], + [$users, $subscriberOne, $subscriberTwo, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->publishEventWithRouting('merchant.created', '1'); + + $commandBusChild = [ + 'details' => ['name' => 'Command Bus'], + 'children' => [ + ['details' => ['name' => 'Command Handler: ' . $users::class . '::register'], 'children' => []], + ], + ]; + self::compareTreesByDetails( + [ + [ + 'details' => ['name' => 'Event Bus'], + 'children' => [ + ['details' => ['name' => 'Event Handler: ' . $subscriberOne::class . '::merchantToUser'], 'children' => [$commandBusChild]], + ['details' => ['name' => 'Event Handler: ' . $subscriberTwo::class . '::merchantToUser'], 'children' => [$commandBusChild]], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + $this->assertCount(7, self::spanNamesWithoutFrameworkNoise($recorder->recordedSpans())); + } + + public function test_tracing_tree_with_three_levels_of_nesting(): void + { + $recorder = new InMemorySpanRecorder(); + $merchants = new class () { + #[CommandHandler('merchant.create')] + public function create(string $merchantId, EventBus $eventBus): void + { + $eventBus->publishWithRouting('merchant.created', $merchantId); + } + }; + $users = $this->users(); + $subscriber = $this->merchantSubscriber(); + + EcotoneLite::bootstrapFlowTesting( + [$merchants::class, $users::class, $subscriber::class], + [$merchants, $users, $subscriber, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->sendCommandWithRouting('merchant.create', '1'); + + self::compareTreesByDetails( + [ + [ + 'details' => ['name' => 'Command Bus'], + 'children' => [ + [ + 'details' => ['name' => 'Command Handler: ' . $merchants::class . '::create'], + 'children' => [ + [ + 'details' => ['name' => 'Event Bus'], + 'children' => [ + [ + 'details' => ['name' => 'Event Handler: ' . $subscriber::class . '::merchantToUser'], + 'children' => [ + [ + 'details' => ['name' => 'Command Bus'], + 'children' => [ + ['details' => ['name' => 'Command Handler: ' . $users::class . '::register'], 'children' => []], + ], + ], + ], + ], + ], + ], + ], + ], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + } + + public function test_every_span_of_one_flow_shares_the_trace_id_of_its_root(): void + { + $recorder = new InMemorySpanRecorder(); + $users = $this->users(); + $subscriber = $this->merchantSubscriber(); + + EcotoneLite::bootstrapFlowTesting( + [$users::class, $subscriber::class], + [$users, $subscriber, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->publishEventWithRouting('merchant.created', '1'); + + $traceIds = array_unique(array_map(fn (Span $span): string => $span->traceId, $recorder->recordedSpans())); + $this->assertCount(1, $traceIds); + $this->assertCount(1, array_filter($recorder->recordedSpans(), fn (Span $span): bool => $span->isRoot())); + } + + public function test_separate_traces(): void + { + $recorder = new InMemorySpanRecorder(); + $users = $this->users(); + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->sendCommandWithRouting('user.register', '1', metadata: ['flowId' => '1']) + ->sendCommandWithRouting('user.register', '2', metadata: ['flowId' => '2']); + + self::compareTreesByDetails( + [ + [ + 'details' => ['name' => 'Command Bus', 'attributes' => ['flowId' => '1']], + 'children' => [ + ['details' => ['name' => 'Command Handler: ' . $users::class . '::register', 'attributes' => ['flowId' => '1']], 'children' => []], + ], + ], + [ + 'details' => ['name' => 'Command Bus', 'attributes' => ['flowId' => '2']], + 'children' => [ + ['details' => ['name' => 'Command Handler: ' . $users::class . '::register', 'attributes' => ['flowId' => '2']], 'children' => []], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + $traceIds = array_unique(array_map(fn (Span $span): string => $span->traceId, $recorder->recordedSpans())); + $this->assertCount(2, $traceIds); + } + + public function test_user_land_metadata_is_skipped_when_it_is_not_scalar(): void + { + $recorder = new InMemorySpanRecorder(); + $users = $this->users(); + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->sendCommandWithRouting('user.register', '1', metadata: ['tokens' => ['123'], 'user' => new stdClass(), 'user_id' => '123']); + + $node = self::getNodeAtTargetedSpan(['details' => ['name' => 'Command Bus']], self::buildTree($recorder->recordedSpans())); + + $this->assertArrayNotHasKey('user', $node['details']['attributes']); + $this->assertArrayNotHasKey('tokens', $node['details']['attributes']); + $this->assertSame('123', $node['details']['attributes']['user_id']); + } + + public function test_spans_carry_message_identity_and_a_child_message_points_at_its_parent(): void + { + $recorder = new InMemorySpanRecorder(); + $users = $this->users(); + $subscriber = $this->merchantSubscriber(); + $messageId = Uuid::v7()->toRfc4122(); + $correlationId = Uuid::v7()->toRfc4122(); + $timestamp = 1680436648; + + EcotoneLite::bootstrapFlowTesting( + [$users::class, $subscriber::class], + [$users, $subscriber, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->publishEventWithRouting('merchant.created', '1', metadata: [ + MessageHeaders::MESSAGE_ID => $messageId, + MessageHeaders::MESSAGE_CORRELATION_ID => $correlationId, + MessageHeaders::TIMESTAMP => $timestamp, + ]); + + $spans = $recorder->recordedSpans(); + $this->assertCount(4, $spans); + + $this->assertChildSpan('Command Handler: ' . $users::class . '::register', $spans[0], $messageId, $correlationId); + $this->assertChildSpan('Command Bus', $spans[1], $messageId, $correlationId); + $this->assertSpanWith('Event Handler: ' . $subscriber::class . '::merchantToUser', $spans[2], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('Event Bus', $spans[3], $messageId, $correlationId, $timestamp); + } + + public function test_a_failing_handler_marks_its_span_and_the_bus_span_as_failed_with_the_exception_recorded(): void + { + $recorder = new InMemorySpanRecorder(); + $users = new class () { + #[CommandHandler('user.register')] + public function register(string $userId): void + { + throw new InvalidArgumentException('User already registered'); + } + }; + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ); + + try { + $ecotone->sendCommandWithRouting('user.register', '1'); + $this->fail('Expected the handler failure to propagate'); + } catch (InvalidArgumentException) { + } + + $spans = $recorder->recordedSpans(); + $this->assertCount(2, $spans); + foreach ($spans as $span) { + $this->assertSame(SpanStatus::Error, $span->status); + $this->assertSame('User already registered', $span->statusDescription); + $exceptionEvents = array_values(array_filter($span->events, fn ($event): bool => $event->name === 'exception')); + $this->assertCount(1, $exceptionEvents); + $this->assertSame(InvalidArgumentException::class, $exceptionEvents[0]->attributes['exception.type']); + $this->assertSame('User already registered', $exceptionEvents[0]->attributes['exception.message']); + } + } + + public function test_query_bus_and_query_handler_are_traced(): void + { + $recorder = new InMemorySpanRecorder(); + $users = $this->users(); + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->sendQueryWithRouting('user.get'); + + self::compareTreesByDetails( + [ + [ + 'details' => ['name' => 'Query Bus'], + 'children' => [ + ['details' => ['name' => 'Query Handler: ' . $users::class . '::isRegistered'], 'children' => []], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + } + + private function users(): object + { + return new class () { + #[CommandHandler('user.register')] + public function register(string $userId): void + { + } + + #[QueryHandler('user.get')] + public function isRegistered(): bool + { + return true; + } + }; + } + + private function merchantSubscriber(): object + { + return new class () { + #[EventHandler('merchant.created')] + public function merchantToUser(string $merchantId, CommandBus $commandBus): void + { + $commandBus->sendWithRouting('user.register', $merchantId); + } + }; + } + + private function assertSpanWith(string $expectedName, Span $span, string $messageId, string $correlationId, int $timestamp): void + { + $this->assertSame($expectedName, $span->name); + $this->assertSame($messageId, $span->attributes[MessageHeaders::MESSAGE_ID]); + $this->assertSame($correlationId, $span->attributes[MessageHeaders::MESSAGE_CORRELATION_ID]); + $this->assertSame($timestamp, $span->attributes[MessageHeaders::TIMESTAMP]); + $this->assertSame(SpanStatus::Ok, $span->status); + } + + private function assertChildSpan(string $expectedName, Span $span, string $messageId, string $correlationId): void + { + $this->assertSame($expectedName, $span->name); + $this->assertNotSame($messageId, $span->attributes[MessageHeaders::MESSAGE_ID]); + $this->assertSame($messageId, $span->attributes[MessageHeaders::PARENT_MESSAGE_ID]); + $this->assertSame($correlationId, $span->attributes[MessageHeaders::MESSAGE_CORRELATION_ID]); + } +} From d49a2fe5dfb0c6c1dbdaaaaf4b2ac9531a5cfdde Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 16 Oct 2026 08:00:00 +0200 Subject: [PATCH 02/50] feat(tracing): a root span continues the trace its message carries, log lines become span events, and failed sends leave no span open Edge cases proved against the in-memory recorder: a failing or filtering channel interceptor, a message stripped of its carrier, an invalid traceparent, a failing error channel, a failing distributed bus send, and two tracers refused at bootstrap. --- .../RegisterSpanEventLoggerCompilerPass.php | 34 ++ .../src/Tracing/Config/TracingModule.php | 1 + .../Ecotone/src/Tracing/SpanEventLogger.php | 51 +++ .../src/Tracing/TracingInterceptor.php | 6 +- .../tests/Tracing/AsynchronousTracingTest.php | 49 +++ .../Tracing/TracingConfigurationTest.php | 86 ++++ .../tests/Tracing/TracingEdgeCasesTest.php | 380 ++++++++++++++++++ 7 files changed, 604 insertions(+), 3 deletions(-) create mode 100644 packages/Ecotone/src/Tracing/Config/RegisterSpanEventLoggerCompilerPass.php create mode 100644 packages/Ecotone/src/Tracing/SpanEventLogger.php create mode 100644 packages/Ecotone/tests/Tracing/TracingConfigurationTest.php create mode 100644 packages/Ecotone/tests/Tracing/TracingEdgeCasesTest.php diff --git a/packages/Ecotone/src/Tracing/Config/RegisterSpanEventLoggerCompilerPass.php b/packages/Ecotone/src/Tracing/Config/RegisterSpanEventLoggerCompilerPass.php new file mode 100644 index 000000000..a202844cb --- /dev/null +++ b/packages/Ecotone/src/Tracing/Config/RegisterSpanEventLoggerCompilerPass.php @@ -0,0 +1,34 @@ +has(LoggingGateway::class)) { + return; + } + + $loggingGateway = $builder->getDefinition(LoggingGateway::class); + if (is_a($loggingGateway->getClassName(), SubscribableLoggingGateway::class, true)) { + $loggingGateway->addMethodCall('registerLogger', [new Definition(SpanEventLogger::class, [new Reference(Tracer::class)])]); + } + } +} diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index 0d06810a7..ffb14920d 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -79,6 +79,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ? new Definition(EcotoneTracer::class, [new Reference(EcotoneClockInterface::class), new Reference(SpanRecorder::class)]) : new Reference($tracerReferences[0]->referenceName) ); + $messagingConfiguration->addCompilerPass(new RegisterSpanEventLoggerCompilerPass()); $messagingConfiguration->registerServiceDefinition( TracingInterceptor::class, new Definition(TracingInterceptor::class, [new Reference(Tracer::class), new Reference(MessageBuilder::class)]) diff --git a/packages/Ecotone/src/Tracing/SpanEventLogger.php b/packages/Ecotone/src/Tracing/SpanEventLogger.php new file mode 100644 index 000000000..c923d2934 --- /dev/null +++ b/packages/Ecotone/src/Tracing/SpanEventLogger.php @@ -0,0 +1,51 @@ +tracer->currentSpan(); + if ($currentSpan === null) { + return; + } + + $attributes = array_filter( + $context, + fn (string|int $key): bool => in_array($key, [MessageHeaders::MESSAGE_ID, MessageHeaders::MESSAGE_CORRELATION_ID, MessageHeaders::PARENT_MESSAGE_ID], true), + ARRAY_FILTER_USE_KEY + ); + if (isset($context['exception']) && $context['exception'] instanceof Throwable) { + $attributes['exception.type'] = $context['exception']::class; + $attributes['exception.message'] = $context['exception']->getMessage(); + $attributes['exception.stacktrace'] = $context['exception']->getTraceAsString(); + } + + $currentSpan->addEvent((string) $message, $attributes); + } +} diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 3745982d7..03a1f4b50 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -66,7 +66,7 @@ public function traceAsynchronousEndpoint(MethodInvocation $methodInvocation, Me default => 'Endpoint: ' . $headers->get(MessageHeaders::CONSUMER_POLLING_METADATA)->getEndpointId() . ' produced Message', }; - return $this->trace($name, $methodInvocation, $message, SpanKind::Consumer, self::remoteParentOf($message)); + return $this->trace($name, $methodInvocation, $message, SpanKind::Consumer); } public function traceDistributedBus(MethodInvocation $methodInvocation, Message $message): mixed @@ -93,9 +93,9 @@ public static function remoteParentOf(Message $message): ?SpanContext return SpanContext::fromTraceparent($headers->containsKey(MessageHeaders::TRACEPARENT) ? $headers->get(MessageHeaders::TRACEPARENT) : null); } - private function trace(string $name, MethodInvocation $methodInvocation, Message $message, SpanKind $kind = SpanKind::Server, ?SpanContext $remoteParent = null): mixed + private function trace(string $name, MethodInvocation $methodInvocation, Message $message, SpanKind $kind = SpanKind::Server): mixed { - $span = $this->tracer->startSpan($name, $kind, MessageSpanAttributes::from($message), $remoteParent); + $span = $this->tracer->startSpan($name, $kind, MessageSpanAttributes::from($message), $this->tracer->currentSpan() === null ? self::remoteParentOf($message) : null); try { $result = $methodInvocation->proceed(); diff --git a/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php b/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php index f4756d6ec..8dfa65257 100644 --- a/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php +++ b/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php @@ -252,6 +252,55 @@ public function notify(string $userId): void $this->assertCount(1, array_unique(array_map(fn (Span $span): string => $span->traceId, $recorder->recordedSpans()))); } + public function test_adding_logs_as_events_in_span(): void + { + $recorder = new InMemorySpanRecorder(); + $users = new class () { + #[Asynchronous('async_channel')] + #[CommandHandler('user.register', endpointId: 'user.register.endpoint')] + public function register(string $userId, EventBus $eventBus): void + { + $eventBus->publishWithRouting('user.registered', $userId); + } + }; + $notifier = new class () { + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.registered.endpoint')] + public function notify(string $userId): void + { + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$users::class, $notifier::class], + [$users, $notifier, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + ) + ->sendCommandWithRouting('user.register', '1') + ->run('async_channel'); + + $eventBus = self::getNodeAtTargetedSpan( + [ + 'details' => ['name' => 'Command Bus'], + 'child' => [ + 'details' => ['name' => 'Sending to Channel: async_channel'], + 'child' => [ + 'details' => ['name' => 'Receiving from channel: async_channel'], + 'child' => [ + 'details' => ['name' => 'Command Handler: ' . $users::class . '::register'], + 'child' => ['details' => ['name' => 'Event Bus']], + ], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + + $eventNames = array_map(fn ($event): string => $event->name, $eventBus['details']['events']); + $this->assertNotEmpty(array_filter($eventNames, fn (string $name): bool => str_starts_with($name, 'Publishing Event Message'))); + $this->assertNotEmpty(array_filter($eventNames, fn (string $name): bool => str_starts_with($name, 'Collecting message with id'))); + } + public function test_tracing_scheduled_handler(): void { $recorder = new InMemorySpanRecorder(); diff --git a/packages/Ecotone/tests/Tracing/TracingConfigurationTest.php b/packages/Ecotone/tests/Tracing/TracingConfigurationTest.php new file mode 100644 index 000000000..2353d923b --- /dev/null +++ b/packages/Ecotone/tests/Tracing/TracingConfigurationTest.php @@ -0,0 +1,86 @@ +headers = $headers; + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + ) + ->sendCommandWithRouting('user.register', '1') + ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertSame([], $recorder->recordedSpans()); + $this->assertArrayNotHasKey(MessageHeaders::TRACEPARENT, $users->headers); + } + + public function test_with_a_configured_recorder_the_trace_carrier_reaches_the_asynchronous_handler(): void + { + $recorder = new InMemorySpanRecorder(); + $users = new class () { + public array $headers = []; + + #[Asynchronous('async_channel')] + #[CommandHandler('user.register', endpointId: 'user.register.endpoint')] + public function register(string $userId, #[Headers] array $headers): void + { + $this->headers = $headers; + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + ) + ->sendCommandWithRouting('user.register', '1') + ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertMatchesRegularExpression('/^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/', $users->headers[MessageHeaders::TRACEPARENT]); + } + + public function test_two_tracers_are_refused_at_bootstrap(): void + { + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage('Only one Tracer can be active, yet these were registered: first.tracer, second.tracer. Enable a single tracing package.'); + + EcotoneLite::bootstrapFlowTesting( + [], + [], + [new TracerReference('first.tracer'), new TracerReference('second.tracer')], + ); + } +} diff --git a/packages/Ecotone/tests/Tracing/TracingEdgeCasesTest.php b/packages/Ecotone/tests/Tracing/TracingEdgeCasesTest.php new file mode 100644 index 000000000..cd5601220 --- /dev/null +++ b/packages/Ecotone/tests/Tracing/TracingEdgeCasesTest.php @@ -0,0 +1,380 @@ +bootstrapWithEventForwardedToSecondChannel($recorder, [ + SimpleMessageChannelBuilder::createQueueChannel('async_channel'), + ExceptionalQueueChannel::createWithExceptionOnSend('failing_channel'), + ]); + $ecotone->sendCommandWithRouting('user.register', '1'); + + $caught = $this->runConsumer($ecotone); + $this->assertStringContainsString('Exception on send', $caught?->getMessage() ?? ''); + + self::assertEverySpanNamedHasStatus('Sending to Channel: failing_channel', SpanStatus::Error, $recorder); + + $recorder->clear(); + $ecotone->sendCommandWithRouting('user.register', '2'); + $this->assertCount(1, array_filter($recorder->recordedSpans(), fn (Span $span): bool => $span->isRoot())); + $this->assertSame('Command Bus', self::rootSpan($recorder)->name); + } + + public function test_a_channel_interceptor_failing_before_send_leaves_no_open_span_behind(): void + { + $throwingChannelInterceptor = new class () extends AbstractChannelInterceptor { + public function preSend(Message $message, MessageChannel $messageChannel): ?Message + { + throw new RuntimeException('Exception before send'); + } + }; + $recorder = new InMemorySpanRecorder(); + $ecotone = $this->bootstrapWithEventForwardedToSecondChannel($recorder, [ + SimpleMessageChannelBuilder::createQueueChannel('async_channel'), + SimpleMessageChannelBuilder::createQueueChannel('failing_channel'), + SimpleChannelInterceptorBuilder::create('failing_channel', 'throwingChannelInterceptor'), + ], ['throwingChannelInterceptor' => $throwingChannelInterceptor]); + $ecotone->sendCommandWithRouting('user.register', '1'); + + $caught = $this->runConsumer($ecotone); + $this->assertStringContainsString('Exception before send', $caught?->getMessage() ?? ''); + + $recorder->clear(); + $ecotone->sendCommandWithRouting('user.register', '2'); + $this->assertSame('Command Bus', self::rootSpan($recorder)->name); + } + + public function test_a_message_filtered_out_by_a_channel_interceptor_leaves_no_open_span_behind(): void + { + $filteringChannelInterceptor = new class () extends AbstractChannelInterceptor { + public function preSend(Message $message, MessageChannel $messageChannel): ?Message + { + return null; + } + }; + $recorder = new InMemorySpanRecorder(); + $ecotone = $this->bootstrapWithEventForwardedToSecondChannel($recorder, [ + SimpleMessageChannelBuilder::createQueueChannel('async_channel'), + SimpleMessageChannelBuilder::createQueueChannel('failing_channel'), + SimpleChannelInterceptorBuilder::create('failing_channel', 'filteringChannelInterceptor'), + ], ['filteringChannelInterceptor' => $filteringChannelInterceptor]); + $ecotone->sendCommandWithRouting('user.register', '1'); + + $this->assertNull($this->runConsumer($ecotone)); + + $recorder->clear(); + $ecotone->sendCommandWithRouting('user.register', '2'); + $this->assertSame('Command Bus', self::rootSpan($recorder)->name); + } + + public function test_a_consumer_receiving_a_message_without_a_trace_carrier_starts_its_own_trace(): void + { + $rebuildingChannelInterceptor = new class () extends AbstractChannelInterceptor { + public function preSend(Message $message, MessageChannel $messageChannel): ?Message + { + return StaticMessageBuilder::withPayload($message->getPayload())->build(); + } + }; + $recorder = new InMemorySpanRecorder(); + $ecotone = $this->bootstrapWithEventForwardedToSecondChannel($recorder, [ + SimpleMessageChannelBuilder::createQueueChannel('async_channel'), + SimpleMessageChannelBuilder::createQueueChannel('failing_channel'), + SimpleChannelInterceptorBuilder::create('failing_channel', 'rebuildingChannelInterceptor'), + ], ['rebuildingChannelInterceptor' => $rebuildingChannelInterceptor]); + $ecotone->sendCommandWithRouting('user.register', '1'); + $this->assertNull($this->runConsumer($ecotone)); + $recorder->clear(); + + $this->assertNull($this->runConsumer($ecotone, 'failing_channel')); + + $receivingSpan = self::singleSpanNamed('Receiving from channel: failing_channel', $recorder); + $this->assertTrue($receivingSpan->isRoot()); + } + + public function test_an_invalid_trace_carrier_on_the_received_message_is_ignored_and_a_new_trace_starts(): void + { + $corruptingChannelInterceptor = new class () extends AbstractChannelInterceptor { + public function preSend(Message $message, MessageChannel $messageChannel): ?Message + { + return StaticMessageBuilder::fromMessage($message)->setHeader(MessageHeaders::TRACEPARENT, 'not-a-traceparent')->build(); + } + }; + $recorder = new InMemorySpanRecorder(); + $users = $this->asynchronousUsers(); + + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, 'corruptingChannelInterceptor' => $corruptingChannelInterceptor, ...self::recordingSpansInto($recorder)], + [ + self::tracingConfiguration(), + SimpleMessageChannelBuilder::createQueueChannel('async_channel'), + SimpleChannelInterceptorBuilder::create('async_channel', 'corruptingChannelInterceptor')->withPrecedence(-500), + ], + ); + $ecotone->sendCommandWithRouting('user.register', '1'); + $recorder->clear(); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertTrue(self::singleSpanNamed('Receiving from channel: async_channel', $recorder)->isRoot()); + } + + public function test_a_command_carrying_a_trace_carrier_continues_that_trace_as_a_child(): void + { + $recorder = new InMemorySpanRecorder(); + $users = new class () { + #[CommandHandler('user.register')] + public function register(string $userId): void + { + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01']); + + $commandBus = self::singleSpanNamed('Command Bus', $recorder); + $this->assertSame('0af7651916cd43dd8448eb211c80319c', $commandBus->traceId); + $this->assertSame('b7ad6b7169203331', $commandBus->parentSpanId); + $this->assertSame($commandBus->spanId, self::singleSpanNamed('Command Handler: ' . $users::class . '::register', $recorder)->parentSpanId); + } + + public function test_the_trace_carrier_of_the_incoming_command_does_not_leak_into_the_handler_span_parent(): void + { + $recorder = new InMemorySpanRecorder(); + $users = new class () { + #[CommandHandler('user.register')] + public function register(string $userId): void + { + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01']); + + $handler = self::singleSpanNamed('Command Handler: ' . $users::class . '::register', $recorder); + $this->assertNotSame('b7ad6b7169203331', $handler->parentSpanId); + } + + public function test_a_failing_asynchronous_handler_with_a_failing_error_channel_still_ends_every_span(): void + { + $recorder = new InMemorySpanRecorder(); + $users = new class () { + #[Asynchronous('async_channel')] + #[CommandHandler('user.register', endpointId: 'userRegisterEndpoint')] + public function register(string $userId): void + { + throw new RuntimeException('Handler failure'); + } + }; + + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [ + self::tracingConfiguration(), + SimpleMessageChannelBuilder::createQueueChannel('async_channel'), + ExceptionalQueueChannel::createWithExceptionOnSend('error_channel'), + ServiceConfiguration::createWithDefaults()->withDefaultErrorChannel('error_channel'), + ], + enabledModules: [ModulePackageList::CORE_PACKAGE], + ); + $ecotone->sendCommandWithRouting('user.register', '1'); + $recorder->clear(); + + try { + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); + } catch (RuntimeException $failure) { + $this->assertStringContainsString('Exception on send', $failure->getMessage()); + } + + self::assertEverySpanNamedHasStatus('Command Handler: ' . $users::class . '::register', SpanStatus::Error, $recorder); + self::assertEverySpanNamedHasStatus('Sending to Channel: error_channel', SpanStatus::Error, $recorder); + + $recorder->clear(); + $ecotone->sendCommandWithRouting('user.register', '2'); + $this->assertSame('Command Bus', self::rootSpan($recorder)->name); + } + + public function test_distributed_bus_send_is_a_producer_span_carrying_the_trace_to_the_other_service(): void + { + $recorder = new InMemorySpanRecorder(); + $ticketService = new class () { + public ?string $traceparent = null; + + #[CommandHandler('ticket.create')] + public function create(string $description, #[Header(MessageHeaders::TRACEPARENT)] ?string $traceparent = null): void + { + $this->traceparent = $traceparent; + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$ticketService::class], + [$ticketService, ...self::recordingSpansInto($recorder)], + [ + self::tracingConfiguration(), + SimpleMessageChannelBuilder::createQueueChannel('distributed_channel'), + DistributedServiceMap::initialize()->withCommandMapping(targetServiceName: 'ticket_service', channelName: 'distributed_channel'), + ServiceConfiguration::createWithDefaults()->withServiceName('ticket_service'), + ], + licenceKey: LicenceTesting::VALID_LICENCE, + ) + ->getDistributedBus()->sendCommand('ticket_service', 'ticket.create', 'User changed billing address'); + + $distributedBus = self::singleSpanNamed('Distributed Bus', $recorder); + $this->assertSame(SpanKind::Producer, $distributedBus->kind); + $this->assertSame($distributedBus->spanId, self::singleSpanNamed('Sending to Channel: distributed_channel', $recorder)->parentSpanId); + } + + public function test_a_failing_distributed_bus_send_fails_the_span_and_leaves_no_open_span_behind(): void + { + $recorder = new InMemorySpanRecorder(); + $users = new class () { + #[CommandHandler('user.register')] + public function register(string $userId): void + { + } + }; + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [ + self::tracingConfiguration(), + ExceptionalQueueChannel::createWithExceptionOnSend('distributed_channel'), + DistributedServiceMap::initialize()->withCommandMapping(targetServiceName: 'ticket_service', channelName: 'distributed_channel'), + ServiceConfiguration::createWithDefaults()->withServiceName('user_service'), + ], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + try { + $ecotone->getDistributedBus()->sendCommand('ticket_service', 'ticket.create', 'User changed billing address'); + $this->fail('Expected the send failure to propagate'); + } catch (RuntimeException $failure) { + $this->assertStringContainsString('Exception on send', $failure->getMessage()); + } + + $this->assertSame(SpanStatus::Error, self::singleSpanNamed('Distributed Bus', $recorder)->status); + $recorder->clear(); + $ecotone->sendCommandWithRouting('user.register', '1'); + $this->assertSame('Command Bus', self::rootSpan($recorder)->name); + } + + private function bootstrapWithEventForwardedToSecondChannel(InMemorySpanRecorder $recorder, array $channelsToRegister, array $servicesToRegister = []): FlowTestSupport + { + $users = new class () { + #[Asynchronous('async_channel')] + #[CommandHandler('user.register', endpointId: 'userRegisterEndpoint')] + public function register(string $userId, EventBus $eventBus): void + { + $eventBus->publishWithRouting('user.registered', $userId); + } + }; + $notifier = new class () { + #[Asynchronous('failing_channel')] + #[EventHandler('user.registered', endpointId: 'userNotifierEndpoint')] + public function notify(string $userId): void + { + } + }; + + return EcotoneLite::bootstrapFlowTesting( + [$users::class, $notifier::class], + [$users, $notifier, ...self::recordingSpansInto($recorder), ...$servicesToRegister], + [self::tracingConfiguration(), ...$channelsToRegister], + enabledModules: [ModulePackageList::CORE_PACKAGE], + ); + } + + private function asynchronousUsers(): object + { + return new class () { + #[Asynchronous('async_channel')] + #[CommandHandler('user.register', endpointId: 'user.register.endpoint')] + public function register(string $userId): void + { + } + }; + } + + private function runConsumer(FlowTestSupport $ecotone, string $channel = 'async_channel'): ?Throwable + { + try { + $ecotone->run($channel, ExecutionPollingMetadata::createWithTestingSetup()); + } catch (Throwable $failure) { + return $failure; + } + + return null; + } + + private static function singleSpanNamed(string $name, InMemorySpanRecorder $recorder): Span + { + $spans = array_values(array_filter($recorder->recordedSpans(), fn (Span $span): bool => $span->name === $name)); + self::assertCount(1, $spans, 'Expected exactly one span named ' . $name . ', got: ' . implode(', ', self::spanNamesWithoutFrameworkNoise($recorder->recordedSpans()))); + + return $spans[0]; + } + + private static function assertEverySpanNamedHasStatus(string $name, SpanStatus $status, InMemorySpanRecorder $recorder): void + { + $spans = array_values(array_filter($recorder->recordedSpans(), fn (Span $span): bool => $span->name === $name)); + self::assertNotEmpty($spans, 'Expected spans named ' . $name . ', got: ' . implode(', ', self::spanNamesWithoutFrameworkNoise($recorder->recordedSpans()))); + foreach ($spans as $span) { + self::assertSame($status, $span->status, $name); + } + } + + private static function rootSpan(InMemorySpanRecorder $recorder): Span + { + $roots = array_values(array_filter($recorder->recordedSpans(), fn (Span $span): bool => $span->isRoot())); + self::assertCount(1, $roots, 'Expected exactly one root span, got: ' . implode(', ', array_map(fn (Span $span): string => $span->name, $roots))); + + return $roots[0]; + } +} From 5f7e5a9c511a4dfa5307e637a73a059fa7d2ee8f Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 16 Oct 2026 08:00:00 +0200 Subject: [PATCH 03/50] refactor(open-telemetry): the package implements Core's Tracer instead of producing spans itself OpenTelemetryTracer starts live SDK spans with the ids Core reports and hands every ended span to the configured recorders; the module registers it through a TracerReference only when a TracerProviderInterface was given, so a flow test without one traces through Core's own tracer. TracerInterceptor, TracingChannelInterceptor, AddSpanEventLogger and the JSON carrier header are gone, and TracingConfiguration moved to Ecotone\Api\OpenTelemetry\OpenTelemetryConfiguration. The 41 package tests pass unchanged apart from that rename. --- composer.json | 1 + .../Api/OpenTelemetryConfiguration.php | 32 +++ packages/OpenTelemetry/composer.json | 1 + packages/OpenTelemetry/phpstan.neon | 1 + .../OpenTelemetry/src/AddSpanEventLogger.php | 45 ----- .../OpenTelemetry/src/ChannelSendSpan.php | 49 ----- .../src/Configuration/OpenTelemetryModule.php | 90 +++------ ...RegisterAddSpanEventLoggerCompilerPass.php | 28 --- .../Configuration/TracingConfiguration.php | 62 ------ .../src/OpenTelemetryActiveSpan.php | 102 ++++++++++ .../OpenTelemetry/src/OpenTelemetryTracer.php | 97 +++++++++ .../OpenTelemetry/src/TracerInterceptor.php | 190 ------------------ .../src/TracingChannelAdapterBuilder.php | 42 ---- .../src/TracingChannelInterceptor.php | 98 --------- .../Integration/ForcedTraceFlushTest.php | 4 +- .../Integration/TracingScopeCleanupTest.php | 2 +- .../tests/Integration/TracingTreeTest.php | 10 +- phpstan.neon | 1 + upgrade-2.0.md | 40 ++++ upgrade/namespace-map-2.0.csv | 1 + 20 files changed, 308 insertions(+), 588 deletions(-) create mode 100644 packages/OpenTelemetry/Api/OpenTelemetryConfiguration.php delete mode 100644 packages/OpenTelemetry/src/AddSpanEventLogger.php delete mode 100644 packages/OpenTelemetry/src/ChannelSendSpan.php delete mode 100644 packages/OpenTelemetry/src/Configuration/RegisterAddSpanEventLoggerCompilerPass.php delete mode 100644 packages/OpenTelemetry/src/Configuration/TracingConfiguration.php create mode 100644 packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php create mode 100644 packages/OpenTelemetry/src/OpenTelemetryTracer.php delete mode 100644 packages/OpenTelemetry/src/TracerInterceptor.php delete mode 100644 packages/OpenTelemetry/src/TracingChannelAdapterBuilder.php delete mode 100644 packages/OpenTelemetry/src/TracingChannelInterceptor.php diff --git a/composer.json b/composer.json index 02e91e6c5..4eab2a099 100644 --- a/composer.json +++ b/composer.json @@ -53,6 +53,7 @@ "Ecotone\\Api\\Laravel\\": "packages/Laravel/Api", "Ecotone\\Api\\Tempest\\": "packages/Tempest/Api", "Ecotone\\Api\\Kafka\\": "packages/Kafka/Api", + "Ecotone\\Api\\OpenTelemetry\\": "packages/OpenTelemetry/Api", "Ecotone\\Api\\Symfony\\": "packages/Symfony/Api", "Ecotone\\Amqp\\": "packages/Amqp/src", "Ecotone\\AnnotationFinder\\": "packages/Ecotone/src/AnnotationFinder/", diff --git a/packages/OpenTelemetry/Api/OpenTelemetryConfiguration.php b/packages/OpenTelemetry/Api/OpenTelemetryConfiguration.php new file mode 100644 index 000000000..334723f87 --- /dev/null +++ b/packages/OpenTelemetry/Api/OpenTelemetryConfiguration.php @@ -0,0 +1,32 @@ + $forceFlush]); + } + + public function withForceFlushOnAsynchronousMessageHandled(bool $forceFlush): self + { + return new self(...[...get_object_vars($this), 'forceFlushOnAsynchronousMessageHandled' => $forceFlush]); + } +} diff --git a/packages/OpenTelemetry/composer.json b/packages/OpenTelemetry/composer.json index e29995db4..86ed649bd 100644 --- a/packages/OpenTelemetry/composer.json +++ b/packages/OpenTelemetry/composer.json @@ -36,6 +36,7 @@ ], "autoload": { "psr-4": { + "Ecotone\\Api\\OpenTelemetry\\": "Api/", "Ecotone\\OpenTelemetry\\": "src" } }, diff --git a/packages/OpenTelemetry/phpstan.neon b/packages/OpenTelemetry/phpstan.neon index 9af9c6bf4..bdfa3905b 100644 --- a/packages/OpenTelemetry/phpstan.neon +++ b/packages/OpenTelemetry/phpstan.neon @@ -2,6 +2,7 @@ parameters: level: 3 reportUnmatchedIgnoredErrors: false paths: + - Api - src excludePaths: - src/Support/OTelTracer.php diff --git a/packages/OpenTelemetry/src/AddSpanEventLogger.php b/packages/OpenTelemetry/src/AddSpanEventLogger.php deleted file mode 100644 index 02bd5c79d..000000000 --- a/packages/OpenTelemetry/src/AddSpanEventLogger.php +++ /dev/null @@ -1,45 +0,0 @@ - in_array($key, [MessageHeaders::MESSAGE_ID, MessageHeaders::MESSAGE_CORRELATION_ID, MessageHeaders::PARENT_MESSAGE_ID]), - ARRAY_FILTER_USE_KEY - ); - - if (isset($context['exception']) && $context['exception'] instanceof Throwable) { - $exception = $context['exception']; - $attributes['exceptionMessage'] = $exception->getMessage(); - $attributes['exceptionClass'] = $exception::class; - $attributes['exceptionTrace'] = $exception->getTraceAsString(); - } - - Span::getCurrent()->addEvent( - $message, - $attributes - ); - } -} diff --git a/packages/OpenTelemetry/src/ChannelSendSpan.php b/packages/OpenTelemetry/src/ChannelSendSpan.php deleted file mode 100644 index 39ec6868a..000000000 --- a/packages/OpenTelemetry/src/ChannelSendSpan.php +++ /dev/null @@ -1,49 +0,0 @@ -close(StatusCode::STATUS_OK, null); - } - - public function closeWithFailure(Throwable $exception): void - { - if ($this->isClosed) { - return; - } - - $this->span->recordException($exception); - $this->close(StatusCode::STATUS_ERROR, $exception->getMessage()); - } - - private function close(string $statusCode, ?string $statusDescription): void - { - if ($this->isClosed) { - return; - } - $this->isClosed = true; - - $this->span->setStatus($statusCode, $statusDescription); - $this->scope->detach(); - $this->span->end(); - } -} diff --git a/packages/OpenTelemetry/src/Configuration/OpenTelemetryModule.php b/packages/OpenTelemetry/src/Configuration/OpenTelemetryModule.php index a0b072f82..f64bafb6f 100644 --- a/packages/OpenTelemetry/src/Configuration/OpenTelemetryModule.php +++ b/packages/OpenTelemetry/src/Configuration/OpenTelemetryModule.php @@ -6,18 +6,16 @@ use Ecotone\AnnotationFinder\AnnotationFinder; use Ecotone\Api\Attribute\AsynchronousRunningEndpoint; -use Ecotone\Api\Attribute\CommandHandler; -use Ecotone\Api\Attribute\EventHandler; -use Ecotone\Api\Attribute\InternalHandler; use Ecotone\Api\Attribute\ModuleAnnotation; -use Ecotone\Api\Attribute\QueryHandler; -use Ecotone\Api\ExtensionObject\MessageChannelBuilder; use Ecotone\Api\ExtensionObject\ModulePackageList; +use Ecotone\Api\ExtensionObject\ServiceConfiguration; use Ecotone\Api\Gateway\CommandBus; use Ecotone\Api\Gateway\DistributedBus; -use Ecotone\Api\Gateway\EventBus; +use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Gateway\QueryBus; use Ecotone\Api\Interceptor\Precedence; +use Ecotone\Api\OpenTelemetry\OpenTelemetryConfiguration; +use Ecotone\Api\Tracing\SpanRecorder; use Ecotone\Lite\Test\Configuration\ProvidedReferences; use Ecotone\Lite\Test\Configuration\ResolvesExtensionObjectsForFlowTesting; use Ecotone\Messaging\Config\Annotation\AnnotationModule; @@ -29,12 +27,9 @@ use Ecotone\Messaging\Config\ModuleReferenceSearchService; use Ecotone\Messaging\Handler\InterfaceToCallRegistry; use Ecotone\Messaging\Handler\Processor\MethodInvoker\AroundInterceptorBuilder; -use Ecotone\Messaging\Handler\Processor\MethodInvoker\MethodInterceptorBuilder; -use Ecotone\Messaging\Support\MessageBuilder; use Ecotone\OpenTelemetry\EcotoneForcedTraceFlush; -use Ecotone\OpenTelemetry\TracerInterceptor; -use Ecotone\OpenTelemetry\TracingChannelAdapterBuilder; -use OpenTelemetry\API\Trace\NoopTracerProvider; +use Ecotone\OpenTelemetry\OpenTelemetryTracer; +use Ecotone\Tracing\Config\TracerReference; use OpenTelemetry\API\Trace\TracerProviderInterface; #[ModuleAnnotation] @@ -48,62 +43,38 @@ public static function create(AnnotationFinder $annotationRegistrationService, I return new self(); } + public function getModuleExtensions(ServiceConfiguration $serviceConfiguration, array $serviceExtensions, ?InterfaceToCallRegistry $interfaceToCallRegistry = null): array + { + if (ExtensionObjectResolver::contains(NoTracerProviderGiven::class, $serviceExtensions)) { + return []; + } + + return [new TracerReference(OpenTelemetryTracer::class)]; + } + public function prepare(Configuration $messagingConfiguration, array $extensionObjects, ModuleReferenceSearchService $moduleReferenceSearchService, InterfaceToCallRegistry $interfaceToCallRegistry): void { - $tracingConfiguration = ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $extensionObjects, TracingConfiguration::createWithDefaults()); - $messageChannelBuilders = ExtensionObjectResolver::resolve(MessageChannelBuilder::class, $extensionObjects); + $openTelemetryConfiguration = ExtensionObjectResolver::resolveUnique(OpenTelemetryConfiguration::class, $extensionObjects, OpenTelemetryConfiguration::createWithDefaults()); if (ExtensionObjectResolver::contains(NoTracerProviderGiven::class, $extensionObjects)) { - $messagingConfiguration->registerServiceDefinition(TracerProviderInterface::class, new Definition(NoopTracerProvider::class)); + return; } - $messagingConfiguration->addCompilerPass(new RegisterAddSpanEventLoggerCompilerPass()); - $messagingConfiguration->addCompilerPass(new TraceDynamicConsistencyBoundaryCompilerPass()); - $messagingConfiguration->registerServiceDefinition( - TracerInterceptor::class, - new Definition(TracerInterceptor::class, [ + OpenTelemetryTracer::class, + new Definition(OpenTelemetryTracer::class, [ new Reference(TracerProviderInterface::class), - new Reference(MessageBuilder::class), + new Reference(EcotoneClockInterface::class), + new Reference(SpanRecorder::class), ]) ); - if ($tracingConfiguration->higherThanOrEqualTo(TracingConfiguration::TRACING_LEVEL_FRAMEWORK)) { - $this->registerTracerFor('trace', '*', $messagingConfiguration, $interfaceToCallRegistry); - } - - foreach ($messageChannelBuilders as $messageChannelBuilder) { - if ($messageChannelBuilder->isPollable()) { - $messagingConfiguration->registerChannelInterceptor(new TracingChannelAdapterBuilder($messageChannelBuilder->getMessageChannelName())); - } - } - - $this->registerTracerFor('traceCommandHandler', CommandHandler::class, $messagingConfiguration, $interfaceToCallRegistry); - $this->registerTracerFor('traceQueryHandler', QueryHandler::class, $messagingConfiguration, $interfaceToCallRegistry); - $this->registerTracerFor('traceEventHandler', EventHandler::class, $messagingConfiguration, $interfaceToCallRegistry); - $this->registerTracerFor('traceMessageHandler', InternalHandler::class, $messagingConfiguration, $interfaceToCallRegistry); - $this->registerTracerFor('traceCommandBus', CommandBus::class, $messagingConfiguration, $interfaceToCallRegistry); - $this->registerTracerFor('traceQueryBus', QueryBus::class, $messagingConfiguration, $interfaceToCallRegistry); - $this->registerTracerFor('traceEventBus', EventBus::class, $messagingConfiguration, $interfaceToCallRegistry); - $this->registerTracerFor('traceAsynchronousEndpoint', AsynchronousRunningEndpoint::class, $messagingConfiguration, $interfaceToCallRegistry); - $this->registerTracerFor('traceDistributedBus', DistributedBus::class, $messagingConfiguration, $interfaceToCallRegistry); - - $messagingConfiguration->registerBeforeMethodInterceptor( - MethodInterceptorBuilder::create( - new Definition(TracerInterceptor::class, [ - Reference::to(TracerProviderInterface::class), - Reference::to(MessageBuilder::class), - ]), - $interfaceToCallRegistry->getFor(TracerInterceptor::class, 'provideContextForDistributedBus'), - 0, - DistributedBus::class - ) - ); + $messagingConfiguration->addCompilerPass(new TraceDynamicConsistencyBoundaryCompilerPass()); $pointcut = ''; - if ($tracingConfiguration->isFlushForcedOnBusExecution()) { + if ($openTelemetryConfiguration->forceFlushOnBusExecution) { $pointcut = CommandBus::class . '||' . QueryBus::class . '||' . DistributedBus::class; } - if ($tracingConfiguration->isForceFlushOnAsynchronousMessageHandled()) { + if ($openTelemetryConfiguration->forceFlushOnAsynchronousMessageHandled) { $pointcut .= $pointcut !== '' ? '||' . AsynchronousRunningEndpoint::class : AsynchronousRunningEndpoint::class; } @@ -139,17 +110,4 @@ public function getModulePackageName(): string { return ModulePackageList::TRACING_PACKAGE; } - - private function registerTracerFor(string $tracingMethodToInvoke, string $pointcut, Configuration $messagingConfiguration, InterfaceToCallRegistry $interfaceToCallRegistry): void - { - $messagingConfiguration - ->registerAroundMethodInterceptor( - AroundInterceptorBuilder::create( - TracerInterceptor::class, - $interfaceToCallRegistry->getFor(TracerInterceptor::class, $tracingMethodToInvoke), - Precedence::TRACING_PRECEDENCE, - $pointcut - ) - ); - } } diff --git a/packages/OpenTelemetry/src/Configuration/RegisterAddSpanEventLoggerCompilerPass.php b/packages/OpenTelemetry/src/Configuration/RegisterAddSpanEventLoggerCompilerPass.php deleted file mode 100644 index eac4e6188..000000000 --- a/packages/OpenTelemetry/src/Configuration/RegisterAddSpanEventLoggerCompilerPass.php +++ /dev/null @@ -1,28 +0,0 @@ -has(LoggingGateway::class)) { - $loggingGatewayDefinition = $builder->getDefinition(LoggingGateway::class); - - if (is_a($loggingGatewayDefinition->getClassName(), SubscribableLoggingGateway::class, true)) { - $loggingGatewayDefinition->addMethodCall('registerLogger', [new Definition(AddSpanEventLogger::class)]); - } - } - } -} diff --git a/packages/OpenTelemetry/src/Configuration/TracingConfiguration.php b/packages/OpenTelemetry/src/Configuration/TracingConfiguration.php deleted file mode 100644 index 139a02cb6..000000000 --- a/packages/OpenTelemetry/src/Configuration/TracingConfiguration.php +++ /dev/null @@ -1,62 +0,0 @@ -tracingLevel >= $tracingLevel; - } - - public function withForceFlushOnBusExecution(bool $forceFlush): self - { - $self = clone $this; - $self->forceFlushOnBusExecution = $forceFlush; - - return $self; - } - - public function withForceFlushOnAsynchronousMessageHandled(bool $forceFlushOnAsynchronousMessageHandled): self - { - $self = clone $this; - $self->forceFlushOnAsynchronousMessageHandled = $forceFlushOnAsynchronousMessageHandled; - - return $self; - } - - public function isFlushForcedOnBusExecution(): bool - { - return $this->forceFlushOnBusExecution; - } - - public function isForceFlushOnAsynchronousMessageHandled(): bool - { - return $this->forceFlushOnAsynchronousMessageHandled; - } -} diff --git a/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php b/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php new file mode 100644 index 000000000..3bf871489 --- /dev/null +++ b/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php @@ -0,0 +1,102 @@ + $attributes + */ + public function __construct( + private SpanInterface $span, + private ScopeInterface $scope, + private SpanContext $context, + private ?string $parentSpanId, + private string $name, + private SpanKind $kind, + private array $attributes, + private DatePoint $startedAt, + private EcotoneClockInterface $clock, + private OpenTelemetryTracer $tracer, + ) { + } + + public function context(): SpanContext + { + return $this->context; + } + + public function addEvent(string $name, array $attributes = []): void + { + $this->span->addEvent($name, $attributes); + $this->events[] = new SpanEvent($name, $attributes, $this->clock->now()); + } + + public function end(): void + { + $this->finish(StatusCode::STATUS_OK, SpanStatus::Ok, null); + } + + public function fail(Throwable $failure): void + { + if ($this->ended) { + return; + } + + $this->span->recordException($failure); + $this->events[] = new SpanEvent('exception', [ + 'exception.type' => $failure::class, + 'exception.message' => $failure->getMessage(), + 'exception.stacktrace' => $failure->getTraceAsString(), + ], $this->clock->now()); + $this->finish(StatusCode::STATUS_ERROR, SpanStatus::Error, $failure->getMessage()); + } + + private function finish(string $statusCode, SpanStatus $status, ?string $statusDescription): void + { + if ($this->ended) { + return; + } + $this->ended = true; + + $this->span->setStatus($statusCode, $statusDescription); + $this->scope->detach(); + $this->span->end(); + + $this->tracer->endSpan($this, new Span( + $this->context->traceId, + $this->context->spanId, + $this->parentSpanId, + $this->name, + $this->kind, + $this->attributes, + $status, + $statusDescription, + $this->events, + $this->startedAt, + $this->clock->now(), + )); + } +} diff --git a/packages/OpenTelemetry/src/OpenTelemetryTracer.php b/packages/OpenTelemetry/src/OpenTelemetryTracer.php new file mode 100644 index 000000000..6b7bd6572 --- /dev/null +++ b/packages/OpenTelemetry/src/OpenTelemetryTracer.php @@ -0,0 +1,97 @@ +spanId ?? $this->currentSpan()?->context()->spanId; + + $spanBuilder = $this->tracerProvider + ->getTracer(EcotoneSpanBuilder::ECOTONE_TRACER_NAME) + ->spanBuilder($name) + ->setSpanKind(self::kindOf($kind)) + ->setAttributes($attributes); + if ($remoteParent !== null) { + $spanBuilder->setParent(TraceContextPropagator::getInstance()->extract([MessageHeaders::TRACEPARENT => $remoteParent->toTraceparent()])); + } + + $span = $spanBuilder->startSpan(); + $scope = $span->activate(); + $context = $span->getContext(); + + $activeSpan = new OpenTelemetryActiveSpan( + $span, + $scope, + new SpanContext($context->getTraceId(), $context->getSpanId(), $context->isSampled()), + $parentSpanId, + $name, + $kind, + $attributes, + $this->clock->now(), + $this->clock, + $this, + ); + $this->activeSpans[] = $activeSpan; + + return $activeSpan; + } + + public function currentSpan(): ?ActiveSpan + { + return $this->activeSpans[count($this->activeSpans) - 1] ?? null; + } + + public function endSpan(OpenTelemetryActiveSpan $activeSpan, Span $span): void + { + $position = array_search($activeSpan, $this->activeSpans, true); + if ($position !== false) { + array_splice($this->activeSpans, $position, 1); + } + + $this->spanRecorder->record($span); + } + + private static function kindOf(SpanKind $kind): int + { + return match ($kind) { + SpanKind::Internal => OpenTelemetrySpanKind::KIND_INTERNAL, + SpanKind::Server => OpenTelemetrySpanKind::KIND_SERVER, + SpanKind::Client => OpenTelemetrySpanKind::KIND_CLIENT, + SpanKind::Producer => OpenTelemetrySpanKind::KIND_PRODUCER, + SpanKind::Consumer => OpenTelemetrySpanKind::KIND_CONSUMER, + }; + } +} diff --git a/packages/OpenTelemetry/src/TracerInterceptor.php b/packages/OpenTelemetry/src/TracerInterceptor.php deleted file mode 100644 index 466111a3e..000000000 --- a/packages/OpenTelemetry/src/TracerInterceptor.php +++ /dev/null @@ -1,190 +0,0 @@ -getHeaders()->containsKey(TracingChannelInterceptor::TRACING_CARRIER_HEADER) ? json_decode($message->getHeaders()->get(TracingChannelInterceptor::TRACING_CARRIER_HEADER), true) : []; - $parentContext = TraceContextPropagator::getInstance()->extract($carrier); - - $scope = $parentContext->activate(); - try { - return $this->trace( - $message->getHeaders()->containsKey(MessageHeaders::POLLED_CHANNEL_NAME) - ? 'Receiving from channel: ' . $message->getHeaders()->get(MessageHeaders::POLLED_CHANNEL_NAME) - : ($message->getHeaders()->containsKey(MessageHeaders::INBOUND_REQUEST_CHANNEL) - ? 'Receiving from inbound channel adapter: ' . $message->getHeaders()->get(MessageHeaders::INBOUND_REQUEST_CHANNEL) - : 'Endpoint: ' . $message->getHeaders()->get(MessageHeaders::CONSUMER_POLLING_METADATA)->getEndpointId() . ' produced Message'), - $methodInvocation, - $message, - spanKind: SpanKind::KIND_CONSUMER, - ); - } finally { - $scope->detach(); - } - } - - public function provideContextForDistributedBus(Message $message): Message - { - $ctx = Span::getCurrent()->storeInContext(Context::getCurrent()); - $carrier = []; - TraceContextPropagator::getInstance()->inject($carrier, null, $ctx); - - return $this->messageBuilder->fromMessage($message) - ->setHeader(TracingChannelInterceptor::TRACING_CARRIER_HEADER, json_encode($carrier)) - ->build(); - } - - public function traceDistributedBus(MethodInvocation $methodInvocation, Message $message): mixed - { - $span = EcotoneSpanBuilder::create($message, 'Distributed Bus', $this->tracerProvider, SpanKind::KIND_PRODUCER) - ->startSpan(); - $spanScope = $span->activate(); - - try { - $result = $methodInvocation->proceed(); - } catch (Throwable $exception) { - $span->recordException($exception); - $this->closeSpan($span, $spanScope, StatusCode::STATUS_ERROR, $exception->getMessage()); - - throw $exception; - } - - $this->closeSpan($span, $spanScope, StatusCode::STATUS_OK, null); - - return $result; - } - - public function traceCommandHandler(MethodInvocation $methodInvocation, Message $message) - { - return $this->trace( - 'Command Handler: ' . $methodInvocation->getName(), - $methodInvocation, - $message, - ); - } - - public function traceQueryHandler(MethodInvocation $methodInvocation, Message $message) - { - return $this->trace( - 'Query Handler: ' . $methodInvocation->getName(), - $methodInvocation, - $message, - ); - } - - public function traceEventHandler(MethodInvocation $methodInvocation, Message $message) - { - return $this->trace( - 'Event Handler: ' . $methodInvocation->getName(), - $methodInvocation, - $message, - ); - } - - public function traceMessageHandler(MethodInvocation $methodInvocation, Message $message) - { - return $this->trace( - 'Message Handler: ' . $methodInvocation->getName(), - $methodInvocation, - $message, - ); - } - - public function traceCommandBus(MethodInvocation $methodInvocation, Message $message) - { - return $this->trace( - 'Command Bus', - $methodInvocation, - $message, - ); - } - - public function traceEventBus(MethodInvocation $methodInvocation, Message $message) - { - return $this->trace( - 'Event Bus', - $methodInvocation, - $message, - ); - } - - public function traceQueryBus(MethodInvocation $methodInvocation, Message $message) - { - return $this->trace( - 'Query Bus', - $methodInvocation, - $message, - ); - } - - public function trace( - string $type, - MethodInvocation $methodInvocation, - Message $message, - array $attributes = [], - int $spanKind = SpanKind::KIND_SERVER, - ) { - $span = EcotoneSpanBuilder::create( - $message, - $type, - $this->tracerProvider, - $spanKind - ) - ->startSpan(); - $spanScope = $span->activate(); - - try { - $result = $methodInvocation->proceed(); - } catch (Throwable $exception) { - $span->recordException($exception); - $this->closeSpan($span, $spanScope, StatusCode::STATUS_ERROR, $exception->getMessage()); - - throw $exception; - } - - $this->closeSpan($span, $spanScope, StatusCode::STATUS_OK, null); - - return $result; - } - - private function closeSpan(SpanInterface $span, ScopeInterface $spanScope, string $statusCode, ?string $descriptionStatusCode): void - { - $span->setStatus($statusCode, $descriptionStatusCode); - $spanScope->detach(); - $span->end(); - } -} diff --git a/packages/OpenTelemetry/src/TracingChannelAdapterBuilder.php b/packages/OpenTelemetry/src/TracingChannelAdapterBuilder.php deleted file mode 100644 index 626afa18e..000000000 --- a/packages/OpenTelemetry/src/TracingChannelAdapterBuilder.php +++ /dev/null @@ -1,42 +0,0 @@ -channelName; - } - - public function getPrecedence(): int - { - return Precedence::DEFAULT_PRECEDENCE; - } - - public function compile(MessagingContainerBuilder $builder): Definition - { - return new Definition(TracingChannelInterceptor::class, [ - $this->channelName, - new Reference(TracerProviderInterface::class), - new Reference(MessageBuilder::class), - ]); - } -} diff --git a/packages/OpenTelemetry/src/TracingChannelInterceptor.php b/packages/OpenTelemetry/src/TracingChannelInterceptor.php deleted file mode 100644 index d6c651032..000000000 --- a/packages/OpenTelemetry/src/TracingChannelInterceptor.php +++ /dev/null @@ -1,98 +0,0 @@ -channelName, $this->tracerProvider, SpanKind::KIND_PRODUCER) - ->startSpan(); - - $scope = $span->activate(); - $this->openSpans[] = new ChannelSendSpan($span, $scope); - - $ctx = $span->storeInContext(Context::getCurrent()); - $carrier = []; - TraceContextPropagator::getInstance()->inject($carrier, null, $ctx); - - return $this->messageBuilder->fromMessage($message) - ->setHeader(self::TRACING_CARRIER_HEADER, json_encode($carrier)) - ->build(); - } - - public function postSend(Message $message, MessageChannel $messageChannel): void - { - } - - public function afterSendCompletion(Message $message, MessageChannel $messageChannel, ?Throwable $exception): bool - { - $channelSendSpan = array_pop($this->openSpans); - - if ($exception !== null) { - $channelSendSpan?->closeWithFailure($exception); - } else { - $channelSendSpan?->closeWithSuccess(); - } - - return false; - } - - public function preReceive(MessageChannel $messageChannel): bool - { - return true; - } - - public function afterReceiveCompletion(?Message $message, MessageChannel $messageChannel, ?Throwable $exception): void - { - if ($exception !== null && $message !== null) { - // @TODO test - $carrier = $message->getHeaders()->containsKey(self::TRACING_CARRIER_HEADER) ? json_decode($message->getHeaders()->get(self::TRACING_CARRIER_HEADER), true) : []; - $context = TraceContextPropagator::getInstance()->extract($carrier); - - $span = EcotoneSpanBuilder::create($message, 'Asynchronous Channel: ' . $this->channelName, $this->tracerProvider, SpanKind::KIND_CONSUMER) - ->setParent($context) - ->startSpan(); - - $span->setStatus(StatusCode::STATUS_ERROR); - $span->end(); - } - } - - public function postReceive(Message $message, MessageChannel $messageChannel): ?Message - { - return $message; - } -} diff --git a/packages/OpenTelemetry/tests/Integration/ForcedTraceFlushTest.php b/packages/OpenTelemetry/tests/Integration/ForcedTraceFlushTest.php index 637aef79a..4b7520643 100644 --- a/packages/OpenTelemetry/tests/Integration/ForcedTraceFlushTest.php +++ b/packages/OpenTelemetry/tests/Integration/ForcedTraceFlushTest.php @@ -11,7 +11,7 @@ use Ecotone\Api\ExtensionObject\ModulePackageList; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; use Ecotone\Api\Lite\EcotoneLite; -use Ecotone\OpenTelemetry\Configuration\TracingConfiguration; +use Ecotone\Api\OpenTelemetry\OpenTelemetryConfiguration; use OpenTelemetry\API\Trace\NoopTracerProvider; use OpenTelemetry\API\Trace\TracerProviderInterface; use OpenTelemetry\SDK\Trace\SpanExporter\InMemoryExporter; @@ -84,7 +84,7 @@ public function place(string $orderId): void EcotoneLite::bootstrapFlowTesting( [$handler::class], [$handler, TracerProviderInterface::class => new NoopTracerProvider()], - extensionObjects: [TracingConfiguration::createWithDefaults()->withForceFlushOnBusExecution(false)->withForceFlushOnAsynchronousMessageHandled(false)], + extensionObjects: [OpenTelemetryConfiguration::createWithDefaults()->withForceFlushOnBusExecution(false)->withForceFlushOnAsynchronousMessageHandled(false)], enabledModules: [ModulePackageList::TRACING_PACKAGE], )->sendCommandWithRouting('order.place', 'order-1'); diff --git a/packages/OpenTelemetry/tests/Integration/TracingScopeCleanupTest.php b/packages/OpenTelemetry/tests/Integration/TracingScopeCleanupTest.php index fddbcd6de..630e198fc 100644 --- a/packages/OpenTelemetry/tests/Integration/TracingScopeCleanupTest.php +++ b/packages/OpenTelemetry/tests/Integration/TracingScopeCleanupTest.php @@ -22,8 +22,8 @@ use Ecotone\Messaging\Channel\AbstractChannelInterceptor; use Ecotone\Messaging\Channel\ExceptionalQueueChannel; use Ecotone\Messaging\Channel\SimpleChannelInterceptorBuilder; -use Ecotone\Test\StaticMessageBuilder; use Ecotone\Test\LicenceTesting; +use Ecotone\Test\StaticMessageBuilder; use OpenTelemetry\API\Trace\TracerProviderInterface; use OpenTelemetry\SDK\Trace\SpanExporter\InMemoryExporter; use RuntimeException; diff --git a/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php b/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php index 018dc3529..63d23315c 100644 --- a/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php +++ b/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php @@ -9,7 +9,7 @@ use Ecotone\Api\ExtensionObject\ModulePackageList; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; use Ecotone\Api\Lite\EcotoneLite; -use Ecotone\OpenTelemetry\Configuration\TracingConfiguration; +use Ecotone\Api\OpenTelemetry\OpenTelemetryConfiguration; use Ecotone\OpenTelemetry\Support\OTelTracer; use InvalidArgumentException; use Monolog\Handler\StreamHandler; @@ -192,7 +192,7 @@ public function test_disabling_force_flushing_traces() ClockFactory::getDefault() ) )], - extensionObjects: [TracingConfiguration::createWithDefaults()->withForceFlushOnBusExecution(false)->withForceFlushOnAsynchronousMessageHandled(false)], + extensionObjects: [OpenTelemetryConfiguration::createWithDefaults()->withForceFlushOnBusExecution(false)->withForceFlushOnAsynchronousMessageHandled(false)], enabledModules: [ModulePackageList::TRACING_PACKAGE], ) ->sendCommand(new RegisterUser('1')); @@ -215,7 +215,7 @@ public function test_force_flushing_traces() ClockFactory::getDefault() ) )], - extensionObjects: [TracingConfiguration::createWithDefaults()->withForceFlushOnBusExecution(true)->withForceFlushOnAsynchronousMessageHandled(true)], + extensionObjects: [OpenTelemetryConfiguration::createWithDefaults()->withForceFlushOnBusExecution(true)->withForceFlushOnAsynchronousMessageHandled(true)], enabledModules: [ModulePackageList::TRACING_PACKAGE], ) ->sendCommand(new RegisterUser('1')); @@ -248,7 +248,7 @@ public function test_traces_are_force_flushed_when_synchronous_exception_happens ClockFactory::getDefault() ) )], - extensionObjects: [TracingConfiguration::createWithDefaults()->withForceFlushOnBusExecution(true)->withForceFlushOnAsynchronousMessageHandled(true)], + extensionObjects: [OpenTelemetryConfiguration::createWithDefaults()->withForceFlushOnBusExecution(true)->withForceFlushOnAsynchronousMessageHandled(true)], enabledModules: [ModulePackageList::TRACING_PACKAGE], ); @@ -738,7 +738,7 @@ public function MANUAL_test_jaeger_command_event_command_flow() TracerProviderInterface::class => $tracerProvider, ], extensionObjects: [ - TracingConfiguration::createWithDefaults(), + OpenTelemetryConfiguration::createWithDefaults(), SimpleMessageChannelBuilder::createQueueChannel('async_channel'), ], enabledModules: [ModulePackageList::TRACING_PACKAGE, ], diff --git a/phpstan.neon b/phpstan.neon index b4d659d4d..99029567d 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -20,6 +20,7 @@ parameters: - packages/Ecotone/Api - packages/JmsConverter/Api - packages/Kafka/Api + - packages/OpenTelemetry/Api - packages/Laravel/Api - packages/PdoEventSourcing/Api - packages/Redis/Api diff --git a/upgrade-2.0.md b/upgrade-2.0.md index 9c8fa9d3d..af091bf63 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -2952,6 +2952,46 @@ flushes only a provider that can — an SDK one. Any other provider handles mess forced flush left on. Bind an SDK tracer provider (`OpenTelemetry\SDK\Trace\TracerProvider`) when you want spans actually flushed. +### Tracing is part of Core, and OpenTelemetry is one of its exporters + +**Before:** every span was produced by the `ecotone/open-telemetry` package: `TracerInterceptor` around the buses and +handlers, `TracingChannelInterceptor` on every pollable channel, and `AddSpanEventLogger` turning log lines into span +events. The trace context travelled between processes as a JSON `ecotoneTracingCarrier` header, the parent of a +synchronous span came from OpenTelemetry's ambient context, and without that package there was no trace at all. + +**Now:** Core produces the spans. `Ecotone\Api\Tracing` holds the model — `Span`, `SpanKind`, `SpanStatus`, +`SpanEvent` — plus the `SpanRecorder` seam and an `InMemorySpanRecorder`; `TracingConfiguration::createWithDefaults() +->withSpanRecorder($referenceName)` names the recorders that receive every ended span. The trace context travels as +the W3C `traceparent` and `tracestate` headers (`MessageHeaders::TRACEPARENT`, `MessageHeaders::TRACESTATE`), which +are framework headers: they are not copied into the next message as user metadata, are not reported as span +attributes, and a root span continues the trace a message already carries, so a command sent with a `traceparent` +header from an HTTP request becomes a child of that request's span. Span names, kinds, attributes and the +`exception` event are unchanged. The tracing interceptors are registered only when a recorder is configured or a +tracing package hands over its `Tracer`, so an application without tracing pays nothing. `ecotone/open-telemetry` +becomes an implementation of that `Tracer`: it starts live SDK spans with the same ids Core reports, keeps the +forced flush and the Dynamic Consistency Boundary spans, and records nothing when a flow test is given no +`TracerProviderInterface`, in which case Core's own tracer serves the configured recorders. Its configuration moved +to `Ecotone\Api\OpenTelemetry\OpenTelemetryConfiguration`; the never-settable tracing level is gone. +`MessageHeaders::TEMPORARY_SPAN_CONTEXT_HEADER`, which nothing set, is removed. + +**How to adapt:** rename `Ecotone\OpenTelemetry\Configuration\TracingConfiguration` to +`Ecotone\Api\OpenTelemetry\OpenTelemetryConfiguration`; `withForceFlushOnBusExecution()` and +`withForceFlushOnAsynchronousMessageHandled()` keep their names. A consumer outside Ecotone that read the +`ecotoneTracingCarrier` header reads `traceparent` instead, which any OpenTelemetry propagator already does. To +assert on spans in a flow test without the OpenTelemetry SDK, register an `InMemorySpanRecorder` as a service and +name it in `TracingConfiguration`: + +```php +$recorder = new InMemorySpanRecorder(); +$ecotone = EcotoneLite::bootstrapFlowTesting( + [OrderService::class], + [new OrderService(), InMemorySpanRecorder::class => $recorder], + [TracingConfiguration::createWithDefaults()->withSpanRecorder(InMemorySpanRecorder::class)], +); +$ecotone->sendCommand(new PlaceOrder('1')); +$recorder->recordedSpans(); // Span[] with traceId, spanId, parentSpanId, name, kind, status, attributes, events +``` + ## 15. Testing and developer-experience changes These changes make tests and error messages say what happens, so that failures point at the real cause. Most of them diff --git a/upgrade/namespace-map-2.0.csv b/upgrade/namespace-map-2.0.csv index b9cc13073..3cfe48645 100644 --- a/upgrade/namespace-map-2.0.csv +++ b/upgrade/namespace-map-2.0.csv @@ -224,3 +224,4 @@ Ecotone\DataProtection\Configuration\DataProtectionConfiguration,Ecotone\Api\Dat Ecotone\DataProtection\Configuration\ChannelProtectionConfiguration,Ecotone\Api\DataProtection\ChannelProtectionConfiguration Ecotone\DataProtection\Encryption\Key,Ecotone\Api\DataProtection\Key Ecotone\EventSourcing\EventStreamEmitter,Ecotone\Api\EventSourcing\EventStreamEmitter +Ecotone\OpenTelemetry\Configuration\TracingConfiguration,Ecotone\Api\OpenTelemetry\OpenTelemetryConfiguration From aeefb5f5e9fdcd7f5fe9051485d333fc91d045ba Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 16 Oct 2026 08:00:00 +0200 Subject: [PATCH 04/50] feat(tracing): every event route is a producer span, a projection run is a consumer span, and spans carry the messaging semantic conventions The event bus route resolver is a replaceable service the tracing module wraps, so each route resolved for an event becomes a 'Routing to: ' or 'Routing to Projection: ' span between the Event Bus and the handler, synchronous or asynchronous, and a failing synchronous handler fails its route. The projection trigger endpoint carries ProjectionTriggerEndpoint, which the tracing interceptor turns into a 'Projection: ' consumer span. Every span names messaging.system, its operation type, its destination and its endpoint or projection through SpanAttribute. --- .../Ecotone/Api/Tracing/SpanAttribute.php | 21 ++ .../Config/MessageHandlerRoutingModule.php | 20 +- .../Config/Routing/BusRoutingMapBuilder.php | 18 +- .../Projecting/Config/ProjectingModule.php | 26 +- .../Projecting/ProjectionTriggerEndpoint.php | 18 ++ .../Config/TraceEventRoutesCompilerPass.php | 31 +++ .../src/Tracing/Config/TracingModule.php | 3 + .../src/Tracing/MessageSpanAttributes.php | 2 + .../src/Tracing/TracedRouteProcessor.php | 72 ++++++ .../src/Tracing/TracingChannelInterceptor.php | 13 +- .../src/Tracing/TracingInterceptor.php | 65 +++-- .../src/Tracing/TracingRouteResolver.php | 25 ++ .../tests/Tracing/AsynchronousTracingTest.php | 10 +- .../tests/Tracing/RouteTracingTest.php | 236 ++++++++++++++++++ .../Ecotone/tests/Tracing/TracingTestCase.php | 8 +- .../Ecotone/tests/Tracing/TracingTreeTest.php | 43 +++- .../MerchantSubscriberOne.php | 2 +- .../MerchantSubscriberTwo.php | 2 +- .../CorrelatedHeadersPropagationTest.php | 16 +- .../tests/Integration/TracingTreeTest.php | 70 ++++-- upgrade-2.0.md | 14 +- 21 files changed, 627 insertions(+), 88 deletions(-) create mode 100644 packages/Ecotone/Api/Tracing/SpanAttribute.php create mode 100644 packages/Ecotone/src/Projecting/ProjectionTriggerEndpoint.php create mode 100644 packages/Ecotone/src/Tracing/Config/TraceEventRoutesCompilerPass.php create mode 100644 packages/Ecotone/src/Tracing/TracedRouteProcessor.php create mode 100644 packages/Ecotone/src/Tracing/TracingRouteResolver.php create mode 100644 packages/Ecotone/tests/Tracing/RouteTracingTest.php diff --git a/packages/Ecotone/Api/Tracing/SpanAttribute.php b/packages/Ecotone/Api/Tracing/SpanAttribute.php new file mode 100644 index 000000000..76a90526d --- /dev/null +++ b/packages/Ecotone/Api/Tracing/SpanAttribute.php @@ -0,0 +1,21 @@ +registerMessageHandler( MessageProcessorActivatorBuilder::create() ->withInputChannelName(MessageBusChannel::COMMAND_CHANNEL_NAME_BY_NAME) - ->chain($this->buildRouterProcessor($commandBusRoutingConfig, MessageBusChannel::COMMAND_CHANNEL_NAME_BY_NAME, $isRunningForFlowTesting)) + ->chain($this->buildRouterProcessor($messagingConfiguration, $commandBusRoutingConfig, MessageBusChannel::COMMAND_CHANNEL_NAME_BY_NAME, $isRunningForFlowTesting)) ) ->registerMessageHandler( MessageProcessorActivatorBuilder::create() ->withInputChannelName(MessageBusChannel::QUERY_CHANNEL_NAME_BY_NAME) - ->chain($this->buildRouterProcessor($queryBusRouting, MessageBusChannel::QUERY_CHANNEL_NAME_BY_NAME, $isRunningForFlowTesting)) + ->chain($this->buildRouterProcessor($messagingConfiguration, $queryBusRouting, MessageBusChannel::QUERY_CHANNEL_NAME_BY_NAME, $isRunningForFlowTesting)) ) ->registerMessageHandler( MessageProcessorActivatorBuilder::create() @@ -186,7 +188,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO MessageBusChannel::EVENT_CHANNEL_NAME_BY_NAME, Reference::to(MessageBuilder::class), ])) - ->chain($this->buildRouterProcessor($eventBusRouting, MessageBusChannel::EVENT_CHANNEL_NAME_BY_NAME, $isRunningForFlowTesting, false)) + ->chain($this->buildRouterProcessor($messagingConfiguration, $eventBusRouting, MessageBusChannel::EVENT_CHANNEL_NAME_BY_NAME, $isRunningForFlowTesting, false)) ); $messagingConfiguration->registerServiceDefinition( @@ -232,7 +234,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ); } - private function buildRouterProcessor(BusRoutingMapBuilder $busRoutingConfig, string $channel, bool $isRunningForFlowTesting, bool $isResolutionRequired = true): Definition + private function buildRouterProcessor(Configuration $messagingConfiguration, BusRoutingMapBuilder $busRoutingConfig, string $channel, bool $isRunningForFlowTesting, bool $isResolutionRequired = true): Definition { $busRouteSelectorClass = match ($channel) { MessageBusChannel::COMMAND_CHANNEL_NAME_BY_NAME => CommandBusRouteSelector::class, @@ -249,10 +251,16 @@ private function buildRouterProcessor(BusRoutingMapBuilder $busRoutingConfig, st $busRouteSelectorArguments[] = $isRunningForFlowTesting; } + $routeResolver = new Definition(RouteToChannelResolver::class, [new Reference(ChannelResolver::class)]); + if ($channel === MessageBusChannel::EVENT_CHANNEL_NAME_BY_NAME) { + $messagingConfiguration->registerServiceDefinition(self::EVENT_BUS_ROUTE_RESOLVER, $routeResolver); + $routeResolver = new Reference(self::EVENT_BUS_ROUTE_RESOLVER); + } + return new Definition(RouterProcessor::class, [ new Definition($busRouteSelectorClass, $busRouteSelectorArguments), - new Definition(RouteToChannelResolver::class, [new Reference(ChannelResolver::class)]), - $isResolutionRequired, // Single route if resolution is required + $routeResolver, + $isResolutionRequired, ]); } diff --git a/packages/Ecotone/src/Modelling/Config/Routing/BusRoutingMapBuilder.php b/packages/Ecotone/src/Modelling/Config/Routing/BusRoutingMapBuilder.php index bf3bfdcf0..6238471a9 100644 --- a/packages/Ecotone/src/Modelling/Config/Routing/BusRoutingMapBuilder.php +++ b/packages/Ecotone/src/Modelling/Config/Routing/BusRoutingMapBuilder.php @@ -25,6 +25,8 @@ class BusRoutingMapBuilder extends BusRoutingMap { + private const DESTINATION_CHANNEL_SUFFIX = '.target'; + /** * @param array $routingEventHandlers */ @@ -63,11 +65,25 @@ public function getRoutesFromAnnotatedFinding(AnnotatedFinding $registration, In /** * @return string the destination channel name */ + public static function destinationChannelFor(string $endpointId): string + { + return $endpointId . self::DESTINATION_CHANNEL_SUFFIX; + } + + public static function endpointIdOfDestinationChannel(string $destinationChannel): ?string + { + if (! str_ends_with($destinationChannel, self::DESTINATION_CHANNEL_SUFFIX)) { + return null; + } + + return substr($destinationChannel, 0, -strlen(self::DESTINATION_CHANNEL_SUFFIX)); + } + public function addRoutesFromAnnotatedFinding(AnnotatedFinding $registration, InterfaceToCallRegistry $interfaceToCallRegistry): ?string { /** @var InputOutputEndpointAnnotation $annotationForMethod */ $annotationForMethod = $registration->getAnnotationForMethod(); - $destinationChannel = $annotationForMethod->getEndpointId() . '.target'; + $destinationChannel = self::destinationChannelFor($annotationForMethod->getEndpointId()); $routes = $this->getRoutesFromAnnotatedFinding($registration, $interfaceToCallRegistry); $priority = PriorityBasedOnType::fromAnnotatedFinding($registration); diff --git a/packages/Ecotone/src/Projecting/Config/ProjectingModule.php b/packages/Ecotone/src/Projecting/Config/ProjectingModule.php index 3a3ec5225..6863e7719 100644 --- a/packages/Ecotone/src/Projecting/Config/ProjectingModule.php +++ b/packages/Ecotone/src/Projecting/Config/ProjectingModule.php @@ -41,6 +41,7 @@ use Ecotone\Projecting\ProjectingManager; use Ecotone\Projecting\ProjectionRegistry; use Ecotone\Projecting\ProjectionStateStorageRegistry; +use Ecotone\Projecting\ProjectionTriggerEndpoint; use Ecotone\Projecting\SinglePartitionProvider; use Ecotone\Projecting\StreamFilterRegistry; use Ecotone\Projecting\StreamSourceRegistry; @@ -53,6 +54,8 @@ #[ModuleAnnotation] final class ProjectingModule implements AnnotationModule { + private const PROJECTING_MANAGER_CHANNEL_PREFIX = 'projecting_manager_handler:'; + public static function getProjectorExecutorReference(string $projectionName): string { return 'projection_executor:' . $projectionName; @@ -135,19 +138,19 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ->withEndpointId(self::endpointIdForProjection($projectionName)) ->withInputChannelName(self::inputChannelForProjectingManager($projectionName)); + $triggerEndpointAnnotations = [AttributeDefinition::fromObject(new ProjectionTriggerEndpoint($projectionName))]; $asyncAttribute = $projectionBuilder instanceof EcotoneProjectionExecutorBuilder ? $projectionBuilder->getAsyncAttribute() : null; if ($asyncAttribute !== null) { $endpointAnnotations = $asyncAttribute->asynchronousExecution; if ($messagingConfiguration->isRunningForEnterpriseLicence()) { $endpointAnnotations = array_merge($endpointAnnotations, [new WithoutDatabaseTransaction(), new WithoutMessageCollector()]); } - $handlerBuilder = $handlerBuilder->withEndpointAnnotations([ - AttributeDefinition::fromObject(new Asynchronous( - $asyncAttribute->getChannelName(), - $endpointAnnotations, - )), - ]); + $triggerEndpointAnnotations[] = AttributeDefinition::fromObject(new Asynchronous( + $asyncAttribute->getChannelName(), + $endpointAnnotations, + )); } + $handlerBuilder = $handlerBuilder->withEndpointAnnotations($triggerEndpointAnnotations); $messagingConfiguration->registerMessageHandler($handlerBuilder); @@ -233,6 +236,15 @@ private static function endpointIdForProjection(string $projectionName): string public static function inputChannelForProjectingManager(string $projectionName): string { - return 'projecting_manager_handler:' . $projectionName; + return self::PROJECTING_MANAGER_CHANNEL_PREFIX . $projectionName; + } + + public static function projectionNameOfTriggerChannel(string $channelName): ?string + { + if (! str_starts_with($channelName, self::PROJECTING_MANAGER_CHANNEL_PREFIX)) { + return null; + } + + return substr($channelName, strlen(self::PROJECTING_MANAGER_CHANNEL_PREFIX)); } } diff --git a/packages/Ecotone/src/Projecting/ProjectionTriggerEndpoint.php b/packages/Ecotone/src/Projecting/ProjectionTriggerEndpoint.php new file mode 100644 index 000000000..b1ce1f9a9 --- /dev/null +++ b/packages/Ecotone/src/Projecting/ProjectionTriggerEndpoint.php @@ -0,0 +1,18 @@ +has(MessageHandlerRoutingModule::EVENT_BUS_ROUTE_RESOLVER)) { + return; + } + + $builder->replace(MessageHandlerRoutingModule::EVENT_BUS_ROUTE_RESOLVER, new Definition(TracingRouteResolver::class, [ + $builder->getDefinition(MessageHandlerRoutingModule::EVENT_BUS_ROUTE_RESOLVER), + new Reference(Tracer::class), + ])); + } +} diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index ffb14920d..352edc415 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -37,6 +37,7 @@ use Ecotone\Messaging\Handler\Processor\MethodInvoker\MethodInterceptorBuilder; use Ecotone\Messaging\Support\MessageBuilder; use Ecotone\Modelling\MessageHandling\MetadataPropagator\MessageHeadersPropagatorInterceptor; +use Ecotone\Projecting\ProjectionTriggerEndpoint; use Ecotone\Tracing\CompositeSpanRecorder; use Ecotone\Tracing\EcotoneTracer; use Ecotone\Tracing\Tracer; @@ -80,6 +81,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO : new Reference($tracerReferences[0]->referenceName) ); $messagingConfiguration->addCompilerPass(new RegisterSpanEventLoggerCompilerPass()); + $messagingConfiguration->addCompilerPass(new TraceEventRoutesCompilerPass()); $messagingConfiguration->registerServiceDefinition( TracingInterceptor::class, new Definition(TracingInterceptor::class, [new Reference(Tracer::class), new Reference(MessageBuilder::class)]) @@ -95,6 +97,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO 'traceEventBus' => EventBus::class, 'traceAsynchronousEndpoint' => AsynchronousRunningEndpoint::class, 'traceDistributedBus' => DistributedBus::class, + 'traceProjection' => ProjectionTriggerEndpoint::class, ] as $method => $pointcut) { $messagingConfiguration->registerAroundMethodInterceptor( AroundInterceptorBuilder::create( diff --git a/packages/Ecotone/src/Tracing/MessageSpanAttributes.php b/packages/Ecotone/src/Tracing/MessageSpanAttributes.php index 938bbceba..7db3d3498 100644 --- a/packages/Ecotone/src/Tracing/MessageSpanAttributes.php +++ b/packages/Ecotone/src/Tracing/MessageSpanAttributes.php @@ -6,6 +6,7 @@ use Ecotone\Api\Messaging\Message; use Ecotone\Api\Messaging\MessageHeaders; +use Ecotone\Api\Tracing\SpanAttribute; use Ecotone\Messaging\Support\FrameworkHeaders; use function is_scalar; @@ -34,6 +35,7 @@ public static function from(Message $message): array $attributes[MessageHeaders::PARENT_MESSAGE_ID] = $headers->getParentId(); } $attributes[MessageHeaders::TIMESTAMP] = $headers->getTimestamp(); + $attributes[SpanAttribute::MESSAGING_SYSTEM] = SpanAttribute::ECOTONE; return $attributes; } diff --git a/packages/Ecotone/src/Tracing/TracedRouteProcessor.php b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php new file mode 100644 index 000000000..1b847a129 --- /dev/null +++ b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php @@ -0,0 +1,72 @@ +tracer->startSpan($this->spanName(), SpanKind::Producer, [ + ...MessageSpanAttributes::from($message), + SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND, + SpanAttribute::MESSAGING_DESTINATION_NAME => $this->routeName, + ...$this->destinationAttributes(), + ]); + + try { + $result = $this->routeProcessor->process($message); + } catch (Throwable $failure) { + $span->fail($failure); + + throw $failure; + } + + $span->end(); + + return $result; + } + + private function spanName(): string + { + $projectionName = ProjectingModule::projectionNameOfTriggerChannel($this->routeName); + if ($projectionName !== null) { + return 'Routing to Projection: ' . $projectionName; + } + + return 'Routing to: ' . (BusRoutingMapBuilder::endpointIdOfDestinationChannel($this->routeName) ?? $this->routeName); + } + + /** + * @return array + */ + private function destinationAttributes(): array + { + $projectionName = ProjectingModule::projectionNameOfTriggerChannel($this->routeName); + if ($projectionName !== null) { + return [SpanAttribute::PROJECTION_NAME => $projectionName]; + } + $endpointId = BusRoutingMapBuilder::endpointIdOfDestinationChannel($this->routeName); + + return $endpointId === null ? [] : [SpanAttribute::ENDPOINT_ID => $endpointId]; + } +} diff --git a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php index aa8025614..8c65a340d 100644 --- a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php @@ -9,6 +9,7 @@ use Ecotone\Api\Messaging\Message; use Ecotone\Api\Messaging\MessageChannel; use Ecotone\Api\Messaging\MessageHeaders; +use Ecotone\Api\Tracing\SpanAttribute; use Ecotone\Api\Tracing\SpanKind; use Ecotone\Messaging\Channel\ChannelInterceptor; use Ecotone\Messaging\Support\MessageBuilder; @@ -31,7 +32,11 @@ public function __construct( public function preSend(Message $message, MessageChannel $messageChannel): ?Message { - $span = $this->tracer->startSpan('Sending to Channel: ' . $this->channelName, SpanKind::Producer, MessageSpanAttributes::from($message)); + $span = $this->tracer->startSpan('Sending to Channel: ' . $this->channelName, SpanKind::Producer, [ + ...MessageSpanAttributes::from($message), + SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND, + SpanAttribute::MESSAGING_DESTINATION_NAME => $this->channelName, + ]); $this->openSendSpans[] = $span; return $this->messageBuilder->fromMessage($message) @@ -67,7 +72,11 @@ public function afterReceiveCompletion(?Message $message, MessageChannel $messag } $this->tracer - ->startSpan('Asynchronous Channel: ' . $this->channelName, SpanKind::Consumer, MessageSpanAttributes::from($message), TracingInterceptor::remoteParentOf($message)) + ->startSpan('Asynchronous Channel: ' . $this->channelName, SpanKind::Consumer, [ + ...MessageSpanAttributes::from($message), + SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS, + SpanAttribute::MESSAGING_DESTINATION_NAME => $this->channelName, + ], TracingInterceptor::remoteParentOf($message)) ->fail($exception); } diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 03a1f4b50..c47e160a0 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -4,11 +4,17 @@ namespace Ecotone\Tracing; +use Ecotone\Api\Attribute\CommandHandler; +use Ecotone\Api\Attribute\EventHandler; +use Ecotone\Api\Attribute\InternalHandler; +use Ecotone\Api\Attribute\QueryHandler; use Ecotone\Api\Interceptor\MethodInvocation; use Ecotone\Api\Messaging\Message; use Ecotone\Api\Messaging\MessageHeaders; +use Ecotone\Api\Tracing\SpanAttribute; use Ecotone\Api\Tracing\SpanKind; use Ecotone\Messaging\Support\MessageBuilder; +use Ecotone\Projecting\ProjectionTriggerEndpoint; use Throwable; /** @@ -24,37 +30,37 @@ public function __construct( public function traceCommandBus(MethodInvocation $methodInvocation, Message $message): mixed { - return $this->trace('Command Bus', $methodInvocation, $message); + return $this->trace('Command Bus', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); } public function traceEventBus(MethodInvocation $methodInvocation, Message $message): mixed { - return $this->trace('Event Bus', $methodInvocation, $message); + return $this->trace('Event Bus', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); } public function traceQueryBus(MethodInvocation $methodInvocation, Message $message): mixed { - return $this->trace('Query Bus', $methodInvocation, $message); + return $this->trace('Query Bus', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); } - public function traceCommandHandler(MethodInvocation $methodInvocation, Message $message): mixed + public function traceCommandHandler(MethodInvocation $methodInvocation, Message $message, CommandHandler $endpoint): mixed { - return $this->trace('Command Handler: ' . $methodInvocation->getName(), $methodInvocation, $message); + return $this->trace('Command Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes($endpoint->getEndpointId())); } - public function traceQueryHandler(MethodInvocation $methodInvocation, Message $message): mixed + public function traceQueryHandler(MethodInvocation $methodInvocation, Message $message, QueryHandler $endpoint): mixed { - return $this->trace('Query Handler: ' . $methodInvocation->getName(), $methodInvocation, $message); + return $this->trace('Query Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes($endpoint->getEndpointId())); } - public function traceEventHandler(MethodInvocation $methodInvocation, Message $message): mixed + public function traceEventHandler(MethodInvocation $methodInvocation, Message $message, EventHandler $endpoint): mixed { - return $this->trace('Event Handler: ' . $methodInvocation->getName(), $methodInvocation, $message); + return $this->trace('Event Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes($endpoint->getEndpointId())); } - public function traceMessageHandler(MethodInvocation $methodInvocation, Message $message): mixed + public function traceMessageHandler(MethodInvocation $methodInvocation, Message $message, InternalHandler $endpoint): mixed { - return $this->trace('Message Handler: ' . $methodInvocation->getName(), $methodInvocation, $message); + return $this->trace('Message Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes($endpoint->getEndpointId())); } public function traceAsynchronousEndpoint(MethodInvocation $methodInvocation, Message $message): mixed @@ -66,12 +72,25 @@ public function traceAsynchronousEndpoint(MethodInvocation $methodInvocation, Me default => 'Endpoint: ' . $headers->get(MessageHeaders::CONSUMER_POLLING_METADATA)->getEndpointId() . ' produced Message', }; - return $this->trace($name, $methodInvocation, $message, SpanKind::Consumer); + $attributes = [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS]; + if ($headers->containsKey(MessageHeaders::POLLED_CHANNEL_NAME)) { + $attributes[SpanAttribute::MESSAGING_DESTINATION_NAME] = $headers->get(MessageHeaders::POLLED_CHANNEL_NAME); + } + + return $this->trace($name, $methodInvocation, $message, SpanKind::Consumer, $attributes); + } + + public function traceProjection(MethodInvocation $methodInvocation, Message $message, ProjectionTriggerEndpoint $trigger): mixed + { + return $this->trace('Projection: ' . $trigger->projectionName, $methodInvocation, $message, SpanKind::Consumer, [ + SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS, + SpanAttribute::PROJECTION_NAME => $trigger->projectionName, + ]); } public function traceDistributedBus(MethodInvocation $methodInvocation, Message $message): mixed { - return $this->trace('Distributed Bus', $methodInvocation, $message, SpanKind::Producer); + return $this->trace('Distributed Bus', $methodInvocation, $message, SpanKind::Producer, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); } public function provideContextForDistributedBus(Message $message): Message @@ -93,9 +112,25 @@ public static function remoteParentOf(Message $message): ?SpanContext return SpanContext::fromTraceparent($headers->containsKey(MessageHeaders::TRACEPARENT) ? $headers->get(MessageHeaders::TRACEPARENT) : null); } - private function trace(string $name, MethodInvocation $methodInvocation, Message $message, SpanKind $kind = SpanKind::Server): mixed + /** + * @return array + */ + private static function processAttributes(string $endpointId): array + { + $attributes = [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS]; + if ($endpointId !== '') { + $attributes[SpanAttribute::ENDPOINT_ID] = $endpointId; + } + + return $attributes; + } + + /** + * @param array $attributes + */ + private function trace(string $name, MethodInvocation $methodInvocation, Message $message, SpanKind $kind, array $attributes): mixed { - $span = $this->tracer->startSpan($name, $kind, MessageSpanAttributes::from($message), $this->tracer->currentSpan() === null ? self::remoteParentOf($message) : null); + $span = $this->tracer->startSpan($name, $kind, [...MessageSpanAttributes::from($message), ...$attributes], $this->tracer->currentSpan() === null ? self::remoteParentOf($message) : null); try { $result = $methodInvocation->proceed(); diff --git a/packages/Ecotone/src/Tracing/TracingRouteResolver.php b/packages/Ecotone/src/Tracing/TracingRouteResolver.php new file mode 100644 index 000000000..0771ca894 --- /dev/null +++ b/packages/Ecotone/src/Tracing/TracingRouteResolver.php @@ -0,0 +1,25 @@ +routeResolver->resolve($routeName), $this->tracer); + } +} diff --git a/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php b/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php index 8dfa65257..1bf9e766e 100644 --- a/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php +++ b/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php @@ -296,9 +296,13 @@ public function notify(string $userId): void self::buildTree($recorder->recordedSpans()) ); - $eventNames = array_map(fn ($event): string => $event->name, $eventBus['details']['events']); - $this->assertNotEmpty(array_filter($eventNames, fn (string $name): bool => str_starts_with($name, 'Publishing Event Message'))); - $this->assertNotEmpty(array_filter($eventNames, fn (string $name): bool => str_starts_with($name, 'Collecting message with id'))); + $eventBusEvents = array_map(fn ($event): string => $event->name, $eventBus['details']['events']); + $this->assertNotEmpty(array_filter($eventBusEvents, fn (string $name): bool => str_starts_with($name, 'Publishing Event Message'))); + + $route = $eventBus['children'][array_key_first($eventBus['children'])]; + $this->assertSame('Routing to: user.registered.endpoint', $route['details']['name']); + $routeEvents = array_map(fn ($event): string => $event->name, $route['details']['events']); + $this->assertNotEmpty(array_filter($routeEvents, fn (string $name): bool => str_starts_with($name, 'Collecting message with id'))); } public function test_tracing_scheduled_handler(): void diff --git a/packages/Ecotone/tests/Tracing/RouteTracingTest.php b/packages/Ecotone/tests/Tracing/RouteTracingTest.php new file mode 100644 index 000000000..acf5592bf --- /dev/null +++ b/packages/Ecotone/tests/Tracing/RouteTracingTest.php @@ -0,0 +1,236 @@ +publishEventWithRouting('merchant.created', '1'); + + self::compareTreesByDetails( + [ + [ + 'details' => ['name' => 'Event Bus'], + 'children' => [ + [ + 'details' => [ + 'name' => 'Routing to: merchant.notifier', + 'kind' => SpanKind::Producer, + 'attributes' => ['messaging.destination.name' => 'merchant.notifier.target', 'ecotone.endpoint.id' => 'merchant.notifier'], + ], + 'children' => [ + ['details' => ['name' => 'Event Handler: ' . $subscriber::class . '::notify', 'attributes' => ['ecotone.endpoint.id' => 'merchant.notifier']], 'children' => []], + ], + ], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + } + + public function test_a_failing_synchronous_handler_fails_its_route_span(): void + { + $recorder = new InMemorySpanRecorder(); + $subscriber = new class () { + #[EventHandler('merchant.created', endpointId: 'merchant.notifier')] + public function notify(string $merchantId): void + { + throw new RuntimeException('Notification failed'); + } + }; + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$subscriber::class], + [$subscriber, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ); + + try { + $ecotone->publishEventWithRouting('merchant.created', '1'); + $this->fail('Expected the handler failure to propagate'); + } catch (RuntimeException) { + } + + $this->assertSame(SpanStatus::Error, self::spanNamed('Routing to: merchant.notifier', $recorder)->status); + } + + public function test_an_asynchronous_event_route_leads_through_the_channel_to_its_handler(): void + { + $recorder = new InMemorySpanRecorder(); + $subscriber = new class () { + #[Asynchronous('async_channel')] + #[EventHandler('merchant.created', endpointId: 'merchant.notifier')] + public function notify(string $merchantId): void + { + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$subscriber::class], + [$subscriber, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + ) + ->publishEventWithRouting('merchant.created', '1') + ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::getNodeAtTargetedSpan( + [ + 'details' => ['name' => 'Event Bus'], + 'child' => [ + 'details' => ['name' => 'Routing to: merchant.notifier'], + 'child' => [ + 'details' => ['name' => 'Sending to Channel: async_channel', 'attributes' => ['messaging.destination.name' => 'async_channel']], + 'child' => [ + 'details' => ['name' => 'Receiving from channel: async_channel', 'attributes' => ['messaging.destination.name' => 'async_channel']], + 'child' => ['details' => ['name' => 'Event Handler: ' . $subscriber::class . '::notify']], + ], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + } + + public function test_an_event_driven_projection_is_a_consumer_span_under_its_route(): void + { + $recorder = new InMemorySpanRecorder(); + $projection = new #[Projection('orders'), FromStream('orders_stream')] class () { + #[EventHandler('order.placed')] + public function whenPlaced(array $event): void + { + } + }; + + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$projection::class], + [$projection, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ); + $ecotone->withEvents([Event::createWithType('order.placed', ['orderId' => '1'])]); + + $ecotone->publishEventWithRouting('order.placed', []); + + self::compareTreesByDetails( + [ + [ + 'details' => ['name' => 'Event Bus'], + 'children' => [ + [ + 'details' => ['name' => 'Routing to Projection: orders', 'kind' => SpanKind::Producer, 'attributes' => ['ecotone.projection.name' => 'orders']], + 'children' => [ + ['details' => ['name' => 'Projection: orders', 'kind' => SpanKind::Consumer, 'attributes' => ['ecotone.projection.name' => 'orders']], 'children' => []], + ], + ], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + } + + public function test_an_asynchronous_projection_is_a_consumer_span_after_its_channel(): void + { + $recorder = new InMemorySpanRecorder(); + $projection = new #[Projection('orders'), FromStream('orders_stream'), Asynchronous('async_channel')] class () { + #[EventHandler('order.placed')] + public function whenPlaced(array $event): void + { + } + }; + + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$projection::class], + [$projection, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $ecotone->withEvents([Event::createWithType('order.placed', ['orderId' => '1'])]); + + $ecotone->publishEventWithRouting('order.placed', []) + ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::getNodeAtTargetedSpan( + [ + 'details' => ['name' => 'Event Bus'], + 'child' => [ + 'details' => ['name' => 'Routing to Projection: orders'], + 'child' => [ + 'details' => ['name' => 'Sending to Channel: async_channel'], + 'child' => [ + 'details' => ['name' => 'Receiving from channel: async_channel'], + 'child' => ['details' => ['name' => 'Projection: orders', 'kind' => SpanKind::Consumer]], + ], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + } + + public function test_every_span_names_the_messaging_system_and_handler_spans_name_their_endpoint(): void + { + $recorder = new InMemorySpanRecorder(); + $users = new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId): void + { + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->sendCommandWithRouting('user.register', '1'); + + foreach ($recorder->recordedSpans() as $span) { + $this->assertSame('ecotone', $span->attributes['messaging.system'], $span->name); + } + $this->assertSame('user.registration', self::spanNamed('Command Handler: ' . $users::class . '::register', $recorder)->attributes['ecotone.endpoint.id']); + } + + private static function spanNamed(string $name, InMemorySpanRecorder $recorder): Span + { + $spans = array_values(array_filter($recorder->recordedSpans(), fn (Span $span): bool => $span->name === $name)); + self::assertNotEmpty($spans, 'Expected a span named ' . $name . ', got: ' . implode(', ', self::spanNamesWithoutFrameworkNoise($recorder->recordedSpans()))); + + return $spans[0]; + } +} diff --git a/packages/Ecotone/tests/Tracing/TracingTestCase.php b/packages/Ecotone/tests/Tracing/TracingTestCase.php index 929dde3a6..bd8eb7386 100644 --- a/packages/Ecotone/tests/Tracing/TracingTestCase.php +++ b/packages/Ecotone/tests/Tracing/TracingTestCase.php @@ -154,7 +154,13 @@ public static function getNodeAtTargetedSpan(array $expectedTreeNode, array $col continue; } foreach ($expectedTreeNode['details'] as $expectedKey => $expectedValue) { - self::assertSame($expectedValue, $collectedNode['details'][$expectedKey]); + if ($expectedKey === 'attributes') { + foreach ($expectedValue as $expectedAttributeKey => $expectedAttributeValue) { + self::assertSame($expectedAttributeValue, $collectedNode['details']['attributes'][$expectedAttributeKey] ?? null, "Attribute {$expectedAttributeKey} on span {$collectedNode['details']['name']}"); + } + } else { + self::assertSame($expectedValue, $collectedNode['details'][$expectedKey]); + } } if (! array_key_exists('child', $expectedTreeNode)) { diff --git a/packages/Ecotone/tests/Tracing/TracingTreeTest.php b/packages/Ecotone/tests/Tracing/TracingTreeTest.php index ccee0be13..6d579d0a8 100644 --- a/packages/Ecotone/tests/Tracing/TracingTreeTest.php +++ b/packages/Ecotone/tests/Tracing/TracingTreeTest.php @@ -116,12 +116,17 @@ public function test_tracing_tree_with_two_levels_of_nesting(): void 'details' => ['name' => 'Event Bus'], 'children' => [ [ - 'details' => ['name' => 'Event Handler: ' . $subscriber::class . '::merchantToUser'], + 'details' => ['name' => 'Routing to: merchant.to.user', 'kind' => SpanKind::Producer], 'children' => [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'Event Handler: ' . $subscriber::class . '::merchantToUser'], 'children' => [ - ['details' => ['name' => 'Command Handler: ' . $users::class . '::register'], 'children' => []], + [ + 'details' => ['name' => 'Command Bus'], + 'children' => [ + ['details' => ['name' => 'Command Handler: ' . $users::class . '::register'], 'children' => []], + ], + ], ], ], ], @@ -139,7 +144,7 @@ public function test_tracing_tree_with_two_branches_on_same_level(): void $users = $this->users(); $subscriberOne = $this->merchantSubscriber(); $subscriberTwo = new class () { - #[EventHandler('merchant.created')] + #[EventHandler('merchant.created', endpointId: 'merchant.to.second.user')] public function merchantToUser(string $merchantId, CommandBus $commandBus): void { $commandBus->sendWithRouting('user.register', $merchantId . '-second'); @@ -164,14 +169,20 @@ public function merchantToUser(string $merchantId, CommandBus $commandBus): void [ 'details' => ['name' => 'Event Bus'], 'children' => [ - ['details' => ['name' => 'Event Handler: ' . $subscriberOne::class . '::merchantToUser'], 'children' => [$commandBusChild]], - ['details' => ['name' => 'Event Handler: ' . $subscriberTwo::class . '::merchantToUser'], 'children' => [$commandBusChild]], + [ + 'details' => ['name' => 'Routing to: merchant.to.user'], + 'children' => [['details' => ['name' => 'Event Handler: ' . $subscriberOne::class . '::merchantToUser'], 'children' => [$commandBusChild]]], + ], + [ + 'details' => ['name' => 'Routing to: merchant.to.second.user'], + 'children' => [['details' => ['name' => 'Event Handler: ' . $subscriberTwo::class . '::merchantToUser'], 'children' => [$commandBusChild]]], + ], ], ], ], self::buildTree($recorder->recordedSpans()) ); - $this->assertCount(7, self::spanNamesWithoutFrameworkNoise($recorder->recordedSpans())); + $this->assertCount(9, self::spanNamesWithoutFrameworkNoise($recorder->recordedSpans())); } public function test_tracing_tree_with_three_levels_of_nesting(): void @@ -206,12 +217,17 @@ public function create(string $merchantId, EventBus $eventBus): void 'details' => ['name' => 'Event Bus'], 'children' => [ [ - 'details' => ['name' => 'Event Handler: ' . $subscriber::class . '::merchantToUser'], + 'details' => ['name' => 'Routing to: merchant.to.user'], 'children' => [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'Event Handler: ' . $subscriber::class . '::merchantToUser'], 'children' => [ - ['details' => ['name' => 'Command Handler: ' . $users::class . '::register'], 'children' => []], + [ + 'details' => ['name' => 'Command Bus'], + 'children' => [ + ['details' => ['name' => 'Command Handler: ' . $users::class . '::register'], 'children' => []], + ], + ], ], ], ], @@ -319,12 +335,13 @@ public function test_spans_carry_message_identity_and_a_child_message_points_at_ ]); $spans = $recorder->recordedSpans(); - $this->assertCount(4, $spans); + $this->assertCount(5, $spans); $this->assertChildSpan('Command Handler: ' . $users::class . '::register', $spans[0], $messageId, $correlationId); $this->assertChildSpan('Command Bus', $spans[1], $messageId, $correlationId); $this->assertSpanWith('Event Handler: ' . $subscriber::class . '::merchantToUser', $spans[2], $messageId, $correlationId, $timestamp); - $this->assertSpanWith('Event Bus', $spans[3], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('Routing to: merchant.to.user', $spans[3], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('Event Bus', $spans[4], $messageId, $correlationId, $timestamp); } public function test_a_failing_handler_marks_its_span_and_the_bus_span_as_failed_with_the_exception_recorded(): void @@ -405,7 +422,7 @@ public function isRegistered(): bool private function merchantSubscriber(): object { return new class () { - #[EventHandler('merchant.created')] + #[EventHandler('merchant.created', endpointId: 'merchant.to.user')] public function merchantToUser(string $merchantId, CommandBus $commandBus): void { $commandBus->sendWithRouting('user.register', $merchantId); diff --git a/packages/OpenTelemetry/tests/Fixture/CommandEventFlow/MerchantSubscriberOne.php b/packages/OpenTelemetry/tests/Fixture/CommandEventFlow/MerchantSubscriberOne.php index 7ee9ac7cf..b1e51eaa5 100644 --- a/packages/OpenTelemetry/tests/Fixture/CommandEventFlow/MerchantSubscriberOne.php +++ b/packages/OpenTelemetry/tests/Fixture/CommandEventFlow/MerchantSubscriberOne.php @@ -12,7 +12,7 @@ */ final class MerchantSubscriberOne { - #[EventHandler] + #[EventHandler(endpointId: 'merchant.subscriber.one')] public function merchantToUser(MerchantCreated $event, CommandBus $commandBus): void { $commandBus->send(new RegisterUser($event->merchantId)); diff --git a/packages/OpenTelemetry/tests/Fixture/CommandEventFlow/MerchantSubscriberTwo.php b/packages/OpenTelemetry/tests/Fixture/CommandEventFlow/MerchantSubscriberTwo.php index 7c4bc2d13..a1221a68c 100644 --- a/packages/OpenTelemetry/tests/Fixture/CommandEventFlow/MerchantSubscriberTwo.php +++ b/packages/OpenTelemetry/tests/Fixture/CommandEventFlow/MerchantSubscriberTwo.php @@ -12,7 +12,7 @@ */ final class MerchantSubscriberTwo { - #[EventHandler] + #[EventHandler(endpointId: 'merchant.subscriber.two')] public function merchantToUser(MerchantCreated $event, CommandBus $commandBus): void { $commandBus->send(new RegisterUser($event->merchantId . '-second')); diff --git a/packages/OpenTelemetry/tests/Integration/CorrelatedHeadersPropagationTest.php b/packages/OpenTelemetry/tests/Integration/CorrelatedHeadersPropagationTest.php index 51e93107a..54d3e8675 100644 --- a/packages/OpenTelemetry/tests/Integration/CorrelatedHeadersPropagationTest.php +++ b/packages/OpenTelemetry/tests/Integration/CorrelatedHeadersPropagationTest.php @@ -81,14 +81,14 @@ public function test_tracing_with_two_levels_of_nesting() /** @var ImmutableSpan[] $spans */ $spans = $exporter->getSpans(); - /** Command Bus and Command Handler */ - $this->assertCount(4, $spans); + $this->assertCount(5, $spans); $this->assertChildSpan('Command Handler: ' . User::class . '::register', $spans[0], $messageId, $correlationId); $this->assertChildSpan('Command Bus', $spans[1], $messageId, $correlationId); $this->assertSpanWith('Event Handler: ' . MerchantSubscriberOne::class . '::merchantToUser', $spans[2], $messageId, $correlationId, $timestamp); - $this->assertSpanWith('Event Bus', $spans[3], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('Routing to: merchant.subscriber.one', $spans[3], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('Event Bus', $spans[4], $messageId, $correlationId, $timestamp); } public function test_tracing_with_three_levels_of_nesting() @@ -112,17 +112,17 @@ public function test_tracing_with_three_levels_of_nesting() /** @var ImmutableSpan[] $spans */ $spans = $exporter->getSpans(); - /** Command Bus and Command Handler */ - $this->assertCount(6, $spans); + $this->assertCount(7, $spans); $this->assertChildSpan('Command Handler: ' . User::class . '::register', $spans[0], $spans[2]->getAttributes()->get(MessageHeaders::MESSAGE_ID), $correlationId); $this->assertChildSpan('Command Bus', $spans[1], $spans[2]->getAttributes()->get(MessageHeaders::MESSAGE_ID), $correlationId); $this->assertChildSpan('Event Handler: ' . MerchantSubscriberOne::class . '::merchantToUser', $spans[2], $messageId, $correlationId); - $this->assertChildSpan('Event Bus', $spans[3], $messageId, $correlationId); + $this->assertChildSpan('Routing to: merchant.subscriber.one', $spans[3], $messageId, $correlationId); + $this->assertChildSpan('Event Bus', $spans[4], $messageId, $correlationId); - $this->assertSpanWith('Command Handler: ' . Merchant::class . '::create', $spans[4], $messageId, $correlationId, $timestamp); - $this->assertSpanWith('Command Bus', $spans[5], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('Command Handler: ' . Merchant::class . '::create', $spans[5], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('Command Bus', $spans[6], $messageId, $correlationId, $timestamp); } public function test_tracing_same_message_for_asynchronous_scenario() diff --git a/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php b/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php index 63d23315c..342f13a8d 100644 --- a/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php +++ b/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php @@ -335,14 +335,19 @@ public function test_tracing_with_two_levels_of_nesting() 'details' => ['name' => 'Event Bus'], 'children' => [ [ - 'details' => ['name' => 'Event Handler: ' . MerchantSubscriberOne::class . '::merchantToUser'], + 'details' => ['name' => 'Routing to: merchant.subscriber.one'], 'children' => [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'Event Handler: ' . MerchantSubscriberOne::class . '::merchantToUser'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], - 'children' => [], + 'details' => ['name' => 'Command Bus'], + 'children' => [ + [ + 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], + 'children' => [], + ], + ], ], ], ], @@ -372,28 +377,38 @@ public function test_tracing_with_two_levels_of_nesting_and_two_branches_on_same 'details' => ['name' => 'Event Bus'], 'children' => [ [ - 'details' => ['name' => 'Event Handler: ' . MerchantSubscriberOne::class . '::merchantToUser'], + 'details' => ['name' => 'Routing to: merchant.subscriber.one'], 'children' => [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'Event Handler: ' . MerchantSubscriberOne::class . '::merchantToUser'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], - 'children' => [], + 'details' => ['name' => 'Command Bus'], + 'children' => [ + [ + 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], + 'children' => [], + ], + ], ], ], ], ], ], [ - 'details' => ['name' => 'Event Handler: ' . MerchantSubscriberTwo::class . '::merchantToUser'], + 'details' => ['name' => 'Routing to: merchant.subscriber.two'], 'children' => [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'Event Handler: ' . MerchantSubscriberTwo::class . '::merchantToUser'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], - 'children' => [], + 'details' => ['name' => 'Command Bus'], + 'children' => [ + [ + 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], + 'children' => [], + ], + ], ], ], ], @@ -429,14 +444,19 @@ public function test_tracing_with_three_levels_of_nesting() 'details' => ['name' => 'Event Bus'], 'children' => [ [ - 'details' => ['name' => 'Event Handler: ' . MerchantSubscriberOne::class . '::merchantToUser'], + 'details' => ['name' => 'Routing to: merchant.subscriber.one'], 'children' => [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'Event Handler: ' . MerchantSubscriberOne::class . '::merchantToUser'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], - 'children' => [], + 'details' => ['name' => 'Command Bus'], + 'children' => [ + [ + 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], + 'children' => [], + ], + ], ], ], ], @@ -528,17 +548,13 @@ public function test_adding_logs_as_events_in_span() self::buildTree($exporter) ); - /** @var Event $event */ - $event = $result['details']['events'][0]; - $this->stringStartsWith( - 'Publishing Event Message', - )->evaluate($event->getName()); - - /** @var Event $event */ - $event = $result['details']['events'][2]; - $this->stringStartsWith( - 'Collecting message with id', - )->evaluate($event->getName()); + $eventBusEvents = array_map(fn (Event $event): string => $event->getName(), $result['details']['events']); + $this->assertNotEmpty(array_filter($eventBusEvents, fn (string $name): bool => str_starts_with($name, 'Publishing Event Message'))); + + $route = $result['children'][array_key_first($result['children'])]; + $this->assertSame('Routing to: user.registered', $route['details']['name']); + $routeEvents = array_map(fn (Event $event): string => $event->getName(), $route['details']['events']); + $this->assertNotEmpty(array_filter($routeEvents, fn (string $name): bool => str_starts_with($name, 'Collecting message with id'))); } public function test_two_traces_with_asynchronous_handlers() diff --git a/upgrade-2.0.md b/upgrade-2.0.md index af091bf63..e8a010d5d 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -2965,8 +2965,15 @@ synchronous span came from OpenTelemetry's ambient context, and without that pac the W3C `traceparent` and `tracestate` headers (`MessageHeaders::TRACEPARENT`, `MessageHeaders::TRACESTATE`), which are framework headers: they are not copied into the next message as user metadata, are not reported as span attributes, and a root span continues the trace a message already carries, so a command sent with a `traceparent` -header from an HTTP request becomes a child of that request's span. Span names, kinds, attributes and the -`exception` event are unchanged. The tracing interceptors are registered only when a recorder is configured or a +header from an HTTP request becomes a child of that request's span. Span names, kinds, the message attributes and the +`exception` event are unchanged, and three things are added. Every route the event bus resolves is a PRODUCER span, +`Routing to: ` or `Routing to Projection: `, between the `Event Bus` span and the handler it +leads to, synchronous or asynchronous, so a trace shows which handlers an event was dispatched to and a failing +synchronous handler fails its route. An event-driven projection run is a CONSUMER span, `Projection: `. And +every span carries the OpenTelemetry messaging attributes: `messaging.system` is `ecotone`, +`messaging.operation.type` is `send` for buses, routes and channel sends and `process` for handlers, consumers and +projections, `messaging.destination.name` names the route or channel, and `ecotone.endpoint.id` or +`ecotone.projection.name` name the handler; the names are constants on `Ecotone\Api\Tracing\SpanAttribute`. The tracing interceptors are registered only when a recorder is configured or a tracing package hands over its `Tracer`, so an application without tracing pays nothing. `ecotone/open-telemetry` becomes an implementation of that `Tracer`: it starts live SDK spans with the same ids Core reports, keeps the forced flush and the Dynamic Consistency Boundary spans, and records nothing when a flow test is given no @@ -2974,7 +2981,8 @@ forced flush and the Dynamic Consistency Boundary spans, and records nothing whe to `Ecotone\Api\OpenTelemetry\OpenTelemetryConfiguration`; the never-settable tracing level is gone. `MessageHeaders::TEMPORARY_SPAN_CONTEXT_HEADER`, which nothing set, is removed. -**How to adapt:** rename `Ecotone\OpenTelemetry\Configuration\TracingConfiguration` to +**How to adapt:** a trace tree asserted in a test gains the `Routing to:` level under `Event Bus`; give the handler +an explicit `endpointId` to make that span's name stable. Rename `Ecotone\OpenTelemetry\Configuration\TracingConfiguration` to `Ecotone\Api\OpenTelemetry\OpenTelemetryConfiguration`; `withForceFlushOnBusExecution()` and `withForceFlushOnAsynchronousMessageHandled()` keep their names. A consumer outside Ecotone that read the `ecotoneTracingCarrier` header reads `traceparent` instead, which any OpenTelemetry propagator already does. To From dcf1382d7d8c819f391a4be03edc106520744824 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 16 Oct 2026 08:00:00 +0200 Subject: [PATCH 05/50] feat(tracing): stored traces answer whether a trace or one of its handlers completed, and notify subscribers after commit Enterprise. TracingConfiguration::withStoredTraces() turns every span into a stored one: a unit of work buffers its spans and writes them once, inside the transaction, before commit; the root span is written when it ends, after commit, and only then are #[OnTraceCompleted] and #[OnSpanEnded] subscribers notified. The TraceCompletion gateway reads the ledger derived from the spans: a trace is complete when its root ended and every route it sent has an ended handler span, a destination is complete when its handler ended, and await() polls until then or fails naming what is still pending. SpanContext moved to Api so an application can mint the trace id it later asks about. Ships with the in-memory store; the DBAL store follows. --- packages/Ecotone/Api/Tracing/OnSpanEnded.php | 23 ++ .../Ecotone/Api/Tracing/OnTraceCompleted.php | 23 ++ .../{src => Api}/Tracing/SpanContext.php | 2 +- packages/Ecotone/Api/Tracing/SpanEnded.php | 16 + .../Ecotone/Api/Tracing/TraceCompleted.php | 16 + .../Ecotone/Api/Tracing/TraceCompletion.php | 22 ++ .../Api/Tracing/TraceCompletionTimedOut.php | 41 +++ .../Api/Tracing/TracingConfiguration.php | 12 +- packages/Ecotone/src/Tracing/ActiveSpan.php | 1 + .../Tracing/Config/TraceStoreReference.php | 15 + .../src/Tracing/Config/TracingModule.php | 143 ++++++++- .../Ecotone/src/Tracing/EcotoneActiveSpan.php | 1 + .../Ecotone/src/Tracing/EcotoneTracer.php | 1 + .../src/Tracing/ExpiredTraceRemover.php | 41 +++ .../src/Tracing/InMemoryTraceStore.php | 71 +++++ .../src/Tracing/StoredTraceCompletion.php | 80 +++++ .../src/Tracing/StoringSpanRecorder.php | 117 +++++++ packages/Ecotone/src/Tracing/TraceLedger.php | 76 +++++ .../Ecotone/src/Tracing/TraceProgress.php | 86 +++++ .../src/Tracing/TraceRecordingInterceptor.php | 49 +++ packages/Ecotone/src/Tracing/TraceStore.php | 39 +++ packages/Ecotone/src/Tracing/Tracer.php | 1 + .../src/Tracing/TracingInterceptor.php | 1 + .../tests/Tracing/TraceCompletionTest.php | 297 ++++++++++++++++++ .../src/OpenTelemetryActiveSpan.php | 2 +- .../OpenTelemetry/src/OpenTelemetryTracer.php | 2 +- 26 files changed, 1165 insertions(+), 13 deletions(-) create mode 100644 packages/Ecotone/Api/Tracing/OnSpanEnded.php create mode 100644 packages/Ecotone/Api/Tracing/OnTraceCompleted.php rename packages/Ecotone/{src => Api}/Tracing/SpanContext.php (97%) create mode 100644 packages/Ecotone/Api/Tracing/SpanEnded.php create mode 100644 packages/Ecotone/Api/Tracing/TraceCompleted.php create mode 100644 packages/Ecotone/Api/Tracing/TraceCompletion.php create mode 100644 packages/Ecotone/Api/Tracing/TraceCompletionTimedOut.php create mode 100644 packages/Ecotone/src/Tracing/Config/TraceStoreReference.php create mode 100644 packages/Ecotone/src/Tracing/ExpiredTraceRemover.php create mode 100644 packages/Ecotone/src/Tracing/InMemoryTraceStore.php create mode 100644 packages/Ecotone/src/Tracing/StoredTraceCompletion.php create mode 100644 packages/Ecotone/src/Tracing/StoringSpanRecorder.php create mode 100644 packages/Ecotone/src/Tracing/TraceLedger.php create mode 100644 packages/Ecotone/src/Tracing/TraceProgress.php create mode 100644 packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php create mode 100644 packages/Ecotone/src/Tracing/TraceStore.php create mode 100644 packages/Ecotone/tests/Tracing/TraceCompletionTest.php diff --git a/packages/Ecotone/Api/Tracing/OnSpanEnded.php b/packages/Ecotone/Api/Tracing/OnSpanEnded.php new file mode 100644 index 000000000..85b2ed90a --- /dev/null +++ b/packages/Ecotone/Api/Tracing/OnSpanEnded.php @@ -0,0 +1,23 @@ +inMilliseconds(), + $pendingDestinations === [] ? 'the root of the trace, which was not recorded yet' : implode(', ', $pendingDestinations), + )); + } + + public static function forDestination(string $traceId, string $destination, Duration $timeout): self + { + return new self(sprintf( + 'Trace %s did not complete for %s within %d milliseconds. Run the consumer of the channel that handler is attached to, or await with a longer timeout.', + $traceId, + $destination, + $timeout->inMilliseconds(), + )); + } +} diff --git a/packages/Ecotone/Api/Tracing/TracingConfiguration.php b/packages/Ecotone/Api/Tracing/TracingConfiguration.php index 47464010d..418706e5b 100644 --- a/packages/Ecotone/Api/Tracing/TracingConfiguration.php +++ b/packages/Ecotone/Api/Tracing/TracingConfiguration.php @@ -4,6 +4,8 @@ namespace Ecotone\Api\Tracing; +use Ecotone\Api\Scheduling\Duration; + /** * licence Apache-2.0 */ @@ -14,16 +16,24 @@ final class TracingConfiguration */ private function __construct( public readonly array $spanRecorderReferenceNames, + public readonly bool $storedTraces, + public readonly bool $storedTracesInMemory, + public readonly Duration $storedTracesRetention, ) { } public static function createWithDefaults(): self { - return new self([]); + return new self([], false, false, Duration::minutes(60 * 24 * 7)); } public function withSpanRecorder(string $referenceName): self { return new self(...[...get_object_vars($this), 'spanRecorderReferenceNames' => [...$this->spanRecorderReferenceNames, $referenceName]]); } + + public function withStoredTraces(bool $inMemory = false, ?Duration $retention = null): self + { + return new self(...[...get_object_vars($this), 'storedTraces' => true, 'storedTracesInMemory' => $inMemory, 'storedTracesRetention' => $retention ?? $this->storedTracesRetention]); + } } diff --git a/packages/Ecotone/src/Tracing/ActiveSpan.php b/packages/Ecotone/src/Tracing/ActiveSpan.php index 43cc2a08b..aab782105 100644 --- a/packages/Ecotone/src/Tracing/ActiveSpan.php +++ b/packages/Ecotone/src/Tracing/ActiveSpan.php @@ -4,6 +4,7 @@ namespace Ecotone\Tracing; +use Ecotone\Api\Tracing\SpanContext; use Throwable; /** diff --git a/packages/Ecotone/src/Tracing/Config/TraceStoreReference.php b/packages/Ecotone/src/Tracing/Config/TraceStoreReference.php new file mode 100644 index 000000000..7212a7be2 --- /dev/null +++ b/packages/Ecotone/src/Tracing/Config/TraceStoreReference.php @@ -0,0 +1,15 @@ + 1) { throw ConfigurationException::create('Only one Tracer can be active, yet these were registered: ' . implode(', ', array_map(fn (TracerReference $reference): string => $reference->referenceName, $tracerReferences)) . '. Enable a single tracing package.'); } + $this->refuseMisconfiguredStoredTraces($messagingConfiguration, $tracingConfiguration); - if ($tracingConfiguration->spanRecorderReferenceNames === [] && $tracerReferences === []) { + if ($tracingConfiguration->spanRecorderReferenceNames === [] && $tracerReferences === [] && ! $tracingConfiguration->storedTraces) { return; } - $messagingConfiguration->registerServiceDefinition( - SpanRecorder::class, - new Definition(CompositeSpanRecorder::class, [ - array_map(fn (string $referenceName): Reference => new Reference($referenceName), $tracingConfiguration->spanRecorderReferenceNames), - ]) - ); + $spanRecorders = array_map(fn (string $referenceName): Reference => new Reference($referenceName), $tracingConfiguration->spanRecorderReferenceNames); + if ($tracingConfiguration->storedTraces) { + $this->registerStoredTraces($messagingConfiguration, $extensionObjects, $tracingConfiguration, $interfaceToCallRegistry); + $spanRecorders[] = new Reference(StoringSpanRecorder::class); + } + + $messagingConfiguration->registerServiceDefinition(SpanRecorder::class, new Definition(CompositeSpanRecorder::class, [$spanRecorders])); $messagingConfiguration->registerServiceDefinition( Tracer::class, $tracerReferences === [] @@ -91,7 +127,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO 'traceCommandHandler' => CommandHandler::class, 'traceQueryHandler' => QueryHandler::class, 'traceEventHandler' => EventHandler::class, - 'traceMessageHandler' => InternalHandler::class . '&¬(' . MessageHeadersPropagatorInterceptor::class . ')', + 'traceMessageHandler' => InternalHandler::class . '&¬(' . MessageHeadersPropagatorInterceptor::class . ')&¬(' . OnTraceCompleted::class . ')&¬(' . OnSpanEnded::class . ')', 'traceCommandBus' => CommandBus::class, 'traceQueryBus' => QueryBus::class, 'traceEventBus' => EventBus::class, @@ -129,4 +165,93 @@ public function getModulePackageName(): string { return ModulePackageList::CORE_PACKAGE; } + + private function refuseMisconfiguredStoredTraces(Configuration $messagingConfiguration, TracingConfiguration $tracingConfiguration): void + { + $subscribers = [...$this->traceCompletedSubscribers, ...$this->spanEndedSubscribers]; + if (($tracingConfiguration->storedTraces || $subscribers !== []) && ! $messagingConfiguration->isRunningForEnterpriseLicence()) { + throw LicensingException::create('Stored traces and TraceCompletion require an Ecotone Enterprise licence.'); + } + if ($tracingConfiguration->storedTraces || $subscribers === []) { + return; + } + + throw ConfigurationException::create( + implode(', ', $subscribers) . ' needs stored traces, which decide when a trace is complete. Register TracingConfiguration::createWithDefaults()->withStoredTraces() from a #[ServiceContext] method.' + ); + } + + /** + * @param object[] $extensionObjects + */ + private function registerStoredTraces(Configuration $messagingConfiguration, array $extensionObjects, TracingConfiguration $tracingConfiguration, InterfaceToCallRegistry $interfaceToCallRegistry): void + { + $traceStoreReferences = ExtensionObjectResolver::resolve(TraceStoreReference::class, $extensionObjects); + $messagingConfiguration->registerServiceDefinition( + TraceStore::class, + $traceStoreReferences === [] || $tracingConfiguration->storedTracesInMemory + ? new Definition(InMemoryTraceStore::class) + : new Reference($traceStoreReferences[0]->referenceName) + ); + $messagingConfiguration->registerServiceDefinition( + StoringSpanRecorder::class, + new Definition(StoringSpanRecorder::class, [ + new Reference(TraceStore::class), + new Reference(MessagingEntrypointService::class), + $this->spanEndedSubscribers !== [], + $this->traceCompletedSubscribers !== [], + ]) + ); + $messagingConfiguration->registerServiceDefinition( + TraceRecordingInterceptor::class, + new Definition(TraceRecordingInterceptor::class, [new Reference(StoringSpanRecorder::class)]) + ); + $messagingConfiguration->registerAroundMethodInterceptor( + AroundInterceptorBuilder::create( + TraceRecordingInterceptor::class, + $interfaceToCallRegistry->getFor(TraceRecordingInterceptor::class, 'recordUnitOfWork'), + Precedence::DATABASE_TRANSACTION_PRECEDENCE + 1, + CommandBus::class . '||' . EventBus::class . '||' . QueryBus::class . '||' . AsynchronousRunningEndpoint::class + ) + ); + $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnTraceCompleted::CHANNEL)); + $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnSpanEnded::CHANNEL)); + + $messagingConfiguration->registerServiceDefinition( + StoredTraceCompletion::class, + new Definition(StoredTraceCompletion::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class)]) + ); + foreach ([ + 'isCompleted' => [GatewayPayloadBuilder::create('traceId')], + 'isCompletedFor' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('destination', StoredTraceCompletion::DESTINATION_HEADER)], + 'await' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('timeout', StoredTraceCompletion::TIMEOUT_HEADER)], + 'awaitFor' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('destination', StoredTraceCompletion::DESTINATION_HEADER), GatewayHeaderBuilder::create('timeout', StoredTraceCompletion::TIMEOUT_HEADER)], + ] as $method => $parameterConverters) { + $channel = 'ecotone.tracing.completion.' . $method; + $messagingConfiguration + ->registerMessageHandler(ServiceActivatorBuilder::create(StoredTraceCompletion::class, $method)->withInputChannelName($channel)) + ->registerGatewayBuilder(GatewayProxyBuilder::create(TraceCompletion::class, TraceCompletion::class, $method, $channel)->withParameterConverters($parameterConverters)); + } + + $messagingConfiguration->registerServiceDefinition( + ExpiredTraceRemover::class, + new Definition(ExpiredTraceRemover::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $tracingConfiguration->storedTracesRetention->inMilliseconds()]) + ); + $messagingConfiguration + ->registerMessageHandler(ServiceActivatorBuilder::create(ExpiredTraceRemover::class, 'removeExpiredTraces')->withInputChannelName(self::REMOVE_EXPIRED_TRACES_CHANNEL)) + ->registerConsoleCommand(ConsoleCommandConfiguration::create(self::REMOVE_EXPIRED_TRACES_CHANNEL, 'ecotone:tracing:remove-expired-traces', [], 'Removes stored traces older than the configured retention')); + } + + /** + * @return string[] + */ + private static function placementsOf(AnnotationFinder $annotationFinder, string $attributeClassName): array + { + $placements = []; + foreach ($annotationFinder->findAnnotatedMethods($attributeClassName) as $annotatedMethod) { + $placements[] = '#[' . (new ReflectionClass($attributeClassName))->getShortName() . '] on ' . $annotatedMethod->getClassName() . '::' . $annotatedMethod->getMethodName(); + } + + return $placements; + } } diff --git a/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php b/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php index 3fb8cc4da..b6488ce48 100644 --- a/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php +++ b/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php @@ -7,6 +7,7 @@ use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Scheduling\DatePoint; use Ecotone\Api\Tracing\Span; +use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\SpanEvent; use Ecotone\Api\Tracing\SpanKind; use Ecotone\Api\Tracing\SpanStatus; diff --git a/packages/Ecotone/src/Tracing/EcotoneTracer.php b/packages/Ecotone/src/Tracing/EcotoneTracer.php index 55c008afe..a3d1b1073 100644 --- a/packages/Ecotone/src/Tracing/EcotoneTracer.php +++ b/packages/Ecotone/src/Tracing/EcotoneTracer.php @@ -10,6 +10,7 @@ use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Tracing\Span; +use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\SpanKind; use Ecotone\Api\Tracing\SpanRecorder; diff --git a/packages/Ecotone/src/Tracing/ExpiredTraceRemover.php b/packages/Ecotone/src/Tracing/ExpiredTraceRemover.php new file mode 100644 index 000000000..8c1acdc65 --- /dev/null +++ b/packages/Ecotone/src/Tracing/ExpiredTraceRemover.php @@ -0,0 +1,41 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +final class ExpiredTraceRemover +{ + public function __construct( + private TraceStore $traceStore, + private EcotoneClockInterface $clock, + private int $retentionInMilliseconds, + ) { + } + + public function removeExpiredTraces(): void + { + $this->traceStore->removeTracesStartedBefore($this->clock->now()->sub(Duration::milliseconds($this->retentionInMilliseconds))); + } +} diff --git a/packages/Ecotone/src/Tracing/InMemoryTraceStore.php b/packages/Ecotone/src/Tracing/InMemoryTraceStore.php new file mode 100644 index 000000000..54e8256ec --- /dev/null +++ b/packages/Ecotone/src/Tracing/InMemoryTraceStore.php @@ -0,0 +1,71 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +final class InMemoryTraceStore implements TraceStore +{ + /** @var array */ + private array $spansByTrace = []; + /** @var array */ + private array $completedTraces = []; + + public function record(array $spans): array + { + $touchedTraces = []; + foreach ($spans as $span) { + $this->spansByTrace[$span->traceId][] = $span; + $touchedTraces[$span->traceId] = true; + } + + $completedNow = []; + foreach (array_keys($touchedTraces) as $traceId) { + if (isset($this->completedTraces[$traceId]) || ! $this->progressOf($traceId)->isCompleted()) { + continue; + } + $this->completedTraces[$traceId] = true; + $completedNow[] = $traceId; + } + + return $completedNow; + } + + public function progressOf(string $traceId): TraceProgress + { + return TraceLedger::progressOf($traceId, $this->spansByTrace[$traceId] ?? []); + } + + public function removeTracesStartedBefore(DatePoint $threshold): void + { + foreach ($this->spansByTrace as $traceId => $spans) { + foreach ($spans as $span) { + if ($span->startedAt < $threshold) { + unset($this->spansByTrace[$traceId], $this->completedTraces[$traceId]); + continue 2; + } + } + } + } +} diff --git a/packages/Ecotone/src/Tracing/StoredTraceCompletion.php b/packages/Ecotone/src/Tracing/StoredTraceCompletion.php new file mode 100644 index 000000000..c3f6eeabe --- /dev/null +++ b/packages/Ecotone/src/Tracing/StoredTraceCompletion.php @@ -0,0 +1,80 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +final class StoredTraceCompletion +{ + public const DESTINATION_HEADER = 'ecotone.tracing.destination'; + public const TIMEOUT_HEADER = 'ecotone.tracing.timeout'; + private const POLL_INTERVAL_IN_MILLISECONDS = 50; + + public function __construct( + private TraceStore $traceStore, + private EcotoneClockInterface $clock, + ) { + } + + public function isCompleted(string $traceId): bool + { + return $this->traceStore->progressOf($traceId)->isCompleted(); + } + + public function isCompletedFor(string $traceId, #[Header(self::DESTINATION_HEADER)] string $destination): bool + { + return $this->traceStore->progressOf($traceId)->isCompletedFor($destination); + } + + public function await(string $traceId, #[Header(self::TIMEOUT_HEADER)] Duration $timeout): void + { + $deadline = $this->clock->now()->add($timeout); + while (true) { + $progress = $this->traceStore->progressOf($traceId); + if ($progress->isCompleted()) { + return; + } + if ($this->clock->now() >= $deadline) { + throw TraceCompletionTimedOut::forTrace($traceId, $timeout, $progress->pendingDestinations()); + } + $this->clock->sleep(Duration::milliseconds(self::POLL_INTERVAL_IN_MILLISECONDS)); + } + } + + public function awaitFor(string $traceId, #[Header(self::DESTINATION_HEADER)] string $destination, #[Header(self::TIMEOUT_HEADER)] Duration $timeout): void + { + $deadline = $this->clock->now()->add($timeout); + while (true) { + if ($this->traceStore->progressOf($traceId)->isCompletedFor($destination)) { + return; + } + if ($this->clock->now() >= $deadline) { + throw TraceCompletionTimedOut::forDestination($traceId, $destination, $timeout); + } + $this->clock->sleep(Duration::milliseconds(self::POLL_INTERVAL_IN_MILLISECONDS)); + } + } +} diff --git a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php new file mode 100644 index 000000000..791bed09f --- /dev/null +++ b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php @@ -0,0 +1,117 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +final class StoringSpanRecorder implements SpanRecorder +{ + /** @var Span[] */ + private array $buffer = []; + private bool $scopeActive = false; + /** @var Span[] */ + private array $endedSpansToPublish = []; + /** @var string[] */ + private array $completedTracesToPublish = []; + + public function __construct( + private TraceStore $traceStore, + private MessagingEntrypointService $messagingEntrypoint, + private bool $publishesEndedSpans, + private bool $publishesCompletedTraces, + ) { + } + + public function record(Span $span): void + { + if ($this->scopeActive) { + $this->buffer[] = $span; + + return; + } + + $this->write([...$this->buffer, $span]); + $this->buffer = []; + $this->publish(); + } + + public function openScope(): void + { + $this->scopeActive = true; + } + + public function isScopeActive(): bool + { + return $this->scopeActive; + } + + public function flush(): void + { + $this->write($this->buffer); + $this->buffer = []; + } + + public function closeScope(): void + { + $this->scopeActive = false; + } + + /** + * @param Span[] $spans + */ + private function write(array $spans): void + { + if ($spans === []) { + return; + } + + $completedTraces = $this->traceStore->record($spans); + if ($this->publishesEndedSpans) { + $this->endedSpansToPublish = [...$this->endedSpansToPublish, ...$spans]; + } + if ($this->publishesCompletedTraces) { + $this->completedTracesToPublish = [...$this->completedTracesToPublish, ...$completedTraces]; + } + } + + private function publish(): void + { + $endedSpans = $this->endedSpansToPublish; + $completedTraces = $this->completedTracesToPublish; + $this->endedSpansToPublish = []; + $this->completedTracesToPublish = []; + + foreach ($endedSpans as $span) { + $this->messagingEntrypoint->send(new SpanEnded($span), OnSpanEnded::CHANNEL); + } + foreach ($completedTraces as $traceId) { + $this->messagingEntrypoint->send(new TraceCompleted($traceId), OnTraceCompleted::CHANNEL); + } + } +} diff --git a/packages/Ecotone/src/Tracing/TraceLedger.php b/packages/Ecotone/src/Tracing/TraceLedger.php new file mode 100644 index 000000000..fb7a0c826 --- /dev/null +++ b/packages/Ecotone/src/Tracing/TraceLedger.php @@ -0,0 +1,76 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +final class TraceLedger +{ + /** + * @param Span[] $spans + */ + public static function progressOf(string $traceId, array $spans): TraceProgress + { + if ($spans === []) { + return TraceProgress::unknown($traceId); + } + + $recordedSpanIds = []; + foreach ($spans as $span) { + $recordedSpanIds[$span->spanId] = true; + } + + $rootEnded = false; + $sent = []; + $processed = []; + $failed = []; + foreach ($spans as $span) { + if ($span->parentSpanId === null || ! isset($recordedSpanIds[$span->parentSpanId])) { + $rootEnded = true; + } + $destination = self::destinationOf($span); + if ($destination === null) { + continue; + } + $operation = $span->attributes[SpanAttribute::MESSAGING_OPERATION_TYPE] ?? null; + if ($operation === SpanAttribute::OPERATION_SEND) { + $sent[$destination] = ($sent[$destination] ?? 0) + 1; + } elseif ($operation === SpanAttribute::OPERATION_PROCESS && $span->status === SpanStatus::Error) { + $failed[$destination] = ($failed[$destination] ?? 0) + 1; + } elseif ($operation === SpanAttribute::OPERATION_PROCESS) { + $processed[$destination] = ($processed[$destination] ?? 0) + 1; + } + } + + return new TraceProgress($traceId, true, $rootEnded, $sent, $processed, $failed); + } + + public static function destinationOf(Span $span): ?string + { + $destination = $span->attributes[SpanAttribute::ENDPOINT_ID] ?? $span->attributes[SpanAttribute::PROJECTION_NAME] ?? null; + + return $destination === null ? null : (string) $destination; + } +} diff --git a/packages/Ecotone/src/Tracing/TraceProgress.php b/packages/Ecotone/src/Tracing/TraceProgress.php new file mode 100644 index 000000000..dc26e59fa --- /dev/null +++ b/packages/Ecotone/src/Tracing/TraceProgress.php @@ -0,0 +1,86 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +final class TraceProgress +{ + /** + * @param array $sent + * @param array $processed + * @param array $failed + */ + public function __construct( + public readonly string $traceId, + public readonly bool $recorded, + public readonly bool $rootEnded, + public readonly array $sent, + public readonly array $processed, + public readonly array $failed, + ) { + } + + public static function unknown(string $traceId): self + { + return new self($traceId, false, false, [], [], []); + } + + public function isCompleted(): bool + { + return $this->recorded && $this->rootEnded && $this->pendingDestinations() === []; + } + + public function isCompletedFor(string $destination): bool + { + if (! $this->recorded) { + return false; + } + $processed = $this->processed[$destination] ?? 0; + + return $processed > 0 && $processed >= ($this->sent[$destination] ?? 0); + } + + /** + * @return string[] + */ + public function pendingDestinations(): array + { + $pending = []; + foreach ($this->sent as $destination => $sent) { + if (($this->processed[$destination] ?? 0) < $sent) { + $pending[] = $destination; + } + } + + return $pending; + } + + /** + * @return string[] + */ + public function destinations(): array + { + return array_keys($this->sent + $this->processed + $this->failed); + } +} diff --git a/packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php b/packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php new file mode 100644 index 000000000..12ea406c8 --- /dev/null +++ b/packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php @@ -0,0 +1,49 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +final class TraceRecordingInterceptor +{ + public function __construct(private StoringSpanRecorder $recorder) + { + } + + public function recordUnitOfWork(MethodInvocation $methodInvocation): mixed + { + if ($this->recorder->isScopeActive()) { + return $methodInvocation->proceed(); + } + + $this->recorder->openScope(); + try { + $result = $methodInvocation->proceed(); + $this->recorder->flush(); + + return $result; + } finally { + $this->recorder->closeScope(); + } + } +} diff --git a/packages/Ecotone/src/Tracing/TraceStore.php b/packages/Ecotone/src/Tracing/TraceStore.php new file mode 100644 index 000000000..e18e88a59 --- /dev/null +++ b/packages/Ecotone/src/Tracing/TraceStore.php @@ -0,0 +1,39 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +interface TraceStore +{ + /** + * @param Span[] $spans + * @return string[] identifiers of the traces this recording completed + */ + public function record(array $spans): array; + + public function progressOf(string $traceId): TraceProgress; + + public function removeTracesStartedBefore(DatePoint $threshold): void; +} diff --git a/packages/Ecotone/src/Tracing/Tracer.php b/packages/Ecotone/src/Tracing/Tracer.php index 5b278ec71..e7d348315 100644 --- a/packages/Ecotone/src/Tracing/Tracer.php +++ b/packages/Ecotone/src/Tracing/Tracer.php @@ -4,6 +4,7 @@ namespace Ecotone\Tracing; +use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\SpanKind; /** diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index c47e160a0..6aa6b2465 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -12,6 +12,7 @@ use Ecotone\Api\Messaging\Message; use Ecotone\Api\Messaging\MessageHeaders; use Ecotone\Api\Tracing\SpanAttribute; +use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\SpanKind; use Ecotone\Messaging\Support\MessageBuilder; use Ecotone\Projecting\ProjectionTriggerEndpoint; diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php new file mode 100644 index 000000000..51b504b12 --- /dev/null +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -0,0 +1,297 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + * + * @internal + */ +final class TraceCompletionTest extends TestCase +{ + public function test_a_command_handled_synchronously_is_completed_when_the_bus_returns(): void + { + $ecotone = $this->bootstrap([$this->users()]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.registration')); + } + + public function test_an_asynchronous_handler_keeps_the_trace_pending_until_its_consumer_ran(): void + { + $ecotone = $this->bootstrap([$this->users(), $this->asynchronousNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.registration')); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + public function test_an_asynchronous_projection_keeps_the_trace_pending_until_it_projected(): void + { + $projection = new #[Projection('users_list'), FromStream('users_stream'), Asynchronous('async_channel')] class () { + #[EventHandler('user.registered')] + public function whenRegistered(array $event): void + { + } + }; + $ecotone = $this->bootstrap([$projection], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $ecotone->withEvents([Event::createWithType('user.registered', ['userId' => '1'])]); + $trace = SpanContext::newTrace(); + + $ecotone->publishEventWithRouting('user.registered', [], metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'users_list')); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'users_list')); + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pending(): void + { + $ecotone = $this->bootstrap([$this->users(), $this->asynchronousNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->expectException(TraceCompletionTimedOut::class); + $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier. Run the consumer of the channel those handlers are attached to, or await with a longer timeout."); + + $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); + } + + public function test_awaiting_a_completed_trace_returns_at_once(): void + { + $ecotone = $this->bootstrap([$this->users()]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); + $this->completion($ecotone)->awaitFor($trace->traceId, 'user.registration', Duration::milliseconds(100)); + + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + public function test_a_trace_nobody_recorded_is_not_completed(): void + { + $ecotone = $this->bootstrap([$this->users()]); + + $this->assertFalse($this->completion($ecotone)->isCompleted(SpanContext::newTrace()->traceId)); + } + + public function test_a_failing_asynchronous_handler_leaves_the_trace_pending(): void + { + $notifier = new class () { + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId): void + { + throw new RuntimeException('Notification failed'); + } + }; + $ecotone = $this->bootstrap([$this->users(), $notifier], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + try { + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + } catch (RuntimeException) { + } + + $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); + } + + public function test_a_trace_completed_subscriber_is_called_once_after_the_last_handler_committed(): void + { + $subscriber = new class () { + /** @var string[] */ + public array $completedTraces = []; + + #[OnTraceCompleted] + public function whenCompleted(TraceCompleted $event): void + { + $this->completedTraces[] = $event->traceId; + } + }; + $ecotone = $this->bootstrap([$this->users(), $this->asynchronousNotifier(), $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + $this->assertSame([], $subscriber->completedTraces); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $this->assertSame([$trace->traceId], $subscriber->completedTraces); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $this->assertSame([$trace->traceId], $subscriber->completedTraces); + } + + public function test_a_synchronous_trace_notifies_its_subscriber_when_the_bus_returns(): void + { + $subscriber = new class () { + /** @var string[] */ + public array $completedTraces = []; + + #[OnTraceCompleted] + public function whenCompleted(TraceCompleted $event): void + { + $this->completedTraces[] = $event->traceId; + } + }; + $ecotone = $this->bootstrap([$this->users(), $subscriber]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->assertSame([$trace->traceId], $subscriber->completedTraces); + } + + public function test_a_span_ended_subscriber_receives_every_stored_span_of_the_trace(): void + { + $subscriber = new class () { + /** @var string[] */ + public array $spanNames = []; + + #[OnSpanEnded] + public function whenEnded(SpanEnded $event): void + { + $this->spanNames[] = $event->span->name; + } + }; + $users = $this->users(); + $ecotone = $this->bootstrap([$users, $subscriber]); + + $ecotone->sendCommandWithRouting('user.register', '1'); + + $this->assertContains('Command Bus', $subscriber->spanNames); + $this->assertContains('Command Handler: ' . $users::class . '::register', $subscriber->spanNames); + } + + public function test_stored_traces_require_an_enterprise_licence(): void + { + $this->expectException(LicensingException::class); + $this->expectExceptionMessage('Stored traces and TraceCompletion require an Ecotone Enterprise licence.'); + + EcotoneLite::bootstrapFlowTesting( + [], + [], + [TracingConfiguration::createWithDefaults()->withStoredTraces()], + ); + } + + public function test_a_trace_completed_subscriber_without_stored_traces_is_refused_at_bootstrap(): void + { + $subscriber = new class () { + #[OnTraceCompleted] + public function whenCompleted(TraceCompleted $event): void + { + } + }; + + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage('#[OnTraceCompleted] on ' . $subscriber::class . '::whenCompleted needs stored traces, which decide when a trace is complete. Register TracingConfiguration::createWithDefaults()->withStoredTraces() from a #[ServiceContext] method.'); + + EcotoneLite::bootstrapFlowTesting( + [$subscriber::class], + [$subscriber], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + + /** + * @param object[] $services + * @param object[] $extensionObjects + */ + private function bootstrap(array $services, array $extensionObjects = []): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + array_map(fn (object $service): string => $service::class, $services), + $services, + [TracingConfiguration::createWithDefaults()->withStoredTraces(), ...$extensionObjects], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + + private function completion(FlowTestSupport $ecotone): TraceCompletion + { + return $ecotone->getGateway(TraceCompletion::class); + } + + private function users(): object + { + return new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId, EventBus $eventBus): void + { + $eventBus->publishWithRouting('user.registered', $userId); + } + }; + } + + private function asynchronousNotifier(): object + { + return new class () { + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId): void + { + } + }; + } +} diff --git a/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php b/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php index 3bf871489..2e0ab6c9b 100644 --- a/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php +++ b/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php @@ -7,11 +7,11 @@ use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Scheduling\DatePoint; use Ecotone\Api\Tracing\Span; +use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\SpanEvent; use Ecotone\Api\Tracing\SpanKind; use Ecotone\Api\Tracing\SpanStatus; use Ecotone\Tracing\ActiveSpan; -use Ecotone\Tracing\SpanContext; use OpenTelemetry\API\Trace\SpanInterface; use OpenTelemetry\API\Trace\StatusCode; use OpenTelemetry\Context\ScopeInterface; diff --git a/packages/OpenTelemetry/src/OpenTelemetryTracer.php b/packages/OpenTelemetry/src/OpenTelemetryTracer.php index 6b7bd6572..72cd862f0 100644 --- a/packages/OpenTelemetry/src/OpenTelemetryTracer.php +++ b/packages/OpenTelemetry/src/OpenTelemetryTracer.php @@ -11,10 +11,10 @@ use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Messaging\MessageHeaders; use Ecotone\Api\Tracing\Span; +use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\SpanKind; use Ecotone\Api\Tracing\SpanRecorder; use Ecotone\Tracing\ActiveSpan; -use Ecotone\Tracing\SpanContext; use Ecotone\Tracing\Tracer; use OpenTelemetry\API\Trace\Propagation\TraceContextPropagator; use OpenTelemetry\API\Trace\SpanKind as OpenTelemetrySpanKind; From 7c8dddfbe91d197fd353cfb702e52440e721188a Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 16 Oct 2026 08:00:00 +0200 Subject: [PATCH 06/50] feat(dbal): stored traces live in ecotone_traces and ecotone_spans, written inside the handler's transaction Enterprise. DbalTraceStore claims a row per trace, locks it while it appends the unit of work's spans and decides completion from the same ledger the in-memory store uses, and marks the trace completed exactly once so subscribers are notified once across concurrent consumers. The tables come from ecotone:migration:database:setup, a flow test without the connection falls back to the in-memory store, and ecotone:tracing:remove-expired-traces applies the configured retention. One conformance suite runs both stores, plus the cross-instance visibility case only the database holds. --- .../FlowTestingDbalConnections.php | 16 +- .../Dbal/src/Database/SpansTableManager.php | 125 ++++++++++ .../Dbal/src/Database/TracesTableManager.php | 117 +++++++++ packages/Dbal/src/Tracing/DbalTraceStore.php | 231 ++++++++++++++++++ .../Dbal/src/Tracing/DbalTracingModule.php | 119 +++++++++ .../Conformance/TraceStoreConformanceTest.php | 217 ++++++++++++++++ packages/Dbal/tests/DbalMessagingTestCase.php | 2 + .../src/Messaging/Config/ModuleClassList.php | 2 + 8 files changed, 828 insertions(+), 1 deletion(-) create mode 100644 packages/Dbal/src/Database/SpansTableManager.php create mode 100644 packages/Dbal/src/Database/TracesTableManager.php create mode 100644 packages/Dbal/src/Tracing/DbalTraceStore.php create mode 100644 packages/Dbal/src/Tracing/DbalTracingModule.php create mode 100644 packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php diff --git a/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php b/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php index 8a88768e2..e2ef69956 100644 --- a/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php +++ b/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php @@ -8,6 +8,7 @@ use Ecotone\Api\Dbal\ExtensionObject\DbalConfiguration; use Ecotone\Api\Dbal\ExtensionObject\DbalConnectionReference; use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; +use Ecotone\Api\Tracing\TracingConfiguration; use Ecotone\Lite\Test\Configuration\ProvidedReferences; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\ExtensionObjectResolver; use Ecotone\Messaging\Config\ConfigurationException; @@ -48,9 +49,12 @@ public function resolveExtensionObjects(): array $dbalConfiguration = ExtensionObjectResolver::resolveUnique(DbalConfiguration::class, $this->extensionObjects, DbalConfiguration::createWithDefaults()); + $tracingConfiguration = ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $this->extensionObjects, TracingConfiguration::createWithDefaults()); + return [ - ...array_filter($this->extensionObjects, fn (object $extensionObject) => ! $extensionObject instanceof DbalConfiguration), + ...array_filter($this->extensionObjects, fn (object $extensionObject) => ! $extensionObject instanceof DbalConfiguration && ! $extensionObject instanceof TracingConfiguration), $this->inMemoryWhereTheConnectionIsMissing($dbalConfiguration), + $this->tracesInMemoryWhereTheConnectionIsMissing($tracingConfiguration, $dbalConfiguration), ]; } @@ -100,6 +104,16 @@ private function inMemoryWhereTheConnectionIsMissing(DbalConfiguration $dbalConf return $dbalConfiguration; } + private function tracesInMemoryWhereTheConnectionIsMissing(TracingConfiguration $tracingConfiguration, DbalConfiguration $dbalConfiguration): TracingConfiguration + { + $connectionReferenceName = $dbalConfiguration->defaultConnectionReferenceNames[0] ?? DbalConnectionReference::DEFAULT; + if (! $tracingConfiguration->storedTraces || $tracingConfiguration->storedTracesInMemory || $this->has($connectionReferenceName)) { + return $tracingConfiguration; + } + + return $tracingConfiguration->withStoredTraces(inMemory: true, retention: $tracingConfiguration->storedTracesRetention); + } + private function refuseDbalChannelsWithoutTheirConnection(): void { foreach (ExtensionObjectResolver::resolve(DbalBackedMessageChannelBuilder::class, $this->extensionObjects) as $channelBuilder) { diff --git a/packages/Dbal/src/Database/SpansTableManager.php b/packages/Dbal/src/Database/SpansTableManager.php new file mode 100644 index 000000000..eed1df4c4 --- /dev/null +++ b/packages/Dbal/src/Database/SpansTableManager.php @@ -0,0 +1,125 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +final class SpansTableManager implements DbalTableManager +{ + public const DEFAULT_TABLE = 'ecotone_spans'; + public const FEATURE_NAME = 'tracing'; + + public function __construct( + private string $tableName, + private bool $isUsed, + private ?string $consoleInvocationPrefix = null, + private ?string $connectionReferenceName = null, + ) { + } + + public function getFeatureName(): string + { + return self::FEATURE_NAME; + } + + public function isUsed(): bool + { + return $this->isUsed; + } + + public function getTableName(): string + { + return $this->tableName; + } + + public function getCreateTableSql(Connection $connection): string|array + { + return DbalIdempotentDdl::withIfNotExists($connection->getDatabasePlatform()->getCreateTableSQL($this->buildTableSchema())); + } + + public function getDropTableSql(Connection $connection): string + { + return "DROP TABLE IF EXISTS {$this->tableName}"; + } + + public function createTable(Connection $connection): void + { + if ($this->isInitialized($connection)) { + return; + } + + try { + $connection->createSchemaManager()->createTable($this->buildTableSchema()); + } catch (TableExistsException) { + } + } + + public function getMissingTableInstructions(): string + { + return MissingTableInstructions::build(self::FEATURE_NAME, $this->tableName, $this->consoleInvocationPrefix, $this->connectionReferenceName); + } + + public function dropTable(Connection $connection): void + { + $schemaManager = $connection->createSchemaManager(); + if (! $schemaManager->tablesExist([$this->tableName])) { + return; + } + + $schemaManager->dropTable($this->tableName); + } + + public function isInitialized(Connection $connection): bool + { + return $connection->createSchemaManager()->tableExists($this->tableName); + } + + public function getDefinition(): Definition + { + return new Definition(self::class, [$this->tableName, $this->isUsed, $this->consoleInvocationPrefix, $this->connectionReferenceName]); + } + + private function buildTableSchema(): Table + { + $table = new Table($this->tableName); + $table->addColumn('span_id', Types::STRING, ['length' => 16]); + $table->addColumn('trace_id', Types::STRING, ['length' => 32]); + $table->addColumn('parent_span_id', Types::STRING, ['length' => 16, 'notnull' => false]); + $table->addColumn('name', Types::STRING, ['length' => 255]); + $table->addColumn('kind', Types::STRING, ['length' => 16]); + $table->addColumn('status', Types::STRING, ['length' => 16]); + $table->addColumn('status_description', Types::TEXT, ['notnull' => false]); + $table->addColumn('attributes', Types::TEXT); + $table->addColumn('events', Types::TEXT); + $table->addColumn('started_at', Types::BIGINT); + $table->addColumn('ended_at', Types::BIGINT); + $table->setPrimaryKey(['span_id']); + $table->addIndex(['trace_id']); + + return $table; + } +} diff --git a/packages/Dbal/src/Database/TracesTableManager.php b/packages/Dbal/src/Database/TracesTableManager.php new file mode 100644 index 000000000..04214f143 --- /dev/null +++ b/packages/Dbal/src/Database/TracesTableManager.php @@ -0,0 +1,117 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +final class TracesTableManager implements DbalTableManager +{ + public const DEFAULT_TABLE = 'ecotone_traces'; + public const FEATURE_NAME = 'tracing'; + + public function __construct( + private string $tableName, + private bool $isUsed, + private ?string $consoleInvocationPrefix = null, + private ?string $connectionReferenceName = null, + ) { + } + + public function getFeatureName(): string + { + return self::FEATURE_NAME; + } + + public function isUsed(): bool + { + return $this->isUsed; + } + + public function getTableName(): string + { + return $this->tableName; + } + + public function getCreateTableSql(Connection $connection): string|array + { + return DbalIdempotentDdl::withIfNotExists($connection->getDatabasePlatform()->getCreateTableSQL($this->buildTableSchema())); + } + + public function getDropTableSql(Connection $connection): string + { + return "DROP TABLE IF EXISTS {$this->tableName}"; + } + + public function createTable(Connection $connection): void + { + if ($this->isInitialized($connection)) { + return; + } + + try { + $connection->createSchemaManager()->createTable($this->buildTableSchema()); + } catch (TableExistsException) { + } + } + + public function getMissingTableInstructions(): string + { + return MissingTableInstructions::build(self::FEATURE_NAME, $this->tableName, $this->consoleInvocationPrefix, $this->connectionReferenceName); + } + + public function dropTable(Connection $connection): void + { + $schemaManager = $connection->createSchemaManager(); + if (! $schemaManager->tablesExist([$this->tableName])) { + return; + } + + $schemaManager->dropTable($this->tableName); + } + + public function isInitialized(Connection $connection): bool + { + return $connection->createSchemaManager()->tableExists($this->tableName); + } + + public function getDefinition(): Definition + { + return new Definition(self::class, [$this->tableName, $this->isUsed, $this->consoleInvocationPrefix, $this->connectionReferenceName]); + } + + private function buildTableSchema(): Table + { + $table = new Table($this->tableName); + $table->addColumn('trace_id', Types::STRING, ['length' => 32]); + $table->addColumn('started_at', Types::BIGINT); + $table->addColumn('completed_at', Types::BIGINT, ['notnull' => false]); + $table->setPrimaryKey(['trace_id']); + $table->addIndex(['started_at']); + + return $table; + } +} diff --git a/packages/Dbal/src/Tracing/DbalTraceStore.php b/packages/Dbal/src/Tracing/DbalTraceStore.php new file mode 100644 index 000000000..41e846310 --- /dev/null +++ b/packages/Dbal/src/Tracing/DbalTraceStore.php @@ -0,0 +1,231 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +final class DbalTraceStore implements TraceStore +{ + public function __construct( + private ConnectionFactory $connectionFactory, + private TracesTableManager $tracesTableManager, + private SpansTableManager $spansTableManager, + private EcotoneClockInterface $clock, + ) { + } + + public function record(array $spans): array + { + $spansByTrace = []; + foreach ($spans as $span) { + $spansByTrace[$span->traceId][] = $span; + } + + $completedNow = []; + $connection = $this->connection(); + foreach ($spansByTrace as $traceId => $traceSpans) { + $this->onTables(function () use ($connection, $traceId, $traceSpans, &$completedNow): void { + $this->claimTrace($connection, $traceId, $traceSpans); + $alreadyCompleted = $this->lockTrace($connection, $traceId); + foreach ($traceSpans as $span) { + $this->insertSpan($connection, $span); + } + if ($alreadyCompleted || ! TraceLedger::progressOf($traceId, $this->spansOf($connection, $traceId))->isCompleted()) { + return; + } + if ($this->markCompleted($connection, $traceId)) { + $completedNow[] = $traceId; + } + }); + } + + return $completedNow; + } + + public function progressOf(string $traceId): TraceProgress + { + return $this->onTables(fn (): TraceProgress => TraceLedger::progressOf($traceId, $this->spansOf($this->connection(), $traceId))); + } + + public function removeTracesStartedBefore(DatePoint $threshold): void + { + $connection = $this->connection(); + $this->onTables(function () use ($connection, $threshold): void { + $traceIds = $connection->createQueryBuilder() + ->select('trace_id') + ->from($this->tracesTableManager->getTableName()) + ->andWhere('started_at < :threshold') + ->setParameter('threshold', $threshold->unixTime()->inMicroseconds(), Types::BIGINT) + ->executeQuery() + ->fetchFirstColumn(); + if ($traceIds === []) { + return; + } + + foreach ([$this->spansTableManager->getTableName(), $this->tracesTableManager->getTableName()] as $tableName) { + $connection->createQueryBuilder() + ->delete($tableName) + ->andWhere('trace_id IN (:traceIds)') + ->setParameter('traceIds', $traceIds, ArrayParameterType::STRING) + ->executeStatement(); + } + }); + } + + /** + * @param Span[] $traceSpans + */ + private function claimTrace(Connection $connection, string $traceId, array $traceSpans): void + { + $startedAt = min(array_map(fn (Span $span): int => $span->startedAt->unixTime()->inMicroseconds(), $traceSpans)); + $tableName = $this->tracesTableManager->getTableName(); + $sql = $connection->getDatabasePlatform() instanceof AbstractMySQLPlatform + ? "INSERT INTO {$tableName} (trace_id, started_at, completed_at) VALUES (:traceId, :startedAt, NULL) ON DUPLICATE KEY UPDATE trace_id = trace_id" + : "INSERT INTO {$tableName} (trace_id, started_at, completed_at) VALUES (:traceId, :startedAt, NULL) ON CONFLICT (trace_id) DO NOTHING"; + + $connection->executeStatement($sql, ['traceId' => $traceId, 'startedAt' => $startedAt], ['startedAt' => Types::BIGINT]); + } + + private function lockTrace(Connection $connection, string $traceId): bool + { + $sql = "SELECT completed_at FROM {$this->tracesTableManager->getTableName()} WHERE trace_id = :traceId"; + if (! $connection->getDatabasePlatform() instanceof SQLitePlatform) { + $sql .= ' FOR UPDATE'; + } + + return $connection->fetchOne($sql, ['traceId' => $traceId]) !== null; + } + + private function markCompleted(Connection $connection, string $traceId): bool + { + return $connection->executeStatement( + "UPDATE {$this->tracesTableManager->getTableName()} SET completed_at = :completedAt WHERE trace_id = :traceId AND completed_at IS NULL", + ['completedAt' => $this->clock->now()->unixTime()->inMicroseconds(), 'traceId' => $traceId], + ['completedAt' => Types::BIGINT], + ) === 1; + } + + private function insertSpan(Connection $connection, Span $span): void + { + $connection->insert($this->spansTableManager->getTableName(), [ + 'span_id' => $span->spanId, + 'trace_id' => $span->traceId, + 'parent_span_id' => $span->parentSpanId, + 'name' => $span->name, + 'kind' => $span->kind->value, + 'status' => $span->status->value, + 'status_description' => $span->statusDescription, + 'attributes' => json_encode($span->attributes, JSON_THROW_ON_ERROR | JSON_FORCE_OBJECT), + 'events' => json_encode(array_map(fn (SpanEvent $event): array => [ + 'name' => $event->name, + 'attributes' => (object) $event->attributes, + 'at' => $event->at->unixTime()->inMicroseconds(), + ], $span->events), JSON_THROW_ON_ERROR), + 'started_at' => $span->startedAt->unixTime()->inMicroseconds(), + 'ended_at' => $span->endedAt->unixTime()->inMicroseconds(), + ], [ + 'started_at' => Types::BIGINT, + 'ended_at' => Types::BIGINT, + ]); + } + + /** + * @return Span[] + */ + private function spansOf(Connection $connection, string $traceId): array + { + $rows = $connection->createQueryBuilder() + ->select('*') + ->from($this->spansTableManager->getTableName()) + ->andWhere('trace_id = :traceId') + ->setParameter('traceId', $traceId) + ->orderBy('ended_at') + ->executeQuery() + ->fetchAllAssociative(); + + return array_map(fn (array $row): Span => new Span( + $row['trace_id'], + $row['span_id'], + $row['parent_span_id'], + $row['name'], + SpanKind::from($row['kind']), + json_decode($row['attributes'], true, flags: JSON_THROW_ON_ERROR), + SpanStatus::from($row['status']), + $row['status_description'], + array_map( + fn (array $event): SpanEvent => new SpanEvent($event['name'], $event['attributes'], self::datePointOf((int) $event['at'])), + json_decode($row['events'], true, flags: JSON_THROW_ON_ERROR), + ), + self::datePointOf((int) $row['started_at']), + self::datePointOf((int) $row['ended_at']), + ), $rows); + } + + private static function datePointOf(int $unixTimeInMicroseconds): DatePoint + { + return (new DatePoint('@0'))->setUnixTime(Duration::microseconds($unixTimeInMicroseconds)); + } + + private function onTables(callable $statement): mixed + { + try { + return $statement(); + } catch (Throwable $failure) { + throw MissingTableFailure::explained(MissingTableFailure::explained($failure, $this->tracesTableManager), $this->spansTableManager); + } + } + + private function connection(): Connection + { + /** @var DbalContext $context */ + $context = CachedConnectionFactory::createFor(new DbalReconnectableConnectionFactory($this->connectionFactory, $this->clock))->createContext(); + + return $context->getDbalConnection(); + } +} diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php new file mode 100644 index 000000000..35b6cda7e --- /dev/null +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -0,0 +1,119 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +final class DbalTracingModule extends NoExternalConfigurationModule implements AnnotationModule +{ + public static function create(AnnotationFinder $annotationRegistrationService, InterfaceToCallRegistry $interfaceToCallRegistry): static + { + return new self(); + } + + public function prepare(Configuration $messagingConfiguration, array $extensionObjects, ModuleReferenceSearchService $moduleReferenceSearchService, InterfaceToCallRegistry $interfaceToCallRegistry): void + { + $tracingConfiguration = ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $extensionObjects, TracingConfiguration::createWithDefaults()); + $connectionReference = self::connectionReferenceOf($extensionObjects); + $consoleInvocationPrefix = ConsoleInvocationResolver::resolveConsolePrefix(ExtensionObjectResolver::resolveUnique(BootstrapConfiguration::class, $extensionObjects, BootstrapConfiguration::createWithDefaults())); + $storesOnDatabase = self::storesOnDatabase($tracingConfiguration); + + $messagingConfiguration->registerServiceDefinition( + TracesTableManager::class, + new Definition(TracesTableManager::class, [TracesTableManager::DEFAULT_TABLE, $storesOnDatabase, $consoleInvocationPrefix, $connectionReference]) + ); + $messagingConfiguration->registerServiceDefinition( + SpansTableManager::class, + new Definition(SpansTableManager::class, [SpansTableManager::DEFAULT_TABLE, $storesOnDatabase, $consoleInvocationPrefix, $connectionReference]) + ); + if (! $storesOnDatabase) { + return; + } + + $messagingConfiguration->registerServiceDefinition( + DbalTraceStore::class, + new Definition(DbalTraceStore::class, [ + new Reference($connectionReference), + new Reference(TracesTableManager::class), + new Reference(SpansTableManager::class), + new Reference(EcotoneClockInterface::class), + ]) + ); + } + + public function getModuleExtensions(ServiceConfiguration $serviceConfiguration, array $serviceExtensions): array + { + $connectionReference = self::connectionReferenceOf($serviceExtensions); + $extensions = [ + new DbalTableManagerReference(TracesTableManager::class, $connectionReference), + new DbalTableManagerReference(SpansTableManager::class, $connectionReference), + ]; + if (self::storesOnDatabase(ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $serviceExtensions, TracingConfiguration::createWithDefaults()))) { + $extensions[] = new TraceStoreReference(DbalTraceStore::class); + } + + return $extensions; + } + + public function getModulePackageName(): string + { + return ModulePackageList::DBAL_PACKAGE; + } + + private static function storesOnDatabase(TracingConfiguration $tracingConfiguration): bool + { + return $tracingConfiguration->storedTraces && ! $tracingConfiguration->storedTracesInMemory; + } + + /** + * @param object[] $extensionObjects + */ + private static function connectionReferenceOf(array $extensionObjects): string + { + $dbalConfiguration = ExtensionObjectResolver::resolveUnique(DbalConfiguration::class, $extensionObjects, DbalConfiguration::createWithDefaults()); + + return $dbalConfiguration->defaultConnectionReferenceNames[0] ?? DbalConnectionReference::DEFAULT; + } +} diff --git a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php new file mode 100644 index 000000000..da76710c0 --- /dev/null +++ b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php @@ -0,0 +1,217 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + * + * @internal + */ +final class TraceStoreConformanceTest extends DbalMessagingTestCase +{ + public static function implementations(): iterable + { + yield 'in-memory' => ['in-memory']; + yield 'dbal' => ['dbal']; + } + + #[DataProvider('implementations')] + public function test_a_synchronous_flow_is_completed_when_the_bus_returns(string $implementation): void + { + $ecotone = $this->bootstrap($implementation, [$this->users()]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.registration')); + } + + #[DataProvider('implementations')] + public function test_an_asynchronous_handler_keeps_the_trace_pending_until_its_consumer_ran(string $implementation): void + { + $ecotone = $this->bootstrap($implementation, [$this->users(), $this->asynchronousNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); + self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + #[DataProvider('implementations')] + public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pending(string $implementation): void + { + $ecotone = $this->bootstrap($implementation, [$this->users(), $this->asynchronousNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->expectException(TraceCompletionTimedOut::class); + $this->expectExceptionMessage('Still pending: user.notifier.'); + + $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); + } + + #[DataProvider('implementations')] + public function test_a_trace_completed_subscriber_is_called_once_after_the_last_handler_committed(string $implementation): void + { + $subscriber = new class () { + /** @var string[] */ + public array $completedTraces = []; + + #[OnTraceCompleted] + public function whenCompleted(TraceCompleted $event): void + { + $this->completedTraces[] = $event->traceId; + } + }; + $ecotone = $this->bootstrap($implementation, [$this->users(), $this->asynchronousNotifier(), $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + self::assertSame([], $subscriber->completedTraces); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertSame([$trace->traceId], $subscriber->completedTraces); + } + + #[DataProvider('implementations')] + public function test_a_failing_asynchronous_handler_leaves_the_trace_pending(string $implementation): void + { + $notifier = new class () { + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId): void + { + throw new RuntimeException('Notification failed'); + } + }; + $ecotone = $this->bootstrap($implementation, [$this->users(), $notifier], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + try { + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + } catch (RuntimeException) { + } + + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + #[DataProvider('implementations')] + public function test_traces_older_than_the_retention_are_removed_by_the_console_command(string $implementation): void + { + $ecotone = $this->bootstrap($implementation, [$this->users()], [], Duration::minutes(10)); + $expired = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $expired->toTraceparent()]); + + $ecotone->advanceTimeBy(TimeSpan::withMinutes(11)); + $recent = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '2', metadata: [MessageHeaders::TRACEPARENT => $recent->toTraceparent()]); + + $ecotone->runConsoleCommand('ecotone:tracing:remove-expired-traces'); + + self::assertFalse($this->completion($ecotone)->isCompleted($expired->traceId)); + self::assertTrue($this->completion($ecotone)->isCompleted($recent->traceId)); + } + + public function test_completion_recorded_by_one_instance_is_visible_to_another_sharing_the_database(): void + { + $producer = $this->bootstrap('dbal', [$this->users()]); + $observer = $this->bootstrap('dbal', [$this->users()]); + $trace = SpanContext::newTrace(); + + $producer->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + self::assertTrue($this->completion($observer)->isCompleted($trace->traceId)); + } + + /** + * @param object[] $services + * @param object[] $extensionObjects + */ + private function bootstrap(string $implementation, array $services, array $extensionObjects = [], ?Duration $retention = null): FlowTestSupport + { + return $this->bootstrapFlowTesting( + classesToResolve: array_map(fn (object $service): string => $service::class, $services), + containerOrAvailableServices: [...$services, DbalConnectionFactory::class => $this->getConnectionFactory()], + extensionObjects: [TracingConfiguration::createWithDefaults()->withStoredTraces(inMemory: $implementation === 'in-memory', retention: $retention), ...$extensionObjects], + enabledModules: [ModulePackageList::DBAL_PACKAGE], + licenceKey: LicenceTesting::VALID_LICENCE, + )->setupDatabaseTables(); + } + + private function completion(FlowTestSupport $ecotone): TraceCompletion + { + return $ecotone->getGateway(TraceCompletion::class); + } + + private function users(): object + { + return new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId, EventBus $eventBus): void + { + $eventBus->publishWithRouting('user.registered', $userId); + } + }; + } + + private function asynchronousNotifier(): object + { + return new class () { + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId): void + { + } + }; + } +} diff --git a/packages/Dbal/tests/DbalMessagingTestCase.php b/packages/Dbal/tests/DbalMessagingTestCase.php index 09f9535ac..3ba52857e 100644 --- a/packages/Dbal/tests/DbalMessagingTestCase.php +++ b/packages/Dbal/tests/DbalMessagingTestCase.php @@ -79,6 +79,8 @@ public static function cleanUpDbalTables(Connection $connection): void self::deleteTable(DbalDeadLetterStore::DEFAULT_DEAD_LETTER_TABLE, $connection); self::deleteTable(DbalDocumentStore::ECOTONE_DOCUMENT_STORE, $connection); self::deleteTable(DbalHandledMessages::DEFAULT_DEDUPLICATION_TABLE, $connection); + self::deleteTable(\Ecotone\Dbal\Database\SpansTableManager::DEFAULT_TABLE, $connection); + self::deleteTable(\Ecotone\Dbal\Database\TracesTableManager::DEFAULT_TABLE, $connection); self::deleteTable('persons', $connection); self::deleteTable('activities', $connection); } diff --git a/packages/Ecotone/src/Messaging/Config/ModuleClassList.php b/packages/Ecotone/src/Messaging/Config/ModuleClassList.php index a58db0032..b2aecec9c 100644 --- a/packages/Ecotone/src/Messaging/Config/ModuleClassList.php +++ b/packages/Ecotone/src/Messaging/Config/ModuleClassList.php @@ -24,6 +24,7 @@ use Ecotone\Dbal\MultiTenant\Module\MultiTenantConnectionFactoryModule; use Ecotone\Dbal\ObjectManager\ObjectManagerModule; use Ecotone\Dbal\Recoverability\DbalDeadLetterModule; +use Ecotone\Dbal\Tracing\DbalTracingModule; use Ecotone\EventSourcing\Config\EventSourcingModule; use Ecotone\EventSourcing\Config\EventSourcingProjectingModule; use Ecotone\EventSourcing\Tagging\Config\EventTaggingModule; @@ -172,6 +173,7 @@ class ModuleClassList DbalBusinessMethodModule::class, MultiTenantConnectionFactoryModule::class, DatabaseSetupModule::class, + DbalTracingModule::class, ]; public const REDIS_MODULES = [ From 2900ffa1226e531ca87335c1d6d0f77850dd1aef Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 16 Oct 2026 08:00:00 +0200 Subject: [PATCH 07/50] feat(tracing): a route carries its destination in tracestate, so a handler without an explicit endpointId still completes its route The route span and the handler span used to read the endpoint id from two attribute instances, which generate different ids when none is declared, so such a trace never completed. The route now writes the destination into the W3C tracestate header, which travels with the message through synchronous and asynchronous channels, and the handler span names that destination. The upgrade guide gains the stored traces entry and the planned depth completion and opt-in recording. --- .../Config/TraceEventRoutesCompilerPass.php | 2 + packages/Ecotone/src/Tracing/TraceState.php | 42 +++++++++++++++++++ .../src/Tracing/TracedRouteProcessor.php | 15 +++++++ .../src/Tracing/TracingInterceptor.php | 8 ++-- .../src/Tracing/TracingRouteResolver.php | 4 +- .../tests/Tracing/TraceCompletionTest.php | 16 +++++++ upgrade-2.0.md | 38 +++++++++++++++++ 7 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 packages/Ecotone/src/Tracing/TraceState.php diff --git a/packages/Ecotone/src/Tracing/Config/TraceEventRoutesCompilerPass.php b/packages/Ecotone/src/Tracing/Config/TraceEventRoutesCompilerPass.php index faa23b451..bce7dd9b2 100644 --- a/packages/Ecotone/src/Tracing/Config/TraceEventRoutesCompilerPass.php +++ b/packages/Ecotone/src/Tracing/Config/TraceEventRoutesCompilerPass.php @@ -8,6 +8,7 @@ use Ecotone\Messaging\Config\Container\ContainerBuilder; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\Reference; +use Ecotone\Messaging\Support\MessageBuilder; use Ecotone\Modelling\Config\MessageHandlerRoutingModule; use Ecotone\Tracing\Tracer; use Ecotone\Tracing\TracingRouteResolver; @@ -26,6 +27,7 @@ public function process(ContainerBuilder $builder): void $builder->replace(MessageHandlerRoutingModule::EVENT_BUS_ROUTE_RESOLVER, new Definition(TracingRouteResolver::class, [ $builder->getDefinition(MessageHandlerRoutingModule::EVENT_BUS_ROUTE_RESOLVER), new Reference(Tracer::class), + new Reference(MessageBuilder::class), ])); } } diff --git a/packages/Ecotone/src/Tracing/TraceState.php b/packages/Ecotone/src/Tracing/TraceState.php new file mode 100644 index 000000000..f0944795d --- /dev/null +++ b/packages/Ecotone/src/Tracing/TraceState.php @@ -0,0 +1,42 @@ +getHeaders(); + if (! $headers->containsKey(MessageHeaders::TRACESTATE)) { + return null; + } + foreach (explode(',', (string) $headers->get(MessageHeaders::TRACESTATE)) as $entry) { + $entry = trim($entry); + if (str_starts_with($entry, self::DESTINATION_ENTRY_PREFIX)) { + return rawurldecode(substr($entry, strlen(self::DESTINATION_ENTRY_PREFIX))); + } + } + + return null; + } +} diff --git a/packages/Ecotone/src/Tracing/TracedRouteProcessor.php b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php index 1b847a129..96ee63f65 100644 --- a/packages/Ecotone/src/Tracing/TracedRouteProcessor.php +++ b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php @@ -5,9 +5,11 @@ namespace Ecotone\Tracing; use Ecotone\Api\Messaging\Message; +use Ecotone\Api\Messaging\MessageHeaders; use Ecotone\Api\Tracing\SpanAttribute; use Ecotone\Api\Tracing\SpanKind; use Ecotone\Messaging\Handler\MessageProcessor; +use Ecotone\Messaging\Support\MessageBuilder; use Ecotone\Modelling\Config\Routing\BusRoutingMapBuilder; use Ecotone\Projecting\Config\ProjectingModule; use Throwable; @@ -21,6 +23,7 @@ public function __construct( private string $routeName, private MessageProcessor $routeProcessor, private Tracer $tracer, + private MessageBuilder $messageBuilder, ) { } @@ -33,6 +36,13 @@ public function process(Message $message): ?Message ...$this->destinationAttributes(), ]); + $destination = $this->destination(); + if ($destination !== null) { + $message = $this->messageBuilder->fromMessage($message) + ->setHeader(MessageHeaders::TRACESTATE, TraceState::withDestination($destination)) + ->build(); + } + try { $result = $this->routeProcessor->process($message); } catch (Throwable $failure) { @@ -56,6 +66,11 @@ private function spanName(): string return 'Routing to: ' . (BusRoutingMapBuilder::endpointIdOfDestinationChannel($this->routeName) ?? $this->routeName); } + private function destination(): ?string + { + return ProjectingModule::projectionNameOfTriggerChannel($this->routeName) ?? BusRoutingMapBuilder::endpointIdOfDestinationChannel($this->routeName); + } + /** * @return array */ diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 6aa6b2465..256cfecfe 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -46,22 +46,22 @@ public function traceQueryBus(MethodInvocation $methodInvocation, Message $messa public function traceCommandHandler(MethodInvocation $methodInvocation, Message $message, CommandHandler $endpoint): mixed { - return $this->trace('Command Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes($endpoint->getEndpointId())); + return $this->trace('Command Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes(TraceState::destinationOf($message) ?? $endpoint->getEndpointId())); } public function traceQueryHandler(MethodInvocation $methodInvocation, Message $message, QueryHandler $endpoint): mixed { - return $this->trace('Query Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes($endpoint->getEndpointId())); + return $this->trace('Query Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes(TraceState::destinationOf($message) ?? $endpoint->getEndpointId())); } public function traceEventHandler(MethodInvocation $methodInvocation, Message $message, EventHandler $endpoint): mixed { - return $this->trace('Event Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes($endpoint->getEndpointId())); + return $this->trace('Event Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes(TraceState::destinationOf($message) ?? $endpoint->getEndpointId())); } public function traceMessageHandler(MethodInvocation $methodInvocation, Message $message, InternalHandler $endpoint): mixed { - return $this->trace('Message Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes($endpoint->getEndpointId())); + return $this->trace('Message Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes(TraceState::destinationOf($message) ?? $endpoint->getEndpointId())); } public function traceAsynchronousEndpoint(MethodInvocation $methodInvocation, Message $message): mixed diff --git a/packages/Ecotone/src/Tracing/TracingRouteResolver.php b/packages/Ecotone/src/Tracing/TracingRouteResolver.php index 0771ca894..72217310a 100644 --- a/packages/Ecotone/src/Tracing/TracingRouteResolver.php +++ b/packages/Ecotone/src/Tracing/TracingRouteResolver.php @@ -6,6 +6,7 @@ use Ecotone\Messaging\Handler\MessageProcessor; use Ecotone\Messaging\Handler\Router\RouteResolver; +use Ecotone\Messaging\Support\MessageBuilder; /** * licence Apache-2.0 @@ -15,11 +16,12 @@ final class TracingRouteResolver implements RouteResolver public function __construct( private RouteResolver $routeResolver, private Tracer $tracer, + private MessageBuilder $messageBuilder, ) { } public function resolve(string $routeName): MessageProcessor { - return new TracedRouteProcessor($routeName, $this->routeResolver->resolve($routeName), $this->tracer); + return new TracedRouteProcessor($routeName, $this->routeResolver->resolve($routeName), $this->tracer, $this->messageBuilder); } } diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index 51b504b12..06bb7669f 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -127,6 +127,22 @@ public function test_awaiting_a_completed_trace_returns_at_once(): void $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } + public function test_a_synchronous_handler_without_an_explicit_endpoint_id_still_completes_its_route(): void + { + $notifier = new class () { + #[EventHandler('user.registered')] + public function notify(string $userId): void + { + } + }; + $ecotone = $this->bootstrap([$this->users(), $notifier]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + public function test_a_trace_nobody_recorded_is_not_completed(): void { $ecotone = $this->bootstrap([$this->users()]); diff --git a/upgrade-2.0.md b/upgrade-2.0.md index e8a010d5d..0d64ea92e 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3000,6 +3000,43 @@ $ecotone->sendCommand(new PlaceOrder('1')); $recorder->recordedSpans(); // Span[] with traceId, spanId, parentSpanId, name, kind, status, attributes, events ``` +### Stored traces answer "has it completed yet?" (Enterprise) + +**Before:** an application that wrote through a command and then read through an asynchronous projection had no way +to know when the projection had caught up: it polled its own read model, or guessed with a delay. Spans existed only +in an external collector, after the fact. + +**Now:** `TracingConfiguration::createWithDefaults()->withStoredTraces()` keeps every span in the database as well, +in `ecotone_traces` and `ecotone_spans`, created by `ecotone:migration:database:setup`. A unit of work, a bus call +or an asynchronous consumption, buffers its spans and writes them once, inside its transaction before the commit; +the root span is written when it ends, after the commit. From those spans the `TraceCompletion` gateway answers +`isCompleted($traceId)`, true once the root ended and every route the trace sent has an ended handler span, +`isCompletedFor($traceId, $destination)` for one `endpointId` or projection name, and `await()` / `awaitFor()`, +which poll until then or throw `TraceCompletionTimedOut` naming what is still pending. A method with +`#[OnTraceCompleted]` receives a `TraceCompleted` once per trace, and one with `#[OnSpanEnded]` receives every +stored span, both after the commit that produced them, so a websocket push or a cache refresh never runs ahead of +the data. The trace id is the one in the `traceparent` header: mint it with `SpanContext::newTrace()` and send it +with the command, or take it from the HTTP request the command came from. A flow test given no connection keeps the +store in memory. `ecotone:tracing:remove-expired-traces` deletes traces older than the retention passed to +`withStoredTraces()`, seven days by default. + +```php +$trace = SpanContext::newTrace(); +$commandBus->sendWithRouting('order.place', $order, metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); +$traceCompletion->awaitFor($trace->traceId, 'orders_list', Duration::seconds(5)); + +final class OrdersWebsocket +{ + #[OnTraceCompleted] + public function push(TraceCompleted $event): void + { + } +} +``` + +**How to adapt:** nothing is required. To use it, register `withStoredTraces()`, run the setup command for the two +tables, and give each handler you want to ask about an explicit `endpointId`. + ## 15. Testing and developer-experience changes These changes make tests and error messages say what happens, so that failures point at the real cause. Most of them @@ -3323,6 +3360,7 @@ normal section with "How to adapt" steps when it ships. | Simpler EcotoneLite testing | Flow tests load every installed package with in-memory test profiles, instead of Core only. (The other half — an in-memory queue channel for every unconfigured `#[Asynchronous]` channel — has shipped; see §1) | `docs/superpowers/research/ecotone-lite-testing-simplification/report.md` (section "Implementation sketch"; no final design yet) | | Service cache directory | Replace the cache-directory setting on `ServiceConfiguration` with an explicit bootstrap parameter; shared cache-clear command | `docs/superpowers/research/service-cache-directory/report.md` (section "Implementation plan"; overlaps with §12) | | `ecotone:describe` introspection | A read-only API and console command that prints how messaging is configured: channels (type, delayable, consumer command), asynchronous endpoints (channel, delay, retry and dead letter), the error channel policy, converters, projections with their commands, and how each handler resolves its aggregate or saga identifier. Answers the question coding agents ask in almost every session without reading configuration files | Not designed yet — to be discussed. Input: agent benchmark findings (catalogue item A6, "How is messaging configured here?") | +| Trace completion by depth, and recording only requested traces | Stored traces (§14) answer whether a whole trace or one destination completed. Two parts of the agreed design are not implemented: `#[OnTraceDepthCompleted]`, notified when every handler at one depth of the trace tree ended (the command handler is depth 1, the event handlers it triggered depth 2, and so on), which needs the depth carried in `tracestate` across the asynchronous boundary; and recording only traces that ask for it through a `tracestate` entry, so that an application can keep storing off for ordinary traffic and on for the requests that await completion. Today `withStoredTraces()` stores every trace and relies on the retention command | Design recorded in the conversation that produced §14's tracing sections (the ledger, the unit-of-work flush before commit, publication after commit); no spec file yet | | Stateless `#[Apply]` | A second, opt-in fold style: the handler **returns** the changed state instead of mutating `$this`, for both event-sourced aggregates and DCB decision models. A command handler then runs against the last state the fold returned. The existing mutating style stays and needs no change. Detected from the declared return type at configuration time — `void` is stateful, the model's own type is stateless — which is unambiguous because a non-void `#[Apply]` is already rejected at bootstrap today. A stateless state object may carry its own `#[Version]`; **userland increments it in each apply** and Ecotone verifies it increased on every applied event, which also makes a fully `readonly` state class possible on PHP 8.2. Returning the wrong type is rejected rather than ignored | `docs/superpowers/specs/2026-09-30-stateless-event-sourcing-handler-design.md` — researched against Axon 5, Marten, Akka/Pekko, Emmett/Decider and Eventuous, with the fold cost measured. Maintainer decisions on all four open questions are recorded in it. Note the measured hazard: a state object that copies a growing collection makes the fold O(n²) (19.9 µs/event by 10k events against 6.3–6.8 µs today), so the documentation must lead with the shape that avoids it | --- From 94ea063d04441a20b2e115d8f40517b5bf13a89e Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 08/50] fix(tracing): a trace stays pending until every channel send was consumed, its root is the span that started it, a failed unit of work discards its spans, and a subscriber's bus calls do not notify it again An asynchronous command handler, or any message sent to a channel outside an event route, completed its trace the moment the bus returned: only event routes recorded a destination. The ledger now also pairs every successful channel send with the consumer span that processed the same message id, and names the channel when it is pending. Any span whose parent was not stored counted as the ended root, so a consumer that stored its spans before, or instead of, its producer completed the trace on its own. The span started outside any other span, unless it is a consumer, now carries ecotone.trace.root and only that ends the root. A failed unit of work kept its buffered spans and wrote them with the failed outer span, so a handler that succeeded before the rollback counted as completed and a send the rollback removed counted as pending forever. The buffer is discarded on failure. Publication ran inside the span end and traced the subscriber's own bus calls, so a subscriber that sent a command completed a new trace and was notified again without end. Notifications gathered while publishing are dropped. --- .../Conformance/TraceStoreConformanceTest.php | 81 ++++++++++- .../Ecotone/Api/Tracing/SpanAttribute.php | 1 + .../src/Tracing/StoringSpanRecorder.php | 27 +++- packages/Ecotone/src/Tracing/TraceLedger.php | 35 ++++- .../Ecotone/src/Tracing/TraceProgress.php | 16 ++- .../src/Tracing/TraceRecordingInterceptor.php | 5 + .../src/Tracing/TracingInterceptor.php | 6 +- .../tests/Tracing/TraceCompletionTest.php | 128 +++++++++++++++++- upgrade-2.0.md | 19 ++- 9 files changed, 295 insertions(+), 23 deletions(-) diff --git a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php index da76710c0..48fc94ffe 100644 --- a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php +++ b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php @@ -7,6 +7,7 @@ use Ecotone\Api\Attribute\Asynchronous; use Ecotone\Api\Attribute\CommandHandler; use Ecotone\Api\Attribute\EventHandler; +use Ecotone\Api\Dbal\ExtensionObject\DbalBackedMessageChannelBuilder; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\ModulePackageList; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; @@ -92,7 +93,7 @@ public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pen $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $this->expectException(TraceCompletionTimedOut::class); - $this->expectExceptionMessage('Still pending: user.notifier.'); + $this->expectExceptionMessage('Still pending: user.notifier, async_channel.'); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); } @@ -162,6 +163,84 @@ public function test_traces_older_than_the_retention_are_removed_by_the_console_ self::assertTrue($this->completion($ecotone)->isCompleted($recent->traceId)); } + #[DataProvider('implementations')] + public function test_an_asynchronous_command_keeps_the_trace_pending_until_its_consumer_ran(string $implementation): void + { + $orders = new class () { + #[Asynchronous('async_channel')] + #[CommandHandler('order.place', endpointId: 'order.placing')] + public function place(string $orderId): void + { + } + }; + $ecotone = $this->bootstrap($implementation, [$orders], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); + self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + public function test_a_consumer_alone_does_not_complete_a_trace_whose_producer_stored_nothing(): void + { + $producer = $this->bootstrapFlowTesting( + classesToResolve: [$this->asynchronousNotifier()::class], + containerOrAvailableServices: [$this->asynchronousNotifier(), DbalConnectionFactory::class => $this->getConnectionFactory()], + extensionObjects: [DbalBackedMessageChannelBuilder::create('async_channel')], + enabledModules: [ModulePackageList::DBAL_PACKAGE], + )->setupDatabaseTables(); + $consumer = $this->bootstrap('dbal', [$this->asynchronousNotifier()], [DbalBackedMessageChannelBuilder::create('async_channel')]); + $trace = SpanContext::newTrace(); + + $producer->publishEventWithRouting('user.registered', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + $consumer->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertTrue($this->completion($consumer)->isCompletedFor($trace->traceId, 'user.notifier')); + self::assertFalse($this->completion($consumer)->isCompleted($trace->traceId)); + } + + public function test_a_send_rolled_back_with_its_failed_handler_is_not_awaited_after_the_retry_succeeded(): void + { + $notifier = new class () { + public int $attempts = 0; + + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId, EventBus $eventBus): void + { + $this->attempts++; + $eventBus->publishWithRouting('user.notified', $userId); + if ($this->attempts === 1) { + throw new RuntimeException('Notification failed once'); + } + } + + #[Asynchronous('audit_channel')] + #[EventHandler('user.notified', endpointId: 'user.auditor')] + public function audit(string $userId): void + { + } + }; + $ecotone = $this->bootstrap('dbal', [$this->users(), $notifier], [DbalBackedMessageChannelBuilder::create('async_channel'), DbalBackedMessageChannelBuilder::create('audit_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + try { + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + } catch (RuntimeException) { + } + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $ecotone->run('audit_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertSame(2, $notifier->attempts); + self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + public function test_completion_recorded_by_one_instance_is_visible_to_another_sharing_the_database(): void { $producer = $this->bootstrap('dbal', [$this->users()]); diff --git a/packages/Ecotone/Api/Tracing/SpanAttribute.php b/packages/Ecotone/Api/Tracing/SpanAttribute.php index 76a90526d..60452ef07 100644 --- a/packages/Ecotone/Api/Tracing/SpanAttribute.php +++ b/packages/Ecotone/Api/Tracing/SpanAttribute.php @@ -14,6 +14,7 @@ final class SpanAttribute public const MESSAGING_DESTINATION_NAME = 'messaging.destination.name'; public const ENDPOINT_ID = 'ecotone.endpoint.id'; public const PROJECTION_NAME = 'ecotone.projection.name'; + public const TRACE_ROOT = 'ecotone.trace.root'; public const ECOTONE = 'ecotone'; public const OPERATION_SEND = 'send'; diff --git a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php index 791bed09f..30b002bbe 100644 --- a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php +++ b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php @@ -39,6 +39,7 @@ final class StoringSpanRecorder implements SpanRecorder private array $endedSpansToPublish = []; /** @var string[] */ private array $completedTracesToPublish = []; + private bool $publishing = false; public function __construct( private TraceStore $traceStore, @@ -82,6 +83,11 @@ public function closeScope(): void $this->scopeActive = false; } + public function discard(): void + { + $this->buffer = []; + } + /** * @param Span[] $spans */ @@ -92,6 +98,9 @@ private function write(array $spans): void } $completedTraces = $this->traceStore->record($spans); + if ($this->publishing) { + return; + } if ($this->publishesEndedSpans) { $this->endedSpansToPublish = [...$this->endedSpansToPublish, ...$spans]; } @@ -102,16 +111,24 @@ private function write(array $spans): void private function publish(): void { + if ($this->publishing) { + return; + } $endedSpans = $this->endedSpansToPublish; $completedTraces = $this->completedTracesToPublish; $this->endedSpansToPublish = []; $this->completedTracesToPublish = []; - foreach ($endedSpans as $span) { - $this->messagingEntrypoint->send(new SpanEnded($span), OnSpanEnded::CHANNEL); - } - foreach ($completedTraces as $traceId) { - $this->messagingEntrypoint->send(new TraceCompleted($traceId), OnTraceCompleted::CHANNEL); + $this->publishing = true; + try { + foreach ($endedSpans as $span) { + $this->messagingEntrypoint->send(new SpanEnded($span), OnSpanEnded::CHANNEL); + } + foreach ($completedTraces as $traceId) { + $this->messagingEntrypoint->send(new TraceCompleted($traceId), OnTraceCompleted::CHANNEL); + } + } finally { + $this->publishing = false; } } } diff --git a/packages/Ecotone/src/Tracing/TraceLedger.php b/packages/Ecotone/src/Tracing/TraceLedger.php index fb7a0c826..3e83e4e18 100644 --- a/packages/Ecotone/src/Tracing/TraceLedger.php +++ b/packages/Ecotone/src/Tracing/TraceLedger.php @@ -4,8 +4,10 @@ namespace Ecotone\Tracing; +use Ecotone\Api\Messaging\MessageHeaders; use Ecotone\Api\Tracing\Span; use Ecotone\Api\Tracing\SpanAttribute; +use Ecotone\Api\Tracing\SpanKind; use Ecotone\Api\Tracing\SpanStatus; /** @@ -37,21 +39,29 @@ public static function progressOf(string $traceId, array $spans): TraceProgress return TraceProgress::unknown($traceId); } - $recordedSpanIds = []; - foreach ($spans as $span) { - $recordedSpanIds[$span->spanId] = true; - } - $rootEnded = false; $sent = []; $processed = []; $failed = []; + $sentDeliveries = []; + $processedDeliveries = []; foreach ($spans as $span) { - if ($span->parentSpanId === null || ! isset($recordedSpanIds[$span->parentSpanId])) { + if ($span->attributes[SpanAttribute::TRACE_ROOT] ?? false) { $rootEnded = true; } $destination = self::destinationOf($span); if ($destination === null) { + $delivery = self::deliveryOf($span); + if ($delivery === null || $span->status === SpanStatus::Error) { + continue; + } + [$channel, $messageId] = $delivery; + if ($span->kind === SpanKind::Producer) { + $sentDeliveries[$channel][$messageId] = ($sentDeliveries[$channel][$messageId] ?? 0) + 1; + } elseif ($span->kind === SpanKind::Consumer) { + $processedDeliveries[$channel][$messageId] = ($processedDeliveries[$channel][$messageId] ?? 0) + 1; + } + continue; } $operation = $span->attributes[SpanAttribute::MESSAGING_OPERATION_TYPE] ?? null; @@ -64,7 +74,7 @@ public static function progressOf(string $traceId, array $spans): TraceProgress } } - return new TraceProgress($traceId, true, $rootEnded, $sent, $processed, $failed); + return new TraceProgress($traceId, true, $rootEnded, $sent, $processed, $failed, $sentDeliveries, $processedDeliveries); } public static function destinationOf(Span $span): ?string @@ -73,4 +83,15 @@ public static function destinationOf(Span $span): ?string return $destination === null ? null : (string) $destination; } + + /** + * @return array{string, string}|null + */ + private static function deliveryOf(Span $span): ?array + { + $channel = $span->attributes[SpanAttribute::MESSAGING_DESTINATION_NAME] ?? null; + $messageId = $span->attributes[MessageHeaders::MESSAGE_ID] ?? null; + + return $channel === null || $messageId === null ? null : [(string) $channel, (string) $messageId]; + } } diff --git a/packages/Ecotone/src/Tracing/TraceProgress.php b/packages/Ecotone/src/Tracing/TraceProgress.php index dc26e59fa..e926d6b64 100644 --- a/packages/Ecotone/src/Tracing/TraceProgress.php +++ b/packages/Ecotone/src/Tracing/TraceProgress.php @@ -30,6 +30,8 @@ final class TraceProgress * @param array $sent * @param array $processed * @param array $failed + * @param array> $sentDeliveries + * @param array> $processedDeliveries */ public function __construct( public readonly string $traceId, @@ -38,12 +40,14 @@ public function __construct( public readonly array $sent, public readonly array $processed, public readonly array $failed, + public readonly array $sentDeliveries, + public readonly array $processedDeliveries, ) { } public static function unknown(string $traceId): self { - return new self($traceId, false, false, [], [], []); + return new self($traceId, false, false, [], [], [], [], []); } public function isCompleted(): bool @@ -72,8 +76,16 @@ public function pendingDestinations(): array $pending[] = $destination; } } + foreach ($this->sentDeliveries as $channel => $messages) { + foreach ($messages as $messageId => $sent) { + if (($this->processedDeliveries[$channel][$messageId] ?? 0) < $sent) { + $pending[] = $channel; + break; + } + } + } - return $pending; + return array_values(array_unique($pending)); } /** diff --git a/packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php b/packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php index 12ea406c8..6aece78db 100644 --- a/packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php @@ -5,6 +5,7 @@ namespace Ecotone\Tracing; use Ecotone\Api\Interceptor\MethodInvocation; +use Throwable; /** * licence Enterprise @@ -42,6 +43,10 @@ public function recordUnitOfWork(MethodInvocation $methodInvocation): mixed $this->recorder->flush(); return $result; + } catch (Throwable $failure) { + $this->recorder->discard(); + + throw $failure; } finally { $this->recorder->closeScope(); } diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 256cfecfe..77d75f8ec 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -131,7 +131,11 @@ private static function processAttributes(string $endpointId): array */ private function trace(string $name, MethodInvocation $methodInvocation, Message $message, SpanKind $kind, array $attributes): mixed { - $span = $this->tracer->startSpan($name, $kind, [...MessageSpanAttributes::from($message), ...$attributes], $this->tracer->currentSpan() === null ? self::remoteParentOf($message) : null); + $isLocalRoot = $this->tracer->currentSpan() === null; + if ($isLocalRoot && $kind !== SpanKind::Consumer) { + $attributes[SpanAttribute::TRACE_ROOT] = true; + } + $span = $this->tracer->startSpan($name, $kind, [...MessageSpanAttributes::from($message), ...$attributes], $isLocalRoot ? self::remoteParentOf($message) : null); try { $result = $methodInvocation->proceed(); diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index 06bb7669f..d012af96b 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -10,6 +10,7 @@ use Ecotone\Api\EventSourcing\Event; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; +use Ecotone\Api\Gateway\CommandBus; use Ecotone\Api\Gateway\EventBus; use Ecotone\Api\Lite\EcotoneLite; use Ecotone\Api\Lite\Test\FlowTestSupport; @@ -110,7 +111,7 @@ public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pen $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $this->expectException(TraceCompletionTimedOut::class); - $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier. Run the consumer of the channel those handlers are attached to, or await with a longer timeout."); + $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the channel those handlers are attached to, or await with a longer timeout."); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); } @@ -239,6 +240,131 @@ public function whenEnded(SpanEnded $event): void $this->assertContains('Command Handler: ' . $users::class . '::register', $subscriber->spanNames); } + public function test_an_asynchronous_command_keeps_the_trace_pending_until_its_consumer_ran(): void + { + $orders = new class () { + #[Asynchronous('async_channel')] + #[CommandHandler('order.place', endpointId: 'order.placing')] + public function place(string $orderId): void + { + } + }; + $ecotone = $this->bootstrap([$orders], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + public function test_awaiting_an_asynchronous_command_names_its_channel_as_pending(): void + { + $orders = new class () { + #[Asynchronous('async_channel')] + #[CommandHandler('order.place', endpointId: 'order.placing')] + public function place(string $orderId): void + { + } + }; + $ecotone = $this->bootstrap([$orders], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->expectException(TraceCompletionTimedOut::class); + $this->expectExceptionMessage('Still pending: async_channel.'); + + $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); + } + + public function test_a_handler_that_succeeded_inside_a_failed_unit_of_work_is_not_completed(): void + { + $users = new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId, CommandBus $commandBus): void + { + $commandBus->sendWithRouting('audit.record', $userId); + + throw new RuntimeException('Registration failed after the audit was recorded'); + } + + #[CommandHandler('audit.record', endpointId: 'audit.recording')] + public function record(string $userId): void + { + } + }; + $ecotone = $this->bootstrap([$users]); + $trace = SpanContext::newTrace(); + + try { + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + $this->fail('Expected the registration failure to propagate'); + } catch (RuntimeException) { + } + + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'audit.recording')); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.registration')); + } + + public function test_a_trace_completed_subscriber_sending_a_command_does_not_notify_itself_again(): void + { + $subscriber = new class () { + public int $notifications = 0; + + #[OnTraceCompleted] + public function whenCompleted(TraceCompleted $event, CommandBus $commandBus): void + { + $this->notifications++; + if ($this->notifications < 5) { + $commandBus->sendWithRouting('cache.refresh', $event->traceId); + } + } + + #[CommandHandler('cache.refresh', endpointId: 'cache.refreshing')] + public function refresh(string $traceId): void + { + } + }; + $ecotone = $this->bootstrap([$this->users(), $subscriber]); + + $ecotone->sendCommandWithRouting('user.register', '1'); + + $this->assertSame(1, $subscriber->notifications); + } + + public function test_a_span_ended_subscriber_sending_a_command_does_not_notify_itself_again(): void + { + $subscriber = new class () { + /** @var string[] */ + public array $notifiedTraces = []; + + #[OnSpanEnded] + public function whenEnded(SpanEnded $event, CommandBus $commandBus): void + { + $this->notifiedTraces[] = $event->span->traceId; + if (count($this->notifiedTraces) < 20) { + $commandBus->sendWithRouting('cache.refresh', $event->span->traceId); + } + } + + #[CommandHandler('cache.refresh', endpointId: 'cache.refreshing')] + public function refresh(string $traceId): void + { + } + }; + $ecotone = $this->bootstrap([$this->users(), $subscriber]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->assertSame([$trace->traceId], array_values(array_unique($subscriber->notifiedTraces))); + } + public function test_stored_traces_require_an_enterprise_licence(): void { $this->expectException(LicensingException::class); diff --git a/upgrade-2.0.md b/upgrade-2.0.md index 0d64ea92e..e996b7890 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3010,12 +3010,19 @@ in an external collector, after the fact. in `ecotone_traces` and `ecotone_spans`, created by `ecotone:migration:database:setup`. A unit of work, a bus call or an asynchronous consumption, buffers its spans and writes them once, inside its transaction before the commit; the root span is written when it ends, after the commit. From those spans the `TraceCompletion` gateway answers -`isCompleted($traceId)`, true once the root ended and every route the trace sent has an ended handler span, -`isCompletedFor($traceId, $destination)` for one `endpointId` or projection name, and `await()` / `awaitFor()`, -which poll until then or throw `TraceCompletionTimedOut` naming what is still pending. A method with -`#[OnTraceCompleted]` receives a `TraceCompleted` once per trace, and one with `#[OnSpanEnded]` receives every -stored span, both after the commit that produced them, so a websocket push or a cache refresh never runs ahead of -the data. The trace id is the one in the `traceparent` header: mint it with `SpanContext::newTrace()` and send it +`isCompleted($traceId)`, true once the root ended, every route the trace sent has an ended handler span, and +every message it sent to a channel was consumed from that channel, so an asynchronous command, event or projection +keeps the trace pending until its consumer processed that exact message; `isCompletedFor($traceId, $destination)` +for one `endpointId` or projection name; and `await()` / `awaitFor()`, which poll until then or throw +`TraceCompletionTimedOut` naming the handlers and channels still pending. The root is the span that started the +trace in this process, a bus call or a handler reached outside any other span, never a consumer: a trace whose +producer stored nothing is never completed by its consumers alone. A unit of work that fails discards the spans it +buffered, so a handler that succeeded inside a rolled-back transaction, and a message whose send rolled back with +it, are not counted; the failed outer span is still stored. A method with `#[OnTraceCompleted]` receives a +`TraceCompleted` once per trace, and one with `#[OnSpanEnded]` receives every stored span, both after the commit +that produced them, so a websocket push or a cache refresh never runs ahead of the data. Work a subscriber does +through a bus is stored like any other trace but never notifies subscribers in turn, so a subscriber that sends a +command does not call itself again. The trace id is the one in the `traceparent` header: mint it with `SpanContext::newTrace()` and send it with the command, or take it from the HTTP request the command came from. A flow test given no connection keeps the store in memory. `ecotone:tracing:remove-expired-traces` deletes traces older than the retention passed to `withStoredTraces()`, seven days by default. From 8f20da67987ff745cd4628f4d235b2a25a0ca82b Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 09/50] feat(tracing): a failed unit of work stores nothing unless storeFailuresOutsideTransaction opts in to the write after the rollback The failed outer span was always written after the rollback, the one stored-trace write that happens outside a transaction. It is now dropped by default, and TracingConfiguration::withStoredTraces(storeFailuresOutsideTransaction: true) restores it. The flow-testing fallback that moves stored traces in memory carries the flag through. --- .../FlowTestingDbalConnections.php | 2 +- .../Conformance/TraceStoreConformanceTest.php | 66 ++++++++++++++++++- .../Api/Tracing/TracingConfiguration.php | 7 +- .../src/Tracing/Config/TracingModule.php | 1 + .../src/Tracing/StoringSpanRecorder.php | 5 ++ .../tests/Tracing/TraceCompletionTest.php | 63 ++++++++++++++++++ upgrade-2.0.md | 4 +- 7 files changed, 141 insertions(+), 7 deletions(-) diff --git a/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php b/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php index e2ef69956..6ed50c04a 100644 --- a/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php +++ b/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php @@ -111,7 +111,7 @@ private function tracesInMemoryWhereTheConnectionIsMissing(TracingConfiguration return $tracingConfiguration; } - return $tracingConfiguration->withStoredTraces(inMemory: true, retention: $tracingConfiguration->storedTracesRetention); + return $tracingConfiguration->withStoredTraces(inMemory: true, retention: $tracingConfiguration->storedTracesRetention, storeFailuresOutsideTransaction: $tracingConfiguration->storedFailuresOutsideTransaction); } private function refuseDbalChannelsWithoutTheirConnection(): void diff --git a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php index 48fc94ffe..28f047958 100644 --- a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php +++ b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php @@ -16,8 +16,11 @@ use Ecotone\Api\Messaging\MessageHeaders; use Ecotone\Api\Scheduling\Duration; use Ecotone\Api\Scheduling\TimeSpan; +use Ecotone\Api\Tracing\OnSpanEnded; use Ecotone\Api\Tracing\OnTraceCompleted; use Ecotone\Api\Tracing\SpanContext; +use Ecotone\Api\Tracing\SpanEnded; +use Ecotone\Api\Tracing\SpanStatus; use Ecotone\Api\Tracing\TraceCompleted; use Ecotone\Api\Tracing\TraceCompletion; use Ecotone\Api\Tracing\TraceCompletionTimedOut; @@ -241,6 +244,39 @@ public function audit(string $userId): void self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } + #[DataProvider('implementations')] + public function test_a_failing_asynchronous_handler_stores_nothing_by_default(string $implementation): void + { + $subscriber = $this->spanEndedSubscriber(); + $ecotone = $this->bootstrap($implementation, [$this->users(), $this->failingNotifier(), $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $ecotone->sendCommandWithRouting('user.register', '1'); + $storedBeforeTheConsumerRan = $subscriber->endedSpans; + + try { + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + } catch (RuntimeException) { + } + + self::assertSame($storedBeforeTheConsumerRan, $subscriber->endedSpans); + } + + #[DataProvider('implementations')] + public function test_a_failing_asynchronous_handler_stores_its_failed_consumer_span_when_failures_may_be_stored_outside_the_transaction(string $implementation): void + { + $subscriber = $this->spanEndedSubscriber(); + $ecotone = $this->bootstrap($implementation, [$this->users(), $this->failingNotifier(), $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')], storeFailuresOutsideTransaction: true); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + try { + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + } catch (RuntimeException) { + } + + self::assertSame(SpanStatus::Error, $subscriber->endedSpans['Receiving from channel: async_channel']); + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + } + public function test_completion_recorded_by_one_instance_is_visible_to_another_sharing_the_database(): void { $producer = $this->bootstrap('dbal', [$this->users()]); @@ -256,12 +292,12 @@ public function test_completion_recorded_by_one_instance_is_visible_to_another_s * @param object[] $services * @param object[] $extensionObjects */ - private function bootstrap(string $implementation, array $services, array $extensionObjects = [], ?Duration $retention = null): FlowTestSupport + private function bootstrap(string $implementation, array $services, array $extensionObjects = [], ?Duration $retention = null, bool $storeFailuresOutsideTransaction = false): FlowTestSupport { return $this->bootstrapFlowTesting( classesToResolve: array_map(fn (object $service): string => $service::class, $services), containerOrAvailableServices: [...$services, DbalConnectionFactory::class => $this->getConnectionFactory()], - extensionObjects: [TracingConfiguration::createWithDefaults()->withStoredTraces(inMemory: $implementation === 'in-memory', retention: $retention), ...$extensionObjects], + extensionObjects: [TracingConfiguration::createWithDefaults()->withStoredTraces(inMemory: $implementation === 'in-memory', retention: $retention, storeFailuresOutsideTransaction: $storeFailuresOutsideTransaction), ...$extensionObjects], enabledModules: [ModulePackageList::DBAL_PACKAGE], licenceKey: LicenceTesting::VALID_LICENCE, )->setupDatabaseTables(); @@ -283,6 +319,32 @@ public function register(string $userId, EventBus $eventBus): void }; } + private function failingNotifier(): object + { + return new class () { + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId): void + { + throw new RuntimeException('Notification failed'); + } + }; + } + + private function spanEndedSubscriber(): object + { + return new class () { + /** @var array */ + public array $endedSpans = []; + + #[OnSpanEnded] + public function whenEnded(SpanEnded $event): void + { + $this->endedSpans[$event->span->name] = $event->span->status; + } + }; + } + private function asynchronousNotifier(): object { return new class () { diff --git a/packages/Ecotone/Api/Tracing/TracingConfiguration.php b/packages/Ecotone/Api/Tracing/TracingConfiguration.php index 418706e5b..35dfaf785 100644 --- a/packages/Ecotone/Api/Tracing/TracingConfiguration.php +++ b/packages/Ecotone/Api/Tracing/TracingConfiguration.php @@ -19,12 +19,13 @@ private function __construct( public readonly bool $storedTraces, public readonly bool $storedTracesInMemory, public readonly Duration $storedTracesRetention, + public readonly bool $storedFailuresOutsideTransaction, ) { } public static function createWithDefaults(): self { - return new self([], false, false, Duration::minutes(60 * 24 * 7)); + return new self([], false, false, Duration::minutes(60 * 24 * 7), false); } public function withSpanRecorder(string $referenceName): self @@ -32,8 +33,8 @@ public function withSpanRecorder(string $referenceName): self return new self(...[...get_object_vars($this), 'spanRecorderReferenceNames' => [...$this->spanRecorderReferenceNames, $referenceName]]); } - public function withStoredTraces(bool $inMemory = false, ?Duration $retention = null): self + public function withStoredTraces(bool $inMemory = false, ?Duration $retention = null, bool $storeFailuresOutsideTransaction = false): self { - return new self(...[...get_object_vars($this), 'storedTraces' => true, 'storedTracesInMemory' => $inMemory, 'storedTracesRetention' => $retention ?? $this->storedTracesRetention]); + return new self(...[...get_object_vars($this), 'storedTraces' => true, 'storedTracesInMemory' => $inMemory, 'storedTracesRetention' => $retention ?? $this->storedTracesRetention, 'storedFailuresOutsideTransaction' => $storeFailuresOutsideTransaction]); } } diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index 3bde63a13..232409c25 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -200,6 +200,7 @@ private function registerStoredTraces(Configuration $messagingConfiguration, arr new Reference(MessagingEntrypointService::class), $this->spanEndedSubscribers !== [], $this->traceCompletedSubscribers !== [], + $tracingConfiguration->storedFailuresOutsideTransaction, ]) ); $messagingConfiguration->registerServiceDefinition( diff --git a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php index 30b002bbe..af99e251d 100644 --- a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php +++ b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php @@ -9,6 +9,7 @@ use Ecotone\Api\Tracing\Span; use Ecotone\Api\Tracing\SpanEnded; use Ecotone\Api\Tracing\SpanRecorder; +use Ecotone\Api\Tracing\SpanStatus; use Ecotone\Api\Tracing\TraceCompleted; use Ecotone\Messaging\Gateway\MessagingEntrypointService; @@ -46,6 +47,7 @@ public function __construct( private MessagingEntrypointService $messagingEntrypoint, private bool $publishesEndedSpans, private bool $publishesCompletedTraces, + private bool $storesFailuresOutsideTransaction, ) { } @@ -56,6 +58,9 @@ public function record(Span $span): void return; } + if ($span->status === SpanStatus::Error && ! $this->storesFailuresOutsideTransaction) { + return; + } $this->write([...$this->buffer, $span]); $this->buffer = []; diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index d012af96b..7f7310013 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -22,6 +22,7 @@ use Ecotone\Api\Tracing\OnTraceCompleted; use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\SpanEnded; +use Ecotone\Api\Tracing\SpanStatus; use Ecotone\Api\Tracing\TraceCompleted; use Ecotone\Api\Tracing\TraceCompletion; use Ecotone\Api\Tracing\TraceCompletionTimedOut; @@ -365,6 +366,43 @@ public function refresh(string $traceId): void $this->assertSame([$trace->traceId], array_values(array_unique($subscriber->notifiedTraces))); } + public function test_a_failed_unit_of_work_stores_nothing_by_default(): void + { + $subscriber = $this->spanEndedSubscriber(); + $ecotone = $this->bootstrap([$this->failingUsers(), $subscriber]); + $trace = SpanContext::newTrace(); + + try { + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + $this->fail('Expected the registration failure to propagate'); + } catch (RuntimeException) { + } + + $this->assertSame([], $subscriber->endedSpans); + $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + public function test_a_failed_unit_of_work_stores_its_failed_root_span_when_failures_may_be_stored_outside_the_transaction(): void + { + $subscriber = $this->spanEndedSubscriber(); + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$this->failingUsers()::class, $subscriber::class], + [$this->failingUsers(), $subscriber], + [TracingConfiguration::createWithDefaults()->withStoredTraces(storeFailuresOutsideTransaction: true)], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $trace = SpanContext::newTrace(); + + try { + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + $this->fail('Expected the registration failure to propagate'); + } catch (RuntimeException) { + } + + $this->assertSame(['Command Bus' => SpanStatus::Error], $subscriber->endedSpans); + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + public function test_stored_traces_require_an_enterprise_licence(): void { $this->expectException(LicensingException::class); @@ -426,6 +464,31 @@ public function register(string $userId, EventBus $eventBus): void }; } + private function failingUsers(): object + { + return new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId): void + { + throw new RuntimeException('Registration failed'); + } + }; + } + + private function spanEndedSubscriber(): object + { + return new class () { + /** @var array */ + public array $endedSpans = []; + + #[OnSpanEnded] + public function whenEnded(SpanEnded $event): void + { + $this->endedSpans[$event->span->name] = $event->span->status; + } + }; + } + private function asynchronousNotifier(): object { return new class () { diff --git a/upgrade-2.0.md b/upgrade-2.0.md index e996b7890..df02aba7d 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3018,7 +3018,9 @@ for one `endpointId` or projection name; and `await()` / `awaitFor()`, which pol trace in this process, a bus call or a handler reached outside any other span, never a consumer: a trace whose producer stored nothing is never completed by its consumers alone. A unit of work that fails discards the spans it buffered, so a handler that succeeded inside a rolled-back transaction, and a message whose send rolled back with -it, are not counted; the failed outer span is still stored. A method with `#[OnTraceCompleted]` receives a +it, are not counted. By default nothing of a failed unit of work is stored, because the only moment left to store it +is after the rollback, outside any transaction; `withStoredTraces(storeFailuresOutsideTransaction: true)` opts in +to storing the failed outer span then, so a failed trace shows its error and counts as complete. A method with `#[OnTraceCompleted]` receives a `TraceCompleted` once per trace, and one with `#[OnSpanEnded]` receives every stored span, both after the commit that produced them, so a websocket push or a cache refresh never runs ahead of the data. Work a subscriber does through a bus is stored like any other trace but never notifies subscribers in turn, so a subscriber that sends a From 1060c08f830ec0f5d6b09c8a02c542282751a57c Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 10/50] test(tracing): each Enterprise entry point of stored traces is refused without a licence, and the refusal names what to remove Only withStoredTraces() without a licence was tested; #[OnTraceCompleted] and #[OnSpanEnded] reach the same guard but had no test of their own, and the message named neither the configuration that triggered it nor the way out besides buying a licence. It now lists the usages found and says to remove them to run without one. --- .../src/Tracing/Config/TracingModule.php | 10 ++++- .../tests/Tracing/TraceCompletionTest.php | 39 ++++++++++++++++++- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index 232409c25..09a8256ad 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -169,8 +169,14 @@ public function getModulePackageName(): string private function refuseMisconfiguredStoredTraces(Configuration $messagingConfiguration, TracingConfiguration $tracingConfiguration): void { $subscribers = [...$this->traceCompletedSubscribers, ...$this->spanEndedSubscribers]; - if (($tracingConfiguration->storedTraces || $subscribers !== []) && ! $messagingConfiguration->isRunningForEnterpriseLicence()) { - throw LicensingException::create('Stored traces and TraceCompletion require an Ecotone Enterprise licence.'); + $enterpriseUsages = [...($tracingConfiguration->storedTraces ? ['TracingConfiguration::withStoredTraces()'] : []), ...$subscribers]; + if ($enterpriseUsages !== [] && ! $messagingConfiguration->isRunningForEnterpriseLicence()) { + throw LicensingException::create(sprintf( + '%s %s an Ecotone Enterprise licence. Remove %s to run without one.', + implode(' and ', $enterpriseUsages), + count($enterpriseUsages) === 1 ? 'requires' : 'require', + count($enterpriseUsages) === 1 ? 'it' : 'them', + )); } if ($tracingConfiguration->storedTraces || $subscribers === []) { return; diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index 7f7310013..6754538a4 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -406,7 +406,7 @@ public function test_a_failed_unit_of_work_stores_its_failed_root_span_when_fail public function test_stored_traces_require_an_enterprise_licence(): void { $this->expectException(LicensingException::class); - $this->expectExceptionMessage('Stored traces and TraceCompletion require an Ecotone Enterprise licence.'); + $this->expectExceptionMessage('TracingConfiguration::withStoredTraces() requires an Ecotone Enterprise licence. Remove it to run without one.'); EcotoneLite::bootstrapFlowTesting( [], @@ -415,6 +415,43 @@ public function test_stored_traces_require_an_enterprise_licence(): void ); } + public function test_a_trace_completed_subscriber_requires_an_enterprise_licence(): void + { + $subscriber = new class () { + #[OnTraceCompleted] + public function whenCompleted(TraceCompleted $event): void + { + } + }; + + $this->expectException(LicensingException::class); + $this->expectExceptionMessage('TracingConfiguration::withStoredTraces() and #[OnTraceCompleted] on ' . $subscriber::class . '::whenCompleted require an Ecotone Enterprise licence. Remove them to run without one.'); + + EcotoneLite::bootstrapFlowTesting( + [$subscriber::class], + [$subscriber], + [TracingConfiguration::createWithDefaults()->withStoredTraces()], + ); + } + + public function test_a_span_ended_subscriber_requires_an_enterprise_licence_even_without_stored_traces(): void + { + $subscriber = new class () { + #[OnSpanEnded] + public function whenEnded(SpanEnded $event): void + { + } + }; + + $this->expectException(LicensingException::class); + $this->expectExceptionMessage('#[OnSpanEnded] on ' . $subscriber::class . '::whenEnded requires an Ecotone Enterprise licence. Remove it to run without one.'); + + EcotoneLite::bootstrapFlowTesting( + [$subscriber::class], + [$subscriber], + ); + } + public function test_a_trace_completed_subscriber_without_stored_traces_is_refused_at_bootstrap(): void { $subscriber = new class () { From f45344f96ce12656a74d1a3faad328edb96cadee Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 11/50] refactor(tracing): span names follow the OpenTelemetry messaging convention, {operation} {target}, with the handler's qualified name in code.function.name The 1.x names were sentences, Sending to Channel: orders, Command Handler: App\OrderService::place, inconsistent in case and long. They are now send command, send event, send query, send distributed, route , send , receive , process and process , so a trace UI groups them as it groups any other messaging span. The fully qualified handler stays on the span as code.function.name. --- .../Tests/Tracing/TracingStackTest.php | 54 ++++---- .../Conformance/TraceStoreConformanceTest.php | 2 +- .../Ecotone/Api/Tracing/SpanAttribute.php | 1 + .../src/Tracing/TracedRouteProcessor.php | 4 +- .../src/Tracing/TracingChannelInterceptor.php | 4 +- .../src/Tracing/TracingInterceptor.php | 44 ++++--- .../tests/Tracing/AsynchronousTracingTest.php | 66 +++++----- .../tests/Tracing/RouteTracingTest.php | 36 ++--- .../tests/Tracing/TraceCompletionTest.php | 6 +- .../tests/Tracing/TracingEdgeCasesTest.php | 32 ++--- .../Ecotone/tests/Tracing/TracingTestCase.php | 2 +- .../Ecotone/tests/Tracing/TracingTreeTest.php | 72 +++++----- .../CorrelatedHeadersPropagationTest.php | 34 ++--- .../tests/Integration/TracingTestCase.php | 2 +- .../tests/Integration/TracingTreeTest.php | 124 +++++++++--------- upgrade-2.0.md | 29 ++-- 16 files changed, 264 insertions(+), 248 deletions(-) diff --git a/Monorepo/CrossModuleTests/Tests/Tracing/TracingStackTest.php b/Monorepo/CrossModuleTests/Tests/Tracing/TracingStackTest.php index 6880a298f..c82951ed7 100644 --- a/Monorepo/CrossModuleTests/Tests/Tracing/TracingStackTest.php +++ b/Monorepo/CrossModuleTests/Tests/Tracing/TracingStackTest.php @@ -10,17 +10,17 @@ use Ecotone\Api\Messaging\ConfiguredMessagingSystem; use Illuminate\Foundation\Http\Kernel as LaravelKernel; use Monorepo\CrossModuleTests\Tests\FullAppTestCase; -use Monorepo\ExampleApp\Common\Domain\Notification\NotificationSubscriber; use Monorepo\ExampleApp\Common\Domain\Order\Command\PlaceOrder; -use Monorepo\ExampleApp\Common\Domain\Order\Order; use Monorepo\ExampleApp\Common\Domain\Order\ShippingAddress; use Monorepo\ExampleApp\Common\Infrastructure\Configuration; -use Monorepo\ExampleApp\Common\Infrastructure\Output; use Monorepo\ExampleApp\ExampleAppCaseTrait; use OpenTelemetry\SDK\Trace\SpanExporter\InMemoryExporter; use Psr\Container\ContainerInterface; use Test\Ecotone\OpenTelemetry\Integration\TracingTestCase; +/** + * @internal + */ final class TracingStackTest extends FullAppTestCase { use ExampleAppCaseTrait; @@ -103,47 +103,47 @@ private function assertTracing(InMemoryExporter $exporter): void TracingTestCase::compareTreesByDetails( [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . Order::class . '::create'], - 'children' => [] + 'details' => ['name' => 'process Order::create'], + 'children' => [], ], [ - 'details' => ['name' => 'Event Bus'], - 'children' => [] + 'details' => ['name' => 'send event'], + 'children' => [], ], [ - 'details' => ['name' => 'Sending to Channel: delivery'], - 'children' => [] + 'details' => ['name' => 'send delivery'], + 'children' => [], ], [ - 'details' => ['name' => 'Sending to Channel: notifications'], + 'details' => ['name' => 'send notifications'], 'children' => [ [ - 'details' => ['name' => 'Receiving from channel: notifications'], + 'details' => ['name' => 'receive notifications'], 'children' => [ [ - 'details' => ['name' => 'Event Handler: ' . NotificationSubscriber::class . '::whenOrderWasPlaced'], - 'children' => [] - ] - ] - ] - ] - ] - ] + 'details' => ['name' => 'process NotificationSubscriber::whenOrderWasPlaced'], + 'children' => [], + ], + ], + ], + ], + ], + ], ], [ - 'details' => ['name' => 'Query Bus'], + 'details' => ['name' => 'send query'], 'children' => [ [ - 'details' => ['name' => 'Query Handler: ' . Output::class . '::getMessages'], - 'children' => [] - ] - ] - ] + 'details' => ['name' => 'process Output::getMessages'], + 'children' => [], + ], + ], + ], ], TracingTestCase::buildTree($exporter) ); } -} \ No newline at end of file +} diff --git a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php index 28f047958..346099df2 100644 --- a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php +++ b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php @@ -273,7 +273,7 @@ public function test_a_failing_asynchronous_handler_stores_its_failed_consumer_s } catch (RuntimeException) { } - self::assertSame(SpanStatus::Error, $subscriber->endedSpans['Receiving from channel: async_channel']); + self::assertSame(SpanStatus::Error, $subscriber->endedSpans['receive async_channel']); self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); } diff --git a/packages/Ecotone/Api/Tracing/SpanAttribute.php b/packages/Ecotone/Api/Tracing/SpanAttribute.php index 60452ef07..8276c763e 100644 --- a/packages/Ecotone/Api/Tracing/SpanAttribute.php +++ b/packages/Ecotone/Api/Tracing/SpanAttribute.php @@ -15,6 +15,7 @@ final class SpanAttribute public const ENDPOINT_ID = 'ecotone.endpoint.id'; public const PROJECTION_NAME = 'ecotone.projection.name'; public const TRACE_ROOT = 'ecotone.trace.root'; + public const CODE_FUNCTION_NAME = 'code.function.name'; public const ECOTONE = 'ecotone'; public const OPERATION_SEND = 'send'; diff --git a/packages/Ecotone/src/Tracing/TracedRouteProcessor.php b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php index 96ee63f65..6c7a739cf 100644 --- a/packages/Ecotone/src/Tracing/TracedRouteProcessor.php +++ b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php @@ -60,10 +60,10 @@ private function spanName(): string { $projectionName = ProjectingModule::projectionNameOfTriggerChannel($this->routeName); if ($projectionName !== null) { - return 'Routing to Projection: ' . $projectionName; + return 'route ' . $projectionName; } - return 'Routing to: ' . (BusRoutingMapBuilder::endpointIdOfDestinationChannel($this->routeName) ?? $this->routeName); + return 'route ' . (BusRoutingMapBuilder::endpointIdOfDestinationChannel($this->routeName) ?? $this->routeName); } private function destination(): ?string diff --git a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php index 8c65a340d..befefdf56 100644 --- a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php @@ -32,7 +32,7 @@ public function __construct( public function preSend(Message $message, MessageChannel $messageChannel): ?Message { - $span = $this->tracer->startSpan('Sending to Channel: ' . $this->channelName, SpanKind::Producer, [ + $span = $this->tracer->startSpan('send ' . $this->channelName, SpanKind::Producer, [ ...MessageSpanAttributes::from($message), SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND, SpanAttribute::MESSAGING_DESTINATION_NAME => $this->channelName, @@ -72,7 +72,7 @@ public function afterReceiveCompletion(?Message $message, MessageChannel $messag } $this->tracer - ->startSpan('Asynchronous Channel: ' . $this->channelName, SpanKind::Consumer, [ + ->startSpan('receive ' . $this->channelName, SpanKind::Consumer, [ ...MessageSpanAttributes::from($message), SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS, SpanAttribute::MESSAGING_DESTINATION_NAME => $this->channelName, diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 77d75f8ec..2e8debfd5 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -31,46 +31,46 @@ public function __construct( public function traceCommandBus(MethodInvocation $methodInvocation, Message $message): mixed { - return $this->trace('Command Bus', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); + return $this->trace('send command', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); } public function traceEventBus(MethodInvocation $methodInvocation, Message $message): mixed { - return $this->trace('Event Bus', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); + return $this->trace('send event', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); } public function traceQueryBus(MethodInvocation $methodInvocation, Message $message): mixed { - return $this->trace('Query Bus', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); + return $this->trace('send query', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); } public function traceCommandHandler(MethodInvocation $methodInvocation, Message $message, CommandHandler $endpoint): mixed { - return $this->trace('Command Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes(TraceState::destinationOf($message) ?? $endpoint->getEndpointId())); + return $this->traceHandler($methodInvocation, $message, TraceState::destinationOf($message) ?? $endpoint->getEndpointId()); } public function traceQueryHandler(MethodInvocation $methodInvocation, Message $message, QueryHandler $endpoint): mixed { - return $this->trace('Query Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes(TraceState::destinationOf($message) ?? $endpoint->getEndpointId())); + return $this->traceHandler($methodInvocation, $message, TraceState::destinationOf($message) ?? $endpoint->getEndpointId()); } public function traceEventHandler(MethodInvocation $methodInvocation, Message $message, EventHandler $endpoint): mixed { - return $this->trace('Event Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes(TraceState::destinationOf($message) ?? $endpoint->getEndpointId())); + return $this->traceHandler($methodInvocation, $message, TraceState::destinationOf($message) ?? $endpoint->getEndpointId()); } public function traceMessageHandler(MethodInvocation $methodInvocation, Message $message, InternalHandler $endpoint): mixed { - return $this->trace('Message Handler: ' . $methodInvocation->getName(), $methodInvocation, $message, SpanKind::Server, self::processAttributes(TraceState::destinationOf($message) ?? $endpoint->getEndpointId())); + return $this->traceHandler($methodInvocation, $message, TraceState::destinationOf($message) ?? $endpoint->getEndpointId()); } public function traceAsynchronousEndpoint(MethodInvocation $methodInvocation, Message $message): mixed { $headers = $message->getHeaders(); $name = match (true) { - $headers->containsKey(MessageHeaders::POLLED_CHANNEL_NAME) => 'Receiving from channel: ' . $headers->get(MessageHeaders::POLLED_CHANNEL_NAME), - $headers->containsKey(MessageHeaders::INBOUND_REQUEST_CHANNEL) => 'Receiving from inbound channel adapter: ' . $headers->get(MessageHeaders::INBOUND_REQUEST_CHANNEL), - default => 'Endpoint: ' . $headers->get(MessageHeaders::CONSUMER_POLLING_METADATA)->getEndpointId() . ' produced Message', + $headers->containsKey(MessageHeaders::POLLED_CHANNEL_NAME) => 'receive ' . $headers->get(MessageHeaders::POLLED_CHANNEL_NAME), + $headers->containsKey(MessageHeaders::INBOUND_REQUEST_CHANNEL) => 'receive ' . $headers->get(MessageHeaders::INBOUND_REQUEST_CHANNEL), + default => 'poll ' . $headers->get(MessageHeaders::CONSUMER_POLLING_METADATA)->getEndpointId(), }; $attributes = [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS]; @@ -83,7 +83,7 @@ public function traceAsynchronousEndpoint(MethodInvocation $methodInvocation, Me public function traceProjection(MethodInvocation $methodInvocation, Message $message, ProjectionTriggerEndpoint $trigger): mixed { - return $this->trace('Projection: ' . $trigger->projectionName, $methodInvocation, $message, SpanKind::Consumer, [ + return $this->trace('process ' . $trigger->projectionName, $methodInvocation, $message, SpanKind::Consumer, [ SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS, SpanAttribute::PROJECTION_NAME => $trigger->projectionName, ]); @@ -91,7 +91,7 @@ public function traceProjection(MethodInvocation $methodInvocation, Message $mes public function traceDistributedBus(MethodInvocation $methodInvocation, Message $message): mixed { - return $this->trace('Distributed Bus', $methodInvocation, $message, SpanKind::Producer, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); + return $this->trace('send distributed', $methodInvocation, $message, SpanKind::Producer, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); } public function provideContextForDistributedBus(Message $message): Message @@ -113,17 +113,25 @@ public static function remoteParentOf(Message $message): ?SpanContext return SpanContext::fromTraceparent($headers->containsKey(MessageHeaders::TRACEPARENT) ? $headers->get(MessageHeaders::TRACEPARENT) : null); } - /** - * @return array - */ - private static function processAttributes(string $endpointId): array + private function traceHandler(MethodInvocation $methodInvocation, Message $message, string $endpointId): mixed { - $attributes = [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS]; + $attributes = [ + SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS, + SpanAttribute::CODE_FUNCTION_NAME => $methodInvocation->getName(), + ]; if ($endpointId !== '') { $attributes[SpanAttribute::ENDPOINT_ID] = $endpointId; } - return $attributes; + return $this->trace('process ' . self::shortNameOf($methodInvocation->getName()), $methodInvocation, $message, SpanKind::Server, $attributes); + } + + private static function shortNameOf(string $classAndMethod): string + { + [$class, $method] = explode('::', $classAndMethod, 2); + $class = explode("\0", $class, 2)[0]; + + return substr($class, (int) strrpos('\\' . $class, '\\')) . '::' . $method; } /** diff --git a/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php b/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php index 1bf9e766e..4bd4e568f 100644 --- a/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php +++ b/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php @@ -42,15 +42,15 @@ public function test_tracing_with_asynchronous_handler(): void self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ [ - 'details' => ['name' => 'Sending to Channel: async_channel', 'kind' => SpanKind::Producer], + 'details' => ['name' => 'send async_channel', 'kind' => SpanKind::Producer], 'children' => [ [ - 'details' => ['name' => 'Receiving from channel: async_channel', 'kind' => SpanKind::Consumer], + 'details' => ['name' => 'receive async_channel', 'kind' => SpanKind::Consumer], 'children' => [ - ['details' => ['name' => 'Command Handler: ' . $users::class . '::register'], 'children' => []], + ['details' => ['name' => 'process class@anonymous::register'], 'children' => []], ], ], ], @@ -99,10 +99,10 @@ public function test_tracing_same_message_for_asynchronous_scenario(): void $node = self::getNodeAtTargetedSpan( [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'child' => [ - 'details' => ['name' => 'Sending to Channel: async_channel'], - 'child' => ['details' => ['name' => 'Receiving from channel: async_channel']], + 'details' => ['name' => 'send async_channel'], + 'child' => ['details' => ['name' => 'receive async_channel']], ], ], self::buildTree($recorder->recordedSpans()) @@ -130,15 +130,15 @@ public function test_two_traces_with_asynchronous_handlers(): void ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 2)); $expectedFlow = fn (string $flowId): array => [ - 'details' => ['name' => 'Command Bus', 'attributes' => ['flowId' => $flowId]], + 'details' => ['name' => 'send command', 'attributes' => ['flowId' => $flowId]], 'children' => [ [ - 'details' => ['name' => 'Sending to Channel: async_channel', 'attributes' => ['flowId' => $flowId]], + 'details' => ['name' => 'send async_channel', 'attributes' => ['flowId' => $flowId]], 'children' => [ [ - 'details' => ['name' => 'Receiving from channel: async_channel', 'attributes' => ['flowId' => $flowId]], + 'details' => ['name' => 'receive async_channel', 'attributes' => ['flowId' => $flowId]], 'children' => [ - ['details' => ['name' => 'Command Handler: ' . $users::class . '::register', 'attributes' => ['flowId' => $flowId]], 'children' => []], + ['details' => ['name' => 'process class@anonymous::register', 'attributes' => ['flowId' => $flowId]], 'children' => []], ], ], ], @@ -162,7 +162,7 @@ public function test_user_headers_travel_with_the_message_into_every_span(): voi ->sendCommandWithRouting('user.register', '1', metadata: ['user_id' => '123']) ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - foreach (['Command Bus', 'Sending to Channel: async_channel', 'Receiving from channel: async_channel', 'Command Handler: ' . $users::class . '::register'] as $spanName) { + foreach (['send command', 'send async_channel', 'receive async_channel', 'process class@anonymous::register'] as $spanName) { $spans = array_values(array_filter($recorder->recordedSpans(), fn (Span $span): bool => $span->name === $spanName)); $this->assertCount(1, $spans, $spanName); $this->assertSame('123', $spans[0]->attributes['user_id'], $spanName); @@ -216,16 +216,16 @@ public function notify(string $userId): void self::getNodeAtTargetedSpan( [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'child' => [ - 'details' => ['name' => 'Sending to Channel: async_channel'], + 'details' => ['name' => 'send async_channel'], 'child' => [ - 'details' => ['name' => 'Receiving from channel: async_channel'], + 'details' => ['name' => 'receive async_channel'], 'child' => [ - 'details' => ['name' => 'Sending to Channel: async_channel'], + 'details' => ['name' => 'send async_channel'], 'child' => [ - 'details' => ['name' => 'Receiving from channel: async_channel'], - 'child' => ['details' => ['name' => 'Event Handler: ' . $notifier::class . '::notify']], + 'details' => ['name' => 'receive async_channel'], + 'child' => ['details' => ['name' => 'process class@anonymous::notify']], ], ], ], @@ -235,14 +235,14 @@ public function notify(string $userId): void ); self::getNodeAtTargetedSpan( [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'child' => [ - 'details' => ['name' => 'Sending to Channel: async_channel'], + 'details' => ['name' => 'send async_channel'], 'child' => [ - 'details' => ['name' => 'Receiving from channel: async_channel'], + 'details' => ['name' => 'receive async_channel'], 'child' => [ - 'details' => ['name' => 'Command Handler: ' . $users::class . '::register'], - 'child' => ['details' => ['name' => 'Event Bus']], + 'details' => ['name' => 'process class@anonymous::register'], + 'child' => ['details' => ['name' => 'send event']], ], ], ], @@ -281,14 +281,14 @@ public function notify(string $userId): void $eventBus = self::getNodeAtTargetedSpan( [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'child' => [ - 'details' => ['name' => 'Sending to Channel: async_channel'], + 'details' => ['name' => 'send async_channel'], 'child' => [ - 'details' => ['name' => 'Receiving from channel: async_channel'], + 'details' => ['name' => 'receive async_channel'], 'child' => [ - 'details' => ['name' => 'Command Handler: ' . $users::class . '::register'], - 'child' => ['details' => ['name' => 'Event Bus']], + 'details' => ['name' => 'process class@anonymous::register'], + 'child' => ['details' => ['name' => 'send event']], ], ], ], @@ -300,7 +300,7 @@ public function notify(string $userId): void $this->assertNotEmpty(array_filter($eventBusEvents, fn (string $name): bool => str_starts_with($name, 'Publishing Event Message'))); $route = $eventBus['children'][array_key_first($eventBus['children'])]; - $this->assertSame('Routing to: user.registered.endpoint', $route['details']['name']); + $this->assertSame('route user.registered.endpoint', $route['details']['name']); $routeEvents = array_map(fn ($event): string => $event->name, $route['details']['events']); $this->assertNotEmpty(array_filter($routeEvents, fn (string $name): bool => str_starts_with($name, 'Collecting message with id'))); } @@ -325,8 +325,8 @@ public function handle(): void self::compareTreesByDetails( [ - ['details' => ['name' => 'Receiving from inbound channel adapter: nullChannel', 'kind' => SpanKind::Consumer], 'children' => []], - ['details' => ['name' => 'Receiving from inbound channel adapter: nullChannel', 'kind' => SpanKind::Consumer], 'children' => []], + ['details' => ['name' => 'receive nullChannel', 'kind' => SpanKind::Consumer], 'children' => []], + ['details' => ['name' => 'receive nullChannel', 'kind' => SpanKind::Consumer], 'children' => []], ], self::buildTree($recorder->recordedSpans()) ); @@ -346,9 +346,9 @@ public function test_tracing_workflow_scheduled_handler(): void ->run('scheduled_handler', ExecutionPollingMetadata::createWithDefaults()->withExecutionAmountLimit(2)); $expectedRun = [ - 'details' => ['name' => 'Receiving from inbound channel adapter: add'], + 'details' => ['name' => 'receive add'], 'children' => [ - ['details' => ['name' => 'Message Handler: ' . $workflow::class . '::add'], 'children' => []], + ['details' => ['name' => 'process class@anonymous::add'], 'children' => []], ], ]; self::compareTreesByDetails([$expectedRun, $expectedRun], self::buildTree($recorder->recordedSpans())); diff --git a/packages/Ecotone/tests/Tracing/RouteTracingTest.php b/packages/Ecotone/tests/Tracing/RouteTracingTest.php index acf5592bf..245d5e151 100644 --- a/packages/Ecotone/tests/Tracing/RouteTracingTest.php +++ b/packages/Ecotone/tests/Tracing/RouteTracingTest.php @@ -46,16 +46,16 @@ public function notify(string $merchantId): void self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Event Bus'], + 'details' => ['name' => 'send event'], 'children' => [ [ 'details' => [ - 'name' => 'Routing to: merchant.notifier', + 'name' => 'route merchant.notifier', 'kind' => SpanKind::Producer, 'attributes' => ['messaging.destination.name' => 'merchant.notifier.target', 'ecotone.endpoint.id' => 'merchant.notifier'], ], 'children' => [ - ['details' => ['name' => 'Event Handler: ' . $subscriber::class . '::notify', 'attributes' => ['ecotone.endpoint.id' => 'merchant.notifier']], 'children' => []], + ['details' => ['name' => 'process class@anonymous::notify', 'attributes' => ['ecotone.endpoint.id' => 'merchant.notifier']], 'children' => []], ], ], ], @@ -87,7 +87,7 @@ public function notify(string $merchantId): void } catch (RuntimeException) { } - $this->assertSame(SpanStatus::Error, self::spanNamed('Routing to: merchant.notifier', $recorder)->status); + $this->assertSame(SpanStatus::Error, self::spanNamed('route merchant.notifier', $recorder)->status); } public function test_an_asynchronous_event_route_leads_through_the_channel_to_its_handler(): void @@ -111,14 +111,14 @@ public function notify(string $merchantId): void self::getNodeAtTargetedSpan( [ - 'details' => ['name' => 'Event Bus'], + 'details' => ['name' => 'send event'], 'child' => [ - 'details' => ['name' => 'Routing to: merchant.notifier'], + 'details' => ['name' => 'route merchant.notifier'], 'child' => [ - 'details' => ['name' => 'Sending to Channel: async_channel', 'attributes' => ['messaging.destination.name' => 'async_channel']], + 'details' => ['name' => 'send async_channel', 'attributes' => ['messaging.destination.name' => 'async_channel']], 'child' => [ - 'details' => ['name' => 'Receiving from channel: async_channel', 'attributes' => ['messaging.destination.name' => 'async_channel']], - 'child' => ['details' => ['name' => 'Event Handler: ' . $subscriber::class . '::notify']], + 'details' => ['name' => 'receive async_channel', 'attributes' => ['messaging.destination.name' => 'async_channel']], + 'child' => ['details' => ['name' => 'process class@anonymous::notify']], ], ], ], @@ -149,12 +149,12 @@ public function whenPlaced(array $event): void self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Event Bus'], + 'details' => ['name' => 'send event'], 'children' => [ [ - 'details' => ['name' => 'Routing to Projection: orders', 'kind' => SpanKind::Producer, 'attributes' => ['ecotone.projection.name' => 'orders']], + 'details' => ['name' => 'route orders', 'kind' => SpanKind::Producer, 'attributes' => ['ecotone.projection.name' => 'orders']], 'children' => [ - ['details' => ['name' => 'Projection: orders', 'kind' => SpanKind::Consumer, 'attributes' => ['ecotone.projection.name' => 'orders']], 'children' => []], + ['details' => ['name' => 'process orders', 'kind' => SpanKind::Consumer, 'attributes' => ['ecotone.projection.name' => 'orders']], 'children' => []], ], ], ], @@ -187,14 +187,14 @@ public function whenPlaced(array $event): void self::getNodeAtTargetedSpan( [ - 'details' => ['name' => 'Event Bus'], + 'details' => ['name' => 'send event'], 'child' => [ - 'details' => ['name' => 'Routing to Projection: orders'], + 'details' => ['name' => 'route orders'], 'child' => [ - 'details' => ['name' => 'Sending to Channel: async_channel'], + 'details' => ['name' => 'send async_channel'], 'child' => [ - 'details' => ['name' => 'Receiving from channel: async_channel'], - 'child' => ['details' => ['name' => 'Projection: orders', 'kind' => SpanKind::Consumer]], + 'details' => ['name' => 'receive async_channel'], + 'child' => ['details' => ['name' => 'process orders', 'kind' => SpanKind::Consumer]], ], ], ], @@ -223,7 +223,7 @@ public function register(string $userId): void foreach ($recorder->recordedSpans() as $span) { $this->assertSame('ecotone', $span->attributes['messaging.system'], $span->name); } - $this->assertSame('user.registration', self::spanNamed('Command Handler: ' . $users::class . '::register', $recorder)->attributes['ecotone.endpoint.id']); + $this->assertSame('user.registration', self::spanNamed('process class@anonymous::register', $recorder)->attributes['ecotone.endpoint.id']); } private static function spanNamed(string $name, InMemorySpanRecorder $recorder): Span diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index 6754538a4..025848cfc 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -237,8 +237,8 @@ public function whenEnded(SpanEnded $event): void $ecotone->sendCommandWithRouting('user.register', '1'); - $this->assertContains('Command Bus', $subscriber->spanNames); - $this->assertContains('Command Handler: ' . $users::class . '::register', $subscriber->spanNames); + $this->assertContains('send command', $subscriber->spanNames); + $this->assertContains('process class@anonymous::register', $subscriber->spanNames); } public function test_an_asynchronous_command_keeps_the_trace_pending_until_its_consumer_ran(): void @@ -399,7 +399,7 @@ public function test_a_failed_unit_of_work_stores_its_failed_root_span_when_fail } catch (RuntimeException) { } - $this->assertSame(['Command Bus' => SpanStatus::Error], $subscriber->endedSpans); + $this->assertSame(['send command' => SpanStatus::Error], $subscriber->endedSpans); $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } diff --git a/packages/Ecotone/tests/Tracing/TracingEdgeCasesTest.php b/packages/Ecotone/tests/Tracing/TracingEdgeCasesTest.php index cd5601220..95e72b7d5 100644 --- a/packages/Ecotone/tests/Tracing/TracingEdgeCasesTest.php +++ b/packages/Ecotone/tests/Tracing/TracingEdgeCasesTest.php @@ -49,12 +49,12 @@ public function test_a_failed_send_to_an_asynchronous_channel_fails_the_send_spa $caught = $this->runConsumer($ecotone); $this->assertStringContainsString('Exception on send', $caught?->getMessage() ?? ''); - self::assertEverySpanNamedHasStatus('Sending to Channel: failing_channel', SpanStatus::Error, $recorder); + self::assertEverySpanNamedHasStatus('send failing_channel', SpanStatus::Error, $recorder); $recorder->clear(); $ecotone->sendCommandWithRouting('user.register', '2'); $this->assertCount(1, array_filter($recorder->recordedSpans(), fn (Span $span): bool => $span->isRoot())); - $this->assertSame('Command Bus', self::rootSpan($recorder)->name); + $this->assertSame('send command', self::rootSpan($recorder)->name); } public function test_a_channel_interceptor_failing_before_send_leaves_no_open_span_behind(): void @@ -78,7 +78,7 @@ public function preSend(Message $message, MessageChannel $messageChannel): ?Mess $recorder->clear(); $ecotone->sendCommandWithRouting('user.register', '2'); - $this->assertSame('Command Bus', self::rootSpan($recorder)->name); + $this->assertSame('send command', self::rootSpan($recorder)->name); } public function test_a_message_filtered_out_by_a_channel_interceptor_leaves_no_open_span_behind(): void @@ -101,7 +101,7 @@ public function preSend(Message $message, MessageChannel $messageChannel): ?Mess $recorder->clear(); $ecotone->sendCommandWithRouting('user.register', '2'); - $this->assertSame('Command Bus', self::rootSpan($recorder)->name); + $this->assertSame('send command', self::rootSpan($recorder)->name); } public function test_a_consumer_receiving_a_message_without_a_trace_carrier_starts_its_own_trace(): void @@ -124,7 +124,7 @@ public function preSend(Message $message, MessageChannel $messageChannel): ?Mess $this->assertNull($this->runConsumer($ecotone, 'failing_channel')); - $receivingSpan = self::singleSpanNamed('Receiving from channel: failing_channel', $recorder); + $receivingSpan = self::singleSpanNamed('receive failing_channel', $recorder); $this->assertTrue($receivingSpan->isRoot()); } @@ -153,7 +153,7 @@ public function preSend(Message $message, MessageChannel $messageChannel): ?Mess $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - $this->assertTrue(self::singleSpanNamed('Receiving from channel: async_channel', $recorder)->isRoot()); + $this->assertTrue(self::singleSpanNamed('receive async_channel', $recorder)->isRoot()); } public function test_a_command_carrying_a_trace_carrier_continues_that_trace_as_a_child(): void @@ -173,10 +173,10 @@ public function register(string $userId): void ) ->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01']); - $commandBus = self::singleSpanNamed('Command Bus', $recorder); + $commandBus = self::singleSpanNamed('send command', $recorder); $this->assertSame('0af7651916cd43dd8448eb211c80319c', $commandBus->traceId); $this->assertSame('b7ad6b7169203331', $commandBus->parentSpanId); - $this->assertSame($commandBus->spanId, self::singleSpanNamed('Command Handler: ' . $users::class . '::register', $recorder)->parentSpanId); + $this->assertSame($commandBus->spanId, self::singleSpanNamed('process class@anonymous::register', $recorder)->parentSpanId); } public function test_the_trace_carrier_of_the_incoming_command_does_not_leak_into_the_handler_span_parent(): void @@ -196,7 +196,7 @@ public function register(string $userId): void ) ->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01']); - $handler = self::singleSpanNamed('Command Handler: ' . $users::class . '::register', $recorder); + $handler = self::singleSpanNamed('process class@anonymous::register', $recorder); $this->assertNotSame('b7ad6b7169203331', $handler->parentSpanId); } @@ -232,12 +232,12 @@ public function register(string $userId): void $this->assertStringContainsString('Exception on send', $failure->getMessage()); } - self::assertEverySpanNamedHasStatus('Command Handler: ' . $users::class . '::register', SpanStatus::Error, $recorder); - self::assertEverySpanNamedHasStatus('Sending to Channel: error_channel', SpanStatus::Error, $recorder); + self::assertEverySpanNamedHasStatus('process class@anonymous::register', SpanStatus::Error, $recorder); + self::assertEverySpanNamedHasStatus('send error_channel', SpanStatus::Error, $recorder); $recorder->clear(); $ecotone->sendCommandWithRouting('user.register', '2'); - $this->assertSame('Command Bus', self::rootSpan($recorder)->name); + $this->assertSame('send command', self::rootSpan($recorder)->name); } public function test_distributed_bus_send_is_a_producer_span_carrying_the_trace_to_the_other_service(): void @@ -266,9 +266,9 @@ public function create(string $description, #[Header(MessageHeaders::TRACEPARENT ) ->getDistributedBus()->sendCommand('ticket_service', 'ticket.create', 'User changed billing address'); - $distributedBus = self::singleSpanNamed('Distributed Bus', $recorder); + $distributedBus = self::singleSpanNamed('send distributed', $recorder); $this->assertSame(SpanKind::Producer, $distributedBus->kind); - $this->assertSame($distributedBus->spanId, self::singleSpanNamed('Sending to Channel: distributed_channel', $recorder)->parentSpanId); + $this->assertSame($distributedBus->spanId, self::singleSpanNamed('send distributed_channel', $recorder)->parentSpanId); } public function test_a_failing_distributed_bus_send_fails_the_span_and_leaves_no_open_span_behind(): void @@ -299,10 +299,10 @@ public function register(string $userId): void $this->assertStringContainsString('Exception on send', $failure->getMessage()); } - $this->assertSame(SpanStatus::Error, self::singleSpanNamed('Distributed Bus', $recorder)->status); + $this->assertSame(SpanStatus::Error, self::singleSpanNamed('send distributed', $recorder)->status); $recorder->clear(); $ecotone->sendCommandWithRouting('user.register', '1'); - $this->assertSame('Command Bus', self::rootSpan($recorder)->name); + $this->assertSame('send command', self::rootSpan($recorder)->name); } private function bootstrapWithEventForwardedToSecondChannel(InMemorySpanRecorder $recorder, array $channelsToRegister, array $servicesToRegister = []): FlowTestSupport diff --git a/packages/Ecotone/tests/Tracing/TracingTestCase.php b/packages/Ecotone/tests/Tracing/TracingTestCase.php index bd8eb7386..696dd04e6 100644 --- a/packages/Ecotone/tests/Tracing/TracingTestCase.php +++ b/packages/Ecotone/tests/Tracing/TracingTestCase.php @@ -17,7 +17,7 @@ */ abstract class TracingTestCase extends TestCase { - private const INTERNAL_NOISE_SPAN_NAME_PREFIX = 'Message Handler: Ecotone\Modelling\MessageHandling\MetadataPropagator\MessageHeadersPropagatorInterceptor::'; + private const INTERNAL_NOISE_SPAN_NAME_PREFIX = 'process MessageHeadersPropagatorInterceptor::'; protected static function recordingSpansInto(InMemorySpanRecorder $recorder): array { diff --git a/packages/Ecotone/tests/Tracing/TracingTreeTest.php b/packages/Ecotone/tests/Tracing/TracingTreeTest.php index 6d579d0a8..4787f331a 100644 --- a/packages/Ecotone/tests/Tracing/TracingTreeTest.php +++ b/packages/Ecotone/tests/Tracing/TracingTreeTest.php @@ -47,10 +47,10 @@ public function register(string $userId): void self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Command Bus', 'kind' => SpanKind::Server, 'parent_id' => null], + 'details' => ['name' => 'send command', 'kind' => SpanKind::Server, 'parent_id' => null], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . $users::class . '::register', 'kind' => SpanKind::Server], + 'details' => ['name' => 'process class@anonymous::register', 'kind' => SpanKind::Server], 'children' => [], ], ], @@ -86,10 +86,10 @@ public function handle(Message $message): void self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ - ['details' => ['name' => 'Command Handler: ' . $handler::class . '::handleCommand'], 'children' => []], - ['details' => ['name' => 'Message Handler: ' . $handler::class . '::handle'], 'children' => []], + ['details' => ['name' => 'process class@anonymous::handleCommand'], 'children' => []], + ['details' => ['name' => 'process class@anonymous::handle'], 'children' => []], ], ], ], @@ -113,18 +113,18 @@ public function test_tracing_tree_with_two_levels_of_nesting(): void self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Event Bus'], + 'details' => ['name' => 'send event'], 'children' => [ [ - 'details' => ['name' => 'Routing to: merchant.to.user', 'kind' => SpanKind::Producer], + 'details' => ['name' => 'route merchant.to.user', 'kind' => SpanKind::Producer], 'children' => [ [ - 'details' => ['name' => 'Event Handler: ' . $subscriber::class . '::merchantToUser'], + 'details' => ['name' => 'process class@anonymous::merchantToUser'], 'children' => [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ - ['details' => ['name' => 'Command Handler: ' . $users::class . '::register'], 'children' => []], + ['details' => ['name' => 'process class@anonymous::register'], 'children' => []], ], ], ], @@ -159,23 +159,23 @@ public function merchantToUser(string $merchantId, CommandBus $commandBus): void ->publishEventWithRouting('merchant.created', '1'); $commandBusChild = [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ - ['details' => ['name' => 'Command Handler: ' . $users::class . '::register'], 'children' => []], + ['details' => ['name' => 'process class@anonymous::register'], 'children' => []], ], ]; self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Event Bus'], + 'details' => ['name' => 'send event'], 'children' => [ [ - 'details' => ['name' => 'Routing to: merchant.to.user'], - 'children' => [['details' => ['name' => 'Event Handler: ' . $subscriberOne::class . '::merchantToUser'], 'children' => [$commandBusChild]]], + 'details' => ['name' => 'route merchant.to.user'], + 'children' => [['details' => ['name' => 'process class@anonymous::merchantToUser'], 'children' => [$commandBusChild]]], ], [ - 'details' => ['name' => 'Routing to: merchant.to.second.user'], - 'children' => [['details' => ['name' => 'Event Handler: ' . $subscriberTwo::class . '::merchantToUser'], 'children' => [$commandBusChild]]], + 'details' => ['name' => 'route merchant.to.second.user'], + 'children' => [['details' => ['name' => 'process class@anonymous::merchantToUser'], 'children' => [$commandBusChild]]], ], ], ], @@ -208,24 +208,24 @@ public function create(string $merchantId, EventBus $eventBus): void self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . $merchants::class . '::create'], + 'details' => ['name' => 'process class@anonymous::create'], 'children' => [ [ - 'details' => ['name' => 'Event Bus'], + 'details' => ['name' => 'send event'], 'children' => [ [ - 'details' => ['name' => 'Routing to: merchant.to.user'], + 'details' => ['name' => 'route merchant.to.user'], 'children' => [ [ - 'details' => ['name' => 'Event Handler: ' . $subscriber::class . '::merchantToUser'], + 'details' => ['name' => 'process class@anonymous::merchantToUser'], 'children' => [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ - ['details' => ['name' => 'Command Handler: ' . $users::class . '::register'], 'children' => []], + ['details' => ['name' => 'process class@anonymous::register'], 'children' => []], ], ], ], @@ -277,15 +277,15 @@ public function test_separate_traces(): void self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Command Bus', 'attributes' => ['flowId' => '1']], + 'details' => ['name' => 'send command', 'attributes' => ['flowId' => '1']], 'children' => [ - ['details' => ['name' => 'Command Handler: ' . $users::class . '::register', 'attributes' => ['flowId' => '1']], 'children' => []], + ['details' => ['name' => 'process class@anonymous::register', 'attributes' => ['flowId' => '1']], 'children' => []], ], ], [ - 'details' => ['name' => 'Command Bus', 'attributes' => ['flowId' => '2']], + 'details' => ['name' => 'send command', 'attributes' => ['flowId' => '2']], 'children' => [ - ['details' => ['name' => 'Command Handler: ' . $users::class . '::register', 'attributes' => ['flowId' => '2']], 'children' => []], + ['details' => ['name' => 'process class@anonymous::register', 'attributes' => ['flowId' => '2']], 'children' => []], ], ], ], @@ -307,7 +307,7 @@ public function test_user_land_metadata_is_skipped_when_it_is_not_scalar(): void ) ->sendCommandWithRouting('user.register', '1', metadata: ['tokens' => ['123'], 'user' => new stdClass(), 'user_id' => '123']); - $node = self::getNodeAtTargetedSpan(['details' => ['name' => 'Command Bus']], self::buildTree($recorder->recordedSpans())); + $node = self::getNodeAtTargetedSpan(['details' => ['name' => 'send command']], self::buildTree($recorder->recordedSpans())); $this->assertArrayNotHasKey('user', $node['details']['attributes']); $this->assertArrayNotHasKey('tokens', $node['details']['attributes']); @@ -337,11 +337,11 @@ public function test_spans_carry_message_identity_and_a_child_message_points_at_ $spans = $recorder->recordedSpans(); $this->assertCount(5, $spans); - $this->assertChildSpan('Command Handler: ' . $users::class . '::register', $spans[0], $messageId, $correlationId); - $this->assertChildSpan('Command Bus', $spans[1], $messageId, $correlationId); - $this->assertSpanWith('Event Handler: ' . $subscriber::class . '::merchantToUser', $spans[2], $messageId, $correlationId, $timestamp); - $this->assertSpanWith('Routing to: merchant.to.user', $spans[3], $messageId, $correlationId, $timestamp); - $this->assertSpanWith('Event Bus', $spans[4], $messageId, $correlationId, $timestamp); + $this->assertChildSpan('process class@anonymous::register', $spans[0], $messageId, $correlationId); + $this->assertChildSpan('send command', $spans[1], $messageId, $correlationId); + $this->assertSpanWith('process class@anonymous::merchantToUser', $spans[2], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('route merchant.to.user', $spans[3], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('send event', $spans[4], $messageId, $correlationId, $timestamp); } public function test_a_failing_handler_marks_its_span_and_the_bus_span_as_failed_with_the_exception_recorded(): void @@ -393,9 +393,9 @@ public function test_query_bus_and_query_handler_are_traced(): void self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Query Bus'], + 'details' => ['name' => 'send query'], 'children' => [ - ['details' => ['name' => 'Query Handler: ' . $users::class . '::isRegistered'], 'children' => []], + ['details' => ['name' => 'process class@anonymous::isRegistered'], 'children' => []], ], ], ], diff --git a/packages/OpenTelemetry/tests/Integration/CorrelatedHeadersPropagationTest.php b/packages/OpenTelemetry/tests/Integration/CorrelatedHeadersPropagationTest.php index 54d3e8675..9edc7d882 100644 --- a/packages/OpenTelemetry/tests/Integration/CorrelatedHeadersPropagationTest.php +++ b/packages/OpenTelemetry/tests/Integration/CorrelatedHeadersPropagationTest.php @@ -56,8 +56,8 @@ public function test_tracing_with_single_levels_of_nesting() /** Command Bus and Command Handler */ $this->assertCount(2, $spans); - $this->assertSpanWith('Command Handler: ' . User::class . '::register', $spans[0], $messageId, $correlationId, $timestamp); - $this->assertSpanWith('Command Bus', $spans[1], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('process User::register', $spans[0], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('send command', $spans[1], $messageId, $correlationId, $timestamp); } public function test_tracing_with_two_levels_of_nesting() @@ -83,12 +83,12 @@ public function test_tracing_with_two_levels_of_nesting() $spans = $exporter->getSpans(); $this->assertCount(5, $spans); - $this->assertChildSpan('Command Handler: ' . User::class . '::register', $spans[0], $messageId, $correlationId); - $this->assertChildSpan('Command Bus', $spans[1], $messageId, $correlationId); + $this->assertChildSpan('process User::register', $spans[0], $messageId, $correlationId); + $this->assertChildSpan('send command', $spans[1], $messageId, $correlationId); - $this->assertSpanWith('Event Handler: ' . MerchantSubscriberOne::class . '::merchantToUser', $spans[2], $messageId, $correlationId, $timestamp); - $this->assertSpanWith('Routing to: merchant.subscriber.one', $spans[3], $messageId, $correlationId, $timestamp); - $this->assertSpanWith('Event Bus', $spans[4], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('process MerchantSubscriberOne::merchantToUser', $spans[2], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('route merchant.subscriber.one', $spans[3], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('send event', $spans[4], $messageId, $correlationId, $timestamp); } public function test_tracing_with_three_levels_of_nesting() @@ -114,15 +114,15 @@ public function test_tracing_with_three_levels_of_nesting() $spans = $exporter->getSpans(); $this->assertCount(7, $spans); - $this->assertChildSpan('Command Handler: ' . User::class . '::register', $spans[0], $spans[2]->getAttributes()->get(MessageHeaders::MESSAGE_ID), $correlationId); - $this->assertChildSpan('Command Bus', $spans[1], $spans[2]->getAttributes()->get(MessageHeaders::MESSAGE_ID), $correlationId); + $this->assertChildSpan('process User::register', $spans[0], $spans[2]->getAttributes()->get(MessageHeaders::MESSAGE_ID), $correlationId); + $this->assertChildSpan('send command', $spans[1], $spans[2]->getAttributes()->get(MessageHeaders::MESSAGE_ID), $correlationId); - $this->assertChildSpan('Event Handler: ' . MerchantSubscriberOne::class . '::merchantToUser', $spans[2], $messageId, $correlationId); - $this->assertChildSpan('Routing to: merchant.subscriber.one', $spans[3], $messageId, $correlationId); - $this->assertChildSpan('Event Bus', $spans[4], $messageId, $correlationId); + $this->assertChildSpan('process MerchantSubscriberOne::merchantToUser', $spans[2], $messageId, $correlationId); + $this->assertChildSpan('route merchant.subscriber.one', $spans[3], $messageId, $correlationId); + $this->assertChildSpan('send event', $spans[4], $messageId, $correlationId); - $this->assertSpanWith('Command Handler: ' . Merchant::class . '::create', $spans[5], $messageId, $correlationId, $timestamp); - $this->assertSpanWith('Command Bus', $spans[6], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('process Merchant::create', $spans[5], $messageId, $correlationId, $timestamp); + $this->assertSpanWith('send command', $spans[6], $messageId, $correlationId, $timestamp); } public function test_tracing_same_message_for_asynchronous_scenario() @@ -148,11 +148,11 @@ public function test_tracing_same_message_for_asynchronous_scenario() $node = $this->getNodeAtTargetedSpan( [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'child' => [ - 'details' => ['name' => 'Sending to Channel: async_channel'], + 'details' => ['name' => 'send async_channel'], 'child' => [ - 'details' => ['name' => 'Receiving from channel: async_channel'], + 'details' => ['name' => 'receive async_channel'], ], ], ], diff --git a/packages/OpenTelemetry/tests/Integration/TracingTestCase.php b/packages/OpenTelemetry/tests/Integration/TracingTestCase.php index da69f9f41..2c38cf41f 100644 --- a/packages/OpenTelemetry/tests/Integration/TracingTestCase.php +++ b/packages/OpenTelemetry/tests/Integration/TracingTestCase.php @@ -28,7 +28,7 @@ public static function prepareTracer(SpanExporterInterface $exporter): TracerPro ); } - private const INTERNAL_NOISE_SPAN_NAME_PREFIX = 'Message Handler: Ecotone\Modelling\MessageHandling\MetadataPropagator\MessageHeadersPropagatorInterceptor::'; + private const INTERNAL_NOISE_SPAN_NAME_PREFIX = 'process MessageHeadersPropagatorInterceptor::'; public static function buildTree(InMemoryExporter $exporter): array { diff --git a/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php b/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php index 342f13a8d..6cac4d107 100644 --- a/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php +++ b/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php @@ -57,10 +57,10 @@ public function test_tracing_tree_with_single_levels_of_nesting() self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], + 'details' => ['name' => 'process User::register'], 'children' => [], ], ], @@ -84,14 +84,14 @@ public function test_tracing_tree_with_two_levels_of_nesting_and_message_handler self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . ExampleMessageHandler::class . '::handleCommand'], + 'details' => ['name' => 'process ExampleMessageHandler::handleCommand'], 'children' => [], ], [ - 'details' => ['name' => 'Message Handler: ' . ExampleMessageHandler::class . '::handle'], + 'details' => ['name' => 'process ExampleMessageHandler::handle'], 'children' => [], ], ], @@ -115,11 +115,11 @@ public function test_tracing_scheduled_handler() self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Receiving from inbound channel adapter: nullChannel'], + 'details' => ['name' => 'receive nullChannel'], 'children' => [], ], [ - 'details' => ['name' => 'Receiving from inbound channel adapter: nullChannel'], + 'details' => ['name' => 'receive nullChannel'], 'children' => [], ], ], @@ -141,19 +141,19 @@ public function test_tracing_workflow_scheduled_handler() self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Receiving from inbound channel adapter: add'], + 'details' => ['name' => 'receive add'], 'children' => [ [ - 'details' => ['name' => 'Message Handler: ' . WorkflowScheduledHandler::class . '::add'], + 'details' => ['name' => 'process WorkflowScheduledHandler::add'], 'children' => [], ], ], ], [ - 'details' => ['name' => 'Receiving from inbound channel adapter: add'], + 'details' => ['name' => 'receive add'], 'children' => [ [ - 'details' => ['name' => 'Message Handler: ' . WorkflowScheduledHandler::class . '::add'], + 'details' => ['name' => 'process WorkflowScheduledHandler::add'], 'children' => [], ], ], @@ -223,10 +223,10 @@ public function test_force_flushing_traces() self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], + 'details' => ['name' => 'process User::register'], 'children' => [], ], ], @@ -267,9 +267,9 @@ public function test_traces_are_force_flushed_when_synchronous_exception_happens $node = $this->getNodeAtTargetedSpan( [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'child' => [ - 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], + 'details' => ['name' => 'process User::register'], ], ], self::buildTree($exporter) @@ -296,19 +296,19 @@ public function test_separate_traces() self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Command Bus', 'attributes' => ['flowId' => '1']], + 'details' => ['name' => 'send command', 'attributes' => ['flowId' => '1']], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . User::class . '::register', 'attributes' => ['flowId' => '1']], + 'details' => ['name' => 'process User::register', 'attributes' => ['flowId' => '1']], 'children' => [], ], ], ], [ - 'details' => ['name' => 'Command Bus', 'attributes' => ['flowId' => '2']], + 'details' => ['name' => 'send command', 'attributes' => ['flowId' => '2']], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . User::class . '::register', 'attributes' => ['flowId' => '2']], + 'details' => ['name' => 'process User::register', 'attributes' => ['flowId' => '2']], 'children' => [], ], ], @@ -332,19 +332,19 @@ public function test_tracing_with_two_levels_of_nesting() self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Event Bus'], + 'details' => ['name' => 'send event'], 'children' => [ [ - 'details' => ['name' => 'Routing to: merchant.subscriber.one'], + 'details' => ['name' => 'route merchant.subscriber.one'], 'children' => [ [ - 'details' => ['name' => 'Event Handler: ' . MerchantSubscriberOne::class . '::merchantToUser'], + 'details' => ['name' => 'process MerchantSubscriberOne::merchantToUser'], 'children' => [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], + 'details' => ['name' => 'process User::register'], 'children' => [], ], ], @@ -374,19 +374,19 @@ public function test_tracing_with_two_levels_of_nesting_and_two_branches_on_same self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Event Bus'], + 'details' => ['name' => 'send event'], 'children' => [ [ - 'details' => ['name' => 'Routing to: merchant.subscriber.one'], + 'details' => ['name' => 'route merchant.subscriber.one'], 'children' => [ [ - 'details' => ['name' => 'Event Handler: ' . MerchantSubscriberOne::class . '::merchantToUser'], + 'details' => ['name' => 'process MerchantSubscriberOne::merchantToUser'], 'children' => [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], + 'details' => ['name' => 'process User::register'], 'children' => [], ], ], @@ -396,16 +396,16 @@ public function test_tracing_with_two_levels_of_nesting_and_two_branches_on_same ], ], [ - 'details' => ['name' => 'Routing to: merchant.subscriber.two'], + 'details' => ['name' => 'route merchant.subscriber.two'], 'children' => [ [ - 'details' => ['name' => 'Event Handler: ' . MerchantSubscriberTwo::class . '::merchantToUser'], + 'details' => ['name' => 'process MerchantSubscriberTwo::merchantToUser'], 'children' => [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], + 'details' => ['name' => 'process User::register'], 'children' => [], ], ], @@ -435,25 +435,25 @@ public function test_tracing_with_three_levels_of_nesting() self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . Merchant::class . '::create'], + 'details' => ['name' => 'process Merchant::create'], 'children' => [ [ - 'details' => ['name' => 'Event Bus'], + 'details' => ['name' => 'send event'], 'children' => [ [ - 'details' => ['name' => 'Routing to: merchant.subscriber.one'], + 'details' => ['name' => 'route merchant.subscriber.one'], 'children' => [ [ - 'details' => ['name' => 'Event Handler: ' . MerchantSubscriberOne::class . '::merchantToUser'], + 'details' => ['name' => 'process MerchantSubscriberOne::merchantToUser'], 'children' => [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . User::class . '::register'], + 'details' => ['name' => 'process User::register'], 'children' => [], ], ], @@ -490,16 +490,16 @@ public function test_tracing_with_asynchronous_handler() self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'children' => [ [ - 'details' => ['name' => 'Sending to Channel: async_channel'], + 'details' => ['name' => 'send async_channel'], 'children' => [ [ - 'details' => ['name' => 'Receiving from channel: async_channel'], + 'details' => ['name' => 'receive async_channel'], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . \Test\Ecotone\OpenTelemetry\Fixture\AsynchronousFlow\User::class . '::register'], + 'details' => ['name' => 'process User::register'], 'children' => [], ], ], @@ -534,13 +534,13 @@ public function test_adding_logs_as_events_in_span() $result = self::getNodeAtTargetedSpan( [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], 'child' => [ - 'details' => ['name' => 'Sending to Channel: async_channel'], + 'details' => ['name' => 'send async_channel'], 'child' => [ - 'details' => ['name' => 'Receiving from channel: async_channel'], + 'details' => ['name' => 'receive async_channel'], 'child' => [ - 'details' => ['name' => 'Event Bus'], + 'details' => ['name' => 'send event'], ], ], ], @@ -552,7 +552,7 @@ public function test_adding_logs_as_events_in_span() $this->assertNotEmpty(array_filter($eventBusEvents, fn (string $name): bool => str_starts_with($name, 'Publishing Event Message'))); $route = $result['children'][array_key_first($result['children'])]; - $this->assertSame('Routing to: user.registered', $route['details']['name']); + $this->assertSame('route user.registered', $route['details']['name']); $routeEvents = array_map(fn (Event $event): string => $event->getName(), $route['details']['events']); $this->assertNotEmpty(array_filter($routeEvents, fn (string $name): bool => str_starts_with($name, 'Collecting message with id'))); } @@ -576,16 +576,16 @@ public function test_two_traces_with_asynchronous_handlers() self::compareTreesByDetails( [ [ - 'details' => ['name' => 'Command Bus', 'attributes' => ['flowId' => '1']], + 'details' => ['name' => 'send command', 'attributes' => ['flowId' => '1']], 'children' => [ [ - 'details' => ['name' => 'Sending to Channel: async_channel', 'attributes' => ['flowId' => '1']], + 'details' => ['name' => 'send async_channel', 'attributes' => ['flowId' => '1']], 'children' => [ [ - 'details' => ['name' => 'Receiving from channel: async_channel', 'attributes' => ['flowId' => '1']], + 'details' => ['name' => 'receive async_channel', 'attributes' => ['flowId' => '1']], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . \Test\Ecotone\OpenTelemetry\Fixture\AsynchronousFlow\User::class . '::register', 'attributes' => ['flowId' => '1']], + 'details' => ['name' => 'process User::register', 'attributes' => ['flowId' => '1']], 'children' => [], ], ], @@ -595,16 +595,16 @@ public function test_two_traces_with_asynchronous_handlers() ], ], [ - 'details' => ['name' => 'Command Bus', 'attributes' => ['flowId' => '2']], + 'details' => ['name' => 'send command', 'attributes' => ['flowId' => '2']], 'children' => [ [ - 'details' => ['name' => 'Sending to Channel: async_channel', 'attributes' => ['flowId' => '2']], + 'details' => ['name' => 'send async_channel', 'attributes' => ['flowId' => '2']], 'children' => [ [ - 'details' => ['name' => 'Receiving from channel: async_channel', 'attributes' => ['flowId' => '2']], + 'details' => ['name' => 'receive async_channel', 'attributes' => ['flowId' => '2']], 'children' => [ [ - 'details' => ['name' => 'Command Handler: ' . \Test\Ecotone\OpenTelemetry\Fixture\AsynchronousFlow\User::class . '::register', 'attributes' => ['flowId' => '2']], + 'details' => ['name' => 'process User::register', 'attributes' => ['flowId' => '2']], 'children' => [], ], ], @@ -641,25 +641,25 @@ public function test_passing_user_specific_headers() [ [ 'details' => [ - 'name' => 'Command Bus', + 'name' => 'send command', 'attributes' => ['user_id' => '123'], ], 'children' => [ [ 'details' => [ - 'name' => 'Sending to Channel: async_channel', + 'name' => 'send async_channel', 'attributes' => ['user_id' => '123'], ], 'children' => [ [ 'details' => [ - 'name' => 'Receiving from channel: async_channel', + 'name' => 'receive async_channel', 'attributes' => ['user_id' => '123'], ], 'children' => [ [ 'details' => [ - 'name' => 'Command Handler: ' . \Test\Ecotone\OpenTelemetry\Fixture\AsynchronousFlow\User::class . '::register', + 'name' => 'process User::register', 'attributes' => ['user_id' => '123'], ], 'children' => [], @@ -693,7 +693,7 @@ public function test_user_land_metadata_should_be_skipped_in_case_is_not_scalar( $node = $this->getNodeAtTargetedSpan( [ - 'details' => ['name' => 'Command Bus'], + 'details' => ['name' => 'send command'], ], self::buildTree($exporter) ); diff --git a/upgrade-2.0.md b/upgrade-2.0.md index df02aba7d..f3f477f0f 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -2965,15 +2965,19 @@ synchronous span came from OpenTelemetry's ambient context, and without that pac the W3C `traceparent` and `tracestate` headers (`MessageHeaders::TRACEPARENT`, `MessageHeaders::TRACESTATE`), which are framework headers: they are not copied into the next message as user metadata, are not reported as span attributes, and a root span continues the trace a message already carries, so a command sent with a `traceparent` -header from an HTTP request becomes a child of that request's span. Span names, kinds, the message attributes and the -`exception` event are unchanged, and three things are added. Every route the event bus resolves is a PRODUCER span, -`Routing to: ` or `Routing to Projection: `, between the `Event Bus` span and the handler it -leads to, synchronous or asynchronous, so a trace shows which handlers an event was dispatched to and a failing -synchronous handler fails its route. An event-driven projection run is a CONSUMER span, `Projection: `. And -every span carries the OpenTelemetry messaging attributes: `messaging.system` is `ecotone`, -`messaging.operation.type` is `send` for buses, routes and channel sends and `process` for handlers, consumers and -projections, `messaging.destination.name` names the route or channel, and `ecotone.endpoint.id` or -`ecotone.projection.name` name the handler; the names are constants on `Ecotone\Api\Tracing\SpanAttribute`. The tracing interceptors are registered only when a recorder is configured or a +header from an HTTP request becomes a child of that request's span. Span kinds, the message attributes and the +`exception` event are unchanged. Span names follow the OpenTelemetry messaging convention, `{operation} {target}`, +short and in lower case, with the detail in attributes: a bus call is `send command`, `send event`, `send query` or +`send distributed`; a channel send is `send ` and its consumption `receive `; a handler is +`process ` with the short class name, the fully qualified one in `code.function.name`; a projection +run is `process `. Three spans are added. Every route the event bus resolves is a PRODUCER span, +`route ` or `route `, between the `send event` span and the handler it leads to, synchronous +or asynchronous, so a trace shows which handlers an event was dispatched to and a failing synchronous handler fails +its route. An event-driven projection run is a CONSUMER span. And every span carries the OpenTelemetry messaging +attributes: `messaging.system` is `ecotone`, `messaging.operation.type` is `send` for buses, routes and channel sends +and `process` for handlers, consumers and projections, `messaging.destination.name` names the route or channel, and +`ecotone.endpoint.id` or `ecotone.projection.name` name the handler; the names are constants on +`Ecotone\Api\Tracing\SpanAttribute`. The tracing interceptors are registered only when a recorder is configured or a tracing package hands over its `Tracer`, so an application without tracing pays nothing. `ecotone/open-telemetry` becomes an implementation of that `Tracer`: it starts live SDK spans with the same ids Core reports, keeps the forced flush and the Dynamic Consistency Boundary spans, and records nothing when a flow test is given no @@ -2981,8 +2985,11 @@ forced flush and the Dynamic Consistency Boundary spans, and records nothing whe to `Ecotone\Api\OpenTelemetry\OpenTelemetryConfiguration`; the never-settable tracing level is gone. `MessageHeaders::TEMPORARY_SPAN_CONTEXT_HEADER`, which nothing set, is removed. -**How to adapt:** a trace tree asserted in a test gains the `Routing to:` level under `Event Bus`; give the handler -an explicit `endpointId` to make that span's name stable. Rename `Ecotone\OpenTelemetry\Configuration\TracingConfiguration` to +**How to adapt:** rename dashboards and alerts keyed on the 1.x span names: `Command Bus` is `send command`, +`Event Bus` is `send event`, `Query Bus` is `send query`, `Sending to Channel: orders` is `send orders`, +`Receiving from channel: orders` is `receive orders`, `Command Handler: App\OrderService::place` is +`process OrderService::place`, and `Projection: orders` is `process orders`. A trace tree asserted in a test gains +the `route` level under `send event`; give the handler an explicit `endpointId` to make that span's name stable. Rename `Ecotone\OpenTelemetry\Configuration\TracingConfiguration` to `Ecotone\Api\OpenTelemetry\OpenTelemetryConfiguration`; `withForceFlushOnBusExecution()` and `withForceFlushOnAsynchronousMessageHandled()` keep their names. A consumer outside Ecotone that read the `ecotoneTracingCarrier` header reads `traceparent` instead, which any OpenTelemetry propagator already does. To From f7a821d03b0f795d170ebf314702734061734c7b Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 12/50] refactor(tracing): follow main's renames, the completion gateway headers use the ecotone_{module}_{feature}_{action} convention ServiceActivatorBuilder became MessageProcessorActivatorBuilder::createForMethod() and DbalConnectionFactory moved to Api on main. The gateway headers of TraceCompletion are ecotone_tracing_completion_destination and ecotone_tracing_completion_timeout; traceparent and tracestate keep their W3C names because they leave the process. --- .../Dbal/tests/Conformance/TraceStoreConformanceTest.php | 2 +- packages/Ecotone/src/Tracing/Config/TracingModule.php | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php index 346099df2..c972e8637 100644 --- a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php +++ b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php @@ -7,6 +7,7 @@ use Ecotone\Api\Attribute\Asynchronous; use Ecotone\Api\Attribute\CommandHandler; use Ecotone\Api\Attribute\EventHandler; +use Ecotone\Api\Dbal\DbalConnectionFactory; use Ecotone\Api\Dbal\ExtensionObject\DbalBackedMessageChannelBuilder; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -25,7 +26,6 @@ use Ecotone\Api\Tracing\TraceCompletion; use Ecotone\Api\Tracing\TraceCompletionTimedOut; use Ecotone\Api\Tracing\TracingConfiguration; -use Ecotone\Dbal\Connection\DbalConnectionFactory; use Ecotone\Test\LicenceTesting; use PHPUnit\Framework\Attributes\DataProvider; use RuntimeException; diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index 09a8256ad..137b12a81 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -44,7 +44,7 @@ use Ecotone\Messaging\Handler\InterfaceToCallRegistry; use Ecotone\Messaging\Handler\Processor\MethodInvoker\AroundInterceptorBuilder; use Ecotone\Messaging\Handler\Processor\MethodInvoker\MethodInterceptorBuilder; -use Ecotone\Messaging\Handler\ServiceActivator\ServiceActivatorBuilder; +use Ecotone\Messaging\Handler\ServiceActivator\MessageProcessorActivatorBuilder; use Ecotone\Messaging\Support\LicensingException; use Ecotone\Messaging\Support\MessageBuilder; use Ecotone\Modelling\MessageHandling\MetadataPropagator\MessageHeadersPropagatorInterceptor; @@ -236,7 +236,7 @@ private function registerStoredTraces(Configuration $messagingConfiguration, arr ] as $method => $parameterConverters) { $channel = 'ecotone.tracing.completion.' . $method; $messagingConfiguration - ->registerMessageHandler(ServiceActivatorBuilder::create(StoredTraceCompletion::class, $method)->withInputChannelName($channel)) + ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod(StoredTraceCompletion::class, $method)->withInputChannelName($channel)) ->registerGatewayBuilder(GatewayProxyBuilder::create(TraceCompletion::class, TraceCompletion::class, $method, $channel)->withParameterConverters($parameterConverters)); } @@ -245,7 +245,7 @@ private function registerStoredTraces(Configuration $messagingConfiguration, arr new Definition(ExpiredTraceRemover::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $tracingConfiguration->storedTracesRetention->inMilliseconds()]) ); $messagingConfiguration - ->registerMessageHandler(ServiceActivatorBuilder::create(ExpiredTraceRemover::class, 'removeExpiredTraces')->withInputChannelName(self::REMOVE_EXPIRED_TRACES_CHANNEL)) + ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod(ExpiredTraceRemover::class, 'removeExpiredTraces')->withInputChannelName(self::REMOVE_EXPIRED_TRACES_CHANNEL)) ->registerConsoleCommand(ConsoleCommandConfiguration::create(self::REMOVE_EXPIRED_TRACES_CHANNEL, 'ecotone:tracing:remove-expired-traces', [], 'Removes stored traces older than the configured retention')); } From ad6a66ae2a4f56fb0d4fde99aef1b838de728be5 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 13/50] test(tracing): DBAL stored traces and the OpenTelemetry exporter run side by side across an asynchronous DBAL channel A cross-module flow test enables the DBAL and tracing packages together with a real database and an in-memory OpenTelemetry exporter: one traceparent is exported by the SDK, stored in ecotone_traces and ecotone_spans, kept pending until the DBAL channel's consumer ran, and notified once; a second application on the same database sees the completion. --- .../StoredTracesWithOpenTelemetryTest.php | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 Monorepo/CrossModuleTests/Tests/Tracing/StoredTracesWithOpenTelemetryTest.php diff --git a/Monorepo/CrossModuleTests/Tests/Tracing/StoredTracesWithOpenTelemetryTest.php b/Monorepo/CrossModuleTests/Tests/Tracing/StoredTracesWithOpenTelemetryTest.php new file mode 100644 index 000000000..4aadfd1b1 --- /dev/null +++ b/Monorepo/CrossModuleTests/Tests/Tracing/StoredTracesWithOpenTelemetryTest.php @@ -0,0 +1,177 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + * + * @internal + */ +final class StoredTracesWithOpenTelemetryTest extends TestCase +{ + protected function setUp(): void + { + MessagingTestCase::cleanUpDbal(); + } + + public function test_one_trace_is_exported_to_open_telemetry_and_stored_in_the_database_across_an_asynchronous_dbal_channel(): void + { + $exporter = new InMemoryExporter(new ArrayObject()); + $subscriber = $this->traceCompletedSubscriber(); + $ecotone = $this->bootstrap($exporter, [$this->users(), $this->asynchronousNotifier(), $subscriber]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + self::assertSame( + ['process class@anonymous::register', 'route user.notifier', 'send command', 'send event', 'send notifications'], + self::exportedSpanNamesOf($exporter, $trace->traceId), + ); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.registration')); + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertSame([], $subscriber->completedTraces); + + $ecotone->run('notifications', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertSame( + ['process class@anonymous::notify', 'process class@anonymous::register', 'receive notifications', 'route user.notifier', 'send command', 'send event', 'send notifications'], + self::exportedSpanNamesOf($exporter, $trace->traceId), + ); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); + self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertSame([$trace->traceId], $subscriber->completedTraces); + } + + public function test_a_second_application_on_the_same_database_sees_the_completion_the_first_one_stored(): void + { + $producer = $this->bootstrap(new InMemoryExporter(new ArrayObject()), [$this->users()]); + $observer = $this->bootstrap(new InMemoryExporter(new ArrayObject()), [$this->users()]); + $trace = SpanContext::newTrace(); + + $producer->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + self::assertTrue($this->completion($observer)->isCompleted($trace->traceId)); + } + + /** + * @return string[] + */ + private static function exportedSpanNamesOf(InMemoryExporter $exporter, string $traceId): array + { + $names = []; + /** @var SpanDataInterface $span */ + foreach ($exporter->getSpans() as $span) { + if ($span->getContext()->getTraceId() !== $traceId || str_starts_with($span->getName(), 'process MessageHeadersPropagatorInterceptor::')) { + continue; + } + $names[] = preg_replace('/^process class@anonymous\S*::/', 'process class@anonymous::', $span->getName()); + } + sort($names); + + return $names; + } + + /** + * @param object[] $services + */ + private function bootstrap(InMemoryExporter $exporter, array $services): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + array_map(fn (object $service): string => $service::class, $services), + [ + ...$services, + DbalConnectionFactory::class => new DbalConnectionFactory(getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@127.0.0.1:5432/ecotone'), + TracerProviderInterface::class => TracingTestCase::prepareTracer($exporter), + ], + [ + TracingConfiguration::createWithDefaults()->withStoredTraces(), + DbalBackedMessageChannelBuilder::create('notifications'), + ], + enabledModules: [ModulePackageList::DBAL_PACKAGE, ModulePackageList::TRACING_PACKAGE], + licenceKey: LicenceTesting::VALID_LICENCE, + )->setupDatabaseTables(); + } + + private function completion(FlowTestSupport $ecotone): TraceCompletion + { + return $ecotone->getGateway(TraceCompletion::class); + } + + private function users(): object + { + return new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId, EventBus $eventBus): void + { + $eventBus->publishWithRouting('user.registered', $userId); + } + }; + } + + private function asynchronousNotifier(): object + { + return new class () { + #[Asynchronous('notifications')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId): void + { + } + }; + } + + private function traceCompletedSubscriber(): object + { + return new class () { + /** @var string[] */ + public array $completedTraces = []; + + #[OnTraceCompleted] + public function whenCompleted(TraceCompleted $event): void + { + $this->completedTraces[] = $event->traceId; + } + }; + } +} From cfcbc81d9cc6820fc80d7ccc5a68557525fa5230 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 14/50] fix(tracing): a consumer whose handler failure was handled by the error channel or the acknowledgement strategy is stored as failed, not as processed The consumer span wraps the error-channel and acknowledgement interceptors, which swallow the handler's exception when the consumer does not stop on error, so the span ended Ok and paired the delivery as processed while the message was requeued or dead-lettered. The recorder now remembers which open span wrapped a discarded unit of work and stores that span as failed, unless a later retry in the same span flushed successfully. --- .../Conformance/TraceStoreConformanceTest.php | 13 ++++ .../src/Tracing/StoringSpanRecorder.php | 53 ++++++++++++++ .../tests/Tracing/TraceCompletionTest.php | 71 +++++++++++++++++++ 3 files changed, 137 insertions(+) diff --git a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php index c972e8637..8c9e9b349 100644 --- a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php +++ b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php @@ -11,6 +11,7 @@ use Ecotone\Api\Dbal\ExtensionObject\DbalBackedMessageChannelBuilder; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\ModulePackageList; +use Ecotone\Api\ExtensionObject\ServiceConfiguration; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; use Ecotone\Api\Gateway\EventBus; use Ecotone\Api\Lite\Test\FlowTestSupport; @@ -277,6 +278,18 @@ public function test_a_failing_asynchronous_handler_stores_its_failed_consumer_s self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); } + #[DataProvider('implementations')] + public function test_a_consumer_that_swallowed_its_handler_failure_leaves_the_trace_pending(string $implementation): void + { + $ecotone = $this->bootstrap($implementation, [$this->users(), $this->failingNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel'), ServiceConfiguration::createWithDefaults()->withDefaultErrorChannel('errorChannel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); + + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + } + public function test_completion_recorded_by_one_instance_is_visible_to_another_sharing_the_database(): void { $producer = $this->bootstrap('dbal', [$this->users()]); diff --git a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php index af99e251d..310bd3b4b 100644 --- a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php +++ b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php @@ -41,6 +41,8 @@ final class StoringSpanRecorder implements SpanRecorder /** @var string[] */ private array $completedTracesToPublish = []; private bool $publishing = false; + /** @var array */ + private array $spansWrappingAFailedUnitOfWork = []; public function __construct( private TraceStore $traceStore, @@ -58,6 +60,10 @@ public function record(Span $span): void return; } + if (isset($this->spansWrappingAFailedUnitOfWork[$span->spanId])) { + unset($this->spansWrappingAFailedUnitOfWork[$span->spanId]); + $span = self::failedByItsUnitOfWork($span); + } if ($span->status === SpanStatus::Error && ! $this->storesFailuresOutsideTransaction) { return; } @@ -79,6 +85,9 @@ public function isScopeActive(): bool public function flush(): void { + foreach (self::outerParentsOf($this->buffer) as $spanId) { + unset($this->spansWrappingAFailedUnitOfWork[$spanId]); + } $this->write($this->buffer); $this->buffer = []; } @@ -90,9 +99,53 @@ public function closeScope(): void public function discard(): void { + foreach (self::outerParentsOf($this->buffer) as $spanId) { + $this->spansWrappingAFailedUnitOfWork[$spanId] = true; + } $this->buffer = []; } + /** + * @param Span[] $spans + * @return string[] + */ + private static function outerParentsOf(array $spans): array + { + $spanIds = []; + foreach ($spans as $span) { + $spanIds[$span->spanId] = true; + } + $parents = []; + foreach ($spans as $span) { + if ($span->parentSpanId !== null && ! isset($spanIds[$span->parentSpanId])) { + $parents[$span->parentSpanId] = true; + } + } + + return array_keys($parents); + } + + private static function failedByItsUnitOfWork(Span $span): Span + { + if ($span->status === SpanStatus::Error) { + return $span; + } + + return new Span( + $span->traceId, + $span->spanId, + $span->parentSpanId, + $span->name, + $span->kind, + $span->attributes, + SpanStatus::Error, + 'The unit of work this span wraps failed and its failure was handled by the error channel or the acknowledgement strategy', + $span->events, + $span->startedAt, + $span->endedAt, + ); + } + /** * @param Span[] $spans */ diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index 025848cfc..212214492 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -9,6 +9,7 @@ use Ecotone\Api\Attribute\EventHandler; use Ecotone\Api\EventSourcing\Event; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; +use Ecotone\Api\ExtensionObject\ServiceConfiguration; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; use Ecotone\Api\Gateway\CommandBus; use Ecotone\Api\Gateway\EventBus; @@ -366,6 +367,76 @@ public function refresh(string $traceId): void $this->assertSame([$trace->traceId], array_values(array_unique($subscriber->notifiedTraces))); } + public function test_a_consumer_that_swallowed_its_handler_failure_leaves_the_trace_pending(): void + { + $orders = new class () { + #[Asynchronous('async_channel')] + #[CommandHandler('order.place', endpointId: 'order.placing')] + public function place(string $orderId): void + { + throw new RuntimeException('Placing failed'); + } + }; + $ecotone = $this->bootstrap([$orders], [SimpleMessageChannelBuilder::createQueueChannel('async_channel'), ServiceConfiguration::createWithDefaults()->withDefaultErrorChannel('errorChannel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); + + $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); + } + + public function test_a_consumer_that_swallowed_its_handler_failure_stores_a_failed_consumer_span_when_failures_may_be_stored(): void + { + $orders = new class () { + #[Asynchronous('async_channel')] + #[CommandHandler('order.place', endpointId: 'order.placing')] + public function place(string $orderId): void + { + throw new RuntimeException('Placing failed'); + } + }; + $subscriber = $this->spanEndedSubscriber(); + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$orders::class, $subscriber::class], + [$orders, $subscriber], + [TracingConfiguration::createWithDefaults()->withStoredTraces(storeFailuresOutsideTransaction: true), SimpleMessageChannelBuilder::createQueueChannel('async_channel'), ServiceConfiguration::createWithDefaults()->withDefaultErrorChannel('errorChannel')], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); + + $this->assertSame(SpanStatus::Error, $subscriber->endedSpans['receive async_channel']); + $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + public function test_a_consumer_whose_handler_failed_once_and_succeeded_on_the_instant_retry_completes_the_trace(): void + { + $orders = new class () { + public int $attempts = 0; + + #[Asynchronous('async_channel')] + #[CommandHandler('order.place', endpointId: 'order.placing')] + public function place(string $orderId): void + { + if (++$this->attempts === 1) { + throw new RuntimeException('Placing failed once'); + } + } + }; + $ecotone = $this->bootstrap([$orders], [SimpleMessageChannelBuilder::createQueueChannel('async_channel'), ServiceConfiguration::createWithDefaults()->withDefaultErrorChannel('errorChannel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); + + $this->assertSame(2, $orders->attempts); + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + public function test_a_failed_unit_of_work_stores_nothing_by_default(): void { $subscriber = $this->spanEndedSubscriber(); From 656550a332044b00d58553b1bc65ed3d9b42beb9 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 15/50] refactor(tracing): the trace store appends inside the transaction and claims completion after the commit, without a row lock SELECT ... FOR UPDATE on the trace row was a pessimistic lock, which rule 8 forbids, and it still could not protect a decision made from a transaction's own snapshot. The flush inside the transaction now only appends. The write of the span wrapping the unit of work, which follows the commit, reads what every unit of work has committed by then and claims completion with one conditional update, so exactly one of two concurrent finishers claims it and neither can miss the other's commit. --- packages/Dbal/src/Tracing/DbalTraceStore.php | 30 +++++-------------- .../src/Tracing/InMemoryTraceStore.php | 19 +++++------- .../src/Tracing/StoringSpanRecorder.php | 25 +++++++++++++--- packages/Ecotone/src/Tracing/TraceStore.php | 5 ++-- upgrade-2.0.md | 5 +++- 5 files changed, 44 insertions(+), 40 deletions(-) diff --git a/packages/Dbal/src/Tracing/DbalTraceStore.php b/packages/Dbal/src/Tracing/DbalTraceStore.php index 41e846310..e6d4be94a 100644 --- a/packages/Dbal/src/Tracing/DbalTraceStore.php +++ b/packages/Dbal/src/Tracing/DbalTraceStore.php @@ -7,7 +7,6 @@ use Doctrine\DBAL\ArrayParameterType; use Doctrine\DBAL\Connection; use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; -use Doctrine\DBAL\Platforms\SQLitePlatform; use Doctrine\DBAL\Types\Types; use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Scheduling\DatePoint; @@ -60,32 +59,29 @@ public function __construct( ) { } - public function record(array $spans): array + public function append(array $spans): void { $spansByTrace = []; foreach ($spans as $span) { $spansByTrace[$span->traceId][] = $span; } - $completedNow = []; $connection = $this->connection(); foreach ($spansByTrace as $traceId => $traceSpans) { - $this->onTables(function () use ($connection, $traceId, $traceSpans, &$completedNow): void { + $this->onTables(function () use ($connection, $traceId, $traceSpans): void { $this->claimTrace($connection, $traceId, $traceSpans); - $alreadyCompleted = $this->lockTrace($connection, $traceId); foreach ($traceSpans as $span) { $this->insertSpan($connection, $span); } - if ($alreadyCompleted || ! TraceLedger::progressOf($traceId, $this->spansOf($connection, $traceId))->isCompleted()) { - return; - } - if ($this->markCompleted($connection, $traceId)) { - $completedNow[] = $traceId; - } }); } + } + + public function markCompletedIfDone(string $traceId): bool + { + $connection = $this->connection(); - return $completedNow; + return $this->onTables(fn (): bool => TraceLedger::progressOf($traceId, $this->spansOf($connection, $traceId))->isCompleted() && $this->markCompleted($connection, $traceId)); } public function progressOf(string $traceId): TraceProgress @@ -132,16 +128,6 @@ private function claimTrace(Connection $connection, string $traceId, array $trac $connection->executeStatement($sql, ['traceId' => $traceId, 'startedAt' => $startedAt], ['startedAt' => Types::BIGINT]); } - private function lockTrace(Connection $connection, string $traceId): bool - { - $sql = "SELECT completed_at FROM {$this->tracesTableManager->getTableName()} WHERE trace_id = :traceId"; - if (! $connection->getDatabasePlatform() instanceof SQLitePlatform) { - $sql .= ' FOR UPDATE'; - } - - return $connection->fetchOne($sql, ['traceId' => $traceId]) !== null; - } - private function markCompleted(Connection $connection, string $traceId): bool { return $connection->executeStatement( diff --git a/packages/Ecotone/src/Tracing/InMemoryTraceStore.php b/packages/Ecotone/src/Tracing/InMemoryTraceStore.php index 54e8256ec..7296ac53c 100644 --- a/packages/Ecotone/src/Tracing/InMemoryTraceStore.php +++ b/packages/Ecotone/src/Tracing/InMemoryTraceStore.php @@ -32,24 +32,21 @@ final class InMemoryTraceStore implements TraceStore /** @var array */ private array $completedTraces = []; - public function record(array $spans): array + public function append(array $spans): void { - $touchedTraces = []; foreach ($spans as $span) { $this->spansByTrace[$span->traceId][] = $span; - $touchedTraces[$span->traceId] = true; } + } - $completedNow = []; - foreach (array_keys($touchedTraces) as $traceId) { - if (isset($this->completedTraces[$traceId]) || ! $this->progressOf($traceId)->isCompleted()) { - continue; - } - $this->completedTraces[$traceId] = true; - $completedNow[] = $traceId; + public function markCompletedIfDone(string $traceId): bool + { + if (isset($this->completedTraces[$traceId]) || ! $this->progressOf($traceId)->isCompleted()) { + return false; } + $this->completedTraces[$traceId] = true; - return $completedNow; + return true; } public function progressOf(string $traceId): TraceProgress diff --git a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php index 310bd3b4b..596089f29 100644 --- a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php +++ b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php @@ -68,7 +68,7 @@ public function record(Span $span): void return; } - $this->write([...$this->buffer, $span]); + $this->write([...$this->buffer, $span], true); $this->buffer = []; $this->publish(); } @@ -88,7 +88,7 @@ public function flush(): void foreach (self::outerParentsOf($this->buffer) as $spanId) { unset($this->spansWrappingAFailedUnitOfWork[$spanId]); } - $this->write($this->buffer); + $this->write($this->buffer, false); $this->buffer = []; } @@ -149,13 +149,14 @@ private static function failedByItsUnitOfWork(Span $span): Span /** * @param Span[] $spans */ - private function write(array $spans): void + private function write(array $spans, bool $decidesCompletion): void { if ($spans === []) { return; } - $completedTraces = $this->traceStore->record($spans); + $this->traceStore->append($spans); + $completedTraces = $decidesCompletion ? $this->completedAmong($spans) : []; if ($this->publishing) { return; } @@ -167,6 +168,22 @@ private function write(array $spans): void } } + /** + * @param Span[] $spans + * @return string[] + */ + private function completedAmong(array $spans): array + { + $completed = []; + foreach (array_unique(array_map(fn (Span $span): string => $span->traceId, $spans)) as $traceId) { + if ($this->traceStore->markCompletedIfDone($traceId)) { + $completed[] = $traceId; + } + } + + return $completed; + } + private function publish(): void { if ($this->publishing) { diff --git a/packages/Ecotone/src/Tracing/TraceStore.php b/packages/Ecotone/src/Tracing/TraceStore.php index e18e88a59..6de1a4700 100644 --- a/packages/Ecotone/src/Tracing/TraceStore.php +++ b/packages/Ecotone/src/Tracing/TraceStore.php @@ -29,9 +29,10 @@ interface TraceStore { /** * @param Span[] $spans - * @return string[] identifiers of the traces this recording completed */ - public function record(array $spans): array; + public function append(array $spans): void; + + public function markCompletedIfDone(string $traceId): bool; public function progressOf(string $traceId): TraceProgress; diff --git a/upgrade-2.0.md b/upgrade-2.0.md index 740e4ef8b..249904a9a 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3095,7 +3095,10 @@ in an external collector, after the fact. **Now:** `TracingConfiguration::createWithDefaults()->withStoredTraces()` keeps every span in the database as well, in `ecotone_traces` and `ecotone_spans`, created by `ecotone:migration:database:setup`. A unit of work, a bus call or an asynchronous consumption, buffers its spans and writes them once, inside its transaction before the commit; -the root span is written when it ends, after the commit. From those spans the `TraceCompletion` gateway answers +the span that wraps the unit of work is written when it ends, after the commit, and that write is the one that +decides whether the trace is complete, reading what every other unit of work has committed by then. No row is +locked: completion is claimed with a single conditional update, so two consumers finishing the last two handlers of +one trace both see each other's commit and exactly one of them claims it. From those spans the `TraceCompletion` gateway answers `isCompleted($traceId)`, true once the root ended, every route the trace sent has an ended handler span, and every message it sent to a channel was consumed from that channel, so an asynchronous command, event or projection keeps the trace pending until its consumer processed that exact message; `isCompletedFor($traceId, $destination)` From 27814009fc57619544ec54b038cc2f64b60b0670 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 16/50] fix(tracing): a user gateway opens the trace and the unit of work, and a handler's destination comes from its registration instead of tracestate A flow entered through a #[MessageGateway] or #[OrchestratorGateway] had no span around it, so every step of an orchestrator started its own trace and a stored trace had no unit of work to buffer in. Both are now traced as send or send orchestrator and open the recording scope, like a bus. The route stamped its destination into tracestate so that a handler without an explicit endpointId could still name it, but a reply copies the request headers, so the handler behind an outputChannelName hop was counted under the previous destination and one delivery completed the destination of two. The module now reads every handler's registered endpoint id at compile time, the same attribute instances the routing module used, and the interceptor looks its own up by class and method. tracestate is no longer written. Flow shapes now covered: outputChannelName chains, splitters, routers, orchestrators entered through their gateway and reached from a command handler, message gateways, and a command bus inside a command handler. --- .../Config/TraceEventRoutesCompilerPass.php | 2 - .../src/Tracing/Config/TracingModule.php | 35 +- packages/Ecotone/src/Tracing/TraceState.php | 42 --- .../src/Tracing/TracedRouteProcessor.php | 15 - .../src/Tracing/TracingInterceptor.php | 26 +- .../src/Tracing/TracingRouteResolver.php | 4 +- .../tests/Tracing/FlowShapeTracingTest.php | 329 ++++++++++++++++++ .../tests/Tracing/TraceCompletionTest.php | 74 ++++ 8 files changed, 456 insertions(+), 71 deletions(-) delete mode 100644 packages/Ecotone/src/Tracing/TraceState.php create mode 100644 packages/Ecotone/tests/Tracing/FlowShapeTracingTest.php diff --git a/packages/Ecotone/src/Tracing/Config/TraceEventRoutesCompilerPass.php b/packages/Ecotone/src/Tracing/Config/TraceEventRoutesCompilerPass.php index bce7dd9b2..faa23b451 100644 --- a/packages/Ecotone/src/Tracing/Config/TraceEventRoutesCompilerPass.php +++ b/packages/Ecotone/src/Tracing/Config/TraceEventRoutesCompilerPass.php @@ -8,7 +8,6 @@ use Ecotone\Messaging\Config\Container\ContainerBuilder; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\Reference; -use Ecotone\Messaging\Support\MessageBuilder; use Ecotone\Modelling\Config\MessageHandlerRoutingModule; use Ecotone\Tracing\Tracer; use Ecotone\Tracing\TracingRouteResolver; @@ -27,7 +26,6 @@ public function process(ContainerBuilder $builder): void $builder->replace(MessageHandlerRoutingModule::EVENT_BUS_ROUTE_RESOLVER, new Definition(TracingRouteResolver::class, [ $builder->getDefinition(MessageHandlerRoutingModule::EVENT_BUS_ROUTE_RESOLVER), new Reference(Tracer::class), - new Reference(MessageBuilder::class), ])); } } diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index 137b12a81..3447b16dd 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -12,7 +12,9 @@ use Ecotone\Api\Attribute\CommandHandler; use Ecotone\Api\Attribute\EventHandler; use Ecotone\Api\Attribute\InternalHandler; +use Ecotone\Api\Attribute\MessageGateway; use Ecotone\Api\Attribute\ModuleAnnotation; +use Ecotone\Api\Attribute\OrchestratorGateway; use Ecotone\Api\Attribute\QueryHandler; use Ecotone\Api\ExtensionObject\MessageChannelBuilder; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -62,8 +64,6 @@ use function implode; -use ReflectionClass; - #[ModuleAnnotation] /** * licence Apache-2.0 @@ -71,14 +71,21 @@ final class TracingModule extends NoExternalConfigurationModule implements AnnotationModule { private const REMOVE_EXPIRED_TRACES_CHANNEL = 'ecotone.tracing.removeExpiredTraces'; + private const USER_MESSAGE_GATEWAYS_POINTCUT = MessageGateway::class . '&¬(' . CommandBus::class . ')&¬(' . EventBus::class . ')&¬(' . QueryBus::class . ')'; /** * @param string[] $traceCompletedSubscribers * @param string[] $spanEndedSubscribers */ + /** + * @param string[] $traceCompletedSubscribers + * @param string[] $spanEndedSubscribers + * @param array $endpointIdsByHandler + */ private function __construct( private array $traceCompletedSubscribers, private array $spanEndedSubscribers, + private array $endpointIdsByHandler, ) { } @@ -87,9 +94,25 @@ public static function create(AnnotationFinder $annotationRegistrationService, I return new self( self::placementsOf($annotationRegistrationService, OnTraceCompleted::class), self::placementsOf($annotationRegistrationService, OnSpanEnded::class), + self::endpointIdsByHandler($annotationRegistrationService), ); } + /** + * @return array + */ + private static function endpointIdsByHandler(AnnotationFinder $annotationFinder): array + { + $endpointIds = []; + foreach ([CommandHandler::class, QueryHandler::class, EventHandler::class, InternalHandler::class] as $attributeClassName) { + foreach ($annotationFinder->findAnnotatedMethods($attributeClassName) as $annotatedMethod) { + $endpointIds[$annotatedMethod->getClassName() . '::' . $annotatedMethod->getMethodName()] = $annotatedMethod->getAnnotationForMethod()->getEndpointId(); + } + } + + return $endpointIds; + } + public function prepare(Configuration $messagingConfiguration, array $extensionObjects, ModuleReferenceSearchService $moduleReferenceSearchService, InterfaceToCallRegistry $interfaceToCallRegistry): void { $tracingConfiguration = ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $extensionObjects, TracingConfiguration::createWithDefaults()); @@ -120,7 +143,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO $messagingConfiguration->addCompilerPass(new TraceEventRoutesCompilerPass()); $messagingConfiguration->registerServiceDefinition( TracingInterceptor::class, - new Definition(TracingInterceptor::class, [new Reference(Tracer::class), new Reference(MessageBuilder::class)]) + new Definition(TracingInterceptor::class, [new Reference(Tracer::class), new Reference(MessageBuilder::class), $this->endpointIdsByHandler]) ); foreach ([ @@ -134,6 +157,8 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO 'traceAsynchronousEndpoint' => AsynchronousRunningEndpoint::class, 'traceDistributedBus' => DistributedBus::class, 'traceProjection' => ProjectionTriggerEndpoint::class, + 'traceMessageGateway' => self::USER_MESSAGE_GATEWAYS_POINTCUT, + 'traceOrchestratorGateway' => OrchestratorGateway::class, ] as $method => $pointcut) { $messagingConfiguration->registerAroundMethodInterceptor( AroundInterceptorBuilder::create( @@ -218,7 +243,7 @@ private function registerStoredTraces(Configuration $messagingConfiguration, arr TraceRecordingInterceptor::class, $interfaceToCallRegistry->getFor(TraceRecordingInterceptor::class, 'recordUnitOfWork'), Precedence::DATABASE_TRANSACTION_PRECEDENCE + 1, - CommandBus::class . '||' . EventBus::class . '||' . QueryBus::class . '||' . AsynchronousRunningEndpoint::class + CommandBus::class . '||' . EventBus::class . '||' . QueryBus::class . '||' . AsynchronousRunningEndpoint::class . '||' . OrchestratorGateway::class . '||(' . self::USER_MESSAGE_GATEWAYS_POINTCUT . ')' ) ); $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnTraceCompleted::CHANNEL)); @@ -256,7 +281,7 @@ private static function placementsOf(AnnotationFinder $annotationFinder, string { $placements = []; foreach ($annotationFinder->findAnnotatedMethods($attributeClassName) as $annotatedMethod) { - $placements[] = '#[' . (new ReflectionClass($attributeClassName))->getShortName() . '] on ' . $annotatedMethod->getClassName() . '::' . $annotatedMethod->getMethodName(); + $placements[] = '#[' . substr((string) strrchr('\\' . $attributeClassName, '\\'), 1) . '] on ' . $annotatedMethod->getClassName() . '::' . $annotatedMethod->getMethodName(); } return $placements; diff --git a/packages/Ecotone/src/Tracing/TraceState.php b/packages/Ecotone/src/Tracing/TraceState.php deleted file mode 100644 index f0944795d..000000000 --- a/packages/Ecotone/src/Tracing/TraceState.php +++ /dev/null @@ -1,42 +0,0 @@ -getHeaders(); - if (! $headers->containsKey(MessageHeaders::TRACESTATE)) { - return null; - } - foreach (explode(',', (string) $headers->get(MessageHeaders::TRACESTATE)) as $entry) { - $entry = trim($entry); - if (str_starts_with($entry, self::DESTINATION_ENTRY_PREFIX)) { - return rawurldecode(substr($entry, strlen(self::DESTINATION_ENTRY_PREFIX))); - } - } - - return null; - } -} diff --git a/packages/Ecotone/src/Tracing/TracedRouteProcessor.php b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php index 6c7a739cf..6d76f2771 100644 --- a/packages/Ecotone/src/Tracing/TracedRouteProcessor.php +++ b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php @@ -5,11 +5,9 @@ namespace Ecotone\Tracing; use Ecotone\Api\Messaging\Message; -use Ecotone\Api\Messaging\MessageHeaders; use Ecotone\Api\Tracing\SpanAttribute; use Ecotone\Api\Tracing\SpanKind; use Ecotone\Messaging\Handler\MessageProcessor; -use Ecotone\Messaging\Support\MessageBuilder; use Ecotone\Modelling\Config\Routing\BusRoutingMapBuilder; use Ecotone\Projecting\Config\ProjectingModule; use Throwable; @@ -23,7 +21,6 @@ public function __construct( private string $routeName, private MessageProcessor $routeProcessor, private Tracer $tracer, - private MessageBuilder $messageBuilder, ) { } @@ -36,13 +33,6 @@ public function process(Message $message): ?Message ...$this->destinationAttributes(), ]); - $destination = $this->destination(); - if ($destination !== null) { - $message = $this->messageBuilder->fromMessage($message) - ->setHeader(MessageHeaders::TRACESTATE, TraceState::withDestination($destination)) - ->build(); - } - try { $result = $this->routeProcessor->process($message); } catch (Throwable $failure) { @@ -66,11 +56,6 @@ private function spanName(): string return 'route ' . (BusRoutingMapBuilder::endpointIdOfDestinationChannel($this->routeName) ?? $this->routeName); } - private function destination(): ?string - { - return ProjectingModule::projectionNameOfTriggerChannel($this->routeName) ?? BusRoutingMapBuilder::endpointIdOfDestinationChannel($this->routeName); - } - /** * @return array */ diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 2e8debfd5..66ff69528 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -7,6 +7,7 @@ use Ecotone\Api\Attribute\CommandHandler; use Ecotone\Api\Attribute\EventHandler; use Ecotone\Api\Attribute\InternalHandler; +use Ecotone\Api\Attribute\MessageGateway; use Ecotone\Api\Attribute\QueryHandler; use Ecotone\Api\Interceptor\MethodInvocation; use Ecotone\Api\Messaging\Message; @@ -23,9 +24,13 @@ */ final class TracingInterceptor { + /** + * @param array $endpointIdsByHandler + */ public function __construct( private Tracer $tracer, private MessageBuilder $messageBuilder, + private array $endpointIdsByHandler, ) { } @@ -46,22 +51,22 @@ public function traceQueryBus(MethodInvocation $methodInvocation, Message $messa public function traceCommandHandler(MethodInvocation $methodInvocation, Message $message, CommandHandler $endpoint): mixed { - return $this->traceHandler($methodInvocation, $message, TraceState::destinationOf($message) ?? $endpoint->getEndpointId()); + return $this->traceHandler($methodInvocation, $message, $this->endpointIdsByHandler[$methodInvocation->getName()] ?? $endpoint->getEndpointId()); } public function traceQueryHandler(MethodInvocation $methodInvocation, Message $message, QueryHandler $endpoint): mixed { - return $this->traceHandler($methodInvocation, $message, TraceState::destinationOf($message) ?? $endpoint->getEndpointId()); + return $this->traceHandler($methodInvocation, $message, $this->endpointIdsByHandler[$methodInvocation->getName()] ?? $endpoint->getEndpointId()); } public function traceEventHandler(MethodInvocation $methodInvocation, Message $message, EventHandler $endpoint): mixed { - return $this->traceHandler($methodInvocation, $message, TraceState::destinationOf($message) ?? $endpoint->getEndpointId()); + return $this->traceHandler($methodInvocation, $message, $this->endpointIdsByHandler[$methodInvocation->getName()] ?? $endpoint->getEndpointId()); } public function traceMessageHandler(MethodInvocation $methodInvocation, Message $message, InternalHandler $endpoint): mixed { - return $this->traceHandler($methodInvocation, $message, TraceState::destinationOf($message) ?? $endpoint->getEndpointId()); + return $this->traceHandler($methodInvocation, $message, $this->endpointIdsByHandler[$methodInvocation->getName()] ?? $endpoint->getEndpointId()); } public function traceAsynchronousEndpoint(MethodInvocation $methodInvocation, Message $message): mixed @@ -89,6 +94,19 @@ public function traceProjection(MethodInvocation $methodInvocation, Message $mes ]); } + public function traceMessageGateway(MethodInvocation $methodInvocation, Message $message, MessageGateway $gateway): mixed + { + return $this->trace('send ' . $gateway->requestChannel, $methodInvocation, $message, SpanKind::Server, [ + SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND, + SpanAttribute::MESSAGING_DESTINATION_NAME => $gateway->requestChannel, + ]); + } + + public function traceOrchestratorGateway(MethodInvocation $methodInvocation, Message $message): mixed + { + return $this->trace('send orchestrator', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); + } + public function traceDistributedBus(MethodInvocation $methodInvocation, Message $message): mixed { return $this->trace('send distributed', $methodInvocation, $message, SpanKind::Producer, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); diff --git a/packages/Ecotone/src/Tracing/TracingRouteResolver.php b/packages/Ecotone/src/Tracing/TracingRouteResolver.php index 72217310a..0771ca894 100644 --- a/packages/Ecotone/src/Tracing/TracingRouteResolver.php +++ b/packages/Ecotone/src/Tracing/TracingRouteResolver.php @@ -6,7 +6,6 @@ use Ecotone\Messaging\Handler\MessageProcessor; use Ecotone\Messaging\Handler\Router\RouteResolver; -use Ecotone\Messaging\Support\MessageBuilder; /** * licence Apache-2.0 @@ -16,12 +15,11 @@ final class TracingRouteResolver implements RouteResolver public function __construct( private RouteResolver $routeResolver, private Tracer $tracer, - private MessageBuilder $messageBuilder, ) { } public function resolve(string $routeName): MessageProcessor { - return new TracedRouteProcessor($routeName, $this->routeResolver->resolve($routeName), $this->tracer, $this->messageBuilder); + return new TracedRouteProcessor($routeName, $this->routeResolver->resolve($routeName), $this->tracer); } } diff --git a/packages/Ecotone/tests/Tracing/FlowShapeTracingTest.php b/packages/Ecotone/tests/Tracing/FlowShapeTracingTest.php new file mode 100644 index 000000000..0297ba536 --- /dev/null +++ b/packages/Ecotone/tests/Tracing/FlowShapeTracingTest.php @@ -0,0 +1,329 @@ +sendCommandWithRouting('order.place', '1'); + + $this->assertSame( + ['process class@anonymous::place', 'process class@anonymous::store', 'process class@anonymous::validate', 'send command'], + self::spanNames($recorder), + ); + $this->assertSame( + ['process class@anonymous::place' => 'order.placing', 'process class@anonymous::store' => 'order.storing', 'process class@anonymous::validate' => 'order.validating'], + self::endpointIdsByHandlerSpan($recorder), + ); + $this->assertCount(1, self::traceIds($recorder)); + } + + public function test_a_splitter_fans_one_command_out_into_one_handler_span_per_item(): void + { + $recorder = new InMemorySpanRecorder(); + $imports = new class () { + #[CommandHandler('batch.import', outputChannelName: 'batch.split', endpointId: 'batch.importing')] + public function import(array $items): array + { + return $items; + } + + #[Splitter('batch.split', outputChannelName: 'item.handle')] + public function split(array $items): array + { + return $items; + } + + #[InternalHandler('item.handle', endpointId: 'item.handling')] + public function handle(string $item): void + { + } + }; + + self::bootstrap($recorder, [$imports])->sendCommandWithRouting('batch.import', ['a', 'b']); + + $this->assertSame( + ['process class@anonymous::handle', 'process class@anonymous::handle', 'process class@anonymous::import', 'send command'], + self::spanNames($recorder), + ); + $this->assertSame(['item.handling', 'item.handling'], self::endpointIdsOfSpansNamed('process class@anonymous::handle', $recorder)); + $this->assertCount(1, self::traceIds($recorder)); + } + + public function test_a_router_leads_the_trace_to_the_handler_it_chose(): void + { + $recorder = new InMemorySpanRecorder(); + $shipments = new class () { + #[CommandHandler('shipment.dispatch', outputChannelName: 'shipment.route', endpointId: 'shipment.dispatching')] + public function dispatch(string $shipmentId): string + { + return $shipmentId; + } + + #[Router('shipment.route')] + public function route(string $shipmentId): string + { + return 'shipment.express'; + } + + #[InternalHandler('shipment.express', endpointId: 'shipment.expressing')] + public function express(string $shipmentId): void + { + } + + #[InternalHandler('shipment.standard', endpointId: 'shipment.standard.handling')] + public function standard(string $shipmentId): void + { + } + }; + + self::bootstrap($recorder, [$shipments])->sendCommandWithRouting('shipment.dispatch', '1'); + + $this->assertSame( + ['process class@anonymous::dispatch', 'process class@anonymous::express', 'send command'], + self::spanNames($recorder), + ); + $this->assertSame(['shipment.expressing'], self::endpointIdsOfSpansNamed('process class@anonymous::express', $recorder)); + } + + public function test_an_orchestrator_gateway_starting_the_flow_traces_each_step_under_one_trace(): void + { + $recorder = new InMemorySpanRecorder(); + $steps = new class () { + #[InternalHandler('approval.validate', endpointId: 'approval.validating')] + public function validate(string $request): string + { + return $request; + } + + #[InternalHandler('approval.notify', endpointId: 'approval.notifying')] + public function notify(string $request): string + { + return $request; + } + }; + $ecotone = self::bootstrap($recorder, [$steps], [ApprovalWorkflow::class], LicenceTesting::VALID_LICENCE); + + $ecotone->getGateway(ApprovalWorkflow::class)->start(['approval.validate', 'approval.notify'], 'request-1'); + + $this->assertSame( + ['process class@anonymous::notify', 'process class@anonymous::validate', 'send orchestrator'], + self::spanNames($recorder), + ); + $this->assertSame( + ['process class@anonymous::notify' => 'approval.notifying', 'process class@anonymous::validate' => 'approval.validating'], + self::endpointIdsByHandlerSpan($recorder), + ); + $this->assertCount(1, self::traceIds($recorder)); + } + + public function test_an_orchestrator_reached_from_a_command_handler_continues_that_command_trace(): void + { + $recorder = new InMemorySpanRecorder(); + $approvals = new class () { + #[CommandHandler('approval.request', outputChannelName: 'approval.workflow', endpointId: 'approval.requesting')] + public function request(string $request): string + { + return $request; + } + + #[Orchestrator('approval.workflow')] + public function workflow(): array + { + return ['approval.validate', 'approval.notify']; + } + + #[InternalHandler('approval.validate', endpointId: 'approval.validating')] + public function validate(string $request): string + { + return $request; + } + + #[InternalHandler('approval.notify', endpointId: 'approval.notifying')] + public function notify(string $request): string + { + return $request; + } + }; + + self::bootstrap($recorder, [$approvals], [], LicenceTesting::VALID_LICENCE)->sendCommandWithRouting('approval.request', 'request-1'); + + $this->assertSame( + ['process class@anonymous::notify', 'process class@anonymous::request', 'process class@anonymous::validate', 'send command'], + self::spanNames($recorder), + ); + $this->assertSame( + ['process class@anonymous::notify' => 'approval.notifying', 'process class@anonymous::request' => 'approval.requesting', 'process class@anonymous::validate' => 'approval.validating'], + self::endpointIdsByHandlerSpan($recorder), + ); + $this->assertCount(1, self::traceIds($recorder)); + } + + public function test_a_message_gateway_starting_the_flow_is_the_root_of_the_handler_it_reaches(): void + { + $recorder = new InMemorySpanRecorder(); + $notifications = new class () { + #[InternalHandler('notification.send', endpointId: 'notification.sending')] + public function send(string $text): void + { + } + }; + $ecotone = self::bootstrap($recorder, [$notifications], [NotificationGateway::class]); + + $ecotone->getGateway(NotificationGateway::class)->send('hello'); + + $this->assertSame(['process class@anonymous::send', 'send notification.send'], self::spanNames($recorder)); + $this->assertCount(1, self::traceIds($recorder)); + $this->assertSame(['notification.sending'], self::endpointIdsOfSpansNamed('process class@anonymous::send', $recorder)); + } + + public function test_a_command_bus_called_inside_a_command_handler_nests_the_second_command_under_the_first_handler(): void + { + $recorder = new InMemorySpanRecorder(); + $users = new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId, CommandBus $commandBus): void + { + $commandBus->sendWithRouting('audit.record', $userId); + } + + #[CommandHandler('audit.record', endpointId: 'audit.recording')] + public function record(string $userId): void + { + } + }; + + self::bootstrap($recorder, [$users])->sendCommandWithRouting('user.register', '1'); + + self::getNodeAtTargetedSpan( + [ + 'details' => ['name' => 'send command'], + 'child' => [ + 'details' => ['name' => 'process class@anonymous::register', 'attributes' => [SpanAttribute::ENDPOINT_ID => 'user.registration']], + 'child' => [ + 'details' => ['name' => 'send command'], + 'child' => ['details' => ['name' => 'process class@anonymous::record', 'attributes' => [SpanAttribute::ENDPOINT_ID => 'audit.recording']]], + ], + ], + ], + self::buildTree($recorder->recordedSpans()) + ); + } + + /** + * @param object[] $services + * @param class-string[] $gateways + */ + private static function bootstrap(InMemorySpanRecorder $recorder, array $services, array $gateways = [], ?string $licenceKey = null): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + [...array_map(fn (object $service): string => $service::class, $services), ...$gateways], + [...$services, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + licenceKey: $licenceKey, + ); + } + + /** + * @return string[] + */ + private static function spanNames(InMemorySpanRecorder $recorder): array + { + $names = array_map( + fn (string $name): string => preg_replace('/^process class@anonymous\S*::/', 'process class@anonymous::', $name), + self::spanNamesWithoutFrameworkNoise($recorder->recordedSpans()), + ); + sort($names); + + return $names; + } + + /** + * @return array + */ + private static function endpointIdsByHandlerSpan(InMemorySpanRecorder $recorder): array + { + $endpointIds = []; + foreach ($recorder->recordedSpans() as $span) { + if (isset($span->attributes[SpanAttribute::ENDPOINT_ID])) { + $endpointIds[preg_replace('/^process class@anonymous\S*::/', 'process class@anonymous::', $span->name)] = $span->attributes[SpanAttribute::ENDPOINT_ID]; + } + } + ksort($endpointIds); + + return $endpointIds; + } + + /** + * @return string[] + */ + private static function endpointIdsOfSpansNamed(string $name, InMemorySpanRecorder $recorder): array + { + return array_values(array_map( + fn (Span $span): string => $span->attributes[SpanAttribute::ENDPOINT_ID] ?? '', + array_filter($recorder->recordedSpans(), fn (Span $span): bool => preg_replace('/^process class@anonymous\S*::/', 'process class@anonymous::', $span->name) === $name), + )); + } + + /** + * @return string[] + */ + private static function traceIds(InMemorySpanRecorder $recorder): array + { + return array_values(array_unique(array_map(fn (Span $span): string => $span->traceId, $recorder->recordedSpans()))); + } +} + +interface ApprovalWorkflow +{ + #[OrchestratorGateway] + public function start(array $routing, mixed $data): mixed; +} + +interface NotificationGateway +{ + #[MessageGateway('notification.send')] + public function send(string $text): void; +} diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index 212214492..a4e951803 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -7,6 +7,9 @@ use Ecotone\Api\Attribute\Asynchronous; use Ecotone\Api\Attribute\CommandHandler; use Ecotone\Api\Attribute\EventHandler; +use Ecotone\Api\Attribute\Headers; +use Ecotone\Api\Attribute\InternalHandler; +use Ecotone\Api\Attribute\MessageGateway; use Ecotone\Api\EventSourcing\Event; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\ServiceConfiguration; @@ -437,6 +440,71 @@ public function place(string $orderId): void $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } + public function test_an_output_channel_after_an_asynchronous_handler_does_not_complete_that_handler_for_a_delivery_still_queued(): void + { + $notifier = new class () { + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', outputChannelName: 'audit.record', endpointId: 'user.notifier')] + public function notify(string $userId): string + { + return $userId; + } + + #[InternalHandler('audit.record', endpointId: 'audit.recording')] + public function record(string $userId): void + { + } + }; + $ecotone = $this->bootstrap([$notifier], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->publishEventWithRouting('user.registered', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + $ecotone->publishEventWithRouting('user.registered', '2', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 1)); + + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'audit.recording')); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); + $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 1)); + + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + public function test_a_flow_started_through_a_message_gateway_is_pending_until_its_asynchronous_handler_ran(): void + { + $notifications = new class () { + #[InternalHandler('notification.send', outputChannelName: 'notification.deliver', endpointId: 'notification.sending')] + public function send(string $text): string + { + return $text; + } + + #[Asynchronous('async_channel')] + #[InternalHandler('notification.deliver', endpointId: 'notification.delivering')] + public function deliver(string $text): void + { + } + }; + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$notifications::class, TracedNotificationGateway::class], + [$notifications], + [TracingConfiguration::createWithDefaults()->withStoredTraces(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $trace = SpanContext::newTrace(); + + $ecotone->getGateway(TracedNotificationGateway::class)->send('hello', [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'notification.sending')); + $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + public function test_a_failed_unit_of_work_stores_nothing_by_default(): void { $subscriber = $this->spanEndedSubscriber(); @@ -608,3 +676,9 @@ public function notify(string $userId): void }; } } + +interface TracedNotificationGateway +{ + #[MessageGateway('notification.send')] + public function send(string $text, #[Headers] array $metadata): void; +} From a0c4d64a4be1a37454ebc8340ba570dfd1eb707b Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 17/50] feat(tracing): withSpanAttributesFromHeaders() names the headers reported as attributes, framework attribute names cannot be forged, and stored traces without a store are refused at bootstrap Every scalar user header became a span attribute, which stored tokens and personal data for the retention period and let a header named ecotone.endpoint.id or ecotone.trace.root pose as a framework attribute. The attribute builder is now a service configured with the headers to report, defaulting to all, and never reports a header under the ecotone., messaging. or code. prefixes. withStoredTraces() with no TraceStoreReference and inMemory false used to fall back to a per-process store in production; it is refused with both ways out. The OpenTelemetry package has a test that it traces nothing until a TracerProviderInterface is registered. --- .../Api/Tracing/TracingConfiguration.php | 10 +++- .../Config/TraceEventRoutesCompilerPass.php | 2 + .../TracingChannelInterceptorBuilder.php | 2 + .../src/Tracing/Config/TracingModule.php | 10 +++- .../src/Tracing/MessageSpanAttributes.php | 26 +++++++++- .../src/Tracing/TracedRouteProcessor.php | 3 +- .../src/Tracing/TracingChannelInterceptor.php | 5 +- .../src/Tracing/TracingInterceptor.php | 3 +- .../src/Tracing/TracingRouteResolver.php | 3 +- .../tests/Tracing/TraceCompletionTest.php | 15 ++++++ .../Tracing/TracingConfigurationTest.php | 50 +++++++++++++++++++ .../TracingDisabledWithoutProviderTest.php | 48 ++++++++++++++++++ upgrade-2.0.md | 10 +++- 13 files changed, 176 insertions(+), 11 deletions(-) create mode 100644 packages/OpenTelemetry/tests/Integration/TracingDisabledWithoutProviderTest.php diff --git a/packages/Ecotone/Api/Tracing/TracingConfiguration.php b/packages/Ecotone/Api/Tracing/TracingConfiguration.php index 35dfaf785..1a3042b72 100644 --- a/packages/Ecotone/Api/Tracing/TracingConfiguration.php +++ b/packages/Ecotone/Api/Tracing/TracingConfiguration.php @@ -13,6 +13,7 @@ final class TracingConfiguration { /** * @param string[] $spanRecorderReferenceNames + * @param string[] $spanAttributeHeaders */ private function __construct( public readonly array $spanRecorderReferenceNames, @@ -20,12 +21,14 @@ private function __construct( public readonly bool $storedTracesInMemory, public readonly Duration $storedTracesRetention, public readonly bool $storedFailuresOutsideTransaction, + public readonly bool $spanAttributesFromAllHeaders, + public readonly array $spanAttributeHeaders, ) { } public static function createWithDefaults(): self { - return new self([], false, false, Duration::minutes(60 * 24 * 7), false); + return new self([], false, false, Duration::minutes(60 * 24 * 7), false, true, []); } public function withSpanRecorder(string $referenceName): self @@ -33,6 +36,11 @@ public function withSpanRecorder(string $referenceName): self return new self(...[...get_object_vars($this), 'spanRecorderReferenceNames' => [...$this->spanRecorderReferenceNames, $referenceName]]); } + public function withSpanAttributesFromHeaders(string ...$headerNames): self + { + return new self(...[...get_object_vars($this), 'spanAttributesFromAllHeaders' => false, 'spanAttributeHeaders' => array_values($headerNames)]); + } + public function withStoredTraces(bool $inMemory = false, ?Duration $retention = null, bool $storeFailuresOutsideTransaction = false): self { return new self(...[...get_object_vars($this), 'storedTraces' => true, 'storedTracesInMemory' => $inMemory, 'storedTracesRetention' => $retention ?? $this->storedTracesRetention, 'storedFailuresOutsideTransaction' => $storeFailuresOutsideTransaction]); diff --git a/packages/Ecotone/src/Tracing/Config/TraceEventRoutesCompilerPass.php b/packages/Ecotone/src/Tracing/Config/TraceEventRoutesCompilerPass.php index faa23b451..71d91ba22 100644 --- a/packages/Ecotone/src/Tracing/Config/TraceEventRoutesCompilerPass.php +++ b/packages/Ecotone/src/Tracing/Config/TraceEventRoutesCompilerPass.php @@ -9,6 +9,7 @@ use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\Reference; use Ecotone\Modelling\Config\MessageHandlerRoutingModule; +use Ecotone\Tracing\MessageSpanAttributes; use Ecotone\Tracing\Tracer; use Ecotone\Tracing\TracingRouteResolver; @@ -26,6 +27,7 @@ public function process(ContainerBuilder $builder): void $builder->replace(MessageHandlerRoutingModule::EVENT_BUS_ROUTE_RESOLVER, new Definition(TracingRouteResolver::class, [ $builder->getDefinition(MessageHandlerRoutingModule::EVENT_BUS_ROUTE_RESOLVER), new Reference(Tracer::class), + new Reference(MessageSpanAttributes::class), ])); } } diff --git a/packages/Ecotone/src/Tracing/Config/TracingChannelInterceptorBuilder.php b/packages/Ecotone/src/Tracing/Config/TracingChannelInterceptorBuilder.php index 1742f31e0..2b802cddc 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingChannelInterceptorBuilder.php +++ b/packages/Ecotone/src/Tracing/Config/TracingChannelInterceptorBuilder.php @@ -10,6 +10,7 @@ use Ecotone\Messaging\Config\Container\MessagingContainerBuilder; use Ecotone\Messaging\Config\Container\Reference; use Ecotone\Messaging\Support\MessageBuilder; +use Ecotone\Tracing\MessageSpanAttributes; use Ecotone\Tracing\Tracer; use Ecotone\Tracing\TracingChannelInterceptor; @@ -38,6 +39,7 @@ public function compile(MessagingContainerBuilder $builder): Definition $this->channelName, new Reference(Tracer::class), new Reference(MessageBuilder::class), + new Reference(MessageSpanAttributes::class), ]); } } diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index 3447b16dd..ff39f6b59 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -55,6 +55,7 @@ use Ecotone\Tracing\EcotoneTracer; use Ecotone\Tracing\ExpiredTraceRemover; use Ecotone\Tracing\InMemoryTraceStore; +use Ecotone\Tracing\MessageSpanAttributes; use Ecotone\Tracing\StoredTraceCompletion; use Ecotone\Tracing\StoringSpanRecorder; use Ecotone\Tracing\Tracer; @@ -143,7 +144,11 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO $messagingConfiguration->addCompilerPass(new TraceEventRoutesCompilerPass()); $messagingConfiguration->registerServiceDefinition( TracingInterceptor::class, - new Definition(TracingInterceptor::class, [new Reference(Tracer::class), new Reference(MessageBuilder::class), $this->endpointIdsByHandler]) + new Definition(TracingInterceptor::class, [new Reference(Tracer::class), new Reference(MessageBuilder::class), new Reference(MessageSpanAttributes::class), $this->endpointIdsByHandler]) + ); + $messagingConfiguration->registerServiceDefinition( + MessageSpanAttributes::class, + new Definition(MessageSpanAttributes::class, [$tracingConfiguration->spanAttributesFromAllHeaders, $tracingConfiguration->spanAttributeHeaders]) ); foreach ([ @@ -218,6 +223,9 @@ private function refuseMisconfiguredStoredTraces(Configuration $messagingConfigu private function registerStoredTraces(Configuration $messagingConfiguration, array $extensionObjects, TracingConfiguration $tracingConfiguration, InterfaceToCallRegistry $interfaceToCallRegistry): void { $traceStoreReferences = ExtensionObjectResolver::resolve(TraceStoreReference::class, $extensionObjects); + if ($traceStoreReferences === [] && ! $tracingConfiguration->storedTracesInMemory) { + throw ConfigurationException::create('TracingConfiguration::withStoredTraces() needs a trace store and none is registered. Enable the DBAL package with a connection to store traces in the database, or call withStoredTraces(inMemory: true) to keep them in this process only.'); + } $messagingConfiguration->registerServiceDefinition( TraceStore::class, $traceStoreReferences === [] || $tracingConfiguration->storedTracesInMemory diff --git a/packages/Ecotone/src/Tracing/MessageSpanAttributes.php b/packages/Ecotone/src/Tracing/MessageSpanAttributes.php index 7db3d3498..4eb28a9f7 100644 --- a/packages/Ecotone/src/Tracing/MessageSpanAttributes.php +++ b/packages/Ecotone/src/Tracing/MessageSpanAttributes.php @@ -16,14 +16,25 @@ */ final class MessageSpanAttributes { + private const RESERVED_PREFIXES = ['ecotone.', 'messaging.', 'code.']; + + /** + * @param string[] $headerNames + */ + public function __construct( + private bool $fromAllHeaders, + private array $headerNames, + ) { + } + /** * @return array */ - public static function from(Message $message): array + public function from(Message $message): array { $attributes = []; foreach (FrameworkHeaders::unsetAllFrameworkHeaders($message->getHeaders()->headers()) as $key => $value) { - if (is_scalar($value)) { + if (is_scalar($value) && $this->isReported((string) $key)) { $attributes[$key] = $value; } } @@ -39,4 +50,15 @@ public static function from(Message $message): array return $attributes; } + + private function isReported(string $headerName): bool + { + foreach (self::RESERVED_PREFIXES as $prefix) { + if (str_starts_with($headerName, $prefix)) { + return false; + } + } + + return $this->fromAllHeaders || in_array($headerName, $this->headerNames, true); + } } diff --git a/packages/Ecotone/src/Tracing/TracedRouteProcessor.php b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php index 6d76f2771..66b298e71 100644 --- a/packages/Ecotone/src/Tracing/TracedRouteProcessor.php +++ b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php @@ -21,13 +21,14 @@ public function __construct( private string $routeName, private MessageProcessor $routeProcessor, private Tracer $tracer, + private MessageSpanAttributes $messageSpanAttributes, ) { } public function process(Message $message): ?Message { $span = $this->tracer->startSpan($this->spanName(), SpanKind::Producer, [ - ...MessageSpanAttributes::from($message), + ...$this->messageSpanAttributes->from($message), SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND, SpanAttribute::MESSAGING_DESTINATION_NAME => $this->routeName, ...$this->destinationAttributes(), diff --git a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php index befefdf56..e8e2a60ea 100644 --- a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php @@ -27,13 +27,14 @@ public function __construct( private string $channelName, private Tracer $tracer, private MessageBuilder $messageBuilder, + private MessageSpanAttributes $messageSpanAttributes, ) { } public function preSend(Message $message, MessageChannel $messageChannel): ?Message { $span = $this->tracer->startSpan('send ' . $this->channelName, SpanKind::Producer, [ - ...MessageSpanAttributes::from($message), + ...$this->messageSpanAttributes->from($message), SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND, SpanAttribute::MESSAGING_DESTINATION_NAME => $this->channelName, ]); @@ -73,7 +74,7 @@ public function afterReceiveCompletion(?Message $message, MessageChannel $messag $this->tracer ->startSpan('receive ' . $this->channelName, SpanKind::Consumer, [ - ...MessageSpanAttributes::from($message), + ...$this->messageSpanAttributes->from($message), SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS, SpanAttribute::MESSAGING_DESTINATION_NAME => $this->channelName, ], TracingInterceptor::remoteParentOf($message)) diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 66ff69528..19463a741 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -30,6 +30,7 @@ final class TracingInterceptor public function __construct( private Tracer $tracer, private MessageBuilder $messageBuilder, + private MessageSpanAttributes $messageSpanAttributes, private array $endpointIdsByHandler, ) { } @@ -161,7 +162,7 @@ private function trace(string $name, MethodInvocation $methodInvocation, Message if ($isLocalRoot && $kind !== SpanKind::Consumer) { $attributes[SpanAttribute::TRACE_ROOT] = true; } - $span = $this->tracer->startSpan($name, $kind, [...MessageSpanAttributes::from($message), ...$attributes], $isLocalRoot ? self::remoteParentOf($message) : null); + $span = $this->tracer->startSpan($name, $kind, [...$this->messageSpanAttributes->from($message), ...$attributes], $isLocalRoot ? self::remoteParentOf($message) : null); try { $result = $methodInvocation->proceed(); diff --git a/packages/Ecotone/src/Tracing/TracingRouteResolver.php b/packages/Ecotone/src/Tracing/TracingRouteResolver.php index 0771ca894..82560a66f 100644 --- a/packages/Ecotone/src/Tracing/TracingRouteResolver.php +++ b/packages/Ecotone/src/Tracing/TracingRouteResolver.php @@ -15,11 +15,12 @@ final class TracingRouteResolver implements RouteResolver public function __construct( private RouteResolver $routeResolver, private Tracer $tracer, + private MessageSpanAttributes $messageSpanAttributes, ) { } public function resolve(string $routeName): MessageProcessor { - return new TracedRouteProcessor($routeName, $this->routeResolver->resolve($routeName), $this->tracer); + return new TracedRouteProcessor($routeName, $this->routeResolver->resolve($routeName), $this->tracer, $this->messageSpanAttributes); } } diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index a4e951803..db4f56e4e 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -12,6 +12,7 @@ use Ecotone\Api\Attribute\MessageGateway; use Ecotone\Api\EventSourcing\Event; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; +use Ecotone\Api\ExtensionObject\ModulePackageList; use Ecotone\Api\ExtensionObject\ServiceConfiguration; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; use Ecotone\Api\Gateway\CommandBus; @@ -542,6 +543,20 @@ public function test_a_failed_unit_of_work_stores_its_failed_root_span_when_fail $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } + public function test_stored_traces_without_a_trace_store_are_refused_at_bootstrap(): void + { + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage('TracingConfiguration::withStoredTraces() needs a trace store and none is registered. Enable the DBAL package with a connection to store traces in the database, or call withStoredTraces(inMemory: true) to keep them in this process only.'); + + EcotoneLite::bootstrapFlowTesting( + [], + [], + [TracingConfiguration::createWithDefaults()->withStoredTraces()], + enabledModules: [ModulePackageList::CORE_PACKAGE], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + public function test_stored_traces_require_an_enterprise_licence(): void { $this->expectException(LicensingException::class); diff --git a/packages/Ecotone/tests/Tracing/TracingConfigurationTest.php b/packages/Ecotone/tests/Tracing/TracingConfigurationTest.php index 2353d923b..6f1c9b344 100644 --- a/packages/Ecotone/tests/Tracing/TracingConfigurationTest.php +++ b/packages/Ecotone/tests/Tracing/TracingConfigurationTest.php @@ -72,6 +72,56 @@ public function register(string $userId, #[Headers] array $headers): void $this->assertMatchesRegularExpression('/^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/', $users->headers[MessageHeaders::TRACEPARENT]); } + public function test_only_the_configured_headers_become_span_attributes(): void + { + $recorder = new InMemorySpanRecorder(); + $users = new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId): void + { + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()->withSpanAttributesFromHeaders('tenant')], + ) + ->sendCommandWithRouting('user.register', '1', metadata: ['tenant' => 'a', 'authorization' => 'Bearer secret']); + + foreach ($recorder->recordedSpans() as $span) { + $this->assertSame('a', $span->attributes['tenant'], $span->name); + $this->assertArrayNotHasKey('authorization', $span->attributes, $span->name); + } + } + + public function test_a_user_header_named_like_a_framework_attribute_does_not_become_one(): void + { + $recorder = new InMemorySpanRecorder(); + $users = new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId): void + { + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users, ...self::recordingSpansInto($recorder)], + [self::tracingConfiguration()], + ) + ->sendCommandWithRouting('user.register', '1', metadata: ['ecotone.endpoint.id' => 'forged', 'ecotone.trace.root' => true, 'messaging.system' => 'forged']); + + $spansByName = []; + foreach ($recorder->recordedSpans() as $span) { + $spansByName[preg_replace('/^process class@anonymous\\S*::/', 'process class@anonymous::', $span->name)] = $span; + $this->assertSame('ecotone', $span->attributes['messaging.system'], $span->name); + } + $this->assertSame('user.registration', $spansByName['process class@anonymous::register']->attributes['ecotone.endpoint.id'] ?? null); + $this->assertArrayNotHasKey('ecotone.endpoint.id', $spansByName['send command']->attributes); + $this->assertArrayNotHasKey('ecotone.trace.root', $spansByName['process class@anonymous::register']->attributes); + } + public function test_two_tracers_are_refused_at_bootstrap(): void { $this->expectException(ConfigurationException::class); diff --git a/packages/OpenTelemetry/tests/Integration/TracingDisabledWithoutProviderTest.php b/packages/OpenTelemetry/tests/Integration/TracingDisabledWithoutProviderTest.php new file mode 100644 index 000000000..5ad111ff5 --- /dev/null +++ b/packages/OpenTelemetry/tests/Integration/TracingDisabledWithoutProviderTest.php @@ -0,0 +1,48 @@ +headers = $headers; + } + }; + + EcotoneLite::bootstrapFlowTesting( + [$users::class], + [$users], + [SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + enabledModules: [ModulePackageList::TRACING_PACKAGE], + ) + ->sendCommandWithRouting('user.register', '1') + ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertNotSame([], $users->headers); + $this->assertArrayNotHasKey(MessageHeaders::TRACEPARENT, $users->headers); + } +} diff --git a/upgrade-2.0.md b/upgrade-2.0.md index 249904a9a..389f476af 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3056,7 +3056,12 @@ its route. An event-driven projection run is a CONSUMER span. And every span car attributes: `messaging.system` is `ecotone`, `messaging.operation.type` is `send` for buses, routes and channel sends and `process` for handlers, consumers and projections, `messaging.destination.name` names the route or channel, and `ecotone.endpoint.id` or `ecotone.projection.name` name the handler; the names are constants on -`Ecotone\Api\Tracing\SpanAttribute`. The tracing interceptors are registered only when a recorder is configured or a +`Ecotone\Api\Tracing\SpanAttribute`. Every scalar user header of the message is a span attribute as well, unless +`TracingConfiguration::createWithDefaults()->withSpanAttributesFromHeaders('tenant', 'orderId')` names the ones to +report, which an application holding tokens or personal data in headers should do; a header whose name starts with +`ecotone.`, `messaging.` or `code.` is never reported, so a message cannot forge the attributes the framework sets. +A flow entered through an application's own `#[MessageGateway]` or `#[OrchestratorGateway]` is traced from that +call, `send ` or `send orchestrator`, so an orchestrator's steps share one trace. The tracing interceptors are registered only when a recorder is configured or a tracing package hands over its `Tracer`, so an application without tracing pays nothing. `ecotone/open-telemetry` becomes an implementation of that `Tracer`: it starts live SDK spans with the same ids Core reports, keeps the forced flush and the Dynamic Consistency Boundary spans, and records nothing when a flow test is given no @@ -3115,7 +3120,8 @@ that produced them, so a websocket push or a cache refresh never runs ahead of t through a bus is stored like any other trace but never notifies subscribers in turn, so a subscriber that sends a command does not call itself again. The trace id is the one in the `traceparent` header: mint it with `SpanContext::newTrace()` and send it with the command, or take it from the HTTP request the command came from. A flow test given no connection keeps the -store in memory. `ecotone:tracing:remove-expired-traces` deletes traces older than the retention passed to +store in memory; an application that enables stored traces with no DBAL package and no `inMemory: true` is refused +at bootstrap, because a store per process could never see a consumer's progress. `ecotone:tracing:remove-expired-traces` deletes traces older than the retention passed to `withStoredTraces()`, seven days by default. ```php From 45a80290138c6790f8fee273814982e1feed3804 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 18/50] fix(tracing): a destination counts messages by id, a channel send names its asynchronous target, the handler span is innermost, and the timeout names failures, unknown traces and unreached destinations A send to an asynchronous handler reached the ledger with no destination unless it came from an event route, so two commands to one handler completed it after the first. The channel send span now carries the target endpoint id read from the routing slip, and a destination tracks the message ids dispatched to it, so a redelivered or replayed message satisfies its own delivery only. A channel send outside any span continues the message's own traceparent, so a dead-letter replay completes the trace it came from. The handler span was the outermost interceptor and ended Ok when an error channel inside it handled the failure; it is now the innermost, at Precedence::TRACING_HANDLER_PRECEDENCE, so the handler's own failure is stored inside the transaction that moved its message. With storeFailuresOutsideTransaction the failed unit's spans are kept. The recording interceptor has its own Precedence::TRACE_RECORDING_PRECEDENCE, the following constants moved up by one, ecotone_spans is keyed by (trace_id, span_id), and TraceCompletion is registered unconditionally, refusing with the configuration it needs when stored traces are off. --- .../Dbal/src/Database/SpansTableManager.php | 3 +- .../Conformance/TraceStoreConformanceTest.php | 37 +++++++- .../Ecotone/Api/Interceptor/Precedence.php | 5 +- .../Ecotone/Api/Tracing/SpanAttribute.php | 1 + .../Api/Tracing/TraceCompletionTimedOut.php | 35 ++++++-- .../AsynchronousModule.php | 4 +- .../src/Tracing/Config/TracingModule.php | 66 +++++++------- .../src/Tracing/StoredTraceCompletion.php | 13 ++- .../src/Tracing/StoringSpanRecorder.php | 4 +- packages/Ecotone/src/Tracing/TraceLedger.php | 27 +++--- .../Ecotone/src/Tracing/TraceProgress.php | 43 ++++++--- .../src/Tracing/TracingChannelInterceptor.php | 33 ++++++- .../src/Tracing/TracingInterceptor.php | 1 + .../Tracing/UnavailableTraceCompletion.php | 40 +++++++++ .../tests/Tracing/TraceCompletionTest.php | 87 ++++++++++++++++++- .../CorrelatedHeadersPropagationTest.php | 4 +- .../tests/Integration/TracingTreeTest.php | 5 +- upgrade-2.0.md | 9 +- 18 files changed, 334 insertions(+), 83 deletions(-) create mode 100644 packages/Ecotone/src/Tracing/UnavailableTraceCompletion.php diff --git a/packages/Dbal/src/Database/SpansTableManager.php b/packages/Dbal/src/Database/SpansTableManager.php index eed1df4c4..c894319ef 100644 --- a/packages/Dbal/src/Database/SpansTableManager.php +++ b/packages/Dbal/src/Database/SpansTableManager.php @@ -117,8 +117,7 @@ private function buildTableSchema(): Table $table->addColumn('events', Types::TEXT); $table->addColumn('started_at', Types::BIGINT); $table->addColumn('ended_at', Types::BIGINT); - $table->setPrimaryKey(['span_id']); - $table->addIndex(['trace_id']); + $table->setPrimaryKey(['trace_id', 'span_id']); return $table; } diff --git a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php index 8c9e9b349..22c28e326 100644 --- a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php +++ b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php @@ -9,7 +9,10 @@ use Ecotone\Api\Attribute\EventHandler; use Ecotone\Api\Dbal\DbalConnectionFactory; use Ecotone\Api\Dbal\ExtensionObject\DbalBackedMessageChannelBuilder; +use Ecotone\Api\Dbal\ExtensionObject\DbalDeadLetterBuilder; +use Ecotone\Api\Dbal\Gateway\DeadLetterGateway; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; +use Ecotone\Api\ExtensionObject\InstantRetryConfiguration; use Ecotone\Api\ExtensionObject\ModulePackageList; use Ecotone\Api\ExtensionObject\ServiceConfiguration; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; @@ -97,7 +100,7 @@ public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pen $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $this->expectException(TraceCompletionTimedOut::class); - $this->expectExceptionMessage('Still pending: user.notifier, async_channel.'); + $this->expectExceptionMessage('Still pending: user.notifier, async_channel. Run the consumer of the pending channel'); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); } @@ -290,6 +293,38 @@ public function test_a_consumer_that_swallowed_its_handler_failure_leaves_the_tr self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); } + #[DataProvider('implementations')] + public function test_a_dead_lettered_message_replayed_later_completes_the_trace_it_came_from(string $implementation): void + { + $notifier = new class () { + public int $attempts = 0; + + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId): void + { + if (++$this->attempts === 1) { + throw new RuntimeException('Notification failed once'); + } + } + }; + $ecotone = $this->bootstrap($implementation, [$this->users(), $notifier], [ + SimpleMessageChannelBuilder::createQueueChannel('async_channel'), + ServiceConfiguration::createWithDefaults()->withDefaultErrorChannel(DbalDeadLetterBuilder::STORE_CHANNEL), + InstantRetryConfiguration::createWithDefaults()->withAsynchronousEndpointsRetry(false), + ]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + + $ecotone->getGateway(DeadLetterGateway::class)->replayAll(); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertSame(2, $notifier->attempts); + self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + public function test_completion_recorded_by_one_instance_is_visible_to_another_sharing_the_database(): void { $producer = $this->bootstrap('dbal', [$this->users()]); diff --git a/packages/Ecotone/Api/Interceptor/Precedence.php b/packages/Ecotone/Api/Interceptor/Precedence.php index ff9516084..5502f6467 100644 --- a/packages/Ecotone/Api/Interceptor/Precedence.php +++ b/packages/Ecotone/Api/Interceptor/Precedence.php @@ -12,6 +12,7 @@ interface Precedence { public const TRACING_PRECEDENCE = Precedence::ERROR_CHANNEL_PRECEDENCE - 101; + public const TRACING_HANDLER_PRECEDENCE = 1000000; public const ASYNCHRONOUS_CONSUMER_INTERCEPTOR_PRECEDENCE = Precedence::ERROR_CHANNEL_PRECEDENCE - 100; public const MESSAGE_ACKNOWLEDGE_PRECEDENCE = Precedence::ERROR_CHANNEL_PRECEDENCE - 99; public const EXCEPTION_LOGGING_PRECEDENCE = Precedence::ERROR_CHANNEL_PRECEDENCE - 1; @@ -30,7 +31,9 @@ interface Precedence public const DATABASE_TRANSACTION_PRECEDENCE = -2000; - public const DELIVERY_CONFIRMATION_PRECEDENCE = self::DATABASE_TRANSACTION_PRECEDENCE + 1; + public const TRACE_RECORDING_PRECEDENCE = self::DATABASE_TRANSACTION_PRECEDENCE + 1; + + public const DELIVERY_CONFIRMATION_PRECEDENCE = self::TRACE_RECORDING_PRECEDENCE + 1; public const COLLECTOR_SENDER_PRECEDENCE = self::DELIVERY_CONFIRMATION_PRECEDENCE + 1; diff --git a/packages/Ecotone/Api/Tracing/SpanAttribute.php b/packages/Ecotone/Api/Tracing/SpanAttribute.php index 8276c763e..0b557179d 100644 --- a/packages/Ecotone/Api/Tracing/SpanAttribute.php +++ b/packages/Ecotone/Api/Tracing/SpanAttribute.php @@ -16,6 +16,7 @@ final class SpanAttribute public const PROJECTION_NAME = 'ecotone.projection.name'; public const TRACE_ROOT = 'ecotone.trace.root'; public const CODE_FUNCTION_NAME = 'code.function.name'; + public const CHANNEL_NAME = 'ecotone.channel.name'; public const ECOTONE = 'ecotone'; public const OPERATION_SEND = 'send'; diff --git a/packages/Ecotone/Api/Tracing/TraceCompletionTimedOut.php b/packages/Ecotone/Api/Tracing/TraceCompletionTimedOut.php index 78748501d..bbd9e7b95 100644 --- a/packages/Ecotone/Api/Tracing/TraceCompletionTimedOut.php +++ b/packages/Ecotone/Api/Tracing/TraceCompletionTimedOut.php @@ -18,24 +18,49 @@ final class TraceCompletionTimedOut extends RuntimeException { /** * @param string[] $pendingDestinations + * @param string[] $failedDestinations */ - public static function forTrace(string $traceId, Duration $timeout, array $pendingDestinations): self + public static function forTrace(string $traceId, Duration $timeout, array $pendingDestinations, array $failedDestinations): self + { + $failed = $failedDestinations === [] ? '' : sprintf(' Failed: %s, whose last attempt failed and whose message is in the error channel or awaiting a retry.', implode(', ', $failedDestinations)); + $pending = array_values(array_diff($pendingDestinations, $failedDestinations)); + $stillPending = $pending === [] ? '' : sprintf(' Still pending: %s.', implode(', ', $pending)); + $advice = $pending === [] ? ' Await with a longer timeout once the failure is resolved.' : ' Run the consumer of the pending channel, or await with a longer timeout.'; + + return new self(sprintf('Trace %s did not complete within %d milliseconds.%s%s%s', $traceId, $timeout->inMilliseconds(), $failed, $stillPending, $advice)); + } + + public static function forUnrecordedTrace(string $traceId, Duration $timeout): self + { + return new self(sprintf( + 'No span was recorded under trace %s within %d milliseconds. Send the command with that traceparent in its metadata, and check that the process handling it stores traces.', + $traceId, + $timeout->inMilliseconds(), + )); + } + + public static function forDestination(string $traceId, string $destination, Duration $timeout, bool $failed): self { return new self(sprintf( - 'Trace %s did not complete within %d milliseconds. Still pending: %s. Run the consumer of the channel those handlers are attached to, or await with a longer timeout.', + 'Trace %s did not complete for %s within %d milliseconds.%s', $traceId, + $destination, $timeout->inMilliseconds(), - $pendingDestinations === [] ? 'the root of the trace, which was not recorded yet' : implode(', ', $pendingDestinations), + $failed ? ' Its last attempt failed and its message is in the error channel or awaiting a retry.' : ' Run the consumer of the channel that handler is attached to, or await with a longer timeout.', )); } - public static function forDestination(string $traceId, string $destination, Duration $timeout): self + /** + * @param string[] $reachedDestinations + */ + public static function forUnreachedDestination(string $traceId, string $destination, Duration $timeout, array $reachedDestinations): self { return new self(sprintf( - 'Trace %s did not complete for %s within %d milliseconds. Run the consumer of the channel that handler is attached to, or await with a longer timeout.', + 'Trace %s was neither sent to nor processed by %s within %d milliseconds. Check the endpointId or projection name; the trace reached: %s.', $traceId, $destination, $timeout->inMilliseconds(), + $reachedDestinations === [] ? 'nothing yet' : implode(', ', $reachedDestinations), )); } } diff --git a/packages/Ecotone/src/Messaging/Config/Annotation/ModuleConfiguration/AsynchronousModule.php b/packages/Ecotone/src/Messaging/Config/Annotation/ModuleConfiguration/AsynchronousModule.php index 89865f029..f055facf0 100644 --- a/packages/Ecotone/src/Messaging/Config/Annotation/ModuleConfiguration/AsynchronousModule.php +++ b/packages/Ecotone/src/Messaging/Config/Annotation/ModuleConfiguration/AsynchronousModule.php @@ -115,9 +115,11 @@ public function getSynchronousChannelFor(string $handlerChannelName, string $end return $handlerChannelName; } + public const HANDLER_EXECUTION_CHANNEL_SUFFIX = '.execute'; + public static function getHandlerExecutionChannel(string $originalInputChannelName): string { - return $originalInputChannelName . '.execute'; + return $originalInputChannelName . self::HANDLER_EXECUTION_CHANNEL_SUFFIX; } /** diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index ff39f6b59..5bcef176f 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -62,6 +62,7 @@ use Ecotone\Tracing\TraceRecordingInterceptor; use Ecotone\Tracing\TraceStore; use Ecotone\Tracing\TracingInterceptor; +use Ecotone\Tracing\UnavailableTraceCompletion; use function implode; @@ -99,6 +100,22 @@ public static function create(AnnotationFinder $annotationRegistrationService, I ); } + private static function registerCompletionGateway(Configuration $messagingConfiguration, string $serviceClass, Definition $service): void + { + $messagingConfiguration->registerServiceDefinition($serviceClass, $service); + foreach ([ + 'isCompleted' => [GatewayPayloadBuilder::create('traceId')], + 'isCompletedFor' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('destination', StoredTraceCompletion::DESTINATION_HEADER)], + 'await' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('timeout', StoredTraceCompletion::TIMEOUT_HEADER)], + 'awaitFor' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('destination', StoredTraceCompletion::DESTINATION_HEADER), GatewayHeaderBuilder::create('timeout', StoredTraceCompletion::TIMEOUT_HEADER)], + ] as $method => $parameterConverters) { + $channel = 'ecotone.tracing.completion.' . $method; + $messagingConfiguration + ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod($serviceClass, $method)->withInputChannelName($channel)) + ->registerGatewayBuilder(GatewayProxyBuilder::create(TraceCompletion::class, TraceCompletion::class, $method, $channel)->withParameterConverters($parameterConverters)); + } + } + /** * @return array */ @@ -123,6 +140,9 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO } $this->refuseMisconfiguredStoredTraces($messagingConfiguration, $tracingConfiguration); + if (! $tracingConfiguration->storedTraces) { + self::registerCompletionGateway($messagingConfiguration, UnavailableTraceCompletion::class, new Definition(UnavailableTraceCompletion::class)); + } if ($tracingConfiguration->spanRecorderReferenceNames === [] && $tracerReferences === [] && ! $tracingConfiguration->storedTraces) { return; } @@ -152,24 +172,24 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ); foreach ([ - 'traceCommandHandler' => CommandHandler::class, - 'traceQueryHandler' => QueryHandler::class, - 'traceEventHandler' => EventHandler::class, - 'traceMessageHandler' => InternalHandler::class . '&¬(' . MessageHeadersPropagatorInterceptor::class . ')&¬(' . OnTraceCompleted::class . ')&¬(' . OnSpanEnded::class . ')', - 'traceCommandBus' => CommandBus::class, - 'traceQueryBus' => QueryBus::class, - 'traceEventBus' => EventBus::class, - 'traceAsynchronousEndpoint' => AsynchronousRunningEndpoint::class, - 'traceDistributedBus' => DistributedBus::class, - 'traceProjection' => ProjectionTriggerEndpoint::class, - 'traceMessageGateway' => self::USER_MESSAGE_GATEWAYS_POINTCUT, - 'traceOrchestratorGateway' => OrchestratorGateway::class, - ] as $method => $pointcut) { + 'traceCommandHandler' => [CommandHandler::class, Precedence::TRACING_HANDLER_PRECEDENCE], + 'traceQueryHandler' => [QueryHandler::class, Precedence::TRACING_HANDLER_PRECEDENCE], + 'traceEventHandler' => [EventHandler::class, Precedence::TRACING_HANDLER_PRECEDENCE], + 'traceMessageHandler' => [InternalHandler::class . '&¬(' . MessageHeadersPropagatorInterceptor::class . ')&¬(' . OnTraceCompleted::class . ')&¬(' . OnSpanEnded::class . ')', Precedence::TRACING_HANDLER_PRECEDENCE], + 'traceProjection' => [ProjectionTriggerEndpoint::class, Precedence::TRACING_HANDLER_PRECEDENCE], + 'traceCommandBus' => [CommandBus::class, Precedence::TRACING_PRECEDENCE], + 'traceQueryBus' => [QueryBus::class, Precedence::TRACING_PRECEDENCE], + 'traceEventBus' => [EventBus::class, Precedence::TRACING_PRECEDENCE], + 'traceAsynchronousEndpoint' => [AsynchronousRunningEndpoint::class, Precedence::TRACING_PRECEDENCE], + 'traceDistributedBus' => [DistributedBus::class, Precedence::TRACING_PRECEDENCE], + 'traceMessageGateway' => [self::USER_MESSAGE_GATEWAYS_POINTCUT, Precedence::TRACING_PRECEDENCE], + 'traceOrchestratorGateway' => [OrchestratorGateway::class, Precedence::TRACING_PRECEDENCE], + ] as $method => [$pointcut, $precedence]) { $messagingConfiguration->registerAroundMethodInterceptor( AroundInterceptorBuilder::create( TracingInterceptor::class, $interfaceToCallRegistry->getFor(TracingInterceptor::class, $method), - Precedence::TRACING_PRECEDENCE, + $precedence, $pointcut ) ); @@ -250,28 +270,14 @@ private function registerStoredTraces(Configuration $messagingConfiguration, arr AroundInterceptorBuilder::create( TraceRecordingInterceptor::class, $interfaceToCallRegistry->getFor(TraceRecordingInterceptor::class, 'recordUnitOfWork'), - Precedence::DATABASE_TRANSACTION_PRECEDENCE + 1, + Precedence::TRACE_RECORDING_PRECEDENCE, CommandBus::class . '||' . EventBus::class . '||' . QueryBus::class . '||' . AsynchronousRunningEndpoint::class . '||' . OrchestratorGateway::class . '||(' . self::USER_MESSAGE_GATEWAYS_POINTCUT . ')' ) ); $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnTraceCompleted::CHANNEL)); $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnSpanEnded::CHANNEL)); - $messagingConfiguration->registerServiceDefinition( - StoredTraceCompletion::class, - new Definition(StoredTraceCompletion::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class)]) - ); - foreach ([ - 'isCompleted' => [GatewayPayloadBuilder::create('traceId')], - 'isCompletedFor' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('destination', StoredTraceCompletion::DESTINATION_HEADER)], - 'await' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('timeout', StoredTraceCompletion::TIMEOUT_HEADER)], - 'awaitFor' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('destination', StoredTraceCompletion::DESTINATION_HEADER), GatewayHeaderBuilder::create('timeout', StoredTraceCompletion::TIMEOUT_HEADER)], - ] as $method => $parameterConverters) { - $channel = 'ecotone.tracing.completion.' . $method; - $messagingConfiguration - ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod(StoredTraceCompletion::class, $method)->withInputChannelName($channel)) - ->registerGatewayBuilder(GatewayProxyBuilder::create(TraceCompletion::class, TraceCompletion::class, $method, $channel)->withParameterConverters($parameterConverters)); - } + self::registerCompletionGateway($messagingConfiguration, StoredTraceCompletion::class, new Definition(StoredTraceCompletion::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class)])); $messagingConfiguration->registerServiceDefinition( ExpiredTraceRemover::class, diff --git a/packages/Ecotone/src/Tracing/StoredTraceCompletion.php b/packages/Ecotone/src/Tracing/StoredTraceCompletion.php index d40226ce8..f13444df7 100644 --- a/packages/Ecotone/src/Tracing/StoredTraceCompletion.php +++ b/packages/Ecotone/src/Tracing/StoredTraceCompletion.php @@ -58,7 +58,9 @@ public function await(string $traceId, #[Header(self::TIMEOUT_HEADER)] Duration return; } if ($this->clock->now() >= $deadline) { - throw TraceCompletionTimedOut::forTrace($traceId, $timeout, $progress->pendingDestinations()); + throw $progress->recorded + ? TraceCompletionTimedOut::forTrace($traceId, $timeout, $progress->pendingDestinations(), $progress->failedDestinations()) + : TraceCompletionTimedOut::forUnrecordedTrace($traceId, $timeout); } $this->clock->sleep(Duration::milliseconds(self::POLL_INTERVAL_IN_MILLISECONDS)); } @@ -68,11 +70,16 @@ public function awaitFor(string $traceId, #[Header(self::DESTINATION_HEADER)] st { $deadline = $this->clock->now()->add($timeout); while (true) { - if ($this->traceStore->progressOf($traceId)->isCompletedFor($destination)) { + $progress = $this->traceStore->progressOf($traceId); + if ($progress->isCompletedFor($destination)) { return; } if ($this->clock->now() >= $deadline) { - throw TraceCompletionTimedOut::forDestination($traceId, $destination, $timeout); + throw match (true) { + ! $progress->recorded => TraceCompletionTimedOut::forUnrecordedTrace($traceId, $timeout), + ! $progress->reached($destination) => TraceCompletionTimedOut::forUnreachedDestination($traceId, $destination, $timeout, $progress->destinations()), + default => TraceCompletionTimedOut::forDestination($traceId, $destination, $timeout, in_array($destination, $progress->failedDestinations(), true)), + }; } $this->clock->sleep(Duration::milliseconds(self::POLL_INTERVAL_IN_MILLISECONDS)); } diff --git a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php index 596089f29..b99326491 100644 --- a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php +++ b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php @@ -102,7 +102,9 @@ public function discard(): void foreach (self::outerParentsOf($this->buffer) as $spanId) { $this->spansWrappingAFailedUnitOfWork[$spanId] = true; } - $this->buffer = []; + if (! $this->storesFailuresOutsideTransaction) { + $this->buffer = []; + } } /** diff --git a/packages/Ecotone/src/Tracing/TraceLedger.php b/packages/Ecotone/src/Tracing/TraceLedger.php index 3e83e4e18..87673c4b4 100644 --- a/packages/Ecotone/src/Tracing/TraceLedger.php +++ b/packages/Ecotone/src/Tracing/TraceLedger.php @@ -49,28 +49,27 @@ public static function progressOf(string $traceId, array $spans): TraceProgress if ($span->attributes[SpanAttribute::TRACE_ROOT] ?? false) { $rootEnded = true; } - $destination = self::destinationOf($span); - if ($destination === null) { - $delivery = self::deliveryOf($span); - if ($delivery === null || $span->status === SpanStatus::Error) { - continue; - } + $delivery = self::deliveryOf($span); + if ($delivery !== null) { [$channel, $messageId] = $delivery; - if ($span->kind === SpanKind::Producer) { - $sentDeliveries[$channel][$messageId] = ($sentDeliveries[$channel][$messageId] ?? 0) + 1; + if ($span->kind === SpanKind::Producer && $span->status !== SpanStatus::Error) { + $sentDeliveries[$channel][$messageId] = true; } elseif ($span->kind === SpanKind::Consumer) { - $processedDeliveries[$channel][$messageId] = ($processedDeliveries[$channel][$messageId] ?? 0) + 1; + $processedDeliveries[$channel][$messageId] = true; } - + } + $destination = self::destinationOf($span); + if ($destination === null) { continue; } $operation = $span->attributes[SpanAttribute::MESSAGING_OPERATION_TYPE] ?? null; + $messageId = (string) ($span->attributes[MessageHeaders::MESSAGE_ID] ?? $span->spanId); if ($operation === SpanAttribute::OPERATION_SEND) { - $sent[$destination] = ($sent[$destination] ?? 0) + 1; + $sent[$destination][$messageId] = true; } elseif ($operation === SpanAttribute::OPERATION_PROCESS && $span->status === SpanStatus::Error) { - $failed[$destination] = ($failed[$destination] ?? 0) + 1; + $failed[$destination][$messageId] = true; } elseif ($operation === SpanAttribute::OPERATION_PROCESS) { - $processed[$destination] = ($processed[$destination] ?? 0) + 1; + $processed[$destination][$messageId] = true; } } @@ -89,7 +88,7 @@ public static function destinationOf(Span $span): ?string */ private static function deliveryOf(Span $span): ?array { - $channel = $span->attributes[SpanAttribute::MESSAGING_DESTINATION_NAME] ?? null; + $channel = $span->attributes[SpanAttribute::CHANNEL_NAME] ?? null; $messageId = $span->attributes[MessageHeaders::MESSAGE_ID] ?? null; return $channel === null || $messageId === null ? null : [(string) $channel, (string) $messageId]; diff --git a/packages/Ecotone/src/Tracing/TraceProgress.php b/packages/Ecotone/src/Tracing/TraceProgress.php index e926d6b64..6dba07395 100644 --- a/packages/Ecotone/src/Tracing/TraceProgress.php +++ b/packages/Ecotone/src/Tracing/TraceProgress.php @@ -27,11 +27,11 @@ final class TraceProgress { /** - * @param array $sent - * @param array $processed - * @param array $failed - * @param array> $sentDeliveries - * @param array> $processedDeliveries + * @param array> $sent message ids dispatched to each destination + * @param array> $processed message ids each destination processed + * @param array> $failed message ids each destination failed on + * @param array> $sentDeliveries message ids sent to each channel + * @param array> $processedDeliveries message ids consumed from each channel */ public function __construct( public readonly string $traceId, @@ -60,9 +60,9 @@ public function isCompletedFor(string $destination): bool if (! $this->recorded) { return false; } - $processed = $this->processed[$destination] ?? 0; + $processed = $this->processed[$destination] ?? []; - return $processed > 0 && $processed >= ($this->sent[$destination] ?? 0); + return $processed !== [] && array_diff_key($this->sent[$destination] ?? [], $processed) === []; } /** @@ -72,22 +72,39 @@ public function pendingDestinations(): array { $pending = []; foreach ($this->sent as $destination => $sent) { - if (($this->processed[$destination] ?? 0) < $sent) { + if (array_diff_key($sent, $this->processed[$destination] ?? []) !== []) { $pending[] = $destination; } } foreach ($this->sentDeliveries as $channel => $messages) { - foreach ($messages as $messageId => $sent) { - if (($this->processedDeliveries[$channel][$messageId] ?? 0) < $sent) { - $pending[] = $channel; - break; - } + if (array_diff_key($messages, $this->processedDeliveries[$channel] ?? []) !== []) { + $pending[] = $channel; } } return array_values(array_unique($pending)); } + /** + * @return string[] + */ + public function failedDestinations(): array + { + $failed = []; + foreach ($this->failed as $destination => $failedMessages) { + if (array_diff_key($failedMessages, $this->processed[$destination] ?? []) !== []) { + $failed[] = $destination; + } + } + + return $failed; + } + + public function reached(string $destination): bool + { + return isset($this->sent[$destination]) || isset($this->processed[$destination]) || isset($this->failed[$destination]); + } + /** * @return string[] */ diff --git a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php index e8e2a60ea..d4c147392 100644 --- a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php @@ -12,7 +12,9 @@ use Ecotone\Api\Tracing\SpanAttribute; use Ecotone\Api\Tracing\SpanKind; use Ecotone\Messaging\Channel\ChannelInterceptor; +use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\AsynchronousModule; use Ecotone\Messaging\Support\MessageBuilder; +use Ecotone\Modelling\Config\Routing\BusRoutingMapBuilder; use Throwable; /** @@ -33,11 +35,22 @@ public function __construct( public function preSend(Message $message, MessageChannel $messageChannel): ?Message { - $span = $this->tracer->startSpan('send ' . $this->channelName, SpanKind::Producer, [ + $attributes = [ ...$this->messageSpanAttributes->from($message), SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND, SpanAttribute::MESSAGING_DESTINATION_NAME => $this->channelName, - ]); + SpanAttribute::CHANNEL_NAME => $this->channelName, + ]; + $targetEndpointId = self::targetEndpointIdOf($message); + if ($targetEndpointId !== null) { + $attributes[SpanAttribute::ENDPOINT_ID] = $targetEndpointId; + } + $span = $this->tracer->startSpan( + 'send ' . $this->channelName, + SpanKind::Producer, + $attributes, + $this->tracer->currentSpan() === null ? TracingInterceptor::remoteParentOf($message) : null, + ); $this->openSendSpans[] = $span; return $this->messageBuilder->fromMessage($message) @@ -45,6 +58,21 @@ public function preSend(Message $message, MessageChannel $messageChannel): ?Mess ->build(); } + private static function targetEndpointIdOf(Message $message): ?string + { + $headers = $message->getHeaders(); + if (! $headers->containsKey(MessageHeaders::ROUTING_SLIP)) { + return null; + } + $routingSlip = explode(',', (string) $headers->get(MessageHeaders::ROUTING_SLIP)); + $executionChannel = $routingSlip[count($routingSlip) - 1]; + if (! str_ends_with($executionChannel, AsynchronousModule::HANDLER_EXECUTION_CHANNEL_SUFFIX)) { + return null; + } + + return BusRoutingMapBuilder::endpointIdOfDestinationChannel(substr($executionChannel, 0, -strlen(AsynchronousModule::HANDLER_EXECUTION_CHANNEL_SUFFIX))); + } + public function postSend(Message $message, MessageChannel $messageChannel): void { } @@ -77,6 +105,7 @@ public function afterReceiveCompletion(?Message $message, MessageChannel $messag ...$this->messageSpanAttributes->from($message), SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS, SpanAttribute::MESSAGING_DESTINATION_NAME => $this->channelName, + SpanAttribute::CHANNEL_NAME => $this->channelName, ], TracingInterceptor::remoteParentOf($message)) ->fail($exception); } diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 19463a741..550928012 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -82,6 +82,7 @@ public function traceAsynchronousEndpoint(MethodInvocation $methodInvocation, Me $attributes = [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS]; if ($headers->containsKey(MessageHeaders::POLLED_CHANNEL_NAME)) { $attributes[SpanAttribute::MESSAGING_DESTINATION_NAME] = $headers->get(MessageHeaders::POLLED_CHANNEL_NAME); + $attributes[SpanAttribute::CHANNEL_NAME] = $headers->get(MessageHeaders::POLLED_CHANNEL_NAME); } return $this->trace($name, $methodInvocation, $message, SpanKind::Consumer, $attributes); diff --git a/packages/Ecotone/src/Tracing/UnavailableTraceCompletion.php b/packages/Ecotone/src/Tracing/UnavailableTraceCompletion.php new file mode 100644 index 000000000..a59aef359 --- /dev/null +++ b/packages/Ecotone/src/Tracing/UnavailableTraceCompletion.php @@ -0,0 +1,40 @@ +withStoredTraces() from a #[ServiceContext] method.'); + } +} diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index db4f56e4e..62cc3deeb 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -117,7 +117,7 @@ public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pen $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $this->expectException(TraceCompletionTimedOut::class); - $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the channel those handlers are attached to, or await with a longer timeout."); + $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the pending channel, or await with a longer timeout."); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); } @@ -283,7 +283,7 @@ public function place(string $orderId): void $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $this->expectException(TraceCompletionTimedOut::class); - $this->expectExceptionMessage('Still pending: async_channel.'); + $this->expectExceptionMessage('Still pending: order.placing, async_channel.'); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); } @@ -506,6 +506,87 @@ public function deliver(string $text): void $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } + public function test_the_completion_gateway_without_stored_traces_names_the_configuration_it_needs(): void + { + $ecotone = EcotoneLite::bootstrapFlowTesting([$this->users()::class], [$this->users()]); + + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage('TraceCompletion needs stored traces, which decide when a trace is complete. Register TracingConfiguration::createWithDefaults()->withStoredTraces() from a #[ServiceContext] method.'); + + $this->completion($ecotone)->isCompleted('0af7651916cd43dd8448eb211c80319c'); + } + + public function test_awaiting_a_trace_whose_handler_failed_names_the_failed_handler(): void + { + $notifier = new class () { + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId): void + { + throw new RuntimeException('Notification failed'); + } + }; + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$this->users()::class, $notifier::class], + [$this->users(), $notifier], + [TracingConfiguration::createWithDefaults()->withStoredTraces(storeFailuresOutsideTransaction: true), SimpleMessageChannelBuilder::createQueueChannel('async_channel'), ServiceConfiguration::createWithDefaults()->withDefaultErrorChannel('errorChannel')], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); + + $this->expectException(TraceCompletionTimedOut::class); + $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Failed: user.notifier, whose last attempt failed and whose message is in the error channel or awaiting a retry. Await with a longer timeout once the failure is resolved."); + + $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); + } + + public function test_awaiting_a_trace_nobody_recorded_says_so(): void + { + $ecotone = $this->bootstrap([$this->users()]); + + $this->expectException(TraceCompletionTimedOut::class); + $this->expectExceptionMessage('No span was recorded under trace 0af7651916cd43dd8448eb211c80319c within 100 milliseconds. Send the command with that traceparent in its metadata, and check that the process handling it stores traces.'); + + $this->completion($ecotone)->await('0af7651916cd43dd8448eb211c80319c', Duration::milliseconds(100)); + } + + public function test_awaiting_a_destination_the_trace_never_reached_names_it_as_unknown(): void + { + $ecotone = $this->bootstrap([$this->users()]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->expectException(TraceCompletionTimedOut::class); + $this->expectExceptionMessage("Trace {$trace->traceId} was neither sent to nor processed by user.notifierr within 100 milliseconds. Check the endpointId or projection name; the trace reached: user.registration."); + + $this->completion($ecotone)->awaitFor($trace->traceId, 'user.notifierr', Duration::milliseconds(100)); + } + + public function test_two_asynchronous_commands_to_one_handler_complete_it_only_after_both_ran(): void + { + $orders = new class () { + #[Asynchronous('async_channel')] + #[CommandHandler('order.place', endpointId: 'order.placing')] + public function place(string $orderId): void + { + } + }; + $ecotone = $this->bootstrap([$orders], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + $ecotone->sendCommandWithRouting('order.place', '2', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 1)); + + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 1)); + + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); + } + public function test_a_failed_unit_of_work_stores_nothing_by_default(): void { $subscriber = $this->spanEndedSubscriber(); @@ -539,7 +620,7 @@ public function test_a_failed_unit_of_work_stores_its_failed_root_span_when_fail } catch (RuntimeException) { } - $this->assertSame(['send command' => SpanStatus::Error], $subscriber->endedSpans); + $this->assertSame(['process class@anonymous::register' => SpanStatus::Error, 'send command' => SpanStatus::Error], array_combine(array_map(fn (string $name): string => preg_replace('/^process class@anonymous\\S*::/', 'process class@anonymous::', $name), array_keys($subscriber->endedSpans)), $subscriber->endedSpans)); $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } diff --git a/packages/OpenTelemetry/tests/Integration/CorrelatedHeadersPropagationTest.php b/packages/OpenTelemetry/tests/Integration/CorrelatedHeadersPropagationTest.php index 9edc7d882..2ab4ccc64 100644 --- a/packages/OpenTelemetry/tests/Integration/CorrelatedHeadersPropagationTest.php +++ b/packages/OpenTelemetry/tests/Integration/CorrelatedHeadersPropagationTest.php @@ -206,13 +206,13 @@ public function test_tracing_with_exception() /** Command Handler */ $this->assertSame(StatusCode::STATUS_ERROR, $spans[0]->getStatus()->getCode()); - $event = $spans[0]->getEvents()[1]; + $event = current(array_filter($spans[0]->getEvents(), fn ($event): bool => $event->getName() === 'exception')); $this->assertSame('exception', $event->getName()); $this->assertSame('User already registered', $event->getAttributes()->get('exception.message')); /** Command Bus */ $this->assertSame(StatusCode::STATUS_ERROR, $spans[1]->getStatus()->getCode()); - $event = $spans[1]->getEvents()[1]; + $event = current(array_filter($spans[1]->getEvents(), fn ($event): bool => $event->getName() === 'exception')); $this->assertSame('exception', $event->getName()); $this->assertSame('User already registered', $event->getAttributes()->get('exception.message')); } diff --git a/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php b/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php index 6cac4d107..867ef2c20 100644 --- a/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php +++ b/packages/OpenTelemetry/tests/Integration/TracingTreeTest.php @@ -275,10 +275,7 @@ public function test_traces_are_force_flushed_when_synchronous_exception_happens self::buildTree($exporter) ); - $this->assertSame( - 'exception', - $node['details']['events'][1]->getName() - ); + $this->assertContains('exception', array_map(fn ($event): string => $event->getName(), $node['details']['events'])); } public function test_separate_traces() diff --git a/upgrade-2.0.md b/upgrade-2.0.md index 389f476af..3c1308897 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3114,7 +3114,14 @@ producer stored nothing is never completed by its consumers alone. A unit of wor buffered, so a handler that succeeded inside a rolled-back transaction, and a message whose send rolled back with it, are not counted. By default nothing of a failed unit of work is stored, because the only moment left to store it is after the rollback, outside any transaction; `withStoredTraces(storeFailuresOutsideTransaction: true)` opts in -to storing the failed outer span then, so a failed trace shows its error and counts as complete. A method with `#[OnTraceCompleted]` receives a +to storing the failed outer span then, so a failed trace shows its error and counts as complete. A destination counts a message as dispatched and as processed by +its message id, so a redelivery of the same message satisfies the delivery it belongs to and no other, and a +message sent to an asynchronous handler is attributed to that handler's `endpointId` whether it was routed by the +event bus, the command bus or an `outputChannelName`. The handler span is the innermost interceptor around the +method, so it records the handler's own failure even when an error channel or a retry handles it, and +`TraceCompletionTimedOut` names the handlers whose last attempt failed, the channels still holding a message, a +trace nobody recorded, and a destination the trace never reached. `TraceCompletion` is always injectable; without +`withStoredTraces()` every call refuses with the configuration it needs. A method with `#[OnTraceCompleted]` receives a `TraceCompleted` once per trace, and one with `#[OnSpanEnded]` receives every stored span, both after the commit that produced them, so a websocket push or a cache refresh never runs ahead of the data. Work a subscriber does through a bus is stored like any other trace but never notifies subscribers in turn, so a subscriber that sends a From 8d48bc9941436e2d3e8346891e3b1dc5e972246f Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 19/50] feat(tracing): completion notifications are durable and repeat when a trace completes again, retention spares pending traces, appends are idempotent, polling projections are refused, and a handler can read its trace id A trace that was handed more work after completing, as when one request sends two commands under one traceparent, reopens on append and notifies again when it completes again. The store records when a completion was notified; a subscriber that throws no longer fails the committed command, and any later unit of work re-delivers a notification left undelivered for thirty seconds. Span inserts are idempotent and the buffer is taken before the write, so a failed write cannot replay the same span ids. The retention command deletes completed traces past the retention and any trace past twice the retention with two statements, instead of loading every id into one IN list and deleting traces whose delayed messages are still queued. isCompletedFor() and awaitFor() refuse a #[Polling] projection with the reason. Ecotone\Api\Tracing\CurrentTrace gives a handler the id of the trace it runs in, and refuses with the way to enable tracing when nothing records one. --- .../Dbal/src/Database/TracesTableManager.php | 2 + packages/Dbal/src/Tracing/DbalTraceStore.php | 77 +++++++------ .../Conformance/TraceStoreConformanceTest.php | 85 +++++++++++++++ .../Conformance/TraceStoreContractTest.php | 101 ++++++++++++++++++ packages/Ecotone/Api/Tracing/CurrentTrace.php | 13 +++ .../src/Tracing/Config/TracingModule.php | 29 ++++- .../src/Tracing/ExpiredTraceRemover.php | 3 +- .../src/Tracing/InMemoryTraceStore.php | 52 ++++++--- .../src/Tracing/StoredTraceCompletion.php | 15 +++ .../src/Tracing/StoringSpanRecorder.php | 31 +++++- packages/Ecotone/src/Tracing/TraceStore.php | 11 +- .../src/Tracing/TracerCurrentTrace.php | 22 ++++ .../Tracing/TracingDisabledCurrentTrace.php | 19 ++++ .../tests/Tracing/TraceCompletionTest.php | 56 ++++++++++ upgrade-2.0.md | 16 ++- 15 files changed, 474 insertions(+), 58 deletions(-) create mode 100644 packages/Dbal/tests/Conformance/TraceStoreContractTest.php create mode 100644 packages/Ecotone/Api/Tracing/CurrentTrace.php create mode 100644 packages/Ecotone/src/Tracing/TracerCurrentTrace.php create mode 100644 packages/Ecotone/src/Tracing/TracingDisabledCurrentTrace.php diff --git a/packages/Dbal/src/Database/TracesTableManager.php b/packages/Dbal/src/Database/TracesTableManager.php index 04214f143..b35a6ef0b 100644 --- a/packages/Dbal/src/Database/TracesTableManager.php +++ b/packages/Dbal/src/Database/TracesTableManager.php @@ -109,8 +109,10 @@ private function buildTableSchema(): Table $table->addColumn('trace_id', Types::STRING, ['length' => 32]); $table->addColumn('started_at', Types::BIGINT); $table->addColumn('completed_at', Types::BIGINT, ['notnull' => false]); + $table->addColumn('notified_at', Types::BIGINT, ['notnull' => false]); $table->setPrimaryKey(['trace_id']); $table->addIndex(['started_at']); + $table->addIndex(['completed_at', 'notified_at']); return $table; } diff --git a/packages/Dbal/src/Tracing/DbalTraceStore.php b/packages/Dbal/src/Tracing/DbalTraceStore.php index e6d4be94a..a76963ffc 100644 --- a/packages/Dbal/src/Tracing/DbalTraceStore.php +++ b/packages/Dbal/src/Tracing/DbalTraceStore.php @@ -4,7 +4,6 @@ namespace Ecotone\Dbal\Tracing; -use Doctrine\DBAL\ArrayParameterType; use Doctrine\DBAL\Connection; use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; use Doctrine\DBAL\Types\Types; @@ -77,11 +76,36 @@ public function append(array $spans): void } } - public function markCompletedIfDone(string $traceId): bool + public function markCompletedIfDone(string $traceId, DatePoint $now): bool { $connection = $this->connection(); - return $this->onTables(fn (): bool => TraceLedger::progressOf($traceId, $this->spansOf($connection, $traceId))->isCompleted() && $this->markCompleted($connection, $traceId)); + return $this->onTables(fn (): bool => TraceLedger::progressOf($traceId, $this->spansOf($connection, $traceId))->isCompleted() && $this->markCompleted($connection, $traceId, $now)); + } + + public function markNotified(string $traceId): void + { + $connection = $this->connection(); + $this->onTables(fn (): int => $connection->executeStatement( + "UPDATE {$this->tracesTableManager->getTableName()} SET notified_at = :notifiedAt WHERE trace_id = :traceId AND completed_at IS NOT NULL AND notified_at IS NULL", + ['notifiedAt' => $this->clock->now()->unixTime()->inMicroseconds(), 'traceId' => $traceId], + ['notifiedAt' => Types::BIGINT], + )); + } + + public function completedButNotNotifiedBefore(DatePoint $completedBefore, int $limit): array + { + return $this->onTables(fn (): array => $this->connection()->createQueryBuilder() + ->select('trace_id') + ->from($this->tracesTableManager->getTableName()) + ->andWhere('completed_at IS NOT NULL') + ->andWhere('notified_at IS NULL') + ->andWhere('completed_at < :completedBefore') + ->setParameter('completedBefore', $completedBefore->unixTime()->inMicroseconds(), Types::BIGINT) + ->orderBy('completed_at') + ->setMaxResults($limit) + ->executeQuery() + ->fetchFirstColumn()); } public function progressOf(string $traceId): TraceProgress @@ -89,28 +113,15 @@ public function progressOf(string $traceId): TraceProgress return $this->onTables(fn (): TraceProgress => TraceLedger::progressOf($traceId, $this->spansOf($this->connection(), $traceId))); } - public function removeTracesStartedBefore(DatePoint $threshold): void + public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore): void { $connection = $this->connection(); - $this->onTables(function () use ($connection, $threshold): void { - $traceIds = $connection->createQueryBuilder() - ->select('trace_id') - ->from($this->tracesTableManager->getTableName()) - ->andWhere('started_at < :threshold') - ->setParameter('threshold', $threshold->unixTime()->inMicroseconds(), Types::BIGINT) - ->executeQuery() - ->fetchFirstColumn(); - if ($traceIds === []) { - return; - } - - foreach ([$this->spansTableManager->getTableName(), $this->tracesTableManager->getTableName()] as $tableName) { - $connection->createQueryBuilder() - ->delete($tableName) - ->andWhere('trace_id IN (:traceIds)') - ->setParameter('traceIds', $traceIds, ArrayParameterType::STRING) - ->executeStatement(); - } + $this->onTables(function () use ($connection, $completedTracesStartedBefore, $anyTraceStartedBefore): void { + $expired = "SELECT trace_id FROM {$this->tracesTableManager->getTableName()} WHERE started_at < :anyStartedBefore OR (completed_at IS NOT NULL AND started_at < :completedStartedBefore)"; + $parameters = ['anyStartedBefore' => $anyTraceStartedBefore->unixTime()->inMicroseconds(), 'completedStartedBefore' => $completedTracesStartedBefore->unixTime()->inMicroseconds()]; + $types = ['anyStartedBefore' => Types::BIGINT, 'completedStartedBefore' => Types::BIGINT]; + $connection->executeStatement("DELETE FROM {$this->spansTableManager->getTableName()} WHERE trace_id IN ({$expired})", $parameters, $types); + $connection->executeStatement("DELETE FROM {$this->tracesTableManager->getTableName()} WHERE started_at < :anyStartedBefore OR (completed_at IS NOT NULL AND started_at < :completedStartedBefore)", $parameters, $types); }); } @@ -122,24 +133,24 @@ private function claimTrace(Connection $connection, string $traceId, array $trac $startedAt = min(array_map(fn (Span $span): int => $span->startedAt->unixTime()->inMicroseconds(), $traceSpans)); $tableName = $this->tracesTableManager->getTableName(); $sql = $connection->getDatabasePlatform() instanceof AbstractMySQLPlatform - ? "INSERT INTO {$tableName} (trace_id, started_at, completed_at) VALUES (:traceId, :startedAt, NULL) ON DUPLICATE KEY UPDATE trace_id = trace_id" - : "INSERT INTO {$tableName} (trace_id, started_at, completed_at) VALUES (:traceId, :startedAt, NULL) ON CONFLICT (trace_id) DO NOTHING"; + ? "INSERT INTO {$tableName} (trace_id, started_at, completed_at, notified_at) VALUES (:traceId, :startedAt, NULL, NULL) ON DUPLICATE KEY UPDATE completed_at = NULL, notified_at = NULL" + : "INSERT INTO {$tableName} (trace_id, started_at, completed_at, notified_at) VALUES (:traceId, :startedAt, NULL, NULL) ON CONFLICT (trace_id) DO UPDATE SET completed_at = NULL, notified_at = NULL"; $connection->executeStatement($sql, ['traceId' => $traceId, 'startedAt' => $startedAt], ['startedAt' => Types::BIGINT]); } - private function markCompleted(Connection $connection, string $traceId): bool + private function markCompleted(Connection $connection, string $traceId, DatePoint $now): bool { return $connection->executeStatement( "UPDATE {$this->tracesTableManager->getTableName()} SET completed_at = :completedAt WHERE trace_id = :traceId AND completed_at IS NULL", - ['completedAt' => $this->clock->now()->unixTime()->inMicroseconds(), 'traceId' => $traceId], + ['completedAt' => $now->unixTime()->inMicroseconds(), 'traceId' => $traceId], ['completedAt' => Types::BIGINT], ) === 1; } private function insertSpan(Connection $connection, Span $span): void { - $connection->insert($this->spansTableManager->getTableName(), [ + $columns = [ 'span_id' => $span->spanId, 'trace_id' => $span->traceId, 'parent_span_id' => $span->parentSpanId, @@ -155,10 +166,12 @@ private function insertSpan(Connection $connection, Span $span): void ], $span->events), JSON_THROW_ON_ERROR), 'started_at' => $span->startedAt->unixTime()->inMicroseconds(), 'ended_at' => $span->endedAt->unixTime()->inMicroseconds(), - ], [ - 'started_at' => Types::BIGINT, - 'ended_at' => Types::BIGINT, - ]); + ]; + $placeholders = implode(', ', array_map(fn (string $column): string => ':' . $column, array_keys($columns))); + $sql = sprintf('INSERT INTO %s (%s) VALUES (%s)', $this->spansTableManager->getTableName(), implode(', ', array_keys($columns)), $placeholders); + $sql .= $connection->getDatabasePlatform() instanceof AbstractMySQLPlatform ? ' ON DUPLICATE KEY UPDATE span_id = span_id' : ' ON CONFLICT (trace_id, span_id) DO NOTHING'; + + $connection->executeStatement($sql, $columns, ['started_at' => Types::BIGINT, 'ended_at' => Types::BIGINT]); } /** diff --git a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php index 22c28e326..014754018 100644 --- a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php +++ b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php @@ -325,6 +325,91 @@ public function notify(string $userId): void self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } + #[DataProvider('implementations')] + public function test_a_pending_trace_older_than_the_retention_survives_until_twice_the_retention(string $implementation): void + { + $ecotone = $this->bootstrap($implementation, [$this->users(), $this->asynchronousNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')], Duration::minutes(10)); + $delayed = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $delayed->toTraceparent()]); + + $ecotone->advanceTimeBy(TimeSpan::withMinutes(11)); + $ecotone->runConsoleCommand('ecotone:tracing:remove-expired-traces'); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + self::assertTrue($this->completion($ecotone)->isCompleted($delayed->traceId)); + + $abandoned = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '2', metadata: [MessageHeaders::TRACEPARENT => $abandoned->toTraceparent()]); + $ecotone->advanceTimeBy(TimeSpan::withMinutes(21)); + $ecotone->runConsoleCommand('ecotone:tracing:remove-expired-traces'); + + self::assertFalse($this->completion($ecotone)->isCompletedFor($abandoned->traceId, 'user.registration')); + } + + #[DataProvider('implementations')] + public function test_a_subscriber_that_throws_does_not_fail_the_committed_command_and_is_notified_again_later(string $implementation): void + { + $subscriber = new class () { + /** @var string[] */ + public array $completedTraces = []; + + #[OnTraceCompleted] + public function whenCompleted(TraceCompleted $event): void + { + $this->completedTraces[] = $event->traceId; + if (count($this->completedTraces) === 1) { + throw new RuntimeException('Websocket is down'); + } + } + }; + $ecotone = $this->bootstrap($implementation, [$this->users(), $subscriber]); + $first = SpanContext::newTrace(); + $second = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $first->toTraceparent()]); + self::assertSame([$first->traceId], $subscriber->completedTraces); + + $ecotone->advanceTimeBy(TimeSpan::withSeconds(31)); + $ecotone->sendCommandWithRouting('user.register', '2', metadata: [MessageHeaders::TRACEPARENT => $second->toTraceparent()]); + + self::assertEqualsCanonicalizing([$first->traceId, $first->traceId, $second->traceId], $subscriber->completedTraces); + } + + #[DataProvider('implementations')] + public function test_a_trace_that_receives_new_work_after_completing_is_notified_again_when_it_completes_again(string $implementation): void + { + $subscriber = new class () { + /** @var string[] */ + public array $completedTraces = []; + + #[OnTraceCompleted] + public function whenCompleted(TraceCompleted $event): void + { + $this->completedTraces[] = $event->traceId; + } + }; + $orders = new class () { + #[Asynchronous('async_channel')] + #[CommandHandler('order.place', endpointId: 'order.placing')] + public function place(string $orderId): void + { + } + }; + $ecotone = $this->bootstrap($implementation, [$this->users(), $orders, $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + self::assertSame([$trace->traceId], $subscriber->completedTraces); + + $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertSame([$trace->traceId], $subscriber->completedTraces); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertSame([$trace->traceId, $trace->traceId], $subscriber->completedTraces); + } + public function test_completion_recorded_by_one_instance_is_visible_to_another_sharing_the_database(): void { $producer = $this->bootstrap('dbal', [$this->users()]); diff --git a/packages/Dbal/tests/Conformance/TraceStoreContractTest.php b/packages/Dbal/tests/Conformance/TraceStoreContractTest.php new file mode 100644 index 000000000..28e72c683 --- /dev/null +++ b/packages/Dbal/tests/Conformance/TraceStoreContractTest.php @@ -0,0 +1,101 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + * + * @internal + */ +final class TraceStoreContractTest extends DbalMessagingTestCase +{ + public static function implementations(): iterable + { + yield 'in-memory' => ['in-memory']; + yield 'dbal' => ['dbal']; + } + + #[DataProvider('implementations')] + public function test_appending_the_same_span_twice_records_it_once(string $implementation): void + { + $store = $this->store($implementation); + $trace = SpanContext::newTrace(); + $root = $this->rootSpan($trace); + + $store->append([$root]); + $store->append([$root]); + + self::assertTrue($store->progressOf($trace->traceId)->isCompleted()); + self::assertTrue($store->markCompletedIfDone($trace->traceId, new DatePoint())); + self::assertFalse($store->markCompletedIfDone($trace->traceId, new DatePoint())); + } + + #[DataProvider('implementations')] + public function test_a_span_appended_after_completion_reopens_the_trace_so_it_can_complete_and_notify_again(string $implementation): void + { + $store = $this->store($implementation); + $trace = SpanContext::newTrace(); + $store->append([$this->rootSpan($trace)]); + self::assertTrue($store->markCompletedIfDone($trace->traceId, new DatePoint())); + $store->markNotified($trace->traceId); + + $store->append([$this->rootSpan($trace)]); + + self::assertTrue($store->markCompletedIfDone($trace->traceId, new DatePoint())); + self::assertSame([$trace->traceId], $store->completedButNotNotifiedBefore((new DatePoint())->modify('+1 minute'), 10)); + } + + private function store(string $implementation): TraceStore + { + if ($implementation === 'in-memory') { + return new InMemoryTraceStore(); + } + $traces = new TracesTableManager(TracesTableManager::DEFAULT_TABLE, true); + $spans = new SpansTableManager(SpansTableManager::DEFAULT_TABLE, true); + $connection = $this->getConnection(); + foreach ([$traces, $spans] as $tableManager) { + if (! $tableManager->isInitialized($connection)) { + $tableManager->createTable($connection); + } + } + + return new DbalTraceStore($this->getConnectionFactory(), $traces, $spans, new NativeClock()); + } + + private function rootSpan(SpanContext $trace): Span + { + return new Span($trace->traceId, $trace->newChild()->spanId, null, 'send command', SpanKind::Server, [SpanAttribute::TRACE_ROOT => true], SpanStatus::Ok, null, [], new DatePoint(), new DatePoint()); + } +} diff --git a/packages/Ecotone/Api/Tracing/CurrentTrace.php b/packages/Ecotone/Api/Tracing/CurrentTrace.php new file mode 100644 index 000000000..8ad7ad1c5 --- /dev/null +++ b/packages/Ecotone/Api/Tracing/CurrentTrace.php @@ -0,0 +1,13 @@ + $endpointIdsByHandler + * @param string[] $pollingProjectionNames */ private function __construct( private array $traceCompletedSubscribers, private array $spanEndedSubscribers, private array $endpointIdsByHandler, + private array $pollingProjectionNames, ) { } @@ -97,9 +104,25 @@ public static function create(AnnotationFinder $annotationRegistrationService, I self::placementsOf($annotationRegistrationService, OnTraceCompleted::class), self::placementsOf($annotationRegistrationService, OnSpanEnded::class), self::endpointIdsByHandler($annotationRegistrationService), + self::pollingProjectionNames($annotationRegistrationService), ); } + /** + * @return string[] + */ + private static function pollingProjectionNames(AnnotationFinder $annotationFinder): array + { + $names = []; + foreach ($annotationFinder->findAnnotatedClasses(Projection::class) as $projectionClass) { + if ($annotationFinder->findAttributeForClass($projectionClass, Polling::class) !== null) { + $names[] = $annotationFinder->getAttributeForClass($projectionClass, Projection::class)->name; + } + } + + return $names; + } + private static function registerCompletionGateway(Configuration $messagingConfiguration, string $serviceClass, Definition $service): void { $messagingConfiguration->registerServiceDefinition($serviceClass, $service); @@ -144,8 +167,11 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO self::registerCompletionGateway($messagingConfiguration, UnavailableTraceCompletion::class, new Definition(UnavailableTraceCompletion::class)); } if ($tracingConfiguration->spanRecorderReferenceNames === [] && $tracerReferences === [] && ! $tracingConfiguration->storedTraces) { + $messagingConfiguration->registerServiceDefinition(CurrentTrace::class, new Definition(TracingDisabledCurrentTrace::class)); + return; } + $messagingConfiguration->registerServiceDefinition(CurrentTrace::class, new Definition(TracerCurrentTrace::class, [new Reference(Tracer::class)])); $spanRecorders = array_map(fn (string $referenceName): Reference => new Reference($referenceName), $tracingConfiguration->spanRecorderReferenceNames); if ($tracingConfiguration->storedTraces) { @@ -260,6 +286,7 @@ private function registerStoredTraces(Configuration $messagingConfiguration, arr $this->spanEndedSubscribers !== [], $this->traceCompletedSubscribers !== [], $tracingConfiguration->storedFailuresOutsideTransaction, + new Reference(EcotoneClockInterface::class), ]) ); $messagingConfiguration->registerServiceDefinition( @@ -277,7 +304,7 @@ private function registerStoredTraces(Configuration $messagingConfiguration, arr $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnTraceCompleted::CHANNEL)); $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnSpanEnded::CHANNEL)); - self::registerCompletionGateway($messagingConfiguration, StoredTraceCompletion::class, new Definition(StoredTraceCompletion::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class)])); + self::registerCompletionGateway($messagingConfiguration, StoredTraceCompletion::class, new Definition(StoredTraceCompletion::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $this->pollingProjectionNames])); $messagingConfiguration->registerServiceDefinition( ExpiredTraceRemover::class, diff --git a/packages/Ecotone/src/Tracing/ExpiredTraceRemover.php b/packages/Ecotone/src/Tracing/ExpiredTraceRemover.php index 8c1acdc65..b094180eb 100644 --- a/packages/Ecotone/src/Tracing/ExpiredTraceRemover.php +++ b/packages/Ecotone/src/Tracing/ExpiredTraceRemover.php @@ -36,6 +36,7 @@ public function __construct( public function removeExpiredTraces(): void { - $this->traceStore->removeTracesStartedBefore($this->clock->now()->sub(Duration::milliseconds($this->retentionInMilliseconds))); + $now = $this->clock->now(); + $this->traceStore->removeExpiredTraces($now->sub(Duration::milliseconds($this->retentionInMilliseconds)), $now->sub(Duration::milliseconds(2 * $this->retentionInMilliseconds))); } } diff --git a/packages/Ecotone/src/Tracing/InMemoryTraceStore.php b/packages/Ecotone/src/Tracing/InMemoryTraceStore.php index 7296ac53c..f310830da 100644 --- a/packages/Ecotone/src/Tracing/InMemoryTraceStore.php +++ b/packages/Ecotone/src/Tracing/InMemoryTraceStore.php @@ -27,41 +27,67 @@ */ final class InMemoryTraceStore implements TraceStore { - /** @var array */ + /** @var array> */ private array $spansByTrace = []; + /** @var array */ + private array $completedAt = []; /** @var array */ - private array $completedTraces = []; + private array $notified = []; public function append(array $spans): void { foreach ($spans as $span) { - $this->spansByTrace[$span->traceId][] = $span; + if (isset($this->spansByTrace[$span->traceId][$span->spanId])) { + continue; + } + $this->spansByTrace[$span->traceId][$span->spanId] = $span; + unset($this->completedAt[$span->traceId], $this->notified[$span->traceId]); } } - public function markCompletedIfDone(string $traceId): bool + public function markCompletedIfDone(string $traceId, DatePoint $now): bool { - if (isset($this->completedTraces[$traceId]) || ! $this->progressOf($traceId)->isCompleted()) { + if (isset($this->completedAt[$traceId]) || ! $this->progressOf($traceId)->isCompleted()) { return false; } - $this->completedTraces[$traceId] = true; + $this->completedAt[$traceId] = $now; return true; } + public function markNotified(string $traceId): void + { + if (isset($this->completedAt[$traceId])) { + $this->notified[$traceId] = true; + } + } + + public function completedButNotNotifiedBefore(DatePoint $completedBefore, int $limit): array + { + $traceIds = []; + foreach ($this->completedAt as $traceId => $completedAt) { + if (count($traceIds) === $limit) { + break; + } + if (! isset($this->notified[$traceId]) && $completedAt < $completedBefore) { + $traceIds[] = $traceId; + } + } + + return $traceIds; + } + public function progressOf(string $traceId): TraceProgress { - return TraceLedger::progressOf($traceId, $this->spansByTrace[$traceId] ?? []); + return TraceLedger::progressOf($traceId, array_values($this->spansByTrace[$traceId] ?? [])); } - public function removeTracesStartedBefore(DatePoint $threshold): void + public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore): void { foreach ($this->spansByTrace as $traceId => $spans) { - foreach ($spans as $span) { - if ($span->startedAt < $threshold) { - unset($this->spansByTrace[$traceId], $this->completedTraces[$traceId]); - continue 2; - } + $startedAt = min(array_map(fn (Span $span): DatePoint => $span->startedAt, $spans)); + if ($startedAt < $anyTraceStartedBefore || (isset($this->completedAt[$traceId]) && $startedAt < $completedTracesStartedBefore)) { + unset($this->spansByTrace[$traceId], $this->completedAt[$traceId], $this->notified[$traceId]); } } } diff --git a/packages/Ecotone/src/Tracing/StoredTraceCompletion.php b/packages/Ecotone/src/Tracing/StoredTraceCompletion.php index f13444df7..374bffc12 100644 --- a/packages/Ecotone/src/Tracing/StoredTraceCompletion.php +++ b/packages/Ecotone/src/Tracing/StoredTraceCompletion.php @@ -8,6 +8,7 @@ use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Scheduling\Duration; use Ecotone\Api\Tracing\TraceCompletionTimedOut; +use Ecotone\Messaging\Config\ConfigurationException; /** * licence Enterprise @@ -33,9 +34,13 @@ final class StoredTraceCompletion public const TIMEOUT_HEADER = 'ecotone_tracing_completion_timeout'; private const POLL_INTERVAL_IN_MILLISECONDS = 50; + /** + * @param string[] $pollingProjectionNames + */ public function __construct( private TraceStore $traceStore, private EcotoneClockInterface $clock, + private array $pollingProjectionNames, ) { } @@ -46,6 +51,8 @@ public function isCompleted(string $traceId): bool public function isCompletedFor(string $traceId, #[Header(self::DESTINATION_HEADER)] string $destination): bool { + $this->refusePollingProjection($destination); + return $this->traceStore->progressOf($traceId)->isCompletedFor($destination); } @@ -68,6 +75,7 @@ public function await(string $traceId, #[Header(self::TIMEOUT_HEADER)] Duration public function awaitFor(string $traceId, #[Header(self::DESTINATION_HEADER)] string $destination, #[Header(self::TIMEOUT_HEADER)] Duration $timeout): void { + $this->refusePollingProjection($destination); $deadline = $this->clock->now()->add($timeout); while (true) { $progress = $this->traceStore->progressOf($traceId); @@ -84,4 +92,11 @@ public function awaitFor(string $traceId, #[Header(self::DESTINATION_HEADER)] st $this->clock->sleep(Duration::milliseconds(self::POLL_INTERVAL_IN_MILLISECONDS)); } } + + private function refusePollingProjection(string $destination): void + { + if (in_array($destination, $this->pollingProjectionNames, true)) { + throw ConfigurationException::create(sprintf('Projection %s runs by #[Polling], so it is not part of the trace that appended its events and TraceCompletion cannot await it. Await the handler that published the event, or make the projection event-driven by removing #[Polling].', $destination)); + } + } } diff --git a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php index b99326491..0fca6a0fe 100644 --- a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php +++ b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php @@ -4,6 +4,8 @@ namespace Ecotone\Tracing; +use Ecotone\Api\Gateway\EcotoneClockInterface; +use Ecotone\Api\Scheduling\Duration; use Ecotone\Api\Tracing\OnSpanEnded; use Ecotone\Api\Tracing\OnTraceCompleted; use Ecotone\Api\Tracing\Span; @@ -12,6 +14,7 @@ use Ecotone\Api\Tracing\SpanStatus; use Ecotone\Api\Tracing\TraceCompleted; use Ecotone\Messaging\Gateway\MessagingEntrypointService; +use Throwable; /** * licence Enterprise @@ -40,6 +43,8 @@ final class StoringSpanRecorder implements SpanRecorder private array $endedSpansToPublish = []; /** @var string[] */ private array $completedTracesToPublish = []; + private const NOTIFICATION_RETRY_DELAY_IN_SECONDS = 30; + private const NOTIFICATION_RETRY_BATCH = 10; private bool $publishing = false; /** @var array */ private array $spansWrappingAFailedUnitOfWork = []; @@ -50,6 +55,7 @@ public function __construct( private bool $publishesEndedSpans, private bool $publishesCompletedTraces, private bool $storesFailuresOutsideTransaction, + private EcotoneClockInterface $clock, ) { } @@ -68,8 +74,9 @@ public function record(Span $span): void return; } - $this->write([...$this->buffer, $span], true); + $spans = [...$this->buffer, $span]; $this->buffer = []; + $this->write($spans, true); $this->publish(); } @@ -88,8 +95,9 @@ public function flush(): void foreach (self::outerParentsOf($this->buffer) as $spanId) { unset($this->spansWrappingAFailedUnitOfWork[$spanId]); } - $this->write($this->buffer, false); + $spans = $this->buffer; $this->buffer = []; + $this->write($spans, false); } public function closeScope(): void @@ -178,7 +186,7 @@ private function completedAmong(array $spans): array { $completed = []; foreach (array_unique(array_map(fn (Span $span): string => $span->traceId, $spans)) as $traceId) { - if ($this->traceStore->markCompletedIfDone($traceId)) { + if ($this->traceStore->markCompletedIfDone($traceId, $this->clock->now())) { $completed[] = $traceId; } } @@ -201,11 +209,24 @@ private function publish(): void foreach ($endedSpans as $span) { $this->messagingEntrypoint->send(new SpanEnded($span), OnSpanEnded::CHANNEL); } - foreach ($completedTraces as $traceId) { - $this->messagingEntrypoint->send(new TraceCompleted($traceId), OnTraceCompleted::CHANNEL); + if ($this->publishesCompletedTraces) { + $completedTraces = [...$completedTraces, ...$this->traceStore->completedButNotNotifiedBefore($this->clock->now()->sub(Duration::seconds(self::NOTIFICATION_RETRY_DELAY_IN_SECONDS)), self::NOTIFICATION_RETRY_BATCH)]; + } + foreach (array_unique($completedTraces) as $traceId) { + $this->notify($traceId); } } finally { $this->publishing = false; } } + + private function notify(string $traceId): void + { + try { + $this->messagingEntrypoint->send(new TraceCompleted($traceId), OnTraceCompleted::CHANNEL); + } catch (Throwable) { + return; + } + $this->traceStore->markNotified($traceId); + } } diff --git a/packages/Ecotone/src/Tracing/TraceStore.php b/packages/Ecotone/src/Tracing/TraceStore.php index 6de1a4700..bf4f3dd64 100644 --- a/packages/Ecotone/src/Tracing/TraceStore.php +++ b/packages/Ecotone/src/Tracing/TraceStore.php @@ -32,9 +32,16 @@ interface TraceStore */ public function append(array $spans): void; - public function markCompletedIfDone(string $traceId): bool; + public function markCompletedIfDone(string $traceId, DatePoint $now): bool; + + public function markNotified(string $traceId): void; + + /** + * @return string[] + */ + public function completedButNotNotifiedBefore(DatePoint $completedBefore, int $limit): array; public function progressOf(string $traceId): TraceProgress; - public function removeTracesStartedBefore(DatePoint $threshold): void; + public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore): void; } diff --git a/packages/Ecotone/src/Tracing/TracerCurrentTrace.php b/packages/Ecotone/src/Tracing/TracerCurrentTrace.php new file mode 100644 index 000000000..c25c143e3 --- /dev/null +++ b/packages/Ecotone/src/Tracing/TracerCurrentTrace.php @@ -0,0 +1,22 @@ +tracer->currentSpan()?->context()->traceId; + } +} diff --git a/packages/Ecotone/src/Tracing/TracingDisabledCurrentTrace.php b/packages/Ecotone/src/Tracing/TracingDisabledCurrentTrace.php new file mode 100644 index 000000000..4f0e3d694 --- /dev/null +++ b/packages/Ecotone/src/Tracing/TracingDisabledCurrentTrace.php @@ -0,0 +1,19 @@ +withSpanRecorder() or ->withStoredTraces() from a #[ServiceContext] method, or install ecotone/open-telemetry with a TracerProviderInterface.'); + } +} diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index 62cc3deeb..03da0cae9 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -10,6 +10,8 @@ use Ecotone\Api\Attribute\Headers; use Ecotone\Api\Attribute\InternalHandler; use Ecotone\Api\Attribute\MessageGateway; +use Ecotone\Api\Attribute\Polling; +use Ecotone\Api\Attribute\Reference; use Ecotone\Api\EventSourcing\Event; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -23,6 +25,7 @@ use Ecotone\Api\Projecting\FromStream; use Ecotone\Api\Projecting\Projection; use Ecotone\Api\Scheduling\Duration; +use Ecotone\Api\Tracing\CurrentTrace; use Ecotone\Api\Tracing\OnSpanEnded; use Ecotone\Api\Tracing\OnTraceCompleted; use Ecotone\Api\Tracing\SpanContext; @@ -587,6 +590,59 @@ public function place(string $orderId): void $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); } + public function test_awaiting_a_polling_projection_is_refused_because_it_runs_outside_the_trace(): void + { + $projection = new #[Projection('orders_list'), FromStream('orders_stream'), Polling('orders_list_polling')] class () { + #[EventHandler('order.placed')] + public function whenPlaced(array $event): void + { + } + }; + $ecotone = $this->bootstrap([$projection]); + + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage("Projection orders_list runs by #[Polling], so it is not part of the trace that appended its events and TraceCompletion cannot await it. Await the handler that published the event, or make the projection event-driven by removing #[Polling]."); + + $this->completion($ecotone)->awaitFor('0af7651916cd43dd8448eb211c80319c', 'orders_list', Duration::milliseconds(10)); + } + + public function test_a_handler_can_read_the_trace_id_it_runs_in(): void + { + $users = new class () { + public ?string $traceId = null; + + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId, #[Reference] CurrentTrace $currentTrace): void + { + $this->traceId = $currentTrace->traceId(); + } + }; + $ecotone = $this->bootstrap([$users]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->assertSame($trace->traceId, $users->traceId); + $this->assertTrue($this->completion($ecotone)->isCompleted($users->traceId)); + } + + public function test_reading_the_current_trace_without_tracing_names_how_to_enable_it(): void + { + $users = new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId, #[Reference] CurrentTrace $currentTrace): void + { + $currentTrace->traceId(); + } + }; + $ecotone = EcotoneLite::bootstrapFlowTesting([$users::class], [$users]); + + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage('No trace is recorded, so there is no current trace id. Enable tracing: register TracingConfiguration::createWithDefaults()->withSpanRecorder() or ->withStoredTraces() from a #[ServiceContext] method, or install ecotone/open-telemetry with a TracerProviderInterface.'); + + $ecotone->sendCommandWithRouting('user.register', '1'); + } + public function test_a_failed_unit_of_work_stores_nothing_by_default(): void { $subscriber = $this->spanEndedSubscriber(); diff --git a/upgrade-2.0.md b/upgrade-2.0.md index 3c1308897..9a42ec1f7 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3122,14 +3122,22 @@ method, so it records the handler's own failure even when an error channel or a `TraceCompletionTimedOut` names the handlers whose last attempt failed, the channels still holding a message, a trace nobody recorded, and a destination the trace never reached. `TraceCompletion` is always injectable; without `withStoredTraces()` every call refuses with the configuration it needs. A method with `#[OnTraceCompleted]` receives a -`TraceCompleted` once per trace, and one with `#[OnSpanEnded]` receives every stored span, both after the commit -that produced them, so a websocket push or a cache refresh never runs ahead of the data. Work a subscriber does +`TraceCompleted` each time the trace reaches completion, so a trace that is handed more work after completing, as +when a request sends two commands under one `traceparent`, notifies again once that work is done; one with +`#[OnSpanEnded]` receives every stored span. Both run after the commit that produced them, so a websocket push or +a cache refresh never runs ahead of the data. A completion notification is durable: the store remembers it was +delivered, a subscriber that throws does not fail the command that already committed, and any later unit of work +in any process re-delivers a notification left undelivered for thirty seconds. A handler that needs the id of the +trace it runs in injects `Ecotone\Api\Tracing\CurrentTrace` and reads `traceId()`. `isCompletedFor()` and +`awaitFor()` refuse a `#[Polling]` projection by name, because a polled run is not part of the trace that appended +its events. Work a subscriber does through a bus is stored like any other trace but never notifies subscribers in turn, so a subscriber that sends a command does not call itself again. The trace id is the one in the `traceparent` header: mint it with `SpanContext::newTrace()` and send it with the command, or take it from the HTTP request the command came from. A flow test given no connection keeps the store in memory; an application that enables stored traces with no DBAL package and no `inMemory: true` is refused -at bootstrap, because a store per process could never see a consumer's progress. `ecotone:tracing:remove-expired-traces` deletes traces older than the retention passed to -`withStoredTraces()`, seven days by default. +at bootstrap, because a store per process could never see a consumer's progress. `ecotone:tracing:remove-expired-traces` deletes completed traces older than the retention passed to +`withStoredTraces()`, seven days by default, and any trace older than twice that, so a message delayed past the +retention still completes the trace it belongs to; the deletion runs as two statements, whatever the volume. ```php $trace = SpanContext::newTrace(); From 8e031c5f268039205d9f52cef1bde5b66a45cfdf Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 20/50] fix(tracing): a delivery keeps its identity through dynamic channels, the outbox and dead-letter replay, a no-op OpenTelemetry provider gets its own span ids, and multi-tenant stored traces use the default connection The first channel a message passes stamps ecotone_tracing_delivery with its name and the message id, so the inner channel of a dynamic channel and a redelivery do not count as deliveries nobody consumes, and the consumer pairs the message with the channel it was first sent to. A high-throughput batch envelope lists the deliveries of its entries, and every entry receives the envelope's traceparent, so the consumer continues the producer's trace instead of starting its own. An OpenTelemetry span that is not recording, as the no-op provider returns, no longer lends its ancestor's ids to every stored span. A multi-tenant application stores traces on the configured default connection, since the write after the unit of work has no tenant, and is refused at bootstrap when there is none. --- .../Dbal/src/Tracing/DbalTracingModule.php | 20 ++++- .../Conformance/TraceStoreConformanceTest.php | 56 ++++++++++++ .../Ecotone/Api/Messaging/MessageHeaders.php | 1 + .../Ecotone/Api/Tracing/SpanAttribute.php | 1 + .../Messaging/Support/FrameworkHeaders.php | 1 + packages/Ecotone/src/Tracing/ActiveSpan.php | 5 ++ .../Ecotone/src/Tracing/EcotoneActiveSpan.php | 5 ++ packages/Ecotone/src/Tracing/TraceLedger.php | 11 +++ .../src/Tracing/TracingChannelInterceptor.php | 71 +++++++++++++-- .../src/Tracing/TracingInterceptor.php | 2 +- .../tests/Tracing/TraceCompletionTest.php | 30 ++++++- .../src/OpenTelemetryActiveSpan.php | 5 ++ .../OpenTelemetry/src/OpenTelemetryTracer.php | 6 +- .../StoredTracesWithNoopProviderTest.php | 89 +++++++++++++++++++ upgrade-2.0.md | 5 +- 15 files changed, 294 insertions(+), 14 deletions(-) create mode 100644 packages/OpenTelemetry/tests/Integration/StoredTracesWithNoopProviderTest.php diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index 35b6cda7e..00b09edb0 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -8,6 +8,8 @@ use Ecotone\Api\Attribute\ModuleAnnotation; use Ecotone\Api\Dbal\ExtensionObject\DbalConfiguration; use Ecotone\Api\Dbal\ExtensionObject\DbalConnectionReference; +use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; +use Ecotone\Api\ExtensionObject\ConnectionReference; use Ecotone\Api\ExtensionObject\ModulePackageList; use Ecotone\Api\ExtensionObject\ServiceConfiguration; use Ecotone\Api\Gateway\EcotoneClockInterface; @@ -20,6 +22,7 @@ use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\NoExternalConfigurationModule; use Ecotone\Messaging\Config\BootstrapConfiguration; use Ecotone\Messaging\Config\Configuration; +use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Config\ConsoleInvocationResolver; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\Reference; @@ -113,7 +116,22 @@ private static function storesOnDatabase(TracingConfiguration $tracingConfigurat private static function connectionReferenceOf(array $extensionObjects): string { $dbalConfiguration = ExtensionObjectResolver::resolveUnique(DbalConfiguration::class, $extensionObjects, DbalConfiguration::createWithDefaults()); + $connectionReference = $dbalConfiguration->defaultConnectionReferenceNames[0] ?? DbalConnectionReference::DEFAULT; + foreach (ExtensionObjectResolver::resolve(MultiTenantConfiguration::class, $extensionObjects) as $multiTenantConfiguration) { + if ($multiTenantConfiguration->getReferenceName() !== $connectionReference) { + continue; + } + if ($multiTenantConfiguration->defaultConnectionName === null) { + if (self::storesOnDatabase(ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $extensionObjects, TracingConfiguration::createWithDefaults()))) { + throw ConfigurationException::create('Stored traces are written after the unit of work ended, when no tenant is active, so they need a connection that does not depend on the tenant. Use MultiTenantConfiguration::createWithDefaultConnection() to name it, or withStoredTraces(inMemory: true) for a single process.'); + } - return $dbalConfiguration->defaultConnectionReferenceNames[0] ?? DbalConnectionReference::DEFAULT; + return $connectionReference; + } + + return $multiTenantConfiguration->defaultConnectionName instanceof ConnectionReference ? $multiTenantConfiguration->defaultConnectionName->getReferenceName() : $multiTenantConfiguration->defaultConnectionName; + } + + return $connectionReference; } } diff --git a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php index 014754018..4e8b34638 100644 --- a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php +++ b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php @@ -10,6 +10,7 @@ use Ecotone\Api\Dbal\DbalConnectionFactory; use Ecotone\Api\Dbal\ExtensionObject\DbalBackedMessageChannelBuilder; use Ecotone\Api\Dbal\ExtensionObject\DbalDeadLetterBuilder; +use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; use Ecotone\Api\Dbal\Gateway\DeadLetterGateway; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\InstantRetryConfiguration; @@ -30,6 +31,7 @@ use Ecotone\Api\Tracing\TraceCompletion; use Ecotone\Api\Tracing\TraceCompletionTimedOut; use Ecotone\Api\Tracing\TracingConfiguration; +use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Test\LicenceTesting; use PHPUnit\Framework\Attributes\DataProvider; use RuntimeException; @@ -410,6 +412,60 @@ public function place(string $orderId): void self::assertSame([$trace->traceId, $trace->traceId], $subscriber->completedTraces); } + public function test_a_channel_published_through_the_outbox_completes_once_its_consumer_ran(): void + { + $ecotone = $this->bootstrap('dbal', [$this->users(), $this->asynchronousNotifier()], [DbalBackedMessageChannelBuilder::create('async_channel')->withHighThroughputPublishing()]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); + self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + public function test_stored_traces_follow_the_default_connection_of_a_multi_tenant_application(): void + { + $ecotone = $this->bootstrapFlowTesting( + classesToResolve: [$this->users()::class, $this->asynchronousNotifier()::class], + containerOrAvailableServices: [$this->users(), $this->asynchronousNotifier(), 'tenant_a_connection' => $this->getConnectionFactory(), 'shared_connection' => $this->getConnectionFactory()], + extensionObjects: [ + TracingConfiguration::createWithDefaults()->withStoredTraces(), + MultiTenantConfiguration::createWithDefaultConnection('tenant', ['tenant_a' => 'tenant_a_connection'], 'shared_connection'), + SimpleMessageChannelBuilder::createQueueChannel('async_channel'), + ], + enabledModules: [ModulePackageList::DBAL_PACKAGE], + licenceKey: LicenceTesting::VALID_LICENCE, + )->setupDatabaseTables(); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent(), 'tenant' => 'tenant_a']); + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + public function test_stored_traces_in_a_multi_tenant_application_without_a_default_connection_are_refused_at_bootstrap(): void + { + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage('Stored traces are written after the unit of work ended, when no tenant is active, so they need a connection that does not depend on the tenant. Use MultiTenantConfiguration::createWithDefaultConnection() to name it, or withStoredTraces(inMemory: true) for a single process.'); + + $this->bootstrapFlowTesting( + classesToResolve: [$this->users()::class], + containerOrAvailableServices: [$this->users(), 'tenant_a_connection' => $this->getConnectionFactory()], + extensionObjects: [ + TracingConfiguration::createWithDefaults()->withStoredTraces(), + MultiTenantConfiguration::create('tenant', ['tenant_a' => 'tenant_a_connection']), + ], + enabledModules: [ModulePackageList::DBAL_PACKAGE], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + public function test_completion_recorded_by_one_instance_is_visible_to_another_sharing_the_database(): void { $producer = $this->bootstrap('dbal', [$this->users()]); diff --git a/packages/Ecotone/Api/Messaging/MessageHeaders.php b/packages/Ecotone/Api/Messaging/MessageHeaders.php index 438023db7..3bdc283eb 100644 --- a/packages/Ecotone/Api/Messaging/MessageHeaders.php +++ b/packages/Ecotone/Api/Messaging/MessageHeaders.php @@ -70,6 +70,7 @@ final class MessageHeaders public const TRACEPARENT = 'traceparent'; public const TRACESTATE = 'tracestate'; + public const TRACING_DELIVERY = 'ecotone_tracing_delivery'; public const EVENT_AGGREGATE_TYPE = '_aggregate_type'; diff --git a/packages/Ecotone/Api/Tracing/SpanAttribute.php b/packages/Ecotone/Api/Tracing/SpanAttribute.php index 0b557179d..edb9dea2e 100644 --- a/packages/Ecotone/Api/Tracing/SpanAttribute.php +++ b/packages/Ecotone/Api/Tracing/SpanAttribute.php @@ -17,6 +17,7 @@ final class SpanAttribute public const TRACE_ROOT = 'ecotone.trace.root'; public const CODE_FUNCTION_NAME = 'code.function.name'; public const CHANNEL_NAME = 'ecotone.channel.name'; + public const BATCH_DELIVERIES = 'ecotone.batch.deliveries'; public const ECOTONE = 'ecotone'; public const OPERATION_SEND = 'send'; diff --git a/packages/Ecotone/src/Messaging/Support/FrameworkHeaders.php b/packages/Ecotone/src/Messaging/Support/FrameworkHeaders.php index ebe380871..5f4e805fb 100644 --- a/packages/Ecotone/src/Messaging/Support/FrameworkHeaders.php +++ b/packages/Ecotone/src/Messaging/Support/FrameworkHeaders.php @@ -151,6 +151,7 @@ private static function getFrameworksHeaderNames(): array MessagingEntrypointService::ENTRYPOINT, MessageHeaders::TRACEPARENT, MessageHeaders::TRACESTATE, + MessageHeaders::TRACING_DELIVERY, ]; } } diff --git a/packages/Ecotone/src/Tracing/ActiveSpan.php b/packages/Ecotone/src/Tracing/ActiveSpan.php index aab782105..3d2af0123 100644 --- a/packages/Ecotone/src/Tracing/ActiveSpan.php +++ b/packages/Ecotone/src/Tracing/ActiveSpan.php @@ -14,6 +14,11 @@ interface ActiveSpan { public function context(): SpanContext; + /** + * @return array + */ + public function attributes(): array; + /** * @param array $attributes */ diff --git a/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php b/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php index b6488ce48..14361c04b 100644 --- a/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php +++ b/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php @@ -37,6 +37,11 @@ public function __construct( ) { } + public function attributes(): array + { + return $this->attributes; + } + public function context(): SpanContext { return $this->context; diff --git a/packages/Ecotone/src/Tracing/TraceLedger.php b/packages/Ecotone/src/Tracing/TraceLedger.php index 87673c4b4..118a322b3 100644 --- a/packages/Ecotone/src/Tracing/TraceLedger.php +++ b/packages/Ecotone/src/Tracing/TraceLedger.php @@ -49,6 +49,17 @@ public static function progressOf(string $traceId, array $spans): TraceProgress if ($span->attributes[SpanAttribute::TRACE_ROOT] ?? false) { $rootEnded = true; } + if (isset($span->attributes[SpanAttribute::BATCH_DELIVERIES], $span->attributes[SpanAttribute::CHANNEL_NAME]) && $span->kind === SpanKind::Producer && $span->status !== SpanStatus::Error) { + $channel = (string) $span->attributes[SpanAttribute::CHANNEL_NAME]; + foreach (json_decode((string) $span->attributes[SpanAttribute::BATCH_DELIVERIES], true, flags: JSON_THROW_ON_ERROR) as $messageId => $endpointId) { + $sentDeliveries[$channel][(string) $messageId] = true; + if ($endpointId !== null) { + $sent[(string) $endpointId][(string) $messageId] = true; + } + } + + continue; + } $delivery = self::deliveryOf($span); if ($delivery !== null) { [$channel, $messageId] = $delivery; diff --git a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php index d4c147392..4d6697e4a 100644 --- a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php @@ -6,6 +6,7 @@ use function array_pop; +use Ecotone\Api\Messaging\BatchMessage; use Ecotone\Api\Messaging\Message; use Ecotone\Api\Messaging\MessageChannel; use Ecotone\Api\Messaging\MessageHeaders; @@ -39,11 +40,10 @@ public function preSend(Message $message, MessageChannel $messageChannel): ?Mess ...$this->messageSpanAttributes->from($message), SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND, SpanAttribute::MESSAGING_DESTINATION_NAME => $this->channelName, - SpanAttribute::CHANNEL_NAME => $this->channelName, ]; - $targetEndpointId = self::targetEndpointIdOf($message); - if ($targetEndpointId !== null) { - $attributes[SpanAttribute::ENDPOINT_ID] = $targetEndpointId; + $isFirstChannel = self::deliveryChannelOf($message) === null; + if ($isFirstChannel) { + $attributes = [...$attributes, ...$this->deliveryAttributes($message)]; } $span = $this->tracer->startSpan( 'send ' . $this->channelName, @@ -53,18 +53,71 @@ public function preSend(Message $message, MessageChannel $messageChannel): ?Mess ); $this->openSendSpans[] = $span; - return $this->messageBuilder->fromMessage($message) - ->setHeader(MessageHeaders::TRACEPARENT, $span->context()->toTraceparent()) - ->build(); + $traceparent = $span->context()->toTraceparent(); + $sent = $this->messageBuilder->fromMessage($message)->setHeader(MessageHeaders::TRACEPARENT, $traceparent); + if ($isFirstChannel) { + $sent = $sent->setHeader(MessageHeaders::TRACING_DELIVERY, $this->channelName . ':' . $message->getHeaders()->getMessageId()); + } + $payload = $message->getPayload(); + if ($payload instanceof BatchMessage) { + $sent = $sent->setPayload(BatchMessage::fromEntries(array_map( + fn (array $entry): array => [...$entry, 'headers' => [...($entry['headers'] ?? []), MessageHeaders::TRACEPARENT => $traceparent]], + $payload->entries, + ))); + } + + return $sent->build(); + } + + public static function deliveryChannelOf(Message $message): ?string + { + $headers = $message->getHeaders(); + if (! $headers->containsKey(MessageHeaders::TRACING_DELIVERY)) { + return null; + } + [$channel, $messageId] = explode(':', (string) $headers->get(MessageHeaders::TRACING_DELIVERY), 2); + + return $messageId === $headers->getMessageId() ? $channel : null; + } + + /** + * @return array + */ + private function deliveryAttributes(Message $message): array + { + $attributes = [SpanAttribute::CHANNEL_NAME => $this->channelName]; + $payload = $message->getPayload(); + if ($payload instanceof BatchMessage) { + $deliveries = []; + foreach ($payload->entries as $entry) { + $headers = $entry['headers'] ?? []; + $deliveries[(string) ($headers[MessageHeaders::MESSAGE_ID] ?? '')] = self::targetEndpointIdOfRoutingSlip((string) ($headers[MessageHeaders::ROUTING_SLIP] ?? '')); + } + $attributes[SpanAttribute::BATCH_DELIVERIES] = json_encode($deliveries, JSON_THROW_ON_ERROR | JSON_FORCE_OBJECT); + + return $attributes; + } + $targetEndpointId = self::targetEndpointIdOf($message); + if ($targetEndpointId !== null) { + $attributes[SpanAttribute::ENDPOINT_ID] = $targetEndpointId; + } + + return $attributes; } private static function targetEndpointIdOf(Message $message): ?string { $headers = $message->getHeaders(); - if (! $headers->containsKey(MessageHeaders::ROUTING_SLIP)) { + + return $headers->containsKey(MessageHeaders::ROUTING_SLIP) ? self::targetEndpointIdOfRoutingSlip((string) $headers->get(MessageHeaders::ROUTING_SLIP)) : null; + } + + private static function targetEndpointIdOfRoutingSlip(string $routingSlipHeader): ?string + { + if ($routingSlipHeader === '') { return null; } - $routingSlip = explode(',', (string) $headers->get(MessageHeaders::ROUTING_SLIP)); + $routingSlip = explode(',', $routingSlipHeader); $executionChannel = $routingSlip[count($routingSlip) - 1]; if (! str_ends_with($executionChannel, AsynchronousModule::HANDLER_EXECUTION_CHANNEL_SUFFIX)) { return null; diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 550928012..860c90045 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -82,7 +82,7 @@ public function traceAsynchronousEndpoint(MethodInvocation $methodInvocation, Me $attributes = [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS]; if ($headers->containsKey(MessageHeaders::POLLED_CHANNEL_NAME)) { $attributes[SpanAttribute::MESSAGING_DESTINATION_NAME] = $headers->get(MessageHeaders::POLLED_CHANNEL_NAME); - $attributes[SpanAttribute::CHANNEL_NAME] = $headers->get(MessageHeaders::POLLED_CHANNEL_NAME); + $attributes[SpanAttribute::CHANNEL_NAME] = TracingChannelInterceptor::deliveryChannelOf($message) ?? $headers->get(MessageHeaders::POLLED_CHANNEL_NAME); } return $this->trace($name, $methodInvocation, $message, SpanKind::Consumer, $attributes); diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index 03da0cae9..5f54c8351 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -13,6 +13,7 @@ use Ecotone\Api\Attribute\Polling; use Ecotone\Api\Attribute\Reference; use Ecotone\Api\EventSourcing\Event; +use Ecotone\Api\ExtensionObject\DynamicMessageChannelBuilder; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\ModulePackageList; use Ecotone\Api\ExtensionObject\ServiceConfiguration; @@ -601,7 +602,7 @@ public function whenPlaced(array $event): void $ecotone = $this->bootstrap([$projection]); $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage("Projection orders_list runs by #[Polling], so it is not part of the trace that appended its events and TraceCompletion cannot await it. Await the handler that published the event, or make the projection event-driven by removing #[Polling]."); + $this->expectExceptionMessage('Projection orders_list runs by #[Polling], so it is not part of the trace that appended its events and TraceCompletion cannot await it. Await the handler that published the event, or make the projection event-driven by removing #[Polling].'); $this->completion($ecotone)->awaitFor('0af7651916cd43dd8448eb211c80319c', 'orders_list', Duration::milliseconds(10)); } @@ -643,6 +644,33 @@ public function register(string $userId, #[Reference] CurrentTrace $currentTrace $ecotone->sendCommandWithRouting('user.register', '1'); } + public function test_a_dynamic_channel_spreading_over_two_channels_completes_once_both_deliveries_were_consumed(): void + { + $orders = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.place', endpointId: 'order.placing')] + public function place(string $orderId): void + { + } + }; + $ecotone = $this->bootstrap([$orders], [ + DynamicMessageChannelBuilder::createRoundRobin('orders', ['orders_a', 'orders_b']), + SimpleMessageChannelBuilder::createQueueChannel('orders_a'), + SimpleMessageChannelBuilder::createQueueChannel('orders_b'), + ]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + $ecotone->sendCommandWithRouting('order.place', '2', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $ecotone->run('orders', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 1)); + $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + + $ecotone->run('orders', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 1)); + + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + public function test_a_failed_unit_of_work_stores_nothing_by_default(): void { $subscriber = $this->spanEndedSubscriber(); diff --git a/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php b/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php index 2e0ab6c9b..d473c616f 100644 --- a/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php +++ b/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php @@ -43,6 +43,11 @@ public function __construct( ) { } + public function attributes(): array + { + return $this->attributes; + } + public function context(): SpanContext { return $this->context; diff --git a/packages/OpenTelemetry/src/OpenTelemetryTracer.php b/packages/OpenTelemetry/src/OpenTelemetryTracer.php index 72cd862f0..0ed01f6d0 100644 --- a/packages/OpenTelemetry/src/OpenTelemetryTracer.php +++ b/packages/OpenTelemetry/src/OpenTelemetryTracer.php @@ -51,11 +51,15 @@ public function startSpan(string $name, SpanKind $kind, array $attributes = [], $span = $spanBuilder->startSpan(); $scope = $span->activate(); $context = $span->getContext(); + $parentContext = $remoteParent ?? $this->currentSpan()?->context(); + $spanContext = $context->isValid() && $span->isRecording() + ? new SpanContext($context->getTraceId(), $context->getSpanId(), $context->isSampled()) + : ($parentContext?->newChild() ?? SpanContext::newTrace()); $activeSpan = new OpenTelemetryActiveSpan( $span, $scope, - new SpanContext($context->getTraceId(), $context->getSpanId(), $context->isSampled()), + $spanContext, $parentSpanId, $name, $kind, diff --git a/packages/OpenTelemetry/tests/Integration/StoredTracesWithNoopProviderTest.php b/packages/OpenTelemetry/tests/Integration/StoredTracesWithNoopProviderTest.php new file mode 100644 index 000000000..79e86785f --- /dev/null +++ b/packages/OpenTelemetry/tests/Integration/StoredTracesWithNoopProviderTest.php @@ -0,0 +1,89 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + * + * @internal + */ +final class StoredTracesWithNoopProviderTest extends TestCase +{ + public function test_a_provider_that_records_nothing_still_gives_every_stored_span_its_own_id(): void + { + $users = new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId, EventBus $eventBus): void + { + $eventBus->publishWithRouting('user.registered', $userId); + } + + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId): void + { + } + }; + $subscriber = new class () { + /** @var string[] */ + public array $spanIds = []; + + #[OnSpanEnded] + public function whenEnded(SpanEnded $event): void + { + $this->spanIds[] = $event->span->spanId; + } + }; + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$users::class, $subscriber::class], + [$users, $subscriber, TracerProviderInterface::class => new NoopTracerProvider()], + [TracingConfiguration::createWithDefaults()->withStoredTraces(inMemory: true), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + enabledModules: [ModulePackageList::TRACING_PACKAGE], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertCount(count($subscriber->spanIds), array_unique($subscriber->spanIds)); + $this->assertGreaterThan(3, count($subscriber->spanIds)); + $this->assertTrue($ecotone->getGateway(TraceCompletion::class)->isCompleted($trace->traceId)); + } +} diff --git a/upgrade-2.0.md b/upgrade-2.0.md index 9a42ec1f7..cf627e7f0 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3130,7 +3130,10 @@ delivered, a subscriber that throws does not fail the command that already commi in any process re-delivers a notification left undelivered for thirty seconds. A handler that needs the id of the trace it runs in injects `Ecotone\Api\Tracing\CurrentTrace` and reads `traceId()`. `isCompletedFor()` and `awaitFor()` refuse a `#[Polling]` projection by name, because a polled run is not part of the trace that appended -its events. Work a subscriber does +its events. A message sent through a dynamic channel, through the high-throughput outbox or replayed from the dead +letter keeps the delivery it started as, so the trace completes when that message is consumed, wherever it landed; +a multi-tenant application stores traces on the connection `MultiTenantConfiguration::createWithDefaultConnection()` +names, because the write after the unit of work has no tenant, and is refused at bootstrap without one. Work a subscriber does through a bus is stored like any other trace but never notifies subscribers in turn, so a subscriber that sends a command does not call itself again. The trace id is the one in the `traceparent` header: mint it with `SpanContext::newTrace()` and send it with the command, or take it from the HTTP request the command came from. A flow test given no connection keeps the From 45d2f034a5159a8e4c0809a025fdac6aeca4564a Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 21/50] feat(tracing): withStoredTraces(onlyTracesCarryingTraceparent: true) stores only requested traces, tracestate travels with the trace, and a flush inserts its spans in one statement A trace whose root continued a traceparent is marked carried, the mark travels across channels and the distributed bus in the W3C tracestate as ecotone=carried next to any other vendor's entry, and the recorder ignores every other span, so ordinary traffic costs no storage. SpanContext carries the trace state, so a vendor entry received with a command reaches the asynchronous handler. The DBAL store inserts all spans of a flush in one statement. --- .../FlowTestingDbalConnections.php | 2 +- packages/Dbal/src/Tracing/DbalTraceStore.php | 62 ++++++++++++------- .../Ecotone/Api/Tracing/SpanAttribute.php | 1 + packages/Ecotone/Api/Tracing/SpanContext.php | 12 +++- .../Api/Tracing/TracingConfiguration.php | 7 ++- .../src/Tracing/Config/TracingModule.php | 1 + .../src/Tracing/StoringSpanRecorder.php | 5 ++ packages/Ecotone/src/Tracing/TraceState.php | 43 +++++++++++++ .../src/Tracing/TracedRouteProcessor.php | 1 + .../src/Tracing/TracingChannelInterceptor.php | 11 +++- .../src/Tracing/TracingInterceptor.php | 34 +++++++++- .../tests/Tracing/TraceCompletionTest.php | 57 +++++++++++++++++ .../OpenTelemetry/src/OpenTelemetryTracer.php | 4 +- upgrade-2.0.md | 5 +- 14 files changed, 206 insertions(+), 39 deletions(-) create mode 100644 packages/Ecotone/src/Tracing/TraceState.php diff --git a/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php b/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php index 6ed50c04a..5f45183c4 100644 --- a/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php +++ b/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php @@ -111,7 +111,7 @@ private function tracesInMemoryWhereTheConnectionIsMissing(TracingConfiguration return $tracingConfiguration; } - return $tracingConfiguration->withStoredTraces(inMemory: true, retention: $tracingConfiguration->storedTracesRetention, storeFailuresOutsideTransaction: $tracingConfiguration->storedFailuresOutsideTransaction); + return $tracingConfiguration->withStoredTraces(inMemory: true, retention: $tracingConfiguration->storedTracesRetention, storeFailuresOutsideTransaction: $tracingConfiguration->storedFailuresOutsideTransaction, onlyTracesCarryingTraceparent: $tracingConfiguration->storedTracesOnlyCarryingTraceparent); } private function refuseDbalChannelsWithoutTheirConnection(): void diff --git a/packages/Dbal/src/Tracing/DbalTraceStore.php b/packages/Dbal/src/Tracing/DbalTraceStore.php index a76963ffc..82a88ebab 100644 --- a/packages/Dbal/src/Tracing/DbalTraceStore.php +++ b/packages/Dbal/src/Tracing/DbalTraceStore.php @@ -69,9 +69,7 @@ public function append(array $spans): void foreach ($spansByTrace as $traceId => $traceSpans) { $this->onTables(function () use ($connection, $traceId, $traceSpans): void { $this->claimTrace($connection, $traceId, $traceSpans); - foreach ($traceSpans as $span) { - $this->insertSpan($connection, $span); - } + $this->insertSpans($connection, $traceSpans); }); } } @@ -148,30 +146,46 @@ private function markCompleted(Connection $connection, string $traceId, DatePoin ) === 1; } - private function insertSpan(Connection $connection, Span $span): void + /** + * @param Span[] $spans + */ + private function insertSpans(Connection $connection, array $spans): void { - $columns = [ - 'span_id' => $span->spanId, - 'trace_id' => $span->traceId, - 'parent_span_id' => $span->parentSpanId, - 'name' => $span->name, - 'kind' => $span->kind->value, - 'status' => $span->status->value, - 'status_description' => $span->statusDescription, - 'attributes' => json_encode($span->attributes, JSON_THROW_ON_ERROR | JSON_FORCE_OBJECT), - 'events' => json_encode(array_map(fn (SpanEvent $event): array => [ - 'name' => $event->name, - 'attributes' => (object) $event->attributes, - 'at' => $event->at->unixTime()->inMicroseconds(), - ], $span->events), JSON_THROW_ON_ERROR), - 'started_at' => $span->startedAt->unixTime()->inMicroseconds(), - 'ended_at' => $span->endedAt->unixTime()->inMicroseconds(), - ]; - $placeholders = implode(', ', array_map(fn (string $column): string => ':' . $column, array_keys($columns))); - $sql = sprintf('INSERT INTO %s (%s) VALUES (%s)', $this->spansTableManager->getTableName(), implode(', ', array_keys($columns)), $placeholders); + $columns = ['span_id', 'trace_id', 'parent_span_id', 'name', 'kind', 'status', 'status_description', 'attributes', 'events', 'started_at', 'ended_at']; + $rows = []; + $parameters = []; + $types = []; + foreach (array_values($spans) as $index => $span) { + $values = [ + 'span_id' => $span->spanId, + 'trace_id' => $span->traceId, + 'parent_span_id' => $span->parentSpanId, + 'name' => $span->name, + 'kind' => $span->kind->value, + 'status' => $span->status->value, + 'status_description' => $span->statusDescription, + 'attributes' => json_encode($span->attributes, JSON_THROW_ON_ERROR | JSON_FORCE_OBJECT), + 'events' => json_encode(array_map(fn (SpanEvent $event): array => [ + 'name' => $event->name, + 'attributes' => (object) $event->attributes, + 'at' => $event->at->unixTime()->inMicroseconds(), + ], $span->events), JSON_THROW_ON_ERROR), + 'started_at' => $span->startedAt->unixTime()->inMicroseconds(), + 'ended_at' => $span->endedAt->unixTime()->inMicroseconds(), + ]; + $placeholders = []; + foreach ($values as $column => $value) { + $placeholders[] = ':' . $column . $index; + $parameters[$column . $index] = $value; + } + $types['started_at' . $index] = Types::BIGINT; + $types['ended_at' . $index] = Types::BIGINT; + $rows[] = '(' . implode(', ', $placeholders) . ')'; + } + $sql = sprintf('INSERT INTO %s (%s) VALUES %s', $this->spansTableManager->getTableName(), implode(', ', $columns), implode(', ', $rows)); $sql .= $connection->getDatabasePlatform() instanceof AbstractMySQLPlatform ? ' ON DUPLICATE KEY UPDATE span_id = span_id' : ' ON CONFLICT (trace_id, span_id) DO NOTHING'; - $connection->executeStatement($sql, $columns, ['started_at' => Types::BIGINT, 'ended_at' => Types::BIGINT]); + $connection->executeStatement($sql, $parameters, $types); } /** diff --git a/packages/Ecotone/Api/Tracing/SpanAttribute.php b/packages/Ecotone/Api/Tracing/SpanAttribute.php index edb9dea2e..e755770c4 100644 --- a/packages/Ecotone/Api/Tracing/SpanAttribute.php +++ b/packages/Ecotone/Api/Tracing/SpanAttribute.php @@ -15,6 +15,7 @@ final class SpanAttribute public const ENDPOINT_ID = 'ecotone.endpoint.id'; public const PROJECTION_NAME = 'ecotone.projection.name'; public const TRACE_ROOT = 'ecotone.trace.root'; + public const TRACE_CARRIED = 'ecotone.trace.carried'; public const CODE_FUNCTION_NAME = 'code.function.name'; public const CHANNEL_NAME = 'ecotone.channel.name'; public const BATCH_DELIVERIES = 'ecotone.batch.deliveries'; diff --git a/packages/Ecotone/Api/Tracing/SpanContext.php b/packages/Ecotone/Api/Tracing/SpanContext.php index de89ad9f6..9b905ba3a 100644 --- a/packages/Ecotone/Api/Tracing/SpanContext.php +++ b/packages/Ecotone/Api/Tracing/SpanContext.php @@ -20,6 +20,7 @@ public function __construct( public readonly string $traceId, public readonly string $spanId, public readonly bool $sampled = true, + public readonly string $traceState = '', ) { } @@ -30,10 +31,15 @@ public static function newTrace(): self public function newChild(): self { - return new self($this->traceId, bin2hex(random_bytes(8)), $this->sampled); + return new self($this->traceId, bin2hex(random_bytes(8)), $this->sampled, $this->traceState); } - public static function fromTraceparent(?string $traceparent): ?self + public function withTraceState(string $traceState): self + { + return new self($this->traceId, $this->spanId, $this->sampled, $traceState); + } + + public static function fromTraceparent(?string $traceparent, string $tracestate = ''): ?self { if ($traceparent === null || preg_match(self::TRACEPARENT_PATTERN, $traceparent, $matches) !== 1) { return null; @@ -42,7 +48,7 @@ public static function fromTraceparent(?string $traceparent): ?self return null; } - return new self($matches[1], $matches[2], (hexdec($matches[3]) & self::SAMPLED_FLAG) === self::SAMPLED_FLAG); + return new self($matches[1], $matches[2], (hexdec($matches[3]) & self::SAMPLED_FLAG) === self::SAMPLED_FLAG, $tracestate); } public function toTraceparent(): string diff --git a/packages/Ecotone/Api/Tracing/TracingConfiguration.php b/packages/Ecotone/Api/Tracing/TracingConfiguration.php index 1a3042b72..499fefb6a 100644 --- a/packages/Ecotone/Api/Tracing/TracingConfiguration.php +++ b/packages/Ecotone/Api/Tracing/TracingConfiguration.php @@ -23,12 +23,13 @@ private function __construct( public readonly bool $storedFailuresOutsideTransaction, public readonly bool $spanAttributesFromAllHeaders, public readonly array $spanAttributeHeaders, + public readonly bool $storedTracesOnlyCarryingTraceparent, ) { } public static function createWithDefaults(): self { - return new self([], false, false, Duration::minutes(60 * 24 * 7), false, true, []); + return new self([], false, false, Duration::minutes(60 * 24 * 7), false, true, [], false); } public function withSpanRecorder(string $referenceName): self @@ -41,8 +42,8 @@ public function withSpanAttributesFromHeaders(string ...$headerNames): self return new self(...[...get_object_vars($this), 'spanAttributesFromAllHeaders' => false, 'spanAttributeHeaders' => array_values($headerNames)]); } - public function withStoredTraces(bool $inMemory = false, ?Duration $retention = null, bool $storeFailuresOutsideTransaction = false): self + public function withStoredTraces(bool $inMemory = false, ?Duration $retention = null, bool $storeFailuresOutsideTransaction = false, bool $onlyTracesCarryingTraceparent = false): self { - return new self(...[...get_object_vars($this), 'storedTraces' => true, 'storedTracesInMemory' => $inMemory, 'storedTracesRetention' => $retention ?? $this->storedTracesRetention, 'storedFailuresOutsideTransaction' => $storeFailuresOutsideTransaction]); + return new self(...[...get_object_vars($this), 'storedTraces' => true, 'storedTracesInMemory' => $inMemory, 'storedTracesRetention' => $retention ?? $this->storedTracesRetention, 'storedFailuresOutsideTransaction' => $storeFailuresOutsideTransaction, 'storedTracesOnlyCarryingTraceparent' => $onlyTracesCarryingTraceparent]); } } diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index beeeb9482..cab5796a7 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -286,6 +286,7 @@ private function registerStoredTraces(Configuration $messagingConfiguration, arr $this->spanEndedSubscribers !== [], $this->traceCompletedSubscribers !== [], $tracingConfiguration->storedFailuresOutsideTransaction, + $tracingConfiguration->storedTracesOnlyCarryingTraceparent, new Reference(EcotoneClockInterface::class), ]) ); diff --git a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php index 0fca6a0fe..90dabcb7c 100644 --- a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php +++ b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php @@ -9,6 +9,7 @@ use Ecotone\Api\Tracing\OnSpanEnded; use Ecotone\Api\Tracing\OnTraceCompleted; use Ecotone\Api\Tracing\Span; +use Ecotone\Api\Tracing\SpanAttribute; use Ecotone\Api\Tracing\SpanEnded; use Ecotone\Api\Tracing\SpanRecorder; use Ecotone\Api\Tracing\SpanStatus; @@ -55,12 +56,16 @@ public function __construct( private bool $publishesEndedSpans, private bool $publishesCompletedTraces, private bool $storesFailuresOutsideTransaction, + private bool $storesOnlyCarriedTraces, private EcotoneClockInterface $clock, ) { } public function record(Span $span): void { + if ($this->storesOnlyCarriedTraces && ! ($span->attributes[SpanAttribute::TRACE_CARRIED] ?? false)) { + return; + } if ($this->scopeActive) { $this->buffer[] = $span; diff --git a/packages/Ecotone/src/Tracing/TraceState.php b/packages/Ecotone/src/Tracing/TraceState.php new file mode 100644 index 000000000..01467a09e --- /dev/null +++ b/packages/Ecotone/src/Tracing/TraceState.php @@ -0,0 +1,43 @@ +getHeaders(); + + return $headers->containsKey(MessageHeaders::TRACESTATE) && self::entryOf((string) $headers->get(MessageHeaders::TRACESTATE)) === self::CARRIED; + } + + public static function withCarried(string $tracestate): string + { + $entries = array_values(array_filter(array_map('trim', explode(',', $tracestate)), fn (string $entry): bool => $entry !== '' && ! str_starts_with($entry, self::VENDOR . '='))); + + return implode(',', [self::VENDOR . '=' . self::CARRIED, ...$entries]); + } + + private static function entryOf(string $tracestate): ?string + { + foreach (explode(',', $tracestate) as $entry) { + $entry = trim($entry); + if (str_starts_with($entry, self::VENDOR . '=')) { + return substr($entry, strlen(self::VENDOR) + 1); + } + } + + return null; + } +} diff --git a/packages/Ecotone/src/Tracing/TracedRouteProcessor.php b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php index 66b298e71..8de3ad7c8 100644 --- a/packages/Ecotone/src/Tracing/TracedRouteProcessor.php +++ b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php @@ -32,6 +32,7 @@ public function process(Message $message): ?Message SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND, SpanAttribute::MESSAGING_DESTINATION_NAME => $this->routeName, ...$this->destinationAttributes(), + ...(TracingInterceptor::isCarried($this->tracer, $message, false) ? [SpanAttribute::TRACE_CARRIED => true] : []), ]); try { diff --git a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php index 4d6697e4a..0d7d4f51a 100644 --- a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php @@ -45,6 +45,10 @@ public function preSend(Message $message, MessageChannel $messageChannel): ?Mess if ($isFirstChannel) { $attributes = [...$attributes, ...$this->deliveryAttributes($message)]; } + $carried = TracingInterceptor::isCarried($this->tracer, $message, false); + if ($carried) { + $attributes[SpanAttribute::TRACE_CARRIED] = true; + } $span = $this->tracer->startSpan( 'send ' . $this->channelName, SpanKind::Producer, @@ -53,15 +57,15 @@ public function preSend(Message $message, MessageChannel $messageChannel): ?Mess ); $this->openSendSpans[] = $span; - $traceparent = $span->context()->toTraceparent(); - $sent = $this->messageBuilder->fromMessage($message)->setHeader(MessageHeaders::TRACEPARENT, $traceparent); + $carrier = TracingInterceptor::carrierOf($span); + $sent = $this->messageBuilder->fromMessage($message)->setMultipleHeaders($carrier); if ($isFirstChannel) { $sent = $sent->setHeader(MessageHeaders::TRACING_DELIVERY, $this->channelName . ':' . $message->getHeaders()->getMessageId()); } $payload = $message->getPayload(); if ($payload instanceof BatchMessage) { $sent = $sent->setPayload(BatchMessage::fromEntries(array_map( - fn (array $entry): array => [...$entry, 'headers' => [...($entry['headers'] ?? []), MessageHeaders::TRACEPARENT => $traceparent]], + fn (array $entry): array => [...$entry, 'headers' => [...($entry['headers'] ?? []), ...$carrier]], $payload->entries, ))); } @@ -159,6 +163,7 @@ public function afterReceiveCompletion(?Message $message, MessageChannel $messag SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS, SpanAttribute::MESSAGING_DESTINATION_NAME => $this->channelName, SpanAttribute::CHANNEL_NAME => $this->channelName, + ...(TraceState::isCarried($message) ? [SpanAttribute::TRACE_CARRIED => true] : []), ], TracingInterceptor::remoteParentOf($message)) ->fail($exception); } diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 860c90045..6ab2a8b3d 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -122,15 +122,42 @@ public function provideContextForDistributedBus(Message $message): Message } return $this->messageBuilder->fromMessage($message) - ->setHeader(MessageHeaders::TRACEPARENT, $currentSpan->context()->toTraceparent()) + ->setMultipleHeaders(self::carrierOf($currentSpan)) ->build(); } + /** + * @return array + */ + public static function carrierOf(ActiveSpan $span): array + { + $carrier = [MessageHeaders::TRACEPARENT => $span->context()->toTraceparent()]; + $traceState = $span->attributes()[SpanAttribute::TRACE_CARRIED] ?? false ? TraceState::withCarried($span->context()->traceState) : $span->context()->traceState; + if ($traceState !== '') { + $carrier[MessageHeaders::TRACESTATE] = $traceState; + } + + return $carrier; + } + + public static function isCarried(Tracer $tracer, Message $message, bool $startsTheTraceHere): bool + { + $current = $tracer->currentSpan(); + if ($current !== null) { + return (bool) ($current->attributes()[SpanAttribute::TRACE_CARRIED] ?? false); + } + + return $startsTheTraceHere ? self::remoteParentOf($message) !== null : TraceState::isCarried($message); + } + public static function remoteParentOf(Message $message): ?SpanContext { $headers = $message->getHeaders(); - return SpanContext::fromTraceparent($headers->containsKey(MessageHeaders::TRACEPARENT) ? $headers->get(MessageHeaders::TRACEPARENT) : null); + return SpanContext::fromTraceparent( + $headers->containsKey(MessageHeaders::TRACEPARENT) ? $headers->get(MessageHeaders::TRACEPARENT) : null, + $headers->containsKey(MessageHeaders::TRACESTATE) ? (string) $headers->get(MessageHeaders::TRACESTATE) : '', + ); } private function traceHandler(MethodInvocation $methodInvocation, Message $message, string $endpointId): mixed @@ -163,6 +190,9 @@ private function trace(string $name, MethodInvocation $methodInvocation, Message if ($isLocalRoot && $kind !== SpanKind::Consumer) { $attributes[SpanAttribute::TRACE_ROOT] = true; } + if (self::isCarried($this->tracer, $message, $isLocalRoot && $kind !== SpanKind::Consumer)) { + $attributes[SpanAttribute::TRACE_CARRIED] = true; + } $span = $this->tracer->startSpan($name, $kind, [...$this->messageSpanAttributes->from($message), ...$attributes], $isLocalRoot ? self::remoteParentOf($message) : null); try { diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index 5f54c8351..d0a3dae5d 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -671,6 +671,63 @@ public function place(string $orderId): void $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } + public function test_with_only_traces_carrying_a_traceparent_stored_a_command_sent_without_one_stores_nothing(): void + { + $subscriber = $this->spanEndedSubscriber(); + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$this->users()::class, $this->asynchronousNotifier()::class, $subscriber::class], + [$this->users(), $this->asynchronousNotifier(), $subscriber], + [TracingConfiguration::createWithDefaults()->withStoredTraces(onlyTracesCarryingTraceparent: true), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $ecotone->sendCommandWithRouting('user.register', '1'); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertSame([], $subscriber->endedSpans); + } + + public function test_with_only_traces_carrying_a_traceparent_stored_a_command_sent_with_one_is_stored_through_its_asynchronous_hop(): void + { + $subscriber = $this->spanEndedSubscriber(); + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$this->users()::class, $this->asynchronousNotifier()::class, $subscriber::class], + [$this->users(), $this->asynchronousNotifier(), $subscriber], + [TracingConfiguration::createWithDefaults()->withStoredTraces(onlyTracesCarryingTraceparent: true), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertArrayHasKey('receive async_channel', $subscriber->endedSpans); + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + public function test_a_vendor_entry_in_tracestate_survives_the_asynchronous_hop(): void + { + $notifier = new class () { + public array $headers = []; + + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId, #[Headers] array $headers): void + { + $this->headers = $headers; + } + }; + $ecotone = $this->bootstrap([$this->users(), $notifier], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent(), MessageHeaders::TRACESTATE => 'congo=t61rcWkgMzE']); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertStringContainsString('congo=t61rcWkgMzE', $notifier->headers[MessageHeaders::TRACESTATE]); + } + public function test_a_failed_unit_of_work_stores_nothing_by_default(): void { $subscriber = $this->spanEndedSubscriber(); diff --git a/packages/OpenTelemetry/src/OpenTelemetryTracer.php b/packages/OpenTelemetry/src/OpenTelemetryTracer.php index 0ed01f6d0..9a0faee8d 100644 --- a/packages/OpenTelemetry/src/OpenTelemetryTracer.php +++ b/packages/OpenTelemetry/src/OpenTelemetryTracer.php @@ -45,7 +45,7 @@ public function startSpan(string $name, SpanKind $kind, array $attributes = [], ->setSpanKind(self::kindOf($kind)) ->setAttributes($attributes); if ($remoteParent !== null) { - $spanBuilder->setParent(TraceContextPropagator::getInstance()->extract([MessageHeaders::TRACEPARENT => $remoteParent->toTraceparent()])); + $spanBuilder->setParent(TraceContextPropagator::getInstance()->extract([MessageHeaders::TRACEPARENT => $remoteParent->toTraceparent(), MessageHeaders::TRACESTATE => $remoteParent->traceState])); } $span = $spanBuilder->startSpan(); @@ -53,7 +53,7 @@ public function startSpan(string $name, SpanKind $kind, array $attributes = [], $context = $span->getContext(); $parentContext = $remoteParent ?? $this->currentSpan()?->context(); $spanContext = $context->isValid() && $span->isRecording() - ? new SpanContext($context->getTraceId(), $context->getSpanId(), $context->isSampled()) + ? new SpanContext($context->getTraceId(), $context->getSpanId(), $context->isSampled(), (string) $context->getTraceState()) : ($parentContext?->newChild() ?? SpanContext::newTrace()); $activeSpan = new OpenTelemetryActiveSpan( diff --git a/upgrade-2.0.md b/upgrade-2.0.md index cf627e7f0..4db2c8e4d 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3136,7 +3136,10 @@ a multi-tenant application stores traces on the connection `MultiTenantConfigura names, because the write after the unit of work has no tenant, and is refused at bootstrap without one. Work a subscriber does through a bus is stored like any other trace but never notifies subscribers in turn, so a subscriber that sends a command does not call itself again. The trace id is the one in the `traceparent` header: mint it with `SpanContext::newTrace()` and send it -with the command, or take it from the HTTP request the command came from. A flow test given no connection keeps the +with the command, or take it from the HTTP request the command came from. `withStoredTraces(onlyTracesCarryingTraceparent: true)` stores only the traces whose root continued a +`traceparent` handed in, from an HTTP request or minted with `SpanContext::newTrace()`, and carries that choice +across channels in the W3C `tracestate` as `ecotone=carried`, next to any other vendor's entry, so ordinary traffic +costs no storage and only the requests that await completion are kept. A flow test given no connection keeps the store in memory; an application that enables stored traces with no DBAL package and no `inMemory: true` is refused at bootstrap, because a store per process could never see a consumer's progress. `ecotone:tracing:remove-expired-traces` deletes completed traces older than the retention passed to `withStoredTraces()`, seven days by default, and any trace older than twice that, so a message delayed past the From 0c60811d7d887c7fb0a2bdbbbdacccfad9e5d1d6 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 22/50] refactor(tracing): stored traces live in their own Enterprise module, handing the core module their recorder through a module extension TracingModule stays Apache-2.0 and knows nothing of stored traces beyond composing the recorders it is handed: the licence guard, the subscriber guard, the store, the recording interceptor, the completion gateway, the retention command and the polling-projection refusal moved to StoredTracesModule, which announces its recorder with a SpanRecorderReference. The completion gateway registration both modules need is a shared helper. --- .../src/Messaging/Config/ModuleClassList.php | 2 + .../Tracing/Config/SpanRecorderReference.php | 15 ++ .../src/Tracing/Config/StoredTracesModule.php | 218 ++++++++++++++++++ .../TraceCompletionGatewayRegistration.php | 36 +++ .../src/Tracing/Config/TracingModule.php | 159 +------------ 5 files changed, 279 insertions(+), 151 deletions(-) create mode 100644 packages/Ecotone/src/Tracing/Config/SpanRecorderReference.php create mode 100644 packages/Ecotone/src/Tracing/Config/StoredTracesModule.php create mode 100644 packages/Ecotone/src/Tracing/Config/TraceCompletionGatewayRegistration.php diff --git a/packages/Ecotone/src/Messaging/Config/ModuleClassList.php b/packages/Ecotone/src/Messaging/Config/ModuleClassList.php index b2aecec9c..8cc403a03 100644 --- a/packages/Ecotone/src/Messaging/Config/ModuleClassList.php +++ b/packages/Ecotone/src/Messaging/Config/ModuleClassList.php @@ -88,6 +88,7 @@ use Ecotone\Sqs\Configuration\SqsModule; use Ecotone\SymfonyBundle\Config\SymfonyConnectionModule; use Ecotone\Tempest\Config\TempestModule; +use Ecotone\Tracing\Config\StoredTracesModule; use Ecotone\Tracing\Config\TracingModule; /** @@ -119,6 +120,7 @@ class ModuleClassList ScheduledModule::class, CollectorModule::class, TracingModule::class, + StoredTracesModule::class, DeliveryConfirmationModule::class, ChannelSetupModule::class, SerializerModule::class, diff --git a/packages/Ecotone/src/Tracing/Config/SpanRecorderReference.php b/packages/Ecotone/src/Tracing/Config/SpanRecorderReference.php new file mode 100644 index 000000000..01864410a --- /dev/null +++ b/packages/Ecotone/src/Tracing/Config/SpanRecorderReference.php @@ -0,0 +1,15 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +#[ModuleAnnotation] +final class StoredTracesModule extends NoExternalConfigurationModule implements AnnotationModule +{ + private const REMOVE_EXPIRED_TRACES_CHANNEL = 'ecotone.tracing.removeExpiredTraces'; + private const USER_MESSAGE_GATEWAYS_POINTCUT = MessageGateway::class . '&¬(' . CommandBus::class . ')&¬(' . EventBus::class . ')&¬(' . QueryBus::class . ')'; + + /** + * @param string[] $traceCompletedSubscribers + * @param string[] $spanEndedSubscribers + * @param string[] $pollingProjectionNames + */ + private function __construct( + private array $traceCompletedSubscribers, + private array $spanEndedSubscribers, + private array $pollingProjectionNames, + ) { + } + + public static function create(AnnotationFinder $annotationRegistrationService, InterfaceToCallRegistry $interfaceToCallRegistry): static + { + return new self( + self::placementsOf($annotationRegistrationService, OnTraceCompleted::class), + self::placementsOf($annotationRegistrationService, OnSpanEnded::class), + self::pollingProjectionNames($annotationRegistrationService), + ); + } + + public function prepare(Configuration $messagingConfiguration, array $extensionObjects, ModuleReferenceSearchService $moduleReferenceSearchService, InterfaceToCallRegistry $interfaceToCallRegistry): void + { + $tracingConfiguration = ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $extensionObjects, TracingConfiguration::createWithDefaults()); + $this->refuseMisconfiguredStoredTraces($messagingConfiguration, $tracingConfiguration); + if (! $tracingConfiguration->storedTraces) { + return; + } + $this->registerStoredTraces($messagingConfiguration, $extensionObjects, $tracingConfiguration, $interfaceToCallRegistry); + } + + public function getModuleExtensions(ServiceConfiguration $serviceConfiguration, array $serviceExtensions): array + { + $tracingConfiguration = ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $serviceExtensions, TracingConfiguration::createWithDefaults()); + + return $tracingConfiguration->storedTraces ? [new SpanRecorderReference(StoringSpanRecorder::class)] : []; + } + + public function getModulePackageName(): string + { + return ModulePackageList::CORE_PACKAGE; + } + + /** + * @return string[] + */ + private static function pollingProjectionNames(AnnotationFinder $annotationFinder): array + { + $names = []; + foreach ($annotationFinder->findAnnotatedClasses(Projection::class) as $projectionClass) { + if ($annotationFinder->findAttributeForClass($projectionClass, Polling::class) !== null) { + $names[] = $annotationFinder->getAttributeForClass($projectionClass, Projection::class)->name; + } + } + + return $names; + } + + private function refuseMisconfiguredStoredTraces(Configuration $messagingConfiguration, TracingConfiguration $tracingConfiguration): void + { + $subscribers = [...$this->traceCompletedSubscribers, ...$this->spanEndedSubscribers]; + $enterpriseUsages = [...($tracingConfiguration->storedTraces ? ['TracingConfiguration::withStoredTraces()'] : []), ...$subscribers]; + if ($enterpriseUsages !== [] && ! $messagingConfiguration->isRunningForEnterpriseLicence()) { + throw LicensingException::create(sprintf( + '%s %s an Ecotone Enterprise licence. Remove %s to run without one.', + implode(' and ', $enterpriseUsages), + count($enterpriseUsages) === 1 ? 'requires' : 'require', + count($enterpriseUsages) === 1 ? 'it' : 'them', + )); + } + if ($tracingConfiguration->storedTraces || $subscribers === []) { + return; + } + + throw ConfigurationException::create( + implode(', ', $subscribers) . ' needs stored traces, which decide when a trace is complete. Register TracingConfiguration::createWithDefaults()->withStoredTraces() from a #[ServiceContext] method.' + ); + } + + /** + * @param object[] $extensionObjects + */ + private function registerStoredTraces(Configuration $messagingConfiguration, array $extensionObjects, TracingConfiguration $tracingConfiguration, InterfaceToCallRegistry $interfaceToCallRegistry): void + { + $traceStoreReferences = ExtensionObjectResolver::resolve(TraceStoreReference::class, $extensionObjects); + if ($traceStoreReferences === [] && ! $tracingConfiguration->storedTracesInMemory) { + throw ConfigurationException::create('TracingConfiguration::withStoredTraces() needs a trace store and none is registered. Enable the DBAL package with a connection to store traces in the database, or call withStoredTraces(inMemory: true) to keep them in this process only.'); + } + $messagingConfiguration->registerServiceDefinition( + TraceStore::class, + $traceStoreReferences === [] || $tracingConfiguration->storedTracesInMemory + ? new Definition(InMemoryTraceStore::class) + : new Reference($traceStoreReferences[0]->referenceName) + ); + $messagingConfiguration->registerServiceDefinition( + StoringSpanRecorder::class, + new Definition(StoringSpanRecorder::class, [ + new Reference(TraceStore::class), + new Reference(MessagingEntrypointService::class), + $this->spanEndedSubscribers !== [], + $this->traceCompletedSubscribers !== [], + $tracingConfiguration->storedFailuresOutsideTransaction, + $tracingConfiguration->storedTracesOnlyCarryingTraceparent, + new Reference(EcotoneClockInterface::class), + ]) + ); + $messagingConfiguration->registerServiceDefinition( + TraceRecordingInterceptor::class, + new Definition(TraceRecordingInterceptor::class, [new Reference(StoringSpanRecorder::class)]) + ); + $messagingConfiguration->registerAroundMethodInterceptor( + AroundInterceptorBuilder::create( + TraceRecordingInterceptor::class, + $interfaceToCallRegistry->getFor(TraceRecordingInterceptor::class, 'recordUnitOfWork'), + Precedence::TRACE_RECORDING_PRECEDENCE, + CommandBus::class . '||' . EventBus::class . '||' . QueryBus::class . '||' . AsynchronousRunningEndpoint::class . '||' . OrchestratorGateway::class . '||(' . self::USER_MESSAGE_GATEWAYS_POINTCUT . ')' + ) + ); + $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnTraceCompleted::CHANNEL)); + $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnSpanEnded::CHANNEL)); + + TraceCompletionGatewayRegistration::register($messagingConfiguration, StoredTraceCompletion::class, new Definition(StoredTraceCompletion::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $this->pollingProjectionNames])); + + $messagingConfiguration->registerServiceDefinition( + ExpiredTraceRemover::class, + new Definition(ExpiredTraceRemover::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $tracingConfiguration->storedTracesRetention->inMilliseconds()]) + ); + $messagingConfiguration + ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod(ExpiredTraceRemover::class, 'removeExpiredTraces')->withInputChannelName(self::REMOVE_EXPIRED_TRACES_CHANNEL)) + ->registerConsoleCommand(ConsoleCommandConfiguration::create(self::REMOVE_EXPIRED_TRACES_CHANNEL, 'ecotone:tracing:remove-expired-traces', [], 'Removes stored traces older than the configured retention')); + } + + /** + * @return string[] + */ + private static function placementsOf(AnnotationFinder $annotationFinder, string $attributeClassName): array + { + $placements = []; + foreach ($annotationFinder->findAnnotatedMethods($attributeClassName) as $annotatedMethod) { + $placements[] = '#[' . substr((string) strrchr('\\' . $attributeClassName, '\\'), 1) . '] on ' . $annotatedMethod->getClassName() . '::' . $annotatedMethod->getMethodName(); + } + + return $placements; + } +} diff --git a/packages/Ecotone/src/Tracing/Config/TraceCompletionGatewayRegistration.php b/packages/Ecotone/src/Tracing/Config/TraceCompletionGatewayRegistration.php new file mode 100644 index 000000000..a9a8cf2d2 --- /dev/null +++ b/packages/Ecotone/src/Tracing/Config/TraceCompletionGatewayRegistration.php @@ -0,0 +1,36 @@ +registerServiceDefinition($serviceClass, $service); + foreach ([ + 'isCompleted' => [GatewayPayloadBuilder::create('traceId')], + 'isCompletedFor' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('destination', StoredTraceCompletion::DESTINATION_HEADER)], + 'await' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('timeout', StoredTraceCompletion::TIMEOUT_HEADER)], + 'awaitFor' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('destination', StoredTraceCompletion::DESTINATION_HEADER), GatewayHeaderBuilder::create('timeout', StoredTraceCompletion::TIMEOUT_HEADER)], + ] as $method => $parameterConverters) { + $channel = 'ecotone.tracing.completion.' . $method; + $messagingConfiguration + ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod($serviceClass, $method)->withInputChannelName($channel)) + ->registerGatewayBuilder(GatewayProxyBuilder::create(TraceCompletion::class, TraceCompletion::class, $method, $channel)->withParameterConverters($parameterConverters)); + } + } +} diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index cab5796a7..965142bc8 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -15,56 +15,39 @@ use Ecotone\Api\Attribute\MessageGateway; use Ecotone\Api\Attribute\ModuleAnnotation; use Ecotone\Api\Attribute\OrchestratorGateway; -use Ecotone\Api\Attribute\Polling; use Ecotone\Api\Attribute\QueryHandler; use Ecotone\Api\ExtensionObject\MessageChannelBuilder; use Ecotone\Api\ExtensionObject\ModulePackageList; -use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; use Ecotone\Api\Gateway\CommandBus; use Ecotone\Api\Gateway\DistributedBus; use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Gateway\EventBus; use Ecotone\Api\Gateway\QueryBus; use Ecotone\Api\Interceptor\Precedence; -use Ecotone\Api\Projecting\Projection; use Ecotone\Api\Tracing\CurrentTrace; use Ecotone\Api\Tracing\OnSpanEnded; use Ecotone\Api\Tracing\OnTraceCompleted; use Ecotone\Api\Tracing\SpanRecorder; -use Ecotone\Api\Tracing\TraceCompletion; use Ecotone\Api\Tracing\TracingConfiguration; use Ecotone\Messaging\Config\Annotation\AnnotationModule; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\ExtensionObjectResolver; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\NoExternalConfigurationModule; use Ecotone\Messaging\Config\Configuration; use Ecotone\Messaging\Config\ConfigurationException; -use Ecotone\Messaging\Config\ConsoleCommandConfiguration; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\Reference; use Ecotone\Messaging\Config\ModuleReferenceSearchService; -use Ecotone\Messaging\Gateway\MessagingEntrypointService; -use Ecotone\Messaging\Handler\Gateway\GatewayProxyBuilder; -use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayHeaderBuilder; -use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayPayloadBuilder; use Ecotone\Messaging\Handler\InterfaceToCallRegistry; use Ecotone\Messaging\Handler\Processor\MethodInvoker\AroundInterceptorBuilder; use Ecotone\Messaging\Handler\Processor\MethodInvoker\MethodInterceptorBuilder; -use Ecotone\Messaging\Handler\ServiceActivator\MessageProcessorActivatorBuilder; -use Ecotone\Messaging\Support\LicensingException; use Ecotone\Messaging\Support\MessageBuilder; use Ecotone\Modelling\MessageHandling\MetadataPropagator\MessageHeadersPropagatorInterceptor; use Ecotone\Projecting\ProjectionTriggerEndpoint; use Ecotone\Tracing\CompositeSpanRecorder; use Ecotone\Tracing\EcotoneTracer; -use Ecotone\Tracing\ExpiredTraceRemover; -use Ecotone\Tracing\InMemoryTraceStore; use Ecotone\Tracing\MessageSpanAttributes; -use Ecotone\Tracing\StoredTraceCompletion; -use Ecotone\Tracing\StoringSpanRecorder; use Ecotone\Tracing\Tracer; use Ecotone\Tracing\TracerCurrentTrace; -use Ecotone\Tracing\TraceRecordingInterceptor; -use Ecotone\Tracing\TraceStore; use Ecotone\Tracing\TracingDisabledCurrentTrace; use Ecotone\Tracing\TracingInterceptor; use Ecotone\Tracing\UnavailableTraceCompletion; @@ -77,7 +60,6 @@ */ final class TracingModule extends NoExternalConfigurationModule implements AnnotationModule { - private const REMOVE_EXPIRED_TRACES_CHANNEL = 'ecotone.tracing.removeExpiredTraces'; private const USER_MESSAGE_GATEWAYS_POINTCUT = MessageGateway::class . '&¬(' . CommandBus::class . ')&¬(' . EventBus::class . ')&¬(' . QueryBus::class . ')'; /** @@ -85,59 +67,19 @@ final class TracingModule extends NoExternalConfigurationModule implements Annot * @param string[] $spanEndedSubscribers */ /** - * @param string[] $traceCompletedSubscribers - * @param string[] $spanEndedSubscribers * @param array $endpointIdsByHandler - * @param string[] $pollingProjectionNames */ private function __construct( - private array $traceCompletedSubscribers, - private array $spanEndedSubscribers, private array $endpointIdsByHandler, - private array $pollingProjectionNames, ) { } public static function create(AnnotationFinder $annotationRegistrationService, InterfaceToCallRegistry $interfaceToCallRegistry): static { - return new self( - self::placementsOf($annotationRegistrationService, OnTraceCompleted::class), - self::placementsOf($annotationRegistrationService, OnSpanEnded::class), - self::endpointIdsByHandler($annotationRegistrationService), - self::pollingProjectionNames($annotationRegistrationService), - ); + return new self(self::endpointIdsByHandler($annotationRegistrationService)); } - /** - * @return string[] - */ - private static function pollingProjectionNames(AnnotationFinder $annotationFinder): array - { - $names = []; - foreach ($annotationFinder->findAnnotatedClasses(Projection::class) as $projectionClass) { - if ($annotationFinder->findAttributeForClass($projectionClass, Polling::class) !== null) { - $names[] = $annotationFinder->getAttributeForClass($projectionClass, Projection::class)->name; - } - } - return $names; - } - - private static function registerCompletionGateway(Configuration $messagingConfiguration, string $serviceClass, Definition $service): void - { - $messagingConfiguration->registerServiceDefinition($serviceClass, $service); - foreach ([ - 'isCompleted' => [GatewayPayloadBuilder::create('traceId')], - 'isCompletedFor' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('destination', StoredTraceCompletion::DESTINATION_HEADER)], - 'await' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('timeout', StoredTraceCompletion::TIMEOUT_HEADER)], - 'awaitFor' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('destination', StoredTraceCompletion::DESTINATION_HEADER), GatewayHeaderBuilder::create('timeout', StoredTraceCompletion::TIMEOUT_HEADER)], - ] as $method => $parameterConverters) { - $channel = 'ecotone.tracing.completion.' . $method; - $messagingConfiguration - ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod($serviceClass, $method)->withInputChannelName($channel)) - ->registerGatewayBuilder(GatewayProxyBuilder::create(TraceCompletion::class, TraceCompletion::class, $method, $channel)->withParameterConverters($parameterConverters)); - } - } /** * @return array @@ -161,23 +103,22 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO if (count($tracerReferences) > 1) { throw ConfigurationException::create('Only one Tracer can be active, yet these were registered: ' . implode(', ', array_map(fn (TracerReference $reference): string => $reference->referenceName, $tracerReferences)) . '. Enable a single tracing package.'); } - $this->refuseMisconfiguredStoredTraces($messagingConfiguration, $tracingConfiguration); + $spanRecorderReferences = ExtensionObjectResolver::resolve(SpanRecorderReference::class, $extensionObjects); if (! $tracingConfiguration->storedTraces) { - self::registerCompletionGateway($messagingConfiguration, UnavailableTraceCompletion::class, new Definition(UnavailableTraceCompletion::class)); + TraceCompletionGatewayRegistration::register($messagingConfiguration, UnavailableTraceCompletion::class, new Definition(UnavailableTraceCompletion::class)); } - if ($tracingConfiguration->spanRecorderReferenceNames === [] && $tracerReferences === [] && ! $tracingConfiguration->storedTraces) { + if ($tracingConfiguration->spanRecorderReferenceNames === [] && $tracerReferences === [] && $spanRecorderReferences === []) { $messagingConfiguration->registerServiceDefinition(CurrentTrace::class, new Definition(TracingDisabledCurrentTrace::class)); return; } $messagingConfiguration->registerServiceDefinition(CurrentTrace::class, new Definition(TracerCurrentTrace::class, [new Reference(Tracer::class)])); - $spanRecorders = array_map(fn (string $referenceName): Reference => new Reference($referenceName), $tracingConfiguration->spanRecorderReferenceNames); - if ($tracingConfiguration->storedTraces) { - $this->registerStoredTraces($messagingConfiguration, $extensionObjects, $tracingConfiguration, $interfaceToCallRegistry); - $spanRecorders[] = new Reference(StoringSpanRecorder::class); - } + $spanRecorders = [ + ...array_map(fn (string $referenceName): Reference => new Reference($referenceName), $tracingConfiguration->spanRecorderReferenceNames), + ...array_map(fn (SpanRecorderReference $reference): Reference => new Reference($reference->referenceName), $spanRecorderReferences), + ]; $messagingConfiguration->registerServiceDefinition(SpanRecorder::class, new Definition(CompositeSpanRecorder::class, [$spanRecorders])); $messagingConfiguration->registerServiceDefinition( @@ -242,90 +183,6 @@ public function getModulePackageName(): string return ModulePackageList::CORE_PACKAGE; } - private function refuseMisconfiguredStoredTraces(Configuration $messagingConfiguration, TracingConfiguration $tracingConfiguration): void - { - $subscribers = [...$this->traceCompletedSubscribers, ...$this->spanEndedSubscribers]; - $enterpriseUsages = [...($tracingConfiguration->storedTraces ? ['TracingConfiguration::withStoredTraces()'] : []), ...$subscribers]; - if ($enterpriseUsages !== [] && ! $messagingConfiguration->isRunningForEnterpriseLicence()) { - throw LicensingException::create(sprintf( - '%s %s an Ecotone Enterprise licence. Remove %s to run without one.', - implode(' and ', $enterpriseUsages), - count($enterpriseUsages) === 1 ? 'requires' : 'require', - count($enterpriseUsages) === 1 ? 'it' : 'them', - )); - } - if ($tracingConfiguration->storedTraces || $subscribers === []) { - return; - } - throw ConfigurationException::create( - implode(', ', $subscribers) . ' needs stored traces, which decide when a trace is complete. Register TracingConfiguration::createWithDefaults()->withStoredTraces() from a #[ServiceContext] method.' - ); - } - /** - * @param object[] $extensionObjects - */ - private function registerStoredTraces(Configuration $messagingConfiguration, array $extensionObjects, TracingConfiguration $tracingConfiguration, InterfaceToCallRegistry $interfaceToCallRegistry): void - { - $traceStoreReferences = ExtensionObjectResolver::resolve(TraceStoreReference::class, $extensionObjects); - if ($traceStoreReferences === [] && ! $tracingConfiguration->storedTracesInMemory) { - throw ConfigurationException::create('TracingConfiguration::withStoredTraces() needs a trace store and none is registered. Enable the DBAL package with a connection to store traces in the database, or call withStoredTraces(inMemory: true) to keep them in this process only.'); - } - $messagingConfiguration->registerServiceDefinition( - TraceStore::class, - $traceStoreReferences === [] || $tracingConfiguration->storedTracesInMemory - ? new Definition(InMemoryTraceStore::class) - : new Reference($traceStoreReferences[0]->referenceName) - ); - $messagingConfiguration->registerServiceDefinition( - StoringSpanRecorder::class, - new Definition(StoringSpanRecorder::class, [ - new Reference(TraceStore::class), - new Reference(MessagingEntrypointService::class), - $this->spanEndedSubscribers !== [], - $this->traceCompletedSubscribers !== [], - $tracingConfiguration->storedFailuresOutsideTransaction, - $tracingConfiguration->storedTracesOnlyCarryingTraceparent, - new Reference(EcotoneClockInterface::class), - ]) - ); - $messagingConfiguration->registerServiceDefinition( - TraceRecordingInterceptor::class, - new Definition(TraceRecordingInterceptor::class, [new Reference(StoringSpanRecorder::class)]) - ); - $messagingConfiguration->registerAroundMethodInterceptor( - AroundInterceptorBuilder::create( - TraceRecordingInterceptor::class, - $interfaceToCallRegistry->getFor(TraceRecordingInterceptor::class, 'recordUnitOfWork'), - Precedence::TRACE_RECORDING_PRECEDENCE, - CommandBus::class . '||' . EventBus::class . '||' . QueryBus::class . '||' . AsynchronousRunningEndpoint::class . '||' . OrchestratorGateway::class . '||(' . self::USER_MESSAGE_GATEWAYS_POINTCUT . ')' - ) - ); - $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnTraceCompleted::CHANNEL)); - $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnSpanEnded::CHANNEL)); - - self::registerCompletionGateway($messagingConfiguration, StoredTraceCompletion::class, new Definition(StoredTraceCompletion::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $this->pollingProjectionNames])); - - $messagingConfiguration->registerServiceDefinition( - ExpiredTraceRemover::class, - new Definition(ExpiredTraceRemover::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $tracingConfiguration->storedTracesRetention->inMilliseconds()]) - ); - $messagingConfiguration - ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod(ExpiredTraceRemover::class, 'removeExpiredTraces')->withInputChannelName(self::REMOVE_EXPIRED_TRACES_CHANNEL)) - ->registerConsoleCommand(ConsoleCommandConfiguration::create(self::REMOVE_EXPIRED_TRACES_CHANNEL, 'ecotone:tracing:remove-expired-traces', [], 'Removes stored traces older than the configured retention')); - } - - /** - * @return string[] - */ - private static function placementsOf(AnnotationFinder $annotationFinder, string $attributeClassName): array - { - $placements = []; - foreach ($annotationFinder->findAnnotatedMethods($attributeClassName) as $annotatedMethod) { - $placements[] = '#[' . substr((string) strrchr('\\' . $attributeClassName, '\\'), 1) . '] on ' . $annotatedMethod->getClassName() . '::' . $annotatedMethod->getMethodName(); - } - - return $placements; - } } From 4787d07b4c1382f63701ad9e1eb2c096eed54a09 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 17 Oct 2026 08:00:00 +0200 Subject: [PATCH 23/50] refactor(tracing): a handler failure marks the span wrapping its unit of work, so the recorder keeps no bookkeeping of foreign spans and the failure shows in OpenTelemetry too The storing recorder remembered the ids of spans wrapping a discarded unit of work and rewrote their status when they arrived, state that outlived the unit of work and was invisible to the OpenTelemetry export. The handler span now marks the active span that wraps it as failed when the handler throws and as recovered when a later attempt succeeds, and that span ends as failed on its own, in both tracers. The recorder's publication queues are gone; it keeps only the spans of the current unit of work until the write that follows the commit. --- packages/Ecotone/src/Tracing/ActiveSpan.php | 4 + .../Ecotone/src/Tracing/EcotoneActiveSpan.php | 17 +++ .../src/Tracing/StoringSpanRecorder.php | 111 ++++-------------- .../src/Tracing/TracingInterceptor.php | 19 ++- .../src/OpenTelemetryActiveSpan.php | 17 +++ .../SwallowedFailureTracingTest.php | 54 +++++++++ upgrade-2.0.md | 4 +- 7 files changed, 138 insertions(+), 88 deletions(-) create mode 100644 packages/OpenTelemetry/tests/Integration/SwallowedFailureTracingTest.php diff --git a/packages/Ecotone/src/Tracing/ActiveSpan.php b/packages/Ecotone/src/Tracing/ActiveSpan.php index 3d2af0123..4d7b58398 100644 --- a/packages/Ecotone/src/Tracing/ActiveSpan.php +++ b/packages/Ecotone/src/Tracing/ActiveSpan.php @@ -26,5 +26,9 @@ public function addEvent(string $name, array $attributes = []): void; public function end(): void; + public function markUnitOfWorkFailed(): void; + + public function markUnitOfWorkRecovered(): void; + public function fail(Throwable $failure): void; } diff --git a/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php b/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php index 14361c04b..ed0e03671 100644 --- a/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php +++ b/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php @@ -18,9 +18,11 @@ */ final class EcotoneActiveSpan implements ActiveSpan { + public const UNIT_OF_WORK_FAILED = 'The unit of work this span wraps failed and its failure was handled by the error channel or the acknowledgement strategy'; /** @var SpanEvent[] */ private array $events = []; private bool $ended = false; + private bool $unitOfWorkFailed = false; /** * @param array $attributes @@ -54,9 +56,24 @@ public function addEvent(string $name, array $attributes = []): void public function end(): void { + if ($this->unitOfWorkFailed) { + $this->finish(SpanStatus::Error, self::UNIT_OF_WORK_FAILED); + + return; + } $this->finish(SpanStatus::Ok, null); } + public function markUnitOfWorkFailed(): void + { + $this->unitOfWorkFailed = true; + } + + public function markUnitOfWorkRecovered(): void + { + $this->unitOfWorkFailed = false; + } + public function fail(Throwable $failure): void { $this->addEvent('exception', [ diff --git a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php index 90dabcb7c..e7fbc7a89 100644 --- a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php +++ b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php @@ -37,18 +37,15 @@ */ final class StoringSpanRecorder implements SpanRecorder { + private const NOTIFICATION_RETRY_DELAY_IN_SECONDS = 30; + private const NOTIFICATION_RETRY_BATCH = 10; + /** @var Span[] */ private array $buffer = []; - private bool $scopeActive = false; /** @var Span[] */ - private array $endedSpansToPublish = []; - /** @var string[] */ - private array $completedTracesToPublish = []; - private const NOTIFICATION_RETRY_DELAY_IN_SECONDS = 30; - private const NOTIFICATION_RETRY_BATCH = 10; + private array $flushedAwaitingCommit = []; + private bool $scopeActive = false; private bool $publishing = false; - /** @var array */ - private array $spansWrappingAFailedUnitOfWork = []; public function __construct( private TraceStore $traceStore, @@ -71,18 +68,14 @@ public function record(Span $span): void return; } - if (isset($this->spansWrappingAFailedUnitOfWork[$span->spanId])) { - unset($this->spansWrappingAFailedUnitOfWork[$span->spanId]); - $span = self::failedByItsUnitOfWork($span); - } + $spans = [...$this->buffer, $span]; + $flushed = $this->flushedAwaitingCommit; + $this->buffer = []; + $this->flushedAwaitingCommit = []; if ($span->status === SpanStatus::Error && ! $this->storesFailuresOutsideTransaction) { return; } - - $spans = [...$this->buffer, $span]; - $this->buffer = []; - $this->write($spans, true); - $this->publish(); + $this->write($spans, $flushed); } public function openScope(): void @@ -97,12 +90,13 @@ public function isScopeActive(): bool public function flush(): void { - foreach (self::outerParentsOf($this->buffer) as $spanId) { - unset($this->spansWrappingAFailedUnitOfWork[$spanId]); - } $spans = $this->buffer; $this->buffer = []; - $this->write($spans, false); + if ($spans === []) { + return; + } + $this->traceStore->append($spans); + $this->flushedAwaitingCommit = [...$this->flushedAwaitingCommit, ...$spans]; } public function closeScope(): void @@ -112,75 +106,24 @@ public function closeScope(): void public function discard(): void { - foreach (self::outerParentsOf($this->buffer) as $spanId) { - $this->spansWrappingAFailedUnitOfWork[$spanId] = true; - } if (! $this->storesFailuresOutsideTransaction) { $this->buffer = []; + $this->flushedAwaitingCommit = []; } } /** * @param Span[] $spans - * @return string[] + * @param Span[] $flushedBefore */ - private static function outerParentsOf(array $spans): array + private function write(array $spans, array $flushedBefore): void { - $spanIds = []; - foreach ($spans as $span) { - $spanIds[$span->spanId] = true; - } - $parents = []; - foreach ($spans as $span) { - if ($span->parentSpanId !== null && ! isset($spanIds[$span->parentSpanId])) { - $parents[$span->parentSpanId] = true; - } - } - - return array_keys($parents); - } - - private static function failedByItsUnitOfWork(Span $span): Span - { - if ($span->status === SpanStatus::Error) { - return $span; - } - - return new Span( - $span->traceId, - $span->spanId, - $span->parentSpanId, - $span->name, - $span->kind, - $span->attributes, - SpanStatus::Error, - 'The unit of work this span wraps failed and its failure was handled by the error channel or the acknowledgement strategy', - $span->events, - $span->startedAt, - $span->endedAt, - ); - } - - /** - * @param Span[] $spans - */ - private function write(array $spans, bool $decidesCompletion): void - { - if ($spans === []) { - return; - } - $this->traceStore->append($spans); - $completedTraces = $decidesCompletion ? $this->completedAmong($spans) : []; + $completedTraces = $this->completedAmong([...$flushedBefore, ...$spans]); if ($this->publishing) { return; } - if ($this->publishesEndedSpans) { - $this->endedSpansToPublish = [...$this->endedSpansToPublish, ...$spans]; - } - if ($this->publishesCompletedTraces) { - $this->completedTracesToPublish = [...$this->completedTracesToPublish, ...$completedTraces]; - } + $this->publish($this->publishesEndedSpans ? [...$flushedBefore, ...$spans] : [], $this->publishesCompletedTraces ? $completedTraces : []); } /** @@ -199,16 +142,12 @@ private function completedAmong(array $spans): array return $completed; } - private function publish(): void + /** + * @param Span[] $endedSpans + * @param string[] $completedTraces + */ + private function publish(array $endedSpans, array $completedTraces): void { - if ($this->publishing) { - return; - } - $endedSpans = $this->endedSpansToPublish; - $completedTraces = $this->completedTracesToPublish; - $this->endedSpansToPublish = []; - $this->completedTracesToPublish = []; - $this->publishing = true; try { foreach ($endedSpans as $span) { diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 6ab2a8b3d..7662c1236 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -170,7 +170,24 @@ private function traceHandler(MethodInvocation $methodInvocation, Message $messa $attributes[SpanAttribute::ENDPOINT_ID] = $endpointId; } - return $this->trace('process ' . self::shortNameOf($methodInvocation->getName()), $methodInvocation, $message, SpanKind::Server, $attributes); + return $this->traceUnitOfWorkStep('process ' . self::shortNameOf($methodInvocation->getName()), $methodInvocation, $message, $attributes); + } + + /** + * @param array $attributes + */ + private function traceUnitOfWorkStep(string $name, MethodInvocation $methodInvocation, Message $message, array $attributes): mixed + { + try { + $result = $this->trace($name, $methodInvocation, $message, SpanKind::Server, $attributes); + } catch (Throwable $failure) { + $this->tracer->currentSpan()?->markUnitOfWorkFailed(); + + throw $failure; + } + $this->tracer->currentSpan()?->markUnitOfWorkRecovered(); + + return $result; } private static function shortNameOf(string $classAndMethod): string diff --git a/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php b/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php index d473c616f..288f57b4e 100644 --- a/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php +++ b/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php @@ -12,6 +12,7 @@ use Ecotone\Api\Tracing\SpanKind; use Ecotone\Api\Tracing\SpanStatus; use Ecotone\Tracing\ActiveSpan; +use Ecotone\Tracing\EcotoneActiveSpan; use OpenTelemetry\API\Trace\SpanInterface; use OpenTelemetry\API\Trace\StatusCode; use OpenTelemetry\Context\ScopeInterface; @@ -25,6 +26,7 @@ final class OpenTelemetryActiveSpan implements ActiveSpan /** @var SpanEvent[] */ private array $events = []; private bool $ended = false; + private bool $unitOfWorkFailed = false; /** * @param array $attributes @@ -61,9 +63,24 @@ public function addEvent(string $name, array $attributes = []): void public function end(): void { + if ($this->unitOfWorkFailed) { + $this->finish(StatusCode::STATUS_ERROR, SpanStatus::Error, EcotoneActiveSpan::UNIT_OF_WORK_FAILED); + + return; + } $this->finish(StatusCode::STATUS_OK, SpanStatus::Ok, null); } + public function markUnitOfWorkFailed(): void + { + $this->unitOfWorkFailed = true; + } + + public function markUnitOfWorkRecovered(): void + { + $this->unitOfWorkFailed = false; + } + public function fail(Throwable $failure): void { if ($this->ended) { diff --git a/packages/OpenTelemetry/tests/Integration/SwallowedFailureTracingTest.php b/packages/OpenTelemetry/tests/Integration/SwallowedFailureTracingTest.php new file mode 100644 index 000000000..b3d5a7985 --- /dev/null +++ b/packages/OpenTelemetry/tests/Integration/SwallowedFailureTracingTest.php @@ -0,0 +1,54 @@ + self::prepareTracer($exporter)], + [SimpleMessageChannelBuilder::createQueueChannel('async_channel'), ServiceConfiguration::createWithDefaults()->withDefaultErrorChannel('errorChannel')], + enabledModules: [ModulePackageList::TRACING_PACKAGE], + ) + ->sendCommandWithRouting('order.place', '1') + ->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); + + $statuses = []; + foreach ($exporter->getSpans() as $span) { + $statuses[$span->getName()] = $span->getStatus()->getCode(); + } + $this->assertSame(StatusCode::STATUS_ERROR, $statuses['receive async_channel']); + $this->assertSame(StatusCode::STATUS_ERROR, $statuses['process ' . substr($orders::class, 0, strpos($orders::class, "\0")) . '::place']); + } +} diff --git a/upgrade-2.0.md b/upgrade-2.0.md index 4db2c8e4d..a5a6400e5 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3118,7 +3118,9 @@ to storing the failed outer span then, so a failed trace shows its error and cou its message id, so a redelivery of the same message satisfies the delivery it belongs to and no other, and a message sent to an asynchronous handler is attributed to that handler's `endpointId` whether it was routed by the event bus, the command bus or an `outputChannelName`. The handler span is the innermost interceptor around the -method, so it records the handler's own failure even when an error channel or a retry handles it, and +method, so it records the handler's own failure even when an error channel or a retry handles it, and the consumer +or bus span that wraps a failed unit of work ends as failed too, in OpenTelemetry as in the store, unless a retry +inside it succeeded; and `TraceCompletionTimedOut` names the handlers whose last attempt failed, the channels still holding a message, a trace nobody recorded, and a destination the trace never reached. `TraceCompletion` is always injectable; without `withStoredTraces()` every call refuses with the configuration it needs. A method with `#[OnTraceCompleted]` receives a From 151702fff63c0de7cbccbd446bb93be1b04108f2 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 24/50] refactor(dbal): the spans table is named ecotone_tracing --- packages/Dbal/src/Database/SpansTableManager.php | 2 +- upgrade-2.0.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/Dbal/src/Database/SpansTableManager.php b/packages/Dbal/src/Database/SpansTableManager.php index c894319ef..13a0d3c34 100644 --- a/packages/Dbal/src/Database/SpansTableManager.php +++ b/packages/Dbal/src/Database/SpansTableManager.php @@ -30,7 +30,7 @@ */ final class SpansTableManager implements DbalTableManager { - public const DEFAULT_TABLE = 'ecotone_spans'; + public const DEFAULT_TABLE = 'ecotone_tracing'; public const FEATURE_NAME = 'tracing'; public function __construct( diff --git a/upgrade-2.0.md b/upgrade-2.0.md index a5a6400e5..a4937bf54 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3098,7 +3098,7 @@ to know when the projection had caught up: it polled its own read model, or gues in an external collector, after the fact. **Now:** `TracingConfiguration::createWithDefaults()->withStoredTraces()` keeps every span in the database as well, -in `ecotone_traces` and `ecotone_spans`, created by `ecotone:migration:database:setup`. A unit of work, a bus call +in `ecotone_traces` and `ecotone_tracing`, created by `ecotone:migration:database:setup`. A unit of work, a bus call or an asynchronous consumption, buffers its spans and writes them once, inside its transaction before the commit; the span that wraps the unit of work is written when it ends, after the commit, and that write is the one that decides whether the trace is complete, reading what every other unit of work has committed by then. No row is From 80f88c65502da16164022681dc7a04bf31bbcdef Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 25/50] fix(tracing): stored traces never write to the database after the transaction, so an outage after a commit cannot fail a consumer or a command The span wrapping a unit of work was inserted when it ended, after the commit, together with the completion claim, and with storeFailuresOutsideTransaction the failed spans were inserted after the rollback. With the database down at that moment the write threw out of a consumer whose message was already acknowledged, or out of a command whose data had committed, instead of leaving the message to the retry. Now the flush inside the transaction writes the buffered spans together with the wrapping span as it stands at that point and claims completion in the same transaction; an interceptor at the new Precedence::TRACE_PUBLICATION_PRECEDENCE, outside the instant retries and the transaction, publishes the committed spans and completions from memory and never touches the database. Queries are no longer recorded. storeFailuresOutsideTransaction, the notified_at column and the post-commit notification retry are removed; notifications are at most once, and failures are visible in OpenTelemetry. AGENTS.md and the coding conventions gain rule 16a: no database write after the transaction, telemetry excepted. --- AGENTS.md | 1 + docs/coding-conventions.md | 42 +++++- .../FlowTestingDbalConnections.php | 2 +- .../Dbal/src/Database/TracesTableManager.php | 2 - packages/Dbal/src/Tracing/DbalTraceStore.php | 29 +--- .../Conformance/TraceStoreConformanceTest.php | 102 ++++++++++---- .../Conformance/TraceStoreContractTest.php | 4 +- .../Ecotone/Api/Interceptor/Precedence.php | 2 + .../Api/Tracing/TraceCompletionTimedOut.php | 20 ++- .../Api/Tracing/TracingConfiguration.php | 7 +- packages/Ecotone/src/Tracing/ActiveSpan.php | 3 + .../src/Tracing/Config/StoredTracesModule.php | 22 +-- .../Ecotone/src/Tracing/EcotoneActiveSpan.php | 17 +++ .../src/Tracing/InMemoryTraceStore.php | 28 +--- .../src/Tracing/StoredTraceCompletion.php | 4 +- .../src/Tracing/StoringSpanRecorder.php | 128 ++++++++---------- packages/Ecotone/src/Tracing/TraceLedger.php | 5 +- .../Ecotone/src/Tracing/TraceProgress.php | 23 +--- .../src/Tracing/TraceRecordingInterceptor.php | 26 +++- packages/Ecotone/src/Tracing/TraceStore.php | 7 - .../tests/Tracing/TraceCompletionTest.php | 60 +------- .../src/OpenTelemetryActiveSpan.php | 17 +++ upgrade-2.0.md | 31 +++-- 23 files changed, 295 insertions(+), 287 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ae5bdd019..978b7ff89 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -115,6 +115,7 @@ commit that evidences it. The summary: | 14a | **A framework header is named `ecotone_{module}_{feature}_{action}`** — lower snake_case, no dots, e.g. `ecotone_dbal_deadletter_limit`, held in one constant. Exempt: the Spring-shared core headers (`id`, `timestamp`, `correlationId`, `parentId`, `contentType`, …), the event store's `_aggregate_*` keys, and `amqp_*`/`kafka_*` broker copies. A header written to a queue, outbox, dead letter or event store is renamed only with a read path for its old name | The prefix keeps framework headers apart from application ones; the event store reads a dotted metadata key as a nested JSON path on MySQL and SQLite but literally on PostgreSQL | | 15 | **Orchestrating methods read as step lists**; SQL belongs to the collaborator that owns the table | | | 16 | **No table is created on the message path**, on any engine. Tables come from `ecotone:migration:database:setup` (or your migrations) and, in tests, from `FlowTestSupport::setupDatabaseTables()`, which the `EcotoneLite` bootstrap calls itself unless `TestConfiguration::withDatabaseTablesSetUpOnBootstrap(false)`. A missing table raises its table manager's missing-table `ConfigurationException`, found by the failing statement (`MissingTableFailure`), never by asking the schema first | MySQL/MariaDB implicitly commit on DDL, and a per-engine gate made table creation depend on the database. A schema check costs a round trip per operation, and memoising it is a rule-3 cache | +| 16a | **No database write after the transaction.** Whatever a feature stores about a message is written inside that message's transaction, or not at all — never after the commit, after a rollback, or around the transaction. Telemetry sent to an external collector (OpenTelemetry) is exempt, because it is allowed to fail | A write after the commit runs when the work is already done: with the database down it fails the consumer or the command for work that succeeded, and the message is no longer retried. Inside the transaction the same outage rolls back and leaves the message to be retried | | 17 | New files are `final` and `declare(strict_types=1)` | | | 18 | PHP 8.1+ features where they say something; named arguments once past two parameters | | diff --git a/docs/coding-conventions.md b/docs/coding-conventions.md index b01898526..a4a2d3ba1 100644 --- a/docs/coding-conventions.md +++ b/docs/coding-conventions.md @@ -15,7 +15,7 @@ branch `dgafka/dcb-docs` and is deliberately untracked here, since `docs/superpo paths resolve on that branch. Commit SHAs cited below resolve on any branch. Rules 1-11 are the maintainer's. Rules 12-19 are conventions the code holds to consistently. Section 20 lists the -mechanics a tool enforces for you, and section 21 the gates and landmines. Lettered sub-rules (1a, 1b, 1c, 6a, 10a, +mechanics a tool enforces for you, and section 21 the gates and landmines. Lettered sub-rules (1a, 1b, 1c, 6a, 10a, 16a, 12a, 13a, 14a) continue the rule they hang off; they are numbered that way so nothing above them ever renumbers. > **Verify names before you write them.** Every attribute, parameter, method and console option in code you write @@ -1141,6 +1141,46 @@ History: `230bcea5d` and `511b31428` first moved this DDL out of the transaction --- +## 16a. No database write after the transaction + +A message's unit of work owns a transaction: the transaction interceptor begins it, the handler and everything it +calls write inside it, and it commits or rolls back as one. Anything a feature records about that message - stored +traces, bookkeeping, a completion flag - is written inside the same transaction, or not at all. + +Never write after the commit, after a rollback, or in an interceptor that wraps the transaction from outside. At that +point the work is already decided. If the database is down, the write throws out of a consumer whose message was +already acknowledged, or out of a command whose data already committed: the caller sees a failure for work that +succeeded, and the message is not retried. The same outage inside the transaction rolls back and leaves the message +to the retry and dead-letter strategy, which is what they exist for. + +Wrong - the root span is inserted when it ends, after the commit: + +```php +public function record(Span $span): void +{ + if ($this->scopeActive) { $this->buffer[] = $span; return; } + $this->traceStore->append([$span]); // after the commit: fails a consumer whose work succeeded +} +``` + +Right - the span that wraps the unit of work is written as it stands at the flush, inside the transaction, and +nothing touches the database once the transaction has returned: + +```php +$result = $methodInvocation->proceed(); +$this->recorder->flush($wrappingSpan?->endedSoFar()); // inside the transaction +``` + +What may happen after the commit is work that does not depend on the database: publishing to subscribers from +memory, and sending telemetry to an external collector such as OpenTelemetry, which is allowed to fail and never +fails the message. A test proves the rule by making the database unavailable right after the commit, with an +`#[Around]` interceptor just outside the transaction, and asserting that the command or consumer still succeeds. + +Evidence: stored traces wrote the root span and claimed completion after the commit, and an outage at that moment +threw the missing-table refusal out of the consumer (`TraceStoreConformanceTest`, the two outage tests). + +--- + ## 17. `final` and `declare(strict_types=1)` on new files 113 of the 121 classes added since 1.x are `final`; 132 of 133 new `src` files declare strict types. Older files diff --git a/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php b/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php index 5f45183c4..2012ca211 100644 --- a/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php +++ b/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php @@ -111,7 +111,7 @@ private function tracesInMemoryWhereTheConnectionIsMissing(TracingConfiguration return $tracingConfiguration; } - return $tracingConfiguration->withStoredTraces(inMemory: true, retention: $tracingConfiguration->storedTracesRetention, storeFailuresOutsideTransaction: $tracingConfiguration->storedFailuresOutsideTransaction, onlyTracesCarryingTraceparent: $tracingConfiguration->storedTracesOnlyCarryingTraceparent); + return $tracingConfiguration->withStoredTraces(inMemory: true, retention: $tracingConfiguration->storedTracesRetention, onlyTracesCarryingTraceparent: $tracingConfiguration->storedTracesOnlyCarryingTraceparent); } private function refuseDbalChannelsWithoutTheirConnection(): void diff --git a/packages/Dbal/src/Database/TracesTableManager.php b/packages/Dbal/src/Database/TracesTableManager.php index b35a6ef0b..04214f143 100644 --- a/packages/Dbal/src/Database/TracesTableManager.php +++ b/packages/Dbal/src/Database/TracesTableManager.php @@ -109,10 +109,8 @@ private function buildTableSchema(): Table $table->addColumn('trace_id', Types::STRING, ['length' => 32]); $table->addColumn('started_at', Types::BIGINT); $table->addColumn('completed_at', Types::BIGINT, ['notnull' => false]); - $table->addColumn('notified_at', Types::BIGINT, ['notnull' => false]); $table->setPrimaryKey(['trace_id']); $table->addIndex(['started_at']); - $table->addIndex(['completed_at', 'notified_at']); return $table; } diff --git a/packages/Dbal/src/Tracing/DbalTraceStore.php b/packages/Dbal/src/Tracing/DbalTraceStore.php index 82a88ebab..66eecd1bf 100644 --- a/packages/Dbal/src/Tracing/DbalTraceStore.php +++ b/packages/Dbal/src/Tracing/DbalTraceStore.php @@ -81,31 +81,6 @@ public function markCompletedIfDone(string $traceId, DatePoint $now): bool return $this->onTables(fn (): bool => TraceLedger::progressOf($traceId, $this->spansOf($connection, $traceId))->isCompleted() && $this->markCompleted($connection, $traceId, $now)); } - public function markNotified(string $traceId): void - { - $connection = $this->connection(); - $this->onTables(fn (): int => $connection->executeStatement( - "UPDATE {$this->tracesTableManager->getTableName()} SET notified_at = :notifiedAt WHERE trace_id = :traceId AND completed_at IS NOT NULL AND notified_at IS NULL", - ['notifiedAt' => $this->clock->now()->unixTime()->inMicroseconds(), 'traceId' => $traceId], - ['notifiedAt' => Types::BIGINT], - )); - } - - public function completedButNotNotifiedBefore(DatePoint $completedBefore, int $limit): array - { - return $this->onTables(fn (): array => $this->connection()->createQueryBuilder() - ->select('trace_id') - ->from($this->tracesTableManager->getTableName()) - ->andWhere('completed_at IS NOT NULL') - ->andWhere('notified_at IS NULL') - ->andWhere('completed_at < :completedBefore') - ->setParameter('completedBefore', $completedBefore->unixTime()->inMicroseconds(), Types::BIGINT) - ->orderBy('completed_at') - ->setMaxResults($limit) - ->executeQuery() - ->fetchFirstColumn()); - } - public function progressOf(string $traceId): TraceProgress { return $this->onTables(fn (): TraceProgress => TraceLedger::progressOf($traceId, $this->spansOf($this->connection(), $traceId))); @@ -131,8 +106,8 @@ private function claimTrace(Connection $connection, string $traceId, array $trac $startedAt = min(array_map(fn (Span $span): int => $span->startedAt->unixTime()->inMicroseconds(), $traceSpans)); $tableName = $this->tracesTableManager->getTableName(); $sql = $connection->getDatabasePlatform() instanceof AbstractMySQLPlatform - ? "INSERT INTO {$tableName} (trace_id, started_at, completed_at, notified_at) VALUES (:traceId, :startedAt, NULL, NULL) ON DUPLICATE KEY UPDATE completed_at = NULL, notified_at = NULL" - : "INSERT INTO {$tableName} (trace_id, started_at, completed_at, notified_at) VALUES (:traceId, :startedAt, NULL, NULL) ON CONFLICT (trace_id) DO UPDATE SET completed_at = NULL, notified_at = NULL"; + ? "INSERT INTO {$tableName} (trace_id, started_at, completed_at) VALUES (:traceId, :startedAt, NULL) ON DUPLICATE KEY UPDATE completed_at = NULL" + : "INSERT INTO {$tableName} (trace_id, started_at, completed_at) VALUES (:traceId, :startedAt, NULL) ON CONFLICT (trace_id) DO UPDATE SET completed_at = NULL"; $connection->executeStatement($sql, ['traceId' => $traceId, 'startedAt' => $startedAt], ['startedAt' => Types::BIGINT]); } diff --git a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php index 4e8b34638..8b137bdc6 100644 --- a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php +++ b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php @@ -4,6 +4,7 @@ namespace Test\Ecotone\Dbal\Conformance; +use Ecotone\Api\Attribute\Around; use Ecotone\Api\Attribute\Asynchronous; use Ecotone\Api\Attribute\CommandHandler; use Ecotone\Api\Attribute\EventHandler; @@ -18,6 +19,8 @@ use Ecotone\Api\ExtensionObject\ServiceConfiguration; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; use Ecotone\Api\Gateway\EventBus; +use Ecotone\Api\Interceptor\MethodInvocation; +use Ecotone\Api\Interceptor\Precedence; use Ecotone\Api\Lite\Test\FlowTestSupport; use Ecotone\Api\Messaging\MessageHeaders; use Ecotone\Api\Scheduling\Duration; @@ -102,7 +105,7 @@ public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pen $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $this->expectException(TraceCompletionTimedOut::class); - $this->expectExceptionMessage('Still pending: user.notifier, async_channel. Run the consumer of the pending channel'); + $this->expectExceptionMessage('Still pending: user.notifier, async_channel. Run the consumer of the pending channel, replay the message from the dead letter'); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); } @@ -266,23 +269,6 @@ public function test_a_failing_asynchronous_handler_stores_nothing_by_default(st self::assertSame($storedBeforeTheConsumerRan, $subscriber->endedSpans); } - #[DataProvider('implementations')] - public function test_a_failing_asynchronous_handler_stores_its_failed_consumer_span_when_failures_may_be_stored_outside_the_transaction(string $implementation): void - { - $subscriber = $this->spanEndedSubscriber(); - $ecotone = $this->bootstrap($implementation, [$this->users(), $this->failingNotifier(), $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')], storeFailuresOutsideTransaction: true); - $trace = SpanContext::newTrace(); - $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - - try { - $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - } catch (RuntimeException) { - } - - self::assertSame(SpanStatus::Error, $subscriber->endedSpans['receive async_channel']); - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); - } - #[DataProvider('implementations')] public function test_a_consumer_that_swallowed_its_handler_failure_leaves_the_trace_pending(string $implementation): void { @@ -348,7 +334,7 @@ public function test_a_pending_trace_older_than_the_retention_survives_until_twi } #[DataProvider('implementations')] - public function test_a_subscriber_that_throws_does_not_fail_the_committed_command_and_is_notified_again_later(string $implementation): void + public function test_a_subscriber_that_throws_does_not_fail_the_committed_command_and_is_not_retried(string $implementation): void { $subscriber = new class () { /** @var string[] */ @@ -370,10 +356,9 @@ public function whenCompleted(TraceCompleted $event): void $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $first->toTraceparent()]); self::assertSame([$first->traceId], $subscriber->completedTraces); - $ecotone->advanceTimeBy(TimeSpan::withSeconds(31)); $ecotone->sendCommandWithRouting('user.register', '2', metadata: [MessageHeaders::TRACEPARENT => $second->toTraceparent()]); - self::assertEqualsCanonicalizing([$first->traceId, $first->traceId, $second->traceId], $subscriber->completedTraces); + self::assertSame([$first->traceId, $second->traceId], $subscriber->completedTraces); } #[DataProvider('implementations')] @@ -466,6 +451,49 @@ classesToResolve: [$this->users()::class], ); } + public function test_a_database_outage_right_after_the_consumer_committed_does_not_fail_the_consumer(): void + { + $notifier = new class () { + public int $handled = 0; + + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId): void + { + $this->handled++; + } + }; + $outage = $this->outageRightAfterTheTransaction(); + $ecotone = $this->bootstrap('dbal', [$this->users(), $notifier, $outage], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $ecotone->sendCommandWithRouting('user.register', '1'); + + $outage->armed = true; + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertSame(1, $notifier->handled); + } + + public function test_a_database_outage_right_after_the_command_committed_does_not_fail_the_command(): void + { + $users = new class () { + public int $registered = 0; + + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId): void + { + $this->registered++; + } + }; + $outage = $this->outageRightAfterTheTransaction(); + $ecotone = $this->bootstrap('dbal', [$users, $outage]); + + $outage->armed = true; + $ecotone->sendCommandWithRouting('user.register', '1'); + + self::assertSame(1, $users->registered); + } + public function test_completion_recorded_by_one_instance_is_visible_to_another_sharing_the_database(): void { $producer = $this->bootstrap('dbal', [$this->users()]); @@ -481,12 +509,12 @@ public function test_completion_recorded_by_one_instance_is_visible_to_another_s * @param object[] $services * @param object[] $extensionObjects */ - private function bootstrap(string $implementation, array $services, array $extensionObjects = [], ?Duration $retention = null, bool $storeFailuresOutsideTransaction = false): FlowTestSupport + private function bootstrap(string $implementation, array $services, array $extensionObjects = [], ?Duration $retention = null): FlowTestSupport { return $this->bootstrapFlowTesting( classesToResolve: array_map(fn (object $service): string => $service::class, $services), containerOrAvailableServices: [...$services, DbalConnectionFactory::class => $this->getConnectionFactory()], - extensionObjects: [TracingConfiguration::createWithDefaults()->withStoredTraces(inMemory: $implementation === 'in-memory', retention: $retention, storeFailuresOutsideTransaction: $storeFailuresOutsideTransaction), ...$extensionObjects], + extensionObjects: [TracingConfiguration::createWithDefaults()->withStoredTraces(inMemory: $implementation === 'in-memory', retention: $retention), ...$extensionObjects], enabledModules: [ModulePackageList::DBAL_PACKAGE], licenceKey: LicenceTesting::VALID_LICENCE, )->setupDatabaseTables(); @@ -508,6 +536,34 @@ public function register(string $userId, EventBus $eventBus): void }; } + private function outageRightAfterTheTransaction(): object + { + return new class ($this->getConnectionFactory()) { + public bool $armed = false; + + public function __construct(private DbalConnectionFactory $connectionFactory) + { + } + + #[Around(precedence: Precedence::DATABASE_TRANSACTION_PRECEDENCE - 1, pointcut: 'Ecotone\\Api\\Gateway\\CommandBus||Ecotone\\Api\\Attribute\\AsynchronousRunningEndpoint')] + public function simulateOutage(MethodInvocation $methodInvocation): mixed + { + $result = $methodInvocation->proceed(); + if (! $this->armed) { + return $result; + } + $schemaManager = $this->connectionFactory->establishConnection()->createSchemaManager(); + foreach (['ecotone_tracing', 'ecotone_traces'] as $table) { + if ($schemaManager->tablesExist([$table])) { + $schemaManager->dropTable($table); + } + } + + return $result; + } + }; + } + private function failingNotifier(): object { return new class () { diff --git a/packages/Dbal/tests/Conformance/TraceStoreContractTest.php b/packages/Dbal/tests/Conformance/TraceStoreContractTest.php index 28e72c683..2f1995d3f 100644 --- a/packages/Dbal/tests/Conformance/TraceStoreContractTest.php +++ b/packages/Dbal/tests/Conformance/TraceStoreContractTest.php @@ -63,18 +63,16 @@ public function test_appending_the_same_span_twice_records_it_once(string $imple } #[DataProvider('implementations')] - public function test_a_span_appended_after_completion_reopens_the_trace_so_it_can_complete_and_notify_again(string $implementation): void + public function test_a_span_appended_after_completion_reopens_the_trace_so_it_can_complete_again(string $implementation): void { $store = $this->store($implementation); $trace = SpanContext::newTrace(); $store->append([$this->rootSpan($trace)]); self::assertTrue($store->markCompletedIfDone($trace->traceId, new DatePoint())); - $store->markNotified($trace->traceId); $store->append([$this->rootSpan($trace)]); self::assertTrue($store->markCompletedIfDone($trace->traceId, new DatePoint())); - self::assertSame([$trace->traceId], $store->completedButNotNotifiedBefore((new DatePoint())->modify('+1 minute'), 10)); } private function store(string $implementation): TraceStore diff --git a/packages/Ecotone/Api/Interceptor/Precedence.php b/packages/Ecotone/Api/Interceptor/Precedence.php index 5502f6467..45bdb89e7 100644 --- a/packages/Ecotone/Api/Interceptor/Precedence.php +++ b/packages/Ecotone/Api/Interceptor/Precedence.php @@ -23,6 +23,8 @@ interface Precedence public const ENDPOINT_HEADERS_PRECEDENCE = -3000; + public const TRACE_PUBLICATION_PRECEDENCE = self::CUSTOM_INSTANT_RETRY_PRECEDENCE - 1; + public const CUSTOM_INSTANT_RETRY_PRECEDENCE = -2003; public const GLOBAL_INSTANT_RETRY_PRECEDENCE = -2002; diff --git a/packages/Ecotone/Api/Tracing/TraceCompletionTimedOut.php b/packages/Ecotone/Api/Tracing/TraceCompletionTimedOut.php index bbd9e7b95..a0e4ee83b 100644 --- a/packages/Ecotone/Api/Tracing/TraceCompletionTimedOut.php +++ b/packages/Ecotone/Api/Tracing/TraceCompletionTimedOut.php @@ -18,16 +18,15 @@ final class TraceCompletionTimedOut extends RuntimeException { /** * @param string[] $pendingDestinations - * @param string[] $failedDestinations */ - public static function forTrace(string $traceId, Duration $timeout, array $pendingDestinations, array $failedDestinations): self + public static function forTrace(string $traceId, Duration $timeout, array $pendingDestinations): self { - $failed = $failedDestinations === [] ? '' : sprintf(' Failed: %s, whose last attempt failed and whose message is in the error channel or awaiting a retry.', implode(', ', $failedDestinations)); - $pending = array_values(array_diff($pendingDestinations, $failedDestinations)); - $stillPending = $pending === [] ? '' : sprintf(' Still pending: %s.', implode(', ', $pending)); - $advice = $pending === [] ? ' Await with a longer timeout once the failure is resolved.' : ' Run the consumer of the pending channel, or await with a longer timeout.'; - - return new self(sprintf('Trace %s did not complete within %d milliseconds.%s%s%s', $traceId, $timeout->inMilliseconds(), $failed, $stillPending, $advice)); + return new self(sprintf( + 'Trace %s did not complete within %d milliseconds. Still pending: %s. Run the consumer of the pending channel, replay the message from the dead letter if its handler failed, or await with a longer timeout.', + $traceId, + $timeout->inMilliseconds(), + implode(', ', $pendingDestinations), + )); } public static function forUnrecordedTrace(string $traceId, Duration $timeout): self @@ -39,14 +38,13 @@ public static function forUnrecordedTrace(string $traceId, Duration $timeout): s )); } - public static function forDestination(string $traceId, string $destination, Duration $timeout, bool $failed): self + public static function forDestination(string $traceId, string $destination, Duration $timeout): self { return new self(sprintf( - 'Trace %s did not complete for %s within %d milliseconds.%s', + 'Trace %s did not complete for %s within %d milliseconds. Run the consumer of the channel that handler is attached to, replay the message from the dead letter if the handler failed, or await with a longer timeout.', $traceId, $destination, $timeout->inMilliseconds(), - $failed ? ' Its last attempt failed and its message is in the error channel or awaiting a retry.' : ' Run the consumer of the channel that handler is attached to, or await with a longer timeout.', )); } diff --git a/packages/Ecotone/Api/Tracing/TracingConfiguration.php b/packages/Ecotone/Api/Tracing/TracingConfiguration.php index 499fefb6a..7b3c549be 100644 --- a/packages/Ecotone/Api/Tracing/TracingConfiguration.php +++ b/packages/Ecotone/Api/Tracing/TracingConfiguration.php @@ -20,7 +20,6 @@ private function __construct( public readonly bool $storedTraces, public readonly bool $storedTracesInMemory, public readonly Duration $storedTracesRetention, - public readonly bool $storedFailuresOutsideTransaction, public readonly bool $spanAttributesFromAllHeaders, public readonly array $spanAttributeHeaders, public readonly bool $storedTracesOnlyCarryingTraceparent, @@ -29,7 +28,7 @@ private function __construct( public static function createWithDefaults(): self { - return new self([], false, false, Duration::minutes(60 * 24 * 7), false, true, [], false); + return new self([], false, false, Duration::minutes(60 * 24 * 7), true, [], false); } public function withSpanRecorder(string $referenceName): self @@ -42,8 +41,8 @@ public function withSpanAttributesFromHeaders(string ...$headerNames): self return new self(...[...get_object_vars($this), 'spanAttributesFromAllHeaders' => false, 'spanAttributeHeaders' => array_values($headerNames)]); } - public function withStoredTraces(bool $inMemory = false, ?Duration $retention = null, bool $storeFailuresOutsideTransaction = false, bool $onlyTracesCarryingTraceparent = false): self + public function withStoredTraces(bool $inMemory = false, ?Duration $retention = null, bool $onlyTracesCarryingTraceparent = false): self { - return new self(...[...get_object_vars($this), 'storedTraces' => true, 'storedTracesInMemory' => $inMemory, 'storedTracesRetention' => $retention ?? $this->storedTracesRetention, 'storedFailuresOutsideTransaction' => $storeFailuresOutsideTransaction, 'storedTracesOnlyCarryingTraceparent' => $onlyTracesCarryingTraceparent]); + return new self(...[...get_object_vars($this), 'storedTraces' => true, 'storedTracesInMemory' => $inMemory, 'storedTracesRetention' => $retention ?? $this->storedTracesRetention, 'storedTracesOnlyCarryingTraceparent' => $onlyTracesCarryingTraceparent]); } } diff --git a/packages/Ecotone/src/Tracing/ActiveSpan.php b/packages/Ecotone/src/Tracing/ActiveSpan.php index 4d7b58398..f1af96a8d 100644 --- a/packages/Ecotone/src/Tracing/ActiveSpan.php +++ b/packages/Ecotone/src/Tracing/ActiveSpan.php @@ -4,6 +4,7 @@ namespace Ecotone\Tracing; +use Ecotone\Api\Tracing\Span; use Ecotone\Api\Tracing\SpanContext; use Throwable; @@ -26,6 +27,8 @@ public function addEvent(string $name, array $attributes = []): void; public function end(): void; + public function endedSoFar(): Span; + public function markUnitOfWorkFailed(): void; public function markUnitOfWorkRecovered(): void; diff --git a/packages/Ecotone/src/Tracing/Config/StoredTracesModule.php b/packages/Ecotone/src/Tracing/Config/StoredTracesModule.php index 93a40f2f1..4e005e8c4 100644 --- a/packages/Ecotone/src/Tracing/Config/StoredTracesModule.php +++ b/packages/Ecotone/src/Tracing/Config/StoredTracesModule.php @@ -42,6 +42,7 @@ use Ecotone\Tracing\InMemoryTraceStore; use Ecotone\Tracing\StoredTraceCompletion; use Ecotone\Tracing\StoringSpanRecorder; +use Ecotone\Tracing\Tracer; use Ecotone\Tracing\TraceRecordingInterceptor; use Ecotone\Tracing\TraceStore; @@ -172,23 +173,24 @@ private function registerStoredTraces(Configuration $messagingConfiguration, arr new Reference(MessagingEntrypointService::class), $this->spanEndedSubscribers !== [], $this->traceCompletedSubscribers !== [], - $tracingConfiguration->storedFailuresOutsideTransaction, $tracingConfiguration->storedTracesOnlyCarryingTraceparent, new Reference(EcotoneClockInterface::class), ]) ); $messagingConfiguration->registerServiceDefinition( TraceRecordingInterceptor::class, - new Definition(TraceRecordingInterceptor::class, [new Reference(StoringSpanRecorder::class)]) - ); - $messagingConfiguration->registerAroundMethodInterceptor( - AroundInterceptorBuilder::create( - TraceRecordingInterceptor::class, - $interfaceToCallRegistry->getFor(TraceRecordingInterceptor::class, 'recordUnitOfWork'), - Precedence::TRACE_RECORDING_PRECEDENCE, - CommandBus::class . '||' . EventBus::class . '||' . QueryBus::class . '||' . AsynchronousRunningEndpoint::class . '||' . OrchestratorGateway::class . '||(' . self::USER_MESSAGE_GATEWAYS_POINTCUT . ')' - ) + new Definition(TraceRecordingInterceptor::class, [new Reference(StoringSpanRecorder::class), new Reference(Tracer::class)]) ); + foreach (['publishAfterCommit' => Precedence::TRACE_PUBLICATION_PRECEDENCE, 'recordUnitOfWork' => Precedence::TRACE_RECORDING_PRECEDENCE] as $method => $precedence) { + $messagingConfiguration->registerAroundMethodInterceptor( + AroundInterceptorBuilder::create( + TraceRecordingInterceptor::class, + $interfaceToCallRegistry->getFor(TraceRecordingInterceptor::class, $method), + $precedence, + CommandBus::class . '||' . EventBus::class . '||' . AsynchronousRunningEndpoint::class . '||' . OrchestratorGateway::class . '||(' . self::USER_MESSAGE_GATEWAYS_POINTCUT . ')' + ) + ); + } $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnTraceCompleted::CHANNEL)); $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnSpanEnded::CHANNEL)); diff --git a/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php b/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php index ed0e03671..bfabdc093 100644 --- a/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php +++ b/packages/Ecotone/src/Tracing/EcotoneActiveSpan.php @@ -64,6 +64,23 @@ public function end(): void $this->finish(SpanStatus::Ok, null); } + public function endedSoFar(): Span + { + return new Span( + $this->context->traceId, + $this->context->spanId, + $this->parentSpanId, + $this->name, + $this->kind, + $this->attributes, + SpanStatus::Ok, + null, + $this->events, + $this->startedAt, + $this->clock->now(), + ); + } + public function markUnitOfWorkFailed(): void { $this->unitOfWorkFailed = true; diff --git a/packages/Ecotone/src/Tracing/InMemoryTraceStore.php b/packages/Ecotone/src/Tracing/InMemoryTraceStore.php index f310830da..6bc02cb99 100644 --- a/packages/Ecotone/src/Tracing/InMemoryTraceStore.php +++ b/packages/Ecotone/src/Tracing/InMemoryTraceStore.php @@ -31,8 +31,6 @@ final class InMemoryTraceStore implements TraceStore private array $spansByTrace = []; /** @var array */ private array $completedAt = []; - /** @var array */ - private array $notified = []; public function append(array $spans): void { @@ -41,7 +39,7 @@ public function append(array $spans): void continue; } $this->spansByTrace[$span->traceId][$span->spanId] = $span; - unset($this->completedAt[$span->traceId], $this->notified[$span->traceId]); + unset($this->completedAt[$span->traceId]); } } @@ -55,28 +53,6 @@ public function markCompletedIfDone(string $traceId, DatePoint $now): bool return true; } - public function markNotified(string $traceId): void - { - if (isset($this->completedAt[$traceId])) { - $this->notified[$traceId] = true; - } - } - - public function completedButNotNotifiedBefore(DatePoint $completedBefore, int $limit): array - { - $traceIds = []; - foreach ($this->completedAt as $traceId => $completedAt) { - if (count($traceIds) === $limit) { - break; - } - if (! isset($this->notified[$traceId]) && $completedAt < $completedBefore) { - $traceIds[] = $traceId; - } - } - - return $traceIds; - } - public function progressOf(string $traceId): TraceProgress { return TraceLedger::progressOf($traceId, array_values($this->spansByTrace[$traceId] ?? [])); @@ -87,7 +63,7 @@ public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, Dat foreach ($this->spansByTrace as $traceId => $spans) { $startedAt = min(array_map(fn (Span $span): DatePoint => $span->startedAt, $spans)); if ($startedAt < $anyTraceStartedBefore || (isset($this->completedAt[$traceId]) && $startedAt < $completedTracesStartedBefore)) { - unset($this->spansByTrace[$traceId], $this->completedAt[$traceId], $this->notified[$traceId]); + unset($this->spansByTrace[$traceId], $this->completedAt[$traceId]); } } } diff --git a/packages/Ecotone/src/Tracing/StoredTraceCompletion.php b/packages/Ecotone/src/Tracing/StoredTraceCompletion.php index 374bffc12..83481777a 100644 --- a/packages/Ecotone/src/Tracing/StoredTraceCompletion.php +++ b/packages/Ecotone/src/Tracing/StoredTraceCompletion.php @@ -66,7 +66,7 @@ public function await(string $traceId, #[Header(self::TIMEOUT_HEADER)] Duration } if ($this->clock->now() >= $deadline) { throw $progress->recorded - ? TraceCompletionTimedOut::forTrace($traceId, $timeout, $progress->pendingDestinations(), $progress->failedDestinations()) + ? TraceCompletionTimedOut::forTrace($traceId, $timeout, $progress->pendingDestinations()) : TraceCompletionTimedOut::forUnrecordedTrace($traceId, $timeout); } $this->clock->sleep(Duration::milliseconds(self::POLL_INTERVAL_IN_MILLISECONDS)); @@ -86,7 +86,7 @@ public function awaitFor(string $traceId, #[Header(self::DESTINATION_HEADER)] st throw match (true) { ! $progress->recorded => TraceCompletionTimedOut::forUnrecordedTrace($traceId, $timeout), ! $progress->reached($destination) => TraceCompletionTimedOut::forUnreachedDestination($traceId, $destination, $timeout, $progress->destinations()), - default => TraceCompletionTimedOut::forDestination($traceId, $destination, $timeout, in_array($destination, $progress->failedDestinations(), true)), + default => TraceCompletionTimedOut::forDestination($traceId, $destination, $timeout), }; } $this->clock->sleep(Duration::milliseconds(self::POLL_INTERVAL_IN_MILLISECONDS)); diff --git a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php index e7fbc7a89..09e9c27c9 100644 --- a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php +++ b/packages/Ecotone/src/Tracing/StoringSpanRecorder.php @@ -5,14 +5,12 @@ namespace Ecotone\Tracing; use Ecotone\Api\Gateway\EcotoneClockInterface; -use Ecotone\Api\Scheduling\Duration; use Ecotone\Api\Tracing\OnSpanEnded; use Ecotone\Api\Tracing\OnTraceCompleted; use Ecotone\Api\Tracing\Span; use Ecotone\Api\Tracing\SpanAttribute; use Ecotone\Api\Tracing\SpanEnded; use Ecotone\Api\Tracing\SpanRecorder; -use Ecotone\Api\Tracing\SpanStatus; use Ecotone\Api\Tracing\TraceCompleted; use Ecotone\Messaging\Gateway\MessagingEntrypointService; use Throwable; @@ -37,22 +35,21 @@ */ final class StoringSpanRecorder implements SpanRecorder { - private const NOTIFICATION_RETRY_DELAY_IN_SECONDS = 30; - private const NOTIFICATION_RETRY_BATCH = 10; - /** @var Span[] */ private array $buffer = []; - /** @var Span[] */ - private array $flushedAwaitingCommit = []; private bool $scopeActive = false; + private bool $unitOfWorkActive = false; private bool $publishing = false; + /** @var Span[] */ + private array $committedSpans = []; + /** @var string[] */ + private array $completedTraces = []; public function __construct( private TraceStore $traceStore, private MessagingEntrypointService $messagingEntrypoint, private bool $publishesEndedSpans, private bool $publishesCompletedTraces, - private bool $storesFailuresOutsideTransaction, private bool $storesOnlyCarriedTraces, private EcotoneClockInterface $clock, ) { @@ -60,22 +57,26 @@ public function __construct( public function record(Span $span): void { - if ($this->storesOnlyCarriedTraces && ! ($span->attributes[SpanAttribute::TRACE_CARRIED] ?? false)) { - return; - } - if ($this->scopeActive) { + if ($this->scopeActive && $this->isStored($span)) { $this->buffer[] = $span; - - return; } - $spans = [...$this->buffer, $span]; - $flushed = $this->flushedAwaitingCommit; - $this->buffer = []; - $this->flushedAwaitingCommit = []; - if ($span->status === SpanStatus::Error && ! $this->storesFailuresOutsideTransaction) { - return; - } - $this->write($spans, $flushed); + } + + public function isUnitOfWorkActive(): bool + { + return $this->unitOfWorkActive; + } + + public function beginUnitOfWork(): void + { + $this->unitOfWorkActive = true; + } + + public function endUnitOfWork(): void + { + $this->unitOfWorkActive = false; + $this->committedSpans = []; + $this->completedTraces = []; } public function openScope(): void @@ -88,89 +89,70 @@ public function isScopeActive(): bool return $this->scopeActive; } - public function flush(): void - { - $spans = $this->buffer; - $this->buffer = []; - if ($spans === []) { - return; - } - $this->traceStore->append($spans); - $this->flushedAwaitingCommit = [...$this->flushedAwaitingCommit, ...$spans]; - } - public function closeScope(): void { $this->scopeActive = false; } - public function discard(): void + public function flush(?Span $wrappingSpanSoFar): void { - if (! $this->storesFailuresOutsideTransaction) { - $this->buffer = []; - $this->flushedAwaitingCommit = []; + $spans = $this->buffer; + $this->buffer = []; + if ($wrappingSpanSoFar !== null && $this->isStored($wrappingSpanSoFar)) { + $spans[] = $wrappingSpanSoFar; } - } - - /** - * @param Span[] $spans - * @param Span[] $flushedBefore - */ - private function write(array $spans, array $flushedBefore): void - { - $this->traceStore->append($spans); - $completedTraces = $this->completedAmong([...$flushedBefore, ...$spans]); - if ($this->publishing) { + if ($spans === []) { return; } - $this->publish($this->publishesEndedSpans ? [...$flushedBefore, ...$spans] : [], $this->publishesCompletedTraces ? $completedTraces : []); - } - /** - * @param Span[] $spans - * @return string[] - */ - private function completedAmong(array $spans): array - { - $completed = []; + $this->traceStore->append($spans); + $completedTraces = []; foreach (array_unique(array_map(fn (Span $span): string => $span->traceId, $spans)) as $traceId) { if ($this->traceStore->markCompletedIfDone($traceId, $this->clock->now())) { - $completed[] = $traceId; + $completedTraces[] = $traceId; } } + if ($this->publishing) { + return; + } + $this->committedSpans = [...$this->committedSpans, ...$spans]; + $this->completedTraces = [...$this->completedTraces, ...$completedTraces]; + } - return $completed; + public function discard(): void + { + $this->buffer = []; } - /** - * @param Span[] $endedSpans - * @param string[] $completedTraces - */ - private function publish(array $endedSpans, array $completedTraces): void + public function publishCommitted(): void { + $endedSpans = $this->publishesEndedSpans ? $this->committedSpans : []; + $completedTraces = $this->publishesCompletedTraces ? $this->completedTraces : []; + $this->endUnitOfWork(); + $this->publishing = true; try { foreach ($endedSpans as $span) { - $this->messagingEntrypoint->send(new SpanEnded($span), OnSpanEnded::CHANNEL); - } - if ($this->publishesCompletedTraces) { - $completedTraces = [...$completedTraces, ...$this->traceStore->completedButNotNotifiedBefore($this->clock->now()->sub(Duration::seconds(self::NOTIFICATION_RETRY_DELAY_IN_SECONDS)), self::NOTIFICATION_RETRY_BATCH)]; + $this->deliver(new SpanEnded($span), OnSpanEnded::CHANNEL); } foreach (array_unique($completedTraces) as $traceId) { - $this->notify($traceId); + $this->deliver(new TraceCompleted($traceId), OnTraceCompleted::CHANNEL); } } finally { $this->publishing = false; } } - private function notify(string $traceId): void + private function isStored(Span $span): bool + { + return ! $this->storesOnlyCarriedTraces || ($span->attributes[SpanAttribute::TRACE_CARRIED] ?? false); + } + + private function deliver(object $notification, string $channel): void { try { - $this->messagingEntrypoint->send(new TraceCompleted($traceId), OnTraceCompleted::CHANNEL); + $this->messagingEntrypoint->send($notification, $channel); } catch (Throwable) { - return; } - $this->traceStore->markNotified($traceId); } } diff --git a/packages/Ecotone/src/Tracing/TraceLedger.php b/packages/Ecotone/src/Tracing/TraceLedger.php index 118a322b3..330eb626d 100644 --- a/packages/Ecotone/src/Tracing/TraceLedger.php +++ b/packages/Ecotone/src/Tracing/TraceLedger.php @@ -42,7 +42,6 @@ public static function progressOf(string $traceId, array $spans): TraceProgress $rootEnded = false; $sent = []; $processed = []; - $failed = []; $sentDeliveries = []; $processedDeliveries = []; foreach ($spans as $span) { @@ -77,14 +76,12 @@ public static function progressOf(string $traceId, array $spans): TraceProgress $messageId = (string) ($span->attributes[MessageHeaders::MESSAGE_ID] ?? $span->spanId); if ($operation === SpanAttribute::OPERATION_SEND) { $sent[$destination][$messageId] = true; - } elseif ($operation === SpanAttribute::OPERATION_PROCESS && $span->status === SpanStatus::Error) { - $failed[$destination][$messageId] = true; } elseif ($operation === SpanAttribute::OPERATION_PROCESS) { $processed[$destination][$messageId] = true; } } - return new TraceProgress($traceId, true, $rootEnded, $sent, $processed, $failed, $sentDeliveries, $processedDeliveries); + return new TraceProgress($traceId, true, $rootEnded, $sent, $processed, $sentDeliveries, $processedDeliveries); } public static function destinationOf(Span $span): ?string diff --git a/packages/Ecotone/src/Tracing/TraceProgress.php b/packages/Ecotone/src/Tracing/TraceProgress.php index 6dba07395..2a41459e8 100644 --- a/packages/Ecotone/src/Tracing/TraceProgress.php +++ b/packages/Ecotone/src/Tracing/TraceProgress.php @@ -29,7 +29,6 @@ final class TraceProgress /** * @param array> $sent message ids dispatched to each destination * @param array> $processed message ids each destination processed - * @param array> $failed message ids each destination failed on * @param array> $sentDeliveries message ids sent to each channel * @param array> $processedDeliveries message ids consumed from each channel */ @@ -39,7 +38,6 @@ public function __construct( public readonly bool $rootEnded, public readonly array $sent, public readonly array $processed, - public readonly array $failed, public readonly array $sentDeliveries, public readonly array $processedDeliveries, ) { @@ -47,7 +45,7 @@ public function __construct( public static function unknown(string $traceId): self { - return new self($traceId, false, false, [], [], [], [], []); + return new self($traceId, false, false, [], [], [], []); } public function isCompleted(): bool @@ -85,24 +83,9 @@ public function pendingDestinations(): array return array_values(array_unique($pending)); } - /** - * @return string[] - */ - public function failedDestinations(): array - { - $failed = []; - foreach ($this->failed as $destination => $failedMessages) { - if (array_diff_key($failedMessages, $this->processed[$destination] ?? []) !== []) { - $failed[] = $destination; - } - } - - return $failed; - } - public function reached(string $destination): bool { - return isset($this->sent[$destination]) || isset($this->processed[$destination]) || isset($this->failed[$destination]); + return isset($this->sent[$destination]) || isset($this->processed[$destination]); } /** @@ -110,6 +93,6 @@ public function reached(string $destination): bool */ public function destinations(): array { - return array_keys($this->sent + $this->processed + $this->failed); + return array_keys($this->sent + $this->processed); } } diff --git a/packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php b/packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php index 6aece78db..13f1441f7 100644 --- a/packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php @@ -27,8 +27,29 @@ */ final class TraceRecordingInterceptor { - public function __construct(private StoringSpanRecorder $recorder) + public function __construct( + private StoringSpanRecorder $recorder, + private Tracer $tracer, + ) { + } + + public function publishAfterCommit(MethodInvocation $methodInvocation): mixed { + if ($this->recorder->isUnitOfWorkActive()) { + return $methodInvocation->proceed(); + } + + $this->recorder->beginUnitOfWork(); + try { + $result = $methodInvocation->proceed(); + } catch (Throwable $failure) { + $this->recorder->endUnitOfWork(); + + throw $failure; + } + $this->recorder->publishCommitted(); + + return $result; } public function recordUnitOfWork(MethodInvocation $methodInvocation): mixed @@ -37,10 +58,11 @@ public function recordUnitOfWork(MethodInvocation $methodInvocation): mixed return $methodInvocation->proceed(); } + $wrappingSpan = $this->tracer->currentSpan(); $this->recorder->openScope(); try { $result = $methodInvocation->proceed(); - $this->recorder->flush(); + $this->recorder->flush($wrappingSpan?->endedSoFar()); return $result; } catch (Throwable $failure) { diff --git a/packages/Ecotone/src/Tracing/TraceStore.php b/packages/Ecotone/src/Tracing/TraceStore.php index bf4f3dd64..bcee0e48b 100644 --- a/packages/Ecotone/src/Tracing/TraceStore.php +++ b/packages/Ecotone/src/Tracing/TraceStore.php @@ -34,13 +34,6 @@ public function append(array $spans): void; public function markCompletedIfDone(string $traceId, DatePoint $now): bool; - public function markNotified(string $traceId): void; - - /** - * @return string[] - */ - public function completedButNotNotifiedBefore(DatePoint $completedBefore, int $limit): array; - public function progressOf(string $traceId): TraceProgress; public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore): void; diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index d0a3dae5d..81f595796 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -121,7 +121,7 @@ public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pen $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $this->expectException(TraceCompletionTimedOut::class); - $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the pending channel, or await with a longer timeout."); + $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the pending channel, replay the message from the dead letter if its handler failed, or await with a longer timeout."); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); } @@ -395,32 +395,6 @@ public function place(string $orderId): void $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); } - public function test_a_consumer_that_swallowed_its_handler_failure_stores_a_failed_consumer_span_when_failures_may_be_stored(): void - { - $orders = new class () { - #[Asynchronous('async_channel')] - #[CommandHandler('order.place', endpointId: 'order.placing')] - public function place(string $orderId): void - { - throw new RuntimeException('Placing failed'); - } - }; - $subscriber = $this->spanEndedSubscriber(); - $ecotone = EcotoneLite::bootstrapFlowTesting( - [$orders::class, $subscriber::class], - [$orders, $subscriber], - [TracingConfiguration::createWithDefaults()->withStoredTraces(storeFailuresOutsideTransaction: true), SimpleMessageChannelBuilder::createQueueChannel('async_channel'), ServiceConfiguration::createWithDefaults()->withDefaultErrorChannel('errorChannel')], - licenceKey: LicenceTesting::VALID_LICENCE, - ); - $trace = SpanContext::newTrace(); - $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - - $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); - - $this->assertSame(SpanStatus::Error, $subscriber->endedSpans['receive async_channel']); - $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); - } - public function test_a_consumer_whose_handler_failed_once_and_succeeded_on_the_instant_retry_completes_the_trace(): void { $orders = new class () { @@ -520,7 +494,7 @@ public function test_the_completion_gateway_without_stored_traces_names_the_conf $this->completion($ecotone)->isCompleted('0af7651916cd43dd8448eb211c80319c'); } - public function test_awaiting_a_trace_whose_handler_failed_names_the_failed_handler(): void + public function test_awaiting_a_trace_whose_handler_failed_names_the_channel_and_how_to_recover(): void { $notifier = new class () { #[Asynchronous('async_channel')] @@ -530,18 +504,13 @@ public function notify(string $userId): void throw new RuntimeException('Notification failed'); } }; - $ecotone = EcotoneLite::bootstrapFlowTesting( - [$this->users()::class, $notifier::class], - [$this->users(), $notifier], - [TracingConfiguration::createWithDefaults()->withStoredTraces(storeFailuresOutsideTransaction: true), SimpleMessageChannelBuilder::createQueueChannel('async_channel'), ServiceConfiguration::createWithDefaults()->withDefaultErrorChannel('errorChannel')], - licenceKey: LicenceTesting::VALID_LICENCE, - ); + $ecotone = $this->bootstrap([$this->users(), $notifier], [SimpleMessageChannelBuilder::createQueueChannel('async_channel'), ServiceConfiguration::createWithDefaults()->withDefaultErrorChannel('errorChannel')]); $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); $this->expectException(TraceCompletionTimedOut::class); - $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Failed: user.notifier, whose last attempt failed and whose message is in the error channel or awaiting a retry. Await with a longer timeout once the failure is resolved."); + $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the pending channel, replay the message from the dead letter if its handler failed, or await with a longer timeout."); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); } @@ -744,27 +713,6 @@ public function test_a_failed_unit_of_work_stores_nothing_by_default(): void $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); } - public function test_a_failed_unit_of_work_stores_its_failed_root_span_when_failures_may_be_stored_outside_the_transaction(): void - { - $subscriber = $this->spanEndedSubscriber(); - $ecotone = EcotoneLite::bootstrapFlowTesting( - [$this->failingUsers()::class, $subscriber::class], - [$this->failingUsers(), $subscriber], - [TracingConfiguration::createWithDefaults()->withStoredTraces(storeFailuresOutsideTransaction: true)], - licenceKey: LicenceTesting::VALID_LICENCE, - ); - $trace = SpanContext::newTrace(); - - try { - $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->fail('Expected the registration failure to propagate'); - } catch (RuntimeException) { - } - - $this->assertSame(['process class@anonymous::register' => SpanStatus::Error, 'send command' => SpanStatus::Error], array_combine(array_map(fn (string $name): string => preg_replace('/^process class@anonymous\\S*::/', 'process class@anonymous::', $name), array_keys($subscriber->endedSpans)), $subscriber->endedSpans)); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); - } - public function test_stored_traces_without_a_trace_store_are_refused_at_bootstrap(): void { $this->expectException(ConfigurationException::class); diff --git a/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php b/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php index 288f57b4e..03d8a0ce9 100644 --- a/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php +++ b/packages/OpenTelemetry/src/OpenTelemetryActiveSpan.php @@ -71,6 +71,23 @@ public function end(): void $this->finish(StatusCode::STATUS_OK, SpanStatus::Ok, null); } + public function endedSoFar(): Span + { + return new Span( + $this->context->traceId, + $this->context->spanId, + $this->parentSpanId, + $this->name, + $this->kind, + $this->attributes, + SpanStatus::Ok, + null, + $this->events, + $this->startedAt, + $this->clock->now(), + ); + } + public function markUnitOfWorkFailed(): void { $this->unitOfWorkFailed = true; diff --git a/upgrade-2.0.md b/upgrade-2.0.md index a4937bf54..9a8807d27 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3099,37 +3099,38 @@ in an external collector, after the fact. **Now:** `TracingConfiguration::createWithDefaults()->withStoredTraces()` keeps every span in the database as well, in `ecotone_traces` and `ecotone_tracing`, created by `ecotone:migration:database:setup`. A unit of work, a bus call -or an asynchronous consumption, buffers its spans and writes them once, inside its transaction before the commit; -the span that wraps the unit of work is written when it ends, after the commit, and that write is the one that -decides whether the trace is complete, reading what every other unit of work has committed by then. No row is -locked: completion is claimed with a single conditional update, so two consumers finishing the last two handlers of -one trace both see each other's commit and exactly one of them claims it. From those spans the `TraceCompletion` gateway answers +or an asynchronous consumption, buffers its spans and writes them once, inside its transaction before the commit, +together with the span that wraps the unit of work as it stands at that moment, and decides in the same transaction +whether the trace is complete. Nothing is written to the database after the commit or after a rollback: a database +that goes down right after a commit cannot fail a consumer whose work already committed, and a failure inside the +transaction rolls the trace back with the work and leaves the message to be retried. Completion is claimed with one +conditional update after the trace row was touched by the same transaction, so a second unit of work finishing the +same trace waits for the first one's commit and, on PostgreSQL and SQLite, reads its spans before deciding. From those spans the `TraceCompletion` gateway answers `isCompleted($traceId)`, true once the root ended, every route the trace sent has an ended handler span, and every message it sent to a channel was consumed from that channel, so an asynchronous command, event or projection keeps the trace pending until its consumer processed that exact message; `isCompletedFor($traceId, $destination)` for one `endpointId` or projection name; and `await()` / `awaitFor()`, which poll until then or throw `TraceCompletionTimedOut` naming the handlers and channels still pending. The root is the span that started the trace in this process, a bus call or a handler reached outside any other span, never a consumer: a trace whose -producer stored nothing is never completed by its consumers alone. A unit of work that fails discards the spans it -buffered, so a handler that succeeded inside a rolled-back transaction, and a message whose send rolled back with -it, are not counted. By default nothing of a failed unit of work is stored, because the only moment left to store it -is after the rollback, outside any transaction; `withStoredTraces(storeFailuresOutsideTransaction: true)` opts in -to storing the failed outer span then, so a failed trace shows its error and counts as complete. A destination counts a message as dispatched and as processed by +producer stored nothing is never completed by its consumers alone. A unit of work that fails stores nothing: a handler +that succeeded inside a rolled-back transaction, and a message whose send rolled back with it, are not counted, and +the failure itself is visible in OpenTelemetry, which may fail on its own without harm, not in the store. A destination counts a message as dispatched and as processed by its message id, so a redelivery of the same message satisfies the delivery it belongs to and no other, and a message sent to an asynchronous handler is attributed to that handler's `endpointId` whether it was routed by the event bus, the command bus or an `outputChannelName`. The handler span is the innermost interceptor around the method, so it records the handler's own failure even when an error channel or a retry handles it, and the consumer or bus span that wraps a failed unit of work ends as failed too, in OpenTelemetry as in the store, unless a retry inside it succeeded; and -`TraceCompletionTimedOut` names the handlers whose last attempt failed, the channels still holding a message, a -trace nobody recorded, and a destination the trace never reached. `TraceCompletion` is always injectable; without +`TraceCompletionTimedOut` names the handlers and channels still pending, points to the dead letter when a handler +failed, and names a trace nobody recorded and a destination the trace never reached. `TraceCompletion` is always injectable; without `withStoredTraces()` every call refuses with the configuration it needs. A method with `#[OnTraceCompleted]` receives a `TraceCompleted` each time the trace reaches completion, so a trace that is handed more work after completing, as when a request sends two commands under one `traceparent`, notifies again once that work is done; one with `#[OnSpanEnded]` receives every stored span. Both run after the commit that produced them, so a websocket push or -a cache refresh never runs ahead of the data. A completion notification is durable: the store remembers it was -delivered, a subscriber that throws does not fail the command that already committed, and any later unit of work -in any process re-delivers a notification left undelivered for thirty seconds. A handler that needs the id of the +a cache refresh never runs ahead of the data. Notifications are published from memory once the transaction +committed, at most once: a subscriber that throws does not fail the command that already committed and is not called +again, and a process that stops between the commit and the publication drops that notification, while +`TraceCompletion` still answers from the stored spans. A handler that needs the id of the trace it runs in injects `Ecotone\Api\Tracing\CurrentTrace` and reads `traceId()`. `isCompletedFor()` and `awaitFor()` refuse a `#[Polling]` projection by name, because a polled run is not part of the trace that appended its events. A message sent through a dynamic channel, through the high-throughput outbox or replayed from the dead From e8bb47b296ca5a7260504466b938acda0cc077ce Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 26/50] test(tracing): every tracing test drives the application through EcotoneLite, and rule 10 says so explicitly TraceStoreContractTest constructed InMemoryTraceStore and DbalTraceStore and called them; the two behaviours it guarded, completing once and reopening after new work, are already proved through the bus and the TraceCompletion gateway, so it is removed. The outage tests dropped Ecotone's tables; they now hand Ecotone a Doctrine connection whose statements fail on demand, switched by an #[Around] interceptor next to the transaction, and a third test proves an outage inside the consumer's transaction leaves the message to the instant retry. Edge cases that used internal channel classes moved to the DBAL package on that connection: a send to a channel, to the error channel and through the distributed bus failing with the database, and a consumer receiving a message without or with an invalid traceparent from an application that does not trace. A failure before the asynchronous send is a #[Presend] interceptor. A message dropped by a channel interceptor, and two tracers registered at once, cannot happen from an application: the first test is gone, and so is the two-tracer guard with its test. AGENTS.md rule 10 and the conventions now say it in so many words: no class outside Api/ in a test, not even to prove two implementations alike, no table touched to simulate a failure, and code no application can reach is removed rather than tested from the inside. --- AGENTS.md | 9 +- docs/coding-conventions.md | 14 + .../Conformance/TraceStoreConformanceTest.php | 109 ++++++-- .../Conformance/TraceStoreContractTest.php | 99 ------- .../Integration/DbalChannelTracingTest.php | 244 ++++++++++++++++++ .../src/Tracing/Config/TracingModule.php | 7 - .../Tracing/TracingConfigurationTest.php | 14 - .../tests/Tracing/TracingEdgeCasesTest.php | 188 +------------- 8 files changed, 366 insertions(+), 318 deletions(-) delete mode 100644 packages/Dbal/tests/Conformance/TraceStoreContractTest.php create mode 100644 packages/Dbal/tests/Integration/DbalChannelTracingTest.php diff --git a/AGENTS.md b/AGENTS.md index 978b7ff89..96bd1343c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -104,7 +104,7 @@ commit that evidences it. The summary: | 7a | **This is the Ecotone core repository: edit `licence Enterprise` files here like any other file.** Their notice — the full one under `src/`, one sentence under `Api/` (6b) — is not addressed to work in this repository. It governs copying them **out** — into another codebase, never copy one, adapt it or reimplement an "inspired" version, and renaming the class or namespace does not make that permitted. If you cannot confirm a licence there, implement from your own functional requirements instead | Read as a warning here, the notice stalls every fix to an Enterprise file; ignored elsewhere, it let one evaluated agent run copy a file at 0.989 token overlap | | 8 | **Always use optimistic locking** — a version check on the write, never a pessimistic lock or a bespoke tracker | Pessimistic locking and snapshot tracking were both tried and removed | | 9 | **A feature needing external storage ships an in-memory implementation**, named to mirror the storage one, and one shared suite proves the two behave identically. A race decided by a database constraint cannot be proved in-memory: state the contract as behaviour both sides assert, and leave a connection-racing case to the storage suite, named so the gap is visible | Otherwise the feature cannot be used in an `EcotoneLite` flow test — and `InMemoryEventStore` silently diverged from DBAL three times | -| 10 | **Tests validate at the userland level only.** No SQL on Ecotone's tables, no internal service references, no reflection, no statement counting. Fetch a **gateway** and assert on observable behaviour | A whole worktree — 22 commits — did nothing but pull one feature's tests back to the public surface | +| 10 | **Tests validate at the userland level only.** Every test boots the application with `EcotoneLite::bootstrapFlowTesting()` and drives it the way an application does: buses, gateways, attributes, `#[ServiceContext]` configuration, console commands. A test never constructs, calls or names a class outside `Api/` — no store, recorder, ledger, module, extension object or channel interceptor from `src/` — never runs SQL or schema calls against Ecotone's tables, and never uses reflection or statement counting. A failure is simulated through a seam the application owns: the connection it hands Ecotone, an `#[Around]` interceptor, a handler that throws. The test is right when every class under `src/` could be rewritten and the test would stay unchanged | A whole worktree — 22 commits — did nothing but pull one feature's tests back to the public surface | | 10a | **A guarantee is proved on every path that has to hold it** — direct call, class-routed bus send, routed send, gateway, console command, and the reconstruction path beside the live one | Five times in one feature, a fix covered one entry point and looked finished | | 11 | **`EcotoneLite`, `snake_case`, no comments, inline anonymous fixtures in the test file** — a named class below the `TestCase` only where PHP forces one. Test-first: RED, GREEN, refactor | An anonymous fixture cannot drift out of the test that owns it | | 12 | **The public surface is `Api/`, a sibling of `src/`** — never `src/Api/`, which breaks Tempest discovery. Everything outside `Api` is `@internal` | | @@ -148,6 +148,13 @@ API moved in 2.0 (`Ecotone\Api\Attribute\CommandHandler`, not the old flat names statement counting, no internal service references — fetch a gateway (`$ecotone->getGateway(EventStore::class)`, `Ecotone\Api\EventSourcing\EventStore`) and drive the public API. This is the most-corrected rule in the repository; see conventions rule 10 +- **Never test an internal class directly**, not even a seam with two implementations. An in-memory store and its + DBAL twin are proved equal by one suite that boots the application twice, once per storage, and drives both through + the same gateway — never by constructing the stores and calling them. A failure the test needs (a database outage, + a broken send) is produced through something the application owns: a Doctrine connection handed to Ecotone, an + `#[Around]` interceptor, a handler that throws — never by dropping or editing Ecotone's tables. If a behaviour + cannot be reached that way, it is not a behaviour the application can observe; remove the code rather than test it + from the inside - **A message channel is never driven directly.** `FlowTestSupport` has no `getMessageChannel()`, `receiveMessageFrom()` or `sendMessageDirectToChannel()`. The application reaches a channel through an `#[Asynchronous]` handler, so the test sends the command or event, calls `run('channel')`, and asserts on what that diff --git a/docs/coding-conventions.md b/docs/coding-conventions.md index a4a2d3ba1..31bc15b52 100644 --- a/docs/coding-conventions.md +++ b/docs/coding-conventions.md @@ -593,6 +593,20 @@ Never, in a test: - use reflection - count SQL statements - name an internal collaborator class in the test's own name +- construct or call any class outside `Api/` — a store, recorder, ledger, module, extension object or channel + interceptor from `src/` — even to prove two implementations of a seam behave alike +- drop, alter or write Ecotone's tables to simulate a failure + +The test of a test: rewrite every class under `src/` the feature touches, keep the behaviour, and the test must not +change. A contract suite that constructs `InMemoryTraceStore` and `DbalTraceStore` and calls `append()` fails that test +the moment the store's methods are renamed, while the behaviour it guards — a trace completes once and notifies once — +is unchanged. Prove it instead through the application: boot it once per storage and drive both through the bus and +the `TraceCompletion` gateway. + +A failure the test needs comes through a seam the application owns. A database outage right after a commit is a +Doctrine connection the application hands Ecotone, whose statements start failing when an `#[Around]` interceptor +just outside the transaction says so — not a `DROP TABLE` on Ecotone's tables. If a path cannot be reached from +userland at all, the application cannot observe it: remove the code instead of testing it from the inside. ```php use Ecotone\Api\EventSourcing\EventStore; diff --git a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php index 8b137bdc6..604e32ebe 100644 --- a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php +++ b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php @@ -4,6 +4,9 @@ namespace Test\Ecotone\Dbal\Conformance; +use Doctrine\DBAL\Cache\QueryCacheProfile; +use Doctrine\DBAL\Connection; +use Doctrine\DBAL\Result; use Ecotone\Api\Attribute\Around; use Ecotone\Api\Attribute\Asynchronous; use Ecotone\Api\Attribute\CommandHandler; @@ -62,6 +65,12 @@ */ final class TraceStoreConformanceTest extends DbalMessagingTestCase { + public function tearDown(): void + { + OutageSimulatingConnection::$databaseDown = false; + parent::tearDown(); + } + public static function implementations(): iterable { yield 'in-memory' => ['in-memory']; @@ -464,7 +473,7 @@ public function notify(string $userId): void } }; $outage = $this->outageRightAfterTheTransaction(); - $ecotone = $this->bootstrap('dbal', [$this->users(), $notifier, $outage], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $ecotone = $this->bootstrapOnAConnectionThatCanGoDown([$this->users(), $notifier, $outage], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); $ecotone->sendCommandWithRouting('user.register', '1'); $outage->armed = true; @@ -486,7 +495,7 @@ public function register(string $userId): void } }; $outage = $this->outageRightAfterTheTransaction(); - $ecotone = $this->bootstrap('dbal', [$users, $outage]); + $ecotone = $this->bootstrapOnAConnectionThatCanGoDown([$users, $outage]); $outage->armed = true; $ecotone->sendCommandWithRouting('user.register', '1'); @@ -494,6 +503,38 @@ public function register(string $userId): void self::assertSame(1, $users->registered); } + public function test_a_database_outage_inside_the_consumer_transaction_leaves_the_message_to_be_retried(): void + { + $notifier = new class () { + public int $handled = 0; + + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId): void + { + $this->handled++; + OutageSimulatingConnection::$databaseDown = $this->handled === 1; + } + }; + $recovery = new class () { + #[Around(precedence: Precedence::DATABASE_TRANSACTION_PRECEDENCE - 1, pointcut: 'Ecotone\\Api\\Attribute\\AsynchronousRunningEndpoint')] + public function comeBackBeforeEachAttempt(MethodInvocation $methodInvocation): mixed + { + OutageSimulatingConnection::$databaseDown = false; + + return $methodInvocation->proceed(); + } + }; + $ecotone = $this->bootstrapOnAConnectionThatCanGoDown([$this->users(), $notifier, $recovery], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertSame(2, $notifier->handled); + self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + public function test_completion_recorded_by_one_instance_is_visible_to_another_sharing_the_database(): void { $producer = $this->bootstrap('dbal', [$this->users()]); @@ -538,25 +579,15 @@ public function register(string $userId, EventBus $eventBus): void private function outageRightAfterTheTransaction(): object { - return new class ($this->getConnectionFactory()) { + return new class () { public bool $armed = false; - public function __construct(private DbalConnectionFactory $connectionFactory) - { - } - #[Around(precedence: Precedence::DATABASE_TRANSACTION_PRECEDENCE - 1, pointcut: 'Ecotone\\Api\\Gateway\\CommandBus||Ecotone\\Api\\Attribute\\AsynchronousRunningEndpoint')] - public function simulateOutage(MethodInvocation $methodInvocation): mixed + public function goDownAfterTheTransaction(MethodInvocation $methodInvocation): mixed { $result = $methodInvocation->proceed(); - if (! $this->armed) { - return $result; - } - $schemaManager = $this->connectionFactory->establishConnection()->createSchemaManager(); - foreach (['ecotone_tracing', 'ecotone_traces'] as $table) { - if ($schemaManager->tablesExist([$table])) { - $schemaManager->dropTable($table); - } + if ($this->armed) { + OutageSimulatingConnection::$databaseDown = true; } return $result; @@ -564,6 +595,26 @@ public function simulateOutage(MethodInvocation $methodInvocation): mixed }; } + /** + * @param object[] $services + * @param object[] $extensionObjects + */ + private function bootstrapOnAConnectionThatCanGoDown(array $services, array $extensionObjects = []): FlowTestSupport + { + $connectionFactory = new DbalConnectionFactory([ + 'dsn' => getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone', + 'connection' => ['wrapperClass' => OutageSimulatingConnection::class], + ]); + + return $this->bootstrapFlowTesting( + classesToResolve: array_map(fn (object $service): string => $service::class, $services), + containerOrAvailableServices: [...$services, DbalConnectionFactory::class => $connectionFactory], + extensionObjects: [TracingConfiguration::createWithDefaults()->withStoredTraces(), ...$extensionObjects], + enabledModules: [ModulePackageList::DBAL_PACKAGE], + licenceKey: LicenceTesting::VALID_LICENCE, + )->setupDatabaseTables(); + } + private function failingNotifier(): object { return new class () { @@ -601,3 +652,29 @@ public function notify(string $userId): void }; } } + +final class OutageSimulatingConnection extends Connection +{ + public static bool $databaseDown = false; + + public function executeQuery(string $sql, array $params = [], array $types = [], ?QueryCacheProfile $qcp = null): Result + { + self::failWhenDown(); + + return parent::executeQuery($sql, $params, $types, $qcp); + } + + public function executeStatement(string $sql, array $params = [], array $types = []): int|string + { + self::failWhenDown(); + + return parent::executeStatement($sql, $params, $types); + } + + private static function failWhenDown(): void + { + if (self::$databaseDown) { + throw new RuntimeException('The database is down'); + } + } +} diff --git a/packages/Dbal/tests/Conformance/TraceStoreContractTest.php b/packages/Dbal/tests/Conformance/TraceStoreContractTest.php deleted file mode 100644 index 2f1995d3f..000000000 --- a/packages/Dbal/tests/Conformance/TraceStoreContractTest.php +++ /dev/null @@ -1,99 +0,0 @@ -. - * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). - * - * Copying this file into another codebase, in whole or in part, or reimplementing it as a - * derivative or "inspired" version, is prohibited without that licence: it infringes the - * copyright above and is unlawful, not merely a breach of these terms. - * - * Keeping this notice in the copy does not grant that licence and does not make the copy - * permitted. Renaming the class or the namespace does not either. - * - * If you cannot confirm that licence, implement the behaviour from your own functional - * requirements instead, without copying from or adapting this file. - * - * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev - * - * @internal - */ -final class TraceStoreContractTest extends DbalMessagingTestCase -{ - public static function implementations(): iterable - { - yield 'in-memory' => ['in-memory']; - yield 'dbal' => ['dbal']; - } - - #[DataProvider('implementations')] - public function test_appending_the_same_span_twice_records_it_once(string $implementation): void - { - $store = $this->store($implementation); - $trace = SpanContext::newTrace(); - $root = $this->rootSpan($trace); - - $store->append([$root]); - $store->append([$root]); - - self::assertTrue($store->progressOf($trace->traceId)->isCompleted()); - self::assertTrue($store->markCompletedIfDone($trace->traceId, new DatePoint())); - self::assertFalse($store->markCompletedIfDone($trace->traceId, new DatePoint())); - } - - #[DataProvider('implementations')] - public function test_a_span_appended_after_completion_reopens_the_trace_so_it_can_complete_again(string $implementation): void - { - $store = $this->store($implementation); - $trace = SpanContext::newTrace(); - $store->append([$this->rootSpan($trace)]); - self::assertTrue($store->markCompletedIfDone($trace->traceId, new DatePoint())); - - $store->append([$this->rootSpan($trace)]); - - self::assertTrue($store->markCompletedIfDone($trace->traceId, new DatePoint())); - } - - private function store(string $implementation): TraceStore - { - if ($implementation === 'in-memory') { - return new InMemoryTraceStore(); - } - $traces = new TracesTableManager(TracesTableManager::DEFAULT_TABLE, true); - $spans = new SpansTableManager(SpansTableManager::DEFAULT_TABLE, true); - $connection = $this->getConnection(); - foreach ([$traces, $spans] as $tableManager) { - if (! $tableManager->isInitialized($connection)) { - $tableManager->createTable($connection); - } - } - - return new DbalTraceStore($this->getConnectionFactory(), $traces, $spans, new NativeClock()); - } - - private function rootSpan(SpanContext $trace): Span - { - return new Span($trace->traceId, $trace->newChild()->spanId, null, 'send command', SpanKind::Server, [SpanAttribute::TRACE_ROOT => true], SpanStatus::Ok, null, [], new DatePoint(), new DatePoint()); - } -} diff --git a/packages/Dbal/tests/Integration/DbalChannelTracingTest.php b/packages/Dbal/tests/Integration/DbalChannelTracingTest.php new file mode 100644 index 000000000..362deeb35 --- /dev/null +++ b/packages/Dbal/tests/Integration/DbalChannelTracingTest.php @@ -0,0 +1,244 @@ +bootstrap($recorder, [$this->asynchronousOrders()], [DbalBackedMessageChannelBuilder::create('orders')]); + + ChannelOutageConnection::$databaseDown = true; + $this->assertNotNull($this->failureOf(fn () => $ecotone->sendCommandWithRouting('order.place', '1'))); + ChannelOutageConnection::$databaseDown = false; + + $this->assertSame(SpanStatus::Error, self::singleSpanNamed('send orders', $recorder)->status); + $recorder->clear(); + $ecotone->sendCommandWithRouting('order.place', '2'); + $this->assertSame('send command', self::rootSpan($recorder)->name); + } + + public function test_a_failing_handler_whose_error_channel_cannot_store_the_message_still_ends_every_span(): void + { + $recorder = new InMemorySpanRecorder(); + $orders = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.place', endpointId: 'order.placing')] + public function place(string $orderId): void + { + ChannelOutageConnection::$databaseDown = true; + + throw new RuntimeException('Handler failure'); + } + }; + $ecotone = $this->bootstrap($recorder, [$orders], [ + DbalBackedMessageChannelBuilder::create('orders'), + DbalBackedMessageChannelBuilder::create('error_channel'), + ServiceConfiguration::createWithDefaults()->withDefaultErrorChannel('error_channel'), + ]); + $ecotone->sendCommandWithRouting('order.place', '1'); + $recorder->clear(); + + $this->failureOf(fn () => $ecotone->run('orders', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false))); + ChannelOutageConnection::$databaseDown = false; + + self::assertEverySpanNamedHasStatus('process class@anonymous::place', SpanStatus::Error, $recorder); + self::assertEverySpanNamedHasStatus('send error_channel', SpanStatus::Error, $recorder); + $recorder->clear(); + $ecotone->sendCommandWithRouting('order.place', '2'); + $this->assertSame('send command', self::rootSpan($recorder)->name); + } + + public function test_a_distributed_send_that_fails_because_the_database_is_down_fails_its_span_and_leaves_no_open_span_behind(): void + { + $recorder = new InMemorySpanRecorder(); + $orders = new class () { + #[CommandHandler('order.place')] + public function place(string $orderId): void + { + } + }; + $ecotone = $this->bootstrap($recorder, [$orders], [ + DbalBackedMessageChannelBuilder::create('distributed_channel'), + DistributedServiceMap::initialize()->withCommandMapping(targetServiceName: 'ticket_service', channelName: 'distributed_channel'), + ServiceConfiguration::createWithDefaults()->withServiceName('order_service'), + ]); + + ChannelOutageConnection::$databaseDown = true; + $this->assertNotNull($this->failureOf(fn () => $ecotone->getDistributedBus()->sendCommand('ticket_service', 'ticket.create', 'User changed billing address'))); + ChannelOutageConnection::$databaseDown = false; + + $this->assertSame(SpanStatus::Error, self::singleSpanNamed('send distributed', $recorder)->status); + $recorder->clear(); + $ecotone->sendCommandWithRouting('order.place', '1'); + $this->assertSame('send command', self::rootSpan($recorder)->name); + } + + public function test_a_consumer_receiving_a_message_sent_by_an_application_without_tracing_starts_its_own_trace(): void + { + $producer = $this->bootstrapWithoutTracing([$this->asynchronousOrders()]); + $recorder = new InMemorySpanRecorder(); + $consumer = $this->bootstrap($recorder, [$this->asynchronousOrders()], [DbalBackedMessageChannelBuilder::create('orders')]); + + $producer->sendCommandWithRouting('order.place', '1'); + $consumer->run('orders', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertTrue(self::singleSpanNamed('receive orders', $recorder)->isRoot()); + } + + public function test_an_invalid_traceparent_on_a_received_message_is_ignored_and_the_consumer_starts_its_own_trace(): void + { + $producer = $this->bootstrapWithoutTracing([$this->asynchronousOrders()]); + $recorder = new InMemorySpanRecorder(); + $consumer = $this->bootstrap($recorder, [$this->asynchronousOrders()], [DbalBackedMessageChannelBuilder::create('orders')]); + + $producer->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => 'not-a-traceparent']); + $consumer->run('orders', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertTrue(self::singleSpanNamed('receive orders', $recorder)->isRoot()); + } + + /** + * @param object[] $services + * @param object[] $extensionObjects + */ + private function bootstrap(InMemorySpanRecorder $recorder, array $services, array $extensionObjects): FlowTestSupport + { + return $this->bootstrapFlowTesting( + classesToResolve: array_map(fn (object $service): string => $service::class, $services), + containerOrAvailableServices: [...$services, InMemorySpanRecorder::class => $recorder, DbalConnectionFactory::class => self::connectionThatCanGoDown()], + extensionObjects: [TracingConfiguration::createWithDefaults()->withSpanRecorder(InMemorySpanRecorder::class), ...$extensionObjects], + enabledModules: [ModulePackageList::DBAL_PACKAGE], + licenceKey: LicenceTesting::VALID_LICENCE, + )->setupDatabaseTables(); + } + + /** + * @param object[] $services + */ + private function bootstrapWithoutTracing(array $services): FlowTestSupport + { + return $this->bootstrapFlowTesting( + classesToResolve: array_map(fn (object $service): string => $service::class, $services), + containerOrAvailableServices: [...$services, DbalConnectionFactory::class => self::connectionThatCanGoDown()], + extensionObjects: [DbalBackedMessageChannelBuilder::create('orders')], + enabledModules: [ModulePackageList::DBAL_PACKAGE], + )->setupDatabaseTables(); + } + + private static function connectionThatCanGoDown(): DbalConnectionFactory + { + return new DbalConnectionFactory([ + 'dsn' => getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone', + 'connection' => ['wrapperClass' => ChannelOutageConnection::class], + ]); + } + + private function asynchronousOrders(): object + { + return new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.place', endpointId: 'order.placing')] + public function place(string $orderId): void + { + } + }; + } + + private function failureOf(callable $action): ?Throwable + { + try { + $action(); + } catch (Throwable $failure) { + return $failure; + } + + return null; + } + + private static function singleSpanNamed(string $name, InMemorySpanRecorder $recorder): Span + { + $spans = array_values(array_filter($recorder->recordedSpans(), fn (Span $span): bool => $span->name === $name)); + self::assertCount(1, $spans, 'Expected exactly one span named ' . $name . ', got: ' . implode(', ', array_map(fn (Span $span): string => $span->name, $recorder->recordedSpans()))); + + return $spans[0]; + } + + private static function assertEverySpanNamedHasStatus(string $name, SpanStatus $status, InMemorySpanRecorder $recorder): void + { + $spans = array_values(array_filter($recorder->recordedSpans(), fn (Span $span): bool => preg_replace('/^process class@anonymous\S*::/', 'process class@anonymous::', $span->name) === $name)); + self::assertNotEmpty($spans, 'Expected spans named ' . $name . ', got: ' . implode(', ', array_map(fn (Span $span): string => $span->name, $recorder->recordedSpans()))); + foreach ($spans as $span) { + self::assertSame($status, $span->status, $name); + } + } + + private static function rootSpan(InMemorySpanRecorder $recorder): Span + { + $roots = array_values(array_filter($recorder->recordedSpans(), fn (Span $span): bool => $span->isRoot())); + self::assertCount(1, $roots, 'Expected exactly one root span, got: ' . implode(', ', array_map(fn (Span $span): string => $span->name, $roots))); + + return $roots[0]; + } +} + +final class ChannelOutageConnection extends Connection +{ + public static bool $databaseDown = false; + + public function executeQuery(string $sql, array $params = [], array $types = [], ?QueryCacheProfile $qcp = null): Result + { + self::failWhenDown(); + + return parent::executeQuery($sql, $params, $types, $qcp); + } + + public function executeStatement(string $sql, array $params = [], array $types = []): int|string + { + self::failWhenDown(); + + return parent::executeStatement($sql, $params, $types); + } + + private static function failWhenDown(): void + { + if (self::$databaseDown) { + throw new RuntimeException('The database is down'); + } + } +} diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index 965142bc8..7fb0e843e 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -5,7 +5,6 @@ namespace Ecotone\Tracing\Config; use function array_map; -use function count; use Ecotone\AnnotationFinder\AnnotationFinder; use Ecotone\Api\Attribute\AsynchronousRunningEndpoint; @@ -33,7 +32,6 @@ use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\ExtensionObjectResolver; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\NoExternalConfigurationModule; use Ecotone\Messaging\Config\Configuration; -use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\Reference; use Ecotone\Messaging\Config\ModuleReferenceSearchService; @@ -52,8 +50,6 @@ use Ecotone\Tracing\TracingInterceptor; use Ecotone\Tracing\UnavailableTraceCompletion; -use function implode; - #[ModuleAnnotation] /** * licence Apache-2.0 @@ -100,9 +96,6 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO { $tracingConfiguration = ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $extensionObjects, TracingConfiguration::createWithDefaults()); $tracerReferences = ExtensionObjectResolver::resolve(TracerReference::class, $extensionObjects); - if (count($tracerReferences) > 1) { - throw ConfigurationException::create('Only one Tracer can be active, yet these were registered: ' . implode(', ', array_map(fn (TracerReference $reference): string => $reference->referenceName, $tracerReferences)) . '. Enable a single tracing package.'); - } $spanRecorderReferences = ExtensionObjectResolver::resolve(SpanRecorderReference::class, $extensionObjects); if (! $tracingConfiguration->storedTraces) { diff --git a/packages/Ecotone/tests/Tracing/TracingConfigurationTest.php b/packages/Ecotone/tests/Tracing/TracingConfigurationTest.php index 6f1c9b344..b2aa540dc 100644 --- a/packages/Ecotone/tests/Tracing/TracingConfigurationTest.php +++ b/packages/Ecotone/tests/Tracing/TracingConfigurationTest.php @@ -12,8 +12,6 @@ use Ecotone\Api\Lite\EcotoneLite; use Ecotone\Api\Messaging\MessageHeaders; use Ecotone\Api\Tracing\InMemorySpanRecorder; -use Ecotone\Messaging\Config\ConfigurationException; -use Ecotone\Tracing\Config\TracerReference; /** * licence Apache-2.0 @@ -121,16 +119,4 @@ public function register(string $userId): void $this->assertArrayNotHasKey('ecotone.endpoint.id', $spansByName['send command']->attributes); $this->assertArrayNotHasKey('ecotone.trace.root', $spansByName['process class@anonymous::register']->attributes); } - - public function test_two_tracers_are_refused_at_bootstrap(): void - { - $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('Only one Tracer can be active, yet these were registered: first.tracer, second.tracer. Enable a single tracing package.'); - - EcotoneLite::bootstrapFlowTesting( - [], - [], - [new TracerReference('first.tracer'), new TracerReference('second.tracer')], - ); - } } diff --git a/packages/Ecotone/tests/Tracing/TracingEdgeCasesTest.php b/packages/Ecotone/tests/Tracing/TracingEdgeCasesTest.php index 95e72b7d5..d963aa17e 100644 --- a/packages/Ecotone/tests/Tracing/TracingEdgeCasesTest.php +++ b/packages/Ecotone/tests/Tracing/TracingEdgeCasesTest.php @@ -8,6 +8,7 @@ use Ecotone\Api\Attribute\CommandHandler; use Ecotone\Api\Attribute\EventHandler; use Ecotone\Api\Attribute\Header; +use Ecotone\Api\Attribute\Presend; use Ecotone\Api\ExtensionObject\DistributedServiceMap; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -16,18 +17,12 @@ use Ecotone\Api\Gateway\EventBus; use Ecotone\Api\Lite\EcotoneLite; use Ecotone\Api\Lite\Test\FlowTestSupport; -use Ecotone\Api\Messaging\Message; -use Ecotone\Api\Messaging\MessageChannel; use Ecotone\Api\Messaging\MessageHeaders; use Ecotone\Api\Tracing\InMemorySpanRecorder; use Ecotone\Api\Tracing\Span; use Ecotone\Api\Tracing\SpanKind; use Ecotone\Api\Tracing\SpanStatus; -use Ecotone\Messaging\Channel\AbstractChannelInterceptor; -use Ecotone\Messaging\Channel\ExceptionalQueueChannel; -use Ecotone\Messaging\Channel\SimpleChannelInterceptorBuilder; use Ecotone\Test\LicenceTesting; -use Ecotone\Test\StaticMessageBuilder; use RuntimeException; use Throwable; @@ -37,30 +32,11 @@ */ final class TracingEdgeCasesTest extends TracingTestCase { - public function test_a_failed_send_to_an_asynchronous_channel_fails_the_send_span_and_the_next_flow_starts_a_fresh_root(): void + public function test_an_interceptor_failing_before_the_asynchronous_send_leaves_no_open_span_behind(): void { - $recorder = new InMemorySpanRecorder(); - $ecotone = $this->bootstrapWithEventForwardedToSecondChannel($recorder, [ - SimpleMessageChannelBuilder::createQueueChannel('async_channel'), - ExceptionalQueueChannel::createWithExceptionOnSend('failing_channel'), - ]); - $ecotone->sendCommandWithRouting('user.register', '1'); - - $caught = $this->runConsumer($ecotone); - $this->assertStringContainsString('Exception on send', $caught?->getMessage() ?? ''); - - self::assertEverySpanNamedHasStatus('send failing_channel', SpanStatus::Error, $recorder); - - $recorder->clear(); - $ecotone->sendCommandWithRouting('user.register', '2'); - $this->assertCount(1, array_filter($recorder->recordedSpans(), fn (Span $span): bool => $span->isRoot())); - $this->assertSame('send command', self::rootSpan($recorder)->name); - } - - public function test_a_channel_interceptor_failing_before_send_leaves_no_open_span_behind(): void - { - $throwingChannelInterceptor = new class () extends AbstractChannelInterceptor { - public function preSend(Message $message, MessageChannel $messageChannel): ?Message + $blockingInterceptor = new class () { + #[Presend(pointcut: EventHandler::class)] + public function block(): void { throw new RuntimeException('Exception before send'); } @@ -69,8 +45,7 @@ public function preSend(Message $message, MessageChannel $messageChannel): ?Mess $ecotone = $this->bootstrapWithEventForwardedToSecondChannel($recorder, [ SimpleMessageChannelBuilder::createQueueChannel('async_channel'), SimpleMessageChannelBuilder::createQueueChannel('failing_channel'), - SimpleChannelInterceptorBuilder::create('failing_channel', 'throwingChannelInterceptor'), - ], ['throwingChannelInterceptor' => $throwingChannelInterceptor]); + ], [$blockingInterceptor]); $ecotone->sendCommandWithRouting('user.register', '1'); $caught = $this->runConsumer($ecotone); @@ -81,81 +56,6 @@ public function preSend(Message $message, MessageChannel $messageChannel): ?Mess $this->assertSame('send command', self::rootSpan($recorder)->name); } - public function test_a_message_filtered_out_by_a_channel_interceptor_leaves_no_open_span_behind(): void - { - $filteringChannelInterceptor = new class () extends AbstractChannelInterceptor { - public function preSend(Message $message, MessageChannel $messageChannel): ?Message - { - return null; - } - }; - $recorder = new InMemorySpanRecorder(); - $ecotone = $this->bootstrapWithEventForwardedToSecondChannel($recorder, [ - SimpleMessageChannelBuilder::createQueueChannel('async_channel'), - SimpleMessageChannelBuilder::createQueueChannel('failing_channel'), - SimpleChannelInterceptorBuilder::create('failing_channel', 'filteringChannelInterceptor'), - ], ['filteringChannelInterceptor' => $filteringChannelInterceptor]); - $ecotone->sendCommandWithRouting('user.register', '1'); - - $this->assertNull($this->runConsumer($ecotone)); - - $recorder->clear(); - $ecotone->sendCommandWithRouting('user.register', '2'); - $this->assertSame('send command', self::rootSpan($recorder)->name); - } - - public function test_a_consumer_receiving_a_message_without_a_trace_carrier_starts_its_own_trace(): void - { - $rebuildingChannelInterceptor = new class () extends AbstractChannelInterceptor { - public function preSend(Message $message, MessageChannel $messageChannel): ?Message - { - return StaticMessageBuilder::withPayload($message->getPayload())->build(); - } - }; - $recorder = new InMemorySpanRecorder(); - $ecotone = $this->bootstrapWithEventForwardedToSecondChannel($recorder, [ - SimpleMessageChannelBuilder::createQueueChannel('async_channel'), - SimpleMessageChannelBuilder::createQueueChannel('failing_channel'), - SimpleChannelInterceptorBuilder::create('failing_channel', 'rebuildingChannelInterceptor'), - ], ['rebuildingChannelInterceptor' => $rebuildingChannelInterceptor]); - $ecotone->sendCommandWithRouting('user.register', '1'); - $this->assertNull($this->runConsumer($ecotone)); - $recorder->clear(); - - $this->assertNull($this->runConsumer($ecotone, 'failing_channel')); - - $receivingSpan = self::singleSpanNamed('receive failing_channel', $recorder); - $this->assertTrue($receivingSpan->isRoot()); - } - - public function test_an_invalid_trace_carrier_on_the_received_message_is_ignored_and_a_new_trace_starts(): void - { - $corruptingChannelInterceptor = new class () extends AbstractChannelInterceptor { - public function preSend(Message $message, MessageChannel $messageChannel): ?Message - { - return StaticMessageBuilder::fromMessage($message)->setHeader(MessageHeaders::TRACEPARENT, 'not-a-traceparent')->build(); - } - }; - $recorder = new InMemorySpanRecorder(); - $users = $this->asynchronousUsers(); - - $ecotone = EcotoneLite::bootstrapFlowTesting( - [$users::class], - [$users, 'corruptingChannelInterceptor' => $corruptingChannelInterceptor, ...self::recordingSpansInto($recorder)], - [ - self::tracingConfiguration(), - SimpleMessageChannelBuilder::createQueueChannel('async_channel'), - SimpleChannelInterceptorBuilder::create('async_channel', 'corruptingChannelInterceptor')->withPrecedence(-500), - ], - ); - $ecotone->sendCommandWithRouting('user.register', '1'); - $recorder->clear(); - - $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - - $this->assertTrue(self::singleSpanNamed('receive async_channel', $recorder)->isRoot()); - } - public function test_a_command_carrying_a_trace_carrier_continues_that_trace_as_a_child(): void { $recorder = new InMemorySpanRecorder(); @@ -200,46 +100,6 @@ public function register(string $userId): void $this->assertNotSame('b7ad6b7169203331', $handler->parentSpanId); } - public function test_a_failing_asynchronous_handler_with_a_failing_error_channel_still_ends_every_span(): void - { - $recorder = new InMemorySpanRecorder(); - $users = new class () { - #[Asynchronous('async_channel')] - #[CommandHandler('user.register', endpointId: 'userRegisterEndpoint')] - public function register(string $userId): void - { - throw new RuntimeException('Handler failure'); - } - }; - - $ecotone = EcotoneLite::bootstrapFlowTesting( - [$users::class], - [$users, ...self::recordingSpansInto($recorder)], - [ - self::tracingConfiguration(), - SimpleMessageChannelBuilder::createQueueChannel('async_channel'), - ExceptionalQueueChannel::createWithExceptionOnSend('error_channel'), - ServiceConfiguration::createWithDefaults()->withDefaultErrorChannel('error_channel'), - ], - enabledModules: [ModulePackageList::CORE_PACKAGE], - ); - $ecotone->sendCommandWithRouting('user.register', '1'); - $recorder->clear(); - - try { - $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); - } catch (RuntimeException $failure) { - $this->assertStringContainsString('Exception on send', $failure->getMessage()); - } - - self::assertEverySpanNamedHasStatus('process class@anonymous::register', SpanStatus::Error, $recorder); - self::assertEverySpanNamedHasStatus('send error_channel', SpanStatus::Error, $recorder); - - $recorder->clear(); - $ecotone->sendCommandWithRouting('user.register', '2'); - $this->assertSame('send command', self::rootSpan($recorder)->name); - } - public function test_distributed_bus_send_is_a_producer_span_carrying_the_trace_to_the_other_service(): void { $recorder = new InMemorySpanRecorder(); @@ -271,40 +131,6 @@ public function create(string $description, #[Header(MessageHeaders::TRACEPARENT $this->assertSame($distributedBus->spanId, self::singleSpanNamed('send distributed_channel', $recorder)->parentSpanId); } - public function test_a_failing_distributed_bus_send_fails_the_span_and_leaves_no_open_span_behind(): void - { - $recorder = new InMemorySpanRecorder(); - $users = new class () { - #[CommandHandler('user.register')] - public function register(string $userId): void - { - } - }; - $ecotone = EcotoneLite::bootstrapFlowTesting( - [$users::class], - [$users, ...self::recordingSpansInto($recorder)], - [ - self::tracingConfiguration(), - ExceptionalQueueChannel::createWithExceptionOnSend('distributed_channel'), - DistributedServiceMap::initialize()->withCommandMapping(targetServiceName: 'ticket_service', channelName: 'distributed_channel'), - ServiceConfiguration::createWithDefaults()->withServiceName('user_service'), - ], - licenceKey: LicenceTesting::VALID_LICENCE, - ); - - try { - $ecotone->getDistributedBus()->sendCommand('ticket_service', 'ticket.create', 'User changed billing address'); - $this->fail('Expected the send failure to propagate'); - } catch (RuntimeException $failure) { - $this->assertStringContainsString('Exception on send', $failure->getMessage()); - } - - $this->assertSame(SpanStatus::Error, self::singleSpanNamed('send distributed', $recorder)->status); - $recorder->clear(); - $ecotone->sendCommandWithRouting('user.register', '1'); - $this->assertSame('send command', self::rootSpan($recorder)->name); - } - private function bootstrapWithEventForwardedToSecondChannel(InMemorySpanRecorder $recorder, array $channelsToRegister, array $servicesToRegister = []): FlowTestSupport { $users = new class () { @@ -324,7 +150,7 @@ public function notify(string $userId): void }; return EcotoneLite::bootstrapFlowTesting( - [$users::class, $notifier::class], + [$users::class, $notifier::class, ...array_map(fn (object $service): string => $service::class, $servicesToRegister)], [$users, $notifier, ...self::recordingSpansInto($recorder), ...$servicesToRegister], [self::tracingConfiguration(), ...$channelsToRegister], enabledModules: [ModulePackageList::CORE_PACKAGE], From 47614a38331cbf3767b52bba9fc0c52bf42d865c Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 27/50] feat(tracing): a flow is tracked by a tracingId header, and TraceCompletion is renamed TracingGateway An application names the flow it wants to track with MessageHeaders::TRACING_ID, a UUID or 32 hexadecimal characters, and passes the same value to TracingGateway, whose methods and TraceCompleted now speak of tracingId. The first span turns it into the W3C traceparent, so a message that already carries a traceparent from an instrumented request keeps that trace; carrying both is refused, and so is a tracingId in neither format, each naming the way out. tracingId is a framework header, not propagated to child messages, and a consumer continues the traceparent its channel stamped. CurrentTrace::traceId() becomes tracingId(). --- .../StoredTracesWithOpenTelemetryTest.php | 8 +- .../Conformance/TraceStoreConformanceTest.php | 12 +-- .../Ecotone/Api/Messaging/MessageHeaders.php | 1 + packages/Ecotone/Api/Tracing/CurrentTrace.php | 2 +- .../Ecotone/Api/Tracing/TraceCompleted.php | 2 +- .../Ecotone/Api/Tracing/TraceCompletion.php | 22 ------ .../Ecotone/Api/Tracing/TracingGateway.php | 22 ++++++ .../Messaging/Support/FrameworkHeaders.php | 1 + .../src/Tracing/Config/StoredTracesModule.php | 4 +- ...ion.php => TracingGatewayRegistration.php} | 16 ++-- .../src/Tracing/Config/TracingModule.php | 4 +- ...ompletion.php => StoredTracingGateway.php} | 34 ++++---- .../src/Tracing/TracerCurrentTrace.php | 2 +- .../Tracing/TracingDisabledCurrentTrace.php | 2 +- packages/Ecotone/src/Tracing/TracingId.php | 31 ++++++++ .../src/Tracing/TracingInterceptor.php | 26 ++++++- .../Tracing/UnavailableTraceCompletion.php | 40 ---------- .../src/Tracing/UnavailableTracingGateway.php | 40 ++++++++++ .../tests/Tracing/TraceCompletionTest.php | 78 ++++++++++++++++--- .../StoredTracesWithNoopProviderTest.php | 4 +- upgrade-2.0.md | 21 +++-- 21 files changed, 246 insertions(+), 126 deletions(-) delete mode 100644 packages/Ecotone/Api/Tracing/TraceCompletion.php create mode 100644 packages/Ecotone/Api/Tracing/TracingGateway.php rename packages/Ecotone/src/Tracing/Config/{TraceCompletionGatewayRegistration.php => TracingGatewayRegistration.php} (62%) rename packages/Ecotone/src/Tracing/{StoredTraceCompletion.php => StoredTracingGateway.php} (67%) create mode 100644 packages/Ecotone/src/Tracing/TracingId.php delete mode 100644 packages/Ecotone/src/Tracing/UnavailableTraceCompletion.php create mode 100644 packages/Ecotone/src/Tracing/UnavailableTracingGateway.php diff --git a/Monorepo/CrossModuleTests/Tests/Tracing/StoredTracesWithOpenTelemetryTest.php b/Monorepo/CrossModuleTests/Tests/Tracing/StoredTracesWithOpenTelemetryTest.php index 4aadfd1b1..b4d616139 100644 --- a/Monorepo/CrossModuleTests/Tests/Tracing/StoredTracesWithOpenTelemetryTest.php +++ b/Monorepo/CrossModuleTests/Tests/Tracing/StoredTracesWithOpenTelemetryTest.php @@ -19,8 +19,8 @@ use Ecotone\Api\Tracing\OnTraceCompleted; use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\TraceCompleted; -use Ecotone\Api\Tracing\TraceCompletion; use Ecotone\Api\Tracing\TracingConfiguration; +use Ecotone\Api\Tracing\TracingGateway; use Ecotone\Test\LicenceTesting; use Monorepo\CrossModuleTests\Tests\MessagingTestCase; use OpenTelemetry\API\Trace\TracerProviderInterface; @@ -134,9 +134,9 @@ private function bootstrap(InMemoryExporter $exporter, array $services): FlowTes )->setupDatabaseTables(); } - private function completion(FlowTestSupport $ecotone): TraceCompletion + private function completion(FlowTestSupport $ecotone): TracingGateway { - return $ecotone->getGateway(TraceCompletion::class); + return $ecotone->getGateway(TracingGateway::class); } private function users(): object @@ -170,7 +170,7 @@ private function traceCompletedSubscriber(): object #[OnTraceCompleted] public function whenCompleted(TraceCompleted $event): void { - $this->completedTraces[] = $event->traceId; + $this->completedTraces[] = $event->tracingId; } }; } diff --git a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php index 604e32ebe..0bc743e95 100644 --- a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php +++ b/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php @@ -34,9 +34,9 @@ use Ecotone\Api\Tracing\SpanEnded; use Ecotone\Api\Tracing\SpanStatus; use Ecotone\Api\Tracing\TraceCompleted; -use Ecotone\Api\Tracing\TraceCompletion; use Ecotone\Api\Tracing\TraceCompletionTimedOut; use Ecotone\Api\Tracing\TracingConfiguration; +use Ecotone\Api\Tracing\TracingGateway; use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Test\LicenceTesting; use PHPUnit\Framework\Attributes\DataProvider; @@ -129,7 +129,7 @@ public function test_a_trace_completed_subscriber_is_called_once_after_the_last_ #[OnTraceCompleted] public function whenCompleted(TraceCompleted $event): void { - $this->completedTraces[] = $event->traceId; + $this->completedTraces[] = $event->tracingId; } }; $ecotone = $this->bootstrap($implementation, [$this->users(), $this->asynchronousNotifier(), $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); @@ -352,7 +352,7 @@ public function test_a_subscriber_that_throws_does_not_fail_the_committed_comman #[OnTraceCompleted] public function whenCompleted(TraceCompleted $event): void { - $this->completedTraces[] = $event->traceId; + $this->completedTraces[] = $event->tracingId; if (count($this->completedTraces) === 1) { throw new RuntimeException('Websocket is down'); } @@ -380,7 +380,7 @@ public function test_a_trace_that_receives_new_work_after_completing_is_notified #[OnTraceCompleted] public function whenCompleted(TraceCompleted $event): void { - $this->completedTraces[] = $event->traceId; + $this->completedTraces[] = $event->tracingId; } }; $orders = new class () { @@ -561,9 +561,9 @@ classesToResolve: array_map(fn (object $service): string => $service::class, $se )->setupDatabaseTables(); } - private function completion(FlowTestSupport $ecotone): TraceCompletion + private function completion(FlowTestSupport $ecotone): TracingGateway { - return $ecotone->getGateway(TraceCompletion::class); + return $ecotone->getGateway(TracingGateway::class); } private function users(): object diff --git a/packages/Ecotone/Api/Messaging/MessageHeaders.php b/packages/Ecotone/Api/Messaging/MessageHeaders.php index 3bdc283eb..db492571c 100644 --- a/packages/Ecotone/Api/Messaging/MessageHeaders.php +++ b/packages/Ecotone/Api/Messaging/MessageHeaders.php @@ -68,6 +68,7 @@ final class MessageHeaders public const REPLY_CONTENT_TYPE = 'ecotone_messaging_reply_content_type'; public const REVISION = 'revision'; + public const TRACING_ID = 'tracingId'; public const TRACEPARENT = 'traceparent'; public const TRACESTATE = 'tracestate'; public const TRACING_DELIVERY = 'ecotone_tracing_delivery'; diff --git a/packages/Ecotone/Api/Tracing/CurrentTrace.php b/packages/Ecotone/Api/Tracing/CurrentTrace.php index 8ad7ad1c5..b45bbeccc 100644 --- a/packages/Ecotone/Api/Tracing/CurrentTrace.php +++ b/packages/Ecotone/Api/Tracing/CurrentTrace.php @@ -9,5 +9,5 @@ */ interface CurrentTrace { - public function traceId(): ?string; + public function tracingId(): ?string; } diff --git a/packages/Ecotone/Api/Tracing/TraceCompleted.php b/packages/Ecotone/Api/Tracing/TraceCompleted.php index 618787ace..3f53e8561 100644 --- a/packages/Ecotone/Api/Tracing/TraceCompleted.php +++ b/packages/Ecotone/Api/Tracing/TraceCompleted.php @@ -10,7 +10,7 @@ */ final class TraceCompleted { - public function __construct(public readonly string $traceId) + public function __construct(public readonly string $tracingId) { } } diff --git a/packages/Ecotone/Api/Tracing/TraceCompletion.php b/packages/Ecotone/Api/Tracing/TraceCompletion.php deleted file mode 100644 index b2a9f123e..000000000 --- a/packages/Ecotone/Api/Tracing/TraceCompletion.php +++ /dev/null @@ -1,22 +0,0 @@ -registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnTraceCompleted::CHANNEL)); $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnSpanEnded::CHANNEL)); - TraceCompletionGatewayRegistration::register($messagingConfiguration, StoredTraceCompletion::class, new Definition(StoredTraceCompletion::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $this->pollingProjectionNames])); + TracingGatewayRegistration::register($messagingConfiguration, StoredTracingGateway::class, new Definition(StoredTracingGateway::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $this->pollingProjectionNames])); $messagingConfiguration->registerServiceDefinition( ExpiredTraceRemover::class, diff --git a/packages/Ecotone/src/Tracing/Config/TraceCompletionGatewayRegistration.php b/packages/Ecotone/src/Tracing/Config/TracingGatewayRegistration.php similarity index 62% rename from packages/Ecotone/src/Tracing/Config/TraceCompletionGatewayRegistration.php rename to packages/Ecotone/src/Tracing/Config/TracingGatewayRegistration.php index a9a8cf2d2..a6c75915b 100644 --- a/packages/Ecotone/src/Tracing/Config/TraceCompletionGatewayRegistration.php +++ b/packages/Ecotone/src/Tracing/Config/TracingGatewayRegistration.php @@ -4,33 +4,33 @@ namespace Ecotone\Tracing\Config; -use Ecotone\Api\Tracing\TraceCompletion; +use Ecotone\Api\Tracing\TracingGateway; use Ecotone\Messaging\Config\Configuration; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Handler\Gateway\GatewayProxyBuilder; use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayHeaderBuilder; use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayPayloadBuilder; use Ecotone\Messaging\Handler\ServiceActivator\MessageProcessorActivatorBuilder; -use Ecotone\Tracing\StoredTraceCompletion; +use Ecotone\Tracing\StoredTracingGateway; /** * licence Apache-2.0 */ -final class TraceCompletionGatewayRegistration +final class TracingGatewayRegistration { public static function register(Configuration $messagingConfiguration, string $serviceClass, Definition $service): void { $messagingConfiguration->registerServiceDefinition($serviceClass, $service); foreach ([ - 'isCompleted' => [GatewayPayloadBuilder::create('traceId')], - 'isCompletedFor' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('destination', StoredTraceCompletion::DESTINATION_HEADER)], - 'await' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('timeout', StoredTraceCompletion::TIMEOUT_HEADER)], - 'awaitFor' => [GatewayPayloadBuilder::create('traceId'), GatewayHeaderBuilder::create('destination', StoredTraceCompletion::DESTINATION_HEADER), GatewayHeaderBuilder::create('timeout', StoredTraceCompletion::TIMEOUT_HEADER)], + 'isCompleted' => [GatewayPayloadBuilder::create('tracingId')], + 'isCompletedFor' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('destination', StoredTracingGateway::DESTINATION_HEADER)], + 'await' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('timeout', StoredTracingGateway::TIMEOUT_HEADER)], + 'awaitFor' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('destination', StoredTracingGateway::DESTINATION_HEADER), GatewayHeaderBuilder::create('timeout', StoredTracingGateway::TIMEOUT_HEADER)], ] as $method => $parameterConverters) { $channel = 'ecotone.tracing.completion.' . $method; $messagingConfiguration ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod($serviceClass, $method)->withInputChannelName($channel)) - ->registerGatewayBuilder(GatewayProxyBuilder::create(TraceCompletion::class, TraceCompletion::class, $method, $channel)->withParameterConverters($parameterConverters)); + ->registerGatewayBuilder(GatewayProxyBuilder::create(TracingGateway::class, TracingGateway::class, $method, $channel)->withParameterConverters($parameterConverters)); } } } diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index 7fb0e843e..0ff0705b0 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -48,7 +48,7 @@ use Ecotone\Tracing\TracerCurrentTrace; use Ecotone\Tracing\TracingDisabledCurrentTrace; use Ecotone\Tracing\TracingInterceptor; -use Ecotone\Tracing\UnavailableTraceCompletion; +use Ecotone\Tracing\UnavailableTracingGateway; #[ModuleAnnotation] /** @@ -99,7 +99,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO $spanRecorderReferences = ExtensionObjectResolver::resolve(SpanRecorderReference::class, $extensionObjects); if (! $tracingConfiguration->storedTraces) { - TraceCompletionGatewayRegistration::register($messagingConfiguration, UnavailableTraceCompletion::class, new Definition(UnavailableTraceCompletion::class)); + TracingGatewayRegistration::register($messagingConfiguration, UnavailableTracingGateway::class, new Definition(UnavailableTracingGateway::class)); } if ($tracingConfiguration->spanRecorderReferenceNames === [] && $tracerReferences === [] && $spanRecorderReferences === []) { $messagingConfiguration->registerServiceDefinition(CurrentTrace::class, new Definition(TracingDisabledCurrentTrace::class)); diff --git a/packages/Ecotone/src/Tracing/StoredTraceCompletion.php b/packages/Ecotone/src/Tracing/StoredTracingGateway.php similarity index 67% rename from packages/Ecotone/src/Tracing/StoredTraceCompletion.php rename to packages/Ecotone/src/Tracing/StoredTracingGateway.php index 83481777a..8c32319e6 100644 --- a/packages/Ecotone/src/Tracing/StoredTraceCompletion.php +++ b/packages/Ecotone/src/Tracing/StoredTracingGateway.php @@ -28,7 +28,7 @@ * * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev */ -final class StoredTraceCompletion +final class StoredTracingGateway { public const DESTINATION_HEADER = 'ecotone_tracing_completion_destination'; public const TIMEOUT_HEADER = 'ecotone_tracing_completion_timeout'; @@ -44,49 +44,53 @@ public function __construct( ) { } - public function isCompleted(string $traceId): bool + public function isCompleted(string $tracingId): bool { - return $this->traceStore->progressOf($traceId)->isCompleted(); + $tracingId = TracingId::normalize($tracingId); + return $this->traceStore->progressOf($tracingId)->isCompleted(); } - public function isCompletedFor(string $traceId, #[Header(self::DESTINATION_HEADER)] string $destination): bool + public function isCompletedFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] string $destination): bool { + $tracingId = TracingId::normalize($tracingId); $this->refusePollingProjection($destination); - return $this->traceStore->progressOf($traceId)->isCompletedFor($destination); + return $this->traceStore->progressOf($tracingId)->isCompletedFor($destination); } - public function await(string $traceId, #[Header(self::TIMEOUT_HEADER)] Duration $timeout): void + public function await(string $tracingId, #[Header(self::TIMEOUT_HEADER)] Duration $timeout): void { + $tracingId = TracingId::normalize($tracingId); $deadline = $this->clock->now()->add($timeout); while (true) { - $progress = $this->traceStore->progressOf($traceId); + $progress = $this->traceStore->progressOf($tracingId); if ($progress->isCompleted()) { return; } if ($this->clock->now() >= $deadline) { throw $progress->recorded - ? TraceCompletionTimedOut::forTrace($traceId, $timeout, $progress->pendingDestinations()) - : TraceCompletionTimedOut::forUnrecordedTrace($traceId, $timeout); + ? TraceCompletionTimedOut::forTrace($tracingId, $timeout, $progress->pendingDestinations()) + : TraceCompletionTimedOut::forUnrecordedTrace($tracingId, $timeout); } $this->clock->sleep(Duration::milliseconds(self::POLL_INTERVAL_IN_MILLISECONDS)); } } - public function awaitFor(string $traceId, #[Header(self::DESTINATION_HEADER)] string $destination, #[Header(self::TIMEOUT_HEADER)] Duration $timeout): void + public function awaitFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] string $destination, #[Header(self::TIMEOUT_HEADER)] Duration $timeout): void { + $tracingId = TracingId::normalize($tracingId); $this->refusePollingProjection($destination); $deadline = $this->clock->now()->add($timeout); while (true) { - $progress = $this->traceStore->progressOf($traceId); + $progress = $this->traceStore->progressOf($tracingId); if ($progress->isCompletedFor($destination)) { return; } if ($this->clock->now() >= $deadline) { throw match (true) { - ! $progress->recorded => TraceCompletionTimedOut::forUnrecordedTrace($traceId, $timeout), - ! $progress->reached($destination) => TraceCompletionTimedOut::forUnreachedDestination($traceId, $destination, $timeout, $progress->destinations()), - default => TraceCompletionTimedOut::forDestination($traceId, $destination, $timeout), + ! $progress->recorded => TraceCompletionTimedOut::forUnrecordedTrace($tracingId, $timeout), + ! $progress->reached($destination) => TraceCompletionTimedOut::forUnreachedDestination($tracingId, $destination, $timeout, $progress->destinations()), + default => TraceCompletionTimedOut::forDestination($tracingId, $destination, $timeout), }; } $this->clock->sleep(Duration::milliseconds(self::POLL_INTERVAL_IN_MILLISECONDS)); @@ -96,7 +100,7 @@ public function awaitFor(string $traceId, #[Header(self::DESTINATION_HEADER)] st private function refusePollingProjection(string $destination): void { if (in_array($destination, $this->pollingProjectionNames, true)) { - throw ConfigurationException::create(sprintf('Projection %s runs by #[Polling], so it is not part of the trace that appended its events and TraceCompletion cannot await it. Await the handler that published the event, or make the projection event-driven by removing #[Polling].', $destination)); + throw ConfigurationException::create(sprintf('Projection %s runs by #[Polling], so it is not part of the trace that appended its events and TracingGateway cannot await it. Await the handler that published the event, or make the projection event-driven by removing #[Polling].', $destination)); } } } diff --git a/packages/Ecotone/src/Tracing/TracerCurrentTrace.php b/packages/Ecotone/src/Tracing/TracerCurrentTrace.php index c25c143e3..bfd1466d8 100644 --- a/packages/Ecotone/src/Tracing/TracerCurrentTrace.php +++ b/packages/Ecotone/src/Tracing/TracerCurrentTrace.php @@ -15,7 +15,7 @@ public function __construct(private Tracer $tracer) { } - public function traceId(): ?string + public function tracingId(): ?string { return $this->tracer->currentSpan()?->context()->traceId; } diff --git a/packages/Ecotone/src/Tracing/TracingDisabledCurrentTrace.php b/packages/Ecotone/src/Tracing/TracingDisabledCurrentTrace.php index 4f0e3d694..7d1898e99 100644 --- a/packages/Ecotone/src/Tracing/TracingDisabledCurrentTrace.php +++ b/packages/Ecotone/src/Tracing/TracingDisabledCurrentTrace.php @@ -12,7 +12,7 @@ */ final class TracingDisabledCurrentTrace implements CurrentTrace { - public function traceId(): ?string + public function tracingId(): ?string { throw ConfigurationException::create('No trace is recorded, so there is no current trace id. Enable tracing: register TracingConfiguration::createWithDefaults()->withSpanRecorder() or ->withStoredTraces() from a #[ServiceContext] method, or install ecotone/open-telemetry with a TracerProviderInterface.'); } diff --git a/packages/Ecotone/src/Tracing/TracingId.php b/packages/Ecotone/src/Tracing/TracingId.php new file mode 100644 index 000000000..6c0172df6 --- /dev/null +++ b/packages/Ecotone/src/Tracing/TracingId.php @@ -0,0 +1,31 @@ +toRfc4122(), or a W3C trace id.', $tracingId)); + } + + public static function normalizedOrNull(string $tracingId): ?string + { + $candidate = strtolower(str_replace('-', '', $tracingId)); + if (preg_match(self::TRACE_ID_PATTERN, $candidate) !== 1 || $candidate === str_repeat('0', 32)) { + return null; + } + + return $candidate; + } +} diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 7662c1236..7484fd932 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -15,6 +15,7 @@ use Ecotone\Api\Tracing\SpanAttribute; use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\SpanKind; +use Ecotone\Messaging\Support\InvalidArgumentException; use Ecotone\Messaging\Support\MessageBuilder; use Ecotone\Projecting\ProjectionTriggerEndpoint; use Throwable; @@ -153,11 +154,29 @@ public static function isCarried(Tracer $tracer, Message $message, bool $startsT public static function remoteParentOf(Message $message): ?SpanContext { $headers = $message->getHeaders(); - - return SpanContext::fromTraceparent( + $traceparent = SpanContext::fromTraceparent( $headers->containsKey(MessageHeaders::TRACEPARENT) ? $headers->get(MessageHeaders::TRACEPARENT) : null, $headers->containsKey(MessageHeaders::TRACESTATE) ? (string) $headers->get(MessageHeaders::TRACESTATE) : '', ); + if ($traceparent !== null || ! $headers->containsKey(MessageHeaders::TRACING_ID)) { + return $traceparent; + } + $tracingId = TracingId::normalizedOrNull((string) $headers->get(MessageHeaders::TRACING_ID)); + + return $tracingId === null ? null : new SpanContext($tracingId, bin2hex(random_bytes(8))); + } + + private static function entryParentOf(Message $message): ?SpanContext + { + $headers = $message->getHeaders(); + if (! $headers->containsKey(MessageHeaders::TRACING_ID)) { + return self::remoteParentOf($message); + } + if ($headers->containsKey(MessageHeaders::TRACEPARENT)) { + throw InvalidArgumentException::create('The message carries both tracingId and traceparent, so it is not clear which trace it belongs to. Pass tracingId to track the flow with TracingGateway, or traceparent when the trace comes from OpenTelemetry instrumentation, not both.'); + } + + return new SpanContext(TracingId::normalize((string) $headers->get(MessageHeaders::TRACING_ID)), bin2hex(random_bytes(8))); } private function traceHandler(MethodInvocation $methodInvocation, Message $message, string $endpointId): mixed @@ -204,13 +223,14 @@ private static function shortNameOf(string $classAndMethod): string private function trace(string $name, MethodInvocation $methodInvocation, Message $message, SpanKind $kind, array $attributes): mixed { $isLocalRoot = $this->tracer->currentSpan() === null; + $remoteParent = ! $isLocalRoot ? null : ($kind === SpanKind::Consumer ? self::remoteParentOf($message) : self::entryParentOf($message)); if ($isLocalRoot && $kind !== SpanKind::Consumer) { $attributes[SpanAttribute::TRACE_ROOT] = true; } if (self::isCarried($this->tracer, $message, $isLocalRoot && $kind !== SpanKind::Consumer)) { $attributes[SpanAttribute::TRACE_CARRIED] = true; } - $span = $this->tracer->startSpan($name, $kind, [...$this->messageSpanAttributes->from($message), ...$attributes], $isLocalRoot ? self::remoteParentOf($message) : null); + $span = $this->tracer->startSpan($name, $kind, [...$this->messageSpanAttributes->from($message), ...$attributes], $remoteParent); try { $result = $methodInvocation->proceed(); diff --git a/packages/Ecotone/src/Tracing/UnavailableTraceCompletion.php b/packages/Ecotone/src/Tracing/UnavailableTraceCompletion.php deleted file mode 100644 index a59aef359..000000000 --- a/packages/Ecotone/src/Tracing/UnavailableTraceCompletion.php +++ /dev/null @@ -1,40 +0,0 @@ -withStoredTraces() from a #[ServiceContext] method.'); - } -} diff --git a/packages/Ecotone/src/Tracing/UnavailableTracingGateway.php b/packages/Ecotone/src/Tracing/UnavailableTracingGateway.php new file mode 100644 index 000000000..ce4d9c0ea --- /dev/null +++ b/packages/Ecotone/src/Tracing/UnavailableTracingGateway.php @@ -0,0 +1,40 @@ +withStoredTraces() from a #[ServiceContext] method.'); + } +} diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php index 81f595796..2e757fd1a 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Ecotone/tests/Tracing/TraceCompletionTest.php @@ -33,9 +33,9 @@ use Ecotone\Api\Tracing\SpanEnded; use Ecotone\Api\Tracing\SpanStatus; use Ecotone\Api\Tracing\TraceCompleted; -use Ecotone\Api\Tracing\TraceCompletion; use Ecotone\Api\Tracing\TraceCompletionTimedOut; use Ecotone\Api\Tracing\TracingConfiguration; +use Ecotone\Api\Tracing\TracingGateway; use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Support\LicensingException; use Ecotone\Test\LicenceTesting; @@ -193,7 +193,7 @@ public function test_a_trace_completed_subscriber_is_called_once_after_the_last_ #[OnTraceCompleted] public function whenCompleted(TraceCompleted $event): void { - $this->completedTraces[] = $event->traceId; + $this->completedTraces[] = $event->tracingId; } }; $ecotone = $this->bootstrap([$this->users(), $this->asynchronousNotifier(), $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); @@ -218,7 +218,7 @@ public function test_a_synchronous_trace_notifies_its_subscriber_when_the_bus_re #[OnTraceCompleted] public function whenCompleted(TraceCompleted $event): void { - $this->completedTraces[] = $event->traceId; + $this->completedTraces[] = $event->tracingId; } }; $ecotone = $this->bootstrap([$this->users(), $subscriber]); @@ -331,7 +331,7 @@ public function whenCompleted(TraceCompleted $event, CommandBus $commandBus): vo { $this->notifications++; if ($this->notifications < 5) { - $commandBus->sendWithRouting('cache.refresh', $event->traceId); + $commandBus->sendWithRouting('cache.refresh', $event->tracingId); } } @@ -489,7 +489,7 @@ public function test_the_completion_gateway_without_stored_traces_names_the_conf $ecotone = EcotoneLite::bootstrapFlowTesting([$this->users()::class], [$this->users()]); $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('TraceCompletion needs stored traces, which decide when a trace is complete. Register TracingConfiguration::createWithDefaults()->withStoredTraces() from a #[ServiceContext] method.'); + $this->expectExceptionMessage('TracingGateway needs stored traces, which decide when a trace is complete. Register TracingConfiguration::createWithDefaults()->withStoredTraces() from a #[ServiceContext] method.'); $this->completion($ecotone)->isCompleted('0af7651916cd43dd8448eb211c80319c'); } @@ -571,7 +571,7 @@ public function whenPlaced(array $event): void $ecotone = $this->bootstrap([$projection]); $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('Projection orders_list runs by #[Polling], so it is not part of the trace that appended its events and TraceCompletion cannot await it. Await the handler that published the event, or make the projection event-driven by removing #[Polling].'); + $this->expectExceptionMessage('Projection orders_list runs by #[Polling], so it is not part of the trace that appended its events and TracingGateway cannot await it. Await the handler that published the event, or make the projection event-driven by removing #[Polling].'); $this->completion($ecotone)->awaitFor('0af7651916cd43dd8448eb211c80319c', 'orders_list', Duration::milliseconds(10)); } @@ -584,7 +584,7 @@ public function test_a_handler_can_read_the_trace_id_it_runs_in(): void #[CommandHandler('user.register', endpointId: 'user.registration')] public function register(string $userId, #[Reference] CurrentTrace $currentTrace): void { - $this->traceId = $currentTrace->traceId(); + $this->traceId = $currentTrace->tracingId(); } }; $ecotone = $this->bootstrap([$users]); @@ -602,7 +602,7 @@ public function test_reading_the_current_trace_without_tracing_names_how_to_enab #[CommandHandler('user.register', endpointId: 'user.registration')] public function register(string $userId, #[Reference] CurrentTrace $currentTrace): void { - $currentTrace->traceId(); + $currentTrace->tracingId(); } }; $ecotone = EcotoneLite::bootstrapFlowTesting([$users::class], [$users]); @@ -697,6 +697,64 @@ public function notify(string $userId, #[Headers] array $headers): void $this->assertStringContainsString('congo=t61rcWkgMzE', $notifier->headers[MessageHeaders::TRACESTATE]); } + public function test_a_command_sent_with_a_tracing_id_is_awaited_by_that_tracing_id(): void + { + $ecotone = $this->bootstrap([$this->users(), $this->asynchronousNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '4bf92f3577b34da6a3ce929d0e0e4736']); + + $this->assertFalse($this->completion($ecotone)->isCompleted('4bf92f3577b34da6a3ce929d0e0e4736')); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $this->assertTrue($this->completion($ecotone)->isCompleted('4bf92f3577b34da6a3ce929d0e0e4736')); + } + + public function test_a_uuid_is_accepted_as_tracing_id_and_awaited_by_the_same_uuid(): void + { + $ecotone = $this->bootstrap([$this->users(), $this->asynchronousNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '0199f2f4-5e3b-7a42-9b1c-2f7d3e8a6c11']); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertTrue($this->completion($ecotone)->isCompleted('0199f2f4-5e3b-7a42-9b1c-2f7d3e8a6c11')); + $this->assertTrue($this->completion($ecotone)->isCompletedFor('0199f2f4-5e3b-7a42-9b1c-2f7d3e8a6c11', 'user.notifier')); + } + + public function test_a_handler_reads_the_tracing_id_the_command_was_sent_with(): void + { + $users = new class () { + public ?string $tracingId = null; + + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId, #[Reference] CurrentTrace $currentTrace): void + { + $this->tracingId = $currentTrace->tracingId(); + } + }; + $ecotone = $this->bootstrap([$users]); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '0199f2f4-5e3b-7a42-9b1c-2f7d3e8a6c11']); + + $this->assertSame('0199f2f45e3b7a429b1c2f7d3e8a6c11', $users->tracingId); + } + + public function test_a_message_carrying_both_a_tracing_id_and_a_traceparent_is_refused(): void + { + $ecotone = $this->bootstrap([$this->users()]); + + $this->expectExceptionMessage('The message carries both tracingId and traceparent, so it is not clear which trace it belongs to. Pass tracingId to track the flow with TracingGateway, or traceparent when the trace comes from OpenTelemetry instrumentation, not both.'); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '4bf92f3577b34da6a3ce929d0e0e4736', MessageHeaders::TRACEPARENT => SpanContext::newTrace()->toTraceparent()]); + } + + public function test_an_invalid_tracing_id_is_refused_naming_the_expected_format(): void + { + $ecotone = $this->bootstrap([$this->users()]); + + $this->expectExceptionMessage('tracingId `order-1` is neither 32 hexadecimal characters nor a UUID. Pass a UUID such as Uuid::v7()->toRfc4122(), or a W3C trace id.'); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => 'order-1']); + } + public function test_a_failed_unit_of_work_stores_nothing_by_default(): void { $subscriber = $this->spanEndedSubscriber(); @@ -809,9 +867,9 @@ private function bootstrap(array $services, array $extensionObjects = []): FlowT ); } - private function completion(FlowTestSupport $ecotone): TraceCompletion + private function completion(FlowTestSupport $ecotone): TracingGateway { - return $ecotone->getGateway(TraceCompletion::class); + return $ecotone->getGateway(TracingGateway::class); } private function users(): object diff --git a/packages/OpenTelemetry/tests/Integration/StoredTracesWithNoopProviderTest.php b/packages/OpenTelemetry/tests/Integration/StoredTracesWithNoopProviderTest.php index 79e86785f..7bd6bfad3 100644 --- a/packages/OpenTelemetry/tests/Integration/StoredTracesWithNoopProviderTest.php +++ b/packages/OpenTelemetry/tests/Integration/StoredTracesWithNoopProviderTest.php @@ -16,8 +16,8 @@ use Ecotone\Api\Tracing\OnSpanEnded; use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\SpanEnded; -use Ecotone\Api\Tracing\TraceCompletion; use Ecotone\Api\Tracing\TracingConfiguration; +use Ecotone\Api\Tracing\TracingGateway; use Ecotone\Test\LicenceTesting; use OpenTelemetry\API\Trace\NoopTracerProvider; use OpenTelemetry\API\Trace\TracerProviderInterface; @@ -84,6 +84,6 @@ public function whenEnded(SpanEnded $event): void $this->assertCount(count($subscriber->spanIds), array_unique($subscriber->spanIds)); $this->assertGreaterThan(3, count($subscriber->spanIds)); - $this->assertTrue($ecotone->getGateway(TraceCompletion::class)->isCompleted($trace->traceId)); + $this->assertTrue($ecotone->getGateway(TracingGateway::class)->isCompleted($trace->traceId)); } } diff --git a/upgrade-2.0.md b/upgrade-2.0.md index 9a8807d27..fe1a441d8 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3105,7 +3105,7 @@ whether the trace is complete. Nothing is written to the database after the comm that goes down right after a commit cannot fail a consumer whose work already committed, and a failure inside the transaction rolls the trace back with the work and leaves the message to be retried. Completion is claimed with one conditional update after the trace row was touched by the same transaction, so a second unit of work finishing the -same trace waits for the first one's commit and, on PostgreSQL and SQLite, reads its spans before deciding. From those spans the `TraceCompletion` gateway answers +same trace waits for the first one's commit and, on PostgreSQL and SQLite, reads its spans before deciding. From those spans the `TracingGateway` gateway answers `isCompleted($traceId)`, true once the root ended, every route the trace sent has an ended handler span, and every message it sent to a channel was consumed from that channel, so an asynchronous command, event or projection keeps the trace pending until its consumer processed that exact message; `isCompletedFor($traceId, $destination)` @@ -3122,7 +3122,7 @@ method, so it records the handler's own failure even when an error channel or a or bus span that wraps a failed unit of work ends as failed too, in OpenTelemetry as in the store, unless a retry inside it succeeded; and `TraceCompletionTimedOut` names the handlers and channels still pending, points to the dead letter when a handler -failed, and names a trace nobody recorded and a destination the trace never reached. `TraceCompletion` is always injectable; without +failed, and names a trace nobody recorded and a destination the trace never reached. `TracingGateway` is always injectable; without `withStoredTraces()` every call refuses with the configuration it needs. A method with `#[OnTraceCompleted]` receives a `TraceCompleted` each time the trace reaches completion, so a trace that is handed more work after completing, as when a request sends two commands under one `traceparent`, notifies again once that work is done; one with @@ -3130,7 +3130,7 @@ when a request sends two commands under one `traceparent`, notifies again once t a cache refresh never runs ahead of the data. Notifications are published from memory once the transaction committed, at most once: a subscriber that throws does not fail the command that already committed and is not called again, and a process that stops between the commit and the publication drops that notification, while -`TraceCompletion` still answers from the stored spans. A handler that needs the id of the +`TracingGateway` still answers from the stored spans. A handler that needs the id of the trace it runs in injects `Ecotone\Api\Tracing\CurrentTrace` and reads `traceId()`. `isCompletedFor()` and `awaitFor()` refuse a `#[Polling]` projection by name, because a polled run is not part of the trace that appended its events. A message sent through a dynamic channel, through the high-throughput outbox or replayed from the dead @@ -3138,8 +3138,13 @@ letter keeps the delivery it started as, so the trace completes when that messag a multi-tenant application stores traces on the connection `MultiTenantConfiguration::createWithDefaultConnection()` names, because the write after the unit of work has no tenant, and is refused at bootstrap without one. Work a subscriber does through a bus is stored like any other trace but never notifies subscribers in turn, so a subscriber that sends a -command does not call itself again. The trace id is the one in the `traceparent` header: mint it with `SpanContext::newTrace()` and send it -with the command, or take it from the HTTP request the command came from. `withStoredTraces(onlyTracesCarryingTraceparent: true)` stores only the traces whose root continued a +command does not call itself again. The flow to track is named by the `tracingId` header, +`MessageHeaders::TRACING_ID`: a UUID or 32 hexadecimal characters chosen by the application and sent with the command, +then passed to `TracingGateway`, whose methods all take that `tracingId`. Ecotone turns it into the W3C `traceparent` +of the first span, so a command that already carries a `traceparent`, as one from an instrumented HTTP request does, +is tracked by that trace id instead; a message carrying both is refused, because it is not clear which one names the +flow, and a `tracingId` that is neither format is refused naming the expected one. A handler reads the id of the flow +it runs in from `CurrentTrace::tracingId()`. `withStoredTraces(onlyTracesCarryingTraceparent: true)` stores only the traces whose root continued a `traceparent` handed in, from an HTTP request or minted with `SpanContext::newTrace()`, and carries that choice across channels in the W3C `tracestate` as `ecotone=carried`, next to any other vendor's entry, so ordinary traffic costs no storage and only the requests that await completion are kept. A flow test given no connection keeps the @@ -3149,9 +3154,9 @@ at bootstrap, because a store per process could never see a consumer's progress. retention still completes the trace it belongs to; the deletion runs as two statements, whatever the volume. ```php -$trace = SpanContext::newTrace(); -$commandBus->sendWithRouting('order.place', $order, metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); -$traceCompletion->awaitFor($trace->traceId, 'orders_list', Duration::seconds(5)); +$tracingId = Uuid::v7()->toRfc4122(); +$commandBus->sendWithRouting('order.place', $order, metadata: [MessageHeaders::TRACING_ID => $tracingId]); +$tracingGateway->awaitFor($tracingId, 'orders_list', Duration::seconds(5)); final class OrdersWebsocket { From 0bbd2e1c53900c74e47a7be540e46464737325ed Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 28/50] refactor(dbal): flow tracking, its TracingGateway and its notifications live in the DBAL package under FlowTrackingConfiguration TracingGateway, #[OnTraceCompleted], #[OnSpanEnded] and what they need only work with storage, so they moved with the store into Ecotone\Api\Dbal\Tracing and one DbalTracingModule. FlowTrackingConfiguration::createWithDefaults() tracks every flow with the gateway and the notifications on; withTracingGateway(false) and withNotifications(false) turn each off while the flows are still tracked, and an attribute left behind with notifications off is refused at bootstrap. withRetention() and withOnlyRequestedFlows() replace the withStoredTraces() arguments, and flow testing switches withInMemoryStorage() on when no connection is given. Core keeps spans, recorders, CurrentTrace and the OpenTelemetry seam, and no longer names a DBAL class; a subscriber's own handler is traced like any other. --- .../FlowTrackingWithNoopProviderTest.php | 16 +- ... => FlowTrackingWithOpenTelemetryTest.php} | 14 +- .../Api/Tracing/FlowTrackingConfiguration.php | 53 +++++ .../Api/Tracing/OnSpanEnded.php | 2 +- .../Api/Tracing/OnTraceCompleted.php | 2 +- .../Api/Tracing/SpanEnded.php | 4 +- .../Api/Tracing/TraceCompleted.php | 2 +- .../Api/Tracing/TraceCompletionTimedOut.php | 2 +- .../Api/Tracing/TracingGateway.php | 2 +- .../FlowTestingDbalConnections.php | 21 +- packages/Dbal/src/Tracing/DbalTraceStore.php | 3 - .../Dbal/src/Tracing/DbalTracingModule.php | 192 +++++++++++++-- .../src/Tracing/ExpiredTraceRemover.php | 2 +- .../src/Tracing/InMemoryTraceStore.php | 2 +- .../src/Tracing/StoredTracingGateway.php | 5 +- .../src/Tracing/StoringSpanRecorder.php | 10 +- .../src/Tracing/TraceLedger.php | 2 +- .../src/Tracing/TraceProgress.php | 2 +- .../src/Tracing/TraceRecordingInterceptor.php | 3 +- .../src/Tracing/TraceStore.php | 2 +- .../src/Tracing/UnavailableTracingGateway.php | 18 +- ...st.php => FlowTrackingConformanceTest.php} | 37 +-- .../Integration/Tracing/FlowTrackingTest.php} | 114 ++++++--- .../Api/Tracing/TracingConfiguration.php | 13 +- .../src/Messaging/Config/ModuleClassList.php | 2 - .../src/Tracing/Config/StoredTracesModule.php | 220 ------------------ .../Tracing/Config/TraceStoreReference.php | 15 -- .../Config/TracingGatewayRegistration.php | 36 --- .../src/Tracing/Config/TracingModule.php | 8 +- .../Tracing/TracingDisabledCurrentTrace.php | 2 +- upgrade-2.0.md | 117 +++++----- 31 files changed, 451 insertions(+), 472 deletions(-) rename packages/OpenTelemetry/tests/Integration/StoredTracesWithNoopProviderTest.php => Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithNoopProviderTest.php (86%) rename Monorepo/CrossModuleTests/Tests/Tracing/{StoredTracesWithOpenTelemetryTest.php => FlowTrackingWithOpenTelemetryTest.php} (93%) create mode 100644 packages/Dbal/Api/Tracing/FlowTrackingConfiguration.php rename packages/{Ecotone => Dbal}/Api/Tracing/OnSpanEnded.php (92%) rename packages/{Ecotone => Dbal}/Api/Tracing/OnTraceCompleted.php (92%) rename packages/{Ecotone => Dbal}/Api/Tracing/SpanEnded.php (78%) rename packages/{Ecotone => Dbal}/Api/Tracing/TraceCompleted.php (87%) rename packages/{Ecotone => Dbal}/Api/Tracing/TraceCompletionTimedOut.php (98%) rename packages/{Ecotone => Dbal}/Api/Tracing/TracingGateway.php (93%) rename packages/{Ecotone => Dbal}/src/Tracing/ExpiredTraceRemover.php (97%) rename packages/{Ecotone => Dbal}/src/Tracing/InMemoryTraceStore.php (98%) rename packages/{Ecotone => Dbal}/src/Tracing/StoredTracingGateway.php (97%) rename packages/{Ecotone => Dbal}/src/Tracing/StoringSpanRecorder.php (95%) rename packages/{Ecotone => Dbal}/src/Tracing/TraceLedger.php (99%) rename packages/{Ecotone => Dbal}/src/Tracing/TraceProgress.php (99%) rename packages/{Ecotone => Dbal}/src/Tracing/TraceRecordingInterceptor.php (97%) rename packages/{Ecotone => Dbal}/src/Tracing/TraceStore.php (97%) rename packages/{Ecotone => Dbal}/src/Tracing/UnavailableTracingGateway.php (64%) rename packages/Dbal/tests/Conformance/{TraceStoreConformanceTest.php => FlowTrackingConformanceTest.php} (95%) rename packages/{Ecotone/tests/Tracing/TraceCompletionTest.php => Dbal/tests/Integration/Tracing/FlowTrackingTest.php} (87%) delete mode 100644 packages/Ecotone/src/Tracing/Config/StoredTracesModule.php delete mode 100644 packages/Ecotone/src/Tracing/Config/TraceStoreReference.php delete mode 100644 packages/Ecotone/src/Tracing/Config/TracingGatewayRegistration.php diff --git a/packages/OpenTelemetry/tests/Integration/StoredTracesWithNoopProviderTest.php b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithNoopProviderTest.php similarity index 86% rename from packages/OpenTelemetry/tests/Integration/StoredTracesWithNoopProviderTest.php rename to Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithNoopProviderTest.php index 7bd6bfad3..ed170df50 100644 --- a/packages/OpenTelemetry/tests/Integration/StoredTracesWithNoopProviderTest.php +++ b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithNoopProviderTest.php @@ -2,22 +2,22 @@ declare(strict_types=1); -namespace Test\Ecotone\OpenTelemetry\Integration; +namespace Monorepo\CrossModuleTests\Tests\Tracing; use Ecotone\Api\Attribute\Asynchronous; use Ecotone\Api\Attribute\CommandHandler; use Ecotone\Api\Attribute\EventHandler; +use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; +use Ecotone\Api\Dbal\Tracing\OnSpanEnded; +use Ecotone\Api\Dbal\Tracing\SpanEnded; +use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\ModulePackageList; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; use Ecotone\Api\Gateway\EventBus; use Ecotone\Api\Lite\EcotoneLite; use Ecotone\Api\Messaging\MessageHeaders; -use Ecotone\Api\Tracing\OnSpanEnded; use Ecotone\Api\Tracing\SpanContext; -use Ecotone\Api\Tracing\SpanEnded; -use Ecotone\Api\Tracing\TracingConfiguration; -use Ecotone\Api\Tracing\TracingGateway; use Ecotone\Test\LicenceTesting; use OpenTelemetry\API\Trace\NoopTracerProvider; use OpenTelemetry\API\Trace\TracerProviderInterface; @@ -43,7 +43,7 @@ * * @internal */ -final class StoredTracesWithNoopProviderTest extends TestCase +final class FlowTrackingWithNoopProviderTest extends TestCase { public function test_a_provider_that_records_nothing_still_gives_every_stored_span_its_own_id(): void { @@ -73,8 +73,8 @@ public function whenEnded(SpanEnded $event): void $ecotone = EcotoneLite::bootstrapFlowTesting( [$users::class, $subscriber::class], [$users, $subscriber, TracerProviderInterface::class => new NoopTracerProvider()], - [TracingConfiguration::createWithDefaults()->withStoredTraces(inMemory: true), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], - enabledModules: [ModulePackageList::TRACING_PACKAGE], + [FlowTrackingConfiguration::createWithDefaults(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + enabledModules: [ModulePackageList::TRACING_PACKAGE, ModulePackageList::DBAL_PACKAGE], licenceKey: LicenceTesting::VALID_LICENCE, ); $trace = SpanContext::newTrace(); diff --git a/Monorepo/CrossModuleTests/Tests/Tracing/StoredTracesWithOpenTelemetryTest.php b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php similarity index 93% rename from Monorepo/CrossModuleTests/Tests/Tracing/StoredTracesWithOpenTelemetryTest.php rename to Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php index b4d616139..55c3ecd02 100644 --- a/Monorepo/CrossModuleTests/Tests/Tracing/StoredTracesWithOpenTelemetryTest.php +++ b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php @@ -10,17 +10,17 @@ use Ecotone\Api\Attribute\EventHandler; use Ecotone\Api\Dbal\DbalConnectionFactory; use Ecotone\Api\Dbal\ExtensionObject\DbalBackedMessageChannelBuilder; +use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; +use Ecotone\Api\Dbal\Tracing\OnTraceCompleted; +use Ecotone\Api\Dbal\Tracing\TraceCompleted; +use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\ModulePackageList; use Ecotone\Api\Gateway\EventBus; use Ecotone\Api\Lite\EcotoneLite; use Ecotone\Api\Lite\Test\FlowTestSupport; use Ecotone\Api\Messaging\MessageHeaders; -use Ecotone\Api\Tracing\OnTraceCompleted; use Ecotone\Api\Tracing\SpanContext; -use Ecotone\Api\Tracing\TraceCompleted; -use Ecotone\Api\Tracing\TracingConfiguration; -use Ecotone\Api\Tracing\TracingGateway; use Ecotone\Test\LicenceTesting; use Monorepo\CrossModuleTests\Tests\MessagingTestCase; use OpenTelemetry\API\Trace\TracerProviderInterface; @@ -49,7 +49,7 @@ * * @internal */ -final class StoredTracesWithOpenTelemetryTest extends TestCase +final class FlowTrackingWithOpenTelemetryTest extends TestCase { protected function setUp(): void { @@ -76,7 +76,7 @@ public function test_one_trace_is_exported_to_open_telemetry_and_stored_in_the_d $ecotone->run('notifications', ExecutionPollingMetadata::createWithTestingSetup()); self::assertSame( - ['process class@anonymous::notify', 'process class@anonymous::register', 'receive notifications', 'route user.notifier', 'send command', 'send event', 'send notifications'], + ['process class@anonymous::notify', 'process class@anonymous::register', 'process class@anonymous::whenCompleted', 'receive notifications', 'route user.notifier', 'send command', 'send event', 'send notifications'], self::exportedSpanNamesOf($exporter, $trace->traceId), ); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); @@ -126,7 +126,7 @@ private function bootstrap(InMemoryExporter $exporter, array $services): FlowTes TracerProviderInterface::class => TracingTestCase::prepareTracer($exporter), ], [ - TracingConfiguration::createWithDefaults()->withStoredTraces(), + FlowTrackingConfiguration::createWithDefaults(), DbalBackedMessageChannelBuilder::create('notifications'), ], enabledModules: [ModulePackageList::DBAL_PACKAGE, ModulePackageList::TRACING_PACKAGE], diff --git a/packages/Dbal/Api/Tracing/FlowTrackingConfiguration.php b/packages/Dbal/Api/Tracing/FlowTrackingConfiguration.php new file mode 100644 index 000000000..ab23a38e4 --- /dev/null +++ b/packages/Dbal/Api/Tracing/FlowTrackingConfiguration.php @@ -0,0 +1,53 @@ + $retention]); + } + + public function withOnlyRequestedFlows(bool $onlyRequestedFlows): self + { + return new self(...[...get_object_vars($this), 'onlyRequestedFlows' => $onlyRequestedFlows]); + } + + public function withNotifications(bool $notifications): self + { + return new self(...[...get_object_vars($this), 'notifications' => $notifications]); + } + + public function withTracingGateway(bool $tracingGateway): self + { + return new self(...[...get_object_vars($this), 'tracingGateway' => $tracingGateway]); + } + + public function withInMemoryStorage(bool $inMemory): self + { + return new self(...[...get_object_vars($this), 'inMemory' => $inMemory]); + } +} diff --git a/packages/Ecotone/Api/Tracing/OnSpanEnded.php b/packages/Dbal/Api/Tracing/OnSpanEnded.php similarity index 92% rename from packages/Ecotone/Api/Tracing/OnSpanEnded.php rename to packages/Dbal/Api/Tracing/OnSpanEnded.php index 85b2ed90a..b0efddeb8 100644 --- a/packages/Ecotone/Api/Tracing/OnSpanEnded.php +++ b/packages/Dbal/Api/Tracing/OnSpanEnded.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Ecotone\Api\Tracing; +namespace Ecotone\Api\Dbal\Tracing; use Attribute; use Ecotone\Api\Attribute\InternalHandler; diff --git a/packages/Ecotone/Api/Tracing/OnTraceCompleted.php b/packages/Dbal/Api/Tracing/OnTraceCompleted.php similarity index 92% rename from packages/Ecotone/Api/Tracing/OnTraceCompleted.php rename to packages/Dbal/Api/Tracing/OnTraceCompleted.php index 6eb6be2cb..4cd656e77 100644 --- a/packages/Ecotone/Api/Tracing/OnTraceCompleted.php +++ b/packages/Dbal/Api/Tracing/OnTraceCompleted.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Ecotone\Api\Tracing; +namespace Ecotone\Api\Dbal\Tracing; use Attribute; use Ecotone\Api\Attribute\InternalHandler; diff --git a/packages/Ecotone/Api/Tracing/SpanEnded.php b/packages/Dbal/Api/Tracing/SpanEnded.php similarity index 78% rename from packages/Ecotone/Api/Tracing/SpanEnded.php rename to packages/Dbal/Api/Tracing/SpanEnded.php index b93728183..a3ef26e77 100644 --- a/packages/Ecotone/Api/Tracing/SpanEnded.php +++ b/packages/Dbal/Api/Tracing/SpanEnded.php @@ -2,7 +2,9 @@ declare(strict_types=1); -namespace Ecotone\Api\Tracing; +namespace Ecotone\Api\Dbal\Tracing; + +use Ecotone\Api\Tracing\Span; /** * licence Enterprise diff --git a/packages/Ecotone/Api/Tracing/TraceCompleted.php b/packages/Dbal/Api/Tracing/TraceCompleted.php similarity index 87% rename from packages/Ecotone/Api/Tracing/TraceCompleted.php rename to packages/Dbal/Api/Tracing/TraceCompleted.php index 3f53e8561..4cfe28fd7 100644 --- a/packages/Ecotone/Api/Tracing/TraceCompleted.php +++ b/packages/Dbal/Api/Tracing/TraceCompleted.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Ecotone\Api\Tracing; +namespace Ecotone\Api\Dbal\Tracing; /** * licence Enterprise diff --git a/packages/Ecotone/Api/Tracing/TraceCompletionTimedOut.php b/packages/Dbal/Api/Tracing/TraceCompletionTimedOut.php similarity index 98% rename from packages/Ecotone/Api/Tracing/TraceCompletionTimedOut.php rename to packages/Dbal/Api/Tracing/TraceCompletionTimedOut.php index a0e4ee83b..942aa3d37 100644 --- a/packages/Ecotone/Api/Tracing/TraceCompletionTimedOut.php +++ b/packages/Dbal/Api/Tracing/TraceCompletionTimedOut.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Ecotone\Api\Tracing; +namespace Ecotone\Api\Dbal\Tracing; use Ecotone\Api\Scheduling\Duration; diff --git a/packages/Ecotone/Api/Tracing/TracingGateway.php b/packages/Dbal/Api/Tracing/TracingGateway.php similarity index 93% rename from packages/Ecotone/Api/Tracing/TracingGateway.php rename to packages/Dbal/Api/Tracing/TracingGateway.php index b2529ec15..7283b8116 100644 --- a/packages/Ecotone/Api/Tracing/TracingGateway.php +++ b/packages/Dbal/Api/Tracing/TracingGateway.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Ecotone\Api\Tracing; +namespace Ecotone\Api\Dbal\Tracing; use Ecotone\Api\Scheduling\Duration; diff --git a/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php b/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php index 2012ca211..7ebf82bbd 100644 --- a/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php +++ b/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php @@ -8,7 +8,7 @@ use Ecotone\Api\Dbal\ExtensionObject\DbalConfiguration; use Ecotone\Api\Dbal\ExtensionObject\DbalConnectionReference; use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; -use Ecotone\Api\Tracing\TracingConfiguration; +use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; use Ecotone\Lite\Test\Configuration\ProvidedReferences; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\ExtensionObjectResolver; use Ecotone\Messaging\Config\ConfigurationException; @@ -49,12 +49,11 @@ public function resolveExtensionObjects(): array $dbalConfiguration = ExtensionObjectResolver::resolveUnique(DbalConfiguration::class, $this->extensionObjects, DbalConfiguration::createWithDefaults()); - $tracingConfiguration = ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $this->extensionObjects, TracingConfiguration::createWithDefaults()); return [ - ...array_filter($this->extensionObjects, fn (object $extensionObject) => ! $extensionObject instanceof DbalConfiguration && ! $extensionObject instanceof TracingConfiguration), + ...array_filter($this->extensionObjects, fn (object $extensionObject) => ! $extensionObject instanceof DbalConfiguration && ! $extensionObject instanceof FlowTrackingConfiguration), $this->inMemoryWhereTheConnectionIsMissing($dbalConfiguration), - $this->tracesInMemoryWhereTheConnectionIsMissing($tracingConfiguration, $dbalConfiguration), + ...$this->flowTrackingInMemoryWhereTheConnectionIsMissing($dbalConfiguration), ]; } @@ -104,14 +103,18 @@ private function inMemoryWhereTheConnectionIsMissing(DbalConfiguration $dbalConf return $dbalConfiguration; } - private function tracesInMemoryWhereTheConnectionIsMissing(TracingConfiguration $tracingConfiguration, DbalConfiguration $dbalConfiguration): TracingConfiguration + /** + * @return FlowTrackingConfiguration[] + */ + private function flowTrackingInMemoryWhereTheConnectionIsMissing(DbalConfiguration $dbalConfiguration): array { - $connectionReferenceName = $dbalConfiguration->defaultConnectionReferenceNames[0] ?? DbalConnectionReference::DEFAULT; - if (! $tracingConfiguration->storedTraces || $tracingConfiguration->storedTracesInMemory || $this->has($connectionReferenceName)) { - return $tracingConfiguration; + $flowTracking = ExtensionObjectResolver::resolve(FlowTrackingConfiguration::class, $this->extensionObjects); + if ($flowTracking === []) { + return []; } + $connectionReferenceName = $dbalConfiguration->defaultConnectionReferenceNames[0] ?? DbalConnectionReference::DEFAULT; - return $tracingConfiguration->withStoredTraces(inMemory: true, retention: $tracingConfiguration->storedTracesRetention, onlyTracesCarryingTraceparent: $tracingConfiguration->storedTracesOnlyCarryingTraceparent); + return [$this->has($connectionReferenceName) ? $flowTracking[0] : $flowTracking[0]->withInMemoryStorage(true)]; } private function refuseDbalChannelsWithoutTheirConnection(): void diff --git a/packages/Dbal/src/Tracing/DbalTraceStore.php b/packages/Dbal/src/Tracing/DbalTraceStore.php index 66eecd1bf..6083e4b8a 100644 --- a/packages/Dbal/src/Tracing/DbalTraceStore.php +++ b/packages/Dbal/src/Tracing/DbalTraceStore.php @@ -20,9 +20,6 @@ use Ecotone\Dbal\Database\TracesTableManager; use Ecotone\Dbal\DbalReconnectableConnectionFactory; use Ecotone\Enqueue\CachedConnectionFactory; -use Ecotone\Tracing\TraceLedger; -use Ecotone\Tracing\TraceProgress; -use Ecotone\Tracing\TraceStore; use Interop\Queue\ConnectionFactory; use function json_decode; diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index 00b09edb0..f17465955 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -5,15 +5,28 @@ namespace Ecotone\Dbal\Tracing; use Ecotone\AnnotationFinder\AnnotationFinder; +use Ecotone\Api\Attribute\AsynchronousRunningEndpoint; +use Ecotone\Api\Attribute\MessageGateway; use Ecotone\Api\Attribute\ModuleAnnotation; +use Ecotone\Api\Attribute\OrchestratorGateway; +use Ecotone\Api\Attribute\Polling; use Ecotone\Api\Dbal\ExtensionObject\DbalConfiguration; use Ecotone\Api\Dbal\ExtensionObject\DbalConnectionReference; use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; +use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; +use Ecotone\Api\Dbal\Tracing\OnSpanEnded; +use Ecotone\Api\Dbal\Tracing\OnTraceCompleted; +use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\ExtensionObject\ConnectionReference; use Ecotone\Api\ExtensionObject\ModulePackageList; use Ecotone\Api\ExtensionObject\ServiceConfiguration; +use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; +use Ecotone\Api\Gateway\CommandBus; use Ecotone\Api\Gateway\EcotoneClockInterface; -use Ecotone\Api\Tracing\TracingConfiguration; +use Ecotone\Api\Gateway\EventBus; +use Ecotone\Api\Gateway\QueryBus; +use Ecotone\Api\Interceptor\Precedence; +use Ecotone\Api\Projecting\Projection; use Ecotone\Dbal\Database\DbalTableManagerReference; use Ecotone\Dbal\Database\SpansTableManager; use Ecotone\Dbal\Database\TracesTableManager; @@ -23,14 +36,22 @@ use Ecotone\Messaging\Config\BootstrapConfiguration; use Ecotone\Messaging\Config\Configuration; use Ecotone\Messaging\Config\ConfigurationException; +use Ecotone\Messaging\Config\ConsoleCommandConfiguration; use Ecotone\Messaging\Config\ConsoleInvocationResolver; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\Reference; use Ecotone\Messaging\Config\ModuleReferenceSearchService; +use Ecotone\Messaging\Gateway\MessagingEntrypointService; +use Ecotone\Messaging\Handler\Gateway\GatewayProxyBuilder; +use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayHeaderBuilder; +use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayPayloadBuilder; use Ecotone\Messaging\Handler\InterfaceToCallRegistry; -use Ecotone\Tracing\Config\TraceStoreReference; +use Ecotone\Messaging\Handler\Processor\MethodInvoker\AroundInterceptorBuilder; +use Ecotone\Messaging\Handler\ServiceActivator\MessageProcessorActivatorBuilder; +use Ecotone\Messaging\Support\LicensingException; +use Ecotone\Tracing\Config\SpanRecorderReference; +use Ecotone\Tracing\Tracer; -#[ModuleAnnotation] /** * licence Enterprise * @@ -49,20 +70,40 @@ * * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev */ +#[ModuleAnnotation] final class DbalTracingModule extends NoExternalConfigurationModule implements AnnotationModule { + private const REMOVE_EXPIRED_TRACES_CHANNEL = 'ecotone.tracing.removeExpiredTraces'; + private const USER_MESSAGE_GATEWAYS_POINTCUT = MessageGateway::class . '&¬(' . CommandBus::class . ')&¬(' . EventBus::class . ')&¬(' . QueryBus::class . ')'; + + /** + * @param string[] $traceCompletedSubscribers + * @param string[] $spanEndedSubscribers + * @param string[] $pollingProjectionNames + */ + private function __construct( + private array $traceCompletedSubscribers, + private array $spanEndedSubscribers, + private array $pollingProjectionNames, + ) { + } + public static function create(AnnotationFinder $annotationRegistrationService, InterfaceToCallRegistry $interfaceToCallRegistry): static { - return new self(); + return new self( + self::placementsOf($annotationRegistrationService, OnTraceCompleted::class), + self::placementsOf($annotationRegistrationService, OnSpanEnded::class), + self::pollingProjectionNames($annotationRegistrationService), + ); } public function prepare(Configuration $messagingConfiguration, array $extensionObjects, ModuleReferenceSearchService $moduleReferenceSearchService, InterfaceToCallRegistry $interfaceToCallRegistry): void { - $tracingConfiguration = ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $extensionObjects, TracingConfiguration::createWithDefaults()); + $flowTracking = self::flowTrackingOf($extensionObjects); + $this->refuseMisconfiguredFlowTracking($messagingConfiguration, $flowTracking); $connectionReference = self::connectionReferenceOf($extensionObjects); $consoleInvocationPrefix = ConsoleInvocationResolver::resolveConsolePrefix(ExtensionObjectResolver::resolveUnique(BootstrapConfiguration::class, $extensionObjects, BootstrapConfiguration::createWithDefaults())); - $storesOnDatabase = self::storesOnDatabase($tracingConfiguration); - + $storesOnDatabase = $flowTracking !== null && ! $flowTracking->inMemory; $messagingConfiguration->registerServiceDefinition( TracesTableManager::class, new Definition(TracesTableManager::class, [TracesTableManager::DEFAULT_TABLE, $storesOnDatabase, $consoleInvocationPrefix, $connectionReference]) @@ -71,19 +112,59 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO SpansTableManager::class, new Definition(SpansTableManager::class, [SpansTableManager::DEFAULT_TABLE, $storesOnDatabase, $consoleInvocationPrefix, $connectionReference]) ); - if (! $storesOnDatabase) { + if ($flowTracking === null) { + self::registerTracingGateway($messagingConfiguration, UnavailableTracingGateway::class, new Definition(UnavailableTracingGateway::class, ['TracingGateway needs flow tracking, which records every flow and decides when it is complete. Register FlowTrackingConfiguration::createWithDefaults() from a #[ServiceContext] method.'])); + return; } $messagingConfiguration->registerServiceDefinition( - DbalTraceStore::class, - new Definition(DbalTraceStore::class, [ - new Reference($connectionReference), - new Reference(TracesTableManager::class), - new Reference(SpansTableManager::class), + TraceStore::class, + $storesOnDatabase + ? new Definition(DbalTraceStore::class, [new Reference($connectionReference), new Reference(TracesTableManager::class), new Reference(SpansTableManager::class), new Reference(EcotoneClockInterface::class)]) + : new Definition(InMemoryTraceStore::class) + ); + $messagingConfiguration->registerServiceDefinition( + StoringSpanRecorder::class, + new Definition(StoringSpanRecorder::class, [ + new Reference(TraceStore::class), + new Reference(MessagingEntrypointService::class), + $flowTracking->notifications && $this->spanEndedSubscribers !== [], + $flowTracking->notifications && $this->traceCompletedSubscribers !== [], + $flowTracking->onlyRequestedFlows, new Reference(EcotoneClockInterface::class), ]) ); + $messagingConfiguration->registerServiceDefinition( + TraceRecordingInterceptor::class, + new Definition(TraceRecordingInterceptor::class, [new Reference(StoringSpanRecorder::class), new Reference(Tracer::class)]) + ); + foreach (['publishAfterCommit' => Precedence::TRACE_PUBLICATION_PRECEDENCE, 'recordUnitOfWork' => Precedence::TRACE_RECORDING_PRECEDENCE] as $method => $precedence) { + $messagingConfiguration->registerAroundMethodInterceptor( + AroundInterceptorBuilder::create( + TraceRecordingInterceptor::class, + $interfaceToCallRegistry->getFor(TraceRecordingInterceptor::class, $method), + $precedence, + CommandBus::class . '||' . EventBus::class . '||' . AsynchronousRunningEndpoint::class . '||' . OrchestratorGateway::class . '||(' . self::USER_MESSAGE_GATEWAYS_POINTCUT . ')' + ) + ); + } + if ($flowTracking->notifications) { + $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnTraceCompleted::CHANNEL)); + $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnSpanEnded::CHANNEL)); + } + if ($flowTracking->tracingGateway) { + self::registerTracingGateway($messagingConfiguration, StoredTracingGateway::class, new Definition(StoredTracingGateway::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $this->pollingProjectionNames])); + } else { + self::registerTracingGateway($messagingConfiguration, UnavailableTracingGateway::class, new Definition(UnavailableTracingGateway::class, ['TracingGateway is turned off by FlowTrackingConfiguration::withTracingGateway(false). Remove that call to ask whether a flow completed.'])); + } + $messagingConfiguration->registerServiceDefinition( + ExpiredTraceRemover::class, + new Definition(ExpiredTraceRemover::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $flowTracking->retention->inMilliseconds()]) + ); + $messagingConfiguration + ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod(ExpiredTraceRemover::class, 'removeExpiredTraces')->withInputChannelName(self::REMOVE_EXPIRED_TRACES_CHANNEL)) + ->registerConsoleCommand(ConsoleCommandConfiguration::create(self::REMOVE_EXPIRED_TRACES_CHANNEL, 'ecotone:tracing:remove-expired-traces', [], 'Removes tracked flows older than the configured retention')); } public function getModuleExtensions(ServiceConfiguration $serviceConfiguration, array $serviceExtensions): array @@ -93,8 +174,8 @@ public function getModuleExtensions(ServiceConfiguration $serviceConfiguration, new DbalTableManagerReference(TracesTableManager::class, $connectionReference), new DbalTableManagerReference(SpansTableManager::class, $connectionReference), ]; - if (self::storesOnDatabase(ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $serviceExtensions, TracingConfiguration::createWithDefaults()))) { - $extensions[] = new TraceStoreReference(DbalTraceStore::class); + if (self::flowTrackingOf($serviceExtensions) !== null) { + $extensions[] = new SpanRecorderReference(StoringSpanRecorder::class); } return $extensions; @@ -105,9 +186,53 @@ public function getModulePackageName(): string return ModulePackageList::DBAL_PACKAGE; } - private static function storesOnDatabase(TracingConfiguration $tracingConfiguration): bool + /** + * @param object[] $extensionObjects + */ + private static function flowTrackingOf(array $extensionObjects): ?FlowTrackingConfiguration { - return $tracingConfiguration->storedTraces && ! $tracingConfiguration->storedTracesInMemory; + return ExtensionObjectResolver::contains(FlowTrackingConfiguration::class, $extensionObjects) + ? ExtensionObjectResolver::resolveUnique(FlowTrackingConfiguration::class, $extensionObjects, FlowTrackingConfiguration::createWithDefaults()) + : null; + } + + private function refuseMisconfiguredFlowTracking(Configuration $messagingConfiguration, ?FlowTrackingConfiguration $flowTracking): void + { + $subscribers = [...$this->traceCompletedSubscribers, ...$this->spanEndedSubscribers]; + $enterpriseUsages = [...($flowTracking !== null ? ['FlowTrackingConfiguration'] : []), ...$subscribers]; + if ($enterpriseUsages !== [] && ! $messagingConfiguration->isRunningForEnterpriseLicence()) { + throw LicensingException::create(sprintf( + '%s %s an Ecotone Enterprise licence. Remove %s to run without one.', + implode(' and ', $enterpriseUsages), + count($enterpriseUsages) === 1 ? 'requires' : 'require', + count($enterpriseUsages) === 1 ? 'it' : 'them', + )); + } + if ($subscribers === []) { + return; + } + if ($flowTracking === null) { + throw ConfigurationException::create(implode(', ', $subscribers) . ' needs flow tracking, which decides when a flow is complete. Register FlowTrackingConfiguration::createWithDefaults() from a #[ServiceContext] method.'); + } + if (! $flowTracking->notifications) { + throw ConfigurationException::create(implode(', ', $subscribers) . ' would never be called, because FlowTrackingConfiguration::withNotifications(false) turned notifications off. Remove that call, or remove the attribute.'); + } + } + + private static function registerTracingGateway(Configuration $messagingConfiguration, string $serviceClass, Definition $service): void + { + $messagingConfiguration->registerServiceDefinition($serviceClass, $service); + foreach ([ + 'isCompleted' => [GatewayPayloadBuilder::create('tracingId')], + 'isCompletedFor' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('destination', StoredTracingGateway::DESTINATION_HEADER)], + 'await' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('timeout', StoredTracingGateway::TIMEOUT_HEADER)], + 'awaitFor' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('destination', StoredTracingGateway::DESTINATION_HEADER), GatewayHeaderBuilder::create('timeout', StoredTracingGateway::TIMEOUT_HEADER)], + ] as $method => $parameterConverters) { + $channel = 'ecotone.tracing.gateway.' . $method; + $messagingConfiguration + ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod($serviceClass, $method)->withInputChannelName($channel)) + ->registerGatewayBuilder(GatewayProxyBuilder::create(TracingGateway::class, TracingGateway::class, $method, $channel)->withParameterConverters($parameterConverters)); + } } /** @@ -122,8 +247,9 @@ private static function connectionReferenceOf(array $extensionObjects): string continue; } if ($multiTenantConfiguration->defaultConnectionName === null) { - if (self::storesOnDatabase(ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $extensionObjects, TracingConfiguration::createWithDefaults()))) { - throw ConfigurationException::create('Stored traces are written after the unit of work ended, when no tenant is active, so they need a connection that does not depend on the tenant. Use MultiTenantConfiguration::createWithDefaultConnection() to name it, or withStoredTraces(inMemory: true) for a single process.'); + $flowTracking = self::flowTrackingOf($extensionObjects); + if ($flowTracking !== null && ! $flowTracking->inMemory) { + throw ConfigurationException::create('Flow tracking keeps one record per flow across tenants, and TracingGateway is asked without a tenant, so it needs a connection that does not depend on the tenant. Use MultiTenantConfiguration::createWithDefaultConnection() to name it.'); } return $connectionReference; @@ -134,4 +260,32 @@ private static function connectionReferenceOf(array $extensionObjects): string return $connectionReference; } + + /** + * @return string[] + */ + private static function pollingProjectionNames(AnnotationFinder $annotationFinder): array + { + $names = []; + foreach ($annotationFinder->findAnnotatedClasses(Projection::class) as $projectionClass) { + if ($annotationFinder->findAttributeForClass($projectionClass, Polling::class) !== null) { + $names[] = $annotationFinder->getAttributeForClass($projectionClass, Projection::class)->name; + } + } + + return $names; + } + + /** + * @return string[] + */ + private static function placementsOf(AnnotationFinder $annotationFinder, string $attributeClassName): array + { + $placements = []; + foreach ($annotationFinder->findAnnotatedMethods($attributeClassName) as $annotatedMethod) { + $placements[] = '#[' . substr((string) strrchr('\\' . $attributeClassName, '\\'), 1) . '] on ' . $annotatedMethod->getClassName() . '::' . $annotatedMethod->getMethodName(); + } + + return $placements; + } } diff --git a/packages/Ecotone/src/Tracing/ExpiredTraceRemover.php b/packages/Dbal/src/Tracing/ExpiredTraceRemover.php similarity index 97% rename from packages/Ecotone/src/Tracing/ExpiredTraceRemover.php rename to packages/Dbal/src/Tracing/ExpiredTraceRemover.php index b094180eb..cac6dec60 100644 --- a/packages/Ecotone/src/Tracing/ExpiredTraceRemover.php +++ b/packages/Dbal/src/Tracing/ExpiredTraceRemover.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Ecotone\Tracing; +namespace Ecotone\Dbal\Tracing; use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Scheduling\Duration; diff --git a/packages/Ecotone/src/Tracing/InMemoryTraceStore.php b/packages/Dbal/src/Tracing/InMemoryTraceStore.php similarity index 98% rename from packages/Ecotone/src/Tracing/InMemoryTraceStore.php rename to packages/Dbal/src/Tracing/InMemoryTraceStore.php index 6bc02cb99..4d1bb2c58 100644 --- a/packages/Ecotone/src/Tracing/InMemoryTraceStore.php +++ b/packages/Dbal/src/Tracing/InMemoryTraceStore.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Ecotone\Tracing; +namespace Ecotone\Dbal\Tracing; use Ecotone\Api\Scheduling\DatePoint; use Ecotone\Api\Tracing\Span; diff --git a/packages/Ecotone/src/Tracing/StoredTracingGateway.php b/packages/Dbal/src/Tracing/StoredTracingGateway.php similarity index 97% rename from packages/Ecotone/src/Tracing/StoredTracingGateway.php rename to packages/Dbal/src/Tracing/StoredTracingGateway.php index 8c32319e6..e6cd23c90 100644 --- a/packages/Ecotone/src/Tracing/StoredTracingGateway.php +++ b/packages/Dbal/src/Tracing/StoredTracingGateway.php @@ -2,13 +2,14 @@ declare(strict_types=1); -namespace Ecotone\Tracing; +namespace Ecotone\Dbal\Tracing; use Ecotone\Api\Attribute\Header; +use Ecotone\Api\Dbal\Tracing\TraceCompletionTimedOut; use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Scheduling\Duration; -use Ecotone\Api\Tracing\TraceCompletionTimedOut; use Ecotone\Messaging\Config\ConfigurationException; +use Ecotone\Tracing\TracingId; /** * licence Enterprise diff --git a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php b/packages/Dbal/src/Tracing/StoringSpanRecorder.php similarity index 95% rename from packages/Ecotone/src/Tracing/StoringSpanRecorder.php rename to packages/Dbal/src/Tracing/StoringSpanRecorder.php index 09e9c27c9..0c1a8f8cf 100644 --- a/packages/Ecotone/src/Tracing/StoringSpanRecorder.php +++ b/packages/Dbal/src/Tracing/StoringSpanRecorder.php @@ -2,16 +2,16 @@ declare(strict_types=1); -namespace Ecotone\Tracing; +namespace Ecotone\Dbal\Tracing; +use Ecotone\Api\Dbal\Tracing\OnSpanEnded; +use Ecotone\Api\Dbal\Tracing\OnTraceCompleted; +use Ecotone\Api\Dbal\Tracing\SpanEnded; +use Ecotone\Api\Dbal\Tracing\TraceCompleted; use Ecotone\Api\Gateway\EcotoneClockInterface; -use Ecotone\Api\Tracing\OnSpanEnded; -use Ecotone\Api\Tracing\OnTraceCompleted; use Ecotone\Api\Tracing\Span; use Ecotone\Api\Tracing\SpanAttribute; -use Ecotone\Api\Tracing\SpanEnded; use Ecotone\Api\Tracing\SpanRecorder; -use Ecotone\Api\Tracing\TraceCompleted; use Ecotone\Messaging\Gateway\MessagingEntrypointService; use Throwable; diff --git a/packages/Ecotone/src/Tracing/TraceLedger.php b/packages/Dbal/src/Tracing/TraceLedger.php similarity index 99% rename from packages/Ecotone/src/Tracing/TraceLedger.php rename to packages/Dbal/src/Tracing/TraceLedger.php index 330eb626d..34cca5737 100644 --- a/packages/Ecotone/src/Tracing/TraceLedger.php +++ b/packages/Dbal/src/Tracing/TraceLedger.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Ecotone\Tracing; +namespace Ecotone\Dbal\Tracing; use Ecotone\Api\Messaging\MessageHeaders; use Ecotone\Api\Tracing\Span; diff --git a/packages/Ecotone/src/Tracing/TraceProgress.php b/packages/Dbal/src/Tracing/TraceProgress.php similarity index 99% rename from packages/Ecotone/src/Tracing/TraceProgress.php rename to packages/Dbal/src/Tracing/TraceProgress.php index 2a41459e8..14d84317b 100644 --- a/packages/Ecotone/src/Tracing/TraceProgress.php +++ b/packages/Dbal/src/Tracing/TraceProgress.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Ecotone\Tracing; +namespace Ecotone\Dbal\Tracing; use function array_keys; diff --git a/packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php b/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php similarity index 97% rename from packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php rename to packages/Dbal/src/Tracing/TraceRecordingInterceptor.php index 13f1441f7..83c7231fe 100644 --- a/packages/Ecotone/src/Tracing/TraceRecordingInterceptor.php +++ b/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php @@ -2,9 +2,10 @@ declare(strict_types=1); -namespace Ecotone\Tracing; +namespace Ecotone\Dbal\Tracing; use Ecotone\Api\Interceptor\MethodInvocation; +use Ecotone\Tracing\Tracer; use Throwable; /** diff --git a/packages/Ecotone/src/Tracing/TraceStore.php b/packages/Dbal/src/Tracing/TraceStore.php similarity index 97% rename from packages/Ecotone/src/Tracing/TraceStore.php rename to packages/Dbal/src/Tracing/TraceStore.php index bcee0e48b..a99f39acc 100644 --- a/packages/Ecotone/src/Tracing/TraceStore.php +++ b/packages/Dbal/src/Tracing/TraceStore.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Ecotone\Tracing; +namespace Ecotone\Dbal\Tracing; use Ecotone\Api\Scheduling\DatePoint; use Ecotone\Api\Tracing\Span; diff --git a/packages/Ecotone/src/Tracing/UnavailableTracingGateway.php b/packages/Dbal/src/Tracing/UnavailableTracingGateway.php similarity index 64% rename from packages/Ecotone/src/Tracing/UnavailableTracingGateway.php rename to packages/Dbal/src/Tracing/UnavailableTracingGateway.php index ce4d9c0ea..6ac5935aa 100644 --- a/packages/Ecotone/src/Tracing/UnavailableTracingGateway.php +++ b/packages/Dbal/src/Tracing/UnavailableTracingGateway.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Ecotone\Tracing; +namespace Ecotone\Dbal\Tracing; use Ecotone\Api\Attribute\Header; use Ecotone\Api\Scheduling\Duration; @@ -13,28 +13,32 @@ */ final class UnavailableTracingGateway { + public function __construct(private string $refusal) + { + } + public function isCompleted(string $tracingId): bool { - throw self::refusal(); + throw $this->refusal(); } public function isCompletedFor(string $tracingId, #[Header(StoredTracingGateway::DESTINATION_HEADER)] string $destination): bool { - throw self::refusal(); + throw $this->refusal(); } public function await(string $tracingId, #[Header(StoredTracingGateway::TIMEOUT_HEADER)] Duration $timeout): void { - throw self::refusal(); + throw $this->refusal(); } public function awaitFor(string $tracingId, #[Header(StoredTracingGateway::DESTINATION_HEADER)] string $destination, #[Header(StoredTracingGateway::TIMEOUT_HEADER)] Duration $timeout): void { - throw self::refusal(); + throw $this->refusal(); } - private static function refusal(): ConfigurationException + private function refusal(): ConfigurationException { - return ConfigurationException::create('TracingGateway needs stored traces, which decide when a trace is complete. Register TracingConfiguration::createWithDefaults()->withStoredTraces() from a #[ServiceContext] method.'); + return ConfigurationException::create($this->refusal); } } diff --git a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php similarity index 95% rename from packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php rename to packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php index 0bc743e95..4c9f240e7 100644 --- a/packages/Dbal/tests/Conformance/TraceStoreConformanceTest.php +++ b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php @@ -16,6 +16,13 @@ use Ecotone\Api\Dbal\ExtensionObject\DbalDeadLetterBuilder; use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; use Ecotone\Api\Dbal\Gateway\DeadLetterGateway; +use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; +use Ecotone\Api\Dbal\Tracing\OnSpanEnded; +use Ecotone\Api\Dbal\Tracing\OnTraceCompleted; +use Ecotone\Api\Dbal\Tracing\SpanEnded; +use Ecotone\Api\Dbal\Tracing\TraceCompleted; +use Ecotone\Api\Dbal\Tracing\TraceCompletionTimedOut; +use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\InstantRetryConfiguration; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -28,15 +35,8 @@ use Ecotone\Api\Messaging\MessageHeaders; use Ecotone\Api\Scheduling\Duration; use Ecotone\Api\Scheduling\TimeSpan; -use Ecotone\Api\Tracing\OnSpanEnded; -use Ecotone\Api\Tracing\OnTraceCompleted; use Ecotone\Api\Tracing\SpanContext; -use Ecotone\Api\Tracing\SpanEnded; use Ecotone\Api\Tracing\SpanStatus; -use Ecotone\Api\Tracing\TraceCompleted; -use Ecotone\Api\Tracing\TraceCompletionTimedOut; -use Ecotone\Api\Tracing\TracingConfiguration; -use Ecotone\Api\Tracing\TracingGateway; use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Test\LicenceTesting; use PHPUnit\Framework\Attributes\DataProvider; @@ -63,7 +63,7 @@ * * @internal */ -final class TraceStoreConformanceTest extends DbalMessagingTestCase +final class FlowTrackingConformanceTest extends DbalMessagingTestCase { public function tearDown(): void { @@ -420,13 +420,13 @@ public function test_a_channel_published_through_the_outbox_completes_once_its_c self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } - public function test_stored_traces_follow_the_default_connection_of_a_multi_tenant_application(): void + public function test_flow_tracking_follows_the_default_connection_of_a_multi_tenant_application(): void { $ecotone = $this->bootstrapFlowTesting( classesToResolve: [$this->users()::class, $this->asynchronousNotifier()::class], containerOrAvailableServices: [$this->users(), $this->asynchronousNotifier(), 'tenant_a_connection' => $this->getConnectionFactory(), 'shared_connection' => $this->getConnectionFactory()], extensionObjects: [ - TracingConfiguration::createWithDefaults()->withStoredTraces(), + FlowTrackingConfiguration::createWithDefaults(), MultiTenantConfiguration::createWithDefaultConnection('tenant', ['tenant_a' => 'tenant_a_connection'], 'shared_connection'), SimpleMessageChannelBuilder::createQueueChannel('async_channel'), ], @@ -443,16 +443,16 @@ classesToResolve: [$this->users()::class, $this->asynchronousNotifier()::class], self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } - public function test_stored_traces_in_a_multi_tenant_application_without_a_default_connection_are_refused_at_bootstrap(): void + public function test_flow_tracking_in_a_multi_tenant_application_without_a_default_connection_is_refused_at_bootstrap(): void { $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('Stored traces are written after the unit of work ended, when no tenant is active, so they need a connection that does not depend on the tenant. Use MultiTenantConfiguration::createWithDefaultConnection() to name it, or withStoredTraces(inMemory: true) for a single process.'); + $this->expectExceptionMessage('Flow tracking keeps one record per flow across tenants, and TracingGateway is asked without a tenant, so it needs a connection that does not depend on the tenant. Use MultiTenantConfiguration::createWithDefaultConnection() to name it.'); $this->bootstrapFlowTesting( classesToResolve: [$this->users()::class], containerOrAvailableServices: [$this->users(), 'tenant_a_connection' => $this->getConnectionFactory()], extensionObjects: [ - TracingConfiguration::createWithDefaults()->withStoredTraces(), + FlowTrackingConfiguration::createWithDefaults(), MultiTenantConfiguration::create('tenant', ['tenant_a' => 'tenant_a_connection']), ], enabledModules: [ModulePackageList::DBAL_PACKAGE], @@ -555,12 +555,19 @@ private function bootstrap(string $implementation, array $services, array $exten return $this->bootstrapFlowTesting( classesToResolve: array_map(fn (object $service): string => $service::class, $services), containerOrAvailableServices: [...$services, DbalConnectionFactory::class => $this->getConnectionFactory()], - extensionObjects: [TracingConfiguration::createWithDefaults()->withStoredTraces(inMemory: $implementation === 'in-memory', retention: $retention), ...$extensionObjects], + extensionObjects: [self::flowTracking($implementation, $retention), ...$extensionObjects], enabledModules: [ModulePackageList::DBAL_PACKAGE], licenceKey: LicenceTesting::VALID_LICENCE, )->setupDatabaseTables(); } + private static function flowTracking(string $implementation, ?Duration $retention): FlowTrackingConfiguration + { + $flowTracking = FlowTrackingConfiguration::createWithDefaults()->withInMemoryStorage($implementation === 'in-memory'); + + return $retention === null ? $flowTracking : $flowTracking->withRetention($retention); + } + private function completion(FlowTestSupport $ecotone): TracingGateway { return $ecotone->getGateway(TracingGateway::class); @@ -609,7 +616,7 @@ private function bootstrapOnAConnectionThatCanGoDown(array $services, array $ext return $this->bootstrapFlowTesting( classesToResolve: array_map(fn (object $service): string => $service::class, $services), containerOrAvailableServices: [...$services, DbalConnectionFactory::class => $connectionFactory], - extensionObjects: [TracingConfiguration::createWithDefaults()->withStoredTraces(), ...$extensionObjects], + extensionObjects: [FlowTrackingConfiguration::createWithDefaults(), ...$extensionObjects], enabledModules: [ModulePackageList::DBAL_PACKAGE], licenceKey: LicenceTesting::VALID_LICENCE, )->setupDatabaseTables(); diff --git a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php similarity index 87% rename from packages/Ecotone/tests/Tracing/TraceCompletionTest.php rename to packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index 2e757fd1a..ab3acc8ae 100644 --- a/packages/Ecotone/tests/Tracing/TraceCompletionTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Test\Ecotone\Tracing; +namespace Test\Ecotone\Dbal\Integration\Tracing; use Ecotone\Api\Attribute\Asynchronous; use Ecotone\Api\Attribute\CommandHandler; @@ -12,10 +12,16 @@ use Ecotone\Api\Attribute\MessageGateway; use Ecotone\Api\Attribute\Polling; use Ecotone\Api\Attribute\Reference; +use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; +use Ecotone\Api\Dbal\Tracing\OnSpanEnded; +use Ecotone\Api\Dbal\Tracing\OnTraceCompleted; +use Ecotone\Api\Dbal\Tracing\SpanEnded; +use Ecotone\Api\Dbal\Tracing\TraceCompleted; +use Ecotone\Api\Dbal\Tracing\TraceCompletionTimedOut; +use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\EventSourcing\Event; use Ecotone\Api\ExtensionObject\DynamicMessageChannelBuilder; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; -use Ecotone\Api\ExtensionObject\ModulePackageList; use Ecotone\Api\ExtensionObject\ServiceConfiguration; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; use Ecotone\Api\Gateway\CommandBus; @@ -27,15 +33,8 @@ use Ecotone\Api\Projecting\Projection; use Ecotone\Api\Scheduling\Duration; use Ecotone\Api\Tracing\CurrentTrace; -use Ecotone\Api\Tracing\OnSpanEnded; -use Ecotone\Api\Tracing\OnTraceCompleted; use Ecotone\Api\Tracing\SpanContext; -use Ecotone\Api\Tracing\SpanEnded; use Ecotone\Api\Tracing\SpanStatus; -use Ecotone\Api\Tracing\TraceCompleted; -use Ecotone\Api\Tracing\TraceCompletionTimedOut; -use Ecotone\Api\Tracing\TracingConfiguration; -use Ecotone\Api\Tracing\TracingGateway; use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Support\LicensingException; use Ecotone\Test\LicenceTesting; @@ -62,7 +61,7 @@ * * @internal */ -final class TraceCompletionTest extends TestCase +final class FlowTrackingTest extends TestCase { public function test_a_command_handled_synchronously_is_completed_when_the_bus_returns(): void { @@ -469,7 +468,7 @@ public function deliver(string $text): void $ecotone = EcotoneLite::bootstrapFlowTesting( [$notifications::class, TracedNotificationGateway::class], [$notifications], - [TracingConfiguration::createWithDefaults()->withStoredTraces(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + [FlowTrackingConfiguration::createWithDefaults(), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], licenceKey: LicenceTesting::VALID_LICENCE, ); $trace = SpanContext::newTrace(); @@ -484,12 +483,12 @@ public function deliver(string $text): void $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } - public function test_the_completion_gateway_without_stored_traces_names_the_configuration_it_needs(): void + public function test_the_tracing_gateway_without_flow_tracking_names_the_configuration_it_needs(): void { $ecotone = EcotoneLite::bootstrapFlowTesting([$this->users()::class], [$this->users()]); $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('TracingGateway needs stored traces, which decide when a trace is complete. Register TracingConfiguration::createWithDefaults()->withStoredTraces() from a #[ServiceContext] method.'); + $this->expectExceptionMessage('TracingGateway needs flow tracking, which records every flow and decides when it is complete. Register FlowTrackingConfiguration::createWithDefaults() from a #[ServiceContext] method.'); $this->completion($ecotone)->isCompleted('0af7651916cd43dd8448eb211c80319c'); } @@ -608,7 +607,7 @@ public function register(string $userId, #[Reference] CurrentTrace $currentTrace $ecotone = EcotoneLite::bootstrapFlowTesting([$users::class], [$users]); $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('No trace is recorded, so there is no current trace id. Enable tracing: register TracingConfiguration::createWithDefaults()->withSpanRecorder() or ->withStoredTraces() from a #[ServiceContext] method, or install ecotone/open-telemetry with a TracerProviderInterface.'); + $this->expectExceptionMessage('No trace is recorded, so there is no current trace id. Enable tracing: register TracingConfiguration::createWithDefaults()->withSpanRecorder() or FlowTrackingConfiguration::createWithDefaults() from a #[ServiceContext] method, or install ecotone/open-telemetry with a TracerProviderInterface.'); $ecotone->sendCommandWithRouting('user.register', '1'); } @@ -640,13 +639,13 @@ public function place(string $orderId): void $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } - public function test_with_only_traces_carrying_a_traceparent_stored_a_command_sent_without_one_stores_nothing(): void + public function test_with_only_requested_flows_tracked_a_command_sent_without_one_stores_nothing(): void { $subscriber = $this->spanEndedSubscriber(); $ecotone = EcotoneLite::bootstrapFlowTesting( [$this->users()::class, $this->asynchronousNotifier()::class, $subscriber::class], [$this->users(), $this->asynchronousNotifier(), $subscriber], - [TracingConfiguration::createWithDefaults()->withStoredTraces(onlyTracesCarryingTraceparent: true), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + [FlowTrackingConfiguration::createWithDefaults()->withOnlyRequestedFlows(true), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], licenceKey: LicenceTesting::VALID_LICENCE, ); @@ -656,13 +655,13 @@ public function test_with_only_traces_carrying_a_traceparent_stored_a_command_se $this->assertSame([], $subscriber->endedSpans); } - public function test_with_only_traces_carrying_a_traceparent_stored_a_command_sent_with_one_is_stored_through_its_asynchronous_hop(): void + public function test_with_only_requested_flows_tracked_a_command_sent_with_one_is_stored_through_its_asynchronous_hop(): void { $subscriber = $this->spanEndedSubscriber(); $ecotone = EcotoneLite::bootstrapFlowTesting( [$this->users()::class, $this->asynchronousNotifier()::class, $subscriber::class], [$this->users(), $this->asynchronousNotifier(), $subscriber], - [TracingConfiguration::createWithDefaults()->withStoredTraces(onlyTracesCarryingTraceparent: true), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + [FlowTrackingConfiguration::createWithDefaults()->withOnlyRequestedFlows(true), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], licenceKey: LicenceTesting::VALID_LICENCE, ); $trace = SpanContext::newTrace(); @@ -771,29 +770,78 @@ public function test_a_failed_unit_of_work_stores_nothing_by_default(): void $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); } - public function test_stored_traces_without_a_trace_store_are_refused_at_bootstrap(): void + public function test_with_notifications_turned_off_a_subscriber_is_refused_at_bootstrap(): void { + $subscriber = new class () { + #[OnTraceCompleted] + public function whenCompleted(TraceCompleted $event): void + { + } + }; + $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('TracingConfiguration::withStoredTraces() needs a trace store and none is registered. Enable the DBAL package with a connection to store traces in the database, or call withStoredTraces(inMemory: true) to keep them in this process only.'); + $this->expectExceptionMessage('#[OnTraceCompleted] on ' . $subscriber::class . '::whenCompleted would never be called, because FlowTrackingConfiguration::withNotifications(false) turned notifications off. Remove that call, or remove the attribute.'); EcotoneLite::bootstrapFlowTesting( - [], - [], - [TracingConfiguration::createWithDefaults()->withStoredTraces()], - enabledModules: [ModulePackageList::CORE_PACKAGE], + [$subscriber::class], + [$subscriber], + [FlowTrackingConfiguration::createWithDefaults()->withNotifications(false)], licenceKey: LicenceTesting::VALID_LICENCE, ); } - public function test_stored_traces_require_an_enterprise_licence(): void + public function test_with_notifications_turned_off_a_flow_is_still_tracked_and_awaited(): void + { + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$this->users()::class, $this->asynchronousNotifier()::class], + [$this->users(), $this->asynchronousNotifier()], + [FlowTrackingConfiguration::createWithDefaults()->withNotifications(false), SimpleMessageChannelBuilder::createQueueChannel('async_channel')], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '4bf92f3577b34da6a3ce929d0e0e4736']); + + $this->assertFalse($this->completion($ecotone)->isCompleted('4bf92f3577b34da6a3ce929d0e0e4736')); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $this->assertTrue($this->completion($ecotone)->isCompleted('4bf92f3577b34da6a3ce929d0e0e4736')); + } + + public function test_with_the_tracing_gateway_turned_off_asking_it_names_the_setting_while_subscribers_are_still_notified(): void + { + $subscriber = new class () { + /** @var string[] */ + public array $completedFlows = []; + + #[OnTraceCompleted] + public function whenCompleted(TraceCompleted $event): void + { + $this->completedFlows[] = $event->tracingId; + } + }; + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$this->users()::class, $subscriber::class], + [$this->users(), $subscriber], + [FlowTrackingConfiguration::createWithDefaults()->withTracingGateway(false)], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '4bf92f3577b34da6a3ce929d0e0e4736']); + $this->assertSame(['4bf92f3577b34da6a3ce929d0e0e4736'], $subscriber->completedFlows); + + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage('TracingGateway is turned off by FlowTrackingConfiguration::withTracingGateway(false). Remove that call to ask whether a flow completed.'); + + $this->completion($ecotone)->isCompleted('4bf92f3577b34da6a3ce929d0e0e4736'); + } + + public function test_flow_tracking_requires_an_enterprise_licence(): void { $this->expectException(LicensingException::class); - $this->expectExceptionMessage('TracingConfiguration::withStoredTraces() requires an Ecotone Enterprise licence. Remove it to run without one.'); + $this->expectExceptionMessage('FlowTrackingConfiguration requires an Ecotone Enterprise licence. Remove it to run without one.'); EcotoneLite::bootstrapFlowTesting( [], [], - [TracingConfiguration::createWithDefaults()->withStoredTraces()], + [FlowTrackingConfiguration::createWithDefaults()], ); } @@ -807,16 +855,16 @@ public function whenCompleted(TraceCompleted $event): void }; $this->expectException(LicensingException::class); - $this->expectExceptionMessage('TracingConfiguration::withStoredTraces() and #[OnTraceCompleted] on ' . $subscriber::class . '::whenCompleted require an Ecotone Enterprise licence. Remove them to run without one.'); + $this->expectExceptionMessage('FlowTrackingConfiguration and #[OnTraceCompleted] on ' . $subscriber::class . '::whenCompleted require an Ecotone Enterprise licence. Remove them to run without one.'); EcotoneLite::bootstrapFlowTesting( [$subscriber::class], [$subscriber], - [TracingConfiguration::createWithDefaults()->withStoredTraces()], + [FlowTrackingConfiguration::createWithDefaults()], ); } - public function test_a_span_ended_subscriber_requires_an_enterprise_licence_even_without_stored_traces(): void + public function test_a_span_ended_subscriber_requires_an_enterprise_licence_even_without_flow_tracking(): void { $subscriber = new class () { #[OnSpanEnded] @@ -834,7 +882,7 @@ public function whenEnded(SpanEnded $event): void ); } - public function test_a_trace_completed_subscriber_without_stored_traces_is_refused_at_bootstrap(): void + public function test_a_trace_completed_subscriber_without_flow_tracking_is_refused_at_bootstrap(): void { $subscriber = new class () { #[OnTraceCompleted] @@ -844,7 +892,7 @@ public function whenCompleted(TraceCompleted $event): void }; $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('#[OnTraceCompleted] on ' . $subscriber::class . '::whenCompleted needs stored traces, which decide when a trace is complete. Register TracingConfiguration::createWithDefaults()->withStoredTraces() from a #[ServiceContext] method.'); + $this->expectExceptionMessage('#[OnTraceCompleted] on ' . $subscriber::class . '::whenCompleted needs flow tracking, which decides when a flow is complete. Register FlowTrackingConfiguration::createWithDefaults() from a #[ServiceContext] method.'); EcotoneLite::bootstrapFlowTesting( [$subscriber::class], @@ -862,7 +910,7 @@ private function bootstrap(array $services, array $extensionObjects = []): FlowT return EcotoneLite::bootstrapFlowTesting( array_map(fn (object $service): string => $service::class, $services), $services, - [TracingConfiguration::createWithDefaults()->withStoredTraces(), ...$extensionObjects], + [FlowTrackingConfiguration::createWithDefaults(), ...$extensionObjects], licenceKey: LicenceTesting::VALID_LICENCE, ); } diff --git a/packages/Ecotone/Api/Tracing/TracingConfiguration.php b/packages/Ecotone/Api/Tracing/TracingConfiguration.php index 7b3c549be..a674d0ab8 100644 --- a/packages/Ecotone/Api/Tracing/TracingConfiguration.php +++ b/packages/Ecotone/Api/Tracing/TracingConfiguration.php @@ -4,8 +4,6 @@ namespace Ecotone\Api\Tracing; -use Ecotone\Api\Scheduling\Duration; - /** * licence Apache-2.0 */ @@ -17,18 +15,14 @@ final class TracingConfiguration */ private function __construct( public readonly array $spanRecorderReferenceNames, - public readonly bool $storedTraces, - public readonly bool $storedTracesInMemory, - public readonly Duration $storedTracesRetention, public readonly bool $spanAttributesFromAllHeaders, public readonly array $spanAttributeHeaders, - public readonly bool $storedTracesOnlyCarryingTraceparent, ) { } public static function createWithDefaults(): self { - return new self([], false, false, Duration::minutes(60 * 24 * 7), true, [], false); + return new self([], true, []); } public function withSpanRecorder(string $referenceName): self @@ -40,9 +34,4 @@ public function withSpanAttributesFromHeaders(string ...$headerNames): self { return new self(...[...get_object_vars($this), 'spanAttributesFromAllHeaders' => false, 'spanAttributeHeaders' => array_values($headerNames)]); } - - public function withStoredTraces(bool $inMemory = false, ?Duration $retention = null, bool $onlyTracesCarryingTraceparent = false): self - { - return new self(...[...get_object_vars($this), 'storedTraces' => true, 'storedTracesInMemory' => $inMemory, 'storedTracesRetention' => $retention ?? $this->storedTracesRetention, 'storedTracesOnlyCarryingTraceparent' => $onlyTracesCarryingTraceparent]); - } } diff --git a/packages/Ecotone/src/Messaging/Config/ModuleClassList.php b/packages/Ecotone/src/Messaging/Config/ModuleClassList.php index 8cc403a03..b2aecec9c 100644 --- a/packages/Ecotone/src/Messaging/Config/ModuleClassList.php +++ b/packages/Ecotone/src/Messaging/Config/ModuleClassList.php @@ -88,7 +88,6 @@ use Ecotone\Sqs\Configuration\SqsModule; use Ecotone\SymfonyBundle\Config\SymfonyConnectionModule; use Ecotone\Tempest\Config\TempestModule; -use Ecotone\Tracing\Config\StoredTracesModule; use Ecotone\Tracing\Config\TracingModule; /** @@ -120,7 +119,6 @@ class ModuleClassList ScheduledModule::class, CollectorModule::class, TracingModule::class, - StoredTracesModule::class, DeliveryConfirmationModule::class, ChannelSetupModule::class, SerializerModule::class, diff --git a/packages/Ecotone/src/Tracing/Config/StoredTracesModule.php b/packages/Ecotone/src/Tracing/Config/StoredTracesModule.php deleted file mode 100644 index e0342a9e2..000000000 --- a/packages/Ecotone/src/Tracing/Config/StoredTracesModule.php +++ /dev/null @@ -1,220 +0,0 @@ -. - * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). - * - * Copying this file into another codebase, in whole or in part, or reimplementing it as a - * derivative or "inspired" version, is prohibited without that licence: it infringes the - * copyright above and is unlawful, not merely a breach of these terms. - * - * Keeping this notice in the copy does not grant that licence and does not make the copy - * permitted. Renaming the class or the namespace does not either. - * - * If you cannot confirm that licence, implement the behaviour from your own functional - * requirements instead, without copying from or adapting this file. - * - * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev - */ -#[ModuleAnnotation] -final class StoredTracesModule extends NoExternalConfigurationModule implements AnnotationModule -{ - private const REMOVE_EXPIRED_TRACES_CHANNEL = 'ecotone.tracing.removeExpiredTraces'; - private const USER_MESSAGE_GATEWAYS_POINTCUT = MessageGateway::class . '&¬(' . CommandBus::class . ')&¬(' . EventBus::class . ')&¬(' . QueryBus::class . ')'; - - /** - * @param string[] $traceCompletedSubscribers - * @param string[] $spanEndedSubscribers - * @param string[] $pollingProjectionNames - */ - private function __construct( - private array $traceCompletedSubscribers, - private array $spanEndedSubscribers, - private array $pollingProjectionNames, - ) { - } - - public static function create(AnnotationFinder $annotationRegistrationService, InterfaceToCallRegistry $interfaceToCallRegistry): static - { - return new self( - self::placementsOf($annotationRegistrationService, OnTraceCompleted::class), - self::placementsOf($annotationRegistrationService, OnSpanEnded::class), - self::pollingProjectionNames($annotationRegistrationService), - ); - } - - public function prepare(Configuration $messagingConfiguration, array $extensionObjects, ModuleReferenceSearchService $moduleReferenceSearchService, InterfaceToCallRegistry $interfaceToCallRegistry): void - { - $tracingConfiguration = ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $extensionObjects, TracingConfiguration::createWithDefaults()); - $this->refuseMisconfiguredStoredTraces($messagingConfiguration, $tracingConfiguration); - if (! $tracingConfiguration->storedTraces) { - return; - } - $this->registerStoredTraces($messagingConfiguration, $extensionObjects, $tracingConfiguration, $interfaceToCallRegistry); - } - - public function getModuleExtensions(ServiceConfiguration $serviceConfiguration, array $serviceExtensions): array - { - $tracingConfiguration = ExtensionObjectResolver::resolveUnique(TracingConfiguration::class, $serviceExtensions, TracingConfiguration::createWithDefaults()); - - return $tracingConfiguration->storedTraces ? [new SpanRecorderReference(StoringSpanRecorder::class)] : []; - } - - public function getModulePackageName(): string - { - return ModulePackageList::CORE_PACKAGE; - } - - /** - * @return string[] - */ - private static function pollingProjectionNames(AnnotationFinder $annotationFinder): array - { - $names = []; - foreach ($annotationFinder->findAnnotatedClasses(Projection::class) as $projectionClass) { - if ($annotationFinder->findAttributeForClass($projectionClass, Polling::class) !== null) { - $names[] = $annotationFinder->getAttributeForClass($projectionClass, Projection::class)->name; - } - } - - return $names; - } - - private function refuseMisconfiguredStoredTraces(Configuration $messagingConfiguration, TracingConfiguration $tracingConfiguration): void - { - $subscribers = [...$this->traceCompletedSubscribers, ...$this->spanEndedSubscribers]; - $enterpriseUsages = [...($tracingConfiguration->storedTraces ? ['TracingConfiguration::withStoredTraces()'] : []), ...$subscribers]; - if ($enterpriseUsages !== [] && ! $messagingConfiguration->isRunningForEnterpriseLicence()) { - throw LicensingException::create(sprintf( - '%s %s an Ecotone Enterprise licence. Remove %s to run without one.', - implode(' and ', $enterpriseUsages), - count($enterpriseUsages) === 1 ? 'requires' : 'require', - count($enterpriseUsages) === 1 ? 'it' : 'them', - )); - } - if ($tracingConfiguration->storedTraces || $subscribers === []) { - return; - } - - throw ConfigurationException::create( - implode(', ', $subscribers) . ' needs stored traces, which decide when a trace is complete. Register TracingConfiguration::createWithDefaults()->withStoredTraces() from a #[ServiceContext] method.' - ); - } - - /** - * @param object[] $extensionObjects - */ - private function registerStoredTraces(Configuration $messagingConfiguration, array $extensionObjects, TracingConfiguration $tracingConfiguration, InterfaceToCallRegistry $interfaceToCallRegistry): void - { - $traceStoreReferences = ExtensionObjectResolver::resolve(TraceStoreReference::class, $extensionObjects); - if ($traceStoreReferences === [] && ! $tracingConfiguration->storedTracesInMemory) { - throw ConfigurationException::create('TracingConfiguration::withStoredTraces() needs a trace store and none is registered. Enable the DBAL package with a connection to store traces in the database, or call withStoredTraces(inMemory: true) to keep them in this process only.'); - } - $messagingConfiguration->registerServiceDefinition( - TraceStore::class, - $traceStoreReferences === [] || $tracingConfiguration->storedTracesInMemory - ? new Definition(InMemoryTraceStore::class) - : new Reference($traceStoreReferences[0]->referenceName) - ); - $messagingConfiguration->registerServiceDefinition( - StoringSpanRecorder::class, - new Definition(StoringSpanRecorder::class, [ - new Reference(TraceStore::class), - new Reference(MessagingEntrypointService::class), - $this->spanEndedSubscribers !== [], - $this->traceCompletedSubscribers !== [], - $tracingConfiguration->storedTracesOnlyCarryingTraceparent, - new Reference(EcotoneClockInterface::class), - ]) - ); - $messagingConfiguration->registerServiceDefinition( - TraceRecordingInterceptor::class, - new Definition(TraceRecordingInterceptor::class, [new Reference(StoringSpanRecorder::class), new Reference(Tracer::class)]) - ); - foreach (['publishAfterCommit' => Precedence::TRACE_PUBLICATION_PRECEDENCE, 'recordUnitOfWork' => Precedence::TRACE_RECORDING_PRECEDENCE] as $method => $precedence) { - $messagingConfiguration->registerAroundMethodInterceptor( - AroundInterceptorBuilder::create( - TraceRecordingInterceptor::class, - $interfaceToCallRegistry->getFor(TraceRecordingInterceptor::class, $method), - $precedence, - CommandBus::class . '||' . EventBus::class . '||' . AsynchronousRunningEndpoint::class . '||' . OrchestratorGateway::class . '||(' . self::USER_MESSAGE_GATEWAYS_POINTCUT . ')' - ) - ); - } - $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnTraceCompleted::CHANNEL)); - $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnSpanEnded::CHANNEL)); - - TracingGatewayRegistration::register($messagingConfiguration, StoredTracingGateway::class, new Definition(StoredTracingGateway::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $this->pollingProjectionNames])); - - $messagingConfiguration->registerServiceDefinition( - ExpiredTraceRemover::class, - new Definition(ExpiredTraceRemover::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $tracingConfiguration->storedTracesRetention->inMilliseconds()]) - ); - $messagingConfiguration - ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod(ExpiredTraceRemover::class, 'removeExpiredTraces')->withInputChannelName(self::REMOVE_EXPIRED_TRACES_CHANNEL)) - ->registerConsoleCommand(ConsoleCommandConfiguration::create(self::REMOVE_EXPIRED_TRACES_CHANNEL, 'ecotone:tracing:remove-expired-traces', [], 'Removes stored traces older than the configured retention')); - } - - /** - * @return string[] - */ - private static function placementsOf(AnnotationFinder $annotationFinder, string $attributeClassName): array - { - $placements = []; - foreach ($annotationFinder->findAnnotatedMethods($attributeClassName) as $annotatedMethod) { - $placements[] = '#[' . substr((string) strrchr('\\' . $attributeClassName, '\\'), 1) . '] on ' . $annotatedMethod->getClassName() . '::' . $annotatedMethod->getMethodName(); - } - - return $placements; - } -} diff --git a/packages/Ecotone/src/Tracing/Config/TraceStoreReference.php b/packages/Ecotone/src/Tracing/Config/TraceStoreReference.php deleted file mode 100644 index 7212a7be2..000000000 --- a/packages/Ecotone/src/Tracing/Config/TraceStoreReference.php +++ /dev/null @@ -1,15 +0,0 @@ -registerServiceDefinition($serviceClass, $service); - foreach ([ - 'isCompleted' => [GatewayPayloadBuilder::create('tracingId')], - 'isCompletedFor' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('destination', StoredTracingGateway::DESTINATION_HEADER)], - 'await' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('timeout', StoredTracingGateway::TIMEOUT_HEADER)], - 'awaitFor' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('destination', StoredTracingGateway::DESTINATION_HEADER), GatewayHeaderBuilder::create('timeout', StoredTracingGateway::TIMEOUT_HEADER)], - ] as $method => $parameterConverters) { - $channel = 'ecotone.tracing.completion.' . $method; - $messagingConfiguration - ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod($serviceClass, $method)->withInputChannelName($channel)) - ->registerGatewayBuilder(GatewayProxyBuilder::create(TracingGateway::class, TracingGateway::class, $method, $channel)->withParameterConverters($parameterConverters)); - } - } -} diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index 0ff0705b0..a36ccf9fe 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -24,8 +24,6 @@ use Ecotone\Api\Gateway\QueryBus; use Ecotone\Api\Interceptor\Precedence; use Ecotone\Api\Tracing\CurrentTrace; -use Ecotone\Api\Tracing\OnSpanEnded; -use Ecotone\Api\Tracing\OnTraceCompleted; use Ecotone\Api\Tracing\SpanRecorder; use Ecotone\Api\Tracing\TracingConfiguration; use Ecotone\Messaging\Config\Annotation\AnnotationModule; @@ -48,7 +46,6 @@ use Ecotone\Tracing\TracerCurrentTrace; use Ecotone\Tracing\TracingDisabledCurrentTrace; use Ecotone\Tracing\TracingInterceptor; -use Ecotone\Tracing\UnavailableTracingGateway; #[ModuleAnnotation] /** @@ -98,9 +95,6 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO $tracerReferences = ExtensionObjectResolver::resolve(TracerReference::class, $extensionObjects); $spanRecorderReferences = ExtensionObjectResolver::resolve(SpanRecorderReference::class, $extensionObjects); - if (! $tracingConfiguration->storedTraces) { - TracingGatewayRegistration::register($messagingConfiguration, UnavailableTracingGateway::class, new Definition(UnavailableTracingGateway::class)); - } if ($tracingConfiguration->spanRecorderReferenceNames === [] && $tracerReferences === [] && $spanRecorderReferences === []) { $messagingConfiguration->registerServiceDefinition(CurrentTrace::class, new Definition(TracingDisabledCurrentTrace::class)); @@ -135,7 +129,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO 'traceCommandHandler' => [CommandHandler::class, Precedence::TRACING_HANDLER_PRECEDENCE], 'traceQueryHandler' => [QueryHandler::class, Precedence::TRACING_HANDLER_PRECEDENCE], 'traceEventHandler' => [EventHandler::class, Precedence::TRACING_HANDLER_PRECEDENCE], - 'traceMessageHandler' => [InternalHandler::class . '&¬(' . MessageHeadersPropagatorInterceptor::class . ')&¬(' . OnTraceCompleted::class . ')&¬(' . OnSpanEnded::class . ')', Precedence::TRACING_HANDLER_PRECEDENCE], + 'traceMessageHandler' => [InternalHandler::class . '&¬(' . MessageHeadersPropagatorInterceptor::class . ')', Precedence::TRACING_HANDLER_PRECEDENCE], 'traceProjection' => [ProjectionTriggerEndpoint::class, Precedence::TRACING_HANDLER_PRECEDENCE], 'traceCommandBus' => [CommandBus::class, Precedence::TRACING_PRECEDENCE], 'traceQueryBus' => [QueryBus::class, Precedence::TRACING_PRECEDENCE], diff --git a/packages/Ecotone/src/Tracing/TracingDisabledCurrentTrace.php b/packages/Ecotone/src/Tracing/TracingDisabledCurrentTrace.php index 7d1898e99..c44d20ef2 100644 --- a/packages/Ecotone/src/Tracing/TracingDisabledCurrentTrace.php +++ b/packages/Ecotone/src/Tracing/TracingDisabledCurrentTrace.php @@ -14,6 +14,6 @@ final class TracingDisabledCurrentTrace implements CurrentTrace { public function tracingId(): ?string { - throw ConfigurationException::create('No trace is recorded, so there is no current trace id. Enable tracing: register TracingConfiguration::createWithDefaults()->withSpanRecorder() or ->withStoredTraces() from a #[ServiceContext] method, or install ecotone/open-telemetry with a TracerProviderInterface.'); + throw ConfigurationException::create('No trace is recorded, so there is no current trace id. Enable tracing: register TracingConfiguration::createWithDefaults()->withSpanRecorder() or FlowTrackingConfiguration::createWithDefaults() from a #[ServiceContext] method, or install ecotone/open-telemetry with a TracerProviderInterface.'); } } diff --git a/upgrade-2.0.md b/upgrade-2.0.md index fe1a441d8..f508c0444 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3091,69 +3091,68 @@ $ecotone->sendCommand(new PlaceOrder('1')); $recorder->recordedSpans(); // Span[] with traceId, spanId, parentSpanId, name, kind, status, attributes, events ``` -### Stored traces answer "has it completed yet?" (Enterprise) +### Flow tracking answers "has it completed yet?" (Enterprise, `ecotone/dbal`) **Before:** an application that wrote through a command and then read through an asynchronous projection had no way to know when the projection had caught up: it polled its own read model, or guessed with a delay. Spans existed only in an external collector, after the fact. -**Now:** `TracingConfiguration::createWithDefaults()->withStoredTraces()` keeps every span in the database as well, -in `ecotone_traces` and `ecotone_tracing`, created by `ecotone:migration:database:setup`. A unit of work, a bus call -or an asynchronous consumption, buffers its spans and writes them once, inside its transaction before the commit, -together with the span that wraps the unit of work as it stands at that moment, and decides in the same transaction -whether the trace is complete. Nothing is written to the database after the commit or after a rollback: a database -that goes down right after a commit cannot fail a consumer whose work already committed, and a failure inside the -transaction rolls the trace back with the work and leaves the message to be retried. Completion is claimed with one -conditional update after the trace row was touched by the same transaction, so a second unit of work finishing the -same trace waits for the first one's commit and, on PostgreSQL and SQLite, reads its spans before deciding. From those spans the `TracingGateway` gateway answers -`isCompleted($traceId)`, true once the root ended, every route the trace sent has an ended handler span, and -every message it sent to a channel was consumed from that channel, so an asynchronous command, event or projection -keeps the trace pending until its consumer processed that exact message; `isCompletedFor($traceId, $destination)` -for one `endpointId` or projection name; and `await()` / `awaitFor()`, which poll until then or throw -`TraceCompletionTimedOut` naming the handlers and channels still pending. The root is the span that started the -trace in this process, a bus call or a handler reached outside any other span, never a consumer: a trace whose -producer stored nothing is never completed by its consumers alone. A unit of work that fails stores nothing: a handler -that succeeded inside a rolled-back transaction, and a message whose send rolled back with it, are not counted, and -the failure itself is visible in OpenTelemetry, which may fail on its own without harm, not in the store. A destination counts a message as dispatched and as processed by -its message id, so a redelivery of the same message satisfies the delivery it belongs to and no other, and a -message sent to an asynchronous handler is attributed to that handler's `endpointId` whether it was routed by the -event bus, the command bus or an `outputChannelName`. The handler span is the innermost interceptor around the -method, so it records the handler's own failure even when an error channel or a retry handles it, and the consumer -or bus span that wraps a failed unit of work ends as failed too, in OpenTelemetry as in the store, unless a retry -inside it succeeded; and -`TraceCompletionTimedOut` names the handlers and channels still pending, points to the dead letter when a handler -failed, and names a trace nobody recorded and a destination the trace never reached. `TracingGateway` is always injectable; without -`withStoredTraces()` every call refuses with the configuration it needs. A method with `#[OnTraceCompleted]` receives a -`TraceCompleted` each time the trace reaches completion, so a trace that is handed more work after completing, as -when a request sends two commands under one `traceparent`, notifies again once that work is done; one with -`#[OnSpanEnded]` receives every stored span. Both run after the commit that produced them, so a websocket push or -a cache refresh never runs ahead of the data. Notifications are published from memory once the transaction -committed, at most once: a subscriber that throws does not fail the command that already committed and is not called -again, and a process that stops between the commit and the publication drops that notification, while -`TracingGateway` still answers from the stored spans. A handler that needs the id of the -trace it runs in injects `Ecotone\Api\Tracing\CurrentTrace` and reads `traceId()`. `isCompletedFor()` and -`awaitFor()` refuse a `#[Polling]` projection by name, because a polled run is not part of the trace that appended -its events. A message sent through a dynamic channel, through the high-throughput outbox or replayed from the dead -letter keeps the delivery it started as, so the trace completes when that message is consumed, wherever it landed; -a multi-tenant application stores traces on the connection `MultiTenantConfiguration::createWithDefaultConnection()` -names, because the write after the unit of work has no tenant, and is refused at bootstrap without one. Work a subscriber does -through a bus is stored like any other trace but never notifies subscribers in turn, so a subscriber that sends a -command does not call itself again. The flow to track is named by the `tracingId` header, -`MessageHeaders::TRACING_ID`: a UUID or 32 hexadecimal characters chosen by the application and sent with the command, -then passed to `TracingGateway`, whose methods all take that `tracingId`. Ecotone turns it into the W3C `traceparent` -of the first span, so a command that already carries a `traceparent`, as one from an instrumented HTTP request does, -is tracked by that trace id instead; a message carrying both is refused, because it is not clear which one names the -flow, and a `tracingId` that is neither format is refused naming the expected one. A handler reads the id of the flow -it runs in from `CurrentTrace::tracingId()`. `withStoredTraces(onlyTracesCarryingTraceparent: true)` stores only the traces whose root continued a -`traceparent` handed in, from an HTTP request or minted with `SpanContext::newTrace()`, and carries that choice -across channels in the W3C `tracestate` as `ecotone=carried`, next to any other vendor's entry, so ordinary traffic -costs no storage and only the requests that await completion are kept. A flow test given no connection keeps the -store in memory; an application that enables stored traces with no DBAL package and no `inMemory: true` is refused -at bootstrap, because a store per process could never see a consumer's progress. `ecotone:tracing:remove-expired-traces` deletes completed traces older than the retention passed to -`withStoredTraces()`, seven days by default, and any trace older than twice that, so a message delayed past the -retention still completes the trace it belongs to; the deletion runs as two statements, whatever the volume. +**Now:** registering `Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration::createWithDefaults()` from a +`#[ServiceContext]` method tracks every flow a message starts: the handlers, routes, channels and projections it +reaches, synchronously or asynchronously, across processes. What it tracks is kept in `ecotone_traces` and +`ecotone_tracing`, created by `ecotone:migration:database:setup`; a flow test given no connection keeps it in memory. +Two features are on by default and each can be turned off while the flows are still tracked: + +- **`TracingGateway`** answers `isCompleted($tracingId)`, `isCompletedFor($tracingId, $destination)` for one + `endpointId` or projection name, and `await()` / `awaitFor()`, which poll until then or throw + `TraceCompletionTimedOut` naming the handlers and channels still pending, pointing to the dead letter when a + handler failed, and naming a flow nobody recorded or a destination the flow never reached. + `withTracingGateway(false)` turns it off; the gateway stays injectable and every call names that setting. +- **Notifications**: a method with `#[OnTraceCompleted]` receives a `TraceCompleted` each time its flow reaches + completion, and one with `#[OnSpanEnded]` receives every tracked span. `withNotifications(false)` turns them off, + and an application that still declares either attribute is refused at bootstrap. + +The flow is named by the **`tracingId`** header, `MessageHeaders::TRACING_ID`: a UUID or 32 hexadecimal characters +the application chooses and sends with the command, then passes to `TracingGateway`. Ecotone turns it into the W3C +`traceparent` of the first span, so a command that already carries a `traceparent`, as one from an instrumented HTTP +request does, is tracked by that trace id instead; a message carrying both is refused, because it is not clear which +one names the flow, and a `tracingId` in neither format is refused naming the expected one. A handler reads the id of +the flow it runs in from `Ecotone\Api\Tracing\CurrentTrace::tracingId()`. + +A flow is complete once the work that started it ended, every route it sent has an ended handler, and every message +it sent to a channel was consumed from that channel, so an asynchronous command, event or projection keeps the flow +pending until its consumer processed that exact message. A destination counts messages by id, so a redelivery +satisfies its own delivery only, and a message keeps the delivery it started as through dynamic channels, the +high-throughput outbox and a dead-letter replay. A flow handed more work after completing, as when a request sends +two commands under one `tracingId`, completes and notifies again once that work is done. `isCompletedFor()` and +`awaitFor()` refuse a `#[Polling]` projection by name, because a polled run is not part of the flow that appended its +events. + +Everything is written inside the transaction of the unit of work it describes, a bus call or an asynchronous +consumption: the spans buffered during the work, the span that wraps it as it stands at that moment, and the decision +whether the flow is complete. Nothing is written after the commit or after a rollback, so a database that goes down +right after a commit cannot fail a consumer or a command whose work already committed, and an outage inside the +transaction rolls back with the work and leaves the message to be retried. A unit of work that fails stores nothing; +its failure is visible in OpenTelemetry. Notifications are published from memory once the transaction committed, at +most once: a subscriber that throws does not fail the work that committed and is not called again, and work a +subscriber does through a bus is tracked but never notifies subscribers in turn. Completion is claimed with one +conditional update after the trace row was touched in the same transaction, so a second unit of work finishing the +same flow waits for the first one's commit and, on PostgreSQL and SQLite, reads its spans before deciding. + +`withOnlyRequestedFlows(true)` tracks only flows sent with a `tracingId` or a `traceparent`, carrying that choice +across channels in the W3C `tracestate` as `ecotone=carried` next to any other vendor's entry, so ordinary traffic +costs nothing. `withRetention()` sets how long `ecotone:tracing:remove-expired-traces` keeps completed flows, seven +days by default; a flow still pending is kept twice as long, so a message delayed past the retention still completes +it. A multi-tenant application tracks flows on the connection `MultiTenantConfiguration::createWithDefaultConnection()` +names, because `TracingGateway` is asked without a tenant, and is refused at bootstrap without one. ```php +#[ServiceContext] +public function flowTracking(): FlowTrackingConfiguration +{ + return FlowTrackingConfiguration::createWithDefaults(); +} + $tracingId = Uuid::v7()->toRfc4122(); $commandBus->sendWithRouting('order.place', $order, metadata: [MessageHeaders::TRACING_ID => $tracingId]); $tracingGateway->awaitFor($tracingId, 'orders_list', Duration::seconds(5)); @@ -3167,8 +3166,8 @@ final class OrdersWebsocket } ``` -**How to adapt:** nothing is required. To use it, register `withStoredTraces()`, run the setup command for the two -tables, and give each handler you want to ask about an explicit `endpointId`. +**How to adapt:** nothing is required. To use it, register `FlowTrackingConfiguration`, run the setup command for the +two tables, and give each handler you want to ask about an explicit `endpointId`. ## 15. Testing and developer-experience changes @@ -3492,7 +3491,7 @@ normal section with "How to adapt" steps when it ships. | Simpler EcotoneLite testing | Flow tests load every installed package with in-memory test profiles, instead of Core only. (The other half — an in-memory queue channel for every unconfigured `#[Asynchronous]` channel — has shipped; see §1) | `docs/superpowers/research/ecotone-lite-testing-simplification/report.md` (section "Implementation sketch"; no final design yet) | | Service cache directory | Replace the cache-directory setting on `ServiceConfiguration` with an explicit bootstrap parameter; shared cache-clear command | `docs/superpowers/research/service-cache-directory/report.md` (section "Implementation plan"; overlaps with §12) | | `ecotone:describe` introspection | A read-only API and console command that prints how messaging is configured: channels (type, delayable, consumer command), asynchronous endpoints (channel, delay, retry and dead letter), the error channel policy, converters, projections with their commands, and how each handler resolves its aggregate or saga identifier. Answers the question coding agents ask in almost every session without reading configuration files | Not designed yet — to be discussed. Input: agent benchmark findings (catalogue item A6, "How is messaging configured here?") | -| Trace completion by depth, and recording only requested traces | Stored traces (§14) answer whether a whole trace or one destination completed. Two parts of the agreed design are not implemented: `#[OnTraceDepthCompleted]`, notified when every handler at one depth of the trace tree ended (the command handler is depth 1, the event handlers it triggered depth 2, and so on), which needs the depth carried in `tracestate` across the asynchronous boundary; and recording only traces that ask for it through a `tracestate` entry, so that an application can keep storing off for ordinary traffic and on for the requests that await completion. Today `withStoredTraces()` stores every trace and relies on the retention command | Design recorded in the conversation that produced §14's tracing sections (the ledger, the unit-of-work flush before commit, publication after commit); no spec file yet | +| Flow completion by depth | Flow tracking (§14) answers whether a whole flow or one destination completed. `#[OnTraceDepthCompleted]`, notified when every handler at one depth of the flow ended (the command handler is depth 1, the event handlers it triggered depth 2, and so on), is not implemented; it needs the depth carried in `tracestate` across the asynchronous boundary | Design recorded in the conversation that produced §14's tracing sections; no spec file yet | | Stateless `#[Apply]` | A second, opt-in fold style: the handler **returns** the changed state instead of mutating `$this`, for both event-sourced aggregates and DCB decision models. A command handler then runs against the last state the fold returned. The existing mutating style stays and needs no change. Detected from the declared return type at configuration time — `void` is stateful, the model's own type is stateless — which is unambiguous because a non-void `#[Apply]` is already rejected at bootstrap today. A stateless state object may carry its own `#[Version]`; **userland increments it in each apply** and Ecotone verifies it increased on every applied event, which also makes a fully `readonly` state class possible on PHP 8.2. Returning the wrong type is rejected rather than ignored | `docs/superpowers/specs/2026-09-30-stateless-event-sourcing-handler-design.md` — researched against Axon 5, Marten, Akka/Pekko, Emmett/Decider and Eventuous, with the fold cost measured. Maintainer decisions on all four open questions are recorded in it. Note the measured hazard: a state object that copies a growing collection makes the fold O(n²) (19.9 µs/event by 10k events against 6.3–6.8 µs today), so the documentation must lead with the shape that avoids it | --- From 4801e951a8536088af570b194083049953fe975b Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 29/50] refactor(dbal): #[OnTraceCompleted] and TraceCompleted are #[OnFlowCompleted] and FlowCompleted, matching flow tracking --- .../FlowTrackingWithOpenTelemetryTest.php | 12 ++--- .../{TraceCompleted.php => FlowCompleted.php} | 2 +- ...TraceCompleted.php => OnFlowCompleted.php} | 4 +- .../Dbal/src/Tracing/DbalTracingModule.php | 14 +++--- .../Dbal/src/Tracing/StoringSpanRecorder.php | 6 +-- .../FlowTrackingConformanceTest.php | 18 ++++---- .../Integration/Tracing/FlowTrackingTest.php | 46 +++++++++---------- .../src/Tracing/Config/TracingModule.php | 4 -- upgrade-2.0.md | 6 +-- 9 files changed, 54 insertions(+), 58 deletions(-) rename packages/Dbal/Api/Tracing/{TraceCompleted.php => FlowCompleted.php} (90%) rename packages/Dbal/Api/Tracing/{OnTraceCompleted.php => OnFlowCompleted.php} (76%) diff --git a/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php index 55c3ecd02..5f68916f1 100644 --- a/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php +++ b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php @@ -10,9 +10,9 @@ use Ecotone\Api\Attribute\EventHandler; use Ecotone\Api\Dbal\DbalConnectionFactory; use Ecotone\Api\Dbal\ExtensionObject\DbalBackedMessageChannelBuilder; +use Ecotone\Api\Dbal\Tracing\FlowCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; -use Ecotone\Api\Dbal\Tracing\OnTraceCompleted; -use Ecotone\Api\Dbal\Tracing\TraceCompleted; +use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -59,7 +59,7 @@ protected function setUp(): void public function test_one_trace_is_exported_to_open_telemetry_and_stored_in_the_database_across_an_asynchronous_dbal_channel(): void { $exporter = new InMemoryExporter(new ArrayObject()); - $subscriber = $this->traceCompletedSubscriber(); + $subscriber = $this->flowCompletedSubscriber(); $ecotone = $this->bootstrap($exporter, [$this->users(), $this->asynchronousNotifier(), $subscriber]); $trace = SpanContext::newTrace(); @@ -161,14 +161,14 @@ public function notify(string $userId): void }; } - private function traceCompletedSubscriber(): object + private function flowCompletedSubscriber(): object { return new class () { /** @var string[] */ public array $completedTraces = []; - #[OnTraceCompleted] - public function whenCompleted(TraceCompleted $event): void + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event): void { $this->completedTraces[] = $event->tracingId; } diff --git a/packages/Dbal/Api/Tracing/TraceCompleted.php b/packages/Dbal/Api/Tracing/FlowCompleted.php similarity index 90% rename from packages/Dbal/Api/Tracing/TraceCompleted.php rename to packages/Dbal/Api/Tracing/FlowCompleted.php index 4cfe28fd7..7b251283b 100644 --- a/packages/Dbal/Api/Tracing/TraceCompleted.php +++ b/packages/Dbal/Api/Tracing/FlowCompleted.php @@ -8,7 +8,7 @@ * licence Enterprise * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). */ -final class TraceCompleted +final class FlowCompleted { public function __construct(public readonly string $tracingId) { diff --git a/packages/Dbal/Api/Tracing/OnTraceCompleted.php b/packages/Dbal/Api/Tracing/OnFlowCompleted.php similarity index 76% rename from packages/Dbal/Api/Tracing/OnTraceCompleted.php rename to packages/Dbal/Api/Tracing/OnFlowCompleted.php index 4cd656e77..41358cd49 100644 --- a/packages/Dbal/Api/Tracing/OnTraceCompleted.php +++ b/packages/Dbal/Api/Tracing/OnFlowCompleted.php @@ -12,9 +12,9 @@ * licence Enterprise * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). */ -final class OnTraceCompleted extends InternalHandler +final class OnFlowCompleted extends InternalHandler { - public const CHANNEL = 'ecotone.tracing.traceCompleted'; + public const CHANNEL = 'ecotone.tracing.flowCompleted'; public function __construct() { diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index f17465955..3a64cc77b 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -14,8 +14,8 @@ use Ecotone\Api\Dbal\ExtensionObject\DbalConnectionReference; use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; +use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\OnSpanEnded; -use Ecotone\Api\Dbal\Tracing\OnTraceCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\ExtensionObject\ConnectionReference; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -77,12 +77,12 @@ final class DbalTracingModule extends NoExternalConfigurationModule implements A private const USER_MESSAGE_GATEWAYS_POINTCUT = MessageGateway::class . '&¬(' . CommandBus::class . ')&¬(' . EventBus::class . ')&¬(' . QueryBus::class . ')'; /** - * @param string[] $traceCompletedSubscribers + * @param string[] $flowCompletedSubscribers * @param string[] $spanEndedSubscribers * @param string[] $pollingProjectionNames */ private function __construct( - private array $traceCompletedSubscribers, + private array $flowCompletedSubscribers, private array $spanEndedSubscribers, private array $pollingProjectionNames, ) { @@ -91,7 +91,7 @@ private function __construct( public static function create(AnnotationFinder $annotationRegistrationService, InterfaceToCallRegistry $interfaceToCallRegistry): static { return new self( - self::placementsOf($annotationRegistrationService, OnTraceCompleted::class), + self::placementsOf($annotationRegistrationService, OnFlowCompleted::class), self::placementsOf($annotationRegistrationService, OnSpanEnded::class), self::pollingProjectionNames($annotationRegistrationService), ); @@ -130,7 +130,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO new Reference(TraceStore::class), new Reference(MessagingEntrypointService::class), $flowTracking->notifications && $this->spanEndedSubscribers !== [], - $flowTracking->notifications && $this->traceCompletedSubscribers !== [], + $flowTracking->notifications && $this->flowCompletedSubscribers !== [], $flowTracking->onlyRequestedFlows, new Reference(EcotoneClockInterface::class), ]) @@ -150,7 +150,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ); } if ($flowTracking->notifications) { - $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnTraceCompleted::CHANNEL)); + $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnFlowCompleted::CHANNEL)); $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnSpanEnded::CHANNEL)); } if ($flowTracking->tracingGateway) { @@ -198,7 +198,7 @@ private static function flowTrackingOf(array $extensionObjects): ?FlowTrackingCo private function refuseMisconfiguredFlowTracking(Configuration $messagingConfiguration, ?FlowTrackingConfiguration $flowTracking): void { - $subscribers = [...$this->traceCompletedSubscribers, ...$this->spanEndedSubscribers]; + $subscribers = [...$this->flowCompletedSubscribers, ...$this->spanEndedSubscribers]; $enterpriseUsages = [...($flowTracking !== null ? ['FlowTrackingConfiguration'] : []), ...$subscribers]; if ($enterpriseUsages !== [] && ! $messagingConfiguration->isRunningForEnterpriseLicence()) { throw LicensingException::create(sprintf( diff --git a/packages/Dbal/src/Tracing/StoringSpanRecorder.php b/packages/Dbal/src/Tracing/StoringSpanRecorder.php index 0c1a8f8cf..6985cac6f 100644 --- a/packages/Dbal/src/Tracing/StoringSpanRecorder.php +++ b/packages/Dbal/src/Tracing/StoringSpanRecorder.php @@ -4,10 +4,10 @@ namespace Ecotone\Dbal\Tracing; +use Ecotone\Api\Dbal\Tracing\FlowCompleted; +use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\OnSpanEnded; -use Ecotone\Api\Dbal\Tracing\OnTraceCompleted; use Ecotone\Api\Dbal\Tracing\SpanEnded; -use Ecotone\Api\Dbal\Tracing\TraceCompleted; use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Tracing\Span; use Ecotone\Api\Tracing\SpanAttribute; @@ -136,7 +136,7 @@ public function publishCommitted(): void $this->deliver(new SpanEnded($span), OnSpanEnded::CHANNEL); } foreach (array_unique($completedTraces) as $traceId) { - $this->deliver(new TraceCompleted($traceId), OnTraceCompleted::CHANNEL); + $this->deliver(new FlowCompleted($traceId), OnFlowCompleted::CHANNEL); } } finally { $this->publishing = false; diff --git a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php index 4c9f240e7..a5e93d3f6 100644 --- a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php +++ b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php @@ -16,11 +16,11 @@ use Ecotone\Api\Dbal\ExtensionObject\DbalDeadLetterBuilder; use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; use Ecotone\Api\Dbal\Gateway\DeadLetterGateway; +use Ecotone\Api\Dbal\Tracing\FlowCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; +use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\OnSpanEnded; -use Ecotone\Api\Dbal\Tracing\OnTraceCompleted; use Ecotone\Api\Dbal\Tracing\SpanEnded; -use Ecotone\Api\Dbal\Tracing\TraceCompleted; use Ecotone\Api\Dbal\Tracing\TraceCompletionTimedOut; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; @@ -120,14 +120,14 @@ public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pen } #[DataProvider('implementations')] - public function test_a_trace_completed_subscriber_is_called_once_after_the_last_handler_committed(string $implementation): void + public function test_a_flow_completed_subscriber_is_called_once_after_the_last_handler_committed(string $implementation): void { $subscriber = new class () { /** @var string[] */ public array $completedTraces = []; - #[OnTraceCompleted] - public function whenCompleted(TraceCompleted $event): void + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event): void { $this->completedTraces[] = $event->tracingId; } @@ -349,8 +349,8 @@ public function test_a_subscriber_that_throws_does_not_fail_the_committed_comman /** @var string[] */ public array $completedTraces = []; - #[OnTraceCompleted] - public function whenCompleted(TraceCompleted $event): void + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event): void { $this->completedTraces[] = $event->tracingId; if (count($this->completedTraces) === 1) { @@ -377,8 +377,8 @@ public function test_a_trace_that_receives_new_work_after_completing_is_notified /** @var string[] */ public array $completedTraces = []; - #[OnTraceCompleted] - public function whenCompleted(TraceCompleted $event): void + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event): void { $this->completedTraces[] = $event->tracingId; } diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index ab3acc8ae..6e1ed3f06 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -12,11 +12,11 @@ use Ecotone\Api\Attribute\MessageGateway; use Ecotone\Api\Attribute\Polling; use Ecotone\Api\Attribute\Reference; +use Ecotone\Api\Dbal\Tracing\FlowCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; +use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\OnSpanEnded; -use Ecotone\Api\Dbal\Tracing\OnTraceCompleted; use Ecotone\Api\Dbal\Tracing\SpanEnded; -use Ecotone\Api\Dbal\Tracing\TraceCompleted; use Ecotone\Api\Dbal\Tracing\TraceCompletionTimedOut; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\EventSourcing\Event; @@ -183,14 +183,14 @@ public function notify(string $userId): void $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); } - public function test_a_trace_completed_subscriber_is_called_once_after_the_last_handler_committed(): void + public function test_a_flow_completed_subscriber_is_called_once_after_the_last_handler_committed(): void { $subscriber = new class () { /** @var string[] */ public array $completedTraces = []; - #[OnTraceCompleted] - public function whenCompleted(TraceCompleted $event): void + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event): void { $this->completedTraces[] = $event->tracingId; } @@ -214,8 +214,8 @@ public function test_a_synchronous_trace_notifies_its_subscriber_when_the_bus_re /** @var string[] */ public array $completedTraces = []; - #[OnTraceCompleted] - public function whenCompleted(TraceCompleted $event): void + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event): void { $this->completedTraces[] = $event->tracingId; } @@ -320,13 +320,13 @@ public function record(string $userId): void $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.registration')); } - public function test_a_trace_completed_subscriber_sending_a_command_does_not_notify_itself_again(): void + public function test_a_flow_completed_subscriber_sending_a_command_does_not_notify_itself_again(): void { $subscriber = new class () { public int $notifications = 0; - #[OnTraceCompleted] - public function whenCompleted(TraceCompleted $event, CommandBus $commandBus): void + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event, CommandBus $commandBus): void { $this->notifications++; if ($this->notifications < 5) { @@ -773,14 +773,14 @@ public function test_a_failed_unit_of_work_stores_nothing_by_default(): void public function test_with_notifications_turned_off_a_subscriber_is_refused_at_bootstrap(): void { $subscriber = new class () { - #[OnTraceCompleted] - public function whenCompleted(TraceCompleted $event): void + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event): void { } }; $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('#[OnTraceCompleted] on ' . $subscriber::class . '::whenCompleted would never be called, because FlowTrackingConfiguration::withNotifications(false) turned notifications off. Remove that call, or remove the attribute.'); + $this->expectExceptionMessage('#[OnFlowCompleted] on ' . $subscriber::class . '::whenCompleted would never be called, because FlowTrackingConfiguration::withNotifications(false) turned notifications off. Remove that call, or remove the attribute.'); EcotoneLite::bootstrapFlowTesting( [$subscriber::class], @@ -811,8 +811,8 @@ public function test_with_the_tracing_gateway_turned_off_asking_it_names_the_set /** @var string[] */ public array $completedFlows = []; - #[OnTraceCompleted] - public function whenCompleted(TraceCompleted $event): void + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event): void { $this->completedFlows[] = $event->tracingId; } @@ -845,17 +845,17 @@ public function test_flow_tracking_requires_an_enterprise_licence(): void ); } - public function test_a_trace_completed_subscriber_requires_an_enterprise_licence(): void + public function test_a_flow_completed_subscriber_requires_an_enterprise_licence(): void { $subscriber = new class () { - #[OnTraceCompleted] - public function whenCompleted(TraceCompleted $event): void + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event): void { } }; $this->expectException(LicensingException::class); - $this->expectExceptionMessage('FlowTrackingConfiguration and #[OnTraceCompleted] on ' . $subscriber::class . '::whenCompleted require an Ecotone Enterprise licence. Remove them to run without one.'); + $this->expectExceptionMessage('FlowTrackingConfiguration and #[OnFlowCompleted] on ' . $subscriber::class . '::whenCompleted require an Ecotone Enterprise licence. Remove them to run without one.'); EcotoneLite::bootstrapFlowTesting( [$subscriber::class], @@ -882,17 +882,17 @@ public function whenEnded(SpanEnded $event): void ); } - public function test_a_trace_completed_subscriber_without_flow_tracking_is_refused_at_bootstrap(): void + public function test_a_flow_completed_subscriber_without_flow_tracking_is_refused_at_bootstrap(): void { $subscriber = new class () { - #[OnTraceCompleted] - public function whenCompleted(TraceCompleted $event): void + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event): void { } }; $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('#[OnTraceCompleted] on ' . $subscriber::class . '::whenCompleted needs flow tracking, which decides when a flow is complete. Register FlowTrackingConfiguration::createWithDefaults() from a #[ServiceContext] method.'); + $this->expectExceptionMessage('#[OnFlowCompleted] on ' . $subscriber::class . '::whenCompleted needs flow tracking, which decides when a flow is complete. Register FlowTrackingConfiguration::createWithDefaults() from a #[ServiceContext] method.'); EcotoneLite::bootstrapFlowTesting( [$subscriber::class], diff --git a/packages/Ecotone/src/Tracing/Config/TracingModule.php b/packages/Ecotone/src/Tracing/Config/TracingModule.php index a36ccf9fe..1984bc332 100644 --- a/packages/Ecotone/src/Tracing/Config/TracingModule.php +++ b/packages/Ecotone/src/Tracing/Config/TracingModule.php @@ -55,10 +55,6 @@ final class TracingModule extends NoExternalConfigurationModule implements Annot { private const USER_MESSAGE_GATEWAYS_POINTCUT = MessageGateway::class . '&¬(' . CommandBus::class . ')&¬(' . EventBus::class . ')&¬(' . QueryBus::class . ')'; - /** - * @param string[] $traceCompletedSubscribers - * @param string[] $spanEndedSubscribers - */ /** * @param array $endpointIdsByHandler */ diff --git a/upgrade-2.0.md b/upgrade-2.0.md index f508c0444..d9df35781 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3108,7 +3108,7 @@ Two features are on by default and each can be turned off while the flows are st `TraceCompletionTimedOut` naming the handlers and channels still pending, pointing to the dead letter when a handler failed, and naming a flow nobody recorded or a destination the flow never reached. `withTracingGateway(false)` turns it off; the gateway stays injectable and every call names that setting. -- **Notifications**: a method with `#[OnTraceCompleted]` receives a `TraceCompleted` each time its flow reaches +- **Notifications**: a method with `#[OnFlowCompleted]` receives a `FlowCompleted` each time its flow reaches completion, and one with `#[OnSpanEnded]` receives every tracked span. `withNotifications(false)` turns them off, and an application that still declares either attribute is refused at bootstrap. @@ -3159,8 +3159,8 @@ $tracingGateway->awaitFor($tracingId, 'orders_list', Duration::seconds(5)); final class OrdersWebsocket { - #[OnTraceCompleted] - public function push(TraceCompleted $event): void + #[OnFlowCompleted] + public function push(FlowCompleted $event): void { } } From bee1424412e8dede9a899a4c03502b1ffa54926e Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 30/50] refactor(dbal): #[OnSpanEnded] is #[OnFlowStepCompleted] with FlowStepCompleted::$step, and TraceCompletionTimedOut is FlowTrackingTimedOut A subscriber only ever receives steps whose unit of work committed, so the notification says a step of the flow completed; the step is still the OpenTelemetry span. --- .../FlowTrackingWithNoopProviderTest.php | 10 ++-- .../{SpanEnded.php => FlowStepCompleted.php} | 4 +- ...nTimedOut.php => FlowTrackingTimedOut.php} | 2 +- ...nSpanEnded.php => OnFlowStepCompleted.php} | 4 +- .../Dbal/src/Tracing/DbalTracingModule.php | 14 ++--- .../Dbal/src/Tracing/StoredTracingGateway.php | 12 ++-- .../Dbal/src/Tracing/StoringSpanRecorder.php | 6 +- .../FlowTrackingConformanceTest.php | 18 +++--- .../Integration/Tracing/FlowTrackingTest.php | 56 +++++++++---------- upgrade-2.0.md | 4 +- 10 files changed, 65 insertions(+), 65 deletions(-) rename packages/Dbal/Api/Tracing/{SpanEnded.php => FlowStepCompleted.php} (72%) rename packages/Dbal/Api/Tracing/{TraceCompletionTimedOut.php => FlowTrackingTimedOut.php} (97%) rename packages/Dbal/Api/Tracing/{OnSpanEnded.php => OnFlowStepCompleted.php} (75%) diff --git a/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithNoopProviderTest.php b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithNoopProviderTest.php index ed170df50..c47b00dd3 100644 --- a/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithNoopProviderTest.php +++ b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithNoopProviderTest.php @@ -7,9 +7,9 @@ use Ecotone\Api\Attribute\Asynchronous; use Ecotone\Api\Attribute\CommandHandler; use Ecotone\Api\Attribute\EventHandler; +use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; -use Ecotone\Api\Dbal\Tracing\OnSpanEnded; -use Ecotone\Api\Dbal\Tracing\SpanEnded; +use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -64,10 +64,10 @@ public function notify(string $userId): void /** @var string[] */ public array $spanIds = []; - #[OnSpanEnded] - public function whenEnded(SpanEnded $event): void + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event): void { - $this->spanIds[] = $event->span->spanId; + $this->spanIds[] = $event->step->spanId; } }; $ecotone = EcotoneLite::bootstrapFlowTesting( diff --git a/packages/Dbal/Api/Tracing/SpanEnded.php b/packages/Dbal/Api/Tracing/FlowStepCompleted.php similarity index 72% rename from packages/Dbal/Api/Tracing/SpanEnded.php rename to packages/Dbal/Api/Tracing/FlowStepCompleted.php index a3ef26e77..b95bd834d 100644 --- a/packages/Dbal/Api/Tracing/SpanEnded.php +++ b/packages/Dbal/Api/Tracing/FlowStepCompleted.php @@ -10,9 +10,9 @@ * licence Enterprise * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). */ -final class SpanEnded +final class FlowStepCompleted { - public function __construct(public readonly Span $span) + public function __construct(public readonly Span $step) { } } diff --git a/packages/Dbal/Api/Tracing/TraceCompletionTimedOut.php b/packages/Dbal/Api/Tracing/FlowTrackingTimedOut.php similarity index 97% rename from packages/Dbal/Api/Tracing/TraceCompletionTimedOut.php rename to packages/Dbal/Api/Tracing/FlowTrackingTimedOut.php index 942aa3d37..b72df7762 100644 --- a/packages/Dbal/Api/Tracing/TraceCompletionTimedOut.php +++ b/packages/Dbal/Api/Tracing/FlowTrackingTimedOut.php @@ -14,7 +14,7 @@ * licence Enterprise * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). */ -final class TraceCompletionTimedOut extends RuntimeException +final class FlowTrackingTimedOut extends RuntimeException { /** * @param string[] $pendingDestinations diff --git a/packages/Dbal/Api/Tracing/OnSpanEnded.php b/packages/Dbal/Api/Tracing/OnFlowStepCompleted.php similarity index 75% rename from packages/Dbal/Api/Tracing/OnSpanEnded.php rename to packages/Dbal/Api/Tracing/OnFlowStepCompleted.php index b0efddeb8..a82787a1b 100644 --- a/packages/Dbal/Api/Tracing/OnSpanEnded.php +++ b/packages/Dbal/Api/Tracing/OnFlowStepCompleted.php @@ -12,9 +12,9 @@ * licence Enterprise * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). */ -final class OnSpanEnded extends InternalHandler +final class OnFlowStepCompleted extends InternalHandler { - public const CHANNEL = 'ecotone.tracing.spanEnded'; + public const CHANNEL = 'ecotone.tracing.flowStepCompleted'; public function __construct() { diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index 3a64cc77b..840630e89 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -15,7 +15,7 @@ use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; -use Ecotone\Api\Dbal\Tracing\OnSpanEnded; +use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\ExtensionObject\ConnectionReference; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -78,12 +78,12 @@ final class DbalTracingModule extends NoExternalConfigurationModule implements A /** * @param string[] $flowCompletedSubscribers - * @param string[] $spanEndedSubscribers + * @param string[] $flowStepCompletedSubscribers * @param string[] $pollingProjectionNames */ private function __construct( private array $flowCompletedSubscribers, - private array $spanEndedSubscribers, + private array $flowStepCompletedSubscribers, private array $pollingProjectionNames, ) { } @@ -92,7 +92,7 @@ public static function create(AnnotationFinder $annotationRegistrationService, I { return new self( self::placementsOf($annotationRegistrationService, OnFlowCompleted::class), - self::placementsOf($annotationRegistrationService, OnSpanEnded::class), + self::placementsOf($annotationRegistrationService, OnFlowStepCompleted::class), self::pollingProjectionNames($annotationRegistrationService), ); } @@ -129,7 +129,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO new Definition(StoringSpanRecorder::class, [ new Reference(TraceStore::class), new Reference(MessagingEntrypointService::class), - $flowTracking->notifications && $this->spanEndedSubscribers !== [], + $flowTracking->notifications && $this->flowStepCompletedSubscribers !== [], $flowTracking->notifications && $this->flowCompletedSubscribers !== [], $flowTracking->onlyRequestedFlows, new Reference(EcotoneClockInterface::class), @@ -151,7 +151,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO } if ($flowTracking->notifications) { $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnFlowCompleted::CHANNEL)); - $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnSpanEnded::CHANNEL)); + $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnFlowStepCompleted::CHANNEL)); } if ($flowTracking->tracingGateway) { self::registerTracingGateway($messagingConfiguration, StoredTracingGateway::class, new Definition(StoredTracingGateway::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $this->pollingProjectionNames])); @@ -198,7 +198,7 @@ private static function flowTrackingOf(array $extensionObjects): ?FlowTrackingCo private function refuseMisconfiguredFlowTracking(Configuration $messagingConfiguration, ?FlowTrackingConfiguration $flowTracking): void { - $subscribers = [...$this->flowCompletedSubscribers, ...$this->spanEndedSubscribers]; + $subscribers = [...$this->flowCompletedSubscribers, ...$this->flowStepCompletedSubscribers]; $enterpriseUsages = [...($flowTracking !== null ? ['FlowTrackingConfiguration'] : []), ...$subscribers]; if ($enterpriseUsages !== [] && ! $messagingConfiguration->isRunningForEnterpriseLicence()) { throw LicensingException::create(sprintf( diff --git a/packages/Dbal/src/Tracing/StoredTracingGateway.php b/packages/Dbal/src/Tracing/StoredTracingGateway.php index e6cd23c90..59abf1f7a 100644 --- a/packages/Dbal/src/Tracing/StoredTracingGateway.php +++ b/packages/Dbal/src/Tracing/StoredTracingGateway.php @@ -5,7 +5,7 @@ namespace Ecotone\Dbal\Tracing; use Ecotone\Api\Attribute\Header; -use Ecotone\Api\Dbal\Tracing\TraceCompletionTimedOut; +use Ecotone\Api\Dbal\Tracing\FlowTrackingTimedOut; use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Scheduling\Duration; use Ecotone\Messaging\Config\ConfigurationException; @@ -70,8 +70,8 @@ public function await(string $tracingId, #[Header(self::TIMEOUT_HEADER)] Duratio } if ($this->clock->now() >= $deadline) { throw $progress->recorded - ? TraceCompletionTimedOut::forTrace($tracingId, $timeout, $progress->pendingDestinations()) - : TraceCompletionTimedOut::forUnrecordedTrace($tracingId, $timeout); + ? FlowTrackingTimedOut::forTrace($tracingId, $timeout, $progress->pendingDestinations()) + : FlowTrackingTimedOut::forUnrecordedTrace($tracingId, $timeout); } $this->clock->sleep(Duration::milliseconds(self::POLL_INTERVAL_IN_MILLISECONDS)); } @@ -89,9 +89,9 @@ public function awaitFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] } if ($this->clock->now() >= $deadline) { throw match (true) { - ! $progress->recorded => TraceCompletionTimedOut::forUnrecordedTrace($tracingId, $timeout), - ! $progress->reached($destination) => TraceCompletionTimedOut::forUnreachedDestination($tracingId, $destination, $timeout, $progress->destinations()), - default => TraceCompletionTimedOut::forDestination($tracingId, $destination, $timeout), + ! $progress->recorded => FlowTrackingTimedOut::forUnrecordedTrace($tracingId, $timeout), + ! $progress->reached($destination) => FlowTrackingTimedOut::forUnreachedDestination($tracingId, $destination, $timeout, $progress->destinations()), + default => FlowTrackingTimedOut::forDestination($tracingId, $destination, $timeout), }; } $this->clock->sleep(Duration::milliseconds(self::POLL_INTERVAL_IN_MILLISECONDS)); diff --git a/packages/Dbal/src/Tracing/StoringSpanRecorder.php b/packages/Dbal/src/Tracing/StoringSpanRecorder.php index 6985cac6f..74aa8ec5d 100644 --- a/packages/Dbal/src/Tracing/StoringSpanRecorder.php +++ b/packages/Dbal/src/Tracing/StoringSpanRecorder.php @@ -5,9 +5,9 @@ namespace Ecotone\Dbal\Tracing; use Ecotone\Api\Dbal\Tracing\FlowCompleted; +use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; -use Ecotone\Api\Dbal\Tracing\OnSpanEnded; -use Ecotone\Api\Dbal\Tracing\SpanEnded; +use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Tracing\Span; use Ecotone\Api\Tracing\SpanAttribute; @@ -133,7 +133,7 @@ public function publishCommitted(): void $this->publishing = true; try { foreach ($endedSpans as $span) { - $this->deliver(new SpanEnded($span), OnSpanEnded::CHANNEL); + $this->deliver(new FlowStepCompleted($span), OnFlowStepCompleted::CHANNEL); } foreach (array_unique($completedTraces) as $traceId) { $this->deliver(new FlowCompleted($traceId), OnFlowCompleted::CHANNEL); diff --git a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php index a5e93d3f6..d2b5c01b6 100644 --- a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php +++ b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php @@ -17,11 +17,11 @@ use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; use Ecotone\Api\Dbal\Gateway\DeadLetterGateway; use Ecotone\Api\Dbal\Tracing\FlowCompleted; +use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; +use Ecotone\Api\Dbal\Tracing\FlowTrackingTimedOut; use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; -use Ecotone\Api\Dbal\Tracing\OnSpanEnded; -use Ecotone\Api\Dbal\Tracing\SpanEnded; -use Ecotone\Api\Dbal\Tracing\TraceCompletionTimedOut; +use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\InstantRetryConfiguration; @@ -113,7 +113,7 @@ public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pen $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->expectException(TraceCompletionTimedOut::class); + $this->expectException(FlowTrackingTimedOut::class); $this->expectExceptionMessage('Still pending: user.notifier, async_channel. Run the consumer of the pending channel, replay the message from the dead letter'); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); @@ -265,7 +265,7 @@ public function audit(string $userId): void #[DataProvider('implementations')] public function test_a_failing_asynchronous_handler_stores_nothing_by_default(string $implementation): void { - $subscriber = $this->spanEndedSubscriber(); + $subscriber = $this->flowStepCompletedSubscriber(); $ecotone = $this->bootstrap($implementation, [$this->users(), $this->failingNotifier(), $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); $ecotone->sendCommandWithRouting('user.register', '1'); $storedBeforeTheConsumerRan = $subscriber->endedSpans; @@ -634,16 +634,16 @@ public function notify(string $userId): void }; } - private function spanEndedSubscriber(): object + private function flowStepCompletedSubscriber(): object { return new class () { /** @var array */ public array $endedSpans = []; - #[OnSpanEnded] - public function whenEnded(SpanEnded $event): void + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event): void { - $this->endedSpans[$event->span->name] = $event->span->status; + $this->endedSpans[$event->step->name] = $event->step->status; } }; } diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index 6e1ed3f06..e97d5a951 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -13,11 +13,11 @@ use Ecotone\Api\Attribute\Polling; use Ecotone\Api\Attribute\Reference; use Ecotone\Api\Dbal\Tracing\FlowCompleted; +use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; +use Ecotone\Api\Dbal\Tracing\FlowTrackingTimedOut; use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; -use Ecotone\Api\Dbal\Tracing\OnSpanEnded; -use Ecotone\Api\Dbal\Tracing\SpanEnded; -use Ecotone\Api\Dbal\Tracing\TraceCompletionTimedOut; +use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\EventSourcing\Event; use Ecotone\Api\ExtensionObject\DynamicMessageChannelBuilder; @@ -119,7 +119,7 @@ public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pen $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->expectException(TraceCompletionTimedOut::class); + $this->expectException(FlowTrackingTimedOut::class); $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the pending channel, replay the message from the dead letter if its handler failed, or await with a longer timeout."); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); @@ -228,16 +228,16 @@ public function whenCompleted(FlowCompleted $event): void $this->assertSame([$trace->traceId], $subscriber->completedTraces); } - public function test_a_span_ended_subscriber_receives_every_stored_span_of_the_trace(): void + public function test_a_flow_step_completed_subscriber_receives_every_stored_span_of_the_trace(): void { $subscriber = new class () { /** @var string[] */ public array $spanNames = []; - #[OnSpanEnded] - public function whenEnded(SpanEnded $event): void + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event): void { - $this->spanNames[] = $event->span->name; + $this->spanNames[] = $event->step->name; } }; $users = $this->users(); @@ -285,7 +285,7 @@ public function place(string $orderId): void $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->expectException(TraceCompletionTimedOut::class); + $this->expectException(FlowTrackingTimedOut::class); $this->expectExceptionMessage('Still pending: order.placing, async_channel.'); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); @@ -346,18 +346,18 @@ public function refresh(string $traceId): void $this->assertSame(1, $subscriber->notifications); } - public function test_a_span_ended_subscriber_sending_a_command_does_not_notify_itself_again(): void + public function test_a_flow_step_completed_subscriber_sending_a_command_does_not_notify_itself_again(): void { $subscriber = new class () { /** @var string[] */ public array $notifiedTraces = []; - #[OnSpanEnded] - public function whenEnded(SpanEnded $event, CommandBus $commandBus): void + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event, CommandBus $commandBus): void { - $this->notifiedTraces[] = $event->span->traceId; + $this->notifiedTraces[] = $event->step->traceId; if (count($this->notifiedTraces) < 20) { - $commandBus->sendWithRouting('cache.refresh', $event->span->traceId); + $commandBus->sendWithRouting('cache.refresh', $event->step->traceId); } } @@ -508,7 +508,7 @@ public function notify(string $userId): void $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); - $this->expectException(TraceCompletionTimedOut::class); + $this->expectException(FlowTrackingTimedOut::class); $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the pending channel, replay the message from the dead letter if its handler failed, or await with a longer timeout."); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); @@ -518,7 +518,7 @@ public function test_awaiting_a_trace_nobody_recorded_says_so(): void { $ecotone = $this->bootstrap([$this->users()]); - $this->expectException(TraceCompletionTimedOut::class); + $this->expectException(FlowTrackingTimedOut::class); $this->expectExceptionMessage('No span was recorded under trace 0af7651916cd43dd8448eb211c80319c within 100 milliseconds. Send the command with that traceparent in its metadata, and check that the process handling it stores traces.'); $this->completion($ecotone)->await('0af7651916cd43dd8448eb211c80319c', Duration::milliseconds(100)); @@ -530,7 +530,7 @@ public function test_awaiting_a_destination_the_trace_never_reached_names_it_as_ $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->expectException(TraceCompletionTimedOut::class); + $this->expectException(FlowTrackingTimedOut::class); $this->expectExceptionMessage("Trace {$trace->traceId} was neither sent to nor processed by user.notifierr within 100 milliseconds. Check the endpointId or projection name; the trace reached: user.registration."); $this->completion($ecotone)->awaitFor($trace->traceId, 'user.notifierr', Duration::milliseconds(100)); @@ -641,7 +641,7 @@ public function place(string $orderId): void public function test_with_only_requested_flows_tracked_a_command_sent_without_one_stores_nothing(): void { - $subscriber = $this->spanEndedSubscriber(); + $subscriber = $this->flowStepCompletedSubscriber(); $ecotone = EcotoneLite::bootstrapFlowTesting( [$this->users()::class, $this->asynchronousNotifier()::class, $subscriber::class], [$this->users(), $this->asynchronousNotifier(), $subscriber], @@ -657,7 +657,7 @@ public function test_with_only_requested_flows_tracked_a_command_sent_without_on public function test_with_only_requested_flows_tracked_a_command_sent_with_one_is_stored_through_its_asynchronous_hop(): void { - $subscriber = $this->spanEndedSubscriber(); + $subscriber = $this->flowStepCompletedSubscriber(); $ecotone = EcotoneLite::bootstrapFlowTesting( [$this->users()::class, $this->asynchronousNotifier()::class, $subscriber::class], [$this->users(), $this->asynchronousNotifier(), $subscriber], @@ -756,7 +756,7 @@ public function test_an_invalid_tracing_id_is_refused_naming_the_expected_format public function test_a_failed_unit_of_work_stores_nothing_by_default(): void { - $subscriber = $this->spanEndedSubscriber(); + $subscriber = $this->flowStepCompletedSubscriber(); $ecotone = $this->bootstrap([$this->failingUsers(), $subscriber]); $trace = SpanContext::newTrace(); @@ -864,17 +864,17 @@ public function whenCompleted(FlowCompleted $event): void ); } - public function test_a_span_ended_subscriber_requires_an_enterprise_licence_even_without_flow_tracking(): void + public function test_a_flow_step_completed_subscriber_requires_an_enterprise_licence_even_without_flow_tracking(): void { $subscriber = new class () { - #[OnSpanEnded] - public function whenEnded(SpanEnded $event): void + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event): void { } }; $this->expectException(LicensingException::class); - $this->expectExceptionMessage('#[OnSpanEnded] on ' . $subscriber::class . '::whenEnded requires an Ecotone Enterprise licence. Remove it to run without one.'); + $this->expectExceptionMessage('#[OnFlowStepCompleted] on ' . $subscriber::class . '::whenEnded requires an Ecotone Enterprise licence. Remove it to run without one.'); EcotoneLite::bootstrapFlowTesting( [$subscriber::class], @@ -942,16 +942,16 @@ public function register(string $userId): void }; } - private function spanEndedSubscriber(): object + private function flowStepCompletedSubscriber(): object { return new class () { /** @var array */ public array $endedSpans = []; - #[OnSpanEnded] - public function whenEnded(SpanEnded $event): void + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event): void { - $this->endedSpans[$event->span->name] = $event->span->status; + $this->endedSpans[$event->step->name] = $event->step->status; } }; } diff --git a/upgrade-2.0.md b/upgrade-2.0.md index d9df35781..699e9f48f 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3105,11 +3105,11 @@ Two features are on by default and each can be turned off while the flows are st - **`TracingGateway`** answers `isCompleted($tracingId)`, `isCompletedFor($tracingId, $destination)` for one `endpointId` or projection name, and `await()` / `awaitFor()`, which poll until then or throw - `TraceCompletionTimedOut` naming the handlers and channels still pending, pointing to the dead letter when a + `FlowTrackingTimedOut` naming the handlers and channels still pending, pointing to the dead letter when a handler failed, and naming a flow nobody recorded or a destination the flow never reached. `withTracingGateway(false)` turns it off; the gateway stays injectable and every call names that setting. - **Notifications**: a method with `#[OnFlowCompleted]` receives a `FlowCompleted` each time its flow reaches - completion, and one with `#[OnSpanEnded]` receives every tracked span. `withNotifications(false)` turns them off, + completion, and one with `#[OnFlowStepCompleted]` receives a `FlowStepCompleted` for every step of the flow whose unit of work committed, a bus call, a handler run, a route, a message sent or received, with the step's span in `$event->step`. `withNotifications(false)` turns them off, and an application that still declares either attribute is refused at bootstrap. The flow is named by the **`tracingId`** header, `MessageHeaders::TRACING_ID`: a UUID or 32 hexadecimal characters From 37ae9e90fff9ad398d38763555b78fcd0c59aeea Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 31/50] fix(dbal): a flow notification is delivered in an untracked trace of its own, so a subscriber can never reopen or re-notify the flow it was told about A subscriber's work, synchronous or sent through an asynchronous channel, joined the flow it was notified about: the consumer re-completed the flow and notified again, forever. A nested unit of work also reset the publishing guard mid-delivery. The notification now starts a root span marked ecotone=untracked in tracestate, nothing carrying that mark is stored, and a subscriber failure is logged instead of vanishing. --- .../FlowTrackingWithOpenTelemetryTest.php | 25 +++- .../Dbal/src/Tracing/DbalTracingModule.php | 4 +- .../Dbal/src/Tracing/StoringSpanRecorder.php | 44 ++----- .../src/Tracing/TraceRecordingInterceptor.php | 31 ++++- .../Integration/Tracing/FlowTrackingTest.php | 115 ++++++++++++++++++ .../Ecotone/Api/Tracing/SpanAttribute.php | 1 + .../Ecotone/src/Tracing/EcotoneTracer.php | 13 +- packages/Ecotone/src/Tracing/TraceState.php | 27 +++- .../src/Tracing/TracedRouteProcessor.php | 2 +- packages/Ecotone/src/Tracing/Tracer.php | 5 + .../src/Tracing/TracingChannelInterceptor.php | 7 +- .../src/Tracing/TracingInterceptor.php | 37 ++++-- .../OpenTelemetry/src/OpenTelemetryTracer.php | 32 ++++- upgrade-2.0.md | 6 +- 14 files changed, 288 insertions(+), 61 deletions(-) diff --git a/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php index 5f68916f1..033c57542 100644 --- a/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php +++ b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php @@ -76,7 +76,7 @@ public function test_one_trace_is_exported_to_open_telemetry_and_stored_in_the_d $ecotone->run('notifications', ExecutionPollingMetadata::createWithTestingSetup()); self::assertSame( - ['process class@anonymous::notify', 'process class@anonymous::register', 'process class@anonymous::whenCompleted', 'receive notifications', 'route user.notifier', 'send command', 'send event', 'send notifications'], + ['process class@anonymous::notify', 'process class@anonymous::register', 'receive notifications', 'route user.notifier', 'send command', 'send event', 'send notifications'], self::exportedSpanNamesOf($exporter, $trace->traceId), ); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); @@ -84,6 +84,29 @@ public function test_one_trace_is_exported_to_open_telemetry_and_stored_in_the_d self::assertSame([$trace->traceId], $subscriber->completedTraces); } + public function test_the_flow_completed_subscriber_is_exported_in_a_trace_of_its_own(): void + { + $exporter = new InMemoryExporter(new ArrayObject()); + $ecotone = $this->bootstrap($exporter, [$this->users(), $this->flowCompletedSubscriber()]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $notificationTraceIds = []; + /** @var SpanDataInterface $span */ + foreach ($exporter->getSpans() as $span) { + if ($span->getName() === 'send ecotone.tracing.flowCompleted') { + $notificationTraceIds[] = $span->getContext()->getTraceId(); + } + } + self::assertCount(1, $notificationTraceIds); + self::assertNotSame($trace->traceId, $notificationTraceIds[0]); + self::assertSame( + ['process class@anonymous::whenCompleted', 'send ecotone.tracing.flowCompleted'], + self::exportedSpanNamesOf($exporter, $notificationTraceIds[0]), + ); + } + public function test_a_second_application_on_the_same_database_sees_the_completion_the_first_one_stored(): void { $producer = $this->bootstrap(new InMemoryExporter(new ArrayObject()), [$this->users()]); diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index 840630e89..fd5875c26 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -46,6 +46,7 @@ use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayHeaderBuilder; use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayPayloadBuilder; use Ecotone\Messaging\Handler\InterfaceToCallRegistry; +use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Ecotone\Messaging\Handler\Processor\MethodInvoker\AroundInterceptorBuilder; use Ecotone\Messaging\Handler\ServiceActivator\MessageProcessorActivatorBuilder; use Ecotone\Messaging\Support\LicensingException; @@ -128,7 +129,6 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO StoringSpanRecorder::class, new Definition(StoringSpanRecorder::class, [ new Reference(TraceStore::class), - new Reference(MessagingEntrypointService::class), $flowTracking->notifications && $this->flowStepCompletedSubscribers !== [], $flowTracking->notifications && $this->flowCompletedSubscribers !== [], $flowTracking->onlyRequestedFlows, @@ -137,7 +137,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ); $messagingConfiguration->registerServiceDefinition( TraceRecordingInterceptor::class, - new Definition(TraceRecordingInterceptor::class, [new Reference(StoringSpanRecorder::class), new Reference(Tracer::class)]) + new Definition(TraceRecordingInterceptor::class, [new Reference(StoringSpanRecorder::class), new Reference(Tracer::class), new Reference(MessagingEntrypointService::class), new Reference(LoggingGateway::class)]) ); foreach (['publishAfterCommit' => Precedence::TRACE_PUBLICATION_PRECEDENCE, 'recordUnitOfWork' => Precedence::TRACE_RECORDING_PRECEDENCE] as $method => $precedence) { $messagingConfiguration->registerAroundMethodInterceptor( diff --git a/packages/Dbal/src/Tracing/StoringSpanRecorder.php b/packages/Dbal/src/Tracing/StoringSpanRecorder.php index 74aa8ec5d..d482f1e76 100644 --- a/packages/Dbal/src/Tracing/StoringSpanRecorder.php +++ b/packages/Dbal/src/Tracing/StoringSpanRecorder.php @@ -6,14 +6,10 @@ use Ecotone\Api\Dbal\Tracing\FlowCompleted; use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; -use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; -use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Tracing\Span; use Ecotone\Api\Tracing\SpanAttribute; use Ecotone\Api\Tracing\SpanRecorder; -use Ecotone\Messaging\Gateway\MessagingEntrypointService; -use Throwable; /** * licence Enterprise @@ -39,7 +35,6 @@ final class StoringSpanRecorder implements SpanRecorder private array $buffer = []; private bool $scopeActive = false; private bool $unitOfWorkActive = false; - private bool $publishing = false; /** @var Span[] */ private array $committedSpans = []; /** @var string[] */ @@ -47,7 +42,6 @@ final class StoringSpanRecorder implements SpanRecorder public function __construct( private TraceStore $traceStore, - private MessagingEntrypointService $messagingEntrypoint, private bool $publishesEndedSpans, private bool $publishesCompletedTraces, private bool $storesOnlyCarriedTraces, @@ -112,9 +106,6 @@ public function flush(?Span $wrappingSpanSoFar): void $completedTraces[] = $traceId; } } - if ($this->publishing) { - return; - } $this->committedSpans = [...$this->committedSpans, ...$spans]; $this->completedTraces = [...$this->completedTraces, ...$completedTraces]; } @@ -124,35 +115,26 @@ public function discard(): void $this->buffer = []; } - public function publishCommitted(): void + /** + * @return array + */ + public function popCommittedNotifications(): array { - $endedSpans = $this->publishesEndedSpans ? $this->committedSpans : []; - $completedTraces = $this->publishesCompletedTraces ? $this->completedTraces : []; + $notifications = [ + ...array_map(fn (Span $span): FlowStepCompleted => new FlowStepCompleted($span), $this->publishesEndedSpans ? $this->committedSpans : []), + ...array_map(fn (string $traceId): FlowCompleted => new FlowCompleted($traceId), $this->publishesCompletedTraces ? array_values(array_unique($this->completedTraces)) : []), + ]; $this->endUnitOfWork(); - $this->publishing = true; - try { - foreach ($endedSpans as $span) { - $this->deliver(new FlowStepCompleted($span), OnFlowStepCompleted::CHANNEL); - } - foreach (array_unique($completedTraces) as $traceId) { - $this->deliver(new FlowCompleted($traceId), OnFlowCompleted::CHANNEL); - } - } finally { - $this->publishing = false; - } + return $notifications; } private function isStored(Span $span): bool { - return ! $this->storesOnlyCarriedTraces || ($span->attributes[SpanAttribute::TRACE_CARRIED] ?? false); - } - - private function deliver(object $notification, string $channel): void - { - try { - $this->messagingEntrypoint->send($notification, $channel); - } catch (Throwable) { + if ($span->attributes[SpanAttribute::TRACE_UNTRACKED] ?? false) { + return false; } + + return ! $this->storesOnlyCarriedTraces || ($span->attributes[SpanAttribute::TRACE_CARRIED] ?? false); } } diff --git a/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php b/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php index 83c7231fe..262958767 100644 --- a/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php +++ b/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php @@ -4,7 +4,15 @@ namespace Ecotone\Dbal\Tracing; +use Ecotone\Api\Dbal\Tracing\FlowCompleted; +use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; +use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; +use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Interceptor\MethodInvocation; +use Ecotone\Api\Tracing\SpanAttribute; +use Ecotone\Api\Tracing\SpanKind; +use Ecotone\Messaging\Gateway\MessagingEntrypointService; +use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Ecotone\Tracing\Tracer; use Throwable; @@ -31,6 +39,8 @@ final class TraceRecordingInterceptor public function __construct( private StoringSpanRecorder $recorder, private Tracer $tracer, + private MessagingEntrypointService $messagingEntrypoint, + private LoggingGateway $logger, ) { } @@ -48,7 +58,9 @@ public function publishAfterCommit(MethodInvocation $methodInvocation): mixed throw $failure; } - $this->recorder->publishCommitted(); + foreach ($this->recorder->popCommittedNotifications() as $notification) { + $this->deliver($notification); + } return $result; } @@ -74,4 +86,21 @@ public function recordUnitOfWork(MethodInvocation $methodInvocation): mixed $this->recorder->closeScope(); } } + + private function deliver(FlowStepCompleted|FlowCompleted $notification): void + { + $channel = $notification instanceof FlowCompleted ? OnFlowCompleted::CHANNEL : OnFlowStepCompleted::CHANNEL; + $span = $this->tracer->startRootSpan('send ' . $channel, SpanKind::Producer, [ + SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND, + SpanAttribute::MESSAGING_DESTINATION_NAME => $channel, + SpanAttribute::TRACE_UNTRACKED => true, + ]); + try { + $this->messagingEntrypoint->send($notification, $channel); + $span->end(); + } catch (Throwable $failure) { + $span->fail($failure); + $this->logger->error(sprintf('A subscriber of %s failed and was skipped: %s', $channel, $failure->getMessage()), [], ['exception' => $failure]); + } + } } diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index e97d5a951..899e4377e 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -38,6 +38,7 @@ use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Support\LicensingException; use Ecotone\Test\LicenceTesting; +use Ecotone\Test\StubLogger; use PHPUnit\Framework\TestCase; use RuntimeException; @@ -374,6 +375,120 @@ public function refresh(string $traceId): void $this->assertSame([$trace->traceId], array_values(array_unique($subscriber->notifiedTraces))); } + public function test_a_failing_flow_completed_subscriber_is_logged_and_does_not_fail_the_command(): void + { + $subscriber = new class () { + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event): void + { + throw new RuntimeException('Cache refresh failed'); + } + }; + $logger = StubLogger::create(); + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$this->users()::class, $subscriber::class], + [$this->users(), $subscriber, 'logger' => $logger], + [FlowTrackingConfiguration::createWithDefaults()], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->assertSame(['A subscriber of ecotone.tracing.flowCompleted failed and was skipped: Cache refresh failed'], $logger->getError()); + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + public function test_a_flow_step_completed_subscriber_is_never_notified_about_the_synchronous_work_it_started(): void + { + $subscriber = new class () { + /** @var string[] */ + public array $stepNames = []; + + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event, CommandBus $commandBus): void + { + $this->stepNames[] = $event->step->name; + if (count($this->stepNames) < 20) { + $commandBus->sendWithRouting('cache.refresh', $event->step->traceId); + } + } + + #[CommandHandler('cache.refresh', endpointId: 'cache.refreshing')] + public function refresh(string $traceId): void + { + } + }; + $ecotone = $this->bootstrap([$this->users(), $subscriber]); + + $ecotone->sendCommandWithRouting('user.register', '1'); + + $this->assertNotContains('process class@anonymous::refresh', $subscriber->stepNames); + $this->assertLessThan(20, count($subscriber->stepNames)); + } + + public function test_a_flow_completed_subscriber_sending_an_asynchronous_command_is_notified_once(): void + { + $subscriber = new class () { + /** @var string[] */ + public array $completedTraces = []; + + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event, CommandBus $commandBus): void + { + $this->completedTraces[] = $event->tracingId; + if (count($this->completedTraces) < 20) { + $commandBus->sendWithRouting('cache.refresh', $event->tracingId); + } + } + + #[Asynchronous('async_channel')] + #[CommandHandler('cache.refresh', endpointId: 'cache.refreshing')] + public function refresh(string $tracingId): void + { + } + }; + $ecotone = $this->bootstrap([$this->users(), $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertSame([$trace->traceId], $subscriber->completedTraces); + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + + public function test_a_flow_step_completed_subscriber_sending_an_asynchronous_command_is_not_notified_about_its_consumer(): void + { + $subscriber = new class () { + /** @var string[] */ + public array $stepNames = []; + + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event, CommandBus $commandBus): void + { + $this->stepNames[] = $event->step->name; + if (count($this->stepNames) < 20) { + $commandBus->sendWithRouting('cache.refresh', $event->step->traceId); + } + } + + #[Asynchronous('async_channel')] + #[CommandHandler('cache.refresh', endpointId: 'cache.refreshing')] + public function refresh(string $traceId): void + { + } + }; + $ecotone = $this->bootstrap([$this->users(), $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $ecotone->sendCommandWithRouting('user.register', '1'); + $notifiedBeforeTheConsumerRan = $subscriber->stepNames; + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertSame($notifiedBeforeTheConsumerRan, $subscriber->stepNames); + } + public function test_a_consumer_that_swallowed_its_handler_failure_leaves_the_trace_pending(): void { $orders = new class () { diff --git a/packages/Ecotone/Api/Tracing/SpanAttribute.php b/packages/Ecotone/Api/Tracing/SpanAttribute.php index e755770c4..d793e5d80 100644 --- a/packages/Ecotone/Api/Tracing/SpanAttribute.php +++ b/packages/Ecotone/Api/Tracing/SpanAttribute.php @@ -16,6 +16,7 @@ final class SpanAttribute public const PROJECTION_NAME = 'ecotone.projection.name'; public const TRACE_ROOT = 'ecotone.trace.root'; public const TRACE_CARRIED = 'ecotone.trace.carried'; + public const TRACE_UNTRACKED = 'ecotone.trace.untracked'; public const CODE_FUNCTION_NAME = 'code.function.name'; public const CHANNEL_NAME = 'ecotone.channel.name'; public const BATCH_DELIVERIES = 'ecotone.batch.deliveries'; diff --git a/packages/Ecotone/src/Tracing/EcotoneTracer.php b/packages/Ecotone/src/Tracing/EcotoneTracer.php index a3d1b1073..3580d2e71 100644 --- a/packages/Ecotone/src/Tracing/EcotoneTracer.php +++ b/packages/Ecotone/src/Tracing/EcotoneTracer.php @@ -30,8 +30,19 @@ public function __construct( public function startSpan(string $name, SpanKind $kind, array $attributes = [], ?SpanContext $remoteParent = null): ActiveSpan { - $parent = $remoteParent ?? $this->currentSpan()?->context(); + return $this->start($name, $kind, $attributes, $remoteParent ?? $this->currentSpan()?->context()); + } + public function startRootSpan(string $name, SpanKind $kind, array $attributes = []): ActiveSpan + { + return $this->start($name, $kind, $attributes, null); + } + + /** + * @param array $attributes + */ + private function start(string $name, SpanKind $kind, array $attributes, ?SpanContext $parent): ActiveSpan + { $span = new EcotoneActiveSpan( $parent?->newChild() ?? SpanContext::newTrace(), $parent?->spanId, diff --git a/packages/Ecotone/src/Tracing/TraceState.php b/packages/Ecotone/src/Tracing/TraceState.php index 01467a09e..41d5f032f 100644 --- a/packages/Ecotone/src/Tracing/TraceState.php +++ b/packages/Ecotone/src/Tracing/TraceState.php @@ -14,19 +14,40 @@ final class TraceState { private const VENDOR = 'ecotone'; private const CARRIED = 'carried'; + private const UNTRACKED = 'untracked'; public static function isCarried(Message $message): bool { - $headers = $message->getHeaders(); + return self::entryOfMessage($message) === self::CARRIED; + } - return $headers->containsKey(MessageHeaders::TRACESTATE) && self::entryOf((string) $headers->get(MessageHeaders::TRACESTATE)) === self::CARRIED; + public static function isUntracked(Message $message): bool + { + return self::entryOfMessage($message) === self::UNTRACKED; } public static function withCarried(string $tracestate): string + { + return self::withEntry($tracestate, self::CARRIED); + } + + public static function withUntracked(string $tracestate): string + { + return self::withEntry($tracestate, self::UNTRACKED); + } + + private static function withEntry(string $tracestate, string $value): string { $entries = array_values(array_filter(array_map('trim', explode(',', $tracestate)), fn (string $entry): bool => $entry !== '' && ! str_starts_with($entry, self::VENDOR . '='))); - return implode(',', [self::VENDOR . '=' . self::CARRIED, ...$entries]); + return implode(',', [self::VENDOR . '=' . $value, ...$entries]); + } + + private static function entryOfMessage(Message $message): ?string + { + $headers = $message->getHeaders(); + + return $headers->containsKey(MessageHeaders::TRACESTATE) ? self::entryOf((string) $headers->get(MessageHeaders::TRACESTATE)) : null; } private static function entryOf(string $tracestate): ?string diff --git a/packages/Ecotone/src/Tracing/TracedRouteProcessor.php b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php index 8de3ad7c8..2df01f053 100644 --- a/packages/Ecotone/src/Tracing/TracedRouteProcessor.php +++ b/packages/Ecotone/src/Tracing/TracedRouteProcessor.php @@ -32,7 +32,7 @@ public function process(Message $message): ?Message SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND, SpanAttribute::MESSAGING_DESTINATION_NAME => $this->routeName, ...$this->destinationAttributes(), - ...(TracingInterceptor::isCarried($this->tracer, $message, false) ? [SpanAttribute::TRACE_CARRIED => true] : []), + ...TracingInterceptor::trackingAttributesOf($this->tracer, $message, false), ]); try { diff --git a/packages/Ecotone/src/Tracing/Tracer.php b/packages/Ecotone/src/Tracing/Tracer.php index e7d348315..a99748a8b 100644 --- a/packages/Ecotone/src/Tracing/Tracer.php +++ b/packages/Ecotone/src/Tracing/Tracer.php @@ -17,5 +17,10 @@ interface Tracer */ public function startSpan(string $name, SpanKind $kind, array $attributes = [], ?SpanContext $remoteParent = null): ActiveSpan; + /** + * @param array $attributes + */ + public function startRootSpan(string $name, SpanKind $kind, array $attributes = []): ActiveSpan; + public function currentSpan(): ?ActiveSpan; } diff --git a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php index 0d7d4f51a..c041ea775 100644 --- a/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingChannelInterceptor.php @@ -45,10 +45,7 @@ public function preSend(Message $message, MessageChannel $messageChannel): ?Mess if ($isFirstChannel) { $attributes = [...$attributes, ...$this->deliveryAttributes($message)]; } - $carried = TracingInterceptor::isCarried($this->tracer, $message, false); - if ($carried) { - $attributes[SpanAttribute::TRACE_CARRIED] = true; - } + $attributes = [...$attributes, ...TracingInterceptor::trackingAttributesOf($this->tracer, $message, false)]; $span = $this->tracer->startSpan( 'send ' . $this->channelName, SpanKind::Producer, @@ -163,7 +160,7 @@ public function afterReceiveCompletion(?Message $message, MessageChannel $messag SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_PROCESS, SpanAttribute::MESSAGING_DESTINATION_NAME => $this->channelName, SpanAttribute::CHANNEL_NAME => $this->channelName, - ...(TraceState::isCarried($message) ? [SpanAttribute::TRACE_CARRIED => true] : []), + ...TracingInterceptor::trackingAttributesCarriedBy($message, false), ], TracingInterceptor::remoteParentOf($message)) ->fail($exception); } diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 7484fd932..94f165ea6 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -133,7 +133,11 @@ public function provideContextForDistributedBus(Message $message): Message public static function carrierOf(ActiveSpan $span): array { $carrier = [MessageHeaders::TRACEPARENT => $span->context()->toTraceparent()]; - $traceState = $span->attributes()[SpanAttribute::TRACE_CARRIED] ?? false ? TraceState::withCarried($span->context()->traceState) : $span->context()->traceState; + $traceState = match (true) { + (bool) ($span->attributes()[SpanAttribute::TRACE_UNTRACKED] ?? false) => TraceState::withUntracked($span->context()->traceState), + (bool) ($span->attributes()[SpanAttribute::TRACE_CARRIED] ?? false) => TraceState::withCarried($span->context()->traceState), + default => $span->context()->traceState, + }; if ($traceState !== '') { $carrier[MessageHeaders::TRACESTATE] = $traceState; } @@ -141,14 +145,33 @@ public static function carrierOf(ActiveSpan $span): array return $carrier; } - public static function isCarried(Tracer $tracer, Message $message, bool $startsTheTraceHere): bool + /** + * @return array + */ + public static function trackingAttributesOf(Tracer $tracer, Message $message, bool $startsTheTraceHere): array { $current = $tracer->currentSpan(); - if ($current !== null) { - return (bool) ($current->attributes()[SpanAttribute::TRACE_CARRIED] ?? false); + if ($current === null) { + return self::trackingAttributesCarriedBy($message, $startsTheTraceHere); + } + + return array_filter([ + SpanAttribute::TRACE_CARRIED => (bool) ($current->attributes()[SpanAttribute::TRACE_CARRIED] ?? false), + SpanAttribute::TRACE_UNTRACKED => (bool) ($current->attributes()[SpanAttribute::TRACE_UNTRACKED] ?? false), + ]); + } + + /** + * @return array + */ + public static function trackingAttributesCarriedBy(Message $message, bool $startsTheTraceHere): array + { + if (TraceState::isUntracked($message)) { + return [SpanAttribute::TRACE_UNTRACKED => true]; } + $carried = $startsTheTraceHere ? self::remoteParentOf($message) !== null : TraceState::isCarried($message); - return $startsTheTraceHere ? self::remoteParentOf($message) !== null : TraceState::isCarried($message); + return $carried ? [SpanAttribute::TRACE_CARRIED => true] : []; } public static function remoteParentOf(Message $message): ?SpanContext @@ -227,9 +250,7 @@ private function trace(string $name, MethodInvocation $methodInvocation, Message if ($isLocalRoot && $kind !== SpanKind::Consumer) { $attributes[SpanAttribute::TRACE_ROOT] = true; } - if (self::isCarried($this->tracer, $message, $isLocalRoot && $kind !== SpanKind::Consumer)) { - $attributes[SpanAttribute::TRACE_CARRIED] = true; - } + $attributes = [...$attributes, ...self::trackingAttributesOf($this->tracer, $message, $isLocalRoot && $kind !== SpanKind::Consumer)]; $span = $this->tracer->startSpan($name, $kind, [...$this->messageSpanAttributes->from($message), ...$attributes], $remoteParent); try { diff --git a/packages/OpenTelemetry/src/OpenTelemetryTracer.php b/packages/OpenTelemetry/src/OpenTelemetryTracer.php index 9a0faee8d..06f6839cd 100644 --- a/packages/OpenTelemetry/src/OpenTelemetryTracer.php +++ b/packages/OpenTelemetry/src/OpenTelemetryTracer.php @@ -17,6 +17,7 @@ use Ecotone\Tracing\ActiveSpan; use Ecotone\Tracing\Tracer; use OpenTelemetry\API\Trace\Propagation\TraceContextPropagator; +use OpenTelemetry\API\Trace\SpanBuilderInterface; use OpenTelemetry\API\Trace\SpanKind as OpenTelemetrySpanKind; use OpenTelemetry\API\Trace\TracerProviderInterface; @@ -37,21 +38,40 @@ public function __construct( public function startSpan(string $name, SpanKind $kind, array $attributes = [], ?SpanContext $remoteParent = null): ActiveSpan { - $parentSpanId = $remoteParent?->spanId ?? $this->currentSpan()?->context()->spanId; + $spanBuilder = $this->spanBuilderFor($name, $kind, $attributes); + if ($remoteParent !== null) { + $spanBuilder->setParent(TraceContextPropagator::getInstance()->extract([MessageHeaders::TRACEPARENT => $remoteParent->toTraceparent(), MessageHeaders::TRACESTATE => $remoteParent->traceState])); + } + + return $this->start($spanBuilder, $name, $kind, $attributes, $remoteParent ?? $this->currentSpan()?->context()); + } + + public function startRootSpan(string $name, SpanKind $kind, array $attributes = []): ActiveSpan + { + return $this->start($this->spanBuilderFor($name, $kind, $attributes)->setParent(false), $name, $kind, $attributes, null); + } - $spanBuilder = $this->tracerProvider + /** + * @param array $attributes + */ + private function spanBuilderFor(string $name, SpanKind $kind, array $attributes): SpanBuilderInterface + { + return $this->tracerProvider ->getTracer(EcotoneSpanBuilder::ECOTONE_TRACER_NAME) ->spanBuilder($name) ->setSpanKind(self::kindOf($kind)) ->setAttributes($attributes); - if ($remoteParent !== null) { - $spanBuilder->setParent(TraceContextPropagator::getInstance()->extract([MessageHeaders::TRACEPARENT => $remoteParent->toTraceparent(), MessageHeaders::TRACESTATE => $remoteParent->traceState])); - } + } + /** + * @param array $attributes + */ + private function start(SpanBuilderInterface $spanBuilder, string $name, SpanKind $kind, array $attributes, ?SpanContext $parentContext): ActiveSpan + { + $parentSpanId = $parentContext?->spanId; $span = $spanBuilder->startSpan(); $scope = $span->activate(); $context = $span->getContext(); - $parentContext = $remoteParent ?? $this->currentSpan()?->context(); $spanContext = $context->isValid() && $span->isRecording() ? new SpanContext($context->getTraceId(), $context->getSpanId(), $context->isSampled(), (string) $context->getTraceState()) : ($parentContext?->newChild() ?? SpanContext::newTrace()); diff --git a/upgrade-2.0.md b/upgrade-2.0.md index 699e9f48f..da1986c36 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3134,8 +3134,10 @@ whether the flow is complete. Nothing is written after the commit or after a rol right after a commit cannot fail a consumer or a command whose work already committed, and an outage inside the transaction rolls back with the work and leaves the message to be retried. A unit of work that fails stores nothing; its failure is visible in OpenTelemetry. Notifications are published from memory once the transaction committed, at -most once: a subscriber that throws does not fail the work that committed and is not called again, and work a -subscriber does through a bus is tracked but never notifies subscribers in turn. Completion is claimed with one +most once: a subscriber that throws does not fail the work that committed, is logged, and is not called again. Each +notification is delivered in a trace of its own, marked `ecotone=untracked` in the W3C `tracestate`, so whatever a +subscriber sends, synchronously or through an asynchronous channel, is neither stored nor notified and can never reopen +the flow it was told about; OpenTelemetry still exports it, as its own trace. Completion is claimed with one conditional update after the trace row was touched in the same transaction, so a second unit of work finishing the same flow waits for the first one's commit and, on PostgreSQL and SQLite, reads its spans before deciding. From 02e0bb9ccc7029a4584068c6dec007a8fabe0f40 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 32/50] fix(dbal): an asynchronous projection is recorded inside its own batch transaction, never after it committed Ecotone turns the endpoint transaction off for an asynchronous projection, which commits once per batch, so flow tracking wrote the projection's spans after the read model had committed - a database outage at that moment failed the consumer for work that succeeded. Flow tracking now joins each batch's transaction and marks the projection processed in the batch that finds it caught up. Paths the application runs with no transaction at all still write when their work ends. --- docs/coding-conventions.md | 12 +- packages/Dbal/src/Tracing/DbalTraceStore.php | 6 + .../Dbal/src/Tracing/DbalTracingModule.php | 9 + .../Dbal/src/Tracing/InMemoryTraceStore.php | 6 + .../Dbal/src/Tracing/StoringSpanRecorder.php | 66 ++++++- .../src/Tracing/TraceRecordingInterceptor.php | 32 +++- packages/Dbal/src/Tracing/TraceStore.php | 3 + .../FlowTrackingConformanceTest.php | 43 +++++ .../Projecting/FlowTrackingProjectionTest.php | 178 ++++++++++++++++++ upgrade-2.0.md | 5 +- 10 files changed, 348 insertions(+), 12 deletions(-) create mode 100644 packages/PdoEventSourcing/tests/Projecting/FlowTrackingProjectionTest.php diff --git a/docs/coding-conventions.md b/docs/coding-conventions.md index 31bc15b52..7d70dce2c 100644 --- a/docs/coding-conventions.md +++ b/docs/coding-conventions.md @@ -1182,16 +1182,24 @@ nothing touches the database once the transaction has returned: ```php $result = $methodInvocation->proceed(); -$this->recorder->flush($wrappingSpan?->endedSoFar()); // inside the transaction +$this->recorder->flushWithOpenSpans(); // inside the transaction ``` +Work that commits by itself - an asynchronous projection commits once per batch, with Ecotone turning the endpoint's +transaction off - is recorded inside each of those commits, never once the work has returned: flow tracking joins +the projection batch's transaction. A path the application chose to run with no transaction at all (the command bus +with `withTransactionOnCommandBus(false)`, an event bus or gateway call from a controller) has no transaction to be +inside, so what it stores is written when its work ends; that is the application's decision, and the default runs +every command and consumer in a transaction. + What may happen after the commit is work that does not depend on the database: publishing to subscribers from memory, and sending telemetry to an external collector such as OpenTelemetry, which is allowed to fail and never fails the message. A test proves the rule by making the database unavailable right after the commit, with an `#[Around]` interceptor just outside the transaction, and asserting that the command or consumer still succeeds. Evidence: stored traces wrote the root span and claimed completion after the commit, and an outage at that moment -threw the missing-table refusal out of the consumer (`TraceStoreConformanceTest`, the two outage tests). +threw the missing-table refusal out of the consumer (`FlowTrackingConformanceTest`, the outage tests); an asynchronous +projection wrote its spans after its own batch committed (`FlowTrackingProjectionTest`). --- diff --git a/packages/Dbal/src/Tracing/DbalTraceStore.php b/packages/Dbal/src/Tracing/DbalTraceStore.php index 6083e4b8a..e5561dd80 100644 --- a/packages/Dbal/src/Tracing/DbalTraceStore.php +++ b/packages/Dbal/src/Tracing/DbalTraceStore.php @@ -4,6 +4,7 @@ namespace Ecotone\Dbal\Tracing; +use Closure; use Doctrine\DBAL\Connection; use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; use Doctrine\DBAL\Types\Types; @@ -95,6 +96,11 @@ public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, Dat }); } + public function withinTransaction(Closure $work): mixed + { + return $this->connection()->transactional($work); + } + /** * @param Span[] $traceSpans */ diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index fd5875c26..c7eca06ef 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -27,6 +27,7 @@ use Ecotone\Api\Gateway\QueryBus; use Ecotone\Api\Interceptor\Precedence; use Ecotone\Api\Projecting\Projection; +use Ecotone\Api\Projecting\ProjectionFlush; use Ecotone\Dbal\Database\DbalTableManagerReference; use Ecotone\Dbal\Database\SpansTableManager; use Ecotone\Dbal\Database\TracesTableManager; @@ -149,6 +150,14 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ) ); } + $messagingConfiguration->registerAroundMethodInterceptor( + AroundInterceptorBuilder::create( + TraceRecordingInterceptor::class, + $interfaceToCallRegistry->getFor(TraceRecordingInterceptor::class, 'recordProjectionBatch'), + Precedence::TRACE_RECORDING_PRECEDENCE, + ProjectionFlush::class + ) + ); if ($flowTracking->notifications) { $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnFlowCompleted::CHANNEL)); $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnFlowStepCompleted::CHANNEL)); diff --git a/packages/Dbal/src/Tracing/InMemoryTraceStore.php b/packages/Dbal/src/Tracing/InMemoryTraceStore.php index 4d1bb2c58..d61150926 100644 --- a/packages/Dbal/src/Tracing/InMemoryTraceStore.php +++ b/packages/Dbal/src/Tracing/InMemoryTraceStore.php @@ -4,6 +4,7 @@ namespace Ecotone\Dbal\Tracing; +use Closure; use Ecotone\Api\Scheduling\DatePoint; use Ecotone\Api\Tracing\Span; @@ -67,4 +68,9 @@ public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, Dat } } } + + public function withinTransaction(Closure $work): mixed + { + return $work(); + } } diff --git a/packages/Dbal/src/Tracing/StoringSpanRecorder.php b/packages/Dbal/src/Tracing/StoringSpanRecorder.php index d482f1e76..0c6bc1a45 100644 --- a/packages/Dbal/src/Tracing/StoringSpanRecorder.php +++ b/packages/Dbal/src/Tracing/StoringSpanRecorder.php @@ -4,12 +4,14 @@ namespace Ecotone\Dbal\Tracing; +use Closure; use Ecotone\Api\Dbal\Tracing\FlowCompleted; use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Tracing\Span; use Ecotone\Api\Tracing\SpanAttribute; use Ecotone\Api\Tracing\SpanRecorder; +use Ecotone\Tracing\ActiveSpan; /** * licence Enterprise @@ -34,6 +36,10 @@ final class StoringSpanRecorder implements SpanRecorder /** @var Span[] */ private array $buffer = []; private bool $scopeActive = false; + private ?ActiveSpan $wrappingSpan = null; + /** @var array */ + private array $storedSpanIds = []; + private bool $projectionBatchActive = false; private bool $unitOfWorkActive = false; /** @var Span[] */ private array $committedSpans = []; @@ -73,9 +79,10 @@ public function endUnitOfWork(): void $this->completedTraces = []; } - public function openScope(): void + public function openScope(?ActiveSpan $wrappingSpan): void { $this->scopeActive = true; + $this->wrappingSpan = $wrappingSpan; } public function isScopeActive(): bool @@ -86,20 +93,67 @@ public function isScopeActive(): bool public function closeScope(): void { $this->scopeActive = false; + $this->wrappingSpan = null; + $this->storedSpanIds = []; } - public function flush(?Span $wrappingSpanSoFar): void + public function beginProjectionBatch(): void { - $spans = $this->buffer; - $this->buffer = []; - if ($wrappingSpanSoFar !== null && $this->isStored($wrappingSpanSoFar)) { - $spans[] = $wrappingSpanSoFar; + $this->projectionBatchActive = true; + } + + public function isProjectionBatchActive(): bool + { + return $this->projectionBatchActive; + } + + public function endProjectionBatch(): void + { + $this->projectionBatchActive = false; + } + + public function withinTransaction(Closure $work): mixed + { + return $this->traceStore->withinTransaction($work); + } + + public function flushBuffered(): void + { + $this->store([]); + } + + public function flushWithOpenSpans(?ActiveSpan ...$innerOpenSpans): void + { + $openSpans = []; + foreach ([$this->wrappingSpan, ...$innerOpenSpans] as $openSpan) { + if ($openSpan !== null) { + $openSpans[$openSpan->context()->spanId] = $openSpan->endedSoFar(); + } } + $this->store(array_values($openSpans)); + } + + /** + * @param Span[] $openSpansSoFar + */ + private function store(array $openSpansSoFar): void + { + $spans = []; + foreach ([...$this->buffer, ...$openSpansSoFar] as $span) { + if ($this->isStored($span) && ! isset($this->storedSpanIds[$span->spanId])) { + $spans[$span->spanId] = $span; + } + } + $this->buffer = []; if ($spans === []) { return; } + $spans = array_values($spans); $this->traceStore->append($spans); + foreach ($spans as $span) { + $this->storedSpanIds[$span->spanId] = true; + } $completedTraces = []; foreach (array_unique(array_map(fn (Span $span): string => $span->traceId, $spans)) as $traceId) { if ($this->traceStore->markCompletedIfDone($traceId, $this->clock->now())) { diff --git a/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php b/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php index 262958767..68f3dd0c5 100644 --- a/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php +++ b/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php @@ -71,11 +71,10 @@ public function recordUnitOfWork(MethodInvocation $methodInvocation): mixed return $methodInvocation->proceed(); } - $wrappingSpan = $this->tracer->currentSpan(); - $this->recorder->openScope(); + $this->recorder->openScope($this->tracer->currentSpan()); try { $result = $methodInvocation->proceed(); - $this->recorder->flush($wrappingSpan?->endedSoFar()); + $this->recorder->flushWithOpenSpans(); return $result; } catch (Throwable $failure) { @@ -87,6 +86,33 @@ public function recordUnitOfWork(MethodInvocation $methodInvocation): mixed } } + public function recordProjectionBatch(MethodInvocation $methodInvocation): mixed + { + if (! $this->recorder->isScopeActive() || $this->recorder->isProjectionBatchActive()) { + return $methodInvocation->proceed(); + } + + $this->recorder->beginProjectionBatch(); + try { + return $this->recorder->withinTransaction(function () use ($methodInvocation): mixed { + $processedEvents = $methodInvocation->proceed(); + if ($processedEvents === 0) { + $this->recorder->flushWithOpenSpans($this->tracer->currentSpan()); + } else { + $this->recorder->flushBuffered(); + } + + return $processedEvents; + }); + } catch (Throwable $failure) { + $this->recorder->discard(); + + throw $failure; + } finally { + $this->recorder->endProjectionBatch(); + } + } + private function deliver(FlowStepCompleted|FlowCompleted $notification): void { $channel = $notification instanceof FlowCompleted ? OnFlowCompleted::CHANNEL : OnFlowStepCompleted::CHANNEL; diff --git a/packages/Dbal/src/Tracing/TraceStore.php b/packages/Dbal/src/Tracing/TraceStore.php index a99f39acc..f567015a7 100644 --- a/packages/Dbal/src/Tracing/TraceStore.php +++ b/packages/Dbal/src/Tracing/TraceStore.php @@ -4,6 +4,7 @@ namespace Ecotone\Dbal\Tracing; +use Closure; use Ecotone\Api\Scheduling\DatePoint; use Ecotone\Api\Tracing\Span; @@ -37,4 +38,6 @@ public function markCompletedIfDone(string $traceId, DatePoint $now): bool; public function progressOf(string $traceId): TraceProgress; public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore): void; + + public function withinTransaction(Closure $work): mixed; } diff --git a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php index d2b5c01b6..4601b5819 100644 --- a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php +++ b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php @@ -23,6 +23,7 @@ use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; +use Ecotone\Api\EventSourcing\Event; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\InstantRetryConfiguration; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -33,6 +34,8 @@ use Ecotone\Api\Interceptor\Precedence; use Ecotone\Api\Lite\Test\FlowTestSupport; use Ecotone\Api\Messaging\MessageHeaders; +use Ecotone\Api\Projecting\FromStream; +use Ecotone\Api\Projecting\Projection; use Ecotone\Api\Scheduling\Duration; use Ecotone\Api\Scheduling\TimeSpan; use Ecotone\Api\Tracing\SpanContext; @@ -503,6 +506,46 @@ public function register(string $userId): void self::assertSame(1, $users->registered); } + public function test_a_database_outage_right_after_an_asynchronous_projection_committed_does_not_fail_its_consumer(): void + { + $projection = new #[Projection('users_list'), FromStream('users_stream'), Asynchronous('async_channel')] class () { + public int $projected = 0; + + #[EventHandler('user.registered')] + public function whenRegistered(array $event): void + { + $this->projected++; + } + }; + $outage = new class () { + public bool $armed = false; + + #[Around(precedence: Precedence::DATABASE_TRANSACTION_PRECEDENCE + 2, pointcut: 'Ecotone\\Api\\Attribute\\AsynchronousRunningEndpoint')] + public function goDownAfterTheProjectionRan(MethodInvocation $methodInvocation): mixed + { + $result = $methodInvocation->proceed(); + if ($this->armed) { + OutageSimulatingConnection::$databaseDown = true; + } + + return $result; + } + }; + $ecotone = $this->bootstrapOnAConnectionThatCanGoDown([$projection, $outage], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $ecotone->withEvents([Event::createWithType('user.registered', ['userId' => '1'])]); + $trace = SpanContext::newTrace(); + $ecotone->publishEventWithRouting('user.registered', [], metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $outage->armed = true; + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $outage->armed = false; + OutageSimulatingConnection::$databaseDown = false; + + self::assertSame(1, $projection->projected); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'users_list')); + self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + public function test_a_database_outage_inside_the_consumer_transaction_leaves_the_message_to_be_retried(): void { $notifier = new class () { diff --git a/packages/PdoEventSourcing/tests/Projecting/FlowTrackingProjectionTest.php b/packages/PdoEventSourcing/tests/Projecting/FlowTrackingProjectionTest.php new file mode 100644 index 000000000..52f9a4193 --- /dev/null +++ b/packages/PdoEventSourcing/tests/Projecting/FlowTrackingProjectionTest.php @@ -0,0 +1,178 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + */ +declare(strict_types=1); + +namespace Test\Ecotone\EventSourcing\Projecting; + +use Doctrine\DBAL\Cache\QueryCacheProfile; +use Doctrine\DBAL\Connection; +use Doctrine\DBAL\Result; +use Ecotone\Api\Attribute\Around; +use Ecotone\Api\Attribute\Asynchronous; +use Ecotone\Api\Attribute\EventHandler; +use Ecotone\Api\Dbal\DbalConnectionFactory; +use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; +use Ecotone\Api\Dbal\Tracing\TracingGateway; +use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; +use Ecotone\Api\ExtensionObject\ModulePackageList; +use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; +use Ecotone\Api\Interceptor\MethodInvocation; +use Ecotone\Api\Interceptor\Precedence; +use Ecotone\Api\Lite\Test\FlowTestSupport; +use Ecotone\Api\Messaging\MessageHeaders; +use Ecotone\Api\Projecting\FromAggregateStream; +use Ecotone\Api\Projecting\Projection; +use Ecotone\Api\Projecting\ProjectionExecution; +use Ecotone\Test\LicenceTesting; +use Ramsey\Uuid\Uuid; +use RuntimeException; +use Test\Ecotone\EventSourcing\Fixture\Ticket\Command\RegisterTicket; +use Test\Ecotone\EventSourcing\Fixture\Ticket\Event\TicketWasRegistered; +use Test\Ecotone\EventSourcing\Fixture\Ticket\Ticket; +use Test\Ecotone\EventSourcing\Fixture\Ticket\TicketEventConverter; + +/** + * licence Enterprise + * @internal + */ +final class FlowTrackingProjectionTest extends ProjectingTestCase +{ + protected function tearDown(): void + { + ProjectionOutageConnection::$databaseDown = false; + parent::tearDown(); + } + + public function test_a_database_outage_right_after_an_asynchronous_projection_committed_does_not_fail_its_consumer(): void + { + $projection = $this->ticketsProjection(); + $outage = $this->outageRightAfterTheConsumerWork(); + $ecotone = $this->bootstrapEcotone([$projection, $outage]); + $tracingId = Uuid::uuid4()->toString(); + $ecotone->sendCommand(new RegisterTicket('ticket-1', 'User1', 'alert'), metadata: [MessageHeaders::TRACING_ID => $tracingId]); + + $outage->armed = true; + $ecotone->run('async_projection', ExecutionPollingMetadata::createWithTestingSetup()); + $outage->armed = false; + ProjectionOutageConnection::$databaseDown = false; + + self::assertSame(1, $projection->projected); + self::assertTrue($ecotone->getGateway(TracingGateway::class)->isCompletedFor($tracingId, 'flow_tracked_tickets')); + self::assertTrue($ecotone->getGateway(TracingGateway::class)->isCompleted($tracingId)); + } + + public function test_an_asynchronous_projection_spread_over_several_batches_completes_the_flow_once_it_caught_up(): void + { + $projection = $this->ticketsProjection(); + $ecotone = $this->bootstrapEcotone([$projection]); + $ecotone->sendCommand(new RegisterTicket('ticket-1', 'User1', 'alert')); + $ecotone->sendCommand(new RegisterTicket('ticket-2', 'User2', 'alert')); + $tracingId = Uuid::uuid4()->toString(); + $ecotone->sendCommand(new RegisterTicket('ticket-3', 'User3', 'alert'), metadata: [MessageHeaders::TRACING_ID => $tracingId]); + + $ecotone->run('async_projection', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertSame(3, $projection->projected); + self::assertTrue($ecotone->getGateway(TracingGateway::class)->isCompleted($tracingId)); + } + + private function ticketsProjection(): object + { + return new #[Projection('flow_tracked_tickets'), Asynchronous('async_projection'), ProjectionExecution(eventLoadingBatchSize: 1), FromAggregateStream(Ticket::class)] class () { + public int $projected = 0; + + #[EventHandler] + public function whenRegistered(TicketWasRegistered $event): void + { + $this->projected++; + } + }; + } + + private function outageRightAfterTheConsumerWork(): object + { + return new class () { + public bool $armed = false; + + #[Around(precedence: Precedence::DATABASE_TRANSACTION_PRECEDENCE + 2, pointcut: 'Ecotone\\Api\\Attribute\\AsynchronousRunningEndpoint')] + public function goDownAfterTheWork(MethodInvocation $methodInvocation): mixed + { + $result = $methodInvocation->proceed(); + if ($this->armed) { + ProjectionOutageConnection::$databaseDown = true; + } + + return $result; + } + }; + } + + /** + * @param object[] $services + */ + private function bootstrapEcotone(array $services): FlowTestSupport + { + $connectionFactory = new DbalConnectionFactory([ + 'dsn' => getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@127.0.0.1:5432/ecotone', + 'connection' => ['wrapperClass' => ProjectionOutageConnection::class], + ]); + + $ecotone = $this->bootstrapFlowTesting( + classesToResolve: [...array_map(fn (object $service): string => $service::class, $services), Ticket::class, TicketEventConverter::class], + containerOrAvailableServices: [...$services, new TicketEventConverter(), DbalConnectionFactory::class => $connectionFactory], + extensionObjects: [ + FlowTrackingConfiguration::createWithDefaults(), + SimpleMessageChannelBuilder::createQueueChannel('async_projection'), + ], + enabledModules: [ModulePackageList::DBAL_PACKAGE, ModulePackageList::EVENT_SOURCING_PACKAGE], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $ecotone->deleteProjection('flow_tracked_tickets')->initializeProjection('flow_tracked_tickets'); + + return $ecotone; + } +} + +final class ProjectionOutageConnection extends Connection +{ + public static bool $databaseDown = false; + + public function executeQuery(string $sql, array $params = [], array $types = [], ?QueryCacheProfile $qcp = null): Result + { + self::failWhenDown(); + + return parent::executeQuery($sql, $params, $types, $qcp); + } + + public function executeStatement(string $sql, array $params = [], array $types = []): int|string + { + self::failWhenDown(); + + return parent::executeStatement($sql, $params, $types); + } + + private static function failWhenDown(): void + { + if (self::$databaseDown) { + throw new RuntimeException('The database is down'); + } + } +} diff --git a/upgrade-2.0.md b/upgrade-2.0.md index da1986c36..f8e702424 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3130,7 +3130,10 @@ events. Everything is written inside the transaction of the unit of work it describes, a bus call or an asynchronous consumption: the spans buffered during the work, the span that wraps it as it stands at that moment, and the decision -whether the flow is complete. Nothing is written after the commit or after a rollback, so a database that goes down +whether the flow is complete. An asynchronous projection, which commits once per batch, is recorded inside each +batch's transaction, and is marked processed in the batch that finds it caught up. A path run with no transaction at +all, an event bus call from a controller or a command bus with `withTransactionOnCommandBus(false)`, has none to be +inside and writes when its work ends. Nothing is written after the commit or after a rollback, so a database that goes down right after a commit cannot fail a consumer or a command whose work already committed, and an outage inside the transaction rolls back with the work and leaves the message to be retried. A unit of work that fails stores nothing; its failure is visible in OpenTelemetry. Notifications are published from memory once the transaction committed, at From d8965fc6be679738eeec7cda6a11c81c72be69a6 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 33/50] fix(dbal): a multi-tenant application tracks each flow in its tenant's database, inside the tenant's transaction Flow tracking wrote on the default connection while the work committed on the tenant's, so a flow could read as completed before the tenant's data committed, and stayed completed when that commit failed. Spans are now written on the tenant's connection, TracingGateway takes the tenant as metadata (refusing a call without one, naming it), and the retention command runs per tenant with --header. --- packages/Dbal/Api/Tracing/TracingGateway.php | 10 +- .../Dbal/src/Tracing/DbalTracingModule.php | 45 ++++--- .../Dbal/src/Tracing/StoredTracingGateway.php | 33 ++++- .../FlowTrackingConformanceTest.php | 117 +++++++++++++----- upgrade-2.0.md | 7 +- 5 files changed, 154 insertions(+), 58 deletions(-) diff --git a/packages/Dbal/Api/Tracing/TracingGateway.php b/packages/Dbal/Api/Tracing/TracingGateway.php index 7283b8116..96f50095a 100644 --- a/packages/Dbal/Api/Tracing/TracingGateway.php +++ b/packages/Dbal/Api/Tracing/TracingGateway.php @@ -4,19 +4,21 @@ namespace Ecotone\Api\Dbal\Tracing; +use Ecotone\Api\Attribute\PropagateHeaders; use Ecotone\Api\Scheduling\Duration; /** * licence Enterprise * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). */ +#[PropagateHeaders] interface TracingGateway { - public function isCompleted(string $tracingId): bool; + public function isCompleted(string $tracingId, array $metadata = []): bool; - public function isCompletedFor(string $tracingId, string $destination): bool; + public function isCompletedFor(string $tracingId, string $destination, array $metadata = []): bool; - public function await(string $tracingId, Duration $timeout): void; + public function await(string $tracingId, Duration $timeout, array $metadata = []): void; - public function awaitFor(string $tracingId, string $destination, Duration $timeout): void; + public function awaitFor(string $tracingId, string $destination, Duration $timeout, array $metadata = []): void; } diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index c7eca06ef..ba3103954 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -17,7 +17,6 @@ use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; -use Ecotone\Api\ExtensionObject\ConnectionReference; use Ecotone\Api\ExtensionObject\ModulePackageList; use Ecotone\Api\ExtensionObject\ServiceConfiguration; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; @@ -45,6 +44,7 @@ use Ecotone\Messaging\Gateway\MessagingEntrypointService; use Ecotone\Messaging\Handler\Gateway\GatewayProxyBuilder; use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayHeaderBuilder; +use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayHeadersBuilder; use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayPayloadBuilder; use Ecotone\Messaging\Handler\InterfaceToCallRegistry; use Ecotone\Messaging\Handler\Logger\LoggingGateway; @@ -163,7 +163,14 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnFlowStepCompleted::CHANNEL)); } if ($flowTracking->tracingGateway) { - self::registerTracingGateway($messagingConfiguration, StoredTracingGateway::class, new Definition(StoredTracingGateway::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $this->pollingProjectionNames])); + $multiTenancy = $flowTracking->inMemory ? null : self::multiTenancyWithoutDefaultConnectionOf($extensionObjects); + self::registerTracingGateway($messagingConfiguration, StoredTracingGateway::class, new Definition(StoredTracingGateway::class, [ + new Reference(TraceStore::class), + new Reference(EcotoneClockInterface::class), + $this->pollingProjectionNames, + $multiTenancy?->tenantHeaderName, + $multiTenancy === null ? null : (string) array_key_first($multiTenancy->tenantToConnectionMapping), + ])); } else { self::registerTracingGateway($messagingConfiguration, UnavailableTracingGateway::class, new Definition(UnavailableTracingGateway::class, ['TracingGateway is turned off by FlowTrackingConfiguration::withTracingGateway(false). Remove that call to ask whether a flow completed.'])); } @@ -232,10 +239,10 @@ private static function registerTracingGateway(Configuration $messagingConfigura { $messagingConfiguration->registerServiceDefinition($serviceClass, $service); foreach ([ - 'isCompleted' => [GatewayPayloadBuilder::create('tracingId')], - 'isCompletedFor' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('destination', StoredTracingGateway::DESTINATION_HEADER)], - 'await' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('timeout', StoredTracingGateway::TIMEOUT_HEADER)], - 'awaitFor' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('destination', StoredTracingGateway::DESTINATION_HEADER), GatewayHeaderBuilder::create('timeout', StoredTracingGateway::TIMEOUT_HEADER)], + 'isCompleted' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeadersBuilder::create('metadata')], + 'isCompletedFor' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('destination', StoredTracingGateway::DESTINATION_HEADER), GatewayHeadersBuilder::create('metadata')], + 'await' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('timeout', StoredTracingGateway::TIMEOUT_HEADER), GatewayHeadersBuilder::create('metadata')], + 'awaitFor' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('destination', StoredTracingGateway::DESTINATION_HEADER), GatewayHeaderBuilder::create('timeout', StoredTracingGateway::TIMEOUT_HEADER), GatewayHeadersBuilder::create('metadata')], ] as $method => $parameterConverters) { $channel = 'ecotone.tracing.gateway.' . $method; $messagingConfiguration @@ -250,24 +257,22 @@ private static function registerTracingGateway(Configuration $messagingConfigura private static function connectionReferenceOf(array $extensionObjects): string { $dbalConfiguration = ExtensionObjectResolver::resolveUnique(DbalConfiguration::class, $extensionObjects, DbalConfiguration::createWithDefaults()); - $connectionReference = $dbalConfiguration->defaultConnectionReferenceNames[0] ?? DbalConnectionReference::DEFAULT; - foreach (ExtensionObjectResolver::resolve(MultiTenantConfiguration::class, $extensionObjects) as $multiTenantConfiguration) { - if ($multiTenantConfiguration->getReferenceName() !== $connectionReference) { - continue; - } - if ($multiTenantConfiguration->defaultConnectionName === null) { - $flowTracking = self::flowTrackingOf($extensionObjects); - if ($flowTracking !== null && ! $flowTracking->inMemory) { - throw ConfigurationException::create('Flow tracking keeps one record per flow across tenants, and TracingGateway is asked without a tenant, so it needs a connection that does not depend on the tenant. Use MultiTenantConfiguration::createWithDefaultConnection() to name it.'); - } - return $connectionReference; - } + return $dbalConfiguration->defaultConnectionReferenceNames[0] ?? DbalConnectionReference::DEFAULT; + } - return $multiTenantConfiguration->defaultConnectionName instanceof ConnectionReference ? $multiTenantConfiguration->defaultConnectionName->getReferenceName() : $multiTenantConfiguration->defaultConnectionName; + /** + * @param object[] $extensionObjects + */ + private static function multiTenancyWithoutDefaultConnectionOf(array $extensionObjects): ?MultiTenantConfiguration + { + foreach (ExtensionObjectResolver::resolve(MultiTenantConfiguration::class, $extensionObjects) as $multiTenantConfiguration) { + if ($multiTenantConfiguration->getReferenceName() === self::connectionReferenceOf($extensionObjects) && $multiTenantConfiguration->defaultConnectionName === null) { + return $multiTenantConfiguration; + } } - return $connectionReference; + return null; } /** diff --git a/packages/Dbal/src/Tracing/StoredTracingGateway.php b/packages/Dbal/src/Tracing/StoredTracingGateway.php index 59abf1f7a..d976e56fd 100644 --- a/packages/Dbal/src/Tracing/StoredTracingGateway.php +++ b/packages/Dbal/src/Tracing/StoredTracingGateway.php @@ -5,10 +5,12 @@ namespace Ecotone\Dbal\Tracing; use Ecotone\Api\Attribute\Header; +use Ecotone\Api\Attribute\Headers; use Ecotone\Api\Dbal\Tracing\FlowTrackingTimedOut; use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Scheduling\Duration; use Ecotone\Messaging\Config\ConfigurationException; +use Ecotone\Messaging\Support\InvalidArgumentException; use Ecotone\Tracing\TracingId; /** @@ -42,25 +44,30 @@ public function __construct( private TraceStore $traceStore, private EcotoneClockInterface $clock, private array $pollingProjectionNames, + private ?string $tenantHeaderName, + private ?string $exampleTenant, ) { } - public function isCompleted(string $tracingId): bool + public function isCompleted(string $tracingId, #[Headers] array $headers): bool { + $this->refuseMissingTenant('isCompleted($tracingId', $headers); $tracingId = TracingId::normalize($tracingId); return $this->traceStore->progressOf($tracingId)->isCompleted(); } - public function isCompletedFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] string $destination): bool + public function isCompletedFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] string $destination, #[Headers] array $headers): bool { + $this->refuseMissingTenant('isCompletedFor($tracingId, $destination', $headers); $tracingId = TracingId::normalize($tracingId); $this->refusePollingProjection($destination); return $this->traceStore->progressOf($tracingId)->isCompletedFor($destination); } - public function await(string $tracingId, #[Header(self::TIMEOUT_HEADER)] Duration $timeout): void + public function await(string $tracingId, #[Header(self::TIMEOUT_HEADER)] Duration $timeout, #[Headers] array $headers): void { + $this->refuseMissingTenant('await($tracingId, $timeout', $headers); $tracingId = TracingId::normalize($tracingId); $deadline = $this->clock->now()->add($timeout); while (true) { @@ -77,8 +84,9 @@ public function await(string $tracingId, #[Header(self::TIMEOUT_HEADER)] Duratio } } - public function awaitFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] string $destination, #[Header(self::TIMEOUT_HEADER)] Duration $timeout): void + public function awaitFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] string $destination, #[Header(self::TIMEOUT_HEADER)] Duration $timeout, #[Headers] array $headers): void { + $this->refuseMissingTenant('awaitFor($tracingId, $destination, $timeout', $headers); $tracingId = TracingId::normalize($tracingId); $this->refusePollingProjection($destination); $deadline = $this->clock->now()->add($timeout); @@ -98,6 +106,23 @@ public function awaitFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] } } + /** + * @param array $headers + */ + private function refuseMissingTenant(string $call, array $headers): void + { + if ($this->tenantHeaderName === null || isset($headers[$this->tenantHeaderName])) { + return; + } + + throw InvalidArgumentException::create(sprintf( + "TracingGateway reads a flow from the database of the tenant it ran for, and this call names no tenant. Pass the tenant as metadata, for example %s, metadata: ['%s' => '%s']).", + $call, + $this->tenantHeaderName, + $this->exampleTenant, + )); + } + private function refusePollingProjection(string $destination): void { if (in_array($destination, $this->pollingProjectionNames, true)) { diff --git a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php index 4601b5819..f84001da7 100644 --- a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php +++ b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php @@ -29,6 +29,7 @@ use Ecotone\Api\ExtensionObject\ModulePackageList; use Ecotone\Api\ExtensionObject\ServiceConfiguration; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; +use Ecotone\Api\ExtensionObject\TestConfiguration; use Ecotone\Api\Gateway\EventBus; use Ecotone\Api\Interceptor\MethodInvocation; use Ecotone\Api\Interceptor\Precedence; @@ -40,7 +41,6 @@ use Ecotone\Api\Scheduling\TimeSpan; use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\SpanStatus; -use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Test\LicenceTesting; use PHPUnit\Framework\Attributes\DataProvider; use RuntimeException; @@ -423,44 +423,66 @@ public function test_a_channel_published_through_the_outbox_completes_once_its_c self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } - public function test_flow_tracking_follows_the_default_connection_of_a_multi_tenant_application(): void + public function test_flow_tracking_stores_each_flow_in_the_database_of_its_tenant(): void { - $ecotone = $this->bootstrapFlowTesting( - classesToResolve: [$this->users()::class, $this->asynchronousNotifier()::class], - containerOrAvailableServices: [$this->users(), $this->asynchronousNotifier(), 'tenant_a_connection' => $this->getConnectionFactory(), 'shared_connection' => $this->getConnectionFactory()], - extensionObjects: [ - FlowTrackingConfiguration::createWithDefaults(), - MultiTenantConfiguration::createWithDefaultConnection('tenant', ['tenant_a' => 'tenant_a_connection'], 'shared_connection'), - SimpleMessageChannelBuilder::createQueueChannel('async_channel'), - ], - enabledModules: [ModulePackageList::DBAL_PACKAGE], - licenceKey: LicenceTesting::VALID_LICENCE, - )->setupDatabaseTables(); + $ecotone = $this->bootstrapMultiTenant([$this->users(), $this->asynchronousNotifier()], $this->connectionForTenantA()); $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent(), 'tenant' => 'tenant_a']); - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId, metadata: ['tenant' => 'tenant_a'])); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId, metadata: ['tenant' => 'tenant_a'])); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier', metadata: ['tenant' => 'tenant_a'])); + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId, metadata: ['tenant' => 'tenant_b'])); + $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100), metadata: ['tenant' => 'tenant_a']); } - public function test_flow_tracking_in_a_multi_tenant_application_without_a_default_connection_is_refused_at_bootstrap(): void + public function test_a_tenant_flow_is_written_inside_the_tenant_transaction_and_rolls_back_with_it(): void { - $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('Flow tracking keeps one record per flow across tenants, and TracingGateway is asked without a tenant, so it needs a connection that does not depend on the tenant. Use MultiTenantConfiguration::createWithDefaultConnection() to name it.'); + $tenantA = new DbalConnectionFactory([ + 'dsn' => getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone', + 'connection' => ['wrapperClass' => CommitFailingConnection::class], + ]); + $ecotone = $this->bootstrapMultiTenant([$this->users()], $tenantA); + $trace = SpanContext::newTrace(); - $this->bootstrapFlowTesting( - classesToResolve: [$this->users()::class], - containerOrAvailableServices: [$this->users(), 'tenant_a_connection' => $this->getConnectionFactory()], - extensionObjects: [ - FlowTrackingConfiguration::createWithDefaults(), - MultiTenantConfiguration::create('tenant', ['tenant_a' => 'tenant_a_connection']), - ], - enabledModules: [ModulePackageList::DBAL_PACKAGE], - licenceKey: LicenceTesting::VALID_LICENCE, - ); + CommitFailingConnection::$failsCommit = true; + try { + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent(), 'tenant' => 'tenant_a']); + self::fail('The tenant commit was expected to fail'); + } catch (RuntimeException $failure) { + self::assertSame('The commit failed', $failure->getMessage()); + } finally { + CommitFailingConnection::$failsCommit = false; + } + + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId, metadata: ['tenant' => 'tenant_a'])); + } + + public function test_expired_flows_are_removed_from_the_database_of_the_tenant_the_console_command_names(): void + { + $ecotone = $this->bootstrapMultiTenant([$this->users()], $this->connectionForTenantA()); + $ofTenantA = SpanContext::newTrace(); + $ofTenantB = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $ofTenantA->toTraceparent(), 'tenant' => 'tenant_a']); + $ecotone->sendCommandWithRouting('user.register', '2', metadata: [MessageHeaders::TRACEPARENT => $ofTenantB->toTraceparent(), 'tenant' => 'tenant_b']); + $ecotone->advanceTimeBy(TimeSpan::withDays(8)); + + $ecotone->runConsoleCommand('ecotone:tracing:remove-expired-traces', ['header' => ['tenant:tenant_a']]); + + self::assertFalse($this->completion($ecotone)->isCompleted($ofTenantA->traceId, metadata: ['tenant' => 'tenant_a'])); + self::assertTrue($this->completion($ecotone)->isCompleted($ofTenantB->traceId, metadata: ['tenant' => 'tenant_b'])); + } + + public function test_asking_the_tracing_gateway_of_a_multi_tenant_application_without_a_tenant_names_the_metadata_it_needs(): void + { + $ecotone = $this->bootstrapMultiTenant([$this->users()], $this->connectionForTenantA()); + + $this->expectExceptionMessage("TracingGateway reads a flow from the database of the tenant it ran for, and this call names no tenant. Pass the tenant as metadata, for example isCompleted(\$tracingId, metadata: ['tenant' => 'tenant_a'])."); + + $this->completion($ecotone)->isCompleted(SpanContext::newTrace()->traceId); } public function test_a_database_outage_right_after_the_consumer_committed_does_not_fail_the_consumer(): void @@ -627,6 +649,29 @@ public function register(string $userId, EventBus $eventBus): void }; } + /** + * @param object[] $services + */ + private function bootstrapMultiTenant(array $services, DbalConnectionFactory $tenantAConnection): FlowTestSupport + { + $ecotone = $this->bootstrapFlowTesting( + classesToResolve: array_map(fn (object $service): string => $service::class, $services), + containerOrAvailableServices: [...$services, 'tenant_a_connection' => $tenantAConnection, 'tenant_b_connection' => $this->connectionForTenantB()], + extensionObjects: [ + FlowTrackingConfiguration::createWithDefaults(), + MultiTenantConfiguration::create('tenant', ['tenant_a' => 'tenant_a_connection', 'tenant_b' => 'tenant_b_connection']), + SimpleMessageChannelBuilder::createQueueChannel('async_channel'), + ], + enabledModules: [ModulePackageList::DBAL_PACKAGE], + testConfiguration: TestConfiguration::createWithDefaults()->withDatabaseTablesSetUpOnBootstrap(false), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $ecotone->runConsoleCommand('ecotone:migration:database:delete', ['force' => true]); + $ecotone->runConsoleCommand('ecotone:migration:database:setup', ['initialize' => true]); + + return $ecotone; + } + private function outageRightAfterTheTransaction(): object { return new class () { @@ -728,3 +773,19 @@ private static function failWhenDown(): void } } } + +final class CommitFailingConnection extends Connection +{ + public static bool $failsCommit = false; + + public function commit(): void + { + if (self::$failsCommit && $this->getTransactionNestingLevel() === 1) { + $this->rollBack(); + + throw new RuntimeException('The commit failed'); + } + + parent::commit(); + } +} diff --git a/upgrade-2.0.md b/upgrade-2.0.md index f8e702424..ee1fbd173 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3148,8 +3148,11 @@ same flow waits for the first one's commit and, on PostgreSQL and SQLite, reads across channels in the W3C `tracestate` as `ecotone=carried` next to any other vendor's entry, so ordinary traffic costs nothing. `withRetention()` sets how long `ecotone:tracing:remove-expired-traces` keeps completed flows, seven days by default; a flow still pending is kept twice as long, so a message delayed past the retention still completes -it. A multi-tenant application tracks flows on the connection `MultiTenantConfiguration::createWithDefaultConnection()` -names, because `TracingGateway` is asked without a tenant, and is refused at bootstrap without one. +it. A multi-tenant application tracks each flow in the database of its tenant, inside the tenant's transaction, so a +flow is never complete before the tenant's data committed. `TracingGateway` is then asked with the tenant, +`$tracingGateway->await($tracingId, $timeout, metadata: ['tenant' => 'tenant_a'])`, and a call without one is refused +naming that metadata; the retention command runs once per tenant, +`ecotone:tracing:remove-expired-traces --header=tenant:tenant_a`. ```php #[ServiceContext] From 7fb5675645e1913d267b417340ad3e7b918bac14 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 34/50] fix(dbal): a destination is completed only once nothing pending upstream of it can still send it another message isCompletedFor() and awaitFor() returned as soon as everything sent to the destination so far was processed, so a later message the same flow was still going to send it was not waited for. A pending message now blocks a destination unless the destination's own handling caused it, followed through each span's parentId. --- packages/Dbal/src/Tracing/TraceLedger.php | 6 +- packages/Dbal/src/Tracing/TraceProgress.php | 57 +++++++++++++++-- .../FlowTrackingConformanceTest.php | 61 +++++++++++++++++++ .../Integration/Tracing/FlowTrackingTest.php | 7 ++- upgrade-2.0.md | 7 ++- 5 files changed, 126 insertions(+), 12 deletions(-) diff --git a/packages/Dbal/src/Tracing/TraceLedger.php b/packages/Dbal/src/Tracing/TraceLedger.php index 34cca5737..ef253ba36 100644 --- a/packages/Dbal/src/Tracing/TraceLedger.php +++ b/packages/Dbal/src/Tracing/TraceLedger.php @@ -40,11 +40,15 @@ public static function progressOf(string $traceId, array $spans): TraceProgress } $rootEnded = false; + $causedBy = []; $sent = []; $processed = []; $sentDeliveries = []; $processedDeliveries = []; foreach ($spans as $span) { + if (isset($span->attributes[MessageHeaders::MESSAGE_ID], $span->attributes[MessageHeaders::PARENT_MESSAGE_ID])) { + $causedBy[(string) $span->attributes[MessageHeaders::MESSAGE_ID]] = (string) $span->attributes[MessageHeaders::PARENT_MESSAGE_ID]; + } if ($span->attributes[SpanAttribute::TRACE_ROOT] ?? false) { $rootEnded = true; } @@ -81,7 +85,7 @@ public static function progressOf(string $traceId, array $spans): TraceProgress } } - return new TraceProgress($traceId, true, $rootEnded, $sent, $processed, $sentDeliveries, $processedDeliveries); + return new TraceProgress($traceId, true, $rootEnded, $sent, $processed, $sentDeliveries, $processedDeliveries, $causedBy); } public static function destinationOf(Span $span): ?string diff --git a/packages/Dbal/src/Tracing/TraceProgress.php b/packages/Dbal/src/Tracing/TraceProgress.php index 14d84317b..f107c0614 100644 --- a/packages/Dbal/src/Tracing/TraceProgress.php +++ b/packages/Dbal/src/Tracing/TraceProgress.php @@ -27,10 +27,11 @@ final class TraceProgress { /** - * @param array> $sent message ids dispatched to each destination - * @param array> $processed message ids each destination processed - * @param array> $sentDeliveries message ids sent to each channel - * @param array> $processedDeliveries message ids consumed from each channel + * @param array> $sent + * @param array> $processed + * @param array> $sentDeliveries + * @param array> $processedDeliveries + * @param array $causedBy */ public function __construct( public readonly string $traceId, @@ -40,12 +41,13 @@ public function __construct( public readonly array $processed, public readonly array $sentDeliveries, public readonly array $processedDeliveries, + public readonly array $causedBy, ) { } public static function unknown(string $traceId): self { - return new self($traceId, false, false, [], [], [], []); + return new self($traceId, false, false, [], [], [], [], []); } public function isCompleted(): bool @@ -59,8 +61,51 @@ public function isCompletedFor(string $destination): bool return false; } $processed = $this->processed[$destination] ?? []; + if ($processed === [] || array_diff_key($this->sent[$destination] ?? [], $processed) !== []) { + return false; + } + + foreach ($this->pendingMessageIds() as $pendingMessageId) { + if (! $this->isCausedByOneOf($pendingMessageId, $processed)) { + return false; + } + } + + return true; + } + + /** + * @return string[] + */ + private function pendingMessageIds(): array + { + $pending = []; + foreach ($this->sent as $destination => $sent) { + $pending = [...$pending, ...array_keys(array_diff_key($sent, $this->processed[$destination] ?? []))]; + } + foreach ($this->sentDeliveries as $channel => $sent) { + $pending = [...$pending, ...array_keys(array_diff_key($sent, $this->processedDeliveries[$channel] ?? []))]; + } + + return array_map('strval', $pending); + } + + /** + * @param array $causes + */ + private function isCausedByOneOf(string $messageId, array $causes): bool + { + $visited = [$messageId => true]; + $current = $this->causedBy[$messageId] ?? null; + while ($current !== null && ! isset($visited[$current])) { + if (isset($causes[$current])) { + return true; + } + $visited[$current] = true; + $current = $this->causedBy[$current] ?? null; + } - return $processed !== [] && array_diff_key($this->sent[$destination] ?? [], $processed) === []; + return false; } /** diff --git a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php index f84001da7..869544058 100644 --- a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php +++ b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php @@ -170,6 +170,67 @@ public function notify(string $userId): void self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); } + #[DataProvider('implementations')] + public function test_a_destination_is_not_completed_while_work_upstream_of_it_can_still_send_it_more(string $implementation): void + { + $audit = new class () { + #[Asynchronous('audit_channel')] + #[EventHandler('user.*', endpointId: 'audit.recorder')] + public function record(string $userId): void + { + } + }; + $welcomer = new class () { + #[Asynchronous('welcome_channel')] + #[EventHandler('user.registered', endpointId: 'user.welcomer')] + public function welcome(string $userId, EventBus $eventBus): void + { + $eventBus->publishWithRouting('user.welcomed', $userId); + } + }; + $ecotone = $this->bootstrap($implementation, [$this->users(), $audit, $welcomer], [SimpleMessageChannelBuilder::createQueueChannel('audit_channel'), SimpleMessageChannelBuilder::createQueueChannel('welcome_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $ecotone->run('audit_channel', ExecutionPollingMetadata::createWithTestingSetup()); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'audit.recorder')); + + $ecotone->run('welcome_channel', ExecutionPollingMetadata::createWithTestingSetup()); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'audit.recorder')); + + $ecotone->run('audit_channel', ExecutionPollingMetadata::createWithTestingSetup()); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'audit.recorder')); + } + + #[DataProvider('implementations')] + public function test_a_destination_is_completed_while_only_work_it_started_itself_is_still_pending(string $implementation): void + { + $notifier = new class () { + #[Asynchronous('async_channel')] + #[EventHandler('user.registered', endpointId: 'user.notifier')] + public function notify(string $userId, EventBus $eventBus): void + { + $eventBus->publishWithRouting('user.notified', $userId); + } + }; + $archiver = new class () { + #[Asynchronous('archive_channel')] + #[EventHandler('user.notified', endpointId: 'notification.archiver')] + public function archive(string $userId): void + { + } + }; + $ecotone = $this->bootstrap($implementation, [$this->users(), $notifier, $archiver], [SimpleMessageChannelBuilder::createQueueChannel('async_channel'), SimpleMessageChannelBuilder::createQueueChannel('archive_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'notification.archiver')); + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + } + #[DataProvider('implementations')] public function test_traces_older_than_the_retention_are_removed_by_the_console_command(string $implementation): void { diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index 899e4377e..f6ab3a0d2 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -533,7 +533,7 @@ public function place(string $orderId): void $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } - public function test_an_output_channel_after_an_asynchronous_handler_does_not_complete_that_handler_for_a_delivery_still_queued(): void + public function test_an_output_channel_after_an_asynchronous_handler_completes_neither_while_a_delivery_that_feeds_them_is_still_queued(): void { $notifier = new class () { #[Asynchronous('async_channel')] @@ -555,12 +555,13 @@ public function record(string $userId): void $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 1)); - $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'audit.recording')); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'audit.recording')); $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 1)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'audit.recording')); $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } @@ -590,11 +591,11 @@ public function deliver(string $text): void $ecotone->getGateway(TracedNotificationGateway::class)->send('hello', [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'notification.sending')); $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'notification.sending')); $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } diff --git a/upgrade-2.0.md b/upgrade-2.0.md index ee1fbd173..d45c19993 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3104,8 +3104,11 @@ reaches, synchronously or asynchronously, across processes. What it tracks is ke Two features are on by default and each can be turned off while the flows are still tracked: - **`TracingGateway`** answers `isCompleted($tracingId)`, `isCompletedFor($tracingId, $destination)` for one - `endpointId` or projection name, and `await()` / `awaitFor()`, which poll until then or throw - `FlowTrackingTimedOut` naming the handlers and channels still pending, pointing to the dead letter when a + `endpointId` or projection name, and `await()` / `awaitFor()`, which poll until then. A destination is + complete once it processed every message the flow sent it and nothing still pending upstream of it could send it + another: only work its own handling caused, through a new message, may still be pending. A message it forwards + unchanged, through an output channel, is waited for too, since it carries the same message id as an event's copy + queued for a sibling handler. Past the timeout, the awaits throw `FlowTrackingTimedOut` naming the handlers and channels still pending, pointing to the dead letter when a handler failed, and naming a flow nobody recorded or a destination the flow never reached. `withTracingGateway(false)` turns it off; the gateway stays injectable and every call names that setting. - **Notifications**: a method with `#[OnFlowCompleted]` receives a `FlowCompleted` each time its flow reaches From 6031e4bda31e8e04b7e2d2b07e3e25fdc2cd9ead Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 35/50] fix(tracing): a tracingId passed to a bus or gateway from inside a handler is refused, naming the flow the call belongs to A nested call's tracingId was silently ignored: the call joined the outer flow, and awaiting the new id timed out with a message that still said traceparent. The bus and gateway entries now refuse it, and the timeout messages speak of flows and tracingId. --- .../Dbal/Api/Tracing/FlowTrackingTimedOut.php | 8 +-- .../Integration/Tracing/FlowTrackingTest.php | 71 +++++++++++++++++-- .../src/Tracing/TracingInterceptor.php | 27 +++++-- upgrade-2.0.md | 4 +- 4 files changed, 95 insertions(+), 15 deletions(-) diff --git a/packages/Dbal/Api/Tracing/FlowTrackingTimedOut.php b/packages/Dbal/Api/Tracing/FlowTrackingTimedOut.php index b72df7762..280605961 100644 --- a/packages/Dbal/Api/Tracing/FlowTrackingTimedOut.php +++ b/packages/Dbal/Api/Tracing/FlowTrackingTimedOut.php @@ -22,7 +22,7 @@ final class FlowTrackingTimedOut extends RuntimeException public static function forTrace(string $traceId, Duration $timeout, array $pendingDestinations): self { return new self(sprintf( - 'Trace %s did not complete within %d milliseconds. Still pending: %s. Run the consumer of the pending channel, replay the message from the dead letter if its handler failed, or await with a longer timeout.', + 'Flow %s did not complete within %d milliseconds. Still pending: %s. Run the consumer of the pending channel, replay the message from the dead letter if its handler failed, or await with a longer timeout.', $traceId, $timeout->inMilliseconds(), implode(', ', $pendingDestinations), @@ -32,7 +32,7 @@ public static function forTrace(string $traceId, Duration $timeout, array $pendi public static function forUnrecordedTrace(string $traceId, Duration $timeout): self { return new self(sprintf( - 'No span was recorded under trace %s within %d milliseconds. Send the command with that traceparent in its metadata, and check that the process handling it stores traces.', + 'Nothing was recorded for flow %s within %d milliseconds. Send the command with that tracingId in its metadata, and check that the application handling it registers FlowTrackingConfiguration.', $traceId, $timeout->inMilliseconds(), )); @@ -41,7 +41,7 @@ public static function forUnrecordedTrace(string $traceId, Duration $timeout): s public static function forDestination(string $traceId, string $destination, Duration $timeout): self { return new self(sprintf( - 'Trace %s did not complete for %s within %d milliseconds. Run the consumer of the channel that handler is attached to, replay the message from the dead letter if the handler failed, or await with a longer timeout.', + 'Flow %s did not complete for %s within %d milliseconds. Run the consumer of the channel that handler is attached to, replay the message from the dead letter if the handler failed, or await with a longer timeout.', $traceId, $destination, $timeout->inMilliseconds(), @@ -54,7 +54,7 @@ public static function forDestination(string $traceId, string $destination, Dura public static function forUnreachedDestination(string $traceId, string $destination, Duration $timeout, array $reachedDestinations): self { return new self(sprintf( - 'Trace %s was neither sent to nor processed by %s within %d milliseconds. Check the endpointId or projection name; the trace reached: %s.', + 'Flow %s was neither sent to nor processed by %s within %d milliseconds. Check the endpointId or projection name; the flow reached: %s.', $traceId, $destination, $timeout->inMilliseconds(), diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index f6ab3a0d2..b764987a4 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -121,7 +121,7 @@ public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pen $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $this->expectException(FlowTrackingTimedOut::class); - $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the pending channel, replay the message from the dead letter if its handler failed, or await with a longer timeout."); + $this->expectExceptionMessage("Flow {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the pending channel, replay the message from the dead letter if its handler failed, or await with a longer timeout."); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); } @@ -625,7 +625,7 @@ public function notify(string $userId): void $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); $this->expectException(FlowTrackingTimedOut::class); - $this->expectExceptionMessage("Trace {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the pending channel, replay the message from the dead letter if its handler failed, or await with a longer timeout."); + $this->expectExceptionMessage("Flow {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the pending channel, replay the message from the dead letter if its handler failed, or await with a longer timeout."); $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); } @@ -635,7 +635,7 @@ public function test_awaiting_a_trace_nobody_recorded_says_so(): void $ecotone = $this->bootstrap([$this->users()]); $this->expectException(FlowTrackingTimedOut::class); - $this->expectExceptionMessage('No span was recorded under trace 0af7651916cd43dd8448eb211c80319c within 100 milliseconds. Send the command with that traceparent in its metadata, and check that the process handling it stores traces.'); + $this->expectExceptionMessage('Nothing was recorded for flow 0af7651916cd43dd8448eb211c80319c within 100 milliseconds. Send the command with that tracingId in its metadata, and check that the application handling it registers FlowTrackingConfiguration.'); $this->completion($ecotone)->await('0af7651916cd43dd8448eb211c80319c', Duration::milliseconds(100)); } @@ -647,7 +647,7 @@ public function test_awaiting_a_destination_the_trace_never_reached_names_it_as_ $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $this->expectException(FlowTrackingTimedOut::class); - $this->expectExceptionMessage("Trace {$trace->traceId} was neither sent to nor processed by user.notifierr within 100 milliseconds. Check the endpointId or projection name; the trace reached: user.registration."); + $this->expectExceptionMessage("Flow {$trace->traceId} was neither sent to nor processed by user.notifierr within 100 milliseconds. Check the endpointId or projection name; the flow reached: user.registration."); $this->completion($ecotone)->awaitFor($trace->traceId, 'user.notifierr', Duration::milliseconds(100)); } @@ -861,6 +861,69 @@ public function test_a_message_carrying_both_a_tracing_id_and_a_traceparent_is_r $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '4bf92f3577b34da6a3ce929d0e0e4736', MessageHeaders::TRACEPARENT => SpanContext::newTrace()->toTraceparent()]); } + public function test_a_tracing_id_on_a_command_sent_from_inside_a_handler_is_refused_naming_the_flow_it_belongs_to(): void + { + $users = new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId, CommandBus $commandBus): void + { + $commandBus->sendWithRouting('user.activate', $userId, metadata: [MessageHeaders::TRACING_ID => '0af7651916cd43dd8448eb211c80319c']); + } + + #[CommandHandler('user.activate', endpointId: 'user.activation')] + public function activate(string $userId): void + { + } + }; + $ecotone = $this->bootstrap([$users]); + + $this->expectExceptionMessage('tracingId names the flow a message starts, but this message is sent from inside flow 4bf92f3577b34da6a3ce929d0e0e4736, so it belongs to that flow already. Await 4bf92f3577b34da6a3ce929d0e0e4736 with TracingGateway, or send the message from outside a handler, where a flow starts.'); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '4bf92f3577b34da6a3ce929d0e0e4736']); + } + + public function test_a_tracing_id_on_an_event_published_from_inside_a_handler_is_refused_naming_the_flow_it_belongs_to(): void + { + $users = new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId, EventBus $eventBus): void + { + $eventBus->publishWithRouting('user.registered', $userId, metadata: [MessageHeaders::TRACING_ID => '0af7651916cd43dd8448eb211c80319c']); + } + }; + $ecotone = $this->bootstrap([$users]); + + $this->expectExceptionMessage('tracingId names the flow a message starts, but this message is sent from inside flow 4bf92f3577b34da6a3ce929d0e0e4736, so it belongs to that flow already. Await 4bf92f3577b34da6a3ce929d0e0e4736 with TracingGateway, or send the message from outside a handler, where a flow starts.'); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '4bf92f3577b34da6a3ce929d0e0e4736']); + } + + public function test_a_tracing_id_on_a_gateway_call_made_from_inside_a_handler_is_refused_naming_the_flow_it_belongs_to(): void + { + $users = new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId, #[Reference] TracedNotificationGateway $notifications): void + { + $notifications->send('welcome', [MessageHeaders::TRACING_ID => '0af7651916cd43dd8448eb211c80319c']); + } + + #[InternalHandler('notification.send', endpointId: 'notification.sending')] + public function sendNotification(string $text): void + { + } + }; + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$users::class, TracedNotificationGateway::class], + [$users], + [FlowTrackingConfiguration::createWithDefaults()], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $this->expectExceptionMessage('tracingId names the flow a message starts, but this message is sent from inside flow 4bf92f3577b34da6a3ce929d0e0e4736, so it belongs to that flow already. Await 4bf92f3577b34da6a3ce929d0e0e4736 with TracingGateway, or send the message from outside a handler, where a flow starts.'); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '4bf92f3577b34da6a3ce929d0e0e4736']); + } + public function test_an_invalid_tracing_id_is_refused_naming_the_expected_format(): void { $ecotone = $this->bootstrap([$this->users()]); diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 94f165ea6..045d83431 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -38,17 +38,17 @@ public function __construct( public function traceCommandBus(MethodInvocation $methodInvocation, Message $message): mixed { - return $this->trace('send command', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); + return $this->traceEntry('send command', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); } public function traceEventBus(MethodInvocation $methodInvocation, Message $message): mixed { - return $this->trace('send event', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); + return $this->traceEntry('send event', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); } public function traceQueryBus(MethodInvocation $methodInvocation, Message $message): mixed { - return $this->trace('send query', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); + return $this->traceEntry('send query', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); } public function traceCommandHandler(MethodInvocation $methodInvocation, Message $message, CommandHandler $endpoint): mixed @@ -99,7 +99,7 @@ public function traceProjection(MethodInvocation $methodInvocation, Message $mes public function traceMessageGateway(MethodInvocation $methodInvocation, Message $message, MessageGateway $gateway): mixed { - return $this->trace('send ' . $gateway->requestChannel, $methodInvocation, $message, SpanKind::Server, [ + return $this->traceEntry('send ' . $gateway->requestChannel, $methodInvocation, $message, SpanKind::Server, [ SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND, SpanAttribute::MESSAGING_DESTINATION_NAME => $gateway->requestChannel, ]); @@ -107,12 +107,12 @@ public function traceMessageGateway(MethodInvocation $methodInvocation, Message public function traceOrchestratorGateway(MethodInvocation $methodInvocation, Message $message): mixed { - return $this->trace('send orchestrator', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); + return $this->traceEntry('send orchestrator', $methodInvocation, $message, SpanKind::Server, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); } public function traceDistributedBus(MethodInvocation $methodInvocation, Message $message): mixed { - return $this->trace('send distributed', $methodInvocation, $message, SpanKind::Producer, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); + return $this->traceEntry('send distributed', $methodInvocation, $message, SpanKind::Producer, [SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND]); } public function provideContextForDistributedBus(Message $message): Message @@ -240,6 +240,21 @@ private static function shortNameOf(string $classAndMethod): string return substr($class, (int) strrpos('\\' . $class, '\\')) . '::' . $method; } + /** + * @param array $attributes + */ + private function traceEntry(string $name, MethodInvocation $methodInvocation, Message $message, SpanKind $kind, array $attributes): mixed + { + $currentSpan = $this->tracer->currentSpan(); + if ($currentSpan !== null && $message->getHeaders()->containsKey(MessageHeaders::TRACING_ID)) { + $flow = $currentSpan->context()->traceId; + + throw InvalidArgumentException::create("tracingId names the flow a message starts, but this message is sent from inside flow {$flow}, so it belongs to that flow already. Await {$flow} with TracingGateway, or send the message from outside a handler, where a flow starts."); + } + + return $this->trace($name, $methodInvocation, $message, $kind, $attributes); + } + /** * @param array $attributes */ diff --git a/upgrade-2.0.md b/upgrade-2.0.md index d45c19993..d1650b10d 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3119,7 +3119,9 @@ The flow is named by the **`tracingId`** header, `MessageHeaders::TRACING_ID`: a the application chooses and sends with the command, then passes to `TracingGateway`. Ecotone turns it into the W3C `traceparent` of the first span, so a command that already carries a `traceparent`, as one from an instrumented HTTP request does, is tracked by that trace id instead; a message carrying both is refused, because it is not clear which -one names the flow, and a `tracingId` in neither format is refused naming the expected one. A handler reads the id of +one names the flow, and a `tracingId` in neither format is refused naming the expected one. A `tracingId` names where +a flow starts, so one passed to a bus or gateway called from inside a handler is refused, naming the flow the call +already belongs to. A handler reads the id of the flow it runs in from `Ecotone\Api\Tracing\CurrentTrace::tracingId()`. A flow is complete once the work that started it ended, every route it sent has an ended handler, and every message From ed4b36dcac423842249140cfa05baf4fba2bd18e Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 36/50] fix(tracing): flows started by a query, a distributed send or a consumer of an untraced message complete The query and distributed buses were not recorded, and a consumer starting a fresh trace never marked its root, so such flows stayed pending forever and their timeout listed nothing. Both buses are now recorded, a consumer without a remote parent is the root of its flow, and a tracingId that names the flow the call already runs in - as a distributed message carries across services - is no longer refused. --- .../Dbal/src/Tracing/DbalTracingModule.php | 3 +- .../FlowTrackingConformanceTest.php | 28 ++++++++++ .../Integration/Tracing/FlowTrackingTest.php | 51 +++++++++++++++++++ .../src/Tracing/TracingInterceptor.php | 5 +- upgrade-2.0.md | 4 +- 5 files changed, 87 insertions(+), 4 deletions(-) diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index ba3103954..a2d37bd74 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -21,6 +21,7 @@ use Ecotone\Api\ExtensionObject\ServiceConfiguration; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; use Ecotone\Api\Gateway\CommandBus; +use Ecotone\Api\Gateway\DistributedBus; use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Gateway\EventBus; use Ecotone\Api\Gateway\QueryBus; @@ -146,7 +147,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO TraceRecordingInterceptor::class, $interfaceToCallRegistry->getFor(TraceRecordingInterceptor::class, $method), $precedence, - CommandBus::class . '||' . EventBus::class . '||' . AsynchronousRunningEndpoint::class . '||' . OrchestratorGateway::class . '||(' . self::USER_MESSAGE_GATEWAYS_POINTCUT . ')' + CommandBus::class . '||' . EventBus::class . '||' . QueryBus::class . '||' . DistributedBus::class . '||' . AsynchronousRunningEndpoint::class . '||' . OrchestratorGateway::class . '||(' . self::USER_MESSAGE_GATEWAYS_POINTCUT . ')' ) ); } diff --git a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php index 869544058..7a13289f1 100644 --- a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php +++ b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php @@ -10,6 +10,7 @@ use Ecotone\Api\Attribute\Around; use Ecotone\Api\Attribute\Asynchronous; use Ecotone\Api\Attribute\CommandHandler; +use Ecotone\Api\Attribute\Distributed; use Ecotone\Api\Attribute\EventHandler; use Ecotone\Api\Dbal\DbalConnectionFactory; use Ecotone\Api\Dbal\ExtensionObject\DbalBackedMessageChannelBuilder; @@ -24,6 +25,7 @@ use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\EventSourcing\Event; +use Ecotone\Api\ExtensionObject\DistributedServiceMap; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\InstantRetryConfiguration; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -289,6 +291,32 @@ classesToResolve: [$this->asynchronousNotifier()::class], self::assertFalse($this->completion($consumer)->isCompleted($trace->traceId)); } + public function test_a_flow_started_by_a_distributed_command_completes_once_the_receiving_service_handled_it(): void + { + $sharedChannel = SimpleMessageChannelBuilder::createQueueChannel('distributed_ticket_channel'); + $tickets = new class () { + public int $created = 0; + + #[Distributed] + #[CommandHandler('ticket.create', endpointId: 'ticket.creation')] + public function create(string $description): void + { + $this->created++; + } + }; + $userService = $this->bootstrap('dbal', [], [$sharedChannel, DistributedServiceMap::initialize()->withCommandMapping(targetServiceName: 'ticket_service', channelName: 'distributed_ticket_channel'), ServiceConfiguration::createWithDefaults()->withServiceName('user_service')]); + $ticketService = $this->bootstrap('dbal', [$tickets], [$sharedChannel, ServiceConfiguration::createWithDefaults()->withServiceName('ticket_service')]); + $tracingId = '0af7651916cd43dd8448eb211c80319c'; + + $userService->getDistributedBus()->sendCommand('ticket_service', 'ticket.create', 'User changed billing address', metadata: [MessageHeaders::TRACING_ID => $tracingId]); + self::assertFalse($this->completion($userService)->isCompleted($tracingId)); + + $ticketService->run('distributed_ticket_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + self::assertSame(1, $tickets->created); + self::assertTrue($this->completion($userService)->isCompleted($tracingId)); + } + public function test_a_send_rolled_back_with_its_failed_handler_is_not_awaited_after_the_retry_succeeded(): void { $notifier = new class () { diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index b764987a4..188793291 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -11,6 +11,7 @@ use Ecotone\Api\Attribute\InternalHandler; use Ecotone\Api\Attribute\MessageGateway; use Ecotone\Api\Attribute\Polling; +use Ecotone\Api\Attribute\QueryHandler; use Ecotone\Api\Attribute\Reference; use Ecotone\Api\Dbal\Tracing\FlowCompleted; use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; @@ -691,6 +692,56 @@ public function whenPlaced(array $event): void $this->completion($ecotone)->awaitFor('0af7651916cd43dd8448eb211c80319c', 'orders_list', Duration::milliseconds(10)); } + public function test_a_flow_started_by_a_query_completes_once_the_asynchronous_command_it_sent_ran(): void + { + $orders = new class () { + #[QueryHandler('order.quote')] + public function quote(string $orderId, CommandBus $commandBus): string + { + $commandBus->sendWithRouting('order.reserve', $orderId); + + return 'quote-' . $orderId; + } + + #[Asynchronous('async_channel')] + #[CommandHandler('order.reserve', endpointId: 'order.reserving')] + public function reserve(string $orderId): void + { + } + }; + $ecotone = $this->bootstrap([$orders], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $tracingId = '0af7651916cd43dd8448eb211c80319c'; + + $ecotone->sendQueryWithRouting('order.quote', '1', metadata: [MessageHeaders::TRACING_ID => $tracingId]); + $this->assertFalse($this->completion($ecotone)->isCompleted($tracingId)); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $this->assertTrue($this->completion($ecotone)->isCompleted($tracingId)); + } + + public function test_a_flow_a_consumer_starts_for_a_message_from_an_application_without_tracing_completes_once_that_consumer_ran(): void + { + $sharedChannel = SimpleMessageChannelBuilder::createQueueChannel('async_channel'); + $orders = new class () { + public ?string $tracingId = null; + + #[Asynchronous('async_channel')] + #[CommandHandler('order.place', endpointId: 'order.placing')] + public function place(string $orderId, #[Reference] CurrentTrace $currentTrace): void + { + $this->tracingId = $currentTrace->tracingId(); + } + }; + $producer = EcotoneLite::bootstrapFlowTesting([$orders::class], [$orders], [$sharedChannel]); + $consumer = $this->bootstrap([$orders], [$sharedChannel]); + + $producer->sendCommandWithRouting('order.place', '1'); + $consumer->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertNotNull($orders->tracingId); + $this->assertTrue($this->completion($consumer)->isCompleted($orders->tracingId)); + } + public function test_a_handler_can_read_the_trace_id_it_runs_in(): void { $users = new class () { diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index 045d83431..d01e27fc0 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -246,7 +246,8 @@ private static function shortNameOf(string $classAndMethod): string private function traceEntry(string $name, MethodInvocation $methodInvocation, Message $message, SpanKind $kind, array $attributes): mixed { $currentSpan = $this->tracer->currentSpan(); - if ($currentSpan !== null && $message->getHeaders()->containsKey(MessageHeaders::TRACING_ID)) { + $headers = $message->getHeaders(); + if ($currentSpan !== null && $headers->containsKey(MessageHeaders::TRACING_ID) && TracingId::normalizedOrNull((string) $headers->get(MessageHeaders::TRACING_ID)) !== $currentSpan->context()->traceId) { $flow = $currentSpan->context()->traceId; throw InvalidArgumentException::create("tracingId names the flow a message starts, but this message is sent from inside flow {$flow}, so it belongs to that flow already. Await {$flow} with TracingGateway, or send the message from outside a handler, where a flow starts."); @@ -262,7 +263,7 @@ private function trace(string $name, MethodInvocation $methodInvocation, Message { $isLocalRoot = $this->tracer->currentSpan() === null; $remoteParent = ! $isLocalRoot ? null : ($kind === SpanKind::Consumer ? self::remoteParentOf($message) : self::entryParentOf($message)); - if ($isLocalRoot && $kind !== SpanKind::Consumer) { + if ($isLocalRoot && ($kind !== SpanKind::Consumer || $remoteParent === null)) { $attributes[SpanAttribute::TRACE_ROOT] = true; } $attributes = [...$attributes, ...self::trackingAttributesOf($this->tracer, $message, $isLocalRoot && $kind !== SpanKind::Consumer)]; diff --git a/upgrade-2.0.md b/upgrade-2.0.md index d1650b10d..5969eb7a4 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3124,7 +3124,9 @@ a flow starts, so one passed to a bus or gateway called from inside a handler is already belongs to. A handler reads the id of the flow it runs in from `Ecotone\Api\Tracing\CurrentTrace::tracingId()`. -A flow is complete once the work that started it ended, every route it sent has an ended handler, and every message +A flow starts at a command, event or query bus call, a distributed bus send, an application or orchestrator gateway +call, or a consumer receiving a message that carries no trace, such as one from an application without tracing or a +scheduled poll. A flow is complete once the work that started it ended, every route it sent has an ended handler, and every message it sent to a channel was consumed from that channel, so an asynchronous command, event or projection keeps the flow pending until its consumer processed that exact message. A destination counts messages by id, so a redelivery satisfies its own delivery only, and a message keeps the delivery it started as through dynamic channels, the From 9ef69431b3be0cd71cb0f8a05090ed53373ccd3c Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 37/50] fix(tracing): a handler served as a subclass proxy keeps the endpoint id its route was compiled with The handler span looked its endpoint id up by the runtime class, so a container that serves a subclass proxy of the handler missed it, the span got a fresh id and its route stayed pending forever. The lookup now walks up to the declaring class. --- .../Integration/Tracing/FlowTrackingTest.php | 27 +++++++++++++++++++ .../src/Tracing/TracingInterceptor.php | 23 +++++++++++++--- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index 188793291..729fd8277 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -155,6 +155,21 @@ public function notify(string $userId): void $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } + public function test_a_handler_the_container_serves_as_a_subclass_proxy_still_completes_its_route(): void + { + $ecotone = EcotoneLite::bootstrapFlowTesting( + [$this->users()::class, RegistrationNotifier::class], + [$this->users(), RegistrationNotifier::class => new ProxiedRegistrationNotifier()], + [FlowTrackingConfiguration::createWithDefaults()], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + public function test_a_trace_nobody_recorded_is_not_completed(): void { $ecotone = $this->bootstrap([$this->users()]); @@ -1203,3 +1218,15 @@ interface TracedNotificationGateway #[MessageGateway('notification.send')] public function send(string $text, #[Headers] array $metadata): void; } + +class RegistrationNotifier +{ + #[EventHandler('user.registered')] + public function notify(string $userId): void + { + } +} + +final class ProxiedRegistrationNotifier extends RegistrationNotifier +{ +} diff --git a/packages/Ecotone/src/Tracing/TracingInterceptor.php b/packages/Ecotone/src/Tracing/TracingInterceptor.php index d01e27fc0..b25f8292b 100644 --- a/packages/Ecotone/src/Tracing/TracingInterceptor.php +++ b/packages/Ecotone/src/Tracing/TracingInterceptor.php @@ -53,22 +53,22 @@ public function traceQueryBus(MethodInvocation $methodInvocation, Message $messa public function traceCommandHandler(MethodInvocation $methodInvocation, Message $message, CommandHandler $endpoint): mixed { - return $this->traceHandler($methodInvocation, $message, $this->endpointIdsByHandler[$methodInvocation->getName()] ?? $endpoint->getEndpointId()); + return $this->traceHandler($methodInvocation, $message, $this->endpointIdOf($methodInvocation, $endpoint->getEndpointId())); } public function traceQueryHandler(MethodInvocation $methodInvocation, Message $message, QueryHandler $endpoint): mixed { - return $this->traceHandler($methodInvocation, $message, $this->endpointIdsByHandler[$methodInvocation->getName()] ?? $endpoint->getEndpointId()); + return $this->traceHandler($methodInvocation, $message, $this->endpointIdOf($methodInvocation, $endpoint->getEndpointId())); } public function traceEventHandler(MethodInvocation $methodInvocation, Message $message, EventHandler $endpoint): mixed { - return $this->traceHandler($methodInvocation, $message, $this->endpointIdsByHandler[$methodInvocation->getName()] ?? $endpoint->getEndpointId()); + return $this->traceHandler($methodInvocation, $message, $this->endpointIdOf($methodInvocation, $endpoint->getEndpointId())); } public function traceMessageHandler(MethodInvocation $methodInvocation, Message $message, InternalHandler $endpoint): mixed { - return $this->traceHandler($methodInvocation, $message, $this->endpointIdsByHandler[$methodInvocation->getName()] ?? $endpoint->getEndpointId()); + return $this->traceHandler($methodInvocation, $message, $this->endpointIdOf($methodInvocation, $endpoint->getEndpointId())); } public function traceAsynchronousEndpoint(MethodInvocation $methodInvocation, Message $message): mixed @@ -202,6 +202,21 @@ private static function entryParentOf(Message $message): ?SpanContext return new SpanContext(TracingId::normalize((string) $headers->get(MessageHeaders::TRACING_ID)), bin2hex(random_bytes(8))); } + private function endpointIdOf(MethodInvocation $methodInvocation, string $declaredEndpointId): string + { + $object = $methodInvocation->getObjectToInvokeOn(); + $class = is_string($object) ? $object : $object::class; + while ($class !== false) { + $endpointId = $this->endpointIdsByHandler[$class . '::' . $methodInvocation->getMethodName()] ?? null; + if ($endpointId !== null) { + return $endpointId; + } + $class = get_parent_class($class); + } + + return $declaredEndpointId; + } + private function traceHandler(MethodInvocation $methodInvocation, Message $message, string $endpointId): mixed { $attributes = [ From efa4b75e5881adb0bb18b7544fcd78eb9983f1d5 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 38/50] fix(dbal): a flow subscriber whose first parameter cannot receive its notification is refused at bootstrap A #[OnFlowCompleted] or #[OnFlowStepCompleted] method declaring the wrong type failed on every notification, at runtime, after the work committed. The module now checks the first parameter against FlowCompleted or FlowStepCompleted when it prepares, naming the type to declare. --- .../Dbal/src/Tracing/DbalTracingModule.php | 52 +++++++++++++++++++ .../Integration/Tracing/FlowTrackingTest.php | 30 +++++++++++ 2 files changed, 82 insertions(+) diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index a2d37bd74..ff39aee88 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -13,6 +13,8 @@ use Ecotone\Api\Dbal\ExtensionObject\DbalConfiguration; use Ecotone\Api\Dbal\ExtensionObject\DbalConnectionReference; use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; +use Ecotone\Api\Dbal\Tracing\FlowCompleted; +use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; @@ -51,6 +53,7 @@ use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Ecotone\Messaging\Handler\Processor\MethodInvoker\AroundInterceptorBuilder; use Ecotone\Messaging\Handler\ServiceActivator\MessageProcessorActivatorBuilder; +use Ecotone\Messaging\Handler\Type; use Ecotone\Messaging\Support\LicensingException; use Ecotone\Tracing\Config\SpanRecorderReference; use Ecotone\Tracing\Tracer; @@ -82,11 +85,13 @@ final class DbalTracingModule extends NoExternalConfigurationModule implements A /** * @param string[] $flowCompletedSubscribers * @param string[] $flowStepCompletedSubscribers + * @param array $subscriberMethods * @param string[] $pollingProjectionNames */ private function __construct( private array $flowCompletedSubscribers, private array $flowStepCompletedSubscribers, + private array $subscriberMethods, private array $pollingProjectionNames, ) { } @@ -96,6 +101,10 @@ public static function create(AnnotationFinder $annotationRegistrationService, I return new self( self::placementsOf($annotationRegistrationService, OnFlowCompleted::class), self::placementsOf($annotationRegistrationService, OnFlowStepCompleted::class), + [ + ...self::subscriberMethodsOf($annotationRegistrationService, OnFlowCompleted::class, FlowCompleted::class), + ...self::subscriberMethodsOf($annotationRegistrationService, OnFlowStepCompleted::class, FlowStepCompleted::class), + ], self::pollingProjectionNames($annotationRegistrationService), ); } @@ -104,6 +113,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO { $flowTracking = self::flowTrackingOf($extensionObjects); $this->refuseMisconfiguredFlowTracking($messagingConfiguration, $flowTracking); + $this->refuseSubscribersThatCannotReceiveTheirNotification($interfaceToCallRegistry); $connectionReference = self::connectionReferenceOf($extensionObjects); $consoleInvocationPrefix = ConsoleInvocationResolver::resolveConsolePrefix(ExtensionObjectResolver::resolveUnique(BootstrapConfiguration::class, $extensionObjects, BootstrapConfiguration::createWithDefaults())); $storesOnDatabase = $flowTracking !== null && ! $flowTracking->inMemory; @@ -236,6 +246,48 @@ private function refuseMisconfiguredFlowTracking(Configuration $messagingConfigu } } + private function refuseSubscribersThatCannotReceiveTheirNotification(InterfaceToCallRegistry $interfaceToCallRegistry): void + { + foreach ($this->subscriberMethods as $subscriber) { + $interfaceToCall = $interfaceToCallRegistry->getFor($subscriber['class'], $subscriber['method']); + if ($interfaceToCall->hasNoParameters()) { + continue; + } + $declaredType = $interfaceToCall->getFirstParameter()->getTypeDescriptor(); + if ($declaredType->acceptType(Type::create($subscriber['notification']))) { + continue; + } + + throw ConfigurationException::create(sprintf( + '%s on %s::%s is passed a %s as its first parameter, but declares it as %s, so every notification would fail. Declare that parameter as %s.', + $subscriber['attribute'], + $subscriber['class'], + $subscriber['method'], + $subscriber['notification'], + $declaredType->toString(), + $subscriber['notification'], + )); + } + } + + /** + * @return array + */ + private static function subscriberMethodsOf(AnnotationFinder $annotationFinder, string $attributeClassName, string $notificationClassName): array + { + $subscribers = []; + foreach ($annotationFinder->findAnnotatedMethods($attributeClassName) as $annotatedMethod) { + $subscribers[] = [ + 'attribute' => '#[' . substr((string) strrchr('\\' . $attributeClassName, '\\'), 1) . ']', + 'notification' => $notificationClassName, + 'class' => $annotatedMethod->getClassName(), + 'method' => $annotatedMethod->getMethodName(), + ]; + } + + return $subscribers; + } + private static function registerTracingGateway(Configuration $messagingConfiguration, string $serviceClass, Definition $service): void { $messagingConfiguration->registerServiceDefinition($serviceClass, $service); diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index 729fd8277..b6133f7b4 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -1015,6 +1015,36 @@ public function test_a_failed_unit_of_work_stores_nothing_by_default(): void $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); } + public function test_a_flow_completed_subscriber_whose_first_parameter_cannot_receive_the_notification_is_refused_at_bootstrap(): void + { + $subscriber = new class () { + #[OnFlowCompleted] + public function whenCompleted(FlowStepCompleted $event): void + { + } + }; + + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage('#[OnFlowCompleted] on ' . $subscriber::class . '::whenCompleted is passed a ' . FlowCompleted::class . ' as its first parameter, but declares it as ' . FlowStepCompleted::class . ', so every notification would fail. Declare that parameter as ' . FlowCompleted::class . '.'); + + $this->bootstrap([$subscriber]); + } + + public function test_a_flow_step_completed_subscriber_whose_first_parameter_cannot_receive_the_notification_is_refused_at_bootstrap(): void + { + $subscriber = new class () { + #[OnFlowStepCompleted] + public function whenEnded(string $event): void + { + } + }; + + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage('#[OnFlowStepCompleted] on ' . $subscriber::class . '::whenEnded is passed a ' . FlowStepCompleted::class . ' as its first parameter, but declares it as string, so every notification would fail. Declare that parameter as ' . FlowStepCompleted::class . '.'); + + $this->bootstrap([$subscriber]); + } + public function test_with_notifications_turned_off_a_subscriber_is_refused_at_bootstrap(): void { $subscriber = new class () { From 7ef30536f652116c69ca13d1baa951e20d29e4a5 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 39/50] fix(dbal): an #[Asynchronous] flow subscriber and a FlowTrackingConfiguration registered twice are refused at bootstrap #[Asynchronous] on a flow subscriber asked for an endpointId the attribute cannot take; an endpoint attribute can now state why it cannot run asynchronously, and AsynchronousModule refuses with that reason. A second FlowTrackingConfiguration was dropped silently by the flow-testing rewrite; it is now refused, naming how to register one. --- packages/Dbal/Api/Tracing/OnFlowCompleted.php | 5 +++ .../Dbal/Api/Tracing/OnFlowStepCompleted.php | 5 +++ .../FlowTestingDbalConnections.php | 2 +- .../Dbal/src/Tracing/DbalTracingModule.php | 9 ++-- .../Integration/Tracing/FlowTrackingTest.php | 44 +++++++++++++++++++ .../Attribute/IdentifiedAnnotation.php | 5 +++ .../AsynchronousModule.php | 12 +++++ upgrade-2.0.md | 5 ++- 8 files changed, 82 insertions(+), 5 deletions(-) diff --git a/packages/Dbal/Api/Tracing/OnFlowCompleted.php b/packages/Dbal/Api/Tracing/OnFlowCompleted.php index 41358cd49..0a498d0c7 100644 --- a/packages/Dbal/Api/Tracing/OnFlowCompleted.php +++ b/packages/Dbal/Api/Tracing/OnFlowCompleted.php @@ -20,4 +20,9 @@ public function __construct() { parent::__construct(self::CHANNEL); } + + public function asynchronousExecutionRefusal(): string + { + return 'cannot run #[Asynchronous]: a flow notification is delivered from memory once the work committed, so it is never queued. Remove #[Asynchronous], and send a command to an #[Asynchronous] handler from this method for the work that should run in the background.'; + } } diff --git a/packages/Dbal/Api/Tracing/OnFlowStepCompleted.php b/packages/Dbal/Api/Tracing/OnFlowStepCompleted.php index a82787a1b..1453eccf0 100644 --- a/packages/Dbal/Api/Tracing/OnFlowStepCompleted.php +++ b/packages/Dbal/Api/Tracing/OnFlowStepCompleted.php @@ -20,4 +20,9 @@ public function __construct() { parent::__construct(self::CHANNEL); } + + public function asynchronousExecutionRefusal(): string + { + return 'cannot run #[Asynchronous]: a flow notification is delivered from memory once the work committed, so it is never queued. Remove #[Asynchronous], and send a command to an #[Asynchronous] handler from this method for the work that should run in the background.'; + } } diff --git a/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php b/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php index 7ebf82bbd..d8aca7b9e 100644 --- a/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php +++ b/packages/Dbal/src/Configuration/FlowTestingDbalConnections.php @@ -114,7 +114,7 @@ private function flowTrackingInMemoryWhereTheConnectionIsMissing(DbalConfigurati } $connectionReferenceName = $dbalConfiguration->defaultConnectionReferenceNames[0] ?? DbalConnectionReference::DEFAULT; - return [$this->has($connectionReferenceName) ? $flowTracking[0] : $flowTracking[0]->withInMemoryStorage(true)]; + return array_map(fn (FlowTrackingConfiguration $configuration): FlowTrackingConfiguration => $this->has($connectionReferenceName) ? $configuration : $configuration->withInMemoryStorage(true), $flowTracking); } private function refuseDbalChannelsWithoutTheirConnection(): void diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index ff39aee88..acfd42dc8 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -218,9 +218,12 @@ public function getModulePackageName(): string */ private static function flowTrackingOf(array $extensionObjects): ?FlowTrackingConfiguration { - return ExtensionObjectResolver::contains(FlowTrackingConfiguration::class, $extensionObjects) - ? ExtensionObjectResolver::resolveUnique(FlowTrackingConfiguration::class, $extensionObjects, FlowTrackingConfiguration::createWithDefaults()) - : null; + $configurations = ExtensionObjectResolver::resolve(FlowTrackingConfiguration::class, $extensionObjects); + if (count($configurations) > 1) { + throw ConfigurationException::create(sprintf('FlowTrackingConfiguration was registered %d times, so it is not clear which settings apply. Register it once, from one #[ServiceContext] method, chaining every setting on it: FlowTrackingConfiguration::createWithDefaults()->withRetention(...)->withNotifications(...).', count($configurations))); + } + + return $configurations[0] ?? null; } private function refuseMisconfiguredFlowTracking(Configuration $messagingConfiguration, ?FlowTrackingConfiguration $flowTracking): void diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index b6133f7b4..809fe753c 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -1045,6 +1045,50 @@ public function whenEnded(string $event): void $this->bootstrap([$subscriber]); } + public function test_an_asynchronous_flow_completed_subscriber_is_refused_at_bootstrap_naming_how_to_run_its_work_in_the_background(): void + { + $subscriber = new class () { + #[Asynchronous('notifications')] + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event): void + { + } + }; + + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage('#[OnFlowCompleted] on ' . $subscriber::class . '::whenCompleted cannot run #[Asynchronous]: a flow notification is delivered from memory once the work committed, so it is never queued. Remove #[Asynchronous], and send a command to an #[Asynchronous] handler from this method for the work that should run in the background.'); + + $this->bootstrap([$subscriber], [SimpleMessageChannelBuilder::createQueueChannel('notifications')]); + } + + public function test_a_flow_step_completed_subscriber_in_an_asynchronous_class_is_refused_at_bootstrap_naming_how_to_run_its_work_in_the_background(): void + { + $subscriber = new #[Asynchronous('notifications')] class () { + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event): void + { + } + }; + + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage('#[OnFlowStepCompleted] on ' . $subscriber::class . '::whenEnded cannot run #[Asynchronous]: a flow notification is delivered from memory once the work committed, so it is never queued. Remove #[Asynchronous], and send a command to an #[Asynchronous] handler from this method for the work that should run in the background.'); + + $this->bootstrap([$subscriber], [SimpleMessageChannelBuilder::createQueueChannel('notifications')]); + } + + public function test_flow_tracking_configured_twice_is_refused_at_bootstrap(): void + { + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage('FlowTrackingConfiguration was registered 2 times, so it is not clear which settings apply. Register it once, from one #[ServiceContext] method, chaining every setting on it: FlowTrackingConfiguration::createWithDefaults()->withRetention(...)->withNotifications(...).'); + + EcotoneLite::bootstrapFlowTesting( + [$this->users()::class], + [$this->users()], + [FlowTrackingConfiguration::createWithDefaults(), FlowTrackingConfiguration::createWithDefaults()->withNotifications(false)], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + public function test_with_notifications_turned_off_a_subscriber_is_refused_at_bootstrap(): void { $subscriber = new class () { diff --git a/packages/Ecotone/src/Messaging/Attribute/IdentifiedAnnotation.php b/packages/Ecotone/src/Messaging/Attribute/IdentifiedAnnotation.php index d7f32790e..27bf803ca 100644 --- a/packages/Ecotone/src/Messaging/Attribute/IdentifiedAnnotation.php +++ b/packages/Ecotone/src/Messaging/Attribute/IdentifiedAnnotation.php @@ -32,6 +32,11 @@ public function isEndpointIdGenerated(): bool return $this->isGenerated; } + public function asynchronousExecutionRefusal(): ?string + { + return null; + } + public function getEndpointId(): string { return $this->endpointId; diff --git a/packages/Ecotone/src/Messaging/Config/Annotation/ModuleConfiguration/AsynchronousModule.php b/packages/Ecotone/src/Messaging/Config/Annotation/ModuleConfiguration/AsynchronousModule.php index f055facf0..be1236abb 100644 --- a/packages/Ecotone/src/Messaging/Config/Annotation/ModuleConfiguration/AsynchronousModule.php +++ b/packages/Ecotone/src/Messaging/Config/Annotation/ModuleConfiguration/AsynchronousModule.php @@ -4,6 +4,7 @@ namespace Ecotone\Messaging\Config\Annotation\ModuleConfiguration; +use Ecotone\AnnotationFinder\AnnotatedMethod; use Ecotone\AnnotationFinder\AnnotationFinder; use Ecotone\Api\Attribute\Asynchronous; use Ecotone\Api\Attribute\CommandHandler; @@ -18,6 +19,7 @@ use Ecotone\Api\ExtensionObject\ServiceConfiguration; use Ecotone\Api\ExtensionObject\SimpleMessageChannelBuilder; use Ecotone\Messaging\Attribute\EndpointAnnotation; +use Ecotone\Messaging\Attribute\IdentifiedAnnotation; use Ecotone\Messaging\Attribute\StreamBasedSource; use Ecotone\Messaging\Config\Annotation\AnnotatedDefinitionReference; use Ecotone\Messaging\Config\Annotation\AnnotationModule; @@ -71,6 +73,7 @@ public static function create(AnnotationFinder $annotationRegistrationService, I $streamSourcesAsyncEndpoints[$annotationForMethod->getEndpointId()] = $asyncClass->getChannelName(); } else { if ($annotationForMethod instanceof CommandHandler || $annotationForMethod instanceof EventHandler || $annotationForMethod instanceof InternalHandler) { + self::refuseEndpointThatCannotRunAsynchronously($endpoint, $annotationForMethod); if ($annotationForMethod->isEndpointIdGenerated()) { throw ConfigurationException::create("{$endpoint} should have endpointId defined for handling asynchronously"); } @@ -93,6 +96,7 @@ public static function create(AnnotationFinder $annotationRegistrationService, I continue; } if ($annotationForMethod instanceof CommandHandler || $annotationForMethod instanceof EventHandler || $annotationForMethod instanceof InternalHandler) { + self::refuseEndpointThatCannotRunAsynchronously($endpoint, $annotationForMethod); if ($annotationForMethod->isEndpointIdGenerated()) { throw ConfigurationException::create("{$endpoint} should have endpointId defined for handling asynchronously"); } @@ -262,4 +266,12 @@ public function registerDefaultPollingMetadata(BootstrapConfiguration $bootstrap } } } + + private static function refuseEndpointThatCannotRunAsynchronously(AnnotatedMethod $endpoint, IdentifiedAnnotation $annotation): void + { + $refusal = $annotation->asynchronousExecutionRefusal(); + if ($refusal !== null) { + throw ConfigurationException::create(sprintf('#[%s] on %s::%s %s', substr((string) strrchr('\\' . $annotation::class, '\\'), 1), $endpoint->getClassName(), $endpoint->getMethodName(), $refusal)); + } + } } diff --git a/upgrade-2.0.md b/upgrade-2.0.md index 5969eb7a4..90a294dca 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3147,7 +3147,10 @@ its failure is visible in OpenTelemetry. Notifications are published from memory most once: a subscriber that throws does not fail the work that committed, is logged, and is not called again. Each notification is delivered in a trace of its own, marked `ecotone=untracked` in the W3C `tracestate`, so whatever a subscriber sends, synchronously or through an asynchronous channel, is neither stored nor notified and can never reopen -the flow it was told about; OpenTelemetry still exports it, as its own trace. Completion is claimed with one +the flow it was told about; OpenTelemetry still exports it, as its own trace. A subscriber runs synchronously, as that delivery +happens; `#[Asynchronous]` on it is refused at bootstrap, so work meant for the background is a command it sends to an +`#[Asynchronous]` handler. A subscriber whose first parameter is not `FlowCompleted` or `FlowStepCompleted`, and a +`FlowTrackingConfiguration` registered twice, are refused at bootstrap too. Completion is claimed with one conditional update after the trace row was touched in the same transaction, so a second unit of work finishing the same flow waits for the first one's commit and, on PostgreSQL and SQLite, reads its spans before deciding. From 2f692e5a67d400af25d285f6042d527f55b9f11d Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 40/50] docs(conventions): rule 1d - the common usage is the default, and any other usage is refused rather than given a meaning of its own --- AGENTS.md | 1 + docs/coding-conventions.md | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 96bd1343c..46b48db14 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,6 +93,7 @@ commit that evidences it. The summary: | 1 | **Exceptions drive the solution** — name what was wrong, why it cannot work, and *every* way out (the exact API call, attribute, option or command). Assert the message in a test | The message is what the agent or developer reacts to; it is the deliverable | | 1a | A configuration mistake is **refused at bootstrap**, never at runtime, and never read as an empty result — the guard and the message test ship in the commit that states the rule | One worktree of 18 fix commits paid for the rules that shipped without one | | 1c | **A check that can run at compile time runs at compile time** — in a guard the module calls from `prepare()`, which is handed the `InterfaceToCallRegistry` for exactly that. Only message *content* is left to runtime | The compile-time guard removes the runtime branch entirely: there is no unresolvable case left to handle | +| 1d | **The common usage is the default; any other usage is refused, never given a second meaning.** `#[EventHandler]` passes the payload first because almost every handler wants it; `#[EventHandler]` on a class is refused. For every attribute, configuration and gateway, decide each placement, parameter type, combining attribute (`#[Asynchronous]` first), repeated registration and contradicting setting: supported with a test, or refused at bootstrap with a message test. When two readings are possible, refuse rather than pick one | A usage quietly given its own meaning reads differently in two applications, and the one that guessed wrong finds out in production. Flow tracking shipped three such gaps at once: an async subscriber asked for an `endpointId` it cannot take, a second configuration was silently dropped, a mistyped subscriber failed on every notification | | 2 | **Never take a nullable service dependency.** Register unconditionally, decide at runtime. Split the class, add a null-object factory, make the parameter required, or pick between two services in the container | A `?Service = null` is a mode switch in disguise, and the two modes drift | | 2a | **Never default a dependency either.** `$clock = new NativeClock()` is rule 2 with the null spelled out: every construction site that forgets the argument silently runs a second way. A dependency is a required parameter at every site, and the container passes it | The 2.0.0-beta.2 clock regression: two channel adapters passed the clock, six other sites took the default, and a DBAL consumer on real time hung a test on the pinned clock | | 3 | **Constructor-injected services are stateless** — no accumulating properties, no static caches, no per-message collectors. Mutable state goes in a function-scoped immutable value object carried on a message header | A container service is a singleton; anything it remembers leaks into the next message, transaction and retry | diff --git a/docs/coding-conventions.md b/docs/coding-conventions.md index 7d70dce2c..4def16b4c 100644 --- a/docs/coding-conventions.md +++ b/docs/coding-conventions.md @@ -15,7 +15,7 @@ branch `dgafka/dcb-docs` and is deliberately untracked here, since `docs/superpo paths resolve on that branch. Commit SHAs cited below resolve on any branch. Rules 1-11 are the maintainer's. Rules 12-19 are conventions the code holds to consistently. Section 20 lists the -mechanics a tool enforces for you, and section 21 the gates and landmines. Lettered sub-rules (1a, 1b, 1c, 6a, 10a, 16a, +mechanics a tool enforces for you, and section 21 the gates and landmines. Lettered sub-rules (1a, 1b, 1c, 1d, 6a, 10a, 16a, 12a, 13a, 14a) continue the rule they hang off; they are numbered that way so nothing above them ever renumbers. > **Verify names before you write them.** Every attribute, parameter, method and console option in code you write @@ -137,6 +137,39 @@ So, before adding a check: ask whether it could have been answered from the clas and the attributes alone. If it could, it belongs in a guard the module calls, and the test that proves it boots Ecotone and expects a `ConfigurationException`. + +### 1d. The common usage is the default; any other usage is refused, never given a second meaning + +Every attribute, configuration object and gateway defaults to the usage most applications need, so the shortest code +is the right code. `#[EventHandler]` passes the message payload as the first parameter, because that is what almost +every handler wants, and resolves the rest from headers and services; `#[CommandHandler]` without a routing key +routes by the payload's class. Nobody has to opt into the normal case. + +A usage the feature is not meant for is refused at bootstrap, naming the meant one (rules 1, 1a). It is never quietly +accepted with a meaning of its own, because then two applications read the same code differently, and the one that +guessed wrong finds out in production. `#[EventHandler]` on a class is refused by its attribute target. When two +readings are possible, refuse rather than pick one: two configurations of one feature, a message carrying both +`tracingId` and `traceparent`, an attribute combined with one it cannot work with. + +Before shipping an attribute, a configuration or a gateway, go through what an application could write and decide, +for each one, whether it is supported (with a test) or refused (with a message test). None may be left to chance: + +- **Placement** — the attribute's `#[Attribute(...)]` target is as narrow as the meant usage. +- **Parameters** — the first parameter is checked against what it receives (rule 1c), so a wrong type fails at + bootstrap, not on the first message after the work committed. +- **Combinations** — every other attribute that can sit beside it, `#[Asynchronous]` first, is either supported or + refused by the attribute itself, with its own reason. An endpoint attribute that cannot run asynchronously returns + that reason from `asynchronousExecutionRefusal()`, and `AsynchronousModule` uses it instead of asking for an + `endpointId` the attribute cannot take. +- **Repetition** — a configuration object registered twice is refused; the first one never silently wins. +- **Settings that contradict** — an option combination that cannot work, such as a subscriber with notifications + turned off, is refused naming the setting to remove. + +Evidence: flow tracking shipped three of these gaps at once. `#[Asynchronous]` on `#[OnFlowCompleted]` asked for an +`endpointId` the attribute cannot take, a second `FlowTrackingConfiguration` was silently dropped, and a subscriber +with a mistyped first parameter failed on every notification at runtime (`FlowTrackingTest`, the three bootstrap +refusals). + --- ## 2. Never take a nullable service dependency From 665539acc3bdfa459afa652627ad088a59504fd7 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 41/50] feat(dbal): expired flows are removed in batches, each batch in one short transaction ecotone:tracing:remove-expired-traces deleted every expired flow in two statements, so the first run after a long gap held one large delete. It now removes batches of 1000 trace ids - their spans and flow rows together - until none is left; FlowTrackingConfiguration::withRemovalBatchSize() changes the size and refuses one below 1. The upgrade guide now says the command has to be scheduled. --- .../Api/Tracing/FlowTrackingConfiguration.php | 13 ++++++- packages/Dbal/src/Tracing/DbalTraceStore.php | 39 +++++++++++++++---- .../Dbal/src/Tracing/DbalTracingModule.php | 2 +- .../Dbal/src/Tracing/ExpiredTraceRemover.php | 3 +- .../Dbal/src/Tracing/InMemoryTraceStore.php | 2 +- packages/Dbal/src/Tracing/TraceStore.php | 2 +- .../FlowTrackingConformanceTest.php | 36 +++++++++++++++++ upgrade-2.0.md | 4 +- 8 files changed, 87 insertions(+), 14 deletions(-) diff --git a/packages/Dbal/Api/Tracing/FlowTrackingConfiguration.php b/packages/Dbal/Api/Tracing/FlowTrackingConfiguration.php index ab23a38e4..4b60f4422 100644 --- a/packages/Dbal/Api/Tracing/FlowTrackingConfiguration.php +++ b/packages/Dbal/Api/Tracing/FlowTrackingConfiguration.php @@ -5,6 +5,7 @@ namespace Ecotone\Api\Dbal\Tracing; use Ecotone\Api\Scheduling\Duration; +use Ecotone\Messaging\Config\ConfigurationException; /** * licence Enterprise @@ -18,12 +19,13 @@ private function __construct( public readonly bool $notifications, public readonly bool $tracingGateway, public readonly bool $inMemory, + public readonly int $removalBatchSize, ) { } public static function createWithDefaults(): self { - return new self(Duration::minutes(60 * 24 * 7), false, true, true, false); + return new self(Duration::minutes(60 * 24 * 7), false, true, true, false, 1000); } public function withRetention(Duration $retention): self @@ -46,6 +48,15 @@ public function withTracingGateway(bool $tracingGateway): self return new self(...[...get_object_vars($this), 'tracingGateway' => $tracingGateway]); } + public function withRemovalBatchSize(int $removalBatchSize): self + { + if ($removalBatchSize < 1) { + throw ConfigurationException::create("FlowTrackingConfiguration::withRemovalBatchSize({$removalBatchSize}) would never remove an expired flow. Pass 1 or more; the default is 1000."); + } + + return new self(...[...get_object_vars($this), 'removalBatchSize' => $removalBatchSize]); + } + public function withInMemoryStorage(bool $inMemory): self { return new self(...[...get_object_vars($this), 'inMemory' => $inMemory]); diff --git a/packages/Dbal/src/Tracing/DbalTraceStore.php b/packages/Dbal/src/Tracing/DbalTraceStore.php index e5561dd80..6c81c8396 100644 --- a/packages/Dbal/src/Tracing/DbalTraceStore.php +++ b/packages/Dbal/src/Tracing/DbalTraceStore.php @@ -5,6 +5,7 @@ namespace Ecotone\Dbal\Tracing; use Closure; +use Doctrine\DBAL\ArrayParameterType; use Doctrine\DBAL\Connection; use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; use Doctrine\DBAL\Types\Types; @@ -84,16 +85,12 @@ public function progressOf(string $traceId): TraceProgress return $this->onTables(fn (): TraceProgress => TraceLedger::progressOf($traceId, $this->spansOf($this->connection(), $traceId))); } - public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore): void + public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore, int $batchSize): void { $connection = $this->connection(); - $this->onTables(function () use ($connection, $completedTracesStartedBefore, $anyTraceStartedBefore): void { - $expired = "SELECT trace_id FROM {$this->tracesTableManager->getTableName()} WHERE started_at < :anyStartedBefore OR (completed_at IS NOT NULL AND started_at < :completedStartedBefore)"; - $parameters = ['anyStartedBefore' => $anyTraceStartedBefore->unixTime()->inMicroseconds(), 'completedStartedBefore' => $completedTracesStartedBefore->unixTime()->inMicroseconds()]; - $types = ['anyStartedBefore' => Types::BIGINT, 'completedStartedBefore' => Types::BIGINT]; - $connection->executeStatement("DELETE FROM {$this->spansTableManager->getTableName()} WHERE trace_id IN ({$expired})", $parameters, $types); - $connection->executeStatement("DELETE FROM {$this->tracesTableManager->getTableName()} WHERE started_at < :anyStartedBefore OR (completed_at IS NOT NULL AND started_at < :completedStartedBefore)", $parameters, $types); - }); + do { + $removed = $this->onTables(fn (): int => $connection->transactional(fn (): int => $this->removeExpiredBatch($connection, $completedTracesStartedBefore, $anyTraceStartedBefore, $batchSize))); + } while ($removed === $batchSize); } public function withinTransaction(Closure $work): mixed @@ -101,6 +98,32 @@ public function withinTransaction(Closure $work): mixed return $this->connection()->transactional($work); } + private function removeExpiredBatch(Connection $connection, DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore, int $batchSize): int + { + $traceIds = $connection->createQueryBuilder() + ->select('trace_id') + ->from($this->tracesTableManager->getTableName()) + ->where('started_at < :anyStartedBefore OR (completed_at IS NOT NULL AND started_at < :completedStartedBefore)') + ->setParameter('anyStartedBefore', $anyTraceStartedBefore->unixTime()->inMicroseconds(), Types::BIGINT) + ->setParameter('completedStartedBefore', $completedTracesStartedBefore->unixTime()->inMicroseconds(), Types::BIGINT) + ->setMaxResults($batchSize) + ->executeQuery() + ->fetchFirstColumn(); + if ($traceIds === []) { + return 0; + } + + foreach ([$this->spansTableManager->getTableName(), $this->tracesTableManager->getTableName()] as $tableName) { + $connection->createQueryBuilder() + ->delete($tableName) + ->where('trace_id IN (:traceIds)') + ->setParameter('traceIds', $traceIds, ArrayParameterType::STRING) + ->executeStatement(); + } + + return count($traceIds); + } + /** * @param Span[] $traceSpans */ diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index acfd42dc8..5057af511 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -187,7 +187,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO } $messagingConfiguration->registerServiceDefinition( ExpiredTraceRemover::class, - new Definition(ExpiredTraceRemover::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $flowTracking->retention->inMilliseconds()]) + new Definition(ExpiredTraceRemover::class, [new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $flowTracking->retention->inMilliseconds(), $flowTracking->removalBatchSize]) ); $messagingConfiguration ->registerMessageHandler(MessageProcessorActivatorBuilder::createForMethod(ExpiredTraceRemover::class, 'removeExpiredTraces')->withInputChannelName(self::REMOVE_EXPIRED_TRACES_CHANNEL)) diff --git a/packages/Dbal/src/Tracing/ExpiredTraceRemover.php b/packages/Dbal/src/Tracing/ExpiredTraceRemover.php index cac6dec60..e79c39d51 100644 --- a/packages/Dbal/src/Tracing/ExpiredTraceRemover.php +++ b/packages/Dbal/src/Tracing/ExpiredTraceRemover.php @@ -31,12 +31,13 @@ public function __construct( private TraceStore $traceStore, private EcotoneClockInterface $clock, private int $retentionInMilliseconds, + private int $removalBatchSize, ) { } public function removeExpiredTraces(): void { $now = $this->clock->now(); - $this->traceStore->removeExpiredTraces($now->sub(Duration::milliseconds($this->retentionInMilliseconds)), $now->sub(Duration::milliseconds(2 * $this->retentionInMilliseconds))); + $this->traceStore->removeExpiredTraces($now->sub(Duration::milliseconds($this->retentionInMilliseconds)), $now->sub(Duration::milliseconds(2 * $this->retentionInMilliseconds)), $this->removalBatchSize); } } diff --git a/packages/Dbal/src/Tracing/InMemoryTraceStore.php b/packages/Dbal/src/Tracing/InMemoryTraceStore.php index d61150926..502565943 100644 --- a/packages/Dbal/src/Tracing/InMemoryTraceStore.php +++ b/packages/Dbal/src/Tracing/InMemoryTraceStore.php @@ -59,7 +59,7 @@ public function progressOf(string $traceId): TraceProgress return TraceLedger::progressOf($traceId, array_values($this->spansByTrace[$traceId] ?? [])); } - public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore): void + public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore, int $batchSize): void { foreach ($this->spansByTrace as $traceId => $spans) { $startedAt = min(array_map(fn (Span $span): DatePoint => $span->startedAt, $spans)); diff --git a/packages/Dbal/src/Tracing/TraceStore.php b/packages/Dbal/src/Tracing/TraceStore.php index f567015a7..ecc8a016f 100644 --- a/packages/Dbal/src/Tracing/TraceStore.php +++ b/packages/Dbal/src/Tracing/TraceStore.php @@ -37,7 +37,7 @@ public function markCompletedIfDone(string $traceId, DatePoint $now): bool; public function progressOf(string $traceId): TraceProgress; - public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore): void; + public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore, int $batchSize): void; public function withinTransaction(Closure $work): mixed; } diff --git a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php index 7a13289f1..38860c6c1 100644 --- a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php +++ b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php @@ -43,6 +43,7 @@ use Ecotone\Api\Scheduling\TimeSpan; use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\SpanStatus; +use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Test\LicenceTesting; use PHPUnit\Framework\Attributes\DataProvider; use RuntimeException; @@ -233,6 +234,41 @@ public function archive(string $userId): void self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); } + #[DataProvider('implementations')] + public function test_expired_flows_beyond_one_removal_batch_are_all_removed_by_the_console_command(string $implementation): void + { + $ecotone = $this->bootstrapFlowTesting( + classesToResolve: [$this->users()::class], + containerOrAvailableServices: [$this->users(), DbalConnectionFactory::class => $this->getConnectionFactory()], + extensionObjects: [self::flowTracking($implementation, Duration::minutes(10))->withRemovalBatchSize(2)], + enabledModules: [ModulePackageList::DBAL_PACKAGE], + licenceKey: LicenceTesting::VALID_LICENCE, + )->setupDatabaseTables(); + $expired = []; + for ($flow = 1; $flow <= 5; $flow++) { + $expired[] = $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', (string) $flow, metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + } + $ecotone->advanceTimeBy(TimeSpan::withMinutes(11)); + $recent = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '6', metadata: [MessageHeaders::TRACEPARENT => $recent->toTraceparent()]); + + $ecotone->runConsoleCommand('ecotone:tracing:remove-expired-traces'); + + foreach ($expired as $trace) { + self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + } + self::assertTrue($this->completion($ecotone)->isCompleted($recent->traceId)); + } + + public function test_a_removal_batch_size_below_one_is_refused_naming_the_minimum(): void + { + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage('FlowTrackingConfiguration::withRemovalBatchSize(0) would never remove an expired flow. Pass 1 or more; the default is 1000.'); + + FlowTrackingConfiguration::createWithDefaults()->withRemovalBatchSize(0); + } + #[DataProvider('implementations')] public function test_traces_older_than_the_retention_are_removed_by_the_console_command(string $implementation): void { diff --git a/upgrade-2.0.md b/upgrade-2.0.md index 90a294dca..d6a7d6fa1 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3158,7 +3158,9 @@ same flow waits for the first one's commit and, on PostgreSQL and SQLite, reads across channels in the W3C `tracestate` as `ecotone=carried` next to any other vendor's entry, so ordinary traffic costs nothing. `withRetention()` sets how long `ecotone:tracing:remove-expired-traces` keeps completed flows, seven days by default; a flow still pending is kept twice as long, so a message delayed past the retention still completes -it. A multi-tenant application tracks each flow in the database of its tenant, inside the tenant's transaction, so a +it. Nothing runs the command for you: schedule it, from cron or the framework's scheduler, or the tables keep every +flow. It removes expired flows in batches of 1000, each batch's spans and flow rows in one short transaction, until +none is left, so a large backlog never holds one long lock; `withRemovalBatchSize()` changes the size. A multi-tenant application tracks each flow in the database of its tenant, inside the tenant's transaction, so a flow is never complete before the tenant's data committed. `TracingGateway` is then asked with the tenant, `$tracingGateway->await($tracingId, $timeout, metadata: ['tenant' => 'tenant_a'])`, and a call without one is refused naming that metadata; the retention command runs once per tenant, From bd94e98f0ce9a22c9535184096fb62fdff001853 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 42/50] feat(dbal): TracingGateway awaits a class, standing for its projection or every handler in it isCompletedFor() and awaitFor() take a class as the destination: a #[Projection] class resolves to its projection name, any other class to the endpoint ids of every handler method in it, explicit or generated, mapped at compile time. The class completes once every one of its handlers the flow reached ran and nothing upstream can still send it more. A class Ecotone registered no handler or projection for is refused at once instead of timing out. --- .../Dbal/src/Tracing/DbalTracingModule.php | 26 ++++++ .../Dbal/src/Tracing/StoredTracingGateway.php | 29 +++++-- packages/Dbal/src/Tracing/TraceProgress.php | 22 +++-- .../FlowTrackingConformanceTest.php | 27 ++++++ .../Integration/Tracing/FlowTrackingTest.php | 83 +++++++++++++++++++ upgrade-2.0.md | 8 +- 6 files changed, 183 insertions(+), 12 deletions(-) diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index 5057af511..addb5e9a2 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -6,10 +6,14 @@ use Ecotone\AnnotationFinder\AnnotationFinder; use Ecotone\Api\Attribute\AsynchronousRunningEndpoint; +use Ecotone\Api\Attribute\CommandHandler; +use Ecotone\Api\Attribute\EventHandler; +use Ecotone\Api\Attribute\InternalHandler; use Ecotone\Api\Attribute\MessageGateway; use Ecotone\Api\Attribute\ModuleAnnotation; use Ecotone\Api\Attribute\OrchestratorGateway; use Ecotone\Api\Attribute\Polling; +use Ecotone\Api\Attribute\QueryHandler; use Ecotone\Api\Dbal\ExtensionObject\DbalConfiguration; use Ecotone\Api\Dbal\ExtensionObject\DbalConnectionReference; use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; @@ -87,12 +91,14 @@ final class DbalTracingModule extends NoExternalConfigurationModule implements A * @param string[] $flowStepCompletedSubscribers * @param array $subscriberMethods * @param string[] $pollingProjectionNames + * @param array $destinationsByClass */ private function __construct( private array $flowCompletedSubscribers, private array $flowStepCompletedSubscribers, private array $subscriberMethods, private array $pollingProjectionNames, + private array $destinationsByClass, ) { } @@ -106,6 +112,7 @@ public static function create(AnnotationFinder $annotationRegistrationService, I ...self::subscriberMethodsOf($annotationRegistrationService, OnFlowStepCompleted::class, FlowStepCompleted::class), ], self::pollingProjectionNames($annotationRegistrationService), + self::destinationsByClass($annotationRegistrationService), ); } @@ -179,6 +186,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO new Reference(TraceStore::class), new Reference(EcotoneClockInterface::class), $this->pollingProjectionNames, + $this->destinationsByClass, $multiTenancy?->tenantHeaderName, $multiTenancy === null ? null : (string) array_key_first($multiTenancy->tenantToConnectionMapping), ])); @@ -331,6 +339,24 @@ private static function multiTenancyWithoutDefaultConnectionOf(array $extensionO return null; } + /** + * @return array + */ + private static function destinationsByClass(AnnotationFinder $annotationFinder): array + { + $destinations = []; + foreach ([CommandHandler::class, EventHandler::class, QueryHandler::class, InternalHandler::class] as $handlerAttribute) { + foreach ($annotationFinder->findAnnotatedMethods($handlerAttribute) as $annotatedMethod) { + $destinations[$annotatedMethod->getClassName()][] = $annotatedMethod->getAnnotationForMethod()->getEndpointId(); + } + } + foreach ($annotationFinder->findAnnotatedClasses(Projection::class) as $projectionClass) { + $destinations[$projectionClass] = [$annotationFinder->getAttributeForClass($projectionClass, Projection::class)->name]; + } + + return array_map(fn (array $classDestinations): array => array_values(array_unique($classDestinations)), $destinations); + } + /** * @return string[] */ diff --git a/packages/Dbal/src/Tracing/StoredTracingGateway.php b/packages/Dbal/src/Tracing/StoredTracingGateway.php index d976e56fd..df802cf2c 100644 --- a/packages/Dbal/src/Tracing/StoredTracingGateway.php +++ b/packages/Dbal/src/Tracing/StoredTracingGateway.php @@ -39,11 +39,13 @@ final class StoredTracingGateway /** * @param string[] $pollingProjectionNames + * @param array $destinationsByClass */ public function __construct( private TraceStore $traceStore, private EcotoneClockInterface $clock, private array $pollingProjectionNames, + private array $destinationsByClass, private ?string $tenantHeaderName, private ?string $exampleTenant, ) { @@ -60,9 +62,9 @@ public function isCompletedFor(string $tracingId, #[Header(self::DESTINATION_HEA { $this->refuseMissingTenant('isCompletedFor($tracingId, $destination', $headers); $tracingId = TracingId::normalize($tracingId); - $this->refusePollingProjection($destination); + $destinations = $this->destinationsOf($destination); - return $this->traceStore->progressOf($tracingId)->isCompletedFor($destination); + return $this->traceStore->progressOf($tracingId)->isCompletedFor(...$destinations); } public function await(string $tracingId, #[Header(self::TIMEOUT_HEADER)] Duration $timeout, #[Headers] array $headers): void @@ -88,17 +90,17 @@ public function awaitFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] { $this->refuseMissingTenant('awaitFor($tracingId, $destination, $timeout', $headers); $tracingId = TracingId::normalize($tracingId); - $this->refusePollingProjection($destination); + $destinations = $this->destinationsOf($destination); $deadline = $this->clock->now()->add($timeout); while (true) { $progress = $this->traceStore->progressOf($tracingId); - if ($progress->isCompletedFor($destination)) { + if ($progress->isCompletedFor(...$destinations)) { return; } if ($this->clock->now() >= $deadline) { throw match (true) { ! $progress->recorded => FlowTrackingTimedOut::forUnrecordedTrace($tracingId, $timeout), - ! $progress->reached($destination) => FlowTrackingTimedOut::forUnreachedDestination($tracingId, $destination, $timeout, $progress->destinations()), + ! $progress->reached(...$destinations) => FlowTrackingTimedOut::forUnreachedDestination($tracingId, $destination, $timeout, $progress->destinations()), default => FlowTrackingTimedOut::forDestination($tracingId, $destination, $timeout), }; } @@ -123,6 +125,23 @@ private function refuseMissingTenant(string $call, array $headers): void )); } + /** + * @return string[] + */ + private function destinationsOf(string $destination): array + { + $destinations = $this->destinationsByClass[$destination] ?? null; + if ($destinations === null && (class_exists($destination) || interface_exists($destination))) { + throw ConfigurationException::create(sprintf('%s has no handler or projection Ecotone registered, so no flow can reach it. Pass a class with a #[CommandHandler], #[EventHandler], #[QueryHandler] or #[InternalHandler] method, a #[Projection] class, an endpointId or a projection name.', $destination)); + } + $destinations ??= [$destination]; + foreach ($destinations as $resolvedDestination) { + $this->refusePollingProjection($resolvedDestination); + } + + return $destinations; + } + private function refusePollingProjection(string $destination): void { if (in_array($destination, $this->pollingProjectionNames, true)) { diff --git a/packages/Dbal/src/Tracing/TraceProgress.php b/packages/Dbal/src/Tracing/TraceProgress.php index f107c0614..264e6fb93 100644 --- a/packages/Dbal/src/Tracing/TraceProgress.php +++ b/packages/Dbal/src/Tracing/TraceProgress.php @@ -55,13 +55,19 @@ public function isCompleted(): bool return $this->recorded && $this->rootEnded && $this->pendingDestinations() === []; } - public function isCompletedFor(string $destination): bool + public function isCompletedFor(string ...$destinations): bool { if (! $this->recorded) { return false; } - $processed = $this->processed[$destination] ?? []; - if ($processed === [] || array_diff_key($this->sent[$destination] ?? [], $processed) !== []) { + $processed = []; + foreach ($destinations as $destination) { + if (array_diff_key($this->sent[$destination] ?? [], $this->processed[$destination] ?? []) !== []) { + return false; + } + $processed += $this->processed[$destination] ?? []; + } + if ($processed === []) { return false; } @@ -128,9 +134,15 @@ public function pendingDestinations(): array return array_values(array_unique($pending)); } - public function reached(string $destination): bool + public function reached(string ...$destinations): bool { - return isset($this->sent[$destination]) || isset($this->processed[$destination]); + foreach ($destinations as $destination) { + if (isset($this->sent[$destination]) || isset($this->processed[$destination])) { + return true; + } + } + + return false; } /** diff --git a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php index 38860c6c1..4de2a2f80 100644 --- a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php +++ b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php @@ -269,6 +269,33 @@ public function test_a_removal_batch_size_below_one_is_refused_naming_the_minimu FlowTrackingConfiguration::createWithDefaults()->withRemovalBatchSize(0); } + #[DataProvider('implementations')] + public function test_a_handler_class_is_completed_once_every_one_of_its_handlers_the_flow_reached_ran(string $implementation): void + { + $notifications = new class () { + #[Asynchronous('mail_channel')] + #[EventHandler('user.registered', endpointId: 'user.mailer')] + public function mail(string $userId): void + { + } + + #[Asynchronous('sms_channel')] + #[EventHandler('user.registered', endpointId: 'user.texter')] + public function text(string $userId): void + { + } + }; + $ecotone = $this->bootstrap($implementation, [$this->users(), $notifications], [SimpleMessageChannelBuilder::createQueueChannel('mail_channel'), SimpleMessageChannelBuilder::createQueueChannel('sms_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $ecotone->run('mail_channel', ExecutionPollingMetadata::createWithTestingSetup()); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, $notifications::class)); + + $ecotone->run('sms_channel', ExecutionPollingMetadata::createWithTestingSetup()); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, $notifications::class)); + } + #[DataProvider('implementations')] public function test_traces_older_than_the_retention_are_removed_by_the_console_command(string $implementation): void { diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index 809fe753c..b1609b83d 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -115,6 +115,89 @@ public function whenRegistered(array $event): void $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); } + public function test_an_asynchronous_projection_is_awaited_by_its_class(): void + { + $projection = new #[Projection('users_list'), FromStream('users_stream'), Asynchronous('async_channel')] class () { + #[EventHandler('user.registered')] + public function whenRegistered(array $event): void + { + } + }; + $ecotone = $this->bootstrap([$projection], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $ecotone->withEvents([Event::createWithType('user.registered', ['userId' => '1'])]); + $trace = SpanContext::newTrace(); + $ecotone->publishEventWithRouting('user.registered', [], metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, $projection::class)); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, $projection::class)); + $this->completion($ecotone)->awaitFor($trace->traceId, $projection::class, Duration::milliseconds(100)); + } + + public function test_a_handler_class_is_completed_once_every_one_of_its_handlers_the_flow_reached_ran(): void + { + $notifications = new class () { + #[Asynchronous('mail_channel')] + #[EventHandler('user.registered', endpointId: 'user.mailer')] + public function mail(string $userId): void + { + } + + #[Asynchronous('sms_channel')] + #[EventHandler('user.registered', endpointId: 'user.texter')] + public function text(string $userId): void + { + } + }; + $ecotone = $this->bootstrap([$this->users(), $notifications], [SimpleMessageChannelBuilder::createQueueChannel('mail_channel'), SimpleMessageChannelBuilder::createQueueChannel('sms_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $ecotone->run('mail_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, $notifications::class)); + + $ecotone->run('sms_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, $notifications::class)); + } + + public function test_a_synchronous_handler_without_an_endpoint_id_is_awaited_by_its_class(): void + { + $users = $this->users(); + $ecotone = $this->bootstrap([$users]); + $trace = SpanContext::newTrace(); + + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, $users::class)); + } + + public function test_awaiting_a_class_with_no_handler_or_projection_is_refused_naming_what_it_takes(): void + { + $ecotone = $this->bootstrap([$this->users()]); + + $this->expectException(ConfigurationException::class); + $this->expectExceptionMessage(sprintf('%s has no handler or projection Ecotone registered, so no flow can reach it. Pass a class with a #[CommandHandler], #[EventHandler], #[QueryHandler] or #[InternalHandler] method, a #[Projection] class, an endpointId or a projection name.', FlowTrackingTest::class)); + + $this->completion($ecotone)->isCompletedFor(SpanContext::newTrace()->traceId, FlowTrackingTest::class); + } + + public function test_awaiting_a_polling_projection_by_its_class_is_refused_because_it_runs_outside_the_trace(): void + { + $projection = new #[Projection('orders_list'), FromStream('orders_stream'), Polling('orders_list_polling')] class () { + #[EventHandler('order.placed')] + public function whenPlaced(array $event): void + { + } + }; + $ecotone = $this->bootstrap([$projection]); + + $this->expectExceptionMessage('Projection orders_list runs by #[Polling]'); + + $this->completion($ecotone)->isCompletedFor(SpanContext::newTrace()->traceId, $projection::class); + } + public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pending(): void { $ecotone = $this->bootstrap([$this->users(), $this->asynchronousNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); diff --git a/upgrade-2.0.md b/upgrade-2.0.md index d6a7d6fa1..cb157f99b 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3104,11 +3104,15 @@ reaches, synchronously or asynchronously, across processes. What it tracks is ke Two features are on by default and each can be turned off while the flows are still tracked: - **`TracingGateway`** answers `isCompleted($tracingId)`, `isCompletedFor($tracingId, $destination)` for one - `endpointId` or projection name, and `await()` / `awaitFor()`, which poll until then. A destination is + `endpointId`, projection name or class, and `await()` / `awaitFor()`, which poll until then. A destination is complete once it processed every message the flow sent it and nothing still pending upstream of it could send it another: only work its own handling caused, through a new message, may still be pending. A message it forwards unchanged, through an output channel, is waited for too, since it carries the same message id as an event's copy - queued for a sibling handler. Past the timeout, the awaits throw `FlowTrackingTimedOut` naming the handlers and channels still pending, pointing to the dead letter when a + queued for a sibling handler. A class stands for everything in it the flow can reach: a `#[Projection]` class for + its projection, any other class for every handler method in it, endpoint id or not, so + `awaitFor($tracingId, UsersListProjection::class, $timeout)` needs no name to be repeated, and the class completes + once every one of its handlers the flow reached ran. A class with no handler or projection is refused at once. + Past the timeout, the awaits throw `FlowTrackingTimedOut` naming the handlers and channels still pending, pointing to the dead letter when a handler failed, and naming a flow nobody recorded or a destination the flow never reached. `withTracingGateway(false)` turns it off; the gateway stays injectable and every call names that setting. - **Notifications**: a method with `#[OnFlowCompleted]` receives a `FlowCompleted` each time its flow reaches From bb3ebf9de395d1901cc9c6544c65134d7647f365 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 43/50] feat(dbal): TracingGateway refuses to await longer than 15 minutes, pointing to flow notifications An await blocks the calling process while it polls. A timeout above 15 minutes is now refused before any polling, naming #[OnFlowCompleted] and #[OnFlowStepCompleted] so a long flow is reacted to when it completes instead of waited for. --- .../Dbal/src/Tracing/StoredTracingGateway.php | 16 +++++++++ .../Integration/Tracing/FlowTrackingTest.php | 33 +++++++++++++++++++ upgrade-2.0.md | 3 ++ 3 files changed, 52 insertions(+) diff --git a/packages/Dbal/src/Tracing/StoredTracingGateway.php b/packages/Dbal/src/Tracing/StoredTracingGateway.php index df802cf2c..4e6062494 100644 --- a/packages/Dbal/src/Tracing/StoredTracingGateway.php +++ b/packages/Dbal/src/Tracing/StoredTracingGateway.php @@ -36,6 +36,7 @@ final class StoredTracingGateway public const DESTINATION_HEADER = 'ecotone_tracing_completion_destination'; public const TIMEOUT_HEADER = 'ecotone_tracing_completion_timeout'; private const POLL_INTERVAL_IN_MILLISECONDS = 50; + private const LONGEST_AWAIT_IN_MILLISECONDS = 15 * 60 * 1000; /** * @param string[] $pollingProjectionNames @@ -70,6 +71,7 @@ public function isCompletedFor(string $tracingId, #[Header(self::DESTINATION_HEA public function await(string $tracingId, #[Header(self::TIMEOUT_HEADER)] Duration $timeout, #[Headers] array $headers): void { $this->refuseMissingTenant('await($tracingId, $timeout', $headers); + $this->refuseAwaitLongerThanFifteenMinutes('await', $timeout); $tracingId = TracingId::normalize($tracingId); $deadline = $this->clock->now()->add($timeout); while (true) { @@ -89,6 +91,7 @@ public function await(string $tracingId, #[Header(self::TIMEOUT_HEADER)] Duratio public function awaitFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] string $destination, #[Header(self::TIMEOUT_HEADER)] Duration $timeout, #[Headers] array $headers): void { $this->refuseMissingTenant('awaitFor($tracingId, $destination, $timeout', $headers); + $this->refuseAwaitLongerThanFifteenMinutes('awaitFor', $timeout); $tracingId = TracingId::normalize($tracingId); $destinations = $this->destinationsOf($destination); $deadline = $this->clock->now()->add($timeout); @@ -108,6 +111,19 @@ public function awaitFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] } } + private function refuseAwaitLongerThanFifteenMinutes(string $method, Duration $timeout): void + { + if ($timeout->inMilliseconds() <= self::LONGEST_AWAIT_IN_MILLISECONDS) { + return; + } + + throw InvalidArgumentException::create(sprintf( + 'TracingGateway::%s() was asked to wait %d milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Await at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowCompleted] receives a FlowCompleted once the flow completes, and one with #[OnFlowStepCompleted] a FlowStepCompleted for each of its steps.', + $method, + $timeout->inMilliseconds(), + )); + } + /** * @param array $headers */ diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index b1609b83d..facd48ae1 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -37,6 +37,7 @@ use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\SpanStatus; use Ecotone\Messaging\Config\ConfigurationException; +use Ecotone\Messaging\Support\InvalidArgumentException; use Ecotone\Messaging\Support\LicensingException; use Ecotone\Test\LicenceTesting; use Ecotone\Test\StubLogger; @@ -198,6 +199,38 @@ public function whenPlaced(array $event): void $this->completion($ecotone)->isCompletedFor(SpanContext::newTrace()->traceId, $projection::class); } + public function test_awaiting_a_flow_longer_than_fifteen_minutes_is_refused_proposing_notifications(): void + { + $ecotone = $this->bootstrap([$this->users()]); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('TracingGateway::await() was asked to wait 960000 milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Await at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowCompleted] receives a FlowCompleted once the flow completes, and one with #[OnFlowStepCompleted] a FlowStepCompleted for each of its steps.'); + + $this->completion($ecotone)->await(SpanContext::newTrace()->traceId, Duration::minutes(16)); + } + + public function test_awaiting_a_destination_longer_than_fifteen_minutes_is_refused_proposing_notifications(): void + { + $ecotone = $this->bootstrap([$this->users()]); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('TracingGateway::awaitFor() was asked to wait 960000 milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Await at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowCompleted] receives a FlowCompleted once the flow completes, and one with #[OnFlowStepCompleted] a FlowStepCompleted for each of its steps.'); + + $this->completion($ecotone)->awaitFor(SpanContext::newTrace()->traceId, 'user.registration', Duration::minutes(16)); + } + + public function test_awaiting_a_completed_flow_for_exactly_fifteen_minutes_is_allowed(): void + { + $ecotone = $this->bootstrap([$this->users()]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $this->completion($ecotone)->await($trace->traceId, Duration::minutes(15)); + $this->completion($ecotone)->awaitFor($trace->traceId, 'user.registration', Duration::minutes(15)); + + $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + } + public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pending(): void { $ecotone = $this->bootstrap([$this->users(), $this->asynchronousNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); diff --git a/upgrade-2.0.md b/upgrade-2.0.md index cb157f99b..dc0525a1e 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3112,6 +3112,9 @@ Two features are on by default and each can be turned off while the flows are st its projection, any other class for every handler method in it, endpoint id or not, so `awaitFor($tracingId, UsersListProjection::class, $timeout)` needs no name to be repeated, and the class completes once every one of its handlers the flow reached ran. A class with no handler or projection is refused at once. + An await blocks the calling process while it polls, so a timeout above 15 minutes is refused, pointing to + `#[OnFlowCompleted]` and `#[OnFlowStepCompleted]`: a flow that takes longer is reacted to when it completes, not + waited for. Past the timeout, the awaits throw `FlowTrackingTimedOut` naming the handlers and channels still pending, pointing to the dead letter when a handler failed, and naming a flow nobody recorded or a destination the flow never reached. `withTracingGateway(false)` turns it off; the gateway stays injectable and every call names that setting. From 713a2682e4d1d7532ce56d42ec127babdb82bb65 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 44/50] feat(dbal): TracingGateway is isCompletedFor(), isCompletedWithin() returning a boolean, and fetchFlow() returning a Flow isCompletedFor() takes an optional destination and replaces isCompleted(); isCompletedWithin() replaces await() and awaitFor(), returning false once the timeout passed instead of throwing FlowTrackingTimedOut, which is removed. fetchFlow() returns a Flow - isCompleted, pending, reached and its stored steps in start order - or null for a flow nobody recorded, so a false is explained by reading the flow. --- .../FlowTrackingWithNoopProviderTest.php | 2 +- .../FlowTrackingWithOpenTelemetryTest.php | 6 +- packages/Dbal/Api/Tracing/Flow.php | 28 +++ .../Dbal/Api/Tracing/FlowTrackingTimedOut.php | 64 ------- packages/Dbal/Api/Tracing/TracingGateway.php | 8 +- packages/Dbal/src/Tracing/DbalTraceStore.php | 5 + .../Dbal/src/Tracing/DbalTracingModule.php | 5 +- .../Dbal/src/Tracing/InMemoryTraceStore.php | 5 + .../Dbal/src/Tracing/StoredTracingGateway.php | 80 ++++----- packages/Dbal/src/Tracing/TraceStore.php | 5 + .../src/Tracing/UnavailableTracingGateway.php | 14 +- .../FlowTrackingConformanceTest.php | 81 +++++---- .../Integration/Tracing/FlowTrackingTest.php | 167 ++++++++++-------- .../Projecting/FlowTrackingProjectionTest.php | 4 +- upgrade-2.0.md | 35 ++-- 15 files changed, 243 insertions(+), 266 deletions(-) create mode 100644 packages/Dbal/Api/Tracing/Flow.php delete mode 100644 packages/Dbal/Api/Tracing/FlowTrackingTimedOut.php diff --git a/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithNoopProviderTest.php b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithNoopProviderTest.php index c47b00dd3..25e5ff649 100644 --- a/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithNoopProviderTest.php +++ b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithNoopProviderTest.php @@ -84,6 +84,6 @@ public function whenEnded(FlowStepCompleted $event): void $this->assertCount(count($subscriber->spanIds), array_unique($subscriber->spanIds)); $this->assertGreaterThan(3, count($subscriber->spanIds)); - $this->assertTrue($ecotone->getGateway(TracingGateway::class)->isCompleted($trace->traceId)); + $this->assertTrue($ecotone->getGateway(TracingGateway::class)->isCompletedFor($trace->traceId)); } } diff --git a/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php index 033c57542..d4ac92a2c 100644 --- a/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php +++ b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php @@ -70,7 +70,7 @@ public function test_one_trace_is_exported_to_open_telemetry_and_stored_in_the_d self::exportedSpanNamesOf($exporter, $trace->traceId), ); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.registration')); - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); self::assertSame([], $subscriber->completedTraces); $ecotone->run('notifications', ExecutionPollingMetadata::createWithTestingSetup()); @@ -80,7 +80,7 @@ public function test_one_trace_is_exported_to_open_telemetry_and_stored_in_the_d self::exportedSpanNamesOf($exporter, $trace->traceId), ); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); - self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); self::assertSame([$trace->traceId], $subscriber->completedTraces); } @@ -115,7 +115,7 @@ public function test_a_second_application_on_the_same_database_sees_the_completi $producer->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - self::assertTrue($this->completion($observer)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($observer)->isCompletedFor($trace->traceId)); } /** diff --git a/packages/Dbal/Api/Tracing/Flow.php b/packages/Dbal/Api/Tracing/Flow.php new file mode 100644 index 000000000..0b1cb8654 --- /dev/null +++ b/packages/Dbal/Api/Tracing/Flow.php @@ -0,0 +1,28 @@ +inMilliseconds(), - implode(', ', $pendingDestinations), - )); - } - - public static function forUnrecordedTrace(string $traceId, Duration $timeout): self - { - return new self(sprintf( - 'Nothing was recorded for flow %s within %d milliseconds. Send the command with that tracingId in its metadata, and check that the application handling it registers FlowTrackingConfiguration.', - $traceId, - $timeout->inMilliseconds(), - )); - } - - public static function forDestination(string $traceId, string $destination, Duration $timeout): self - { - return new self(sprintf( - 'Flow %s did not complete for %s within %d milliseconds. Run the consumer of the channel that handler is attached to, replay the message from the dead letter if the handler failed, or await with a longer timeout.', - $traceId, - $destination, - $timeout->inMilliseconds(), - )); - } - - /** - * @param string[] $reachedDestinations - */ - public static function forUnreachedDestination(string $traceId, string $destination, Duration $timeout, array $reachedDestinations): self - { - return new self(sprintf( - 'Flow %s was neither sent to nor processed by %s within %d milliseconds. Check the endpointId or projection name; the flow reached: %s.', - $traceId, - $destination, - $timeout->inMilliseconds(), - $reachedDestinations === [] ? 'nothing yet' : implode(', ', $reachedDestinations), - )); - } -} diff --git a/packages/Dbal/Api/Tracing/TracingGateway.php b/packages/Dbal/Api/Tracing/TracingGateway.php index 96f50095a..3238db596 100644 --- a/packages/Dbal/Api/Tracing/TracingGateway.php +++ b/packages/Dbal/Api/Tracing/TracingGateway.php @@ -14,11 +14,9 @@ #[PropagateHeaders] interface TracingGateway { - public function isCompleted(string $tracingId, array $metadata = []): bool; + public function isCompletedFor(string $tracingId, ?string $destination = null, array $metadata = []): bool; - public function isCompletedFor(string $tracingId, string $destination, array $metadata = []): bool; + public function isCompletedWithin(string $tracingId, Duration $timeout, ?string $destination = null, array $metadata = []): bool; - public function await(string $tracingId, Duration $timeout, array $metadata = []): void; - - public function awaitFor(string $tracingId, string $destination, Duration $timeout, array $metadata = []): void; + public function fetchFlow(string $tracingId, array $metadata = []): ?Flow; } diff --git a/packages/Dbal/src/Tracing/DbalTraceStore.php b/packages/Dbal/src/Tracing/DbalTraceStore.php index 6c81c8396..fc08f93a3 100644 --- a/packages/Dbal/src/Tracing/DbalTraceStore.php +++ b/packages/Dbal/src/Tracing/DbalTraceStore.php @@ -80,6 +80,11 @@ public function markCompletedIfDone(string $traceId, DatePoint $now): bool return $this->onTables(fn (): bool => TraceLedger::progressOf($traceId, $this->spansOf($connection, $traceId))->isCompleted() && $this->markCompleted($connection, $traceId, $now)); } + public function stepsOf(string $traceId): array + { + return $this->onTables(fn (): array => $this->spansOf($this->connection(), $traceId)); + } + public function progressOf(string $traceId): TraceProgress { return $this->onTables(fn (): TraceProgress => TraceLedger::progressOf($traceId, $this->spansOf($this->connection(), $traceId))); diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index addb5e9a2..931153554 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -303,10 +303,9 @@ private static function registerTracingGateway(Configuration $messagingConfigura { $messagingConfiguration->registerServiceDefinition($serviceClass, $service); foreach ([ - 'isCompleted' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeadersBuilder::create('metadata')], 'isCompletedFor' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('destination', StoredTracingGateway::DESTINATION_HEADER), GatewayHeadersBuilder::create('metadata')], - 'await' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('timeout', StoredTracingGateway::TIMEOUT_HEADER), GatewayHeadersBuilder::create('metadata')], - 'awaitFor' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('destination', StoredTracingGateway::DESTINATION_HEADER), GatewayHeaderBuilder::create('timeout', StoredTracingGateway::TIMEOUT_HEADER), GatewayHeadersBuilder::create('metadata')], + 'isCompletedWithin' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeaderBuilder::create('timeout', StoredTracingGateway::TIMEOUT_HEADER), GatewayHeaderBuilder::create('destination', StoredTracingGateway::DESTINATION_HEADER), GatewayHeadersBuilder::create('metadata')], + 'fetchFlow' => [GatewayPayloadBuilder::create('tracingId'), GatewayHeadersBuilder::create('metadata')], ] as $method => $parameterConverters) { $channel = 'ecotone.tracing.gateway.' . $method; $messagingConfiguration diff --git a/packages/Dbal/src/Tracing/InMemoryTraceStore.php b/packages/Dbal/src/Tracing/InMemoryTraceStore.php index 502565943..896e38a2c 100644 --- a/packages/Dbal/src/Tracing/InMemoryTraceStore.php +++ b/packages/Dbal/src/Tracing/InMemoryTraceStore.php @@ -59,6 +59,11 @@ public function progressOf(string $traceId): TraceProgress return TraceLedger::progressOf($traceId, array_values($this->spansByTrace[$traceId] ?? [])); } + public function stepsOf(string $traceId): array + { + return array_values($this->spansByTrace[$traceId] ?? []); + } + public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore, int $batchSize): void { foreach ($this->spansByTrace as $traceId => $spans) { diff --git a/packages/Dbal/src/Tracing/StoredTracingGateway.php b/packages/Dbal/src/Tracing/StoredTracingGateway.php index 4e6062494..3c479ac2e 100644 --- a/packages/Dbal/src/Tracing/StoredTracingGateway.php +++ b/packages/Dbal/src/Tracing/StoredTracingGateway.php @@ -6,9 +6,10 @@ use Ecotone\Api\Attribute\Header; use Ecotone\Api\Attribute\Headers; -use Ecotone\Api\Dbal\Tracing\FlowTrackingTimedOut; +use Ecotone\Api\Dbal\Tracing\Flow; use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Scheduling\Duration; +use Ecotone\Api\Tracing\Span; use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Support\InvalidArgumentException; use Ecotone\Tracing\TracingId; @@ -52,74 +53,63 @@ public function __construct( ) { } - public function isCompleted(string $tracingId, #[Headers] array $headers): bool + public function isCompletedFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] ?string $destination, #[Headers] array $headers): bool { - $this->refuseMissingTenant('isCompleted($tracingId', $headers); + $this->refuseMissingTenant('isCompletedFor($tracingId', $headers); $tracingId = TracingId::normalize($tracingId); - return $this->traceStore->progressOf($tracingId)->isCompleted(); - } - public function isCompletedFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] string $destination, #[Headers] array $headers): bool - { - $this->refuseMissingTenant('isCompletedFor($tracingId, $destination', $headers); - $tracingId = TracingId::normalize($tracingId); - $destinations = $this->destinationsOf($destination); - - return $this->traceStore->progressOf($tracingId)->isCompletedFor(...$destinations); + return $this->isCompletedNow($tracingId, $destination === null ? null : $this->destinationsOf($destination)); } - public function await(string $tracingId, #[Header(self::TIMEOUT_HEADER)] Duration $timeout, #[Headers] array $headers): void + public function isCompletedWithin(string $tracingId, #[Header(self::TIMEOUT_HEADER)] Duration $timeout, #[Header(self::DESTINATION_HEADER)] ?string $destination, #[Headers] array $headers): bool { - $this->refuseMissingTenant('await($tracingId, $timeout', $headers); - $this->refuseAwaitLongerThanFifteenMinutes('await', $timeout); + $this->refuseMissingTenant('isCompletedWithin($tracingId, $timeout', $headers); + $this->refuseWaitingLongerThanFifteenMinutes($timeout); $tracingId = TracingId::normalize($tracingId); + $destinations = $destination === null ? null : $this->destinationsOf($destination); $deadline = $this->clock->now()->add($timeout); - while (true) { - $progress = $this->traceStore->progressOf($tracingId); - if ($progress->isCompleted()) { - return; - } + while (! $this->isCompletedNow($tracingId, $destinations)) { if ($this->clock->now() >= $deadline) { - throw $progress->recorded - ? FlowTrackingTimedOut::forTrace($tracingId, $timeout, $progress->pendingDestinations()) - : FlowTrackingTimedOut::forUnrecordedTrace($tracingId, $timeout); + return false; } $this->clock->sleep(Duration::milliseconds(self::POLL_INTERVAL_IN_MILLISECONDS)); } + + return true; } - public function awaitFor(string $tracingId, #[Header(self::DESTINATION_HEADER)] string $destination, #[Header(self::TIMEOUT_HEADER)] Duration $timeout, #[Headers] array $headers): void + public function fetchFlow(string $tracingId, #[Headers] array $headers): ?Flow { - $this->refuseMissingTenant('awaitFor($tracingId, $destination, $timeout', $headers); - $this->refuseAwaitLongerThanFifteenMinutes('awaitFor', $timeout); + $this->refuseMissingTenant('fetchFlow($tracingId', $headers); $tracingId = TracingId::normalize($tracingId); - $destinations = $this->destinationsOf($destination); - $deadline = $this->clock->now()->add($timeout); - while (true) { - $progress = $this->traceStore->progressOf($tracingId); - if ($progress->isCompletedFor(...$destinations)) { - return; - } - if ($this->clock->now() >= $deadline) { - throw match (true) { - ! $progress->recorded => FlowTrackingTimedOut::forUnrecordedTrace($tracingId, $timeout), - ! $progress->reached(...$destinations) => FlowTrackingTimedOut::forUnreachedDestination($tracingId, $destination, $timeout, $progress->destinations()), - default => FlowTrackingTimedOut::forDestination($tracingId, $destination, $timeout), - }; - } - $this->clock->sleep(Duration::milliseconds(self::POLL_INTERVAL_IN_MILLISECONDS)); + $steps = $this->traceStore->stepsOf($tracingId); + if ($steps === []) { + return null; } + usort($steps, fn (Span $first, Span $second): int => $first->startedAt <=> $second->startedAt); + $progress = TraceLedger::progressOf($tracingId, $steps); + + return new Flow($tracingId, $progress->isCompleted(), $progress->pendingDestinations(), $progress->destinations(), $steps); + } + + /** + * @param string[]|null $destinations + */ + private function isCompletedNow(string $tracingId, ?array $destinations): bool + { + $progress = $this->traceStore->progressOf($tracingId); + + return $destinations === null ? $progress->isCompleted() : $progress->isCompletedFor(...$destinations); } - private function refuseAwaitLongerThanFifteenMinutes(string $method, Duration $timeout): void + private function refuseWaitingLongerThanFifteenMinutes(Duration $timeout): void { if ($timeout->inMilliseconds() <= self::LONGEST_AWAIT_IN_MILLISECONDS) { return; } throw InvalidArgumentException::create(sprintf( - 'TracingGateway::%s() was asked to wait %d milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Await at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowCompleted] receives a FlowCompleted once the flow completes, and one with #[OnFlowStepCompleted] a FlowStepCompleted for each of its steps.', - $method, + 'TracingGateway::isCompletedWithin() was asked to wait %d milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Wait at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowCompleted] receives a FlowCompleted once the flow completes, and one with #[OnFlowStepCompleted] a FlowStepCompleted for each of its steps.', $timeout->inMilliseconds(), )); } @@ -161,7 +151,7 @@ private function destinationsOf(string $destination): array private function refusePollingProjection(string $destination): void { if (in_array($destination, $this->pollingProjectionNames, true)) { - throw ConfigurationException::create(sprintf('Projection %s runs by #[Polling], so it is not part of the trace that appended its events and TracingGateway cannot await it. Await the handler that published the event, or make the projection event-driven by removing #[Polling].', $destination)); + throw ConfigurationException::create(sprintf('Projection %s runs by #[Polling], so it is not part of the trace that appended its events and TracingGateway cannot wait for it. Await the handler that published the event, or make the projection event-driven by removing #[Polling].', $destination)); } } } diff --git a/packages/Dbal/src/Tracing/TraceStore.php b/packages/Dbal/src/Tracing/TraceStore.php index ecc8a016f..44d9fb59c 100644 --- a/packages/Dbal/src/Tracing/TraceStore.php +++ b/packages/Dbal/src/Tracing/TraceStore.php @@ -37,6 +37,11 @@ public function markCompletedIfDone(string $traceId, DatePoint $now): bool; public function progressOf(string $traceId): TraceProgress; + /** + * @return Span[] + */ + public function stepsOf(string $traceId): array; + public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore, int $batchSize): void; public function withinTransaction(Closure $work): mixed; diff --git a/packages/Dbal/src/Tracing/UnavailableTracingGateway.php b/packages/Dbal/src/Tracing/UnavailableTracingGateway.php index 6ac5935aa..8e79dbd80 100644 --- a/packages/Dbal/src/Tracing/UnavailableTracingGateway.php +++ b/packages/Dbal/src/Tracing/UnavailableTracingGateway.php @@ -4,8 +4,7 @@ namespace Ecotone\Dbal\Tracing; -use Ecotone\Api\Attribute\Header; -use Ecotone\Api\Scheduling\Duration; +use Ecotone\Api\Dbal\Tracing\Flow; use Ecotone\Messaging\Config\ConfigurationException; /** @@ -17,22 +16,17 @@ public function __construct(private string $refusal) { } - public function isCompleted(string $tracingId): bool + public function isCompletedFor(string $tracingId): bool { throw $this->refusal(); } - public function isCompletedFor(string $tracingId, #[Header(StoredTracingGateway::DESTINATION_HEADER)] string $destination): bool + public function isCompletedWithin(string $tracingId): bool { throw $this->refusal(); } - public function await(string $tracingId, #[Header(StoredTracingGateway::TIMEOUT_HEADER)] Duration $timeout): void - { - throw $this->refusal(); - } - - public function awaitFor(string $tracingId, #[Header(StoredTracingGateway::DESTINATION_HEADER)] string $destination, #[Header(StoredTracingGateway::TIMEOUT_HEADER)] Duration $timeout): void + public function fetchFlow(string $tracingId): ?Flow { throw $this->refusal(); } diff --git a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php index 4de2a2f80..844dc889d 100644 --- a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php +++ b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php @@ -20,7 +20,6 @@ use Ecotone\Api\Dbal\Tracing\FlowCompleted; use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; -use Ecotone\Api\Dbal\Tracing\FlowTrackingTimedOut; use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; @@ -91,7 +90,7 @@ public function test_a_synchronous_flow_is_completed_when_the_bus_returns(string $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.registration')); } @@ -103,26 +102,24 @@ public function test_an_asynchronous_handler_keeps_the_trace_pending_until_its_c $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); - self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } #[DataProvider('implementations')] - public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pending(string $implementation): void + public function test_waiting_for_a_pending_flow_returns_false_and_the_flow_names_what_is_still_pending(string $implementation): void { $ecotone = $this->bootstrap($implementation, [$this->users(), $this->asynchronousNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->expectException(FlowTrackingTimedOut::class); - $this->expectExceptionMessage('Still pending: user.notifier, async_channel. Run the consumer of the pending channel, replay the message from the dead letter'); - - $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); + self::assertFalse($this->completion($ecotone)->isCompletedWithin($trace->traceId, Duration::milliseconds(100))); + self::assertSame(['user.notifier', 'async_channel'], $this->completion($ecotone)->fetchFlow($trace->traceId)->pending); } #[DataProvider('implementations')] @@ -170,7 +167,7 @@ public function notify(string $userId): void } catch (RuntimeException) { } - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); } #[DataProvider('implementations')] @@ -231,7 +228,7 @@ public function archive(string $userId): void self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'notification.archiver')); - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); } #[DataProvider('implementations')] @@ -256,9 +253,9 @@ classesToResolve: [$this->users()::class], $ecotone->runConsoleCommand('ecotone:tracing:remove-expired-traces'); foreach ($expired as $trace) { - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); } - self::assertTrue($this->completion($ecotone)->isCompleted($recent->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($recent->traceId)); } public function test_a_removal_batch_size_below_one_is_refused_naming_the_minimum(): void @@ -309,8 +306,8 @@ public function test_traces_older_than_the_retention_are_removed_by_the_console_ $ecotone->runConsoleCommand('ecotone:tracing:remove-expired-traces'); - self::assertFalse($this->completion($ecotone)->isCompleted($expired->traceId)); - self::assertTrue($this->completion($ecotone)->isCompleted($recent->traceId)); + self::assertFalse($this->completion($ecotone)->isCompletedFor($expired->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($recent->traceId)); } #[DataProvider('implementations')] @@ -328,12 +325,12 @@ public function place(string $orderId): void $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); - self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_a_consumer_alone_does_not_complete_a_trace_whose_producer_stored_nothing(): void @@ -351,7 +348,7 @@ classesToResolve: [$this->asynchronousNotifier()::class], $consumer->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); self::assertTrue($this->completion($consumer)->isCompletedFor($trace->traceId, 'user.notifier')); - self::assertFalse($this->completion($consumer)->isCompleted($trace->traceId)); + self::assertFalse($this->completion($consumer)->isCompletedFor($trace->traceId)); } public function test_a_flow_started_by_a_distributed_command_completes_once_the_receiving_service_handled_it(): void @@ -372,12 +369,12 @@ public function create(string $description): void $tracingId = '0af7651916cd43dd8448eb211c80319c'; $userService->getDistributedBus()->sendCommand('ticket_service', 'ticket.create', 'User changed billing address', metadata: [MessageHeaders::TRACING_ID => $tracingId]); - self::assertFalse($this->completion($userService)->isCompleted($tracingId)); + self::assertFalse($this->completion($userService)->isCompletedFor($tracingId)); $ticketService->run('distributed_ticket_channel', ExecutionPollingMetadata::createWithTestingSetup()); self::assertSame(1, $tickets->created); - self::assertTrue($this->completion($userService)->isCompleted($tracingId)); + self::assertTrue($this->completion($userService)->isCompletedFor($tracingId)); } public function test_a_send_rolled_back_with_its_failed_handler_is_not_awaited_after_the_retry_succeeded(): void @@ -414,7 +411,7 @@ public function audit(string $userId): void $ecotone->run('audit_channel', ExecutionPollingMetadata::createWithTestingSetup()); self::assertSame(2, $notifier->attempts); - self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } #[DataProvider('implementations')] @@ -442,7 +439,7 @@ public function test_a_consumer_that_swallowed_its_handler_failure_leaves_the_tr $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); } #[DataProvider('implementations')] @@ -468,13 +465,13 @@ public function notify(string $userId): void $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); $ecotone->getGateway(DeadLetterGateway::class)->replayAll(); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); self::assertSame(2, $notifier->attempts); - self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } #[DataProvider('implementations')] @@ -487,7 +484,7 @@ public function test_a_pending_trace_older_than_the_retention_survives_until_twi $ecotone->advanceTimeBy(TimeSpan::withMinutes(11)); $ecotone->runConsoleCommand('ecotone:tracing:remove-expired-traces'); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - self::assertTrue($this->completion($ecotone)->isCompleted($delayed->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($delayed->traceId)); $abandoned = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '2', metadata: [MessageHeaders::TRACEPARENT => $abandoned->toTraceparent()]); @@ -552,12 +549,12 @@ public function place(string $orderId): void self::assertSame([$trace->traceId], $subscriber->completedTraces); $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); self::assertSame([$trace->traceId], $subscriber->completedTraces); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); self::assertSame([$trace->traceId, $trace->traceId], $subscriber->completedTraces); } @@ -567,12 +564,12 @@ public function test_a_channel_published_through_the_outbox_completes_once_its_c $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); - self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_flow_tracking_stores_each_flow_in_the_database_of_its_tenant(): void @@ -581,14 +578,16 @@ public function test_flow_tracking_stores_each_flow_in_the_database_of_its_tenan $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent(), 'tenant' => 'tenant_a']); - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId, metadata: ['tenant' => 'tenant_a'])); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, metadata: ['tenant' => 'tenant_a'])); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId, metadata: ['tenant' => 'tenant_a'])); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, metadata: ['tenant' => 'tenant_a'])); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier', metadata: ['tenant' => 'tenant_a'])); - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId, metadata: ['tenant' => 'tenant_b'])); - $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100), metadata: ['tenant' => 'tenant_a']); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, metadata: ['tenant' => 'tenant_b'])); + self::assertTrue($this->completion($ecotone)->isCompletedWithin($trace->traceId, Duration::milliseconds(100), metadata: ['tenant' => 'tenant_a'])); + self::assertTrue($this->completion($ecotone)->fetchFlow($trace->traceId, metadata: ['tenant' => 'tenant_a'])->isCompleted); + self::assertNull($this->completion($ecotone)->fetchFlow($trace->traceId, metadata: ['tenant' => 'tenant_b'])); } public function test_a_tenant_flow_is_written_inside_the_tenant_transaction_and_rolls_back_with_it(): void @@ -610,7 +609,7 @@ public function test_a_tenant_flow_is_written_inside_the_tenant_transaction_and_ CommitFailingConnection::$failsCommit = false; } - self::assertFalse($this->completion($ecotone)->isCompleted($trace->traceId, metadata: ['tenant' => 'tenant_a'])); + self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, metadata: ['tenant' => 'tenant_a'])); } public function test_expired_flows_are_removed_from_the_database_of_the_tenant_the_console_command_names(): void @@ -624,17 +623,17 @@ public function test_expired_flows_are_removed_from_the_database_of_the_tenant_t $ecotone->runConsoleCommand('ecotone:tracing:remove-expired-traces', ['header' => ['tenant:tenant_a']]); - self::assertFalse($this->completion($ecotone)->isCompleted($ofTenantA->traceId, metadata: ['tenant' => 'tenant_a'])); - self::assertTrue($this->completion($ecotone)->isCompleted($ofTenantB->traceId, metadata: ['tenant' => 'tenant_b'])); + self::assertFalse($this->completion($ecotone)->isCompletedFor($ofTenantA->traceId, metadata: ['tenant' => 'tenant_a'])); + self::assertTrue($this->completion($ecotone)->isCompletedFor($ofTenantB->traceId, metadata: ['tenant' => 'tenant_b'])); } public function test_asking_the_tracing_gateway_of_a_multi_tenant_application_without_a_tenant_names_the_metadata_it_needs(): void { $ecotone = $this->bootstrapMultiTenant([$this->users()], $this->connectionForTenantA()); - $this->expectExceptionMessage("TracingGateway reads a flow from the database of the tenant it ran for, and this call names no tenant. Pass the tenant as metadata, for example isCompleted(\$tracingId, metadata: ['tenant' => 'tenant_a'])."); + $this->expectExceptionMessage("TracingGateway reads a flow from the database of the tenant it ran for, and this call names no tenant. Pass the tenant as metadata, for example isCompletedFor(\$tracingId, metadata: ['tenant' => 'tenant_a'])."); - $this->completion($ecotone)->isCompleted(SpanContext::newTrace()->traceId); + $this->completion($ecotone)->isCompletedFor(SpanContext::newTrace()->traceId); } public function test_a_database_outage_right_after_the_consumer_committed_does_not_fail_the_consumer(): void @@ -717,7 +716,7 @@ public function goDownAfterTheProjectionRan(MethodInvocation $methodInvocation): self::assertSame(1, $projection->projected); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'users_list')); - self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_a_database_outage_inside_the_consumer_transaction_leaves_the_message_to_be_retried(): void @@ -749,7 +748,7 @@ public function comeBackBeforeEachAttempt(MethodInvocation $methodInvocation): m $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); self::assertSame(2, $notifier->handled); - self::assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_completion_recorded_by_one_instance_is_visible_to_another_sharing_the_database(): void @@ -760,7 +759,7 @@ public function test_completion_recorded_by_one_instance_is_visible_to_another_s $producer->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - self::assertTrue($this->completion($observer)->isCompleted($trace->traceId)); + self::assertTrue($this->completion($observer)->isCompletedFor($trace->traceId)); } /** diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index facd48ae1..129ba72d4 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -16,7 +16,6 @@ use Ecotone\Api\Dbal\Tracing\FlowCompleted; use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; -use Ecotone\Api\Dbal\Tracing\FlowTrackingTimedOut; use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; @@ -34,6 +33,7 @@ use Ecotone\Api\Projecting\Projection; use Ecotone\Api\Scheduling\Duration; use Ecotone\Api\Tracing\CurrentTrace; +use Ecotone\Api\Tracing\Span; use Ecotone\Api\Tracing\SpanContext; use Ecotone\Api\Tracing\SpanStatus; use Ecotone\Messaging\Config\ConfigurationException; @@ -73,7 +73,7 @@ public function test_a_command_handled_synchronously_is_completed_when_the_bus_r $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.registration')); } @@ -84,14 +84,14 @@ public function test_an_asynchronous_handler_keeps_the_trace_pending_until_its_c $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.registration')); $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_an_asynchronous_projection_keeps_the_trace_pending_until_it_projected(): void @@ -113,7 +113,7 @@ public function whenRegistered(array $event): void $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'users_list')); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_an_asynchronous_projection_is_awaited_by_its_class(): void @@ -134,7 +134,7 @@ public function whenRegistered(array $event): void $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, $projection::class)); - $this->completion($ecotone)->awaitFor($trace->traceId, $projection::class, Duration::milliseconds(100)); + $this->assertTrue($this->completion($ecotone)->isCompletedWithin($trace->traceId, Duration::milliseconds(100), $projection::class)); } public function test_a_handler_class_is_completed_once_every_one_of_its_handlers_the_flow_reached_ran(): void @@ -199,60 +199,82 @@ public function whenPlaced(array $event): void $this->completion($ecotone)->isCompletedFor(SpanContext::newTrace()->traceId, $projection::class); } - public function test_awaiting_a_flow_longer_than_fifteen_minutes_is_refused_proposing_notifications(): void + public function test_waiting_for_a_flow_longer_than_fifteen_minutes_is_refused_proposing_notifications(): void { $ecotone = $this->bootstrap([$this->users()]); $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('TracingGateway::await() was asked to wait 960000 milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Await at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowCompleted] receives a FlowCompleted once the flow completes, and one with #[OnFlowStepCompleted] a FlowStepCompleted for each of its steps.'); + $this->expectExceptionMessage('TracingGateway::isCompletedWithin() was asked to wait 960000 milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Wait at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowCompleted] receives a FlowCompleted once the flow completes, and one with #[OnFlowStepCompleted] a FlowStepCompleted for each of its steps.'); - $this->completion($ecotone)->await(SpanContext::newTrace()->traceId, Duration::minutes(16)); + $this->completion($ecotone)->isCompletedWithin(SpanContext::newTrace()->traceId, Duration::minutes(16)); } - public function test_awaiting_a_destination_longer_than_fifteen_minutes_is_refused_proposing_notifications(): void + public function test_waiting_for_a_destination_longer_than_fifteen_minutes_is_refused_proposing_notifications(): void { $ecotone = $this->bootstrap([$this->users()]); $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('TracingGateway::awaitFor() was asked to wait 960000 milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Await at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowCompleted] receives a FlowCompleted once the flow completes, and one with #[OnFlowStepCompleted] a FlowStepCompleted for each of its steps.'); + $this->expectExceptionMessage('TracingGateway::isCompletedWithin() was asked to wait 960000 milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Wait at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowCompleted] receives a FlowCompleted once the flow completes, and one with #[OnFlowStepCompleted] a FlowStepCompleted for each of its steps.'); - $this->completion($ecotone)->awaitFor(SpanContext::newTrace()->traceId, 'user.registration', Duration::minutes(16)); + $this->completion($ecotone)->isCompletedWithin(SpanContext::newTrace()->traceId, Duration::minutes(16), 'user.registration'); } - public function test_awaiting_a_completed_flow_for_exactly_fifteen_minutes_is_allowed(): void + public function test_waiting_for_a_completed_flow_for_exactly_fifteen_minutes_is_allowed(): void { $ecotone = $this->bootstrap([$this->users()]); $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->completion($ecotone)->await($trace->traceId, Duration::minutes(15)); - $this->completion($ecotone)->awaitFor($trace->traceId, 'user.registration', Duration::minutes(15)); - - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedWithin($trace->traceId, Duration::minutes(15))); + $this->assertTrue($this->completion($ecotone)->isCompletedWithin($trace->traceId, Duration::minutes(15), 'user.registration')); } - public function test_awaiting_a_pending_trace_times_out_naming_what_is_still_pending(): void + public function test_waiting_for_a_pending_flow_returns_false_and_the_flow_names_what_is_still_pending(): void { $ecotone = $this->bootstrap([$this->users(), $this->asynchronousNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->expectException(FlowTrackingTimedOut::class); - $this->expectExceptionMessage("Flow {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the pending channel, replay the message from the dead letter if its handler failed, or await with a longer timeout."); - - $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); + $this->assertFalse($this->completion($ecotone)->isCompletedWithin($trace->traceId, Duration::milliseconds(100))); + $this->assertSame(['user.notifier', 'async_channel'], $this->completion($ecotone)->fetchFlow($trace->traceId)->pending); } - public function test_awaiting_a_completed_trace_returns_at_once(): void + public function test_waiting_for_a_completed_flow_returns_true_at_once(): void { $ecotone = $this->bootstrap([$this->users()]); $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); - $this->completion($ecotone)->awaitFor($trace->traceId, 'user.registration', Duration::milliseconds(100)); + $this->assertTrue($this->completion($ecotone)->isCompletedWithin($trace->traceId, Duration::milliseconds(100))); + $this->assertTrue($this->completion($ecotone)->isCompletedWithin($trace->traceId, Duration::milliseconds(100), 'user.registration')); + } + + public function test_a_fetched_flow_holds_its_steps_what_it_reached_and_what_is_still_pending(): void + { + $ecotone = $this->bootstrap([$this->users(), $this->asynchronousNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $trace = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); + + $pending = $this->completion($ecotone)->fetchFlow($trace->traceId); + $this->assertSame($trace->traceId, $pending->tracingId); + $this->assertFalse($pending->isCompleted); + $this->assertSame(['user.notifier', 'async_channel'], $pending->pending); + $this->assertSame('send command', $pending->steps[0]->name); + + $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); + + $completed = $this->completion($ecotone)->fetchFlow($trace->traceId); + $this->assertTrue($completed->isCompleted); + $this->assertSame([], $completed->pending); + $this->assertEqualsCanonicalizing(['user.registration', 'user.notifier'], $completed->reached); + $this->assertContains('process class@anonymous::notify', array_map(fn (Span $step): string => preg_replace('/^process class@anonymous\S*::/', 'process class@anonymous::', $step->name), $completed->steps)); + } + + public function test_fetching_a_flow_nobody_recorded_returns_null(): void + { + $ecotone = $this->bootstrap([$this->users()]); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertNull($this->completion($ecotone)->fetchFlow('0af7651916cd43dd8448eb211c80319c')); } public function test_a_synchronous_handler_without_an_explicit_endpoint_id_still_completes_its_route(): void @@ -268,7 +290,7 @@ public function notify(string $userId): void $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_a_handler_the_container_serves_as_a_subclass_proxy_still_completes_its_route(): void @@ -283,14 +305,14 @@ public function test_a_handler_the_container_serves_as_a_subclass_proxy_still_co $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_a_trace_nobody_recorded_is_not_completed(): void { $ecotone = $this->bootstrap([$this->users()]); - $this->assertFalse($this->completion($ecotone)->isCompleted(SpanContext::newTrace()->traceId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor(SpanContext::newTrace()->traceId)); } public function test_a_failing_asynchronous_handler_leaves_the_trace_pending(): void @@ -312,7 +334,7 @@ public function notify(string $userId): void } catch (RuntimeException) { } - $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); } @@ -396,16 +418,16 @@ public function place(string $orderId): void $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } - public function test_awaiting_an_asynchronous_command_names_its_channel_as_pending(): void + public function test_waiting_for_an_asynchronous_command_leaves_its_channel_pending(): void { $orders = new class () { #[Asynchronous('async_channel')] @@ -418,10 +440,8 @@ public function place(string $orderId): void $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->expectException(FlowTrackingTimedOut::class); - $this->expectExceptionMessage('Still pending: order.placing, async_channel.'); - - $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); + $this->assertFalse($this->completion($ecotone)->isCompletedWithin($trace->traceId, Duration::milliseconds(100))); + $this->assertSame(['order.placing', 'async_channel'], $this->completion($ecotone)->fetchFlow($trace->traceId)->pending); } public function test_a_handler_that_succeeded_inside_a_failed_unit_of_work_is_not_completed(): void @@ -528,7 +548,7 @@ public function whenCompleted(FlowCompleted $event): void $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $this->assertSame(['A subscriber of ecotone.tracing.flowCompleted failed and was skipped: Cache refresh failed'], $logger->getError()); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_a_flow_step_completed_subscriber_is_never_notified_about_the_synchronous_work_it_started(): void @@ -588,7 +608,7 @@ public function refresh(string $tracingId): void $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); $this->assertSame([$trace->traceId], $subscriber->completedTraces); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_a_flow_step_completed_subscriber_sending_an_asynchronous_command_is_not_notified_about_its_consumer(): void @@ -637,7 +657,7 @@ public function place(string $orderId): void $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); - $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); } @@ -662,7 +682,7 @@ public function place(string $orderId): void $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); $this->assertSame(2, $orders->attempts); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_an_output_channel_after_an_asynchronous_handler_completes_neither_while_a_delivery_that_feeds_them_is_still_queued(): void @@ -689,13 +709,13 @@ public function record(string $userId): void $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'audit.recording')); $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); - $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 1)); $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'audit.recording')); $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_a_flow_started_through_a_message_gateway_is_pending_until_its_asynchronous_handler_ran(): void @@ -723,12 +743,12 @@ public function deliver(string $text): void $ecotone->getGateway(TracedNotificationGateway::class)->send('hello', [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'notification.sending')); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_the_tracing_gateway_without_flow_tracking_names_the_configuration_it_needs(): void @@ -738,7 +758,7 @@ public function test_the_tracing_gateway_without_flow_tracking_names_the_configu $this->expectException(ConfigurationException::class); $this->expectExceptionMessage('TracingGateway needs flow tracking, which records every flow and decides when it is complete. Register FlowTrackingConfiguration::createWithDefaults() from a #[ServiceContext] method.'); - $this->completion($ecotone)->isCompleted('0af7651916cd43dd8448eb211c80319c'); + $this->completion($ecotone)->isCompletedFor('0af7651916cd43dd8448eb211c80319c'); } public function test_awaiting_a_trace_whose_handler_failed_names_the_channel_and_how_to_recover(): void @@ -756,32 +776,25 @@ public function notify(string $userId): void $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup(stopOnError: false)); - $this->expectException(FlowTrackingTimedOut::class); - $this->expectExceptionMessage("Flow {$trace->traceId} did not complete within 100 milliseconds. Still pending: user.notifier, async_channel. Run the consumer of the pending channel, replay the message from the dead letter if its handler failed, or await with a longer timeout."); - - $this->completion($ecotone)->await($trace->traceId, Duration::milliseconds(100)); + $this->assertFalse($this->completion($ecotone)->isCompletedWithin($trace->traceId, Duration::milliseconds(100))); + $this->assertSame(['user.notifier', 'async_channel'], $this->completion($ecotone)->fetchFlow($trace->traceId)->pending); } - public function test_awaiting_a_trace_nobody_recorded_says_so(): void + public function test_waiting_for_a_flow_nobody_recorded_returns_false(): void { $ecotone = $this->bootstrap([$this->users()]); - $this->expectException(FlowTrackingTimedOut::class); - $this->expectExceptionMessage('Nothing was recorded for flow 0af7651916cd43dd8448eb211c80319c within 100 milliseconds. Send the command with that tracingId in its metadata, and check that the application handling it registers FlowTrackingConfiguration.'); - - $this->completion($ecotone)->await('0af7651916cd43dd8448eb211c80319c', Duration::milliseconds(100)); + $this->assertFalse($this->completion($ecotone)->isCompletedWithin('0af7651916cd43dd8448eb211c80319c', Duration::milliseconds(100))); } - public function test_awaiting_a_destination_the_trace_never_reached_names_it_as_unknown(): void + public function test_waiting_for_a_destination_the_flow_never_reached_returns_false_and_the_flow_names_what_it_reached(): void { $ecotone = $this->bootstrap([$this->users()]); $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->expectException(FlowTrackingTimedOut::class); - $this->expectExceptionMessage("Flow {$trace->traceId} was neither sent to nor processed by user.notifierr within 100 milliseconds. Check the endpointId or projection name; the flow reached: user.registration."); - - $this->completion($ecotone)->awaitFor($trace->traceId, 'user.notifierr', Duration::milliseconds(100)); + $this->assertFalse($this->completion($ecotone)->isCompletedWithin($trace->traceId, Duration::milliseconds(100), 'user.notifierr')); + $this->assertSame(['user.registration'], $this->completion($ecotone)->fetchFlow($trace->traceId)->reached); } public function test_two_asynchronous_commands_to_one_handler_complete_it_only_after_both_ran(): void @@ -818,9 +831,9 @@ public function whenPlaced(array $event): void $ecotone = $this->bootstrap([$projection]); $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('Projection orders_list runs by #[Polling], so it is not part of the trace that appended its events and TracingGateway cannot await it. Await the handler that published the event, or make the projection event-driven by removing #[Polling].'); + $this->expectExceptionMessage('Projection orders_list runs by #[Polling], so it is not part of the trace that appended its events and TracingGateway cannot wait for it. Await the handler that published the event, or make the projection event-driven by removing #[Polling].'); - $this->completion($ecotone)->awaitFor('0af7651916cd43dd8448eb211c80319c', 'orders_list', Duration::milliseconds(10)); + $this->completion($ecotone)->isCompletedWithin('0af7651916cd43dd8448eb211c80319c', Duration::milliseconds(10), 'orders_list'); } public function test_a_flow_started_by_a_query_completes_once_the_asynchronous_command_it_sent_ran(): void @@ -844,10 +857,10 @@ public function reserve(string $orderId): void $tracingId = '0af7651916cd43dd8448eb211c80319c'; $ecotone->sendQueryWithRouting('order.quote', '1', metadata: [MessageHeaders::TRACING_ID => $tracingId]); - $this->assertFalse($this->completion($ecotone)->isCompleted($tracingId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($tracingId)); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - $this->assertTrue($this->completion($ecotone)->isCompleted($tracingId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($tracingId)); } public function test_a_flow_a_consumer_starts_for_a_message_from_an_application_without_tracing_completes_once_that_consumer_ran(): void @@ -870,7 +883,7 @@ public function place(string $orderId, #[Reference] CurrentTrace $currentTrace): $consumer->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); $this->assertNotNull($orders->tracingId); - $this->assertTrue($this->completion($consumer)->isCompleted($orders->tracingId)); + $this->assertTrue($this->completion($consumer)->isCompletedFor($orders->tracingId)); } public function test_a_handler_can_read_the_trace_id_it_runs_in(): void @@ -890,7 +903,7 @@ public function register(string $userId, #[Reference] CurrentTrace $currentTrace $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $this->assertSame($trace->traceId, $users->traceId); - $this->assertTrue($this->completion($ecotone)->isCompleted($users->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($users->traceId)); } public function test_reading_the_current_trace_without_tracing_names_how_to_enable_it(): void @@ -929,12 +942,12 @@ public function place(string $orderId): void $ecotone->sendCommandWithRouting('order.place', '2', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); $ecotone->run('orders', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 1)); - $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); $ecotone->run('orders', ExecutionPollingMetadata::createWithTestingSetup(handledMessageLimit: 1)); $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'order.placing')); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_with_only_requested_flows_tracked_a_command_sent_without_one_stores_nothing(): void @@ -965,12 +978,12 @@ public function test_with_only_requested_flows_tracked_a_command_sent_with_one_i $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); $this->assertArrayHasKey('receive async_channel', $subscriber->endedSpans); - $this->assertTrue($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_a_vendor_entry_in_tracestate_survives_the_asynchronous_hop(): void @@ -1000,9 +1013,9 @@ public function test_a_command_sent_with_a_tracing_id_is_awaited_by_that_tracing $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '4bf92f3577b34da6a3ce929d0e0e4736']); - $this->assertFalse($this->completion($ecotone)->isCompleted('4bf92f3577b34da6a3ce929d0e0e4736')); + $this->assertFalse($this->completion($ecotone)->isCompletedFor('4bf92f3577b34da6a3ce929d0e0e4736')); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - $this->assertTrue($this->completion($ecotone)->isCompleted('4bf92f3577b34da6a3ce929d0e0e4736')); + $this->assertTrue($this->completion($ecotone)->isCompletedFor('4bf92f3577b34da6a3ce929d0e0e4736')); } public function test_a_uuid_is_accepted_as_tracing_id_and_awaited_by_the_same_uuid(): void @@ -1012,7 +1025,7 @@ public function test_a_uuid_is_accepted_as_tracing_id_and_awaited_by_the_same_uu $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '0199f2f4-5e3b-7a42-9b1c-2f7d3e8a6c11']); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - $this->assertTrue($this->completion($ecotone)->isCompleted('0199f2f4-5e3b-7a42-9b1c-2f7d3e8a6c11')); + $this->assertTrue($this->completion($ecotone)->isCompletedFor('0199f2f4-5e3b-7a42-9b1c-2f7d3e8a6c11')); $this->assertTrue($this->completion($ecotone)->isCompletedFor('0199f2f4-5e3b-7a42-9b1c-2f7d3e8a6c11', 'user.notifier')); } @@ -1128,7 +1141,7 @@ public function test_a_failed_unit_of_work_stores_nothing_by_default(): void } $this->assertSame([], $subscriber->endedSpans); - $this->assertFalse($this->completion($ecotone)->isCompleted($trace->traceId)); + $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); } public function test_a_flow_completed_subscriber_whose_first_parameter_cannot_receive_the_notification_is_refused_at_bootstrap(): void @@ -1235,9 +1248,9 @@ public function test_with_notifications_turned_off_a_flow_is_still_tracked_and_a ); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '4bf92f3577b34da6a3ce929d0e0e4736']); - $this->assertFalse($this->completion($ecotone)->isCompleted('4bf92f3577b34da6a3ce929d0e0e4736')); + $this->assertFalse($this->completion($ecotone)->isCompletedFor('4bf92f3577b34da6a3ce929d0e0e4736')); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - $this->assertTrue($this->completion($ecotone)->isCompleted('4bf92f3577b34da6a3ce929d0e0e4736')); + $this->assertTrue($this->completion($ecotone)->isCompletedFor('4bf92f3577b34da6a3ce929d0e0e4736')); } public function test_with_the_tracing_gateway_turned_off_asking_it_names_the_setting_while_subscribers_are_still_notified(): void @@ -1265,7 +1278,7 @@ public function whenCompleted(FlowCompleted $event): void $this->expectException(ConfigurationException::class); $this->expectExceptionMessage('TracingGateway is turned off by FlowTrackingConfiguration::withTracingGateway(false). Remove that call to ask whether a flow completed.'); - $this->completion($ecotone)->isCompleted('4bf92f3577b34da6a3ce929d0e0e4736'); + $this->completion($ecotone)->isCompletedFor('4bf92f3577b34da6a3ce929d0e0e4736'); } public function test_flow_tracking_requires_an_enterprise_licence(): void diff --git a/packages/PdoEventSourcing/tests/Projecting/FlowTrackingProjectionTest.php b/packages/PdoEventSourcing/tests/Projecting/FlowTrackingProjectionTest.php index 52f9a4193..907b9c0d7 100644 --- a/packages/PdoEventSourcing/tests/Projecting/FlowTrackingProjectionTest.php +++ b/packages/PdoEventSourcing/tests/Projecting/FlowTrackingProjectionTest.php @@ -76,7 +76,7 @@ public function test_a_database_outage_right_after_an_asynchronous_projection_co self::assertSame(1, $projection->projected); self::assertTrue($ecotone->getGateway(TracingGateway::class)->isCompletedFor($tracingId, 'flow_tracked_tickets')); - self::assertTrue($ecotone->getGateway(TracingGateway::class)->isCompleted($tracingId)); + self::assertTrue($ecotone->getGateway(TracingGateway::class)->isCompletedFor($tracingId)); } public function test_an_asynchronous_projection_spread_over_several_batches_completes_the_flow_once_it_caught_up(): void @@ -91,7 +91,7 @@ public function test_an_asynchronous_projection_spread_over_several_batches_comp $ecotone->run('async_projection', ExecutionPollingMetadata::createWithTestingSetup()); self::assertSame(3, $projection->projected); - self::assertTrue($ecotone->getGateway(TracingGateway::class)->isCompleted($tracingId)); + self::assertTrue($ecotone->getGateway(TracingGateway::class)->isCompletedFor($tracingId)); } private function ticketsProjection(): object diff --git a/upgrade-2.0.md b/upgrade-2.0.md index dc0525a1e..b18092b87 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3103,20 +3103,23 @@ reaches, synchronously or asynchronously, across processes. What it tracks is ke `ecotone_tracing`, created by `ecotone:migration:database:setup`; a flow test given no connection keeps it in memory. Two features are on by default and each can be turned off while the flows are still tracked: -- **`TracingGateway`** answers `isCompleted($tracingId)`, `isCompletedFor($tracingId, $destination)` for one - `endpointId`, projection name or class, and `await()` / `awaitFor()`, which poll until then. A destination is - complete once it processed every message the flow sent it and nothing still pending upstream of it could send it - another: only work its own handling caused, through a new message, may still be pending. A message it forwards - unchanged, through an output channel, is waited for too, since it carries the same message id as an event's copy - queued for a sibling handler. A class stands for everything in it the flow can reach: a `#[Projection]` class for - its projection, any other class for every handler method in it, endpoint id or not, so - `awaitFor($tracingId, UsersListProjection::class, $timeout)` needs no name to be repeated, and the class completes - once every one of its handlers the flow reached ran. A class with no handler or projection is refused at once. - An await blocks the calling process while it polls, so a timeout above 15 minutes is refused, pointing to +- **`TracingGateway`** has three methods. `isCompletedFor($tracingId)` tells whether the flow is complete, and + `isCompletedFor($tracingId, $destination)` whether one `endpointId`, projection name or class is. + `isCompletedWithin($tracingId, $timeout, $destination = null)` polls until that holds and returns `true`, or + returns `false` once the timeout passed; it never throws for a flow that is late. `fetchFlow($tracingId)` returns + a `Flow` with `isCompleted`, `pending` (the handlers and channels still waited for), `reached` (the destinations + the flow was sent to or processed by) and `steps` (its stored spans, in the order they started), or `null` for a + flow nobody recorded, so a `false` is explained by reading the flow rather than by an exception. + A destination is complete once it processed every message the flow sent it and nothing still pending upstream of + it could send it another: only work its own handling caused, through a new message, may still be pending. A + message it forwards unchanged, through an output channel, is waited for too, since it carries the same message id + as an event's copy queued for a sibling handler. A class stands for everything in it the flow can reach: a + `#[Projection]` class for its projection, any other class for every handler method in it, endpoint id or not, so + `isCompletedWithin($tracingId, $timeout, UsersListProjection::class)` needs no name to be repeated, and the class + completes once every one of its handlers the flow reached ran. A class with no handler or projection is refused at + once. Waiting blocks the calling process while it polls, so a timeout above 15 minutes is refused, pointing to `#[OnFlowCompleted]` and `#[OnFlowStepCompleted]`: a flow that takes longer is reacted to when it completes, not waited for. - Past the timeout, the awaits throw `FlowTrackingTimedOut` naming the handlers and channels still pending, pointing to the dead letter when a - handler failed, and naming a flow nobody recorded or a destination the flow never reached. `withTracingGateway(false)` turns it off; the gateway stays injectable and every call names that setting. - **Notifications**: a method with `#[OnFlowCompleted]` receives a `FlowCompleted` each time its flow reaches completion, and one with `#[OnFlowStepCompleted]` receives a `FlowStepCompleted` for every step of the flow whose unit of work committed, a bus call, a handler run, a route, a message sent or received, with the step's span in `$event->step`. `withNotifications(false)` turns them off, @@ -3139,7 +3142,7 @@ pending until its consumer processed that exact message. A destination counts me satisfies its own delivery only, and a message keeps the delivery it started as through dynamic channels, the high-throughput outbox and a dead-letter replay. A flow handed more work after completing, as when a request sends two commands under one `tracingId`, completes and notifies again once that work is done. `isCompletedFor()` and -`awaitFor()` refuse a `#[Polling]` projection by name, because a polled run is not part of the flow that appended its +`isCompletedWithin()` refuse a `#[Polling]` projection by name, because a polled run is not part of the flow that appended its events. Everything is written inside the transaction of the unit of work it describes, a bus call or an asynchronous @@ -3169,7 +3172,7 @@ it. Nothing runs the command for you: schedule it, from cron or the framework's flow. It removes expired flows in batches of 1000, each batch's spans and flow rows in one short transaction, until none is left, so a large backlog never holds one long lock; `withRemovalBatchSize()` changes the size. A multi-tenant application tracks each flow in the database of its tenant, inside the tenant's transaction, so a flow is never complete before the tenant's data committed. `TracingGateway` is then asked with the tenant, -`$tracingGateway->await($tracingId, $timeout, metadata: ['tenant' => 'tenant_a'])`, and a call without one is refused +`$tracingGateway->isCompletedWithin($tracingId, $timeout, metadata: ['tenant' => 'tenant_a'])`, and a call without one is refused naming that metadata; the retention command runs once per tenant, `ecotone:tracing:remove-expired-traces --header=tenant:tenant_a`. @@ -3182,7 +3185,9 @@ public function flowTracking(): FlowTrackingConfiguration $tracingId = Uuid::v7()->toRfc4122(); $commandBus->sendWithRouting('order.place', $order, metadata: [MessageHeaders::TRACING_ID => $tracingId]); -$tracingGateway->awaitFor($tracingId, 'orders_list', Duration::seconds(5)); +if (! $tracingGateway->isCompletedWithin($tracingId, Duration::seconds(5), 'orders_list')) { + $pending = $tracingGateway->fetchFlow($tracingId)->pending; +} final class OrdersWebsocket { From f8516a1eb025ef8e85377515e02523e2d057a924 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 45/50] style(dbal): flow tracking's Api files carry the Enterprise notice in the form main now requires --- packages/Dbal/Api/Tracing/Flow.php | 3 ++- packages/Dbal/Api/Tracing/FlowCompleted.php | 3 ++- packages/Dbal/Api/Tracing/FlowStepCompleted.php | 3 ++- packages/Dbal/Api/Tracing/FlowTrackingConfiguration.php | 3 ++- packages/Dbal/Api/Tracing/OnFlowCompleted.php | 3 ++- packages/Dbal/Api/Tracing/OnFlowStepCompleted.php | 3 ++- packages/Dbal/Api/Tracing/TracingGateway.php | 3 ++- 7 files changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/Dbal/Api/Tracing/Flow.php b/packages/Dbal/Api/Tracing/Flow.php index 0b1cb8654..a007f5e4f 100644 --- a/packages/Dbal/Api/Tracing/Flow.php +++ b/packages/Dbal/Api/Tracing/Flow.php @@ -8,7 +8,8 @@ /** * licence Enterprise - * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Use requires an Ecotone Enterprise licence (see LICENSE-ENTERPRISE; trial: https://ecotone.tech/pricing). */ final class Flow { diff --git a/packages/Dbal/Api/Tracing/FlowCompleted.php b/packages/Dbal/Api/Tracing/FlowCompleted.php index 7b251283b..939474d1e 100644 --- a/packages/Dbal/Api/Tracing/FlowCompleted.php +++ b/packages/Dbal/Api/Tracing/FlowCompleted.php @@ -6,7 +6,8 @@ /** * licence Enterprise - * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Use requires an Ecotone Enterprise licence (see LICENSE-ENTERPRISE; trial: https://ecotone.tech/pricing). */ final class FlowCompleted { diff --git a/packages/Dbal/Api/Tracing/FlowStepCompleted.php b/packages/Dbal/Api/Tracing/FlowStepCompleted.php index b95bd834d..5fcfcfad1 100644 --- a/packages/Dbal/Api/Tracing/FlowStepCompleted.php +++ b/packages/Dbal/Api/Tracing/FlowStepCompleted.php @@ -8,7 +8,8 @@ /** * licence Enterprise - * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Use requires an Ecotone Enterprise licence (see LICENSE-ENTERPRISE; trial: https://ecotone.tech/pricing). */ final class FlowStepCompleted { diff --git a/packages/Dbal/Api/Tracing/FlowTrackingConfiguration.php b/packages/Dbal/Api/Tracing/FlowTrackingConfiguration.php index 4b60f4422..e80e4e4c9 100644 --- a/packages/Dbal/Api/Tracing/FlowTrackingConfiguration.php +++ b/packages/Dbal/Api/Tracing/FlowTrackingConfiguration.php @@ -9,7 +9,8 @@ /** * licence Enterprise - * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Use requires an Ecotone Enterprise licence (see LICENSE-ENTERPRISE; trial: https://ecotone.tech/pricing). */ final class FlowTrackingConfiguration { diff --git a/packages/Dbal/Api/Tracing/OnFlowCompleted.php b/packages/Dbal/Api/Tracing/OnFlowCompleted.php index 0a498d0c7..4a66e313b 100644 --- a/packages/Dbal/Api/Tracing/OnFlowCompleted.php +++ b/packages/Dbal/Api/Tracing/OnFlowCompleted.php @@ -10,7 +10,8 @@ #[Attribute(Attribute::TARGET_METHOD)] /** * licence Enterprise - * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Use requires an Ecotone Enterprise licence (see LICENSE-ENTERPRISE; trial: https://ecotone.tech/pricing). */ final class OnFlowCompleted extends InternalHandler { diff --git a/packages/Dbal/Api/Tracing/OnFlowStepCompleted.php b/packages/Dbal/Api/Tracing/OnFlowStepCompleted.php index 1453eccf0..6361c3901 100644 --- a/packages/Dbal/Api/Tracing/OnFlowStepCompleted.php +++ b/packages/Dbal/Api/Tracing/OnFlowStepCompleted.php @@ -10,7 +10,8 @@ #[Attribute(Attribute::TARGET_METHOD)] /** * licence Enterprise - * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Use requires an Ecotone Enterprise licence (see LICENSE-ENTERPRISE; trial: https://ecotone.tech/pricing). */ final class OnFlowStepCompleted extends InternalHandler { diff --git a/packages/Dbal/Api/Tracing/TracingGateway.php b/packages/Dbal/Api/Tracing/TracingGateway.php index 3238db596..9b92829d5 100644 --- a/packages/Dbal/Api/Tracing/TracingGateway.php +++ b/packages/Dbal/Api/Tracing/TracingGateway.php @@ -9,7 +9,8 @@ /** * licence Enterprise - * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Use requires an Ecotone Enterprise licence (see LICENSE-ENTERPRISE; trial: https://ecotone.tech/pricing). */ #[PropagateHeaders] interface TracingGateway From e40d27594b10fbf31854931cc4495a1044a2428e Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 46/50] build(core): the isolated core phpstan accepts DbalTracingModule in ModuleClassList, as it does every other DBAL module --- packages/Ecotone/phpstan-baseline.neon | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/Ecotone/phpstan-baseline.neon b/packages/Ecotone/phpstan-baseline.neon index 6bc0bb226..5b5c855ef 100644 --- a/packages/Ecotone/phpstan-baseline.neon +++ b/packages/Ecotone/phpstan-baseline.neon @@ -234,6 +234,12 @@ parameters: count: 1 path: src/Messaging/Config/ModuleClassList.php + - + message: '#^Class Ecotone\\Dbal\\Tracing\\DbalTracingModule not found\.$#' + identifier: class.notFound + count: 1 + path: src/Messaging/Config/ModuleClassList.php + - message: '#^Class Ecotone\\EventSourcing\\Config\\EventSourcingModule not found\.$#' identifier: class.notFound From 0803a58c77a1a5e318a0f2dc628a30b8434c3bfe Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 47/50] test(tracing): each scheduled execution is proved its own trace without depending on a 1 ms poller firing twice in one run The test relied on a fixed-rate poller executing twice inside one run(), which the CI runners did not do; its lenient tree comparison then matched one span against both expected nodes. Two runs of one execution each now produce the two polls, and the test asserts both spans, both roots, on different traces. --- .../tests/Tracing/AsynchronousTracingTest.php | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php b/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php index 4bd4e568f..2f1697cf3 100644 --- a/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php +++ b/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php @@ -316,21 +316,22 @@ public function handle(): void } }; - EcotoneLite::bootstrapFlowTesting( + $ecotone = EcotoneLite::bootstrapFlowTesting( [$scheduled::class], [$scheduled, ...self::recordingSpansInto($recorder)], [self::tracingConfiguration()], - ) - ->run('scheduled_handler', ExecutionPollingMetadata::createWithDefaults()->withExecutionAmountLimit(2)); - - self::compareTreesByDetails( - [ - ['details' => ['name' => 'receive nullChannel', 'kind' => SpanKind::Consumer], 'children' => []], - ['details' => ['name' => 'receive nullChannel', 'kind' => SpanKind::Consumer], 'children' => []], - ], - self::buildTree($recorder->recordedSpans()) ); - $this->assertCount(2, array_unique(array_map(fn (Span $span): string => $span->traceId, $recorder->recordedSpans()))); + + $ecotone->run('scheduled_handler', ExecutionPollingMetadata::createWithDefaults()->withExecutionAmountLimit(1)); + $ecotone->run('scheduled_handler', ExecutionPollingMetadata::createWithDefaults()->withExecutionAmountLimit(1)); + + $polls = array_values(array_filter($recorder->recordedSpans(), fn (Span $span): bool => $span->name === 'receive nullChannel')); + $this->assertCount(2, $polls); + foreach ($polls as $poll) { + $this->assertSame(SpanKind::Consumer, $poll->kind); + $this->assertTrue($poll->isRoot()); + } + $this->assertNotSame($polls[0]->traceId, $polls[1]->traceId); } public function test_tracing_workflow_scheduled_handler(): void From 620023942be0f5c37275feb4bda3ef697b4129ae Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 48/50] test(tracing): each scheduled workflow execution is proved its own trace with its handler beneath it, without depending on poller timing Like the scheduled-handler test, it relied on a 1 ms poller firing twice in one run(), and its lenient tree comparison could match both expected runs against one. Two runs of one execution each now produce the two polls, and the test asserts both root spans, each with its own add handler span, on different traces. --- .../tests/Tracing/AsynchronousTracingTest.php | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php b/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php index 2f1697cf3..b48481abe 100644 --- a/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php +++ b/packages/Ecotone/tests/Tracing/AsynchronousTracingTest.php @@ -339,20 +339,26 @@ public function test_tracing_workflow_scheduled_handler(): void $recorder = new InMemorySpanRecorder(); $workflow = $this->workflowScheduledHandler(); - EcotoneLite::bootstrapFlowTesting( + $ecotone = EcotoneLite::bootstrapFlowTesting( [$workflow::class], [$workflow, ...self::recordingSpansInto($recorder)], [self::tracingConfiguration()], - ) - ->run('scheduled_handler', ExecutionPollingMetadata::createWithDefaults()->withExecutionAmountLimit(2)); + ); - $expectedRun = [ - 'details' => ['name' => 'receive add'], - 'children' => [ - ['details' => ['name' => 'process class@anonymous::add'], 'children' => []], - ], - ]; - self::compareTreesByDetails([$expectedRun, $expectedRun], self::buildTree($recorder->recordedSpans())); + $ecotone->run('scheduled_handler', ExecutionPollingMetadata::createWithDefaults()->withExecutionAmountLimit(1)); + $ecotone->run('scheduled_handler', ExecutionPollingMetadata::createWithDefaults()->withExecutionAmountLimit(1)); + + $spans = $recorder->recordedSpans(); + $polls = array_values(array_filter($spans, fn (Span $span): bool => $span->name === 'receive add')); + $this->assertCount(2, $polls); + foreach ($polls as $poll) { + $this->assertTrue($poll->isRoot()); + $handlers = array_values(array_filter($spans, fn (Span $span): bool => $span->parentSpanId === $poll->spanId)); + $this->assertCount(1, $handlers); + $this->assertMatchesRegularExpression('/^process class@anonymous\S*::add$/', $handlers[0]->name); + $this->assertSame($poll->traceId, $handlers[0]->traceId); + } + $this->assertNotSame($polls[0]->traceId, $polls[1]->traceId); } public function test_no_tracing_when_there_is_nothing_to_trace(): void From f6b1c1ee420614785bb8300c1dac33b04bb4a1ed Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sun, 18 Oct 2026 08:00:00 +0200 Subject: [PATCH 49/50] fix(dbal): two parts of one flow finishing at the same moment notify its completion once on MySQL and MariaDB too On REPEATABLE READ the second consumer waited for the first one's commit, then read the flow's spans from its earlier snapshot, missed the first part, and neither part claimed completion, so #[OnFlowCompleted] never fired. The flow row now carries a version: a part writes it only if no other part committed since it read it, and otherwise rolls back with ConcurrencyException so the retry reads the flow from a fresh snapshot. A forked-process race test proves one notification on PostgreSQL, MySQL and MariaDB. --- .../Dbal/src/Database/TracesTableManager.php | 1 + packages/Dbal/src/Tracing/DbalTraceStore.php | 30 ++- .../Tracing/FlowTrackingRaceTest.php | 202 ++++++++++++++++++ .../Api/Messaging/ConcurrencyException.php | 10 + upgrade-2.0.md | 9 +- 5 files changed, 244 insertions(+), 8 deletions(-) create mode 100644 packages/Dbal/tests/Integration/Tracing/FlowTrackingRaceTest.php diff --git a/packages/Dbal/src/Database/TracesTableManager.php b/packages/Dbal/src/Database/TracesTableManager.php index 04214f143..2e9e32511 100644 --- a/packages/Dbal/src/Database/TracesTableManager.php +++ b/packages/Dbal/src/Database/TracesTableManager.php @@ -109,6 +109,7 @@ private function buildTableSchema(): Table $table->addColumn('trace_id', Types::STRING, ['length' => 32]); $table->addColumn('started_at', Types::BIGINT); $table->addColumn('completed_at', Types::BIGINT, ['notnull' => false]); + $table->addColumn('version', Types::INTEGER, ['default' => 1]); $table->setPrimaryKey(['trace_id']); $table->addIndex(['started_at']); diff --git a/packages/Dbal/src/Tracing/DbalTraceStore.php b/packages/Dbal/src/Tracing/DbalTraceStore.php index fc08f93a3..cc6843b60 100644 --- a/packages/Dbal/src/Tracing/DbalTraceStore.php +++ b/packages/Dbal/src/Tracing/DbalTraceStore.php @@ -7,9 +7,11 @@ use Closure; use Doctrine\DBAL\ArrayParameterType; use Doctrine\DBAL\Connection; +use Doctrine\DBAL\Exception\UniqueConstraintViolationException; use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; use Doctrine\DBAL\Types\Types; use Ecotone\Api\Gateway\EcotoneClockInterface; +use Ecotone\Api\Messaging\ConcurrencyException; use Ecotone\Api\Scheduling\DatePoint; use Ecotone\Api\Scheduling\Duration; use Ecotone\Api\Tracing\Span; @@ -134,13 +136,31 @@ private function removeExpiredBatch(Connection $connection, DatePoint $completed */ private function claimTrace(Connection $connection, string $traceId, array $traceSpans): void { - $startedAt = min(array_map(fn (Span $span): int => $span->startedAt->unixTime()->inMicroseconds(), $traceSpans)); $tableName = $this->tracesTableManager->getTableName(); - $sql = $connection->getDatabasePlatform() instanceof AbstractMySQLPlatform - ? "INSERT INTO {$tableName} (trace_id, started_at, completed_at) VALUES (:traceId, :startedAt, NULL) ON DUPLICATE KEY UPDATE completed_at = NULL" - : "INSERT INTO {$tableName} (trace_id, started_at, completed_at) VALUES (:traceId, :startedAt, NULL) ON CONFLICT (trace_id) DO UPDATE SET completed_at = NULL"; + $readVersion = $connection->fetchOne("SELECT version FROM {$tableName} WHERE trace_id = :traceId", ['traceId' => $traceId]); + if ($readVersion === false) { + try { + $connection->insert($tableName, [ + 'trace_id' => $traceId, + 'started_at' => min(array_map(fn (Span $span): int => $span->startedAt->unixTime()->inMicroseconds(), $traceSpans)), + 'completed_at' => null, + 'version' => 1, + ], ['started_at' => Types::BIGINT]); + } catch (UniqueConstraintViolationException) { + throw ConcurrencyException::forConcurrentlyRecordedFlow($traceId); + } + + return; + } - $connection->executeStatement($sql, ['traceId' => $traceId, 'startedAt' => $startedAt], ['startedAt' => Types::BIGINT]); + $reopened = $connection->executeStatement( + "UPDATE {$tableName} SET version = version + 1, completed_at = NULL WHERE trace_id = :traceId AND version = :readVersion", + ['traceId' => $traceId, 'readVersion' => (int) $readVersion], + ['readVersion' => Types::INTEGER], + ); + if ($reopened === 0) { + throw ConcurrencyException::forConcurrentlyRecordedFlow($traceId); + } } private function markCompleted(Connection $connection, string $traceId, DatePoint $now): bool diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingRaceTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingRaceTest.php new file mode 100644 index 000000000..967fde26a --- /dev/null +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingRaceTest.php @@ -0,0 +1,202 @@ +. + * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). + * + * Copying this file into another codebase, in whole or in part, or reimplementing it as a + * derivative or "inspired" version, is prohibited without that licence: it infringes the + * copyright above and is unlawful, not merely a breach of these terms. + * + * Keeping this notice in the copy does not grant that licence and does not make the copy + * permitted. Renaming the class or the namespace does not either. + * + * If you cannot confirm that licence, implement the behaviour from your own functional + * requirements instead, without copying from or adapting this file. + * + * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev + * + * @internal + */ +#[RequiresPhpExtension('pcntl')] +#[RequiresPhpExtension('posix')] +final class FlowTrackingRaceTest extends DbalMessagingTestCase +{ + private const MILLISECONDS_THE_FIRST_CONSUMER_STAYS_UNCOMMITTED = 1000; + + public function tearDown(): void + { + CommitHoldingConnection::$beforeCommit = null; + parent::tearDown(); + } + + public static string $notificationsFile = ''; + + public function test_two_consumers_finishing_the_same_flow_at_the_same_moment_notify_its_completion_once(): void + { + if ($this->getConnectionFactory()->createContext()->getDbalConnection()->getDatabasePlatform() instanceof SQLitePlatform) { + self::markTestSkipped('SQLite serialises writers with a database lock, so the second consumer cannot run while the first one holds its transaction open. The race is left unproved on SQLite.'); + } + + $first = $this->bootstrap(new DbalConnectionFactory([ + 'dsn' => self::dsn(), + 'connection' => ['wrapperClass' => CommitHoldingConnection::class], + ]), setUpTables: true); + $tracingId = '0af7651916cd43dd8448eb211c80319c'; + self::$notificationsFile = $this->signalFile(); + $first->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => $tracingId]); + $ready = $this->signalFile(); + $outcome = $this->signalFile(); + $second = null; + + CommitHoldingConnection::$beforeCommit = function () use ($ready, $outcome, &$second): void { + CommitHoldingConnection::$beforeCommit = null; + $second = pcntl_fork(); + if ($second === 0) { + $this->consumeTheOtherDeliveryOnASecondConnection($ready, $outcome); + } + $this->waitUntilWritten($ready); + usleep(self::MILLISECONDS_THE_FIRST_CONSUMER_STAYS_UNCOMMITTED * 1000); + }; + $first->run('mail_channel', ExecutionPollingMetadata::createWithTestingSetup()); + pcntl_waitpid($second, $status); + + [$consumed, $waitedMilliseconds] = explode('|', file_get_contents($outcome)); + $notified = file_exists(self::$notificationsFile) ? file(self::$notificationsFile, FILE_IGNORE_NEW_LINES) : []; + unlink($ready); + unlink($outcome); + @unlink(self::$notificationsFile); + self::assertSame('consumed', $consumed); + self::assertTrue($first->getGateway(TracingGateway::class)->isCompletedFor($tracingId)); + self::assertSame([$tracingId], $notified); + self::assertGreaterThanOrEqual(self::MILLISECONDS_THE_FIRST_CONSUMER_STAYS_UNCOMMITTED / 2, (int) $waitedMilliseconds); + } + + private function consumeTheOtherDeliveryOnASecondConnection(string $ready, string $outcome): never + { + $consumed = 'not consumed'; + $started = hrtime(true); + try { + $second = $this->bootstrap(new DbalConnectionFactory(self::dsn()), setUpTables: false); + file_put_contents($ready, 'ready'); + $started = hrtime(true); + $second->run('sms_channel', ExecutionPollingMetadata::createWithTestingSetup()); + $consumed = 'consumed'; + } catch (Throwable $failure) { + $consumed = $failure::class . ': ' . $failure->getMessage(); + } + file_put_contents($outcome, $consumed . '|' . (int) ((hrtime(true) - $started) / 1e6)); + + posix_kill(posix_getpid(), SIGKILL); + exit(1); + } + + private function bootstrap(DbalConnectionFactory $connectionFactory, bool $setUpTables): FlowTestSupport + { + $users = new class () { + #[CommandHandler('user.register', endpointId: 'user.registration')] + public function register(string $userId, EventBus $eventBus): void + { + $eventBus->publishWithRouting('user.registered', $userId); + } + }; + $subscriber = new class () { + #[OnFlowCompleted] + public function whenCompleted(FlowCompleted $event): void + { + file_put_contents(FlowTrackingRaceTest::$notificationsFile, $event->tracingId . PHP_EOL, FILE_APPEND | LOCK_EX); + } + }; + $notifications = new class () { + #[Asynchronous('mail_channel')] + #[EventHandler('user.registered', endpointId: 'user.mailer')] + public function mail(string $userId): void + { + } + + #[Asynchronous('sms_channel')] + #[EventHandler('user.registered', endpointId: 'user.texter')] + public function text(string $userId): void + { + } + }; + + return $this->bootstrapFlowTesting( + classesToResolve: [$users::class, $notifications::class, $subscriber::class], + containerOrAvailableServices: [$users, $notifications, $subscriber, DbalConnectionFactory::class => $connectionFactory], + extensionObjects: [ + FlowTrackingConfiguration::createWithDefaults(), + DbalBackedMessageChannelBuilder::create('mail_channel'), + DbalBackedMessageChannelBuilder::create('sms_channel'), + ], + enabledModules: [ModulePackageList::DBAL_PACKAGE], + testConfiguration: TestConfiguration::createWithDefaults()->withDatabaseTablesSetUpOnBootstrap($setUpTables), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + + private static function dsn(): string + { + return getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone'; + } + + private function signalFile(): string + { + $file = tempnam(sys_get_temp_dir(), 'ecotone_flow_race_'); + unlink($file); + + return $file; + } + + private function waitUntilWritten(string $file): void + { + $deadline = hrtime(true) + 30 * 1e9; + while (! file_exists($file) && hrtime(true) < $deadline) { + usleep(10_000); + } + } +} + +final class CommitHoldingConnection extends Connection +{ + /** @var (Closure(): void)|null */ + public static ?Closure $beforeCommit = null; + + public function commit(): void + { + if (self::$beforeCommit !== null && $this->getTransactionNestingLevel() === 1) { + (self::$beforeCommit)(); + } + + parent::commit(); + } +} diff --git a/packages/Ecotone/Api/Messaging/ConcurrencyException.php b/packages/Ecotone/Api/Messaging/ConcurrencyException.php index 0f7d37425..5e061f4bd 100644 --- a/packages/Ecotone/Api/Messaging/ConcurrencyException.php +++ b/packages/Ecotone/Api/Messaging/ConcurrencyException.php @@ -80,6 +80,16 @@ public static function forHandledMessage(string $messageId, string $consumerEndp ), self::errorCode()); } + public static function forConcurrentlyRecordedFlow(string $tracingId): self + { + return new self(sprintf( + 'Flow %s was recorded by another of its units of work after this one read it, so this one cannot tell whether it is the one that completed the flow. ' + . 'Its transaction is rolled back so it can read the flow again: an asynchronous endpoint retries it by default, ' + . 'and a command does when Ecotone retries it with InstantRetryConfiguration::createWithDefaults()->withCommandBusRetry(true, 3, [ConcurrencyException::class]).', + $tracingId, + ), self::errorCode()); + } + private static function whereItIsNow(int $expectedVersion, int $currentVersion): string { return $currentVersion === $expectedVersion diff --git a/upgrade-2.0.md b/upgrade-2.0.md index db157d29c..cadd90d60 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3160,9 +3160,12 @@ subscriber sends, synchronously or through an asynchronous channel, is neither s the flow it was told about; OpenTelemetry still exports it, as its own trace. A subscriber runs synchronously, as that delivery happens; `#[Asynchronous]` on it is refused at bootstrap, so work meant for the background is a command it sends to an `#[Asynchronous]` handler. A subscriber whose first parameter is not `FlowCompleted` or `FlowStepCompleted`, and a -`FlowTrackingConfiguration` registered twice, are refused at bootstrap too. Completion is claimed with one -conditional update after the trace row was touched in the same transaction, so a second unit of work finishing the -same flow waits for the first one's commit and, on PostgreSQL and SQLite, reads its spans before deciding. +`FlowTrackingConfiguration` registered twice, are refused at bootstrap too. Each flow's row carries a version: a unit +of work reads it, and writes the row only if no other part of the same flow committed in between. When two parts of +one flow finish at the same moment, the second one's write fails with `ConcurrencyException`, its transaction rolls +back, and the retry reads the flow again from a fresh snapshot, sees the first part and claims completion, so the flow +notifies exactly once on PostgreSQL, MySQL and MariaDB alike. The retried part runs its handler again: an asynchronous +endpoint retries by default; a command does with `withCommandBusRetry(true, 3, [ConcurrencyException::class])`. `withOnlyRequestedFlows(true)` tracks only flows sent with a `tracingId` or a `traceparent`, carrying that choice across channels in the W3C `tracestate` as `ecotone=carried` next to any other vendor's entry, so ordinary traffic From 0269c70764cda232c2312e14e2dc923488f87984 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Mon, 19 Oct 2026 08:00:00 +0200 Subject: [PATCH 50/50] refactor(dbal): parts of one flow commit without waiting for each other, because completion is read from the steps instead of claimed on a shared flow row #[OnFlowCompleted] needed one part of a flow to claim that it completed it, which made every part write one shared row per flow and, under contention, rolled one part back and ran its handler twice. Completion was always decided from the stored steps, so it is now only read: isCompletedFor() and fetchFlow() answer from ecotone_tracing alone, each step is its own row, and two consumers of one flow commit side by side. - #[OnFlowCompleted], FlowCompleted, the ecotone_traces table and its version are removed; a #[OnFlowStepCompleted] subscriber asks TracingGateway::isCompletedFor() after its commit, so the part that commits last always sees the flow complete - retention keeps every flow for withRetention() from its first step, removed from the spans in batches --- .../FlowTrackingWithOpenTelemetryTest.php | 39 ++-- docs/coding-conventions.md | 2 +- packages/Dbal/Api/Tracing/FlowCompleted.php | 17 -- packages/Dbal/Api/Tracing/OnFlowCompleted.php | 29 --- .../Dbal/src/Database/SpansTableManager.php | 1 + .../Dbal/src/Database/TracesTableManager.php | 118 ------------ packages/Dbal/src/Tracing/DbalTraceStore.php | 93 ++-------- .../Dbal/src/Tracing/DbalTracingModule.php | 25 +-- .../Dbal/src/Tracing/ExpiredTraceRemover.php | 3 +- .../Dbal/src/Tracing/InMemoryTraceStore.php | 20 +-- .../Dbal/src/Tracing/StoredTracingGateway.php | 2 +- .../Dbal/src/Tracing/StoringSpanRecorder.php | 21 +-- .../src/Tracing/TraceRecordingInterceptor.php | 6 +- packages/Dbal/src/Tracing/TraceStore.php | 4 +- .../FlowTrackingConformanceTest.php | 63 +++---- packages/Dbal/tests/DbalMessagingTestCase.php | 1 - .../Tracing/FlowTrackingRaceTest.php | 39 ++-- .../Integration/Tracing/FlowTrackingTest.php | 170 +++--------------- .../Api/Messaging/ConcurrencyException.php | 10 -- upgrade-2.0.md | 51 +++--- 20 files changed, 141 insertions(+), 573 deletions(-) delete mode 100644 packages/Dbal/Api/Tracing/FlowCompleted.php delete mode 100644 packages/Dbal/Api/Tracing/OnFlowCompleted.php delete mode 100644 packages/Dbal/src/Database/TracesTableManager.php diff --git a/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php index d4ac92a2c..26c4da274 100644 --- a/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php +++ b/Monorepo/CrossModuleTests/Tests/Tracing/FlowTrackingWithOpenTelemetryTest.php @@ -10,9 +10,9 @@ use Ecotone\Api\Attribute\EventHandler; use Ecotone\Api\Dbal\DbalConnectionFactory; use Ecotone\Api\Dbal\ExtensionObject\DbalBackedMessageChannelBuilder; -use Ecotone\Api\Dbal\Tracing\FlowCompleted; +use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; -use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; +use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -59,8 +59,7 @@ protected function setUp(): void public function test_one_trace_is_exported_to_open_telemetry_and_stored_in_the_database_across_an_asynchronous_dbal_channel(): void { $exporter = new InMemoryExporter(new ArrayObject()); - $subscriber = $this->flowCompletedSubscriber(); - $ecotone = $this->bootstrap($exporter, [$this->users(), $this->asynchronousNotifier(), $subscriber]); + $ecotone = $this->bootstrap($exporter, [$this->users(), $this->asynchronousNotifier()]); $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); @@ -71,7 +70,6 @@ public function test_one_trace_is_exported_to_open_telemetry_and_stored_in_the_d ); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.registration')); self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); - self::assertSame([], $subscriber->completedTraces); $ecotone->run('notifications', ExecutionPollingMetadata::createWithTestingSetup()); @@ -81,13 +79,12 @@ public function test_one_trace_is_exported_to_open_telemetry_and_stored_in_the_d ); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); - self::assertSame([$trace->traceId], $subscriber->completedTraces); } - public function test_the_flow_completed_subscriber_is_exported_in_a_trace_of_its_own(): void + public function test_each_flow_step_completed_notification_is_exported_in_a_trace_of_its_own(): void { $exporter = new InMemoryExporter(new ArrayObject()); - $ecotone = $this->bootstrap($exporter, [$this->users(), $this->flowCompletedSubscriber()]); + $ecotone = $this->bootstrap($exporter, [$this->users(), $this->flowStepCompletedSubscriber()]); $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); @@ -95,16 +92,18 @@ public function test_the_flow_completed_subscriber_is_exported_in_a_trace_of_its $notificationTraceIds = []; /** @var SpanDataInterface $span */ foreach ($exporter->getSpans() as $span) { - if ($span->getName() === 'send ecotone.tracing.flowCompleted') { + if ($span->getName() === 'send ecotone.tracing.flowStepCompleted') { $notificationTraceIds[] = $span->getContext()->getTraceId(); } } - self::assertCount(1, $notificationTraceIds); - self::assertNotSame($trace->traceId, $notificationTraceIds[0]); - self::assertSame( - ['process class@anonymous::whenCompleted', 'send ecotone.tracing.flowCompleted'], - self::exportedSpanNamesOf($exporter, $notificationTraceIds[0]), - ); + self::assertNotEmpty($notificationTraceIds); + self::assertNotContains($trace->traceId, $notificationTraceIds); + foreach ($notificationTraceIds as $notificationTraceId) { + self::assertSame( + ['process class@anonymous::whenStepCompleted', 'send ecotone.tracing.flowStepCompleted'], + self::exportedSpanNamesOf($exporter, $notificationTraceId), + ); + } } public function test_a_second_application_on_the_same_database_sees_the_completion_the_first_one_stored(): void @@ -184,16 +183,12 @@ public function notify(string $userId): void }; } - private function flowCompletedSubscriber(): object + private function flowStepCompletedSubscriber(): object { return new class () { - /** @var string[] */ - public array $completedTraces = []; - - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event): void + #[OnFlowStepCompleted] + public function whenStepCompleted(FlowStepCompleted $event): void { - $this->completedTraces[] = $event->tracingId; } }; } diff --git a/docs/coding-conventions.md b/docs/coding-conventions.md index 4def16b4c..ed7c041e2 100644 --- a/docs/coding-conventions.md +++ b/docs/coding-conventions.md @@ -165,7 +165,7 @@ for each one, whether it is supported (with a test) or refused (with a message t - **Settings that contradict** — an option combination that cannot work, such as a subscriber with notifications turned off, is refused naming the setting to remove. -Evidence: flow tracking shipped three of these gaps at once. `#[Asynchronous]` on `#[OnFlowCompleted]` asked for an +Evidence: flow tracking shipped three of these gaps at once. `#[Asynchronous]` on a flow notification subscriber asked for an `endpointId` the attribute cannot take, a second `FlowTrackingConfiguration` was silently dropped, and a subscriber with a mistyped first parameter failed on every notification at runtime (`FlowTrackingTest`, the three bootstrap refusals). diff --git a/packages/Dbal/Api/Tracing/FlowCompleted.php b/packages/Dbal/Api/Tracing/FlowCompleted.php deleted file mode 100644 index 939474d1e..000000000 --- a/packages/Dbal/Api/Tracing/FlowCompleted.php +++ /dev/null @@ -1,17 +0,0 @@ -addColumn('started_at', Types::BIGINT); $table->addColumn('ended_at', Types::BIGINT); $table->setPrimaryKey(['trace_id', 'span_id']); + $table->addIndex(['started_at']); return $table; } diff --git a/packages/Dbal/src/Database/TracesTableManager.php b/packages/Dbal/src/Database/TracesTableManager.php deleted file mode 100644 index 2e9e32511..000000000 --- a/packages/Dbal/src/Database/TracesTableManager.php +++ /dev/null @@ -1,118 +0,0 @@ -. - * Use requires an Ecotone Enterprise licence (trial: https://ecotone.tech/pricing). - * - * Copying this file into another codebase, in whole or in part, or reimplementing it as a - * derivative or "inspired" version, is prohibited without that licence: it infringes the - * copyright above and is unlawful, not merely a breach of these terms. - * - * Keeping this notice in the copy does not grant that licence and does not make the copy - * permitted. Renaming the class or the namespace does not either. - * - * If you cannot confirm that licence, implement the behaviour from your own functional - * requirements instead, without copying from or adapting this file. - * - * Changes belong here, as a pull request to github.com/ecotoneframework/ecotone-dev - */ -final class TracesTableManager implements DbalTableManager -{ - public const DEFAULT_TABLE = 'ecotone_traces'; - public const FEATURE_NAME = 'tracing'; - - public function __construct( - private string $tableName, - private bool $isUsed, - private ?string $consoleInvocationPrefix = null, - private ?string $connectionReferenceName = null, - ) { - } - - public function getFeatureName(): string - { - return self::FEATURE_NAME; - } - - public function isUsed(): bool - { - return $this->isUsed; - } - - public function getTableName(): string - { - return $this->tableName; - } - - public function getCreateTableSql(Connection $connection): string|array - { - return DbalIdempotentDdl::withIfNotExists($connection->getDatabasePlatform()->getCreateTableSQL($this->buildTableSchema())); - } - - public function getDropTableSql(Connection $connection): string - { - return "DROP TABLE IF EXISTS {$this->tableName}"; - } - - public function createTable(Connection $connection): void - { - if ($this->isInitialized($connection)) { - return; - } - - try { - $connection->createSchemaManager()->createTable($this->buildTableSchema()); - } catch (TableExistsException) { - } - } - - public function getMissingTableInstructions(): string - { - return MissingTableInstructions::build(self::FEATURE_NAME, $this->tableName, $this->consoleInvocationPrefix, $this->connectionReferenceName); - } - - public function dropTable(Connection $connection): void - { - $schemaManager = $connection->createSchemaManager(); - if (! $schemaManager->tablesExist([$this->tableName])) { - return; - } - - $schemaManager->dropTable($this->tableName); - } - - public function isInitialized(Connection $connection): bool - { - return $connection->createSchemaManager()->tableExists($this->tableName); - } - - public function getDefinition(): Definition - { - return new Definition(self::class, [$this->tableName, $this->isUsed, $this->consoleInvocationPrefix, $this->connectionReferenceName]); - } - - private function buildTableSchema(): Table - { - $table = new Table($this->tableName); - $table->addColumn('trace_id', Types::STRING, ['length' => 32]); - $table->addColumn('started_at', Types::BIGINT); - $table->addColumn('completed_at', Types::BIGINT, ['notnull' => false]); - $table->addColumn('version', Types::INTEGER, ['default' => 1]); - $table->setPrimaryKey(['trace_id']); - $table->addIndex(['started_at']); - - return $table; - } -} diff --git a/packages/Dbal/src/Tracing/DbalTraceStore.php b/packages/Dbal/src/Tracing/DbalTraceStore.php index cc6843b60..1c5a9610a 100644 --- a/packages/Dbal/src/Tracing/DbalTraceStore.php +++ b/packages/Dbal/src/Tracing/DbalTraceStore.php @@ -7,11 +7,9 @@ use Closure; use Doctrine\DBAL\ArrayParameterType; use Doctrine\DBAL\Connection; -use Doctrine\DBAL\Exception\UniqueConstraintViolationException; use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; use Doctrine\DBAL\Types\Types; use Ecotone\Api\Gateway\EcotoneClockInterface; -use Ecotone\Api\Messaging\ConcurrencyException; use Ecotone\Api\Scheduling\DatePoint; use Ecotone\Api\Scheduling\Duration; use Ecotone\Api\Tracing\Span; @@ -21,7 +19,6 @@ use Ecotone\Dbal\Connection\DbalContext; use Ecotone\Dbal\Database\MissingTableFailure; use Ecotone\Dbal\Database\SpansTableManager; -use Ecotone\Dbal\Database\TracesTableManager; use Ecotone\Dbal\DbalReconnectableConnectionFactory; use Ecotone\Enqueue\CachedConnectionFactory; use Interop\Queue\ConnectionFactory; @@ -53,33 +50,15 @@ final class DbalTraceStore implements TraceStore { public function __construct( private ConnectionFactory $connectionFactory, - private TracesTableManager $tracesTableManager, private SpansTableManager $spansTableManager, private EcotoneClockInterface $clock, ) { } public function append(array $spans): void - { - $spansByTrace = []; - foreach ($spans as $span) { - $spansByTrace[$span->traceId][] = $span; - } - - $connection = $this->connection(); - foreach ($spansByTrace as $traceId => $traceSpans) { - $this->onTables(function () use ($connection, $traceId, $traceSpans): void { - $this->claimTrace($connection, $traceId, $traceSpans); - $this->insertSpans($connection, $traceSpans); - }); - } - } - - public function markCompletedIfDone(string $traceId, DatePoint $now): bool { $connection = $this->connection(); - - return $this->onTables(fn (): bool => TraceLedger::progressOf($traceId, $this->spansOf($connection, $traceId))->isCompleted() && $this->markCompleted($connection, $traceId, $now)); + $this->onTables(fn () => $this->insertSpans($connection, $spans)); } public function stepsOf(string $traceId): array @@ -92,11 +71,11 @@ public function progressOf(string $traceId): TraceProgress return $this->onTables(fn (): TraceProgress => TraceLedger::progressOf($traceId, $this->spansOf($this->connection(), $traceId))); } - public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore, int $batchSize): void + public function removeTracesStartedBefore(DatePoint $startedBefore, int $batchSize): void { $connection = $this->connection(); do { - $removed = $this->onTables(fn (): int => $connection->transactional(fn (): int => $this->removeExpiredBatch($connection, $completedTracesStartedBefore, $anyTraceStartedBefore, $batchSize))); + $removed = $this->onTables(fn (): int => $connection->transactional(fn (): int => $this->removeBatchStartedBefore($connection, $startedBefore, $batchSize))); } while ($removed === $batchSize); } @@ -105,14 +84,13 @@ public function withinTransaction(Closure $work): mixed return $this->connection()->transactional($work); } - private function removeExpiredBatch(Connection $connection, DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore, int $batchSize): int + private function removeBatchStartedBefore(Connection $connection, DatePoint $startedBefore, int $batchSize): int { $traceIds = $connection->createQueryBuilder() - ->select('trace_id') - ->from($this->tracesTableManager->getTableName()) - ->where('started_at < :anyStartedBefore OR (completed_at IS NOT NULL AND started_at < :completedStartedBefore)') - ->setParameter('anyStartedBefore', $anyTraceStartedBefore->unixTime()->inMicroseconds(), Types::BIGINT) - ->setParameter('completedStartedBefore', $completedTracesStartedBefore->unixTime()->inMicroseconds(), Types::BIGINT) + ->select('DISTINCT trace_id') + ->from($this->spansTableManager->getTableName()) + ->where('started_at < :startedBefore') + ->setParameter('startedBefore', $startedBefore->unixTime()->inMicroseconds(), Types::BIGINT) ->setMaxResults($batchSize) ->executeQuery() ->fetchFirstColumn(); @@ -120,58 +98,15 @@ private function removeExpiredBatch(Connection $connection, DatePoint $completed return 0; } - foreach ([$this->spansTableManager->getTableName(), $this->tracesTableManager->getTableName()] as $tableName) { - $connection->createQueryBuilder() - ->delete($tableName) - ->where('trace_id IN (:traceIds)') - ->setParameter('traceIds', $traceIds, ArrayParameterType::STRING) - ->executeStatement(); - } + $connection->createQueryBuilder() + ->delete($this->spansTableManager->getTableName()) + ->where('trace_id IN (:traceIds)') + ->setParameter('traceIds', $traceIds, ArrayParameterType::STRING) + ->executeStatement(); return count($traceIds); } - /** - * @param Span[] $traceSpans - */ - private function claimTrace(Connection $connection, string $traceId, array $traceSpans): void - { - $tableName = $this->tracesTableManager->getTableName(); - $readVersion = $connection->fetchOne("SELECT version FROM {$tableName} WHERE trace_id = :traceId", ['traceId' => $traceId]); - if ($readVersion === false) { - try { - $connection->insert($tableName, [ - 'trace_id' => $traceId, - 'started_at' => min(array_map(fn (Span $span): int => $span->startedAt->unixTime()->inMicroseconds(), $traceSpans)), - 'completed_at' => null, - 'version' => 1, - ], ['started_at' => Types::BIGINT]); - } catch (UniqueConstraintViolationException) { - throw ConcurrencyException::forConcurrentlyRecordedFlow($traceId); - } - - return; - } - - $reopened = $connection->executeStatement( - "UPDATE {$tableName} SET version = version + 1, completed_at = NULL WHERE trace_id = :traceId AND version = :readVersion", - ['traceId' => $traceId, 'readVersion' => (int) $readVersion], - ['readVersion' => Types::INTEGER], - ); - if ($reopened === 0) { - throw ConcurrencyException::forConcurrentlyRecordedFlow($traceId); - } - } - - private function markCompleted(Connection $connection, string $traceId, DatePoint $now): bool - { - return $connection->executeStatement( - "UPDATE {$this->tracesTableManager->getTableName()} SET completed_at = :completedAt WHERE trace_id = :traceId AND completed_at IS NULL", - ['completedAt' => $now->unixTime()->inMicroseconds(), 'traceId' => $traceId], - ['completedAt' => Types::BIGINT], - ) === 1; - } - /** * @param Span[] $spans */ @@ -256,7 +191,7 @@ private function onTables(callable $statement): mixed try { return $statement(); } catch (Throwable $failure) { - throw MissingTableFailure::explained(MissingTableFailure::explained($failure, $this->tracesTableManager), $this->spansTableManager); + throw MissingTableFailure::explained($failure, $this->spansTableManager); } } diff --git a/packages/Dbal/src/Tracing/DbalTracingModule.php b/packages/Dbal/src/Tracing/DbalTracingModule.php index 931153554..dded367b4 100644 --- a/packages/Dbal/src/Tracing/DbalTracingModule.php +++ b/packages/Dbal/src/Tracing/DbalTracingModule.php @@ -17,10 +17,8 @@ use Ecotone\Api\Dbal\ExtensionObject\DbalConfiguration; use Ecotone\Api\Dbal\ExtensionObject\DbalConnectionReference; use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; -use Ecotone\Api\Dbal\Tracing\FlowCompleted; use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; -use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -36,7 +34,6 @@ use Ecotone\Api\Projecting\ProjectionFlush; use Ecotone\Dbal\Database\DbalTableManagerReference; use Ecotone\Dbal\Database\SpansTableManager; -use Ecotone\Dbal\Database\TracesTableManager; use Ecotone\Messaging\Config\Annotation\AnnotationModule; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\ExtensionObjectResolver; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\NoExternalConfigurationModule; @@ -87,14 +84,12 @@ final class DbalTracingModule extends NoExternalConfigurationModule implements A private const USER_MESSAGE_GATEWAYS_POINTCUT = MessageGateway::class . '&¬(' . CommandBus::class . ')&¬(' . EventBus::class . ')&¬(' . QueryBus::class . ')'; /** - * @param string[] $flowCompletedSubscribers * @param string[] $flowStepCompletedSubscribers * @param array $subscriberMethods * @param string[] $pollingProjectionNames * @param array $destinationsByClass */ private function __construct( - private array $flowCompletedSubscribers, private array $flowStepCompletedSubscribers, private array $subscriberMethods, private array $pollingProjectionNames, @@ -105,12 +100,8 @@ private function __construct( public static function create(AnnotationFinder $annotationRegistrationService, InterfaceToCallRegistry $interfaceToCallRegistry): static { return new self( - self::placementsOf($annotationRegistrationService, OnFlowCompleted::class), self::placementsOf($annotationRegistrationService, OnFlowStepCompleted::class), - [ - ...self::subscriberMethodsOf($annotationRegistrationService, OnFlowCompleted::class, FlowCompleted::class), - ...self::subscriberMethodsOf($annotationRegistrationService, OnFlowStepCompleted::class, FlowStepCompleted::class), - ], + self::subscriberMethodsOf($annotationRegistrationService, OnFlowStepCompleted::class, FlowStepCompleted::class), self::pollingProjectionNames($annotationRegistrationService), self::destinationsByClass($annotationRegistrationService), ); @@ -124,10 +115,6 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO $connectionReference = self::connectionReferenceOf($extensionObjects); $consoleInvocationPrefix = ConsoleInvocationResolver::resolveConsolePrefix(ExtensionObjectResolver::resolveUnique(BootstrapConfiguration::class, $extensionObjects, BootstrapConfiguration::createWithDefaults())); $storesOnDatabase = $flowTracking !== null && ! $flowTracking->inMemory; - $messagingConfiguration->registerServiceDefinition( - TracesTableManager::class, - new Definition(TracesTableManager::class, [TracesTableManager::DEFAULT_TABLE, $storesOnDatabase, $consoleInvocationPrefix, $connectionReference]) - ); $messagingConfiguration->registerServiceDefinition( SpansTableManager::class, new Definition(SpansTableManager::class, [SpansTableManager::DEFAULT_TABLE, $storesOnDatabase, $consoleInvocationPrefix, $connectionReference]) @@ -141,7 +128,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO $messagingConfiguration->registerServiceDefinition( TraceStore::class, $storesOnDatabase - ? new Definition(DbalTraceStore::class, [new Reference($connectionReference), new Reference(TracesTableManager::class), new Reference(SpansTableManager::class), new Reference(EcotoneClockInterface::class)]) + ? new Definition(DbalTraceStore::class, [new Reference($connectionReference), new Reference(SpansTableManager::class), new Reference(EcotoneClockInterface::class)]) : new Definition(InMemoryTraceStore::class) ); $messagingConfiguration->registerServiceDefinition( @@ -149,9 +136,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO new Definition(StoringSpanRecorder::class, [ new Reference(TraceStore::class), $flowTracking->notifications && $this->flowStepCompletedSubscribers !== [], - $flowTracking->notifications && $this->flowCompletedSubscribers !== [], $flowTracking->onlyRequestedFlows, - new Reference(EcotoneClockInterface::class), ]) ); $messagingConfiguration->registerServiceDefinition( @@ -177,7 +162,6 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ) ); if ($flowTracking->notifications) { - $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnFlowCompleted::CHANNEL)); $messagingConfiguration->registerMessageChannel(SimpleMessageChannelBuilder::createPublishSubscribeChannel(OnFlowStepCompleted::CHANNEL)); } if ($flowTracking->tracingGateway) { @@ -206,7 +190,6 @@ public function getModuleExtensions(ServiceConfiguration $serviceConfiguration, { $connectionReference = self::connectionReferenceOf($serviceExtensions); $extensions = [ - new DbalTableManagerReference(TracesTableManager::class, $connectionReference), new DbalTableManagerReference(SpansTableManager::class, $connectionReference), ]; if (self::flowTrackingOf($serviceExtensions) !== null) { @@ -236,7 +219,7 @@ private static function flowTrackingOf(array $extensionObjects): ?FlowTrackingCo private function refuseMisconfiguredFlowTracking(Configuration $messagingConfiguration, ?FlowTrackingConfiguration $flowTracking): void { - $subscribers = [...$this->flowCompletedSubscribers, ...$this->flowStepCompletedSubscribers]; + $subscribers = $this->flowStepCompletedSubscribers; $enterpriseUsages = [...($flowTracking !== null ? ['FlowTrackingConfiguration'] : []), ...$subscribers]; if ($enterpriseUsages !== [] && ! $messagingConfiguration->isRunningForEnterpriseLicence()) { throw LicensingException::create(sprintf( @@ -250,7 +233,7 @@ private function refuseMisconfiguredFlowTracking(Configuration $messagingConfigu return; } if ($flowTracking === null) { - throw ConfigurationException::create(implode(', ', $subscribers) . ' needs flow tracking, which decides when a flow is complete. Register FlowTrackingConfiguration::createWithDefaults() from a #[ServiceContext] method.'); + throw ConfigurationException::create(implode(', ', $subscribers) . ' needs flow tracking, which records the steps of every flow. Register FlowTrackingConfiguration::createWithDefaults() from a #[ServiceContext] method.'); } if (! $flowTracking->notifications) { throw ConfigurationException::create(implode(', ', $subscribers) . ' would never be called, because FlowTrackingConfiguration::withNotifications(false) turned notifications off. Remove that call, or remove the attribute.'); diff --git a/packages/Dbal/src/Tracing/ExpiredTraceRemover.php b/packages/Dbal/src/Tracing/ExpiredTraceRemover.php index e79c39d51..88af6f18e 100644 --- a/packages/Dbal/src/Tracing/ExpiredTraceRemover.php +++ b/packages/Dbal/src/Tracing/ExpiredTraceRemover.php @@ -37,7 +37,6 @@ public function __construct( public function removeExpiredTraces(): void { - $now = $this->clock->now(); - $this->traceStore->removeExpiredTraces($now->sub(Duration::milliseconds($this->retentionInMilliseconds)), $now->sub(Duration::milliseconds(2 * $this->retentionInMilliseconds)), $this->removalBatchSize); + $this->traceStore->removeTracesStartedBefore($this->clock->now()->sub(Duration::milliseconds($this->retentionInMilliseconds)), $this->removalBatchSize); } } diff --git a/packages/Dbal/src/Tracing/InMemoryTraceStore.php b/packages/Dbal/src/Tracing/InMemoryTraceStore.php index 896e38a2c..4cf565b89 100644 --- a/packages/Dbal/src/Tracing/InMemoryTraceStore.php +++ b/packages/Dbal/src/Tracing/InMemoryTraceStore.php @@ -30,8 +30,6 @@ final class InMemoryTraceStore implements TraceStore { /** @var array> */ private array $spansByTrace = []; - /** @var array */ - private array $completedAt = []; public function append(array $spans): void { @@ -40,20 +38,9 @@ public function append(array $spans): void continue; } $this->spansByTrace[$span->traceId][$span->spanId] = $span; - unset($this->completedAt[$span->traceId]); } } - public function markCompletedIfDone(string $traceId, DatePoint $now): bool - { - if (isset($this->completedAt[$traceId]) || ! $this->progressOf($traceId)->isCompleted()) { - return false; - } - $this->completedAt[$traceId] = $now; - - return true; - } - public function progressOf(string $traceId): TraceProgress { return TraceLedger::progressOf($traceId, array_values($this->spansByTrace[$traceId] ?? [])); @@ -64,12 +51,11 @@ public function stepsOf(string $traceId): array return array_values($this->spansByTrace[$traceId] ?? []); } - public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore, int $batchSize): void + public function removeTracesStartedBefore(DatePoint $startedBefore, int $batchSize): void { foreach ($this->spansByTrace as $traceId => $spans) { - $startedAt = min(array_map(fn (Span $span): DatePoint => $span->startedAt, $spans)); - if ($startedAt < $anyTraceStartedBefore || (isset($this->completedAt[$traceId]) && $startedAt < $completedTracesStartedBefore)) { - unset($this->spansByTrace[$traceId], $this->completedAt[$traceId]); + if (min(array_map(fn (Span $span): DatePoint => $span->startedAt, $spans)) < $startedBefore) { + unset($this->spansByTrace[$traceId]); } } } diff --git a/packages/Dbal/src/Tracing/StoredTracingGateway.php b/packages/Dbal/src/Tracing/StoredTracingGateway.php index 3c479ac2e..86733b050 100644 --- a/packages/Dbal/src/Tracing/StoredTracingGateway.php +++ b/packages/Dbal/src/Tracing/StoredTracingGateway.php @@ -109,7 +109,7 @@ private function refuseWaitingLongerThanFifteenMinutes(Duration $timeout): void } throw InvalidArgumentException::create(sprintf( - 'TracingGateway::isCompletedWithin() was asked to wait %d milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Wait at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowCompleted] receives a FlowCompleted once the flow completes, and one with #[OnFlowStepCompleted] a FlowStepCompleted for each of its steps.', + 'TracingGateway::isCompletedWithin() was asked to wait %d milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Wait at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowStepCompleted] receives a FlowStepCompleted once each step of the flow committed, and can ask TracingGateway::isCompletedFor() whether the flow completed with it.', $timeout->inMilliseconds(), )); } diff --git a/packages/Dbal/src/Tracing/StoringSpanRecorder.php b/packages/Dbal/src/Tracing/StoringSpanRecorder.php index 0c6bc1a45..f2051ab6e 100644 --- a/packages/Dbal/src/Tracing/StoringSpanRecorder.php +++ b/packages/Dbal/src/Tracing/StoringSpanRecorder.php @@ -5,9 +5,7 @@ namespace Ecotone\Dbal\Tracing; use Closure; -use Ecotone\Api\Dbal\Tracing\FlowCompleted; use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; -use Ecotone\Api\Gateway\EcotoneClockInterface; use Ecotone\Api\Tracing\Span; use Ecotone\Api\Tracing\SpanAttribute; use Ecotone\Api\Tracing\SpanRecorder; @@ -43,15 +41,11 @@ final class StoringSpanRecorder implements SpanRecorder private bool $unitOfWorkActive = false; /** @var Span[] */ private array $committedSpans = []; - /** @var string[] */ - private array $completedTraces = []; public function __construct( private TraceStore $traceStore, private bool $publishesEndedSpans, - private bool $publishesCompletedTraces, private bool $storesOnlyCarriedTraces, - private EcotoneClockInterface $clock, ) { } @@ -76,7 +70,6 @@ public function endUnitOfWork(): void { $this->unitOfWorkActive = false; $this->committedSpans = []; - $this->completedTraces = []; } public function openScope(?ActiveSpan $wrappingSpan): void @@ -154,14 +147,7 @@ private function store(array $openSpansSoFar): void foreach ($spans as $span) { $this->storedSpanIds[$span->spanId] = true; } - $completedTraces = []; - foreach (array_unique(array_map(fn (Span $span): string => $span->traceId, $spans)) as $traceId) { - if ($this->traceStore->markCompletedIfDone($traceId, $this->clock->now())) { - $completedTraces[] = $traceId; - } - } $this->committedSpans = [...$this->committedSpans, ...$spans]; - $this->completedTraces = [...$this->completedTraces, ...$completedTraces]; } public function discard(): void @@ -170,14 +156,11 @@ public function discard(): void } /** - * @return array + * @return FlowStepCompleted[] */ public function popCommittedNotifications(): array { - $notifications = [ - ...array_map(fn (Span $span): FlowStepCompleted => new FlowStepCompleted($span), $this->publishesEndedSpans ? $this->committedSpans : []), - ...array_map(fn (string $traceId): FlowCompleted => new FlowCompleted($traceId), $this->publishesCompletedTraces ? array_values(array_unique($this->completedTraces)) : []), - ]; + $notifications = array_map(fn (Span $span): FlowStepCompleted => new FlowStepCompleted($span), $this->publishesEndedSpans ? $this->committedSpans : []); $this->endUnitOfWork(); return $notifications; diff --git a/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php b/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php index 68f3dd0c5..8bc2fe94d 100644 --- a/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php +++ b/packages/Dbal/src/Tracing/TraceRecordingInterceptor.php @@ -4,9 +4,7 @@ namespace Ecotone\Dbal\Tracing; -use Ecotone\Api\Dbal\Tracing\FlowCompleted; use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; -use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Interceptor\MethodInvocation; use Ecotone\Api\Tracing\SpanAttribute; @@ -113,9 +111,9 @@ public function recordProjectionBatch(MethodInvocation $methodInvocation): mixed } } - private function deliver(FlowStepCompleted|FlowCompleted $notification): void + private function deliver(FlowStepCompleted $notification): void { - $channel = $notification instanceof FlowCompleted ? OnFlowCompleted::CHANNEL : OnFlowStepCompleted::CHANNEL; + $channel = OnFlowStepCompleted::CHANNEL; $span = $this->tracer->startRootSpan('send ' . $channel, SpanKind::Producer, [ SpanAttribute::MESSAGING_OPERATION_TYPE => SpanAttribute::OPERATION_SEND, SpanAttribute::MESSAGING_DESTINATION_NAME => $channel, diff --git a/packages/Dbal/src/Tracing/TraceStore.php b/packages/Dbal/src/Tracing/TraceStore.php index 44d9fb59c..2b5c1e4b0 100644 --- a/packages/Dbal/src/Tracing/TraceStore.php +++ b/packages/Dbal/src/Tracing/TraceStore.php @@ -33,8 +33,6 @@ interface TraceStore */ public function append(array $spans): void; - public function markCompletedIfDone(string $traceId, DatePoint $now): bool; - public function progressOf(string $traceId): TraceProgress; /** @@ -42,7 +40,7 @@ public function progressOf(string $traceId): TraceProgress; */ public function stepsOf(string $traceId): array; - public function removeExpiredTraces(DatePoint $completedTracesStartedBefore, DatePoint $anyTraceStartedBefore, int $batchSize): void; + public function removeTracesStartedBefore(DatePoint $startedBefore, int $batchSize): void; public function withinTransaction(Closure $work): mixed; } diff --git a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php index 844dc889d..1a74adfed 100644 --- a/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php +++ b/packages/Dbal/tests/Conformance/FlowTrackingConformanceTest.php @@ -17,10 +17,8 @@ use Ecotone\Api\Dbal\ExtensionObject\DbalDeadLetterBuilder; use Ecotone\Api\Dbal\ExtensionObject\MultiTenantConfiguration; use Ecotone\Api\Dbal\Gateway\DeadLetterGateway; -use Ecotone\Api\Dbal\Tracing\FlowCompleted; use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; -use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\EventSourcing\Event; @@ -123,28 +121,27 @@ public function test_waiting_for_a_pending_flow_returns_false_and_the_flow_names } #[DataProvider('implementations')] - public function test_a_flow_completed_subscriber_is_called_once_after_the_last_handler_committed(string $implementation): void + public function test_a_step_subscriber_asking_the_gateway_sees_the_flow_completed_once_the_last_handler_committed(string $implementation): void { $subscriber = new class () { - /** @var string[] */ - public array $completedTraces = []; + /** @var array */ + public array $completedAfterStep = []; - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event): void + #[OnFlowStepCompleted] + public function whenStepCompleted(FlowStepCompleted $event, TracingGateway $tracingGateway): void { - $this->completedTraces[] = $event->tracingId; + $this->completedAfterStep[$event->step->name] = $tracingGateway->isCompletedFor($event->step->traceId); } }; $ecotone = $this->bootstrap($implementation, [$this->users(), $this->asynchronousNotifier(), $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - self::assertSame([], $subscriber->completedTraces); + self::assertNotContains(true, $subscriber->completedAfterStep); - $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - self::assertSame([$trace->traceId], $subscriber->completedTraces); + self::assertTrue($subscriber->completedAfterStep['receive async_channel']); } #[DataProvider('implementations')] @@ -475,23 +472,16 @@ public function notify(string $userId): void } #[DataProvider('implementations')] - public function test_a_pending_trace_older_than_the_retention_survives_until_twice_the_retention(string $implementation): void + public function test_a_pending_flow_older_than_the_retention_is_removed_like_a_completed_one(string $implementation): void { $ecotone = $this->bootstrap($implementation, [$this->users(), $this->asynchronousNotifier()], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')], Duration::minutes(10)); - $delayed = SpanContext::newTrace(); - $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $delayed->toTraceparent()]); + $abandoned = SpanContext::newTrace(); + $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $abandoned->toTraceparent()]); $ecotone->advanceTimeBy(TimeSpan::withMinutes(11)); $ecotone->runConsoleCommand('ecotone:tracing:remove-expired-traces'); - $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - self::assertTrue($this->completion($ecotone)->isCompletedFor($delayed->traceId)); - - $abandoned = SpanContext::newTrace(); - $ecotone->sendCommandWithRouting('user.register', '2', metadata: [MessageHeaders::TRACEPARENT => $abandoned->toTraceparent()]); - $ecotone->advanceTimeBy(TimeSpan::withMinutes(21)); - $ecotone->runConsoleCommand('ecotone:tracing:remove-expired-traces'); - self::assertFalse($this->completion($ecotone)->isCompletedFor($abandoned->traceId, 'user.registration')); + self::assertNull($this->completion($ecotone)->fetchFlow($abandoned->traceId)); } #[DataProvider('implementations')] @@ -501,10 +491,10 @@ public function test_a_subscriber_that_throws_does_not_fail_the_committed_comman /** @var string[] */ public array $completedTraces = []; - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event): void + #[OnFlowStepCompleted] + public function whenStepCompleted(FlowStepCompleted $event): void { - $this->completedTraces[] = $event->tracingId; + $this->completedTraces[] = $event->step->traceId; if (count($this->completedTraces) === 1) { throw new RuntimeException('Websocket is down'); } @@ -515,26 +505,17 @@ public function whenCompleted(FlowCompleted $event): void $second = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $first->toTraceparent()]); - self::assertSame([$first->traceId], $subscriber->completedTraces); + $stepsOfFirst = count($subscriber->completedTraces); $ecotone->sendCommandWithRouting('user.register', '2', metadata: [MessageHeaders::TRACEPARENT => $second->toTraceparent()]); - self::assertSame([$first->traceId, $second->traceId], $subscriber->completedTraces); + self::assertSame([$first->traceId], array_values(array_unique(array_slice($subscriber->completedTraces, 0, $stepsOfFirst)))); + self::assertSame([$second->traceId], array_values(array_unique(array_slice($subscriber->completedTraces, $stepsOfFirst)))); } #[DataProvider('implementations')] - public function test_a_trace_that_receives_new_work_after_completing_is_notified_again_when_it_completes_again(string $implementation): void + public function test_a_flow_that_receives_new_work_after_completing_is_pending_until_that_work_completes(string $implementation): void { - $subscriber = new class () { - /** @var string[] */ - public array $completedTraces = []; - - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event): void - { - $this->completedTraces[] = $event->tracingId; - } - }; $orders = new class () { #[Asynchronous('async_channel')] #[CommandHandler('order.place', endpointId: 'order.placing')] @@ -542,20 +523,18 @@ public function place(string $orderId): void { } }; - $ecotone = $this->bootstrap($implementation, [$this->users(), $orders, $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); + $ecotone = $this->bootstrap($implementation, [$this->users(), $orders], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); $trace = SpanContext::newTrace(); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - self::assertSame([$trace->traceId], $subscriber->completedTraces); + self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); $ecotone->sendCommandWithRouting('order.place', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); self::assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); - self::assertSame([$trace->traceId], $subscriber->completedTraces); $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); self::assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); - self::assertSame([$trace->traceId, $trace->traceId], $subscriber->completedTraces); } public function test_a_channel_published_through_the_outbox_completes_once_its_consumer_ran(): void diff --git a/packages/Dbal/tests/DbalMessagingTestCase.php b/packages/Dbal/tests/DbalMessagingTestCase.php index d61f09698..69728fda4 100644 --- a/packages/Dbal/tests/DbalMessagingTestCase.php +++ b/packages/Dbal/tests/DbalMessagingTestCase.php @@ -80,7 +80,6 @@ public static function cleanUpDbalTables(Connection $connection): void self::deleteTable(DbalDocumentStore::ECOTONE_DOCUMENT_STORE, $connection); self::deleteTable(DbalHandledMessages::DEFAULT_DEDUPLICATION_TABLE, $connection); self::deleteTable(\Ecotone\Dbal\Database\SpansTableManager::DEFAULT_TABLE, $connection); - self::deleteTable(\Ecotone\Dbal\Database\TracesTableManager::DEFAULT_TABLE, $connection); self::deleteTable('persons', $connection); self::deleteTable('activities', $connection); } diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingRaceTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingRaceTest.php index 967fde26a..c818bb4e4 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingRaceTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingRaceTest.php @@ -12,9 +12,9 @@ use Ecotone\Api\Attribute\EventHandler; use Ecotone\Api\Dbal\DbalConnectionFactory; use Ecotone\Api\Dbal\ExtensionObject\DbalBackedMessageChannelBuilder; -use Ecotone\Api\Dbal\Tracing\FlowCompleted; +use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; -use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; +use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\ExtensionObject\ExecutionPollingMetadata; use Ecotone\Api\ExtensionObject\ModulePackageList; @@ -53,15 +53,15 @@ final class FlowTrackingRaceTest extends DbalMessagingTestCase { private const MILLISECONDS_THE_FIRST_CONSUMER_STAYS_UNCOMMITTED = 1000; + public static string $observationsFile = ''; + public function tearDown(): void { CommitHoldingConnection::$beforeCommit = null; parent::tearDown(); } - public static string $notificationsFile = ''; - - public function test_two_consumers_finishing_the_same_flow_at_the_same_moment_notify_its_completion_once(): void + public function test_two_consumers_of_one_flow_commit_without_waiting_for_each_other_and_the_last_step_sees_the_flow_completed(): void { if ($this->getConnectionFactory()->createContext()->getDbalConnection()->getDatabasePlatform() instanceof SQLitePlatform) { self::markTestSkipped('SQLite serialises writers with a database lock, so the second consumer cannot run while the first one holds its transaction open. The race is left unproved on SQLite.'); @@ -72,7 +72,7 @@ public function test_two_consumers_finishing_the_same_flow_at_the_same_moment_no 'connection' => ['wrapperClass' => CommitHoldingConnection::class], ]), setUpTables: true); $tracingId = '0af7651916cd43dd8448eb211c80319c'; - self::$notificationsFile = $this->signalFile(); + self::$observationsFile = $this->signalFile(); $first->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => $tracingId]); $ready = $this->signalFile(); $outcome = $this->signalFile(); @@ -85,20 +85,21 @@ public function test_two_consumers_finishing_the_same_flow_at_the_same_moment_no $this->consumeTheOtherDeliveryOnASecondConnection($ready, $outcome); } $this->waitUntilWritten($ready); - usleep(self::MILLISECONDS_THE_FIRST_CONSUMER_STAYS_UNCOMMITTED * 1000); + $this->waitUntilWritten($outcome, self::MILLISECONDS_THE_FIRST_CONSUMER_STAYS_UNCOMMITTED); }; $first->run('mail_channel', ExecutionPollingMetadata::createWithTestingSetup()); pcntl_waitpid($second, $status); - [$consumed, $waitedMilliseconds] = explode('|', file_get_contents($outcome)); - $notified = file_exists(self::$notificationsFile) ? file(self::$notificationsFile, FILE_IGNORE_NEW_LINES) : []; + [$consumed, $tookMilliseconds] = explode('|', file_get_contents($outcome)); + $observations = file(self::$observationsFile, FILE_IGNORE_NEW_LINES); unlink($ready); unlink($outcome); - @unlink(self::$notificationsFile); + unlink(self::$observationsFile); self::assertSame('consumed', $consumed); + self::assertLessThan(self::MILLISECONDS_THE_FIRST_CONSUMER_STAYS_UNCOMMITTED, (int) $tookMilliseconds); + self::assertSame(['user.mailer ran', 'user.texter ran'], array_values(array_filter($observations, fn (string $line): bool => str_ends_with($line, 'ran')))); + self::assertSame('flow completed', end($observations)); self::assertTrue($first->getGateway(TracingGateway::class)->isCompletedFor($tracingId)); - self::assertSame([$tracingId], $notified); - self::assertGreaterThanOrEqual(self::MILLISECONDS_THE_FIRST_CONSUMER_STAYS_UNCOMMITTED / 2, (int) $waitedMilliseconds); } private function consumeTheOtherDeliveryOnASecondConnection(string $ready, string $outcome): never @@ -130,10 +131,12 @@ public function register(string $userId, EventBus $eventBus): void } }; $subscriber = new class () { - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event): void + #[OnFlowStepCompleted] + public function whenStepCompleted(FlowStepCompleted $event, TracingGateway $tracingGateway): void { - file_put_contents(FlowTrackingRaceTest::$notificationsFile, $event->tracingId . PHP_EOL, FILE_APPEND | LOCK_EX); + if ($tracingGateway->isCompletedFor($event->step->traceId)) { + file_put_contents(FlowTrackingRaceTest::$observationsFile, 'flow completed' . PHP_EOL, FILE_APPEND | LOCK_EX); + } } }; $notifications = new class () { @@ -141,12 +144,14 @@ public function whenCompleted(FlowCompleted $event): void #[EventHandler('user.registered', endpointId: 'user.mailer')] public function mail(string $userId): void { + file_put_contents(FlowTrackingRaceTest::$observationsFile, 'user.mailer ran' . PHP_EOL, FILE_APPEND | LOCK_EX); } #[Asynchronous('sms_channel')] #[EventHandler('user.registered', endpointId: 'user.texter')] public function text(string $userId): void { + file_put_contents(FlowTrackingRaceTest::$observationsFile, 'user.texter ran' . PHP_EOL, FILE_APPEND | LOCK_EX); } }; @@ -177,9 +182,9 @@ private function signalFile(): string return $file; } - private function waitUntilWritten(string $file): void + private function waitUntilWritten(string $file, int $atMostMilliseconds = 30_000): void { - $deadline = hrtime(true) + 30 * 1e9; + $deadline = hrtime(true) + $atMostMilliseconds * 1e6; while (! file_exists($file) && hrtime(true) < $deadline) { usleep(10_000); } diff --git a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php index 129ba72d4..03729fe7a 100644 --- a/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php +++ b/packages/Dbal/tests/Integration/Tracing/FlowTrackingTest.php @@ -13,10 +13,8 @@ use Ecotone\Api\Attribute\Polling; use Ecotone\Api\Attribute\QueryHandler; use Ecotone\Api\Attribute\Reference; -use Ecotone\Api\Dbal\Tracing\FlowCompleted; use Ecotone\Api\Dbal\Tracing\FlowStepCompleted; use Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration; -use Ecotone\Api\Dbal\Tracing\OnFlowCompleted; use Ecotone\Api\Dbal\Tracing\OnFlowStepCompleted; use Ecotone\Api\Dbal\Tracing\TracingGateway; use Ecotone\Api\EventSourcing\Event; @@ -204,7 +202,7 @@ public function test_waiting_for_a_flow_longer_than_fifteen_minutes_is_refused_p $ecotone = $this->bootstrap([$this->users()]); $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('TracingGateway::isCompletedWithin() was asked to wait 960000 milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Wait at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowCompleted] receives a FlowCompleted once the flow completes, and one with #[OnFlowStepCompleted] a FlowStepCompleted for each of its steps.'); + $this->expectExceptionMessage('TracingGateway::isCompletedWithin() was asked to wait 960000 milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Wait at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowStepCompleted] receives a FlowStepCompleted once each step of the flow committed, and can ask TracingGateway::isCompletedFor() whether the flow completed with it.'); $this->completion($ecotone)->isCompletedWithin(SpanContext::newTrace()->traceId, Duration::minutes(16)); } @@ -214,7 +212,7 @@ public function test_waiting_for_a_destination_longer_than_fifteen_minutes_is_re $ecotone = $this->bootstrap([$this->users()]); $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('TracingGateway::isCompletedWithin() was asked to wait 960000 milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Wait at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowCompleted] receives a FlowCompleted once the flow completes, and one with #[OnFlowStepCompleted] a FlowStepCompleted for each of its steps.'); + $this->expectExceptionMessage('TracingGateway::isCompletedWithin() was asked to wait 960000 milliseconds, but it blocks the calling process for as long as it waits, which past 15 minutes ties that process up for an extensive period. Wait at most 15 minutes, or react to the flow event-driven instead: a method with #[OnFlowStepCompleted] receives a FlowStepCompleted once each step of the flow committed, and can ask TracingGateway::isCompletedFor() whether the flow completed with it.'); $this->completion($ecotone)->isCompletedWithin(SpanContext::newTrace()->traceId, Duration::minutes(16), 'user.registration'); } @@ -338,51 +336,6 @@ public function notify(string $userId): void $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.notifier')); } - public function test_a_flow_completed_subscriber_is_called_once_after_the_last_handler_committed(): void - { - $subscriber = new class () { - /** @var string[] */ - public array $completedTraces = []; - - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event): void - { - $this->completedTraces[] = $event->tracingId; - } - }; - $ecotone = $this->bootstrap([$this->users(), $this->asynchronousNotifier(), $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); - $trace = SpanContext::newTrace(); - - $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->assertSame([], $subscriber->completedTraces); - - $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - $this->assertSame([$trace->traceId], $subscriber->completedTraces); - - $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - $this->assertSame([$trace->traceId], $subscriber->completedTraces); - } - - public function test_a_synchronous_trace_notifies_its_subscriber_when_the_bus_returns(): void - { - $subscriber = new class () { - /** @var string[] */ - public array $completedTraces = []; - - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event): void - { - $this->completedTraces[] = $event->tracingId; - } - }; - $ecotone = $this->bootstrap([$this->users(), $subscriber]); - $trace = SpanContext::newTrace(); - - $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - - $this->assertSame([$trace->traceId], $subscriber->completedTraces); - } - public function test_a_flow_step_completed_subscriber_receives_every_stored_span_of_the_trace(): void { $subscriber = new class () { @@ -473,32 +426,6 @@ public function record(string $userId): void $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId, 'user.registration')); } - public function test_a_flow_completed_subscriber_sending_a_command_does_not_notify_itself_again(): void - { - $subscriber = new class () { - public int $notifications = 0; - - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event, CommandBus $commandBus): void - { - $this->notifications++; - if ($this->notifications < 5) { - $commandBus->sendWithRouting('cache.refresh', $event->tracingId); - } - } - - #[CommandHandler('cache.refresh', endpointId: 'cache.refreshing')] - public function refresh(string $traceId): void - { - } - }; - $ecotone = $this->bootstrap([$this->users(), $subscriber]); - - $ecotone->sendCommandWithRouting('user.register', '1'); - - $this->assertSame(1, $subscriber->notifications); - } - public function test_a_flow_step_completed_subscriber_sending_a_command_does_not_notify_itself_again(): void { $subscriber = new class () { @@ -527,11 +454,11 @@ public function refresh(string $traceId): void $this->assertSame([$trace->traceId], array_values(array_unique($subscriber->notifiedTraces))); } - public function test_a_failing_flow_completed_subscriber_is_logged_and_does_not_fail_the_command(): void + public function test_a_failing_flow_step_completed_subscriber_is_logged_and_does_not_fail_the_command(): void { $subscriber = new class () { - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event): void + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event): void { throw new RuntimeException('Cache refresh failed'); } @@ -547,7 +474,7 @@ public function whenCompleted(FlowCompleted $event): void $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $this->assertSame(['A subscriber of ecotone.tracing.flowCompleted failed and was skipped: Cache refresh failed'], $logger->getError()); + $this->assertSame(['A subscriber of ecotone.tracing.flowStepCompleted failed and was skipped: Cache refresh failed'], array_values(array_unique($logger->getError()))); $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); } @@ -579,38 +506,6 @@ public function refresh(string $traceId): void $this->assertLessThan(20, count($subscriber->stepNames)); } - public function test_a_flow_completed_subscriber_sending_an_asynchronous_command_is_notified_once(): void - { - $subscriber = new class () { - /** @var string[] */ - public array $completedTraces = []; - - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event, CommandBus $commandBus): void - { - $this->completedTraces[] = $event->tracingId; - if (count($this->completedTraces) < 20) { - $commandBus->sendWithRouting('cache.refresh', $event->tracingId); - } - } - - #[Asynchronous('async_channel')] - #[CommandHandler('cache.refresh', endpointId: 'cache.refreshing')] - public function refresh(string $tracingId): void - { - } - }; - $ecotone = $this->bootstrap([$this->users(), $subscriber], [SimpleMessageChannelBuilder::createQueueChannel('async_channel')]); - $trace = SpanContext::newTrace(); - - $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACEPARENT => $trace->toTraceparent()]); - $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - $ecotone->run('async_channel', ExecutionPollingMetadata::createWithTestingSetup()); - - $this->assertSame([$trace->traceId], $subscriber->completedTraces); - $this->assertTrue($this->completion($ecotone)->isCompletedFor($trace->traceId)); - } - public function test_a_flow_step_completed_subscriber_sending_an_asynchronous_command_is_not_notified_about_its_consumer(): void { $subscriber = new class () { @@ -1144,21 +1039,6 @@ public function test_a_failed_unit_of_work_stores_nothing_by_default(): void $this->assertFalse($this->completion($ecotone)->isCompletedFor($trace->traceId)); } - public function test_a_flow_completed_subscriber_whose_first_parameter_cannot_receive_the_notification_is_refused_at_bootstrap(): void - { - $subscriber = new class () { - #[OnFlowCompleted] - public function whenCompleted(FlowStepCompleted $event): void - { - } - }; - - $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('#[OnFlowCompleted] on ' . $subscriber::class . '::whenCompleted is passed a ' . FlowCompleted::class . ' as its first parameter, but declares it as ' . FlowStepCompleted::class . ', so every notification would fail. Declare that parameter as ' . FlowCompleted::class . '.'); - - $this->bootstrap([$subscriber]); - } - public function test_a_flow_step_completed_subscriber_whose_first_parameter_cannot_receive_the_notification_is_refused_at_bootstrap(): void { $subscriber = new class () { @@ -1174,18 +1054,18 @@ public function whenEnded(string $event): void $this->bootstrap([$subscriber]); } - public function test_an_asynchronous_flow_completed_subscriber_is_refused_at_bootstrap_naming_how_to_run_its_work_in_the_background(): void + public function test_an_asynchronous_flow_step_completed_subscriber_is_refused_at_bootstrap_naming_how_to_run_its_work_in_the_background(): void { $subscriber = new class () { #[Asynchronous('notifications')] - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event): void + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event): void { } }; $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('#[OnFlowCompleted] on ' . $subscriber::class . '::whenCompleted cannot run #[Asynchronous]: a flow notification is delivered from memory once the work committed, so it is never queued. Remove #[Asynchronous], and send a command to an #[Asynchronous] handler from this method for the work that should run in the background.'); + $this->expectExceptionMessage('#[OnFlowStepCompleted] on ' . $subscriber::class . '::whenEnded cannot run #[Asynchronous]: a flow notification is delivered from memory once the work committed, so it is never queued. Remove #[Asynchronous], and send a command to an #[Asynchronous] handler from this method for the work that should run in the background.'); $this->bootstrap([$subscriber], [SimpleMessageChannelBuilder::createQueueChannel('notifications')]); } @@ -1221,14 +1101,14 @@ public function test_flow_tracking_configured_twice_is_refused_at_bootstrap(): v public function test_with_notifications_turned_off_a_subscriber_is_refused_at_bootstrap(): void { $subscriber = new class () { - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event): void + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event): void { } }; $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('#[OnFlowCompleted] on ' . $subscriber::class . '::whenCompleted would never be called, because FlowTrackingConfiguration::withNotifications(false) turned notifications off. Remove that call, or remove the attribute.'); + $this->expectExceptionMessage('#[OnFlowStepCompleted] on ' . $subscriber::class . '::whenEnded would never be called, because FlowTrackingConfiguration::withNotifications(false) turned notifications off. Remove that call, or remove the attribute.'); EcotoneLite::bootstrapFlowTesting( [$subscriber::class], @@ -1259,10 +1139,10 @@ public function test_with_the_tracing_gateway_turned_off_asking_it_names_the_set /** @var string[] */ public array $completedFlows = []; - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event): void + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event): void { - $this->completedFlows[] = $event->tracingId; + $this->completedFlows[] = $event->step->traceId; } }; $ecotone = EcotoneLite::bootstrapFlowTesting( @@ -1273,7 +1153,7 @@ public function whenCompleted(FlowCompleted $event): void ); $ecotone->sendCommandWithRouting('user.register', '1', metadata: [MessageHeaders::TRACING_ID => '4bf92f3577b34da6a3ce929d0e0e4736']); - $this->assertSame(['4bf92f3577b34da6a3ce929d0e0e4736'], $subscriber->completedFlows); + $this->assertSame(['4bf92f3577b34da6a3ce929d0e0e4736'], array_values(array_unique($subscriber->completedFlows))); $this->expectException(ConfigurationException::class); $this->expectExceptionMessage('TracingGateway is turned off by FlowTrackingConfiguration::withTracingGateway(false). Remove that call to ask whether a flow completed.'); @@ -1293,17 +1173,17 @@ public function test_flow_tracking_requires_an_enterprise_licence(): void ); } - public function test_a_flow_completed_subscriber_requires_an_enterprise_licence(): void + public function test_a_flow_step_completed_subscriber_requires_an_enterprise_licence(): void { $subscriber = new class () { - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event): void + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event): void { } }; $this->expectException(LicensingException::class); - $this->expectExceptionMessage('FlowTrackingConfiguration and #[OnFlowCompleted] on ' . $subscriber::class . '::whenCompleted require an Ecotone Enterprise licence. Remove them to run without one.'); + $this->expectExceptionMessage('FlowTrackingConfiguration and #[OnFlowStepCompleted] on ' . $subscriber::class . '::whenEnded require an Ecotone Enterprise licence. Remove them to run without one.'); EcotoneLite::bootstrapFlowTesting( [$subscriber::class], @@ -1330,17 +1210,17 @@ public function whenEnded(FlowStepCompleted $event): void ); } - public function test_a_flow_completed_subscriber_without_flow_tracking_is_refused_at_bootstrap(): void + public function test_a_flow_step_completed_subscriber_without_flow_tracking_is_refused_at_bootstrap(): void { $subscriber = new class () { - #[OnFlowCompleted] - public function whenCompleted(FlowCompleted $event): void + #[OnFlowStepCompleted] + public function whenEnded(FlowStepCompleted $event): void { } }; $this->expectException(ConfigurationException::class); - $this->expectExceptionMessage('#[OnFlowCompleted] on ' . $subscriber::class . '::whenCompleted needs flow tracking, which decides when a flow is complete. Register FlowTrackingConfiguration::createWithDefaults() from a #[ServiceContext] method.'); + $this->expectExceptionMessage('#[OnFlowStepCompleted] on ' . $subscriber::class . '::whenEnded needs flow tracking, which records the steps of every flow. Register FlowTrackingConfiguration::createWithDefaults() from a #[ServiceContext] method.'); EcotoneLite::bootstrapFlowTesting( [$subscriber::class], diff --git a/packages/Ecotone/Api/Messaging/ConcurrencyException.php b/packages/Ecotone/Api/Messaging/ConcurrencyException.php index 5e061f4bd..0f7d37425 100644 --- a/packages/Ecotone/Api/Messaging/ConcurrencyException.php +++ b/packages/Ecotone/Api/Messaging/ConcurrencyException.php @@ -80,16 +80,6 @@ public static function forHandledMessage(string $messageId, string $consumerEndp ), self::errorCode()); } - public static function forConcurrentlyRecordedFlow(string $tracingId): self - { - return new self(sprintf( - 'Flow %s was recorded by another of its units of work after this one read it, so this one cannot tell whether it is the one that completed the flow. ' - . 'Its transaction is rolled back so it can read the flow again: an asynchronous endpoint retries it by default, ' - . 'and a command does when Ecotone retries it with InstantRetryConfiguration::createWithDefaults()->withCommandBusRetry(true, 3, [ConcurrencyException::class]).', - $tracingId, - ), self::errorCode()); - } - private static function whereItIsNow(int $expectedVersion, int $currentVersion): string { return $currentVersion === $expectedVersion diff --git a/upgrade-2.0.md b/upgrade-2.0.md index cadd90d60..79f221be0 100644 --- a/upgrade-2.0.md +++ b/upgrade-2.0.md @@ -3099,8 +3099,8 @@ in an external collector, after the fact. **Now:** registering `Ecotone\Api\Dbal\Tracing\FlowTrackingConfiguration::createWithDefaults()` from a `#[ServiceContext]` method tracks every flow a message starts: the handlers, routes, channels and projections it -reaches, synchronously or asynchronously, across processes. What it tracks is kept in `ecotone_traces` and -`ecotone_tracing`, created by `ecotone:migration:database:setup`; a flow test given no connection keeps it in memory. +reaches, synchronously or asynchronously, across processes. What it tracks is kept in one table, `ecotone_tracing`, +created by `ecotone:migration:database:setup`; a flow test given no connection keeps it in memory. Two features are on by default and each can be turned off while the flows are still tracked: - **`TracingGateway`** has three methods. `isCompletedFor($tracingId)` tells whether the flow is complete, and @@ -3118,12 +3118,15 @@ Two features are on by default and each can be turned off while the flows are st `isCompletedWithin($tracingId, $timeout, UsersListProjection::class)` needs no name to be repeated, and the class completes once every one of its handlers the flow reached ran. A class with no handler or projection is refused at once. Waiting blocks the calling process while it polls, so a timeout above 15 minutes is refused, pointing to - `#[OnFlowCompleted]` and `#[OnFlowStepCompleted]`: a flow that takes longer is reacted to when it completes, not - waited for. + `#[OnFlowStepCompleted]`: a flow that takes longer is reacted to as its steps commit, not waited for. `withTracingGateway(false)` turns it off; the gateway stays injectable and every call names that setting. -- **Notifications**: a method with `#[OnFlowCompleted]` receives a `FlowCompleted` each time its flow reaches - completion, and one with `#[OnFlowStepCompleted]` receives a `FlowStepCompleted` for every step of the flow whose unit of work committed, a bus call, a handler run, a route, a message sent or received, with the step's span in `$event->step`. `withNotifications(false)` turns them off, - and an application that still declares either attribute is refused at bootstrap. +- **Notifications**: a method with `#[OnFlowStepCompleted]` receives a `FlowStepCompleted` for every step of the + flow whose unit of work committed, a bus call, a handler run, a route, a message sent or received, with the step's + span in `$event->step`. To react to the whole flow, or to one destination, the subscriber asks + `TracingGateway::isCompletedFor()`: it is called after its unit of work committed, so the part of the flow that + commits last always sees the flow complete. When two parts commit at the same moment, both can see it complete, so + a reaction to completion is delivered at least once. `withNotifications(false)` turns notifications off, and an + application that still declares the attribute is refused at bootstrap. The flow is named by the **`tracingId`** header, `MessageHeaders::TRACING_ID`: a UUID or 32 hexadecimal characters the application chooses and sends with the command, then passes to `TracingGateway`. Ecotone turns it into the W3C @@ -3141,13 +3144,13 @@ it sent to a channel was consumed from that channel, so an asynchronous command, pending until its consumer processed that exact message. A destination counts messages by id, so a redelivery satisfies its own delivery only, and a message keeps the delivery it started as through dynamic channels, the high-throughput outbox and a dead-letter replay. A flow handed more work after completing, as when a request sends -two commands under one `tracingId`, completes and notifies again once that work is done. `isCompletedFor()` and +two commands under one `tracingId`, is pending again until that work is done. `isCompletedFor()` and `isCompletedWithin()` refuse a `#[Polling]` projection by name, because a polled run is not part of the flow that appended its events. Everything is written inside the transaction of the unit of work it describes, a bus call or an asynchronous -consumption: the spans buffered during the work, the span that wraps it as it stands at that moment, and the decision -whether the flow is complete. An asynchronous projection, which commits once per batch, is recorded inside each +consumption: the spans buffered during the work, and the span that wraps it as it stands at that moment. Whether a +flow is complete is never stored: it is decided from the committed spans each time it is asked. An asynchronous projection, which commits once per batch, is recorded inside each batch's transaction, and is marked processed in the batch that finds it caught up. A path run with no transaction at all, an event bus call from a controller or a command bus with `withTransactionOnCommandBus(false)`, has none to be inside and writes when its work ends. Nothing is written after the commit or after a rollback, so a database that goes down @@ -3159,20 +3162,16 @@ notification is delivered in a trace of its own, marked `ecotone=untracked` in t subscriber sends, synchronously or through an asynchronous channel, is neither stored nor notified and can never reopen the flow it was told about; OpenTelemetry still exports it, as its own trace. A subscriber runs synchronously, as that delivery happens; `#[Asynchronous]` on it is refused at bootstrap, so work meant for the background is a command it sends to an -`#[Asynchronous]` handler. A subscriber whose first parameter is not `FlowCompleted` or `FlowStepCompleted`, and a -`FlowTrackingConfiguration` registered twice, are refused at bootstrap too. Each flow's row carries a version: a unit -of work reads it, and writes the row only if no other part of the same flow committed in between. When two parts of -one flow finish at the same moment, the second one's write fails with `ConcurrencyException`, its transaction rolls -back, and the retry reads the flow again from a fresh snapshot, sees the first part and claims completion, so the flow -notifies exactly once on PostgreSQL, MySQL and MariaDB alike. The retried part runs its handler again: an asynchronous -endpoint retries by default; a command does with `withCommandBusRetry(true, 3, [ConcurrencyException::class])`. +`#[Asynchronous]` handler. A subscriber whose first parameter is not `FlowStepCompleted`, and a +`FlowTrackingConfiguration` registered twice, are refused at bootstrap too. Each step is its own row, keyed by its +trace and span id, so parts of one flow committing at the same moment never write the same row and never wait for +each other, on PostgreSQL, MySQL and MariaDB alike. `withOnlyRequestedFlows(true)` tracks only flows sent with a `tracingId` or a `traceparent`, carrying that choice across channels in the W3C `tracestate` as `ecotone=carried` next to any other vendor's entry, so ordinary traffic -costs nothing. `withRetention()` sets how long `ecotone:tracing:remove-expired-traces` keeps completed flows, seven -days by default; a flow still pending is kept twice as long, so a message delayed past the retention still completes -it. Nothing runs the command for you: schedule it, from cron or the framework's scheduler, or the tables keep every -flow. It removes expired flows in batches of 1000, each batch's spans and flow rows in one short transaction, until +costs nothing. `withRetention()` sets how long `ecotone:tracing:remove-expired-traces` keeps a flow, counted from its +first step, seven days by default, completed or not. Nothing runs the command for you: schedule it, from cron or the framework's scheduler, or the tables keep every +flow. It removes expired flows in batches of 1000, each batch's spans in one short transaction, until none is left, so a large backlog never holds one long lock; `withRemovalBatchSize()` changes the size. A multi-tenant application tracks each flow in the database of its tenant, inside the tenant's transaction, so a flow is never complete before the tenant's data committed. `TracingGateway` is then asked with the tenant, `$tracingGateway->isCompletedWithin($tracingId, $timeout, metadata: ['tenant' => 'tenant_a'])`, and a call without one is refused @@ -3194,15 +3193,17 @@ if (! $tracingGateway->isCompletedWithin($tracingId, Duration::seconds(5), 'orde final class OrdersWebsocket { - #[OnFlowCompleted] - public function push(FlowCompleted $event): void + #[OnFlowStepCompleted] + public function push(FlowStepCompleted $event, TracingGateway $tracingGateway): void { + if ($tracingGateway->isCompletedFor($event->step->traceId, 'orders_list')) { + } } } ``` -**How to adapt:** nothing is required. To use it, register `FlowTrackingConfiguration`, run the setup command for the -two tables, and give each handler you want to ask about an explicit `endpointId`. +**How to adapt:** nothing is required. To use it, register `FlowTrackingConfiguration`, run the setup command for its +table, and give each handler you want to ask about an explicit `endpointId`. ## 15. Testing and developer-experience changes