From dcab240a39d85e8e1f7e4d361c500613b0d4f48a Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 22 Jul 2026 14:52:31 +1000 Subject: [PATCH 1/5] EF persistence for failed messages and streamline known endpoints This change introduces the core EF Core schema and entities for storing failed messages, including their associated groups and retry information. It also sets up full-text search capabilities for failed message headers, bodies, and message types in both PostgreSQL and SQL Server. The separate 'insert-only' table and background reconciler for KnownEndpoints have been removed, simplifying how known endpoints are managed. A new `MaxBodySizeToStore` setting is added to control message body storage. --- .../FullTextSearchSql.cs | 29 ++ .../KnownEndpointsReconciler.cs | 46 --- .../20260720230745_Initial.Designer.cs | 93 ------- .../Migrations/20260720230745_Initial.cs | 62 ----- .../20260722061312_Initial.Designer.cs | 262 ++++++++++++++++++ .../Migrations/20260722061312_Initial.cs | 151 ++++++++++ ...260722061316_AddFullTextSearch.Designer.cs | 262 ++++++++++++++++++ .../20260722061316_AddFullTextSearch.cs | 16 ++ ...SqlServiceControlDbContextModelSnapshot.cs | 221 +++++++++++++-- .../PostgreSqlPersistence.cs | 3 - .../PostgreSqlServiceControlDbContext.cs | 11 + .../FullTextSearchSql.cs | 52 ++++ .../KnownEndpointsReconciler.cs | 64 ----- .../20260720230811_Initial.Designer.cs | 83 ------ .../Migrations/20260720230811_Initial.cs | 61 ---- .../20260722061320_Initial.Designer.cs | 212 ++++++++++++++ .../Migrations/20260722061320_Initial.cs | 151 ++++++++++ ...260722061323_AddFullTextSearch.Designer.cs | 212 ++++++++++++++ .../20260722061323_AddFullTextSearch.cs | 24 ++ ...verServiceControlDbContextModelSnapshot.cs | 165 +++++++++-- .../SqlServerPersistence.cs | 3 - .../SqlServerServiceControlDbContext.cs | 10 + .../EFPersistenceConfigurationBase.cs | 21 ++ .../Abstractions/EFPersisterSettings.cs | 1 + .../DbContexts/ServiceControlDbContext.cs | 8 +- .../Entities/FailedMessageEntity.cs | 60 ++++ .../Entities/FailedMessageGroupEntity.cs | 14 + .../Entities/FailedMessageRetryEntity.cs | 8 + .../Entities/KnownEndpointInsertOnlyEntity.cs | 13 - .../FailedMessageConfiguration.cs | 50 ++++ .../FailedMessageGroupConfiguration.cs | 25 ++ .../FailedMessageRetryConfiguration.cs | 15 + .../KnownEndpointConfiguration.cs | 1 - .../KnownEndpointInsertOnlyConfiguration.cs | 21 -- .../EFMonitoringIngestionUnitOfWork.cs | 21 +- .../KnownEndpointsReconcilerTests.cs | 156 ----------- .../KnownEndpointsReconcilerTests.cs | 158 ----------- 37 files changed, 1938 insertions(+), 827 deletions(-) create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs delete mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Infrastructure/KnownEndpointsReconciler.cs delete mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260720230745_Initial.Designer.cs delete mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260720230745_Initial.cs create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061312_Initial.Designer.cs create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061312_Initial.cs create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061316_AddFullTextSearch.Designer.cs create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061316_AddFullTextSearch.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/FullTextSearchSql.cs delete mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Infrastructure/KnownEndpointsReconciler.cs delete mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260720230811_Initial.Designer.cs delete mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260720230811_Initial.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061320_Initial.Designer.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061320_Initial.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061323_AddFullTextSearch.Designer.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061323_AddFullTextSearch.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Entities/FailedMessageEntity.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Entities/FailedMessageGroupEntity.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Entities/FailedMessageRetryEntity.cs delete mode 100644 src/ServiceControl.Persistence.EFCore/Entities/KnownEndpointInsertOnlyEntity.cs create mode 100644 src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs create mode 100644 src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageGroupConfiguration.cs create mode 100644 src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageRetryConfiguration.cs delete mode 100644 src/ServiceControl.Persistence.EFCore/EntityConfigurations/KnownEndpointInsertOnlyConfiguration.cs delete mode 100644 src/ServiceControl.Persistence.Tests.PostgreSql/KnownEndpointsReconcilerTests.cs delete mode 100644 src/ServiceControl.Persistence.Tests.SqlServer/KnownEndpointsReconcilerTests.cs diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs new file mode 100644 index 0000000000..2266b6e087 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs @@ -0,0 +1,29 @@ +namespace ServiceControl.Persistence.EFCore.PostgreSql; + +/// +/// Full text search DDL for the failed messages table. EF Core cannot model a GIN index over an +/// expression, so it is applied by the AddFullTextSearch migration. The statements live here, and +/// not in the migration itself, so that regenerating the migrations with the dotnet-ef CLI only +/// costs a one line migration body. +/// +static class FullTextSearchSql +{ + const string IndexName = "ix_failed_messages_full_text"; + + // 'simple' rather than 'english': message and header content is technical, stemming and + // stopword removal do more harm than good. + // The default parser reads a dotted name as a single host token, so + // "ServiceControl.MessageFailures.MyMessage" would not match a search for "MyMessage". The + // message type is therefore also indexed with its separators replaced by spaces, mirroring + // the SearchableMessageType that MessageTypeEnricher already produces for RavenDB. + public const string Up = $""" + CREATE INDEX {IndexName} + ON failed_messages + USING GIN (to_tsvector('simple', + coalesce(headers_json, '') || ' ' || + coalesce(body_text, '') || ' ' || + replace(replace(coalesce(message_type, ''), '.', ' '), '+', ' '))) + """; + + public const string Down = $"DROP INDEX IF EXISTS {IndexName}"; +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Infrastructure/KnownEndpointsReconciler.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Infrastructure/KnownEndpointsReconciler.cs deleted file mode 100644 index 666312ee06..0000000000 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Infrastructure/KnownEndpointsReconciler.cs +++ /dev/null @@ -1,46 +0,0 @@ -namespace ServiceControl.Persistence.EFCore.PostgreSql.Infrastructure; - -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using ServiceControl.Persistence.EFCore.DbContexts; -using ServiceControl.Persistence.EFCore.Infrastructure; - -class KnownEndpointsReconciler( - ILogger logger, - TimeProvider timeProvider, - IServiceScopeFactory serviceScopeFactory) - : InsertOnlyTableReconciler( - logger, timeProvider, serviceScopeFactory, nameof(KnownEndpointsReconciler)) -{ - protected override Task ReconcileBatch(ServiceControlDbContext dbContext, CancellationToken stoppingToken) => - ReconcileBatch(dbContext, BatchSize, stoppingToken); - - // Static so tests can execute a batch deterministically without the background service's timer loop. - // Must be called within an active transaction because of the pg_try_advisory_xact_lock. - internal static async Task ReconcileBatch(ServiceControlDbContext dbContext, int batchSize, CancellationToken cancellationToken) - { - var sql = """ - WITH lock_check AS ( - SELECT pg_try_advisory_xact_lock(hashtext('known_endpoints_sync')) AS acquired - ), - batch AS ( - SELECT ctid FROM "known_endpoints_insert_only" - WHERE (SELECT acquired FROM lock_check) - LIMIT @batchSize - ), - ins AS ( - INSERT INTO "known_endpoints" ("id", "name", "host_id", "host", "monitored") - SELECT DISTINCT ON ("known_endpoint_id") "known_endpoint_id", "name", "host_id", "host", FALSE - FROM "known_endpoints_insert_only" - WHERE ctid IN (SELECT ctid FROM batch) - ON CONFLICT ("id") DO NOTHING - ) - DELETE FROM "known_endpoints_insert_only" - WHERE ctid IN (SELECT ctid FROM batch); - """; - - var rowsAffected = await dbContext.Database.ExecuteSqlRawAsync(sql, [new Npgsql.NpgsqlParameter("@batchSize", batchSize)], cancellationToken); - return rowsAffected; - } -} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260720230745_Initial.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260720230745_Initial.Designer.cs deleted file mode 100644 index 1a1383b3e2..0000000000 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260720230745_Initial.Designer.cs +++ /dev/null @@ -1,93 +0,0 @@ -// -using System; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; -using ServiceControl.Persistence.EFCore.PostgreSql; - -#nullable disable - -namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations -{ - [DbContext(typeof(PostgreSqlServiceControlDbContext))] - [Migration("20260720230745_Initial")] - partial class Initial - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "10.0.9") - .HasAnnotation("Relational:MaxIdentifierLength", 63); - - NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => - { - b.Property("Id") - .HasColumnType("uuid") - .HasColumnName("id"); - - b.Property("Host") - .IsRequired() - .HasColumnType("text") - .HasColumnName("host"); - - b.Property("HostId") - .HasColumnType("uuid") - .HasColumnName("host_id"); - - b.Property("Monitored") - .HasColumnType("boolean") - .HasColumnName("monitored"); - - b.Property("Name") - .IsRequired() - .HasColumnType("text") - .HasColumnName("name"); - - b.HasKey("Id"); - - b.ToTable("known_endpoints", (string)null); - }); - - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointInsertOnlyEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("bigint") - .HasColumnName("id"); - - NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); - - b.Property("Host") - .IsRequired() - .HasColumnType("text") - .HasColumnName("host"); - - b.Property("HostId") - .HasColumnType("uuid") - .HasColumnName("host_id"); - - b.Property("KnownEndpointId") - .HasColumnType("uuid") - .HasColumnName("known_endpoint_id"); - - b.Property("Name") - .IsRequired() - .HasColumnType("text") - .HasColumnName("name"); - - b.HasKey("Id"); - - b.HasIndex("KnownEndpointId"); - - b.ToTable("known_endpoints_insert_only", (string)null); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260720230745_Initial.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260720230745_Initial.cs deleted file mode 100644 index d2caaa189a..0000000000 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260720230745_Initial.cs +++ /dev/null @@ -1,62 +0,0 @@ -using System; -using Microsoft.EntityFrameworkCore.Migrations; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; - -#nullable disable - -namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations -{ - /// - public partial class Initial : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.CreateTable( - name: "known_endpoints", - columns: table => new - { - id = table.Column(type: "uuid", nullable: false), - name = table.Column(type: "text", nullable: false), - host_id = table.Column(type: "uuid", nullable: false), - host = table.Column(type: "text", nullable: false), - monitored = table.Column(type: "boolean", nullable: false) - }, - constraints: table => - { - table.PrimaryKey("PK_known_endpoints", x => x.id); - }); - - migrationBuilder.CreateTable( - name: "known_endpoints_insert_only", - columns: table => new - { - id = table.Column(type: "bigint", nullable: false) - .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), - known_endpoint_id = table.Column(type: "uuid", nullable: false), - name = table.Column(type: "text", nullable: false), - host_id = table.Column(type: "uuid", nullable: false), - host = table.Column(type: "text", nullable: false) - }, - constraints: table => - { - table.PrimaryKey("PK_known_endpoints_insert_only", x => x.id); - }); - - migrationBuilder.CreateIndex( - name: "IX_known_endpoints_insert_only_known_endpoint_id", - table: "known_endpoints_insert_only", - column: "known_endpoint_id"); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropTable( - name: "known_endpoints"); - - migrationBuilder.DropTable( - name: "known_endpoints_insert_only"); - } - } -} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061312_Initial.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061312_Initial.Designer.cs new file mode 100644 index 0000000000..465e6178c6 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061312_Initial.Designer.cs @@ -0,0 +1,262 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Persistence.EFCore.PostgreSql; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + [DbContext(typeof(PostgreSqlServiceControlDbContext))] + [Migration("20260722061312_Initial")] + partial class Initial + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("body_content_type"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("BodyText") + .HasColumnType("text") + .HasColumnName("body_text"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("conversation_id"); + + b.Property("ExceptionMessage") + .HasColumnType("text") + .HasColumnName("exception_message"); + + b.Property("ExceptionType") + .HasColumnType("text") + .HasColumnName("exception_type"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_time_of_failure"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("LastAttemptedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_attempted_at"); + + b.Property("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property("LastTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_time_of_failure"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasColumnType("text") + .HasColumnName("message_type"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("integer") + .HasColumnName("number_of_processing_attempts"); + + b.Property("QueueAddress") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("queue_address"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_host"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("receiving_endpoint_host_id"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_host"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("sending_endpoint_host_id"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_name"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("StatusChangedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("status_changed_at"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_messages"); + + b.HasIndex("ConversationId") + .HasDatabaseName("ix_failed_messages_conversation_id"); + + b.HasIndex("QueueAddress") + .HasDatabaseName("ix_failed_messages_queue_address"); + + b.HasIndex("ReceivingEndpointName") + .HasDatabaseName("ix_failed_messages_receiving_endpoint_name"); + + b.HasIndex("StatusChangedAt") + .HasDatabaseName("ix_failed_messages_status_changed_at") + .HasFilter("status IN (2, 4)"); + + b.HasIndex("TimeSent") + .HasDatabaseName("ix_failed_messages_time_sent"); + + b.HasIndex("Status", "LastModified") + .HasDatabaseName("ix_failed_messages_status_last_modified"); + + b.ToTable("failed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uuid") + .HasColumnName("failed_message_unique_id"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text") + .HasColumnName("title"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("type"); + + b.HasKey("FailedMessageUniqueId", "GroupId") + .HasName("pk_failed_message_groups"); + + b.HasIndex("GroupId") + .HasDatabaseName("ix_failed_message_groups_group_id"); + + b.ToTable("failed_message_groups", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("RetryId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("retry_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_retries"); + + b.ToTable("failed_message_retries", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Host") + .IsRequired() + .HasColumnType("text") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("Monitored") + .HasColumnType("boolean") + .HasColumnName("monitored"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_known_endpoints"); + + b.ToTable("known_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_failed_message_groups_failed_messages_failed_message_unique"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061312_Initial.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061312_Initial.cs new file mode 100644 index 0000000000..d4b2543aa7 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061312_Initial.cs @@ -0,0 +1,151 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + /// + public partial class Initial : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "failed_message_retries", + columns: table => new + { + unique_message_id = table.Column(type: "uuid", nullable: false), + retry_id = table.Column(type: "character varying(450)", maxLength: 450, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_failed_message_retries", x => x.unique_message_id); + }); + + migrationBuilder.CreateTable( + name: "failed_messages", + columns: table => new + { + unique_message_id = table.Column(type: "uuid", nullable: false), + status = table.Column(type: "integer", nullable: false), + status_changed_at = table.Column(type: "timestamp with time zone", nullable: false), + last_modified = table.Column(type: "timestamp with time zone", nullable: false), + number_of_processing_attempts = table.Column(type: "integer", nullable: false), + first_time_of_failure = table.Column(type: "timestamp with time zone", nullable: false), + last_time_of_failure = table.Column(type: "timestamp with time zone", nullable: false), + last_attempted_at = table.Column(type: "timestamp with time zone", nullable: false), + message_id = table.Column(type: "character varying(450)", maxLength: 450, nullable: true), + message_type = table.Column(type: "text", nullable: true), + time_sent = table.Column(type: "timestamp with time zone", nullable: true), + conversation_id = table.Column(type: "character varying(450)", maxLength: 450, nullable: true), + queue_address = table.Column(type: "character varying(450)", maxLength: 450, nullable: true), + sending_endpoint_name = table.Column(type: "character varying(450)", maxLength: 450, nullable: true), + sending_endpoint_host_id = table.Column(type: "uuid", nullable: true), + sending_endpoint_host = table.Column(type: "character varying(450)", maxLength: 450, nullable: true), + receiving_endpoint_name = table.Column(type: "character varying(450)", maxLength: 450, nullable: true), + receiving_endpoint_host_id = table.Column(type: "uuid", nullable: true), + receiving_endpoint_host = table.Column(type: "character varying(450)", maxLength: 450, nullable: true), + exception_type = table.Column(type: "text", nullable: true), + exception_message = table.Column(type: "text", nullable: true), + is_system_message = table.Column(type: "boolean", nullable: false), + headers_json = table.Column(type: "text", nullable: false), + body_text = table.Column(type: "text", nullable: true), + body_stored_externally = table.Column(type: "boolean", nullable: false), + body_size = table.Column(type: "integer", nullable: false), + body_content_type = table.Column(type: "character varying(450)", maxLength: 450, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_failed_messages", x => x.unique_message_id); + }); + + migrationBuilder.CreateTable( + name: "known_endpoints", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + name = table.Column(type: "text", nullable: false), + host_id = table.Column(type: "uuid", nullable: false), + host = table.Column(type: "text", nullable: false), + monitored = table.Column(type: "boolean", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_known_endpoints", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "failed_message_groups", + columns: table => new + { + failed_message_unique_id = table.Column(type: "uuid", nullable: false), + group_id = table.Column(type: "character varying(64)", maxLength: 64, nullable: false), + title = table.Column(type: "text", nullable: false), + type = table.Column(type: "character varying(255)", maxLength: 255, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_failed_message_groups", x => new { x.failed_message_unique_id, x.group_id }); + table.ForeignKey( + name: "fk_failed_message_groups_failed_messages_failed_message_unique", + column: x => x.failed_message_unique_id, + principalTable: "failed_messages", + principalColumn: "unique_message_id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_failed_message_groups_group_id", + table: "failed_message_groups", + column: "group_id"); + + migrationBuilder.CreateIndex( + name: "ix_failed_messages_conversation_id", + table: "failed_messages", + column: "conversation_id"); + + migrationBuilder.CreateIndex( + name: "ix_failed_messages_queue_address", + table: "failed_messages", + column: "queue_address"); + + migrationBuilder.CreateIndex( + name: "ix_failed_messages_receiving_endpoint_name", + table: "failed_messages", + column: "receiving_endpoint_name"); + + migrationBuilder.CreateIndex( + name: "ix_failed_messages_status_changed_at", + table: "failed_messages", + column: "status_changed_at", + filter: "status IN (2, 4)"); + + migrationBuilder.CreateIndex( + name: "ix_failed_messages_status_last_modified", + table: "failed_messages", + columns: new[] { "status", "last_modified" }); + + migrationBuilder.CreateIndex( + name: "ix_failed_messages_time_sent", + table: "failed_messages", + column: "time_sent"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "failed_message_groups"); + + migrationBuilder.DropTable( + name: "failed_message_retries"); + + migrationBuilder.DropTable( + name: "known_endpoints"); + + migrationBuilder.DropTable( + name: "failed_messages"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061316_AddFullTextSearch.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061316_AddFullTextSearch.Designer.cs new file mode 100644 index 0000000000..fe5a5b6bf3 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061316_AddFullTextSearch.Designer.cs @@ -0,0 +1,262 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Persistence.EFCore.PostgreSql; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + [DbContext(typeof(PostgreSqlServiceControlDbContext))] + [Migration("20260722061316_AddFullTextSearch")] + partial class AddFullTextSearch + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("body_content_type"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("BodyText") + .HasColumnType("text") + .HasColumnName("body_text"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("conversation_id"); + + b.Property("ExceptionMessage") + .HasColumnType("text") + .HasColumnName("exception_message"); + + b.Property("ExceptionType") + .HasColumnType("text") + .HasColumnName("exception_type"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_time_of_failure"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("LastAttemptedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_attempted_at"); + + b.Property("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property("LastTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_time_of_failure"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasColumnType("text") + .HasColumnName("message_type"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("integer") + .HasColumnName("number_of_processing_attempts"); + + b.Property("QueueAddress") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("queue_address"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_host"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("receiving_endpoint_host_id"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_host"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("sending_endpoint_host_id"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_name"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("StatusChangedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("status_changed_at"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_messages"); + + b.HasIndex("ConversationId") + .HasDatabaseName("ix_failed_messages_conversation_id"); + + b.HasIndex("QueueAddress") + .HasDatabaseName("ix_failed_messages_queue_address"); + + b.HasIndex("ReceivingEndpointName") + .HasDatabaseName("ix_failed_messages_receiving_endpoint_name"); + + b.HasIndex("StatusChangedAt") + .HasDatabaseName("ix_failed_messages_status_changed_at") + .HasFilter("status IN (2, 4)"); + + b.HasIndex("TimeSent") + .HasDatabaseName("ix_failed_messages_time_sent"); + + b.HasIndex("Status", "LastModified") + .HasDatabaseName("ix_failed_messages_status_last_modified"); + + b.ToTable("failed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uuid") + .HasColumnName("failed_message_unique_id"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text") + .HasColumnName("title"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("type"); + + b.HasKey("FailedMessageUniqueId", "GroupId") + .HasName("pk_failed_message_groups"); + + b.HasIndex("GroupId") + .HasDatabaseName("ix_failed_message_groups_group_id"); + + b.ToTable("failed_message_groups", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("RetryId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("retry_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_retries"); + + b.ToTable("failed_message_retries", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Host") + .IsRequired() + .HasColumnType("text") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("Monitored") + .HasColumnType("boolean") + .HasColumnName("monitored"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_known_endpoints"); + + b.ToTable("known_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_failed_message_groups_failed_messages_failed_message_unique"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061316_AddFullTextSearch.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061316_AddFullTextSearch.cs new file mode 100644 index 0000000000..dfcd89e387 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260722061316_AddFullTextSearch.cs @@ -0,0 +1,16 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + /// + public partial class AddFullTextSearch : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) => migrationBuilder.Sql(FullTextSearchSql.Up); + + /// + protected override void Down(MigrationBuilder migrationBuilder) => migrationBuilder.Sql(FullTextSearchSql.Down); + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs index 83b17187a4..02d2a95c8d 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs @@ -22,43 +22,203 @@ protected override void BuildModel(ModelBuilder modelBuilder) NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => { - b.Property("Id") + b.Property("UniqueMessageId") .HasColumnType("uuid") - .HasColumnName("id"); + .HasColumnName("unique_message_id"); - b.Property("Host") + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("body_content_type"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("BodyText") + .HasColumnType("text") + .HasColumnName("body_text"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("conversation_id"); + + b.Property("ExceptionMessage") + .HasColumnType("text") + .HasColumnName("exception_message"); + + b.Property("ExceptionType") + .HasColumnType("text") + .HasColumnName("exception_type"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_time_of_failure"); + + b.Property("HeadersJson") .IsRequired() .HasColumnType("text") - .HasColumnName("host"); + .HasColumnName("headers_json"); - b.Property("HostId") + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("LastAttemptedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_attempted_at"); + + b.Property("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property("LastTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_time_of_failure"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasColumnType("text") + .HasColumnName("message_type"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("integer") + .HasColumnName("number_of_processing_attempts"); + + b.Property("QueueAddress") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("queue_address"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_host"); + + b.Property("ReceivingEndpointHostId") .HasColumnType("uuid") - .HasColumnName("host_id"); + .HasColumnName("receiving_endpoint_host_id"); - b.Property("Monitored") - .HasColumnType("boolean") - .HasColumnName("monitored"); + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_name"); - b.Property("Name") + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_host"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("sending_endpoint_host_id"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_name"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("StatusChangedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("status_changed_at"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_messages"); + + b.HasIndex("ConversationId") + .HasDatabaseName("ix_failed_messages_conversation_id"); + + b.HasIndex("QueueAddress") + .HasDatabaseName("ix_failed_messages_queue_address"); + + b.HasIndex("ReceivingEndpointName") + .HasDatabaseName("ix_failed_messages_receiving_endpoint_name"); + + b.HasIndex("StatusChangedAt") + .HasDatabaseName("ix_failed_messages_status_changed_at") + .HasFilter("status IN (2, 4)"); + + b.HasIndex("TimeSent") + .HasDatabaseName("ix_failed_messages_time_sent"); + + b.HasIndex("Status", "LastModified") + .HasDatabaseName("ix_failed_messages_status_last_modified"); + + b.ToTable("failed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uuid") + .HasColumnName("failed_message_unique_id"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property("Title") .IsRequired() .HasColumnType("text") - .HasColumnName("name"); + .HasColumnName("title"); - b.HasKey("Id"); + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("type"); - b.ToTable("known_endpoints", (string)null); + b.HasKey("FailedMessageUniqueId", "GroupId") + .HasName("pk_failed_message_groups"); + + b.HasIndex("GroupId") + .HasDatabaseName("ix_failed_message_groups_group_id"); + + b.ToTable("failed_message_groups", (string)null); }); - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointInsertOnlyEntity", b => + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("bigint") - .HasColumnName("id"); + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("RetryId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("retry_id"); - NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_retries"); + + b.ToTable("failed_message_retries", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); b.Property("Host") .IsRequired() @@ -69,20 +229,29 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("uuid") .HasColumnName("host_id"); - b.Property("KnownEndpointId") - .HasColumnType("uuid") - .HasColumnName("known_endpoint_id"); + b.Property("Monitored") + .HasColumnType("boolean") + .HasColumnName("monitored"); b.Property("Name") .IsRequired() .HasColumnType("text") .HasColumnName("name"); - b.HasKey("Id"); + b.HasKey("Id") + .HasName("pk_known_endpoints"); - b.HasIndex("KnownEndpointId"); + b.ToTable("known_endpoints", (string)null); + }); - b.ToTable("known_endpoints_insert_only", (string)null); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_failed_message_groups_failed_messages_failed_message_unique"); }); #pragma warning restore 612, 618 } diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs index ea32ae9788..bba7a9506a 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs @@ -4,7 +4,6 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql; using Microsoft.Extensions.DependencyInjection; using ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.Persistence.EFCore.DbContexts; -using ServiceControl.Persistence.EFCore.PostgreSql.Infrastructure; class PostgreSqlPersistence(PostgreSqlPersisterSettings settings) : BasePersistence, IPersistence { @@ -13,8 +12,6 @@ public void AddPersistence(IServiceCollection services) RegisterSettings(services); ConfigureDbContext(services); RegisterDataStores(services); - - services.AddHostedService(); } public void AddInstaller(IServiceCollection services) diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlServiceControlDbContext.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlServiceControlDbContext.cs index c62a5c78e1..6164f58d2f 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlServiceControlDbContext.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlServiceControlDbContext.cs @@ -1,7 +1,9 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql; using Microsoft.EntityFrameworkCore; +using ServiceControl.MessageFailures; using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; public class PostgreSqlServiceControlDbContext(DbContextOptions options) : ServiceControlDbContext(options) { @@ -12,4 +14,13 @@ protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) // Use snake_case naming convention for PostgreSQL optionsBuilder.UseSnakeCaseNamingConvention(); } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + + modelBuilder.Entity() + .HasIndex(e => e.StatusChangedAt) + .HasFilter($"status IN ({(int)FailedMessageStatus.Resolved}, {(int)FailedMessageStatus.Archived})"); + } } diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/FullTextSearchSql.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/FullTextSearchSql.cs new file mode 100644 index 0000000000..8f699723c0 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/FullTextSearchSql.cs @@ -0,0 +1,52 @@ +namespace ServiceControl.Persistence.EFCore.SqlServer; + +/// +/// Full text search DDL for the failed messages table. EF Core has no full text support for SQL +/// Server, so it is applied by the AddFullTextSearch migration. The statements live here, and not +/// in the migration itself, so that regenerating the migrations with the dotnet-ef CLI only costs +/// a one line migration body. +/// +static class FullTextSearchSql +{ + const string CatalogName = "ServiceControlFullTextCatalog"; + + // Both statements are guarded: an instance without Full-Text Search installed still migrates, + // it just has no full text index. They also cannot run inside a transaction, so the migration + // passes suppressTransaction. + public const string CreateCatalog = $""" + IF SERVERPROPERTY('IsFullTextInstalled') = 1 + AND NOT EXISTS (SELECT 1 FROM sys.fulltext_catalogs WHERE name = '{CatalogName}') + BEGIN + EXEC('CREATE FULLTEXT CATALOG {CatalogName}'); + END + """; + + // LANGUAGE 0 (neutral) and STOPLIST = OFF keep the word breaker from applying language rules + // and from dropping stopwords, both of which lose matches on technical content. + // The message type needs no dedicated column here: the word breaker splits dotted names, and + // the headers already carry the type. + public const string CreateIndex = $""" + IF SERVERPROPERTY('IsFullTextInstalled') = 1 + AND NOT EXISTS (SELECT 1 FROM sys.fulltext_indexes WHERE object_id = OBJECT_ID('FailedMessages')) + BEGIN + EXEC('CREATE FULLTEXT INDEX ON FailedMessages(HeadersJson LANGUAGE 0, BodyText LANGUAGE 0) + KEY INDEX PK_FailedMessages + ON {CatalogName} + WITH (CHANGE_TRACKING AUTO, STOPLIST = OFF)'); + END + """; + + public const string DropIndex = """ + IF EXISTS (SELECT 1 FROM sys.fulltext_indexes WHERE object_id = OBJECT_ID('FailedMessages')) + BEGIN + DROP FULLTEXT INDEX ON FailedMessages; + END + """; + + public const string DropCatalog = $""" + IF EXISTS (SELECT 1 FROM sys.fulltext_catalogs WHERE name = '{CatalogName}') + BEGIN + DROP FULLTEXT CATALOG {CatalogName}; + END + """; +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Infrastructure/KnownEndpointsReconciler.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Infrastructure/KnownEndpointsReconciler.cs deleted file mode 100644 index 727ecb8e83..0000000000 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Infrastructure/KnownEndpointsReconciler.cs +++ /dev/null @@ -1,64 +0,0 @@ -namespace ServiceControl.Persistence.EFCore.SqlServer.Infrastructure; - -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using ServiceControl.Persistence.EFCore.DbContexts; -using ServiceControl.Persistence.EFCore.Infrastructure; - -class KnownEndpointsReconciler( - ILogger logger, - TimeProvider timeProvider, - IServiceScopeFactory serviceScopeFactory) - : InsertOnlyTableReconciler( - logger, timeProvider, serviceScopeFactory, nameof(KnownEndpointsReconciler)) -{ - protected override Task ReconcileBatch(ServiceControlDbContext dbContext, CancellationToken stoppingToken) => - ReconcileBatch(dbContext, BatchSize, stoppingToken); - - // Static so tests can execute a batch deterministically without the background service's timer loop. - // Must be called within an active transaction because of the sp_getapplock Transaction lock owner. - internal static async Task ReconcileBatch(ServiceControlDbContext dbContext, int batchSize, CancellationToken cancellationToken) - { - var sql = """ - DECLARE @lockResult INT; - EXEC @lockResult = sp_getapplock @Resource = 'known_endpoints_sync', @LockMode = 'Exclusive', @LockOwner = 'Transaction', @LockTimeout = 0; - IF @lockResult < 0 - BEGIN - SELECT 0; - RETURN; - END; - - DECLARE @deleted TABLE ( - KnownEndpointId UNIQUEIDENTIFIER, - Name NVARCHAR(MAX), - HostId UNIQUEIDENTIFIER, - Host NVARCHAR(MAX) - ); - - DELETE TOP (@batchSize) FROM KnownEndpointsInsertOnly - OUTPUT DELETED.KnownEndpointId, DELETED.Name, DELETED.HostId, DELETED.Host - INTO @deleted; - - WITH ranked AS ( - SELECT KnownEndpointId, Name, HostId, Host, - ROW_NUMBER() OVER (PARTITION BY KnownEndpointId ORDER BY (SELECT NULL)) AS rn - FROM @deleted - ), - aggregated AS ( - SELECT KnownEndpointId, Name, HostId, Host - FROM ranked - WHERE rn = 1 - ) - MERGE INTO KnownEndpoints WITH (HOLDLOCK) AS target - USING aggregated AS source - ON target.Id = source.KnownEndpointId - WHEN NOT MATCHED THEN - INSERT (Id, Name, HostId, Host, Monitored) - VALUES (source.KnownEndpointId, source.Name, source.HostId, source.Host, 0); - """; - - var rowsAffected = await dbContext.Database.ExecuteSqlRawAsync(sql, [new Microsoft.Data.SqlClient.SqlParameter("@batchSize", batchSize)], cancellationToken); - return rowsAffected; - } -} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260720230811_Initial.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260720230811_Initial.Designer.cs deleted file mode 100644 index 5901a2770e..0000000000 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260720230811_Initial.Designer.cs +++ /dev/null @@ -1,83 +0,0 @@ -// -using System; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Metadata; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using ServiceControl.Persistence.EFCore.SqlServer; - -#nullable disable - -namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations -{ - [DbContext(typeof(SqlServerServiceControlDbContext))] - [Migration("20260720230811_Initial")] - partial class Initial - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "10.0.9") - .HasAnnotation("Relational:MaxIdentifierLength", 128); - - SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); - - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => - { - b.Property("Id") - .HasColumnType("uniqueidentifier"); - - b.Property("Host") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("HostId") - .HasColumnType("uniqueidentifier"); - - b.Property("Monitored") - .HasColumnType("bit"); - - b.Property("Name") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.HasKey("Id"); - - b.ToTable("KnownEndpoints", (string)null); - }); - - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointInsertOnlyEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("bigint"); - - SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); - - b.Property("Host") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("HostId") - .HasColumnType("uniqueidentifier"); - - b.Property("KnownEndpointId") - .HasColumnType("uniqueidentifier"); - - b.Property("Name") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.HasKey("Id"); - - b.HasIndex("KnownEndpointId"); - - b.ToTable("KnownEndpointsInsertOnly", (string)null); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260720230811_Initial.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260720230811_Initial.cs deleted file mode 100644 index d1af2ee2e1..0000000000 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260720230811_Initial.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations -{ - /// - public partial class Initial : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.CreateTable( - name: "KnownEndpoints", - columns: table => new - { - Id = table.Column(type: "uniqueidentifier", nullable: false), - Name = table.Column(type: "nvarchar(max)", nullable: false), - HostId = table.Column(type: "uniqueidentifier", nullable: false), - Host = table.Column(type: "nvarchar(max)", nullable: false), - Monitored = table.Column(type: "bit", nullable: false) - }, - constraints: table => - { - table.PrimaryKey("PK_KnownEndpoints", x => x.Id); - }); - - migrationBuilder.CreateTable( - name: "KnownEndpointsInsertOnly", - columns: table => new - { - Id = table.Column(type: "bigint", nullable: false) - .Annotation("SqlServer:Identity", "1, 1"), - KnownEndpointId = table.Column(type: "uniqueidentifier", nullable: false), - Name = table.Column(type: "nvarchar(max)", nullable: false), - HostId = table.Column(type: "uniqueidentifier", nullable: false), - Host = table.Column(type: "nvarchar(max)", nullable: false) - }, - constraints: table => - { - table.PrimaryKey("PK_KnownEndpointsInsertOnly", x => x.Id); - }); - - migrationBuilder.CreateIndex( - name: "IX_KnownEndpointsInsertOnly_KnownEndpointId", - table: "KnownEndpointsInsertOnly", - column: "KnownEndpointId"); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropTable( - name: "KnownEndpoints"); - - migrationBuilder.DropTable( - name: "KnownEndpointsInsertOnly"); - } - } -} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061320_Initial.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061320_Initial.Designer.cs new file mode 100644 index 0000000000..27b9d2b309 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061320_Initial.Designer.cs @@ -0,0 +1,212 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Persistence.EFCore.SqlServer; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerServiceControlDbContext))] + [Migration("20260722061320_Initial")] + partial class Initial + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("BodyText") + .HasColumnType("nvarchar(max)"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ExceptionMessage") + .HasColumnType("nvarchar(max)"); + + b.Property("ExceptionType") + .HasColumnType("nvarchar(max)"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("LastAttemptedAt") + .HasColumnType("datetime2"); + + b.Property("LastModified") + .HasColumnType("datetime2"); + + b.Property("LastTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("MessageType") + .HasColumnType("nvarchar(max)"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("int"); + + b.Property("QueueAddress") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("StatusChangedAt") + .HasColumnType("datetime2"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("ConversationId"); + + b.HasIndex("QueueAddress"); + + b.HasIndex("ReceivingEndpointName"); + + b.HasIndex("StatusChangedAt") + .HasFilter("[Status] IN (2, 4)"); + + b.HasIndex("TimeSent"); + + b.HasIndex("Status", "LastModified"); + + b.ToTable("FailedMessages"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uniqueidentifier"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.HasKey("FailedMessageUniqueId", "GroupId"); + + b.HasIndex("GroupId"); + + b.ToTable("FailedMessageGroups"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("RetryId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.ToTable("FailedMessageRetries"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Host") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("Monitored") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("KnownEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061320_Initial.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061320_Initial.cs new file mode 100644 index 0000000000..f15a501188 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061320_Initial.cs @@ -0,0 +1,151 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + /// + public partial class Initial : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "FailedMessageRetries", + columns: table => new + { + UniqueMessageId = table.Column(type: "uniqueidentifier", nullable: false), + RetryId = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_FailedMessageRetries", x => x.UniqueMessageId); + }); + + migrationBuilder.CreateTable( + name: "FailedMessages", + columns: table => new + { + UniqueMessageId = table.Column(type: "uniqueidentifier", nullable: false), + Status = table.Column(type: "int", nullable: false), + StatusChangedAt = table.Column(type: "datetime2", nullable: false), + LastModified = table.Column(type: "datetime2", nullable: false), + NumberOfProcessingAttempts = table.Column(type: "int", nullable: false), + FirstTimeOfFailure = table.Column(type: "datetime2", nullable: false), + LastTimeOfFailure = table.Column(type: "datetime2", nullable: false), + LastAttemptedAt = table.Column(type: "datetime2", nullable: false), + MessageId = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: true), + MessageType = table.Column(type: "nvarchar(max)", nullable: true), + TimeSent = table.Column(type: "datetime2", nullable: true), + ConversationId = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: true), + QueueAddress = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: true), + SendingEndpointName = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: true), + SendingEndpointHostId = table.Column(type: "uniqueidentifier", nullable: true), + SendingEndpointHost = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: true), + ReceivingEndpointName = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: true), + ReceivingEndpointHostId = table.Column(type: "uniqueidentifier", nullable: true), + ReceivingEndpointHost = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: true), + ExceptionType = table.Column(type: "nvarchar(max)", nullable: true), + ExceptionMessage = table.Column(type: "nvarchar(max)", nullable: true), + IsSystemMessage = table.Column(type: "bit", nullable: false), + HeadersJson = table.Column(type: "nvarchar(max)", nullable: false), + BodyText = table.Column(type: "nvarchar(max)", nullable: true), + BodyStoredExternally = table.Column(type: "bit", nullable: false), + BodySize = table.Column(type: "int", nullable: false), + BodyContentType = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_FailedMessages", x => x.UniqueMessageId); + }); + + migrationBuilder.CreateTable( + name: "KnownEndpoints", + columns: table => new + { + Id = table.Column(type: "uniqueidentifier", nullable: false), + Name = table.Column(type: "nvarchar(max)", nullable: false), + HostId = table.Column(type: "uniqueidentifier", nullable: false), + Host = table.Column(type: "nvarchar(max)", nullable: false), + Monitored = table.Column(type: "bit", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_KnownEndpoints", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "FailedMessageGroups", + columns: table => new + { + FailedMessageUniqueId = table.Column(type: "uniqueidentifier", nullable: false), + GroupId = table.Column(type: "nvarchar(64)", maxLength: 64, nullable: false), + Title = table.Column(type: "nvarchar(max)", nullable: false), + Type = table.Column(type: "nvarchar(255)", maxLength: 255, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_FailedMessageGroups", x => new { x.FailedMessageUniqueId, x.GroupId }); + table.ForeignKey( + name: "FK_FailedMessageGroups_FailedMessages_FailedMessageUniqueId", + column: x => x.FailedMessageUniqueId, + principalTable: "FailedMessages", + principalColumn: "UniqueMessageId", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_FailedMessageGroups_GroupId", + table: "FailedMessageGroups", + column: "GroupId"); + + migrationBuilder.CreateIndex( + name: "IX_FailedMessages_ConversationId", + table: "FailedMessages", + column: "ConversationId"); + + migrationBuilder.CreateIndex( + name: "IX_FailedMessages_QueueAddress", + table: "FailedMessages", + column: "QueueAddress"); + + migrationBuilder.CreateIndex( + name: "IX_FailedMessages_ReceivingEndpointName", + table: "FailedMessages", + column: "ReceivingEndpointName"); + + migrationBuilder.CreateIndex( + name: "IX_FailedMessages_Status_LastModified", + table: "FailedMessages", + columns: new[] { "Status", "LastModified" }); + + migrationBuilder.CreateIndex( + name: "IX_FailedMessages_StatusChangedAt", + table: "FailedMessages", + column: "StatusChangedAt", + filter: "[Status] IN (2, 4)"); + + migrationBuilder.CreateIndex( + name: "IX_FailedMessages_TimeSent", + table: "FailedMessages", + column: "TimeSent"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "FailedMessageGroups"); + + migrationBuilder.DropTable( + name: "FailedMessageRetries"); + + migrationBuilder.DropTable( + name: "KnownEndpoints"); + + migrationBuilder.DropTable( + name: "FailedMessages"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061323_AddFullTextSearch.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061323_AddFullTextSearch.Designer.cs new file mode 100644 index 0000000000..4e36cde31e --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061323_AddFullTextSearch.Designer.cs @@ -0,0 +1,212 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Persistence.EFCore.SqlServer; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerServiceControlDbContext))] + [Migration("20260722061323_AddFullTextSearch")] + partial class AddFullTextSearch + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("BodyText") + .HasColumnType("nvarchar(max)"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ExceptionMessage") + .HasColumnType("nvarchar(max)"); + + b.Property("ExceptionType") + .HasColumnType("nvarchar(max)"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("LastAttemptedAt") + .HasColumnType("datetime2"); + + b.Property("LastModified") + .HasColumnType("datetime2"); + + b.Property("LastTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("MessageType") + .HasColumnType("nvarchar(max)"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("int"); + + b.Property("QueueAddress") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("StatusChangedAt") + .HasColumnType("datetime2"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("ConversationId"); + + b.HasIndex("QueueAddress"); + + b.HasIndex("ReceivingEndpointName"); + + b.HasIndex("StatusChangedAt") + .HasFilter("[Status] IN (2, 4)"); + + b.HasIndex("TimeSent"); + + b.HasIndex("Status", "LastModified"); + + b.ToTable("FailedMessages"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uniqueidentifier"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.HasKey("FailedMessageUniqueId", "GroupId"); + + b.HasIndex("GroupId"); + + b.ToTable("FailedMessageGroups"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("RetryId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.ToTable("FailedMessageRetries"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Host") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("Monitored") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("KnownEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061323_AddFullTextSearch.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061323_AddFullTextSearch.cs new file mode 100644 index 0000000000..9393cce434 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260722061323_AddFullTextSearch.cs @@ -0,0 +1,24 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + /// + public partial class AddFullTextSearch : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(FullTextSearchSql.CreateCatalog, suppressTransaction: true); + migrationBuilder.Sql(FullTextSearchSql.CreateIndex, suppressTransaction: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(FullTextSearchSql.DropIndex, suppressTransaction: true); + migrationBuilder.Sql(FullTextSearchSql.DropCatalog, suppressTransaction: true); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs index df880e2f92..bedbda43dc 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs @@ -22,37 +22,159 @@ protected override void BuildModel(ModelBuilder modelBuilder) SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => { - b.Property("Id") + b.Property("UniqueMessageId") .HasColumnType("uniqueidentifier"); - b.Property("Host") + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("BodyText") + .HasColumnType("nvarchar(max)"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ExceptionMessage") + .HasColumnType("nvarchar(max)"); + + b.Property("ExceptionType") + .HasColumnType("nvarchar(max)"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") .IsRequired() .HasColumnType("nvarchar(max)"); - b.Property("HostId") + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("LastAttemptedAt") + .HasColumnType("datetime2"); + + b.Property("LastModified") + .HasColumnType("datetime2"); + + b.Property("LastTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("MessageType") + .HasColumnType("nvarchar(max)"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("int"); + + b.Property("QueueAddress") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHostId") .HasColumnType("uniqueidentifier"); - b.Property("Monitored") - .HasColumnType("bit"); + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); - b.Property("Name") + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("StatusChangedAt") + .HasColumnType("datetime2"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("ConversationId"); + + b.HasIndex("QueueAddress"); + + b.HasIndex("ReceivingEndpointName"); + + b.HasIndex("StatusChangedAt") + .HasFilter("[Status] IN (2, 4)"); + + b.HasIndex("TimeSent"); + + b.HasIndex("Status", "LastModified"); + + b.ToTable("FailedMessages"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uniqueidentifier"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Title") .IsRequired() .HasColumnType("nvarchar(max)"); - b.HasKey("Id"); + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.HasKey("FailedMessageUniqueId", "GroupId"); + + b.HasIndex("GroupId"); - b.ToTable("KnownEndpoints", (string)null); + b.ToTable("FailedMessageGroups"); }); - modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointInsertOnlyEntity", b => + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("bigint"); + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("RetryId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); - SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + b.HasKey("UniqueMessageId"); + + b.ToTable("FailedMessageRetries"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); b.Property("Host") .IsRequired() @@ -61,8 +183,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("HostId") .HasColumnType("uniqueidentifier"); - b.Property("KnownEndpointId") - .HasColumnType("uniqueidentifier"); + b.Property("Monitored") + .HasColumnType("bit"); b.Property("Name") .IsRequired() @@ -70,9 +192,16 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasKey("Id"); - b.HasIndex("KnownEndpointId"); + b.ToTable("KnownEndpoints"); + }); - b.ToTable("KnownEndpointsInsertOnly", (string)null); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); }); #pragma warning restore 612, 618 } diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs index 2882099881..171f08a837 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs @@ -4,7 +4,6 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; using Microsoft.Extensions.DependencyInjection; using ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.Persistence.EFCore.DbContexts; -using ServiceControl.Persistence.EFCore.SqlServer.Infrastructure; class SqlServerPersistence(SqlServerPersisterSettings settings) : BasePersistence, IPersistence { @@ -13,8 +12,6 @@ public void AddPersistence(IServiceCollection services) RegisterSettings(services); ConfigureDbContext(services); RegisterDataStores(services); - - services.AddHostedService(); } public void AddInstaller(IServiceCollection services) diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerServiceControlDbContext.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerServiceControlDbContext.cs index 528ab19b52..8c77803995 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerServiceControlDbContext.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerServiceControlDbContext.cs @@ -1,8 +1,18 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; using Microsoft.EntityFrameworkCore; +using ServiceControl.MessageFailures; using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; public class SqlServerServiceControlDbContext(DbContextOptions options) : ServiceControlDbContext(options) { + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + + modelBuilder.Entity() + .HasIndex(e => e.StatusChangedAt) + .HasFilter($"[Status] IN ({(int)FailedMessageStatus.Resolved}, {(int)FailedMessageStatus.Archived})"); + } } diff --git a/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs b/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs index 54f272d30c..4eca844bee 100644 --- a/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs +++ b/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs @@ -1,6 +1,8 @@ namespace ServiceControl.Persistence.EFCore.Abstractions; +using Microsoft.Extensions.Logging; using ServiceControl.Configuration; +using ServiceControl.Infrastructure; public abstract class EFPersistenceConfigurationBase : IPersistenceConfiguration { @@ -8,9 +10,12 @@ public abstract class EFPersistenceConfigurationBase : IPersistenceConfiguration const string CommandTimeoutKey = "Database/CommandTimeout"; const string MessageBodyStoragePathKey = "MessageBody/StoragePath"; const string MinBodySizeForCompressionKey = "MessageBody/MinCompressionSize"; + const string MaxBodySizeToStoreKey = "MaxBodySizeToStore"; const string ErrorRetentionPeriodKey = "ErrorRetentionPeriod"; const string EnableFullTextSearchOnBodiesKey = "EnableFullTextSearchOnBodies"; + const int MaxBodySizeToStoreDefault = 102400; //100 kb + public PersistenceSettings CreateSettings(SettingsRootNamespace settingsRootNamespace) { var settings = CreateSettings(GetRequiredSetting(settingsRootNamespace, ConnectionStringKey)); @@ -18,6 +23,7 @@ public PersistenceSettings CreateSettings(SettingsRootNamespace settingsRootName settings.CommandTimeout = SettingsReader.Read(settingsRootNamespace, CommandTimeoutKey, 30); settings.MessageBodyStoragePath = SettingsReader.Read(settingsRootNamespace, MessageBodyStoragePathKey); settings.MinBodySizeForCompression = SettingsReader.Read(settingsRootNamespace, MinBodySizeForCompressionKey, 4096); + settings.MaxBodySizeToStore = ReadMaxBodySizeToStore(settingsRootNamespace); settings.ErrorRetentionPeriod = GetRequiredSetting(settingsRootNamespace, ErrorRetentionPeriodKey); settings.EnableFullTextSearchOnBodies = SettingsReader.Read(settingsRootNamespace, EnableFullTextSearchOnBodiesKey, true); @@ -28,6 +34,21 @@ public PersistenceSettings CreateSettings(SettingsRootNamespace settingsRootName protected abstract EFPersisterSettings CreateSettings(string connectionString); + static int ReadMaxBodySizeToStore(SettingsRootNamespace settingsRootNamespace) + { + var maxBodySizeToStore = SettingsReader.Read(settingsRootNamespace, MaxBodySizeToStoreKey, MaxBodySizeToStoreDefault); + + if (maxBodySizeToStore <= 0) + { + LoggerUtil.CreateStaticLogger() + .LogError("MaxBodySizeToStore setting is invalid, 1 is the minimum value. Defaulting to {MaxBodySizeToStoreDefault}", MaxBodySizeToStoreDefault); + + return MaxBodySizeToStoreDefault; + } + + return maxBodySizeToStore; + } + static T GetRequiredSetting(SettingsRootNamespace settingsRootNamespace, string key) { if (SettingsReader.TryRead(settingsRootNamespace, key, out var value)) diff --git a/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersisterSettings.cs b/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersisterSettings.cs index d6a5d6ca38..52f54d4815 100644 --- a/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersisterSettings.cs +++ b/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersisterSettings.cs @@ -9,6 +9,7 @@ public abstract class EFPersisterSettings : PersistenceSettings public TimeSpan ErrorRetentionPeriod { get; set; } public string? MessageBodyStoragePath { get; set; } public int MinBodySizeForCompression { get; set; } = 4096; + public int MaxBodySizeToStore { get; set; } = 102400; public int MaxRetryCount { get; set; } = 5; public int MaxRetryDelayInSeconds { get; set; } = 30; public bool EnableSensitiveDataLogging { get; set; } diff --git a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs index 6a7e0a6fc9..49990d4363 100644 --- a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs +++ b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs @@ -7,7 +7,9 @@ namespace ServiceControl.Persistence.EFCore.DbContexts; public abstract class ServiceControlDbContext(DbContextOptions options) : DbContext(options) { public DbSet KnownEndpoints { get; set; } - public DbSet KnownEndpointsInsertOnly { get; set; } + public DbSet FailedMessages { get; set; } + public DbSet FailedMessageGroups { get; set; } + public DbSet FailedMessageRetries { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { @@ -19,6 +21,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) base.OnModelCreating(modelBuilder); modelBuilder.ApplyConfiguration(new KnownEndpointConfiguration()); - modelBuilder.ApplyConfiguration(new KnownEndpointInsertOnlyConfiguration()); + modelBuilder.ApplyConfiguration(new FailedMessageConfiguration()); + modelBuilder.ApplyConfiguration(new FailedMessageGroupConfiguration()); + modelBuilder.ApplyConfiguration(new FailedMessageRetryConfiguration()); } } diff --git a/src/ServiceControl.Persistence.EFCore/Entities/FailedMessageEntity.cs b/src/ServiceControl.Persistence.EFCore/Entities/FailedMessageEntity.cs new file mode 100644 index 0000000000..f9b7ee810a --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Entities/FailedMessageEntity.cs @@ -0,0 +1,60 @@ +namespace ServiceControl.Persistence.EFCore.Entities; + +using ServiceControl.MessageFailures; + +public class FailedMessageEntity +{ + public Guid UniqueMessageId { get; set; } + + public FailedMessageStatus Status { get; set; } + + public DateTime StatusChangedAt { get; set; } + + public DateTime LastModified { get; set; } + + public int NumberOfProcessingAttempts { get; set; } + + public DateTime FirstTimeOfFailure { get; set; } + + public DateTime LastTimeOfFailure { get; set; } + + public DateTime LastAttemptedAt { get; set; } + + public string? MessageId { get; set; } + + public string? MessageType { get; set; } + + public DateTime? TimeSent { get; set; } + + public string? ConversationId { get; set; } + + public string? QueueAddress { get; set; } + + public string? SendingEndpointName { get; set; } + + public Guid? SendingEndpointHostId { get; set; } + + public string? SendingEndpointHost { get; set; } + + public string? ReceivingEndpointName { get; set; } + + public Guid? ReceivingEndpointHostId { get; set; } + + public string? ReceivingEndpointHost { get; set; } + + public string? ExceptionType { get; set; } + + public string? ExceptionMessage { get; set; } + + public bool IsSystemMessage { get; set; } + + public required string HeadersJson { get; set; } + + public string? BodyText { get; set; } + + public bool BodyStoredExternally { get; set; } + + public int BodySize { get; set; } + + public string? BodyContentType { get; set; } +} diff --git a/src/ServiceControl.Persistence.EFCore/Entities/FailedMessageGroupEntity.cs b/src/ServiceControl.Persistence.EFCore/Entities/FailedMessageGroupEntity.cs new file mode 100644 index 0000000000..cb17ae0f16 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Entities/FailedMessageGroupEntity.cs @@ -0,0 +1,14 @@ +namespace ServiceControl.Persistence.EFCore.Entities; + +// Association between a failed message and the failure groups it belongs to. The rows for a +// message are replaced wholesale on every processing attempt. +public class FailedMessageGroupEntity +{ + public Guid FailedMessageUniqueId { get; set; } + + public required string GroupId { get; set; } + + public required string Title { get; set; } + + public required string Type { get; set; } +} diff --git a/src/ServiceControl.Persistence.EFCore/Entities/FailedMessageRetryEntity.cs b/src/ServiceControl.Persistence.EFCore/Entities/FailedMessageRetryEntity.cs new file mode 100644 index 0000000000..5fe0816f2d --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Entities/FailedMessageRetryEntity.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Persistence.EFCore.Entities; + +public class FailedMessageRetryEntity +{ + public Guid UniqueMessageId { get; set; } + + public string? RetryId { get; set; } +} diff --git a/src/ServiceControl.Persistence.EFCore/Entities/KnownEndpointInsertOnlyEntity.cs b/src/ServiceControl.Persistence.EFCore/Entities/KnownEndpointInsertOnlyEntity.cs deleted file mode 100644 index 00f4274407..0000000000 --- a/src/ServiceControl.Persistence.EFCore/Entities/KnownEndpointInsertOnlyEntity.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace ServiceControl.Persistence.EFCore.Entities; - -public class KnownEndpointInsertOnlyEntity -{ - public long Id { get; set; } - public Guid KnownEndpointId { get; set; } - - public string? Name { get; set; } - - public Guid HostId { get; set; } - - public string? Host { get; set; } -} diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs new file mode 100644 index 0000000000..36a1dfb966 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs @@ -0,0 +1,50 @@ +namespace ServiceControl.Persistence.EFCore.EntityConfigurations; + +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +class FailedMessageConfiguration : IEntityTypeConfiguration +{ + // Indexed and short-by-nature values get a length so that SQL Server can index them, + // nvarchar(max) columns cannot be index key columns. + public const int ShortTextLength = 450; + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(e => e.UniqueMessageId); + builder.Property(e => e.UniqueMessageId).ValueGeneratedNever(); + + builder.Property(e => e.Status).IsRequired(); + builder.Property(e => e.StatusChangedAt).IsRequired(); + builder.Property(e => e.LastModified).IsRequired(); + builder.Property(e => e.NumberOfProcessingAttempts).IsRequired(); + builder.Property(e => e.FirstTimeOfFailure).IsRequired(); + builder.Property(e => e.LastTimeOfFailure).IsRequired(); + builder.Property(e => e.LastAttemptedAt).IsRequired(); + + builder.Property(e => e.MessageId).HasMaxLength(ShortTextLength); + builder.Property(e => e.ConversationId).HasMaxLength(ShortTextLength); + builder.Property(e => e.QueueAddress).HasMaxLength(ShortTextLength); + builder.Property(e => e.SendingEndpointName).HasMaxLength(ShortTextLength); + builder.Property(e => e.SendingEndpointHost).HasMaxLength(ShortTextLength); + builder.Property(e => e.ReceivingEndpointName).HasMaxLength(ShortTextLength); + builder.Property(e => e.ReceivingEndpointHost).HasMaxLength(ShortTextLength); + builder.Property(e => e.BodyContentType).HasMaxLength(ShortTextLength); + + builder.Property(e => e.IsSystemMessage).IsRequired(); + builder.Property(e => e.HeadersJson).IsRequired(); + builder.Property(e => e.BodyStoredExternally).IsRequired(); + builder.Property(e => e.BodySize).IsRequired(); + + builder.HasIndex(e => new { e.Status, e.LastModified }); + builder.HasIndex(e => e.ReceivingEndpointName); + builder.HasIndex(e => e.ConversationId); + builder.HasIndex(e => e.TimeSent); + builder.HasIndex(e => e.QueueAddress); + + // Drives the retention sweep. The index is restricted to the statuses the sweep deletes + // (Resolved and Archived) by a provider specific filter, applied in the provider DbContext. + builder.HasIndex(e => e.StatusChangedAt); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageGroupConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageGroupConfiguration.cs new file mode 100644 index 0000000000..591081cb80 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageGroupConfiguration.cs @@ -0,0 +1,25 @@ +namespace ServiceControl.Persistence.EFCore.EntityConfigurations; + +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +class FailedMessageGroupConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(e => new { e.FailedMessageUniqueId, e.GroupId }); + + // Group ids are deterministic Guid strings + builder.Property(e => e.GroupId).HasMaxLength(64).IsRequired(); + builder.Property(e => e.Title).IsRequired(); + builder.Property(e => e.Type).HasMaxLength(255).IsRequired(); + + builder.HasIndex(e => e.GroupId); + + builder.HasOne() + .WithMany() + .HasForeignKey(e => e.FailedMessageUniqueId) + .OnDelete(DeleteBehavior.Cascade); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageRetryConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageRetryConfiguration.cs new file mode 100644 index 0000000000..ea98f8240f --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageRetryConfiguration.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Persistence.EFCore.EntityConfigurations; + +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +class FailedMessageRetryConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(e => e.UniqueMessageId); + builder.Property(e => e.UniqueMessageId).ValueGeneratedNever(); + builder.Property(e => e.RetryId).HasMaxLength(FailedMessageConfiguration.ShortTextLength); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/KnownEndpointConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/KnownEndpointConfiguration.cs index f82cd1bbe5..4bcb26d5e2 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/KnownEndpointConfiguration.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/KnownEndpointConfiguration.cs @@ -8,7 +8,6 @@ class KnownEndpointConfiguration : IEntityTypeConfiguration { public void Configure(EntityTypeBuilder builder) { - builder.ToTable("KnownEndpoints"); builder.HasKey(e => e.Id); builder.Property(e => e.Id).ValueGeneratedNever(); builder.Property(e => e.Name).IsRequired(); diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/KnownEndpointInsertOnlyConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/KnownEndpointInsertOnlyConfiguration.cs deleted file mode 100644 index ba3ce9f578..0000000000 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/KnownEndpointInsertOnlyConfiguration.cs +++ /dev/null @@ -1,21 +0,0 @@ -namespace ServiceControl.Persistence.EFCore.EntityConfigurations; - -using Entities; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Metadata.Builders; - -class KnownEndpointInsertOnlyConfiguration : IEntityTypeConfiguration -{ - public void Configure(EntityTypeBuilder builder) - { - builder.ToTable("KnownEndpointsInsertOnly"); - builder.HasKey(e => e.Id); - builder.Property(e => e.Id).ValueGeneratedOnAdd(); - builder.Property(e => e.KnownEndpointId).IsRequired(); - builder.Property(e => e.Name).IsRequired(); - builder.Property(e => e.HostId).IsRequired(); - builder.Property(e => e.Host).IsRequired(); - - builder.HasIndex(e => e.KnownEndpointId); - } -} diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFMonitoringIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFMonitoringIngestionUnitOfWork.cs index b37f07d634..32ecd4b4ec 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFMonitoringIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFMonitoringIngestionUnitOfWork.cs @@ -1,25 +1,12 @@ namespace ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; using ServiceControl.Persistence.EFCore.DbContexts; -using ServiceControl.Persistence.EFCore.Entities; using ServiceControl.Persistence.UnitOfWork; +#pragma warning disable CS9113 // Parameter is unread. public class EFMonitoringIngestionUnitOfWork(ServiceControlDbContext dbContext) : IMonitoringIngestionUnitOfWork +#pragma warning restore CS9113 // Parameter is unread. { - readonly ServiceControlDbContext dbContext = dbContext; - - public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint) - { - var entity = new KnownEndpointInsertOnlyEntity - { - KnownEndpointId = knownEndpoint.EndpointDetails.GetDeterministicId(), - Name = knownEndpoint.EndpointDetails.Name, - HostId = knownEndpoint.EndpointDetails.HostId, - Host = knownEndpoint.EndpointDetails.Host - }; - - dbContext.KnownEndpointsInsertOnly.Add(entity); - - return Task.CompletedTask; - } + public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint) => + throw new NotImplementedException(); } diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/KnownEndpointsReconcilerTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/KnownEndpointsReconcilerTests.cs deleted file mode 100644 index a1529d19b1..0000000000 --- a/src/ServiceControl.Persistence.Tests.PostgreSql/KnownEndpointsReconcilerTests.cs +++ /dev/null @@ -1,156 +0,0 @@ -namespace ServiceControl.Persistence.Tests -{ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.EntityFrameworkCore; - using Microsoft.Extensions.DependencyInjection; - using NUnit.Framework; - using ServiceControl.Persistence.EFCore.DbContexts; - using ServiceControl.Persistence.EFCore.Entities; - using ServiceControl.Persistence.EFCore.PostgreSql.Infrastructure; - - [TestFixture] - class KnownEndpointsReconcilerTests : PersistenceTestBase - { - [Test] - public async Task Moves_pending_rows_into_known_endpoints() - { - var row1 = NewInsertOnlyRow("Endpoint1"); - var row2 = NewInsertOnlyRow("Endpoint2"); - await SeedInsertOnlyRows(row1, row2); - - await RunReconcileBatch(); - - var ids = new[] { row1.KnownEndpointId, row2.KnownEndpointId }; - var knownEndpoints = await GetKnownEndpoints(ids); - Assert.That(knownEndpoints, Has.Count.EqualTo(2)); - - var endpoint1 = knownEndpoints.Single(e => e.Id == row1.KnownEndpointId); - using (Assert.EnterMultipleScope()) - { - Assert.That(endpoint1.Name, Is.EqualTo(row1.Name)); - Assert.That(endpoint1.HostId, Is.EqualTo(row1.HostId)); - Assert.That(endpoint1.Host, Is.EqualTo(row1.Host)); - Assert.That(endpoint1.Monitored, Is.False, "Endpoints detected during ingestion should not be monitored by default"); - } - - Assert.That(await CountInsertOnlyRows(ids), Is.Zero, "Reconciled rows should be removed from the insert-only table"); - } - - [Test] - public async Task Duplicate_pending_rows_produce_single_endpoint() - { - var row = NewInsertOnlyRow("Endpoint1"); - var duplicate = NewInsertOnlyRow("Endpoint1"); - duplicate.KnownEndpointId = row.KnownEndpointId; - await SeedInsertOnlyRows(row, duplicate); - - await RunReconcileBatch(); - - var ids = new[] { row.KnownEndpointId }; - using (Assert.EnterMultipleScope()) - { - Assert.That(await GetKnownEndpoints(ids), Has.Count.EqualTo(1)); - Assert.That(await CountInsertOnlyRows(ids), Is.Zero); - } - } - - [Test] - public async Task Existing_endpoints_are_not_modified() - { - var row = NewInsertOnlyRow("Endpoint1"); - await SeedKnownEndpoint(new KnownEndpointEntity - { - Id = row.KnownEndpointId, - Name = row.Name, - HostId = row.HostId, - Host = row.Host, - Monitored = true - }); - await SeedInsertOnlyRows(row); - - await RunReconcileBatch(); - - var ids = new[] { row.KnownEndpointId }; - var knownEndpoints = await GetKnownEndpoints(ids); - using (Assert.EnterMultipleScope()) - { - Assert.That(knownEndpoints, Has.Count.EqualTo(1)); - Assert.That(knownEndpoints[0].Monitored, Is.True, "Reconciliation should not overwrite existing endpoints"); - Assert.That(await CountInsertOnlyRows(ids), Is.Zero); - } - } - - [Test] - public async Task Reconciles_in_batches() - { - var rows = new[] { NewInsertOnlyRow("Endpoint1"), NewInsertOnlyRow("Endpoint2"), NewInsertOnlyRow("Endpoint3") }; - await SeedInsertOnlyRows(rows); - var ids = rows.Select(r => r.KnownEndpointId).ToArray(); - - await RunReconcileBatch(batchSize: 2); - Assert.That(await CountInsertOnlyRows(ids), Is.EqualTo(1), "Only one batch should be processed per call"); - - await RunReconcileBatch(batchSize: 2); - using (Assert.EnterMultipleScope()) - { - Assert.That(await CountInsertOnlyRows(ids), Is.Zero); - Assert.That(await GetKnownEndpoints(ids), Has.Count.EqualTo(3)); - } - } - - static KnownEndpointInsertOnlyEntity NewInsertOnlyRow(string name) => new() - { - KnownEndpointId = Guid.NewGuid(), - Name = name, - HostId = Guid.NewGuid(), - Host = "Host1" - }; - - async Task SeedInsertOnlyRows(params KnownEndpointInsertOnlyEntity[] rows) - { - using var scope = ServiceProvider.CreateScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); - dbContext.KnownEndpointsInsertOnly.AddRange(rows); - await dbContext.SaveChangesAsync(); - } - - async Task SeedKnownEndpoint(KnownEndpointEntity entity) - { - using var scope = ServiceProvider.CreateScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); - dbContext.KnownEndpoints.Add(entity); - await dbContext.SaveChangesAsync(); - } - - async Task RunReconcileBatch(int batchSize = 1000) - { - using var scope = ServiceProvider.CreateScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); - var strategy = dbContext.Database.CreateExecutionStrategy(); - await strategy.ExecuteAsync(async () => - { - await using var transaction = await dbContext.Database.BeginTransactionAsync(); - await KnownEndpointsReconciler.ReconcileBatch(dbContext, batchSize, CancellationToken.None); - await transaction.CommitAsync(); - }); - } - - async Task> GetKnownEndpoints(IReadOnlyCollection ids) - { - using var scope = ServiceProvider.CreateScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); - return await dbContext.KnownEndpoints.AsNoTracking().Where(e => ids.Contains(e.Id)).ToListAsync(); - } - - async Task CountInsertOnlyRows(IReadOnlyCollection ids) - { - using var scope = ServiceProvider.CreateScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); - return await dbContext.KnownEndpointsInsertOnly.CountAsync(e => ids.Contains(e.KnownEndpointId)); - } - } -} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/KnownEndpointsReconcilerTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/KnownEndpointsReconcilerTests.cs deleted file mode 100644 index c9aa851b9d..0000000000 --- a/src/ServiceControl.Persistence.Tests.SqlServer/KnownEndpointsReconcilerTests.cs +++ /dev/null @@ -1,158 +0,0 @@ -namespace ServiceControl.Persistence.Tests -{ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.EntityFrameworkCore; - using Microsoft.Extensions.DependencyInjection; - using NUnit.Framework; - using ServiceControl.Persistence.EFCore.DbContexts; - using ServiceControl.Persistence.EFCore.Entities; - using ServiceControl.Persistence.EFCore.SqlServer.Infrastructure; - - // The SqlServer/PostgreSql persistence test suites share one database/container across all - // tests (no per-test isolation), so assertions here are always scoped to the specific - // KnownEndpointId values each test seeds, rather than the table's total row count. - class KnownEndpointsReconcilerTests : PersistenceTestBase - { - [Test] - public async Task Moves_pending_rows_into_known_endpoints() - { - var row1 = NewInsertOnlyRow("Endpoint1"); - var row2 = NewInsertOnlyRow("Endpoint2"); - await SeedInsertOnlyRows(row1, row2); - - await RunReconcileBatch(); - - var ids = new[] { row1.KnownEndpointId, row2.KnownEndpointId }; - var knownEndpoints = await GetKnownEndpoints(ids); - Assert.That(knownEndpoints, Has.Count.EqualTo(2)); - - var endpoint1 = knownEndpoints.Single(e => e.Id == row1.KnownEndpointId); - using (Assert.EnterMultipleScope()) - { - Assert.That(endpoint1.Name, Is.EqualTo(row1.Name)); - Assert.That(endpoint1.HostId, Is.EqualTo(row1.HostId)); - Assert.That(endpoint1.Host, Is.EqualTo(row1.Host)); - Assert.That(endpoint1.Monitored, Is.False, "Endpoints detected during ingestion should not be monitored by default"); - } - - Assert.That(await CountInsertOnlyRows(ids), Is.Zero, "Reconciled rows should be removed from the insert-only table"); - } - - [Test] - public async Task Duplicate_pending_rows_produce_single_endpoint() - { - var row = NewInsertOnlyRow("Endpoint1"); - var duplicate = NewInsertOnlyRow("Endpoint1"); - duplicate.KnownEndpointId = row.KnownEndpointId; - await SeedInsertOnlyRows(row, duplicate); - - await RunReconcileBatch(); - - var ids = new[] { row.KnownEndpointId }; - using (Assert.EnterMultipleScope()) - { - Assert.That(await GetKnownEndpoints(ids), Has.Count.EqualTo(1)); - Assert.That(await CountInsertOnlyRows(ids), Is.Zero); - } - } - - [Test] - public async Task Existing_endpoints_are_not_modified() - { - var row = NewInsertOnlyRow("Endpoint1"); - await SeedKnownEndpoint(new KnownEndpointEntity - { - Id = row.KnownEndpointId, - Name = row.Name, - HostId = row.HostId, - Host = row.Host, - Monitored = true - }); - await SeedInsertOnlyRows(row); - - await RunReconcileBatch(); - - var ids = new[] { row.KnownEndpointId }; - var knownEndpoints = await GetKnownEndpoints(ids); - using (Assert.EnterMultipleScope()) - { - Assert.That(knownEndpoints, Has.Count.EqualTo(1)); - Assert.That(knownEndpoints[0].Monitored, Is.True, "Reconciliation should not overwrite existing endpoints"); - Assert.That(await CountInsertOnlyRows(ids), Is.Zero); - } - } - - [Test] - public async Task Reconciles_in_batches() - { - var rows = new[] { NewInsertOnlyRow("Endpoint1"), NewInsertOnlyRow("Endpoint2"), NewInsertOnlyRow("Endpoint3") }; - await SeedInsertOnlyRows(rows); - var ids = rows.Select(r => r.KnownEndpointId).ToArray(); - - await RunReconcileBatch(batchSize: 2); - Assert.That(await CountInsertOnlyRows(ids), Is.EqualTo(1), "Only one batch should be processed per call"); - - await RunReconcileBatch(batchSize: 2); - using (Assert.EnterMultipleScope()) - { - Assert.That(await CountInsertOnlyRows(ids), Is.Zero); - Assert.That(await GetKnownEndpoints(ids), Has.Count.EqualTo(3)); - } - } - - static KnownEndpointInsertOnlyEntity NewInsertOnlyRow(string name) => new() - { - KnownEndpointId = Guid.NewGuid(), - Name = name, - HostId = Guid.NewGuid(), - Host = "Host1" - }; - - async Task SeedInsertOnlyRows(params KnownEndpointInsertOnlyEntity[] rows) - { - using var scope = ServiceProvider.CreateScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); - dbContext.KnownEndpointsInsertOnly.AddRange(rows); - await dbContext.SaveChangesAsync(); - } - - async Task SeedKnownEndpoint(KnownEndpointEntity entity) - { - using var scope = ServiceProvider.CreateScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); - dbContext.KnownEndpoints.Add(entity); - await dbContext.SaveChangesAsync(); - } - - async Task RunReconcileBatch(int batchSize = 1000) - { - using var scope = ServiceProvider.CreateScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); - var strategy = dbContext.Database.CreateExecutionStrategy(); - await strategy.ExecuteAsync(async () => - { - await using var transaction = await dbContext.Database.BeginTransactionAsync(); - await KnownEndpointsReconciler.ReconcileBatch(dbContext, batchSize, CancellationToken.None); - await transaction.CommitAsync(); - }); - } - - async Task> GetKnownEndpoints(IReadOnlyCollection ids) - { - using var scope = ServiceProvider.CreateScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); - return await dbContext.KnownEndpoints.AsNoTracking().Where(e => ids.Contains(e.Id)).ToListAsync(); - } - - async Task CountInsertOnlyRows(IReadOnlyCollection ids) - { - using var scope = ServiceProvider.CreateScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); - return await dbContext.KnownEndpointsInsertOnly.CountAsync(e => ids.Contains(e.KnownEndpointId)); - } - } -} From b733de70a8d1cd332e48c616a2698810cc2b62fd Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 23 Jul 2026 11:32:22 +1000 Subject: [PATCH 2/5] Implement relational error ingestion write path This change completes the error ingestion process for PostgreSQL and SQL Server persisters, enabling robust and concurrent storage of failed messages. It introduces a detailed design document (`error-ingestion-design.md`) outlining the complex write path. --- docs/error-ingestion-design.md | 186 +++++++++++ .../PostgreSqlIngestionSqlDialect.cs | 175 +++++++++++ .../PostgreSqlPersistence.cs | 3 + .../SqlServerIngestionSqlDialect.cs | 197 ++++++++++++ .../SqlServerPersistence.cs | 3 + .../FakeBodyStoragePersistence.cs | 4 +- .../UnitOfWork/EFIngestionUnitOfWork.cs | 38 ++- .../EFIngestionUnitOfWorkFactory.cs | 6 +- .../EFMonitoringIngestionUnitOfWork.cs | 13 +- .../EFRecoverabilityIngestionUnitOfWork.cs | 74 ++++- .../UnitOfWork/FailedMessageBatchWriter.cs | 155 ++++++++++ .../UnitOfWork/HeadersJsonContext.cs | 8 + .../UnitOfWork/MessageBodyClassifier.cs | 75 +++++ .../RecordedFailedProcessingAttempt.cs | 32 ++ .../Infrastructure/IBodyStoragePersistence.cs | 6 +- .../Infrastructure/IIngestionSqlDialect.cs | 30 ++ ...ontrol.Persistence.Tests.PostgreSql.csproj | 3 - ...ceControl.Persistence.Tests.RavenDB.csproj | 3 + ...Control.Persistence.Tests.SqlServer.csproj | 3 - .../EFCore/ErrorIngestionBodyTests.cs | 143 +++++++++ .../EFCore/ErrorIngestionConcurrencyTests.cs | 68 ++++ .../ErrorIngestionKnownEndpointTests.cs | 88 ++++++ .../EFCore/ErrorIngestionTestBase.cs | 130 ++++++++ .../EFCore/ErrorIngestionTests.cs | 290 ++++++++++++++++++ .../ExecuteUpdateDeleteTransactionProbe.cs | 47 +++ .../EFCore/IngestedFailure.cs | 99 ++++++ 26 files changed, 1842 insertions(+), 37 deletions(-) create mode 100644 docs/error-ingestion-design.md create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlIngestionSqlDialect.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerIngestionSqlDialect.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/FailedMessageBatchWriter.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/HeadersJsonContext.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/RecordedFailedProcessingAttempt.cs create mode 100644 src/ServiceControl.Persistence.EFCore/Infrastructure/IIngestionSqlDialect.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionBodyTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionConcurrencyTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionKnownEndpointTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTestBase.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/ExecuteUpdateDeleteTransactionProbe.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/IngestedFailure.cs diff --git a/docs/error-ingestion-design.md b/docs/error-ingestion-design.md new file mode 100644 index 0000000000..02938844fe --- /dev/null +++ b/docs/error-ingestion-design.md @@ -0,0 +1,186 @@ +# Error ingestion design (relational persisters) + +## Overview + +The error instance receives failed messages on its input queue and stores them so they can be +queried, grouped, retried, and eventually archived. This document describes how ingestion works +for the relational persisters (PostgreSQL and SQL Server) and explains the design decisions that +are not obvious from the code, in particular why the write path is hand-written SQL rather than +ordinary change-tracked entity saves. + +The unit of ingestion is a **batch**: the transport hands the ingester up to `MaximumConcurrency` +messages at a time, and the whole batch is written in a single database transaction. The relevant +types are `EFIngestionUnitOfWork` (accumulation), `FailedMessageBatchWriter` (the write), and the +per-provider `IIngestionSqlDialect` implementations (the statements that differ by provider). + +## Data model + +The schema stores **one row per failed message**, keyed by `UniqueMessageId`, in the +`FailedMessages` table. There is deliberately **no attempts history table**. A message that fails +repeatedly keeps a single row that records: + +- the **last** processing attempt (every denormalized and payload column comes from it), +- the **number of distinct attempts** (`NumberOfProcessingAttempts`), +- the **failure window** (`FirstTimeOfFailure`, `LastTimeOfFailure`). + +This is a deliberate difference from the document-database persister, which retained an array of +attempts. The read side only ever consumed the last attempt plus the count, so storing the full +history earned nothing and cost write amplification. One consequence: `NumberOfProcessingAttempts` +is no longer capped, whereas the document store capped the retained array at ten. + +### Stored source data vs derived columns + +Only two pieces of data are **stored as source of truth**: the full headers dictionary +(`HeadersJson`) and the message body. Every other column (message type, endpoints, exception +details, timestamps, and so on) is a **derived extraction** written purely so it can be indexed +and queried. On read, the `FailureDetails` object and the metadata dictionary the rest of the +system expects are **reconstructed** from the headers and these columns. Nothing downstream of +ingestion reads a column expecting it to carry information the headers do not already contain. + +`BodyUrl`, `ContentType`, and `ContentLength` are examples worth calling out: the document store +persisted them into a metadata dictionary, but they are all derivable (`BodyUrl` from the +`UniqueMessageId`, the other two from the `BodyContentType` and `BodySize` columns), so they are +not stored again. + +### Body placement + +Bodies are **always** stored. The `MaxBodySizeToStore` setting (default 100 KB) only decides +*where*: + +- text at or under the cap: stored inline in `BodyText`, nothing external. +- text over the cap: the full body goes to external storage, and a **search prefix** of at most + the cap, cut on a valid UTF-8 boundary, is kept inline in `BodyText` so search still works. +- binary, or not strictly UTF-8 decodable, or containing a NUL byte: external storage only, + `BodyText` is null, regardless of size. + +When `BodyStoredExternally` is true the external copy is authoritative and `BodyText` is a +search aid only; it must never be served as the body. `BodySize` is always the true original +size. External writes happen before the row that points at them is committed. + +### Groups, endpoints, retention + +- **Failure groups** live in `FailedMessageGroups`, an association table. A message's group rows + are **replaced wholesale** on every attempt, because grouping is recomputed from the latest + attempt. Group views are query-time aggregates and are not part of ingestion. +- **Known endpoints** are written in the same batch, **insert-if-absent**. An existing endpoint + row is never updated, which preserves the user-controlled `Monitored` flag. +- **Retention** is driven by `StatusChangedAt`. A background sweeper deletes `Resolved` and + `Archived` rows older than `now - ErrorRetentionPeriod`, with the cutoff recomputed on every + run so a changed retention setting takes effect without rewriting rows. A filtered index on + `StatusChangedAt` restricted to those two statuses keeps the sweep cheap. + +### Full-text search + +Search over headers and body is provider-native and set up with raw SQL in the migrations: a +stored generated `tsvector` column with a GIN index on PostgreSQL, and a full-text catalog and +index on SQL Server. This is orthogonal to the write path but is another place where the +relational features we want have no portable expression. + +## The write path + +`RecordFailedProcessingAttempt` is called **concurrently** for the messages in a batch (the +ingester fans the batch out). `RecordKnownEndpoint` and `RecordSuccessfulRetry` are called +afterwards. An EF `DbContext` is not thread-safe and must not be touched from multiple threads, +so the `Record*` methods do no database work at all: they only enqueue into thread-safe +collections. **All** database access happens later, on one thread, in `Complete`. + +`Complete` first waits for any external body writes, then hands the accumulated work to +`FailedMessageBatchWriter.Write`, which: + +1. **Folds** the accumulated attempts in memory into one row per message plus its group rows. The + fold sorts a message's attempts by time, takes the last as the winner, counts distinct attempt + timestamps, and computes the failure window. The result is sorted by `UniqueMessageId` so that + concurrent writers tend to take row locks in the same order. +2. Opens **one transaction** and runs the statements in a fixed order: + 1. upsert the failed-message rows, + 2. delete then re-insert the affected messages' group rows, + 3. insert-if-absent the known endpoints, + 4. resolve confirmed retries (set them `Resolved`, delete their retry rows), + 5. commit. + +The order matters: a message that both fails and is retry-confirmed in the same batch must end +`Resolved`, so the retry resolution runs last. + +## Why the write is raw SQL + +Most of ServiceControl prefers standard abstractions, and the portable parts of this write path do +use them: the group delete and the retry resolution are ordinary set-based EF operations +(`ExecuteDelete`/`ExecuteUpdate`). The **upserts** are hand-written SQL, per provider, behind the +`IIngestionSqlDialect` seam. Three requirements together force that, and no ORM-level API satisfies +all three at once. + +### 1. The upsert is a conditional merge, not a save + +Writing a failed message is not "insert this row" or "update this row". For a message that already +exists the statement must, in one shot: + +- flip the status back to `Unresolved`, and reset the retention clock **only** if the row was + previously resolved or archived, +- add the batch's attempt count, but **not** if the batch merely redelivered the attempt already + stored (equal timestamps), +- widen the failure window (min of the first, max of the last), +- replace every payload column with the incoming values **only if** the incoming attempt is at + least as new as the stored one, so that an out-of-order older attempt still counts but does not + overwrite newer data. + +Those are per-column conditional expressions comparing the incoming row against the pre-update +stored row. A change-tracked save cannot express them: it would have to read every row first, +decide in memory, and write back, which is both slower and a race (see below). The logic has to +execute **inside a single set-based statement** where every guard reads the same consistent row +state. + +### 2. It must be correct under concurrent writers + +The instance runs a single ingestion loop today, but the write path is built so that multiple +instances could ingest against the same database (for example to scale out under load). That means +two transactions can try to write the **same** `UniqueMessageId` at the same time. Two hazards +follow: + +- **Insert races.** Both writers see the row as absent and both insert, colliding on the primary + key. +- **Lost updates.** A read-modify-write done in application code (read the row, add to the count, + write it back) can lose one writer's increment. + +Closing both requires the database's own concurrency-safe upsert primitive, and those are +**provider-specific**: + +- PostgreSQL: `INSERT ... ON CONFLICT (unique_message_id) DO UPDATE`. The conflict clause makes a + concurrent insert fall through to the update instead of failing, and the whole statement is + atomic so the count arithmetic cannot lose an increment. +- SQL Server: `MERGE ... WITH (HOLDLOCK)`. The lock hint serializes concurrent merges on the same + key so the second one sees the row and updates instead of colliding. + +These have no common surface. `ON CONFLICT` and `MERGE` are different grammars with different +concurrency semantics (a plain `MERGE` on PostgreSQL is **not** insert-race safe, which is exactly +why the PostgreSQL side uses `ON CONFLICT` instead). Expressing "the concurrency-safe conditional +upsert for this database" therefore means writing the statement each database actually needs. + +### 3. It is set-based over a whole batch + +A batch can hold many messages. Saving them as tracked entities would be a statement per row and +would, on SQL Server, run into the 2100-parameter limit for larger batches. The dialects instead +send **chunked multi-row statements** (up to 50 rows each): a handful of fixed statement shapes +that the database can cache a plan for, with no per-row round trips and no temporary tables. + +### What stays portable + +Only the genuinely divergent statements are raw. The retry resolution and the group delete are set +based and identical across providers, so they remain EF operations in the shared writer. The raw +SQL is confined to the two dialect classes, one per provider, each responsible only for the +upserts. The guard semantics are kept identical between the two dialects; the shared test suite +runs every ingestion test against both providers to keep them from drifting. + +## Transactions and retries + +Everything in a batch runs in one transaction opened by the writer. The raw dialect commands are +explicitly enlisted onto that transaction, and the EF `ExecuteUpdate`/`ExecuteDelete` operations +participate in it as well, so a failure anywhere rolls the whole batch back. Nothing is committed +piecemeal. + +The transaction is wrapped in the provider's execution strategy so that a transient failure (a +dropped connection, or a deadlock between concurrent writers) retries the **entire** batch. This is +safe because the batch is **idempotent**: re-running it folds to the same rows, the upsert is a +merge, the group rows are deleted and re-inserted, endpoints are insert-if-absent, and retry +resolution is a set update plus delete. Replaying a batch after an ambiguous commit changes +nothing. Stable lock ordering (the fold sorts by `UniqueMessageId`) keeps deadlocks rare in the +first place. diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlIngestionSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlIngestionSqlDialect.cs new file mode 100644 index 0000000000..96421bd7bd --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlIngestionSqlDialect.cs @@ -0,0 +1,175 @@ +namespace ServiceControl.Persistence.EFCore.PostgreSql; + +using System.Text; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +// INSERT ... ON CONFLICT rather than MERGE: PostgreSQL's MERGE can fail with unique_violation +// when two writers insert the same key concurrently, ON CONFLICT cannot. All references to the +// target table inside DO UPDATE read the pre-update row, so the guards are consistent within one +// atomic statement. Rows are chunked to keep statement texts down to a few reusable shapes. +class PostgreSqlIngestionSqlDialect : IIngestionSqlDialect +{ + public async Task UpsertFailedMessages(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken) + { + foreach (var chunk in rows.Chunk(MaxRowsPerStatement)) + { + await Execute( + dbContext, + $""" + INSERT INTO failed_messages ({FailedMessageColumnList}) + VALUES + {ParameterRows(chunk.Length, FailedMessageColumns.Length)} + {OnConflictUpdate} + """, + chunk.Select(FailedMessageValues), + cancellationToken); + } + } + + public async Task InsertGroups(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken) + { + foreach (var chunk in rows.Chunk(MaxRowsPerStatement)) + { + await Execute( + dbContext, + $""" + INSERT INTO failed_message_groups (failed_message_unique_id, group_id, title, type) + VALUES + {ParameterRows(chunk.Length, 4)} + ON CONFLICT (failed_message_unique_id, group_id) DO NOTHING + """, + chunk.Select(group => new object?[] { group.FailedMessageUniqueId, group.GroupId, group.Title, group.Type }), + cancellationToken); + } + } + + public async Task InsertMissingKnownEndpoints(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken) + { + foreach (var chunk in rows.Chunk(MaxRowsPerStatement)) + { + await Execute( + dbContext, + $""" + INSERT INTO known_endpoints (id, name, host_id, host, monitored) + VALUES + {ParameterRows(chunk.Length, 5)} + ON CONFLICT (id) DO NOTHING + """, + chunk.Select(endpoint => new object?[] { endpoint.Id, endpoint.Name, endpoint.HostId, endpoint.Host, endpoint.Monitored }), + cancellationToken); + } + } + + static async Task Execute(ServiceControlDbContext dbContext, string sql, IEnumerable rows, CancellationToken cancellationToken) + { + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.Transaction = (dbContext.Database.CurrentTransaction + ?? throw new InvalidOperationException("Ingestion statements must run inside the batch transaction")).GetDbTransaction(); + command.CommandText = sql; + + var index = 0; + foreach (var row in rows) + { + foreach (var value in row) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = $"@p{index++}"; + parameter.Value = value ?? DBNull.Value; + command.Parameters.Add(parameter); + } + } + + await command.ExecuteNonQueryAsync(cancellationToken); + } + + // The columns the newer attempt wins wholesale + static readonly string[] PayloadColumns = + [ + "message_id", "message_type", "time_sent", "conversation_id", "queue_address", + "sending_endpoint_name", "sending_endpoint_host_id", "sending_endpoint_host", + "receiving_endpoint_name", "receiving_endpoint_host_id", "receiving_endpoint_host", + "exception_type", "exception_message", "is_system_message", + "headers_json", "body_text", "body_stored_externally", "body_size", "body_content_type" + ]; + + // Column order matches FailedMessageValues + static readonly string[] FailedMessageColumns = + [ + "unique_message_id", "status", "status_changed_at", "last_modified", + "number_of_processing_attempts", "first_time_of_failure", "last_time_of_failure", "last_attempted_at", + .. PayloadColumns + ]; + + static object?[] FailedMessageValues(FailedMessageEntity row) => + [ + row.UniqueMessageId, (int)row.Status, row.StatusChangedAt, row.LastModified, + row.NumberOfProcessingAttempts, row.FirstTimeOfFailure, row.LastTimeOfFailure, row.LastAttemptedAt, + row.MessageId, row.MessageType, row.TimeSent, row.ConversationId, row.QueueAddress, + row.SendingEndpointName, row.SendingEndpointHostId, row.SendingEndpointHost, + row.ReceivingEndpointName, row.ReceivingEndpointHostId, row.ReceivingEndpointHost, + row.ExceptionType, row.ExceptionMessage, row.IsSystemMessage, + row.HeadersJson, row.BodyText, row.BodyStoredExternally, row.BodySize, row.BodyContentType + ]; + + static readonly string FailedMessageColumnList = string.Join(", ", FailedMessageColumns); + + static readonly string OnConflictUpdate = BuildOnConflictUpdate(); + + static string BuildOnConflictUpdate() + { + const int unresolved = (int)FailedMessageStatus.Unresolved; + + var sql = new StringBuilder( + $""" + ON CONFLICT (unique_message_id) DO UPDATE SET + status = {unresolved}, + status_changed_at = CASE WHEN failed_messages.status <> {unresolved} THEN excluded.status_changed_at ELSE failed_messages.status_changed_at END, + last_modified = excluded.last_modified, + number_of_processing_attempts = failed_messages.number_of_processing_attempts + + CASE WHEN excluded.last_attempted_at <> failed_messages.last_attempted_at THEN excluded.number_of_processing_attempts ELSE 0 END, + first_time_of_failure = LEAST(failed_messages.first_time_of_failure, excluded.first_time_of_failure), + last_time_of_failure = GREATEST(failed_messages.last_time_of_failure, excluded.last_time_of_failure), + """); + + foreach (var column in PayloadColumns) + { + sql.AppendLine().Append( + $" {column} = CASE WHEN excluded.last_attempted_at >= failed_messages.last_attempted_at THEN excluded.{column} ELSE failed_messages.{column} END,"); + } + + sql.AppendLine().Append(" last_attempted_at = GREATEST(failed_messages.last_attempted_at, excluded.last_attempted_at)"); + + return sql.ToString(); + } + + static string ParameterRows(int rowCount, int columnCount) + { + var sql = new StringBuilder(); + + for (var row = 0; row < rowCount; row++) + { + sql.Append(row == 0 ? "(" : ",\n("); + + for (var column = 0; column < columnCount; column++) + { + if (column > 0) + { + sql.Append(", "); + } + + sql.Append("@p").Append((row * columnCount) + column); + } + + sql.Append(')'); + } + + return sql.ToString(); + } + + const int MaxRowsPerStatement = 50; +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs index bba7a9506a..dabe754e30 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs @@ -4,6 +4,7 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql; using Microsoft.Extensions.DependencyInjection; using ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Infrastructure; class PostgreSqlPersistence(PostgreSqlPersisterSettings settings) : BasePersistence, IPersistence { @@ -12,6 +13,8 @@ public void AddPersistence(IServiceCollection services) RegisterSettings(services); ConfigureDbContext(services); RegisterDataStores(services); + + services.AddSingleton(); } public void AddInstaller(IServiceCollection services) diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerIngestionSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerIngestionSqlDialect.cs new file mode 100644 index 0000000000..86f31948b4 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerIngestionSqlDialect.cs @@ -0,0 +1,197 @@ +namespace ServiceControl.Persistence.EFCore.SqlServer; + +using System.Data; +using System.Text; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +// MERGE WITH (HOLDLOCK) over an inline VALUES source. HOLDLOCK closes the race where two writers +// both miss a key and collide on the insert, and the single statement keeps every guard reading +// the same row state. Rows are chunked to stay clear of the 2100 parameter limit while keeping +// statement texts down to a few reusable shapes. +class SqlServerIngestionSqlDialect : IIngestionSqlDialect +{ + public async Task UpsertFailedMessages(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken) + { + foreach (var chunk in rows.Chunk(MaxRowsPerStatement)) + { + await Execute( + dbContext, + $""" + MERGE [FailedMessages] WITH (HOLDLOCK) AS t + USING (VALUES + {ParameterRows(chunk.Length, FailedMessageColumns.Length)} + ) AS s ({FailedMessageColumnList}) + ON t.[UniqueMessageId] = s.[UniqueMessageId] + {WhenMatchedUpdate} + WHEN NOT MATCHED THEN INSERT ({FailedMessageColumnList}) + VALUES ({FailedMessageSourceColumnList}); + """, + chunk.Select(FailedMessageValues), + cancellationToken); + } + } + + public async Task InsertGroups(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken) + { + foreach (var chunk in rows.Chunk(MaxRowsPerStatement)) + { + await Execute( + dbContext, + $""" + MERGE [FailedMessageGroups] WITH (HOLDLOCK) AS t + USING (VALUES + {ParameterRows(chunk.Length, 4)} + ) AS s ([FailedMessageUniqueId], [GroupId], [Title], [Type]) + ON t.[FailedMessageUniqueId] = s.[FailedMessageUniqueId] AND t.[GroupId] = s.[GroupId] + WHEN NOT MATCHED THEN INSERT ([FailedMessageUniqueId], [GroupId], [Title], [Type]) + VALUES (s.[FailedMessageUniqueId], s.[GroupId], s.[Title], s.[Type]); + """, + chunk.Select(group => new object?[] { group.FailedMessageUniqueId, group.GroupId, group.Title, group.Type }), + cancellationToken); + } + } + + public async Task InsertMissingKnownEndpoints(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken) + { + foreach (var chunk in rows.Chunk(MaxRowsPerStatement)) + { + await Execute( + dbContext, + $""" + MERGE [KnownEndpoints] WITH (HOLDLOCK) AS t + USING (VALUES + {ParameterRows(chunk.Length, 5)} + ) AS s ([Id], [Name], [HostId], [Host], [Monitored]) + ON t.[Id] = s.[Id] + WHEN NOT MATCHED THEN INSERT ([Id], [Name], [HostId], [Host], [Monitored]) + VALUES (s.[Id], s.[Name], s.[HostId], s.[Host], s.[Monitored]); + """, + chunk.Select(endpoint => new object?[] { endpoint.Id, endpoint.Name, endpoint.HostId, endpoint.Host, endpoint.Monitored }), + cancellationToken); + } + } + + static async Task Execute(ServiceControlDbContext dbContext, string sql, IEnumerable rows, CancellationToken cancellationToken) + { + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.Transaction = (dbContext.Database.CurrentTransaction + ?? throw new InvalidOperationException("Ingestion statements must run inside the batch transaction")).GetDbTransaction(); + command.CommandText = sql; + + var index = 0; + foreach (var row in rows) + { + foreach (var value in row) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = $"@p{index++}"; + parameter.Value = value ?? DBNull.Value; + + // Attempt de-duplication compares LastAttemptedAt for equality, so datetime + // parameters must keep datetime2 precision instead of the datetime default. + if (value is DateTime) + { + parameter.DbType = DbType.DateTime2; + } + + command.Parameters.Add(parameter); + } + } + + await command.ExecuteNonQueryAsync(cancellationToken); + } + + // The columns the newer attempt wins wholesale + static readonly string[] PayloadColumns = + [ + "MessageId", "MessageType", "TimeSent", "ConversationId", "QueueAddress", + "SendingEndpointName", "SendingEndpointHostId", "SendingEndpointHost", + "ReceivingEndpointName", "ReceivingEndpointHostId", "ReceivingEndpointHost", + "ExceptionType", "ExceptionMessage", "IsSystemMessage", + "HeadersJson", "BodyText", "BodyStoredExternally", "BodySize", "BodyContentType" + ]; + + // Column order matches FailedMessageValues + static readonly string[] FailedMessageColumns = + [ + "UniqueMessageId", "Status", "StatusChangedAt", "LastModified", + "NumberOfProcessingAttempts", "FirstTimeOfFailure", "LastTimeOfFailure", "LastAttemptedAt", + .. PayloadColumns + ]; + + static object?[] FailedMessageValues(FailedMessageEntity row) => + [ + row.UniqueMessageId, (int)row.Status, row.StatusChangedAt, row.LastModified, + row.NumberOfProcessingAttempts, row.FirstTimeOfFailure, row.LastTimeOfFailure, row.LastAttemptedAt, + row.MessageId, row.MessageType, row.TimeSent, row.ConversationId, row.QueueAddress, + row.SendingEndpointName, row.SendingEndpointHostId, row.SendingEndpointHost, + row.ReceivingEndpointName, row.ReceivingEndpointHostId, row.ReceivingEndpointHost, + row.ExceptionType, row.ExceptionMessage, row.IsSystemMessage, + row.HeadersJson, row.BodyText, row.BodyStoredExternally, row.BodySize, row.BodyContentType + ]; + + static readonly string FailedMessageColumnList = string.Join(", ", FailedMessageColumns.Select(column => $"[{column}]")); + + static readonly string FailedMessageSourceColumnList = string.Join(", ", FailedMessageColumns.Select(column => $"s.[{column}]")); + + static readonly string WhenMatchedUpdate = BuildWhenMatchedUpdate(); + + static string BuildWhenMatchedUpdate() + { + const int unresolved = (int)FailedMessageStatus.Unresolved; + + var sql = new StringBuilder( + $""" + WHEN MATCHED THEN UPDATE SET + [Status] = {unresolved}, + [StatusChangedAt] = CASE WHEN t.[Status] <> {unresolved} THEN s.[StatusChangedAt] ELSE t.[StatusChangedAt] END, + [LastModified] = s.[LastModified], + [NumberOfProcessingAttempts] = t.[NumberOfProcessingAttempts] + + CASE WHEN s.[LastAttemptedAt] <> t.[LastAttemptedAt] THEN s.[NumberOfProcessingAttempts] ELSE 0 END, + [FirstTimeOfFailure] = CASE WHEN s.[FirstTimeOfFailure] < t.[FirstTimeOfFailure] THEN s.[FirstTimeOfFailure] ELSE t.[FirstTimeOfFailure] END, + [LastTimeOfFailure] = CASE WHEN s.[LastTimeOfFailure] > t.[LastTimeOfFailure] THEN s.[LastTimeOfFailure] ELSE t.[LastTimeOfFailure] END, + """); + + foreach (var column in PayloadColumns) + { + sql.AppendLine().Append( + $" [{column}] = CASE WHEN s.[LastAttemptedAt] >= t.[LastAttemptedAt] THEN s.[{column}] ELSE t.[{column}] END,"); + } + + sql.AppendLine().Append(" [LastAttemptedAt] = CASE WHEN s.[LastAttemptedAt] > t.[LastAttemptedAt] THEN s.[LastAttemptedAt] ELSE t.[LastAttemptedAt] END"); + + return sql.ToString(); + } + + static string ParameterRows(int rowCount, int columnCount) + { + var sql = new StringBuilder(); + + for (var row = 0; row < rowCount; row++) + { + sql.Append(row == 0 ? "(" : ",\n("); + + for (var column = 0; column < columnCount; column++) + { + if (column > 0) + { + sql.Append(", "); + } + + sql.Append("@p").Append((row * columnCount) + column); + } + + sql.Append(')'); + } + + return sql.ToString(); + } + + // 27 columns * 50 rows stays well below the 2100 parameter limit + const int MaxRowsPerStatement = 50; +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs index 171f08a837..5e9336e747 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs @@ -4,6 +4,7 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; using Microsoft.Extensions.DependencyInjection; using ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Infrastructure; class SqlServerPersistence(SqlServerPersisterSettings settings) : BasePersistence, IPersistence { @@ -12,6 +13,8 @@ public void AddPersistence(IServiceCollection services) RegisterSettings(services); ConfigureDbContext(services); RegisterDataStores(services); + + services.AddSingleton(); } public void AddInstaller(IServiceCollection services) diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/FakeBodyStoragePersistence.cs b/src/ServiceControl.Persistence.EFCore/Implementation/FakeBodyStoragePersistence.cs index 7aa2199020..c4fe41a092 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/FakeBodyStoragePersistence.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/FakeBodyStoragePersistence.cs @@ -5,6 +5,6 @@ namespace ServiceControl.Persistence.EFCore.Implementation; public class FakeBodyStoragePersistence : IBodyStoragePersistence { public Task DeleteBody(string bodyId, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task ReadBody(string bodyId, DateTime createdOn, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task WriteBody(string bodyId, DateTime createdOn, ReadOnlyMemory body, string contentType, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task ReadBody(string bodyId, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task WriteBody(string bodyId, ReadOnlyMemory body, string contentType, CancellationToken cancellationToken = default) => throw new NotImplementedException(); } diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFIngestionUnitOfWork.cs index 3ead245bb1..07f434269a 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFIngestionUnitOfWork.cs @@ -1,28 +1,55 @@ namespace ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; +using System.Collections.Concurrent; using ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.Persistence.EFCore.DbContexts; using ServiceControl.Persistence.EFCore.Infrastructure; using ServiceControl.Persistence.UnitOfWork; +// RecordFailedProcessingAttempt runs concurrently across the batch, so the Record methods only +// add to thread safe collections. Every database call happens in Complete, on one thread. public class EFIngestionUnitOfWork : IIngestionUnitOfWork { readonly ServiceControlDbContext dbContext; readonly IAsyncDisposable scope; - - public EFIngestionUnitOfWork(IAsyncDisposable scope, ServiceControlDbContext dbContext, IBodyStoragePersistence storagePersistence, EFPersisterSettings settings) + readonly IIngestionSqlDialect dialect; + readonly TimeProvider timeProvider; + readonly ConcurrentQueue failedProcessingAttempts = new(); + readonly ConcurrentQueue bodyWrites = new(); + readonly ConcurrentQueue knownEndpoints = new(); + readonly ConcurrentQueue confirmedRetries = new(); + + public EFIngestionUnitOfWork(IAsyncDisposable scope, ServiceControlDbContext dbContext, IBodyStoragePersistence storagePersistence, EFPersisterSettings settings, IIngestionSqlDialect dialect, TimeProvider timeProvider) { this.scope = scope; this.dbContext = dbContext; - Recoverability = new EFRecoverabilityIngestionUnitOfWork(dbContext, storagePersistence, settings); - Monitoring = new EFMonitoringIngestionUnitOfWork(dbContext); + this.dialect = dialect; + this.timeProvider = timeProvider; + Recoverability = new EFRecoverabilityIngestionUnitOfWork(this, storagePersistence, settings); + Monitoring = new EFMonitoringIngestionUnitOfWork(this); } public IMonitoringIngestionUnitOfWork Monitoring { get; } public IRecoverabilityIngestionUnitOfWork Recoverability { get; } - public Task Complete(CancellationToken cancellationToken) => dbContext.SaveChangesAsync(cancellationToken); + internal void Record(RecordedFailedProcessingAttempt attempt) => failedProcessingAttempts.Enqueue(attempt); + + internal void RecordBodyWrite(Task bodyWrite) => bodyWrites.Enqueue(bodyWrite); + + internal void Record(KnownEndpoint knownEndpoint) => knownEndpoints.Enqueue(knownEndpoint); + + internal void RecordConfirmedRetry(Guid uniqueMessageId) => confirmedRetries.Enqueue(uniqueMessageId); + + public async Task Complete(CancellationToken cancellationToken) + { + // External bodies are written before the rows that point at them + await Task.WhenAll(bodyWrites); + + var writer = new FailedMessageBatchWriter(dbContext, dialect); + + await writer.Write(failedProcessingAttempts, knownEndpoints, confirmedRetries, timeProvider.GetUtcNow().UtcDateTime, cancellationToken); + } public async ValueTask DisposeAsync() { @@ -32,4 +59,3 @@ public async ValueTask DisposeAsync() GC.SuppressFinalize(this); } } - diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFIngestionUnitOfWorkFactory.cs index a425acd23e..9c2422a249 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFIngestionUnitOfWorkFactory.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFIngestionUnitOfWorkFactory.cs @@ -9,14 +9,16 @@ namespace ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; public class EFIngestionUnitOfWorkFactory( IServiceProvider serviceProvider, MinimumRequiredStorageState storageState, - IBodyStoragePersistence storagePersistence) : IIngestionUnitOfWorkFactory + IBodyStoragePersistence storagePersistence, + IIngestionSqlDialect dialect, + TimeProvider timeProvider) : IIngestionUnitOfWorkFactory { public ValueTask StartNew() { var scope = serviceProvider.CreateAsyncScope(); var dbContext = scope.ServiceProvider.GetRequiredService(); var settings = scope.ServiceProvider.GetRequiredService(); - var unitOfWork = new EFIngestionUnitOfWork(scope, dbContext, storagePersistence, settings); + var unitOfWork = new EFIngestionUnitOfWork(scope, dbContext, storagePersistence, settings, dialect, timeProvider); return ValueTask.FromResult(unitOfWork); } diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFMonitoringIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFMonitoringIngestionUnitOfWork.cs index 32ecd4b4ec..19cdf86f54 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFMonitoringIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFMonitoringIngestionUnitOfWork.cs @@ -1,12 +1,13 @@ namespace ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; -using ServiceControl.Persistence.EFCore.DbContexts; using ServiceControl.Persistence.UnitOfWork; -#pragma warning disable CS9113 // Parameter is unread. -public class EFMonitoringIngestionUnitOfWork(ServiceControlDbContext dbContext) : IMonitoringIngestionUnitOfWork -#pragma warning restore CS9113 // Parameter is unread. +public class EFMonitoringIngestionUnitOfWork(EFIngestionUnitOfWork parentUnitOfWork) : IMonitoringIngestionUnitOfWork { - public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint) => - throw new NotImplementedException(); + public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint) + { + parentUnitOfWork.Record(knownEndpoint); + + return Task.CompletedTask; + } } diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFRecoverabilityIngestionUnitOfWork.cs index 5eb0989b71..d33a539cdc 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFRecoverabilityIngestionUnitOfWork.cs @@ -1,24 +1,72 @@ namespace ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; +using System.Text.Json; +using NServiceBus; using NServiceBus.Transport; using ServiceControl.MessageFailures; -using ServiceControl.Persistence.UnitOfWork; -using ServiceControl.Persistence.EFCore.DbContexts; -using ServiceControl.Persistence.EFCore.Infrastructure; -using System; -using System.Collections.Generic; -using System.Threading.Tasks; +using ServiceControl.Operations; using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Persistence.Infrastructure; +using ServiceControl.Persistence.UnitOfWork; -#pragma warning disable CS9113 // Parameter is unread. -public class EFRecoverabilityIngestionUnitOfWork(ServiceControlDbContext dbContext, IBodyStoragePersistence storagePersistence, EFPersisterSettings settings) : IRecoverabilityIngestionUnitOfWork -#pragma warning restore CS9113 // Parameter is unread. +public class EFRecoverabilityIngestionUnitOfWork(EFIngestionUnitOfWork parentUnitOfWork, IBodyStoragePersistence storagePersistence, EFPersisterSettings settings) : IRecoverabilityIngestionUnitOfWork { public Task RecordFailedProcessingAttempt(MessageContext context, FailedMessage.ProcessingAttempt processingAttempt, - List groups) => - throw new NotImplementedException(); + List groups) + { + var uniqueMessageId = context.Headers.UniqueId(); + var contentType = context.Headers.GetValueOrDefault(Headers.ContentType, "text/plain"); + var bodySize = context.Body.Length; + var (bodyText, storeExternally) = MessageBodyClassifier.Classify(context.Headers, context.Body, settings.MaxBodySizeToStore); + + if (storeExternally) + { + parentUnitOfWork.RecordBodyWrite( + storagePersistence.WriteBody(uniqueMessageId, context.Body, contentType)); + } + + var sendingEndpoint = GetMetadata(processingAttempt, "SendingEndpoint"); + var receivingEndpoint = GetMetadata(processingAttempt, "ReceivingEndpoint"); + + parentUnitOfWork.Record(new RecordedFailedProcessingAttempt + { + UniqueMessageId = Guid.Parse(uniqueMessageId), + AttemptedAt = processingAttempt.AttemptedAt, + TimeOfFailure = processingAttempt.FailureDetails.TimeOfFailure, + Groups = groups, + HeadersJson = JsonSerializer.Serialize(processingAttempt.Headers, HeadersJsonContext.Default.DictionaryStringString), + MessageId = processingAttempt.MessageId, + MessageType = GetMetadata(processingAttempt, "MessageType"), + TimeSent = GetMetadata(processingAttempt, "TimeSent"), + ConversationId = GetMetadata(processingAttempt, "ConversationId"), + QueueAddress = context.Headers.GetValueOrDefault(NServiceBus.Faults.FaultsHeaderKeys.FailedQ), + SendingEndpointName = sendingEndpoint?.Name, + SendingEndpointHostId = sendingEndpoint?.HostId, + SendingEndpointHost = sendingEndpoint?.Host, + ReceivingEndpointName = receivingEndpoint?.Name, + ReceivingEndpointHostId = receivingEndpoint?.HostId, + ReceivingEndpointHost = receivingEndpoint?.Host, + ExceptionType = processingAttempt.FailureDetails.Exception?.ExceptionType, + ExceptionMessage = processingAttempt.FailureDetails.Exception?.Message, + IsSystemMessage = GetMetadata(processingAttempt, "IsSystemMessage"), + BodyText = bodyText, + BodyStoredExternally = storeExternally, + BodySize = bodySize, + BodyContentType = contentType + }); + + return Task.CompletedTask; + } + + public Task RecordSuccessfulRetry(string retriedMessageUniqueId) + { + parentUnitOfWork.RecordConfirmedRetry(Guid.Parse(retriedMessageUniqueId)); + + return Task.CompletedTask; + } - public Task RecordSuccessfulRetry(string retriedMessageUniqueId) => - throw new NotImplementedException(); + static T? GetMetadata(FailedMessage.ProcessingAttempt processingAttempt, string key) => + processingAttempt.MessageMetadata.TryGetValue(key, out var value) && value is T typed ? typed : default; } diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/FailedMessageBatchWriter.cs b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/FailedMessageBatchWriter.cs new file mode 100644 index 0000000000..f2974c75b0 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/FailedMessageBatchWriter.cs @@ -0,0 +1,155 @@ +namespace ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; + +using Microsoft.EntityFrameworkCore; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +// Writes one ingestion batch inside a single transaction. The statements providers genuinely +// differ on (the upserts) come from the injected dialect; everything portable stays here as +// set-based EF operations. Statement order matters: a message that fails and is retry-confirmed +// in the same batch must end Resolved. +class FailedMessageBatchWriter(ServiceControlDbContext dbContext, IIngestionSqlDialect dialect) +{ + public async Task Write( + IReadOnlyCollection attempts, + IReadOnlyCollection knownEndpoints, + IReadOnlyCollection confirmedRetries, + DateTime now, + CancellationToken cancellationToken) + { + var (failedMessages, groups) = Fold(attempts, now); + var endpoints = BuildEndpointRows(knownEndpoints); + var retries = confirmedRetries.Distinct().ToArray(); + + if (failedMessages.Count == 0 && endpoints.Count == 0 && retries.Length == 0) + { + return; + } + + var strategy = dbContext.Database.CreateExecutionStrategy(); + await strategy.ExecuteAsync(async ct => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(ct); + + if (failedMessages.Count > 0) + { + await dialect.UpsertFailedMessages(dbContext, failedMessages, ct); + await ReplaceGroups(failedMessages, groups, ct); + } + + if (endpoints.Count > 0) + { + await dialect.InsertMissingKnownEndpoints(dbContext, endpoints, ct); + } + + if (retries.Length > 0) + { + await ResolveRetried(retries, now, ct); + } + + await transaction.CommitAsync(ct); + }, cancellationToken); + } + + static (List Messages, List Groups) Fold( + IReadOnlyCollection attempts, DateTime now) + { + var messages = new List(); + var groups = new List(); + + foreach (var group in attempts.GroupBy(attempt => attempt.UniqueMessageId).OrderBy(group => group.Key)) + { + var ordered = group.OrderBy(attempt => attempt.AttemptedAt).ToList(); + var last = ordered[^1]; + + messages.Add(new FailedMessageEntity + { + UniqueMessageId = group.Key, + Status = FailedMessageStatus.Unresolved, + StatusChangedAt = now, + LastModified = now, + NumberOfProcessingAttempts = ordered.Select(attempt => attempt.AttemptedAt).Distinct().Count(), + FirstTimeOfFailure = ordered.Min(attempt => attempt.TimeOfFailure), + LastTimeOfFailure = ordered.Max(attempt => attempt.TimeOfFailure), + LastAttemptedAt = last.AttemptedAt, + MessageId = last.MessageId, + MessageType = last.MessageType, + TimeSent = last.TimeSent, + ConversationId = last.ConversationId, + QueueAddress = last.QueueAddress, + SendingEndpointName = last.SendingEndpointName, + SendingEndpointHostId = last.SendingEndpointHostId, + SendingEndpointHost = last.SendingEndpointHost, + ReceivingEndpointName = last.ReceivingEndpointName, + ReceivingEndpointHostId = last.ReceivingEndpointHostId, + ReceivingEndpointHost = last.ReceivingEndpointHost, + ExceptionType = last.ExceptionType, + ExceptionMessage = last.ExceptionMessage, + IsSystemMessage = last.IsSystemMessage, + HeadersJson = last.HeadersJson, + BodyText = last.BodyText, + BodyStoredExternally = last.BodyStoredExternally, + BodySize = last.BodySize, + BodyContentType = last.BodyContentType + }); + + groups.AddRange(last.Groups + .Where(failureGroup => failureGroup.Id != null) + .DistinctBy(failureGroup => failureGroup.Id) + .Select(failureGroup => new FailedMessageGroupEntity + { + FailedMessageUniqueId = group.Key, + GroupId = failureGroup.Id, + Title = failureGroup.Title ?? string.Empty, + Type = failureGroup.Type ?? string.Empty + })); + } + + return (messages, groups); + } + + static List BuildEndpointRows(IReadOnlyCollection knownEndpoints) => + [.. knownEndpoints + .Select(knownEndpoint => new KnownEndpointEntity + { + Id = knownEndpoint.EndpointDetails.GetDeterministicId(), + Name = knownEndpoint.EndpointDetails.Name, + HostId = knownEndpoint.EndpointDetails.HostId, + Host = knownEndpoint.EndpointDetails.Host, + Monitored = false + }) + .DistinctBy(endpoint => endpoint.Id)]; + + // Group rows are replaced wholesale on every attempt. When two concurrent writers process the + // same message their delete/insert pairs can interleave into a transient union of both + // attempts' groups; that is accepted, the next attempt replaces it. + async Task ReplaceGroups(List failedMessages, List groups, CancellationToken cancellationToken) + { + var messageIds = failedMessages.Select(message => message.UniqueMessageId).ToArray(); + + await dbContext.FailedMessageGroups + .Where(group => messageIds.Contains(group.FailedMessageUniqueId)) + .ExecuteDeleteAsync(cancellationToken); + + if (groups.Count > 0) + { + await dialect.InsertGroups(dbContext, groups, cancellationToken); + } + } + + async Task ResolveRetried(Guid[] retries, DateTime now, CancellationToken cancellationToken) + { + await dbContext.FailedMessages + .Where(failedMessage => retries.Contains(failedMessage.UniqueMessageId)) + .ExecuteUpdateAsync(setters => setters + .SetProperty(failedMessage => failedMessage.Status, FailedMessageStatus.Resolved) + .SetProperty(failedMessage => failedMessage.StatusChangedAt, now) + .SetProperty(failedMessage => failedMessage.LastModified, now), cancellationToken); + + await dbContext.FailedMessageRetries + .Where(retry => retries.Contains(retry.UniqueMessageId)) + .ExecuteDeleteAsync(cancellationToken); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/HeadersJsonContext.cs b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/HeadersJsonContext.cs new file mode 100644 index 0000000000..c10617e8fe --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/HeadersJsonContext.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; + +using System.Text.Json.Serialization; + +// Source generated serialization for the failed message's headers, which are stored verbatim as +// the HeadersJson column. Avoids the reflection-based serializer on the ingestion hot path. +[JsonSerializable(typeof(Dictionary))] +partial class HeadersJsonContext : JsonSerializerContext; diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs new file mode 100644 index 0000000000..48dc630ac2 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs @@ -0,0 +1,75 @@ +namespace ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; + +using System.Text; +using ServiceControl.Persistence.Infrastructure; + +// Bodies are always stored. MaxBodySizeToStore only decides where: inline in BodyText, or in +// external storage with at most a search prefix left inline. +static class MessageBodyClassifier +{ + public static (string? BodyText, bool StoreExternally) Classify(IReadOnlyDictionary headers, ReadOnlyMemory body, int maxBodySizeToStore) + { + if (body.IsEmpty) + { + return (null, false); + } + + if (headers.IsBinary()) + { + return (null, true); + } + + var span = body.Span; + var overCap = span.Length > maxBodySizeToStore; + var slice = overCap ? span[..Utf8SafeLength(span[..maxBodySizeToStore])] : span; + + string bodyText; + try + { + bodyText = strictUtf8.GetString(slice); + } + catch (DecoderFallbackException) + { + return (null, true); + } + + // NUL bytes decode fine but mean the payload is not really text, and neither database + // stores them in a text column without complaint. + if (bodyText.Contains('\0')) + { + return (null, true); + } + + return (bodyText, overCap); + } + + // Cutting at an arbitrary byte can split a multi byte character, which strict decoding then + // rejects. Walk back to the start of the last character and drop it if it is incomplete. + static int Utf8SafeLength(ReadOnlySpan slice) + { + var leadIndex = slice.Length - 1; + while (leadIndex >= 0 && (slice[leadIndex] & 0xC0) == 0x80) + { + leadIndex--; + } + + if (leadIndex < 0) + { + return 0; + } + + var lead = slice[leadIndex]; + var sequenceLength = lead switch + { + < 0x80 => 1, + >= 0xF0 => 4, + >= 0xE0 => 3, + >= 0xC0 => 2, + _ => 1 // A stray continuation byte, strict decoding rejects it either way + }; + + return slice.Length - leadIndex >= sequenceLength ? slice.Length : leadIndex; + } + + static readonly UTF8Encoding strictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); +} diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/RecordedFailedProcessingAttempt.cs b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/RecordedFailedProcessingAttempt.cs new file mode 100644 index 0000000000..82b6c047f4 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/RecordedFailedProcessingAttempt.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; + +using ServiceControl.MessageFailures; + +sealed class RecordedFailedProcessingAttempt +{ + public required Guid UniqueMessageId { get; init; } + public required DateTime AttemptedAt { get; init; } + public required DateTime TimeOfFailure { get; init; } + public required IReadOnlyList Groups { get; init; } + public required string HeadersJson { get; init; } + + public string? MessageId { get; init; } + public string? MessageType { get; init; } + public DateTime? TimeSent { get; init; } + public string? ConversationId { get; init; } + public string? QueueAddress { get; init; } + public string? SendingEndpointName { get; init; } + public Guid? SendingEndpointHostId { get; init; } + public string? SendingEndpointHost { get; init; } + public string? ReceivingEndpointName { get; init; } + public Guid? ReceivingEndpointHostId { get; init; } + public string? ReceivingEndpointHost { get; init; } + public string? ExceptionType { get; init; } + public string? ExceptionMessage { get; init; } + public bool IsSystemMessage { get; init; } + + public string? BodyText { get; init; } + public bool BodyStoredExternally { get; init; } + public int BodySize { get; init; } + public string? BodyContentType { get; init; } +} diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IBodyStoragePersistence.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IBodyStoragePersistence.cs index b4ef500909..c089275bc1 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/IBodyStoragePersistence.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IBodyStoragePersistence.cs @@ -4,9 +4,11 @@ namespace ServiceControl.Persistence.EFCore.Infrastructure; using System.Threading; using System.Threading.Tasks; +// Bodies are immutable and addressed by bodyId (the UniqueMessageId) alone, so re-failures of the +// same message resolve to the same stored body and writes can be skipped when it already exists. public interface IBodyStoragePersistence { - Task WriteBody(string bodyId, DateTime createdOn, ReadOnlyMemory body, string contentType, CancellationToken cancellationToken = default); - Task ReadBody(string bodyId, DateTime createdOn, CancellationToken cancellationToken = default); + Task WriteBody(string bodyId, ReadOnlyMemory body, string contentType, CancellationToken cancellationToken = default); + Task ReadBody(string bodyId, CancellationToken cancellationToken = default); Task DeleteBody(string bodyId, CancellationToken cancellationToken = default); } diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IIngestionSqlDialect.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IIngestionSqlDialect.cs new file mode 100644 index 0000000000..3e24d92faa --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IIngestionSqlDialect.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Persistence.EFCore.Infrastructure; + +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +/// +/// The provider specific SQL of the error ingestion batch. Implementations run on the DbContext +/// connection inside the transaction the caller has already opened, and every statement must stay +/// correct under concurrent writers: a same-key race between two instances may not fail the batch. +/// +public interface IIngestionSqlDialect +{ + /// + /// One row per message, distinct by UniqueMessageId. Inserts new rows; for existing rows the + /// attempt counter advances (unless the batch merely redelivered the attempt already stored), + /// the failure window widens, and the newer attempt supplies every payload column. + /// + Task UpsertFailedMessages(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken); + + /// + /// Insert if absent. The caller has already deleted the batch's messages' group rows in the + /// same transaction; if-absent keeps a concurrent writer's identical row from failing us. + /// + Task InsertGroups(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken); + + /// + /// Insert if absent, never update: existing endpoints keep their Monitored flag. + /// + Task InsertMissingKnownEndpoints(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken); +} diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/ServiceControl.Persistence.Tests.PostgreSql.csproj b/src/ServiceControl.Persistence.Tests.PostgreSql/ServiceControl.Persistence.Tests.PostgreSql.csproj index c2d8b611a2..59de3364fa 100644 --- a/src/ServiceControl.Persistence.Tests.PostgreSql/ServiceControl.Persistence.Tests.PostgreSql.csproj +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/ServiceControl.Persistence.Tests.PostgreSql.csproj @@ -33,15 +33,12 @@ - - - diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/ServiceControl.Persistence.Tests.RavenDB.csproj b/src/ServiceControl.Persistence.Tests.RavenDB/ServiceControl.Persistence.Tests.RavenDB.csproj index 5631e1959c..85d3b51c65 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/ServiceControl.Persistence.Tests.RavenDB.csproj +++ b/src/ServiceControl.Persistence.Tests.RavenDB/ServiceControl.Persistence.Tests.RavenDB.csproj @@ -25,6 +25,9 @@ + + + diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/ServiceControl.Persistence.Tests.SqlServer.csproj b/src/ServiceControl.Persistence.Tests.SqlServer/ServiceControl.Persistence.Tests.SqlServer.csproj index bfd404dd6b..186518974d 100644 --- a/src/ServiceControl.Persistence.Tests.SqlServer/ServiceControl.Persistence.Tests.SqlServer.csproj +++ b/src/ServiceControl.Persistence.Tests.SqlServer/ServiceControl.Persistence.Tests.SqlServer.csproj @@ -33,15 +33,12 @@ - - - diff --git a/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionBodyTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionBodyTests.cs new file mode 100644 index 0000000000..d2bd359688 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionBodyTests.cs @@ -0,0 +1,143 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; + +// Bodies are always stored. MaxBodySizeToStore only decides whether the body lives inline in +// BodyText or in external storage with at most a search prefix left inline. +class ErrorIngestionBodyTests : ErrorIngestionTestBase +{ + const int Cap = 64; + + [SetUp] + public void ShrinkTheBodyCap() => EFSettings.MaxBodySizeToStore = Cap; + + [Test] + public async Task Text_within_the_cap_is_stored_inline() + { + var failure = new IngestedFailure { Body = Encoding.UTF8.GetBytes("1") }; + + await Ingest(failure); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.BodyText, Is.EqualTo("1")); + Assert.That(row.BodyStoredExternally, Is.False); + Assert.That(row.BodySize, Is.EqualTo(failure.Body.Length)); + } + + Assert.That(RecordedBodies.Written, Is.Empty); + } + + [Test] + public async Task Text_over_the_cap_goes_external_and_leaves_a_search_prefix() + { + // Two byte characters, so the cap falls in the middle of one + var body = Encoding.UTF8.GetBytes(new string('é', Cap)); + var failure = new IngestedFailure { Body = body }; + + await Ingest(failure); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + Assert.That(row.BodyStoredExternally, Is.True); + Assert.That(row.BodyText, Is.Not.Null); + + var prefixBytes = Encoding.UTF8.GetByteCount(row.BodyText); + + using (Assert.EnterMultipleScope()) + { + Assert.That(prefixBytes, Is.LessThanOrEqualTo(Cap), "The prefix must not exceed the cap"); + Assert.That(row.BodyText, Is.EqualTo(new string('é', Cap / 2)), "The prefix must end on a character boundary"); + Assert.That(row.BodySize, Is.EqualTo(body.Length), "BodySize is the original size"); + } + + var written = RecordedBodies.Written.Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(written.BodyId, Is.EqualTo(failure.UniqueMessageIdString)); + Assert.That(written.Body, Is.EqualTo(body), "External storage holds the whole body"); + Assert.That(written.ContentType, Is.EqualTo(failure.ContentType)); + } + } + + [Test] + public async Task A_binary_body_goes_external_whatever_its_size() + { + var failure = new IngestedFailure + { + ContentType = "application/octet-stream", + Body = BitConverter.GetBytes(0xDEADBEEF) + }; + + await Ingest(failure); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.BodyText, Is.Null); + Assert.That(row.BodyStoredExternally, Is.True); + Assert.That(row.BodySize, Is.EqualTo(failure.Body.Length)); + } + + Assert.That(RecordedBodies.Written.Single().Body, Is.EqualTo(failure.Body)); + } + + [Test] + public async Task A_body_that_is_not_valid_utf8_goes_external() + { + var failure = new IngestedFailure { Body = [0xFF, 0xFE, 0xFD] }; + + await Ingest(failure); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.BodyText, Is.Null); + Assert.That(row.BodyStoredExternally, Is.True); + } + } + + [Test] + public async Task A_body_containing_a_nul_goes_external() + { + var failure = new IngestedFailure { Body = Encoding.UTF8.GetBytes("abc\0def") }; + + await Ingest(failure); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.BodyText, Is.Null); + Assert.That(row.BodyStoredExternally, Is.True); + } + } + + [Test] + public async Task An_empty_body_is_stored_nowhere() + { + var failure = new IngestedFailure { Body = [] }; + + await Ingest(failure); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.BodyText, Is.Null); + Assert.That(row.BodyStoredExternally, Is.False); + Assert.That(row.BodySize, Is.Zero); + } + + Assert.That(RecordedBodies.Written, Is.Empty); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionConcurrencyTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionConcurrencyTests.cs new file mode 100644 index 0000000000..c265140b4d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionConcurrencyTests.cs @@ -0,0 +1,68 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Operations; + +class ErrorIngestionConcurrencyTests : ErrorIngestionTestBase +{ + [Test] + public async Task Concurrent_writers_on_the_same_messages_neither_collide_nor_lose_attempts() + { + const int writers = 4; + const int messages = 25; + + var seeds = Enumerable.Range(0, messages).Select(_ => new IngestedFailure()).ToArray(); + var baseTime = seeds[0].AttemptedAt; + + await Task.WhenAll(Enumerable.Range(0, writers).Select(writer => Task.Run(async () => + { + await using var unitOfWork = await UnitOfWorkFactory.StartNew(); + + foreach (var seed in seeds) + { + var attempt = seed.NextAttempt(baseTime.AddMinutes(writer)); + await unitOfWork.Recoverability.RecordFailedProcessingAttempt(attempt.Context, attempt.ProcessingAttempt, attempt.Groups); + } + + await unitOfWork.Complete(TestContext.CurrentContext.CancellationToken); + }))); + + foreach (var seed in seeds) + { + var row = await GetFailedMessage(seed.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(writers), "every writer's distinct attempt must be counted exactly once"); + Assert.That(row.LastAttemptedAt, Is.EqualTo(baseTime.AddMinutes(writers - 1)), "the newest attempt must win regardless of commit order"); + } + } + } + + [Test] + public async Task Concurrent_writers_recording_the_same_endpoint_insert_it_once() + { + const int writers = 8; + + var endpoint = new EndpointDetails { Name = $"Endpoint-{Guid.NewGuid():N}", HostId = Guid.NewGuid(), Host = "Host1" }; + + await Task.WhenAll(Enumerable.Range(0, writers).Select(_ => Task.Run(async () => + { + await using var unitOfWork = await UnitOfWorkFactory.StartNew(); + + await unitOfWork.Monitoring.RecordKnownEndpoint(new KnownEndpoint + { + EndpointDetails = endpoint, + HostDisplayName = endpoint.Host, + Monitored = false + }); + + await unitOfWork.Complete(TestContext.CurrentContext.CancellationToken); + }))); + + Assert.That(await GetKnownEndpoints([endpoint.GetDeterministicId()]), Has.Count.EqualTo(1)); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionKnownEndpointTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionKnownEndpointTests.cs new file mode 100644 index 0000000000..2d9d39db68 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionKnownEndpointTests.cs @@ -0,0 +1,88 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.EFCore.Entities; + +class ErrorIngestionKnownEndpointTests : ErrorIngestionTestBase +{ + [Test] + public async Task An_unknown_endpoint_is_inserted_unmonitored() + { + var endpoint = NewEndpoint(); + + await InBatch(unitOfWork => unitOfWork.Monitoring.RecordKnownEndpoint(new KnownEndpoint + { + EndpointDetails = endpoint, + HostDisplayName = endpoint.Host, + Monitored = false + })); + + var stored = (await GetKnownEndpoints([endpoint.GetDeterministicId()])).Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.Name, Is.EqualTo(endpoint.Name)); + Assert.That(stored.HostId, Is.EqualTo(endpoint.HostId)); + Assert.That(stored.Host, Is.EqualTo(endpoint.Host)); + Assert.That(stored.Monitored, Is.False, "Endpoints discovered during ingestion are not monitored"); + } + } + + [Test] + public async Task A_known_endpoint_keeps_its_monitored_flag() + { + var endpoint = NewEndpoint(); + + await Store(new KnownEndpointEntity + { + Id = endpoint.GetDeterministicId(), + Name = endpoint.Name, + HostId = endpoint.HostId, + Host = endpoint.Host, + Monitored = true + }); + + await InBatch(unitOfWork => unitOfWork.Monitoring.RecordKnownEndpoint(new KnownEndpoint + { + EndpointDetails = endpoint, + HostDisplayName = endpoint.Host, + Monitored = false + })); + + var stored = (await GetKnownEndpoints([endpoint.GetDeterministicId()])).Single(); + + Assert.That(stored.Monitored, Is.True, "Ingestion must never overwrite an existing endpoint"); + } + + [Test] + public async Task The_same_endpoint_twice_in_one_batch_is_inserted_once() + { + var endpoint = NewEndpoint(); + + await InBatch(async unitOfWork => + { + for (var i = 0; i < 2; i++) + { + await unitOfWork.Monitoring.RecordKnownEndpoint(new KnownEndpoint + { + EndpointDetails = endpoint, + HostDisplayName = endpoint.Host, + Monitored = false + }); + } + }); + + Assert.That(await GetKnownEndpoints([endpoint.GetDeterministicId()]), Has.Count.EqualTo(1)); + } + + static EndpointDetails NewEndpoint() => new() + { + Name = $"Endpoint-{Guid.NewGuid():N}", + HostId = Guid.NewGuid(), + Host = "Host1" + }; +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTestBase.cs b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTestBase.cs new file mode 100644 index 0000000000..5bc44fc378 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTestBase.cs @@ -0,0 +1,130 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Persistence.UnitOfWork; + +abstract class ErrorIngestionTestBase : PersistenceTestBase +{ + protected ErrorIngestionTestBase() => + RegisterServices = services => services.AddSingleton(RecordedBodies); + + protected RecordingBodyStoragePersistence RecordedBodies { get; } = new(); + + protected EFPersisterSettings EFSettings => (EFPersisterSettings)PersistenceSettings; + + protected void AdvanceClock(TimeSpan by) => PersistenceTestsContext.FakeTime.Advance(by); + + protected async Task InBatch(Func record) + { + await using var unitOfWork = await UnitOfWorkFactory.StartNew(); + + await record(unitOfWork); + + await unitOfWork.Complete(TestContext.CurrentContext.CancellationToken); + } + + protected Task Ingest(params IngestedFailure[] failures) => + InBatch(async unitOfWork => + { + foreach (var failure in failures) + { + await unitOfWork.Recoverability.RecordFailedProcessingAttempt(failure.Context, failure.ProcessingAttempt, failure.Groups); + } + }); + + protected Task ConfirmRetry(params string[] uniqueMessageIds) => + InBatch(async unitOfWork => + { + foreach (var uniqueMessageId in uniqueMessageIds) + { + await unitOfWork.Recoverability.RecordSuccessfulRetry(uniqueMessageId); + } + }); + + protected async Task GetFailedMessage(Guid uniqueMessageId) + { + var row = await Query(dbContext => dbContext.FailedMessages.AsNoTracking().SingleOrDefaultAsync(m => m.UniqueMessageId == uniqueMessageId)); + + Assert.That(row, Is.Not.Null, $"No failed message row for {uniqueMessageId}"); + + return row; + } + + protected Task FindFailedMessage(Guid uniqueMessageId) => + Query(dbContext => dbContext.FailedMessages.AsNoTracking().SingleOrDefaultAsync(m => m.UniqueMessageId == uniqueMessageId)); + + protected Task> GetGroups(Guid uniqueMessageId) => + Query(dbContext => dbContext.FailedMessageGroups + .AsNoTracking() + .Where(g => g.FailedMessageUniqueId == uniqueMessageId) + .ToListAsync()); + + protected Task> GetKnownEndpoints(IReadOnlyCollection ids) => + Query(dbContext => dbContext.KnownEndpoints.AsNoTracking().Where(e => ids.Contains(e.Id)).ToListAsync()); + + protected async Task Store(params T[] entities) where T : class + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + dbContext.Set().AddRange(entities); + + await dbContext.SaveChangesAsync(TestContext.CurrentContext.CancellationToken); + } + + protected Task CountRetryRows(Guid uniqueMessageId) => + Query(dbContext => dbContext.FailedMessageRetries.AsNoTracking().CountAsync(r => r.UniqueMessageId == uniqueMessageId)); + + async Task Query(Func> query) + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + return await query(dbContext); + } + + protected class RecordingBodyStoragePersistence : IBodyStoragePersistence + { + readonly List written = []; + + public IReadOnlyList Written + { + get + { + lock (written) + { + return [.. written]; + } + } + } + + public Task WriteBody(string bodyId, ReadOnlyMemory body, string contentType, CancellationToken cancellationToken = default) + { + lock (written) + { + written.Add(new StoredBody(bodyId, body.ToArray(), contentType)); + } + + return Task.CompletedTask; + } + + public Task ReadBody(string bodyId, CancellationToken cancellationToken = default) => + throw new NotImplementedException(); + + public Task DeleteBody(string bodyId, CancellationToken cancellationToken = default) => + throw new NotImplementedException(); + + public record StoredBody(string BodyId, byte[] Body, string ContentType); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTests.cs new file mode 100644 index 0000000000..7a4529c1f6 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTests.cs @@ -0,0 +1,290 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.Entities; + +class ErrorIngestionTests : ErrorIngestionTestBase +{ + [Test] + public async Task First_failure_stores_the_message() + { + var failure = new IngestedFailure(); + + await Ingest(failure); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.Status, Is.EqualTo(FailedMessageStatus.Unresolved)); + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(1)); + Assert.That(row.LastAttemptedAt, Is.EqualTo(failure.AttemptedAt)); + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(failure.TimeOfFailure)); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(failure.TimeOfFailure)); + Assert.That(row.MessageId, Is.EqualTo(failure.MessageId)); + Assert.That(row.MessageType, Is.EqualTo(failure.MessageType)); + Assert.That(row.TimeSent, Is.EqualTo(failure.TimeSent)); + Assert.That(row.ConversationId, Is.EqualTo(failure.ConversationId)); + Assert.That(row.QueueAddress, Is.EqualTo(failure.QueueAddress)); + Assert.That(row.SendingEndpointName, Is.EqualTo(failure.SendingEndpoint.Name)); + Assert.That(row.SendingEndpointHostId, Is.EqualTo(failure.SendingEndpoint.HostId)); + Assert.That(row.SendingEndpointHost, Is.EqualTo(failure.SendingEndpoint.Host)); + Assert.That(row.ReceivingEndpointName, Is.EqualTo(failure.ReceivingEndpoint.Name)); + Assert.That(row.ReceivingEndpointHostId, Is.EqualTo(failure.ReceivingEndpoint.HostId)); + Assert.That(row.ReceivingEndpointHost, Is.EqualTo(failure.ReceivingEndpoint.Host)); + Assert.That(row.ExceptionType, Is.EqualTo(failure.ExceptionType)); + Assert.That(row.ExceptionMessage, Is.EqualTo(failure.ExceptionMessage)); + Assert.That(row.IsSystemMessage, Is.False); + Assert.That(row.HeadersJson, Does.Contain(failure.MessageId)); + Assert.That(row.BodyText, Is.EqualTo(Encoding.UTF8.GetString(failure.Body))); + Assert.That(row.BodyStoredExternally, Is.False); + Assert.That(row.BodySize, Is.EqualTo(failure.Body.Length)); + Assert.That(row.BodyContentType, Is.EqualTo(failure.ContentType)); + Assert.That(row.StatusChangedAt, Is.EqualTo(row.LastModified)); + } + + var groups = await GetGroups(failure.UniqueMessageId); + + Assert.That(groups, Has.Count.EqualTo(1)); + using (Assert.EnterMultipleScope()) + { + Assert.That(groups[0].GroupId, Is.EqualTo(failure.Groups[0].Id)); + Assert.That(groups[0].Title, Is.EqualTo(failure.Groups[0].Title)); + Assert.That(groups[0].Type, Is.EqualTo(failure.Groups[0].Type)); + } + } + + [Test] + public async Task Later_attempt_replaces_the_stored_attempt() + { + var first = new IngestedFailure(); + await Ingest(first); + + var second = new IngestedFailure + { + MessageId = first.MessageId, + EndpointName = first.EndpointName, + AttemptedAt = first.AttemptedAt.AddMinutes(5), + TimeOfFailure = first.TimeOfFailure.AddMinutes(5), + ExceptionMessage = "A different failure", + Body = Encoding.UTF8.GetBytes("2"), + Groups = [new FailedMessage.FailureGroup { Id = Guid.NewGuid().ToString(), Title = "Another group", Type = "Exception Type and Stack Trace" }] + }; + await Ingest(second); + + var row = await GetFailedMessage(first.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(2)); + Assert.That(row.LastAttemptedAt, Is.EqualTo(second.AttemptedAt)); + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(first.TimeOfFailure)); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(second.TimeOfFailure)); + Assert.That(row.ExceptionMessage, Is.EqualTo(second.ExceptionMessage)); + Assert.That(row.BodyText, Is.EqualTo("2")); + } + + var groups = await GetGroups(first.UniqueMessageId); + + Assert.That(groups, Has.Count.EqualTo(1), "Groups are replaced wholesale"); + Assert.That(groups[0].GroupId, Is.EqualTo(second.Groups[0].Id)); + } + + [Test] + public async Task Refailure_of_a_resolved_message_restarts_the_retention_clock() + { + var failure = new IngestedFailure(); + await Ingest(failure); + await ConfirmRetry(failure.UniqueMessageIdString); + + var resolved = await GetFailedMessage(failure.UniqueMessageId); + Assert.That(resolved.Status, Is.EqualTo(FailedMessageStatus.Resolved)); + + AdvanceClock(TimeSpan.FromMinutes(1)); + + await Ingest(failure.NextAttempt(failure.AttemptedAt.AddMinutes(1))); + + var refailed = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(refailed.Status, Is.EqualTo(FailedMessageStatus.Unresolved)); + Assert.That(refailed.StatusChangedAt, Is.GreaterThan(resolved.StatusChangedAt)); + } + } + + [Test] + public async Task Refailure_of_an_unresolved_message_keeps_the_retention_clock() + { + var failure = new IngestedFailure(); + await Ingest(failure); + + var first = await GetFailedMessage(failure.UniqueMessageId); + + AdvanceClock(TimeSpan.FromMinutes(1)); + + await Ingest(failure.NextAttempt(failure.AttemptedAt.AddMinutes(1))); + + var second = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(second.Status, Is.EqualTo(FailedMessageStatus.Unresolved)); + Assert.That(second.StatusChangedAt, Is.EqualTo(first.StatusChangedAt)); + Assert.That(second.LastModified, Is.GreaterThan(first.LastModified)); + } + } + + [Test] + public async Task Redelivery_of_a_stored_attempt_is_not_a_new_attempt() + { + var failure = new IngestedFailure(); + + await Ingest(failure); + await Ingest(failure); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(1)); + } + + [Test] + public async Task Two_attempts_of_one_message_in_one_batch_fold_into_one_row() + { + var first = new IngestedFailure(); + var second = first.NextAttempt(first.AttemptedAt.AddMinutes(5)); + + await Ingest(first, second); + + var row = await GetFailedMessage(first.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(2)); + Assert.That(row.LastAttemptedAt, Is.EqualTo(second.AttemptedAt)); + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(first.TimeOfFailure)); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(second.TimeOfFailure)); + } + } + + [Test] + public async Task Duplicate_attempt_within_one_batch_counts_once() + { + var failure = new IngestedFailure(); + + await Ingest(failure, failure); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(1)); + } + + [Test] + public async Task An_older_attempt_counts_but_does_not_overwrite_the_newer_one() + { + var newer = new IngestedFailure { ExceptionMessage = "The newer failure" }; + await Ingest(newer); + + var older = new IngestedFailure + { + MessageId = newer.MessageId, + EndpointName = newer.EndpointName, + AttemptedAt = newer.AttemptedAt.AddMinutes(-5), + TimeOfFailure = newer.TimeOfFailure.AddMinutes(-5), + ExceptionMessage = "The older failure" + }; + await Ingest(older); + + var row = await GetFailedMessage(newer.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(2)); + Assert.That(row.LastAttemptedAt, Is.EqualTo(newer.AttemptedAt)); + Assert.That(row.ExceptionMessage, Is.EqualTo(newer.ExceptionMessage)); + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(older.TimeOfFailure)); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(newer.TimeOfFailure)); + } + } + + [Test] + public async Task A_confirmed_retry_resolves_the_message_and_drops_its_retry_row() + { + var failure = new IngestedFailure(); + await Ingest(failure); + await Store(new FailedMessageRetryEntity { UniqueMessageId = failure.UniqueMessageId, RetryId = "RetryBatches/1" }); + + await ConfirmRetry(failure.UniqueMessageIdString); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.Status, Is.EqualTo(FailedMessageStatus.Resolved)); + Assert.That(row.StatusChangedAt, Is.EqualTo(row.LastModified)); + } + + Assert.That(await CountRetryRows(failure.UniqueMessageId), Is.Zero); + } + + [Test] + public async Task A_failure_confirmed_in_the_same_batch_ends_resolved() + { + var failure = new IngestedFailure(); + + await InBatch(async unitOfWork => + { + await unitOfWork.Recoverability.RecordFailedProcessingAttempt(failure.Context, failure.ProcessingAttempt, failure.Groups); + await unitOfWork.Recoverability.RecordSuccessfulRetry(failure.UniqueMessageIdString); + }); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + Assert.That(row.Status, Is.EqualTo(FailedMessageStatus.Resolved)); + } + + [Test] + public async Task A_batch_can_hold_more_messages_than_a_statement_can_hold_parameters() + { + // SQL Server allows 2100 parameters per statement, which a batch this size would blow + // through several times over if the rows were parameterized instead of bulk copied. + var failures = Enumerable.Range(0, 250).Select(_ => new IngestedFailure()).ToArray(); + + await Ingest(failures); + + foreach (var failure in failures) + { + var row = await GetFailedMessage(failure.UniqueMessageId); + + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(1)); + } + } + + [Test] + public async Task Concurrent_recording_stores_every_message() + { + var failures = Enumerable.Range(0, 32).Select(_ => new IngestedFailure()).ToArray(); + + await InBatch(async unitOfWork => + { + // Mirrors ErrorProcessor, which records the whole batch through Task.WhenAll + await Task.WhenAll(failures.Select(failure => Task.Run(() => + unitOfWork.Recoverability.RecordFailedProcessingAttempt(failure.Context, failure.ProcessingAttempt, failure.Groups)))); + }); + + var stored = new List(); + foreach (var failure in failures) + { + stored.Add(await GetFailedMessage(failure.UniqueMessageId)); + } + + Assert.That(stored.Select(row => row.UniqueMessageId), Is.Unique); + Assert.That(stored, Has.Count.EqualTo(failures.Length)); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/ExecuteUpdateDeleteTransactionProbe.cs b/src/ServiceControl.Persistence.Tests/EFCore/ExecuteUpdateDeleteTransactionProbe.cs new file mode 100644 index 0000000000..011c885903 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/ExecuteUpdateDeleteTransactionProbe.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.DbContexts; + +// Verifies the assumption the batch writer's atomicity rests on: ExecuteUpdate/ExecuteDelete run +// inside the ambient transaction, so a rollback undoes them. +class ExecuteUpdateDeleteTransactionProbe : ErrorIngestionTestBase +{ + [Test] + public async Task ExecuteDelete_and_ExecuteUpdate_are_undone_by_a_rollback() + { + var failure = new IngestedFailure(); + await Ingest(failure); + + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var strategy = dbContext.Database.CreateExecutionStrategy(); + await strategy.ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + + await dbContext.FailedMessages + .Where(m => m.UniqueMessageId == failure.UniqueMessageId) + .ExecuteUpdateAsync(setters => setters.SetProperty(m => m.ExceptionMessage, "changed in the transaction")); + + await dbContext.FailedMessages + .Where(m => m.UniqueMessageId == failure.UniqueMessageId) + .ExecuteDeleteAsync(); + + await transaction.RollbackAsync(); + }); + + var row = await FindFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row, Is.Not.Null, "ExecuteDelete must enlist in the transaction, so a rollback keeps the row"); + Assert.That(row!.ExceptionMessage, Is.EqualTo(failure.ExceptionMessage), "ExecuteUpdate must enlist too, so a rollback reverts the change"); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/IngestedFailure.cs b/src/ServiceControl.Persistence.Tests/EFCore/IngestedFailure.cs new file mode 100644 index 0000000000..fa95d4d694 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/IngestedFailure.cs @@ -0,0 +1,99 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Text; +using NServiceBus.Extensibility; +using NServiceBus.Transport; +using ServiceControl.Contracts.Operations; +using ServiceControl.MessageFailures; +using ServiceControl.Operations; +using ServiceControl.Persistence.Infrastructure; + +class IngestedFailure +{ + public string MessageId { get; init; } = Guid.NewGuid().ToString(); + public string EndpointName { get; init; } = "Sales"; + public string ContentType { get; init; } = "text/xml"; + public byte[] Body { get; init; } = Encoding.UTF8.GetBytes("1"); + public DateTime AttemptedAt { get; init; } = new(2026, 7, 22, 10, 0, 0, DateTimeKind.Utc); + public DateTime TimeOfFailure { get; init; } = new(2026, 7, 22, 10, 0, 0, DateTimeKind.Utc); + public DateTime TimeSent { get; init; } = new(2026, 7, 22, 9, 59, 0, DateTimeKind.Utc); + public string MessageType { get; init; } = "MyCompany.Sales.OrderPlaced"; + public string ConversationId { get; init; } = Guid.NewGuid().ToString(); + public string QueueAddress { get; init; } = "error"; + public string ExceptionType { get; init; } = "System.InvalidOperationException"; + public string ExceptionMessage { get; init; } = "Something went wrong"; + public bool IsSystemMessage { get; init; } + public EndpointDetails SendingEndpoint { get; init; } = new() { Name = "Ordering", Host = "SenderHost", HostId = Guid.NewGuid() }; + public EndpointDetails ReceivingEndpoint { get; init; } = new() { Name = "Sales", Host = "ReceiverHost", HostId = Guid.NewGuid() }; + public List Groups { get; init; } = + [ + new() { Id = Guid.NewGuid().ToString(), Title = "OrderPlaced", Type = "Message Type" } + ]; + + public Dictionary Headers => field ??= new Dictionary + { + [NServiceBus.Headers.MessageId] = MessageId, + [NServiceBus.Headers.ProcessingEndpoint] = EndpointName, + [NServiceBus.Headers.ContentType] = ContentType, + [NServiceBus.Headers.EnclosedMessageTypes] = MessageType, + ["NServiceBus.FailedQ"] = QueueAddress, + ["NServiceBus.ExceptionInfo.ExceptionType"] = ExceptionType, + ["NServiceBus.ExceptionInfo.Message"] = ExceptionMessage + }; + + public string UniqueMessageIdString => Headers.UniqueId(); + + public Guid UniqueMessageId => Guid.Parse(UniqueMessageIdString); + + public MessageContext Context => + new(MessageId, Headers, Body, new TransportTransaction(), "receiveAddress", new ContextBag()); + + public FailedMessage.ProcessingAttempt ProcessingAttempt => new() + { + AttemptedAt = AttemptedAt, + MessageId = MessageId, + Headers = Headers, + MessageMetadata = new Dictionary + { + ["MessageId"] = MessageId, + ["MessageType"] = MessageType, + ["TimeSent"] = TimeSent, + ["ConversationId"] = ConversationId, + ["IsSystemMessage"] = IsSystemMessage, + ["SendingEndpoint"] = SendingEndpoint, + ["ReceivingEndpoint"] = ReceivingEndpoint + }, + FailureDetails = new FailureDetails + { + TimeOfFailure = TimeOfFailure, + AddressOfFailingEndpoint = QueueAddress, + Exception = new ExceptionDetails + { + ExceptionType = ExceptionType, + Message = ExceptionMessage + } + } + }; + + public IngestedFailure NextAttempt(DateTime attemptedAt) => new() + { + MessageId = MessageId, + EndpointName = EndpointName, + AttemptedAt = attemptedAt, + TimeOfFailure = attemptedAt, + ContentType = ContentType, + Body = Body, + MessageType = MessageType, + ConversationId = ConversationId, + QueueAddress = QueueAddress, + ExceptionType = ExceptionType, + ExceptionMessage = ExceptionMessage, + IsSystemMessage = IsSystemMessage, + SendingEndpoint = SendingEndpoint, + ReceivingEndpoint = ReceivingEndpoint, + TimeSent = TimeSent, + Groups = Groups + }; +} From ac025258a7c00a573421b2d675996d33c0374ac9 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 23 Jul 2026 11:53:18 +1000 Subject: [PATCH 3/5] Implement background retention sweep for failed messages Adds a `RetentionSweeper` background service to automatically delete resolved and archived failed messages that have exceeded the configured retention period. The sweeper runs periodically in batches, handling external message bodies and tolerating body deletion failures. --- .../Abstractions/BasePersistence.cs | 3 + .../EditFailedMessagesManager.cs | 1 + .../Implementation/ErrorMessagesDataStore.cs | 5 + .../Implementation/MessageArchiver.cs | 2 + .../Infrastructure/RetentionSweeper.cs | 126 ++++++++++++++++ .../EFCore/ErrorIngestionTestBase.cs | 35 ++++- .../EFCore/RetentionSweepTests.cs | 142 ++++++++++++++++++ 7 files changed, 312 insertions(+), 2 deletions(-) create mode 100644 src/ServiceControl.Persistence.EFCore/Infrastructure/RetentionSweeper.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs diff --git a/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs b/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs index 8a6642c313..c6916fe4ad 100644 --- a/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs +++ b/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs @@ -32,6 +32,9 @@ protected static void RegisterDataStores(IServiceCollection services) services.AddSingleton(p => p.GetRequiredService()); services.AddHostedService(p => p.GetRequiredService()); + services.AddSingleton(); + services.AddHostedService(p => p.GetRequiredService()); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EditFailedMessagesManager.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EditFailedMessagesManager.cs index aa936dd6dc..5a80ec2287 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/EditFailedMessagesManager.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EditFailedMessagesManager.cs @@ -13,6 +13,7 @@ public Task GetCurrentEditingRequestId(string failedMessageId) => public Task SetCurrentEditingRequestId(string editingMessageId) => throw new NotImplementedException(); + // must set StatusChangedAt + LastModified public Task SetFailedMessageAsResolved() => throw new NotImplementedException(); diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/ErrorMessagesDataStore.cs index 2757c59658..c081aa0674 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/ErrorMessagesDataStore.cs @@ -25,6 +25,7 @@ public Task>> GetAllMessagesForSearch(string sea public Task>> SearchEndpointMessages(string endpointName, string searchKeyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null) => throw new NotImplementedException(); + // must set StatusChangedAt + LastModified public Task FailedMessageMarkAsArchived(string failedMessageId) => throw new NotImplementedException(); @@ -79,18 +80,22 @@ public Task GetGroupErrorsCount(string groupId, string status, s public Task>> GetGroup(string groupId, string status, string modified) => throw new NotImplementedException(); + // must set StatusChangedAt + LastModified public Task MarkMessageAsResolved(string failedMessageId) => throw new NotImplementedException(); public Task ProcessPendingRetries(DateTime periodFrom, DateTime periodTo, string queueAddress, Func processCallback) => throw new NotImplementedException(); + // must set StatusChangedAt + LastModified public Task UnArchiveMessagesByRange(DateTime from, DateTime to) => throw new NotImplementedException(); + // must set StatusChangedAt + LastModified public Task UnArchiveMessages(IEnumerable failedMessageIds) => throw new NotImplementedException(); + // must set StatusChangedAt + LastModified public Task RevertRetry(string messageUniqueId) => throw new NotImplementedException(); diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/MessageArchiver.cs b/src/ServiceControl.Persistence.EFCore/Implementation/MessageArchiver.cs index 9ec470e855..e56575df4d 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/MessageArchiver.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/MessageArchiver.cs @@ -6,9 +6,11 @@ namespace ServiceControl.Persistence.EFCore.Implementation; public class MessageArchiver : IArchiveMessages { + // must set StatusChangedAt + LastModified public Task ArchiveAllInGroup(string groupId, AuditUser? initiatedBy = null, string? operationId = null) => throw new NotImplementedException(); + // must set StatusChangedAt + LastModified public Task UnarchiveAllInGroup(string groupId, AuditUser? initiatedBy = null, string? operationId = null) => throw new NotImplementedException(); diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/RetentionSweeper.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/RetentionSweeper.cs new file mode 100644 index 0000000000..8f5d2d165d --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/RetentionSweeper.cs @@ -0,0 +1,126 @@ +namespace ServiceControl.Persistence.EFCore.Infrastructure; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +// Deletes resolved and archived failed messages once they age past the retention period. Runs +// hourly, in bounded batches so it never holds a large delete, and recomputes the cutoff on every +// run so a changed retention setting takes effect without rewriting any row. +public class RetentionSweeper( + ILogger logger, + TimeProvider timeProvider, + IServiceScopeFactory serviceScopeFactory, + IBodyStoragePersistence bodyStorage, + EFPersisterSettings settings) : BackgroundService +{ + const int BatchSize = 1000; + static readonly TimeSpan Interval = TimeSpan.FromHours(1); + static readonly TimeSpan InitialDelay = TimeSpan.FromMinutes(1); + static readonly TimeSpan BatchPause = TimeSpan.FromSeconds(1); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + logger.LogInformation("Starting error retention sweep"); + + try + { + await Task.Delay(InitialDelay, timeProvider, stoppingToken); + + using PeriodicTimer timer = new(Interval, timeProvider); + + do + { + try + { + await Sweep(pace: true, stoppingToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogError(ex, "Error during error retention sweep"); + } + } while (await timer.WaitForNextTickAsync(stoppingToken)); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + logger.LogInformation("Stopping error retention sweep"); + } + } + + // Runs a full sweep immediately, bypassing the timer and the inter-batch pause. + // Intended for tests that need the effect without waiting for the hourly loop. + public Task SweepNow(CancellationToken cancellationToken = default) => Sweep(pace: false, cancellationToken); + + async Task Sweep(bool pace, CancellationToken cancellationToken) + { + var cutoff = timeProvider.GetUtcNow().UtcDateTime - settings.ErrorRetentionPeriod; + + while (!cancellationToken.IsCancellationRequested) + { + using var scope = serviceScopeFactory.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var expired = await dbContext.FailedMessages + .AsNoTracking() + .Where(IsExpired(cutoff)) + .OrderBy(failedMessage => failedMessage.StatusChangedAt) + .Take(BatchSize) + .Select(failedMessage => new { failedMessage.UniqueMessageId, failedMessage.BodyStoredExternally }) + .ToListAsync(cancellationToken); + + if (expired.Count == 0) + { + break; + } + + // External bodies are deleted before the rows. A crash in between leaves rows the next + // sweep re-handles (tolerating the already-missing body); deleting rows first would + // instead leak the external bodies. + foreach (var row in expired.Where(row => row.BodyStoredExternally)) + { + await DeleteExternalBody(row.UniqueMessageId, cancellationToken); + } + + var ids = expired.Select(row => row.UniqueMessageId).ToArray(); + + // The predicate is re-asserted so a message that was re-failed (back to Unresolved) + // between the select and the delete is left alone. The cascade removes its group rows. + await dbContext.FailedMessages + .Where(failedMessage => ids.Contains(failedMessage.UniqueMessageId)) + .Where(IsExpired(cutoff)) + .ExecuteDeleteAsync(cancellationToken); + + if (expired.Count < BatchSize) + { + break; + } + + if (pace) + { + await Task.Delay(BatchPause, timeProvider, cancellationToken); + } + } + } + + async Task DeleteExternalBody(Guid uniqueMessageId, CancellationToken cancellationToken) + { + try + { + await bodyStorage.DeleteBody(uniqueMessageId.ToString(), cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + // Retention must not stall on a missing or unavailable body. + logger.LogWarning(ex, "Could not delete the external body for {UniqueMessageId} during retention", uniqueMessageId); + } + } + + static System.Linq.Expressions.Expression> IsExpired(DateTime cutoff) => + failedMessage => (failedMessage.Status == FailedMessageStatus.Resolved || failedMessage.Status == FailedMessageStatus.Archived) + && failedMessage.StatusChangedAt < cutoff; +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTestBase.cs b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTestBase.cs index 5bc44fc378..32bc18e917 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTestBase.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTestBase.cs @@ -7,6 +7,7 @@ namespace ServiceControl.Persistence.Tests; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using NUnit.Framework; using ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.Persistence.EFCore.DbContexts; @@ -86,6 +87,9 @@ protected async Task Store(params T[] entities) where T : class protected Task CountRetryRows(Guid uniqueMessageId) => Query(dbContext => dbContext.FailedMessageRetries.AsNoTracking().CountAsync(r => r.UniqueMessageId == uniqueMessageId)); + protected Task RunRetentionSweep() => + ServiceProvider.GetServices().OfType().Single().SweepNow(TestContext.CurrentContext.CancellationToken); + async Task Query(Func> query) { using var scope = ServiceProvider.CreateScope(); @@ -97,6 +101,10 @@ async Task Query(Func> query) protected class RecordingBodyStoragePersistence : IBodyStoragePersistence { readonly List written = []; + readonly List deleted = []; + + // Body ids whose deletion should throw, to exercise the sweep's tolerate-missing handling. + public HashSet FailDeleteFor { get; } = []; public IReadOnlyList Written { @@ -109,6 +117,17 @@ public IReadOnlyList Written } } + public IReadOnlyList Deleted + { + get + { + lock (deleted) + { + return [.. deleted]; + } + } + } + public Task WriteBody(string bodyId, ReadOnlyMemory body, string contentType, CancellationToken cancellationToken = default) { lock (written) @@ -122,8 +141,20 @@ public Task WriteBody(string bodyId, ReadOnlyMemory body, string contentTy public Task ReadBody(string bodyId, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task DeleteBody(string bodyId, CancellationToken cancellationToken = default) => - throw new NotImplementedException(); + public Task DeleteBody(string bodyId, CancellationToken cancellationToken = default) + { + if (FailDeleteFor.Contains(bodyId)) + { + throw new InvalidOperationException($"Simulated missing body for {bodyId}"); + } + + lock (deleted) + { + deleted.Add(bodyId); + } + + return Task.CompletedTask; + } public record StoredBody(string BodyId, byte[] Body, string ContentType); } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs new file mode 100644 index 0000000000..0c9b614908 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs @@ -0,0 +1,142 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.Entities; + +class RetentionSweepTests : ErrorIngestionTestBase +{ + [SetUp] + public void SetRetention() => EFSettings.ErrorRetentionPeriod = TimeSpan.FromDays(30); + + DateTime Now => PersistenceTestsContext.FakeTime.GetUtcNow().UtcDateTime; + + [Test] + public async Task Deletes_resolved_and_archived_rows_past_the_cutoff() + { + var oldResolved = await SeedFailedMessage(FailedMessageStatus.Resolved, Now.AddDays(-31)); + var oldArchived = await SeedFailedMessage(FailedMessageStatus.Archived, Now.AddDays(-31)); + + await RunRetentionSweep(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await FindFailedMessage(oldResolved), Is.Null); + Assert.That(await FindFailedMessage(oldArchived), Is.Null); + } + } + + [Test] + public async Task Keeps_resolved_and_archived_rows_within_the_cutoff() + { + var recentResolved = await SeedFailedMessage(FailedMessageStatus.Resolved, Now.AddDays(-29)); + var recentArchived = await SeedFailedMessage(FailedMessageStatus.Archived, Now.AddDays(-29)); + + await RunRetentionSweep(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await FindFailedMessage(recentResolved), Is.Not.Null); + Assert.That(await FindFailedMessage(recentArchived), Is.Not.Null); + } + } + + [Test] + public async Task Never_deletes_unresolved_or_retry_issued_rows_however_old() + { + var ancientUnresolved = await SeedFailedMessage(FailedMessageStatus.Unresolved, Now.AddYears(-5)); + var ancientRetryIssued = await SeedFailedMessage(FailedMessageStatus.RetryIssued, Now.AddYears(-5)); + + await RunRetentionSweep(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await FindFailedMessage(ancientUnresolved), Is.Not.Null); + Assert.That(await FindFailedMessage(ancientRetryIssued), Is.Not.Null); + } + } + + [Test] + public async Task Shrinking_the_retention_takes_effect_on_the_next_run() + { + var message = await SeedFailedMessage(FailedMessageStatus.Resolved, Now.AddDays(-20)); + + await RunRetentionSweep(); + Assert.That(await FindFailedMessage(message), Is.Not.Null, "20 days old is still within the 30 day retention"); + + // No row rewrite, only the setting changes; the next run recomputes the cutoff. + EFSettings.ErrorRetentionPeriod = TimeSpan.FromDays(10); + + await RunRetentionSweep(); + Assert.That(await FindFailedMessage(message), Is.Null, "now past the shrunk 10 day retention"); + } + + [Test] + public async Task Deletes_the_external_bodies_of_swept_rows_only() + { + var externalBody = await SeedFailedMessage(FailedMessageStatus.Resolved, Now.AddDays(-31), bodyStoredExternally: true); + var inlineBody = await SeedFailedMessage(FailedMessageStatus.Resolved, Now.AddDays(-31), bodyStoredExternally: false); + + await RunRetentionSweep(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(RecordedBodies.Deleted, Does.Contain(externalBody.ToString()), "the external body must be deleted"); + Assert.That(RecordedBodies.Deleted, Does.Not.Contain(inlineBody.ToString()), "an inline body needs no external cleanup"); + } + } + + [Test] + public async Task Also_removes_the_group_rows_of_swept_messages() + { + var message = await SeedFailedMessage(FailedMessageStatus.Resolved, Now.AddDays(-31)); + await Store(new FailedMessageGroupEntity { FailedMessageUniqueId = message, GroupId = "group-1", Title = "t", Type = "Message Type" }); + + await RunRetentionSweep(); + + Assert.That(await GetGroups(message), Is.Empty, "the cascade must remove the group rows"); + } + + [Test] + public async Task Tolerates_a_body_that_cannot_be_deleted() + { + var unluckyBody = await SeedFailedMessage(FailedMessageStatus.Resolved, Now.AddDays(-31), bodyStoredExternally: true); + var otherBody = await SeedFailedMessage(FailedMessageStatus.Resolved, Now.AddDays(-31), bodyStoredExternally: true); + RecordedBodies.FailDeleteFor.Add(unluckyBody.ToString()); + + await RunRetentionSweep(); + + using (Assert.EnterMultipleScope()) + { + // The failed delete must not stall retention: both rows are still swept. + Assert.That(await FindFailedMessage(unluckyBody), Is.Null); + Assert.That(await FindFailedMessage(otherBody), Is.Null); + Assert.That(RecordedBodies.Deleted, Does.Contain(otherBody.ToString())); + } + } + + async Task SeedFailedMessage(FailedMessageStatus status, DateTime statusChangedAt, bool bodyStoredExternally = false) + { + var id = Guid.NewGuid(); + + await Store(new FailedMessageEntity + { + UniqueMessageId = id, + Status = status, + StatusChangedAt = statusChangedAt, + LastModified = statusChangedAt, + NumberOfProcessingAttempts = 1, + FirstTimeOfFailure = statusChangedAt, + LastTimeOfFailure = statusChangedAt, + LastAttemptedAt = statusChangedAt, + IsSystemMessage = false, + HeadersJson = "{}", + BodyStoredExternally = bodyStoredExternally, + BodySize = 0 + }); + + return id; + } +} From ed0b1a639fbff8cd3925e074d9891680caf71ba3 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 23 Jul 2026 15:30:35 +1000 Subject: [PATCH 4/5] Updates based on feedback --- docs/error-ingestion-design.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/error-ingestion-design.md b/docs/error-ingestion-design.md index 02938844fe..523fe70515 100644 --- a/docs/error-ingestion-design.md +++ b/docs/error-ingestion-design.md @@ -25,8 +25,10 @@ repeatedly keeps a single row that records: This is a deliberate difference from the document-database persister, which retained an array of attempts. The read side only ever consumed the last attempt plus the count, so storing the full -history earned nothing and cost write amplification. One consequence: `NumberOfProcessingAttempts` -is no longer capped, whereas the document store capped the retained array at ten. +history earned nothing and cost write amplification. It is also an improvement: because the count +is a column rather than the length of a capped array, `NumberOfProcessingAttempts` always reports +the true number of attempts, where Raven's implementation silently stopped counting once the +retained array hit its cap of ten. ### Stored source data vs derived columns @@ -138,11 +140,13 @@ follow: - **Insert races.** Both writers see the row as absent and both insert, colliding on the primary key. -- **Lost updates.** A read-modify-write done in application code (read the row, add to the count, - write it back) can lose one writer's increment. +- **Read-modify-write cost.** EF's optimistic concurrency (a rowversion/xmin token) would catch a + conflicting write instead of silently losing it, but only via a read before every write and a + retry loop per message, the per-row round trip reason 3 rules out, and it still can't express the + conditional merge from reason 1. -Closing both requires the database's own concurrency-safe upsert primitive, and those are -**provider-specific**: +Closing the insert race requires the database's own concurrency-safe upsert primitive, and those +are **provider-specific**: - PostgreSQL: `INSERT ... ON CONFLICT (unique_message_id) DO UPDATE`. The conflict clause makes a concurrent insert fall through to the update instead of failing, and the whole statement is From 5e49c0fa8574be4b29405ef7324697b44f5cb3db Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 23 Jul 2026 18:38:55 +1000 Subject: [PATCH 5/5] Updates based on feedback --- ...2351_AddKnownEndpointMaxLength.Designer.cs | 264 ++++++++++++++++++ ...0260723082351_AddKnownEndpointMaxLength.cs | 54 ++++ ...SqlServiceControlDbContextModelSnapshot.cs | 8 +- ...2337_AddKnownEndpointMaxLength.Designer.cs | 214 ++++++++++++++ ...0260723082337_AddKnownEndpointMaxLength.cs | 54 ++++ ...verServiceControlDbContextModelSnapshot.cs | 8 +- .../Abstractions/BasePersistence.cs | 3 +- .../EFPersistenceConfigurationBase.cs | 12 +- .../Abstractions/EFPersisterSettings.cs | 10 +- .../EntityConfigurations/ColumnLengths.cs | 8 + .../FailedMessageConfiguration.cs | 20 +- .../FailedMessageRetryConfiguration.cs | 2 +- .../KnownEndpointConfiguration.cs | 4 +- 13 files changed, 628 insertions(+), 33 deletions(-) create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260723082351_AddKnownEndpointMaxLength.Designer.cs create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260723082351_AddKnownEndpointMaxLength.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260723082337_AddKnownEndpointMaxLength.Designer.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260723082337_AddKnownEndpointMaxLength.cs create mode 100644 src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260723082351_AddKnownEndpointMaxLength.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260723082351_AddKnownEndpointMaxLength.Designer.cs new file mode 100644 index 0000000000..08116b52f0 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260723082351_AddKnownEndpointMaxLength.Designer.cs @@ -0,0 +1,264 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Persistence.EFCore.PostgreSql; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + [DbContext(typeof(PostgreSqlServiceControlDbContext))] + [Migration("20260723082351_AddKnownEndpointMaxLength")] + partial class AddKnownEndpointMaxLength + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.10") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("body_content_type"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("BodyText") + .HasColumnType("text") + .HasColumnName("body_text"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("conversation_id"); + + b.Property("ExceptionMessage") + .HasColumnType("text") + .HasColumnName("exception_message"); + + b.Property("ExceptionType") + .HasColumnType("text") + .HasColumnName("exception_type"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_time_of_failure"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("LastAttemptedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_attempted_at"); + + b.Property("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property("LastTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_time_of_failure"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasColumnType("text") + .HasColumnName("message_type"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("integer") + .HasColumnName("number_of_processing_attempts"); + + b.Property("QueueAddress") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("queue_address"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_host"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("receiving_endpoint_host_id"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_host"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("sending_endpoint_host_id"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_name"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("StatusChangedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("status_changed_at"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_messages"); + + b.HasIndex("ConversationId") + .HasDatabaseName("ix_failed_messages_conversation_id"); + + b.HasIndex("QueueAddress") + .HasDatabaseName("ix_failed_messages_queue_address"); + + b.HasIndex("ReceivingEndpointName") + .HasDatabaseName("ix_failed_messages_receiving_endpoint_name"); + + b.HasIndex("StatusChangedAt") + .HasDatabaseName("ix_failed_messages_status_changed_at") + .HasFilter("status IN (2, 4)"); + + b.HasIndex("TimeSent") + .HasDatabaseName("ix_failed_messages_time_sent"); + + b.HasIndex("Status", "LastModified") + .HasDatabaseName("ix_failed_messages_status_last_modified"); + + b.ToTable("failed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uuid") + .HasColumnName("failed_message_unique_id"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text") + .HasColumnName("title"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("type"); + + b.HasKey("FailedMessageUniqueId", "GroupId") + .HasName("pk_failed_message_groups"); + + b.HasIndex("GroupId") + .HasDatabaseName("ix_failed_message_groups_group_id"); + + b.ToTable("failed_message_groups", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("RetryId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("retry_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_retries"); + + b.ToTable("failed_message_retries", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("Monitored") + .HasColumnType("boolean") + .HasColumnName("monitored"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_known_endpoints"); + + b.ToTable("known_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_failed_message_groups_failed_messages_failed_message_unique"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260723082351_AddKnownEndpointMaxLength.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260723082351_AddKnownEndpointMaxLength.cs new file mode 100644 index 0000000000..75a997bed4 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260723082351_AddKnownEndpointMaxLength.cs @@ -0,0 +1,54 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + /// + public partial class AddKnownEndpointMaxLength : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "name", + table: "known_endpoints", + type: "character varying(450)", + maxLength: 450, + nullable: false, + oldClrType: typeof(string), + oldType: "text"); + + migrationBuilder.AlterColumn( + name: "host", + table: "known_endpoints", + type: "character varying(450)", + maxLength: 450, + nullable: false, + oldClrType: typeof(string), + oldType: "text"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "name", + table: "known_endpoints", + type: "text", + nullable: false, + oldClrType: typeof(string), + oldType: "character varying(450)", + oldMaxLength: 450); + + migrationBuilder.AlterColumn( + name: "host", + table: "known_endpoints", + type: "text", + nullable: false, + oldClrType: typeof(string), + oldType: "character varying(450)", + oldMaxLength: 450); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs index 02d2a95c8d..fd525ada2c 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs @@ -17,7 +17,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("ProductVersion", "10.0.10") .HasAnnotation("Relational:MaxIdentifierLength", 63); NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); @@ -222,7 +222,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Host") .IsRequired() - .HasColumnType("text") + .HasMaxLength(450) + .HasColumnType("character varying(450)") .HasColumnName("host"); b.Property("HostId") @@ -235,7 +236,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Name") .IsRequired() - .HasColumnType("text") + .HasMaxLength(450) + .HasColumnType("character varying(450)") .HasColumnName("name"); b.HasKey("Id") diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260723082337_AddKnownEndpointMaxLength.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260723082337_AddKnownEndpointMaxLength.Designer.cs new file mode 100644 index 0000000000..686c392038 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260723082337_AddKnownEndpointMaxLength.Designer.cs @@ -0,0 +1,214 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Persistence.EFCore.SqlServer; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerServiceControlDbContext))] + [Migration("20260723082337_AddKnownEndpointMaxLength")] + partial class AddKnownEndpointMaxLength + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.10") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("BodyText") + .HasColumnType("nvarchar(max)"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ExceptionMessage") + .HasColumnType("nvarchar(max)"); + + b.Property("ExceptionType") + .HasColumnType("nvarchar(max)"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("LastAttemptedAt") + .HasColumnType("datetime2"); + + b.Property("LastModified") + .HasColumnType("datetime2"); + + b.Property("LastTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("MessageType") + .HasColumnType("nvarchar(max)"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("int"); + + b.Property("QueueAddress") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("StatusChangedAt") + .HasColumnType("datetime2"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("ConversationId"); + + b.HasIndex("QueueAddress"); + + b.HasIndex("ReceivingEndpointName"); + + b.HasIndex("StatusChangedAt") + .HasFilter("[Status] IN (2, 4)"); + + b.HasIndex("TimeSent"); + + b.HasIndex("Status", "LastModified"); + + b.ToTable("FailedMessages"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uniqueidentifier"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.HasKey("FailedMessageUniqueId", "GroupId"); + + b.HasIndex("GroupId"); + + b.ToTable("FailedMessageGroups"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("RetryId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.ToTable("FailedMessageRetries"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("Monitored") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.ToTable("KnownEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260723082337_AddKnownEndpointMaxLength.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260723082337_AddKnownEndpointMaxLength.cs new file mode 100644 index 0000000000..32233ec132 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260723082337_AddKnownEndpointMaxLength.cs @@ -0,0 +1,54 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + /// + public partial class AddKnownEndpointMaxLength : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "Name", + table: "KnownEndpoints", + type: "nvarchar(450)", + maxLength: 450, + nullable: false, + oldClrType: typeof(string), + oldType: "nvarchar(max)"); + + migrationBuilder.AlterColumn( + name: "Host", + table: "KnownEndpoints", + type: "nvarchar(450)", + maxLength: 450, + nullable: false, + oldClrType: typeof(string), + oldType: "nvarchar(max)"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "Name", + table: "KnownEndpoints", + type: "nvarchar(max)", + nullable: false, + oldClrType: typeof(string), + oldType: "nvarchar(450)", + oldMaxLength: 450); + + migrationBuilder.AlterColumn( + name: "Host", + table: "KnownEndpoints", + type: "nvarchar(max)", + nullable: false, + oldClrType: typeof(string), + oldType: "nvarchar(450)", + oldMaxLength: 450); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs index bedbda43dc..cceec018f8 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs @@ -17,7 +17,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("ProductVersion", "10.0.10") .HasAnnotation("Relational:MaxIdentifierLength", 128); SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); @@ -178,7 +178,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Host") .IsRequired() - .HasColumnType("nvarchar(max)"); + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); b.Property("HostId") .HasColumnType("uniqueidentifier"); @@ -188,7 +189,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Name") .IsRequired() - .HasColumnType("nvarchar(max)"); + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); b.HasKey("Id"); diff --git a/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs b/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs index c6916fe4ad..c4d67e60dc 100644 --- a/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs +++ b/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs @@ -32,8 +32,7 @@ protected static void RegisterDataStores(IServiceCollection services) services.AddSingleton(p => p.GetRequiredService()); services.AddHostedService(p => p.GetRequiredService()); - services.AddSingleton(); - services.AddHostedService(p => p.GetRequiredService()); + services.AddHostedService(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs b/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs index 4eca844bee..fba0f10e3d 100644 --- a/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs +++ b/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs @@ -14,15 +14,13 @@ public abstract class EFPersistenceConfigurationBase : IPersistenceConfiguration const string ErrorRetentionPeriodKey = "ErrorRetentionPeriod"; const string EnableFullTextSearchOnBodiesKey = "EnableFullTextSearchOnBodies"; - const int MaxBodySizeToStoreDefault = 102400; //100 kb - public PersistenceSettings CreateSettings(SettingsRootNamespace settingsRootNamespace) { var settings = CreateSettings(GetRequiredSetting(settingsRootNamespace, ConnectionStringKey)); - settings.CommandTimeout = SettingsReader.Read(settingsRootNamespace, CommandTimeoutKey, 30); + settings.CommandTimeout = SettingsReader.Read(settingsRootNamespace, CommandTimeoutKey, EFPersisterSettings.DefaultCommandTimeout); settings.MessageBodyStoragePath = SettingsReader.Read(settingsRootNamespace, MessageBodyStoragePathKey); - settings.MinBodySizeForCompression = SettingsReader.Read(settingsRootNamespace, MinBodySizeForCompressionKey, 4096); + settings.MinBodySizeForCompression = SettingsReader.Read(settingsRootNamespace, MinBodySizeForCompressionKey, EFPersisterSettings.DefaultMinBodySizeForCompression); settings.MaxBodySizeToStore = ReadMaxBodySizeToStore(settingsRootNamespace); settings.ErrorRetentionPeriod = GetRequiredSetting(settingsRootNamespace, ErrorRetentionPeriodKey); settings.EnableFullTextSearchOnBodies = SettingsReader.Read(settingsRootNamespace, EnableFullTextSearchOnBodiesKey, true); @@ -36,14 +34,14 @@ public PersistenceSettings CreateSettings(SettingsRootNamespace settingsRootName static int ReadMaxBodySizeToStore(SettingsRootNamespace settingsRootNamespace) { - var maxBodySizeToStore = SettingsReader.Read(settingsRootNamespace, MaxBodySizeToStoreKey, MaxBodySizeToStoreDefault); + var maxBodySizeToStore = SettingsReader.Read(settingsRootNamespace, MaxBodySizeToStoreKey, EFPersisterSettings.DefaultMaxBodySizeToStore); if (maxBodySizeToStore <= 0) { LoggerUtil.CreateStaticLogger() - .LogError("MaxBodySizeToStore setting is invalid, 1 is the minimum value. Defaulting to {MaxBodySizeToStoreDefault}", MaxBodySizeToStoreDefault); + .LogError("MaxBodySizeToStore setting is invalid, 1 is the minimum value. Defaulting to {MaxBodySizeToStoreDefault}", EFPersisterSettings.DefaultMaxBodySizeToStore); - return MaxBodySizeToStoreDefault; + return EFPersisterSettings.DefaultMaxBodySizeToStore; } return maxBodySizeToStore; diff --git a/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersisterSettings.cs b/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersisterSettings.cs index 52f54d4815..3b919453c4 100644 --- a/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersisterSettings.cs +++ b/src/ServiceControl.Persistence.EFCore/Abstractions/EFPersisterSettings.cs @@ -4,12 +4,16 @@ public abstract class EFPersisterSettings : PersistenceSettings { public static readonly TimeSpan MigrationCommandTimeout = TimeSpan.FromMinutes(40); + public const int DefaultCommandTimeout = 30; + public const int DefaultMinBodySizeForCompression = 4096; + public const int DefaultMaxBodySizeToStore = 102400; // 100 kb + public required string ConnectionString { get; set; } - public int CommandTimeout { get; set; } = 30; + public int CommandTimeout { get; set; } = DefaultCommandTimeout; public TimeSpan ErrorRetentionPeriod { get; set; } public string? MessageBodyStoragePath { get; set; } - public int MinBodySizeForCompression { get; set; } = 4096; - public int MaxBodySizeToStore { get; set; } = 102400; + public int MinBodySizeForCompression { get; set; } = DefaultMinBodySizeForCompression; + public int MaxBodySizeToStore { get; set; } = DefaultMaxBodySizeToStore; public int MaxRetryCount { get; set; } = 5; public int MaxRetryDelayInSeconds { get; set; } = 30; public bool EnableSensitiveDataLogging { get; set; } diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs new file mode 100644 index 0000000000..140d945469 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Persistence.EFCore.EntityConfigurations; + +static class ColumnLengths +{ + // Indexed and short-by-nature values get a length so that SQL Server can index them, + // nvarchar(max) columns cannot be index key columns. + public const int ShortTextLength = 450; +} diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs index 36a1dfb966..1702985c21 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs @@ -6,10 +6,6 @@ namespace ServiceControl.Persistence.EFCore.EntityConfigurations; class FailedMessageConfiguration : IEntityTypeConfiguration { - // Indexed and short-by-nature values get a length so that SQL Server can index them, - // nvarchar(max) columns cannot be index key columns. - public const int ShortTextLength = 450; - public void Configure(EntityTypeBuilder builder) { builder.HasKey(e => e.UniqueMessageId); @@ -23,14 +19,14 @@ public void Configure(EntityTypeBuilder builder) builder.Property(e => e.LastTimeOfFailure).IsRequired(); builder.Property(e => e.LastAttemptedAt).IsRequired(); - builder.Property(e => e.MessageId).HasMaxLength(ShortTextLength); - builder.Property(e => e.ConversationId).HasMaxLength(ShortTextLength); - builder.Property(e => e.QueueAddress).HasMaxLength(ShortTextLength); - builder.Property(e => e.SendingEndpointName).HasMaxLength(ShortTextLength); - builder.Property(e => e.SendingEndpointHost).HasMaxLength(ShortTextLength); - builder.Property(e => e.ReceivingEndpointName).HasMaxLength(ShortTextLength); - builder.Property(e => e.ReceivingEndpointHost).HasMaxLength(ShortTextLength); - builder.Property(e => e.BodyContentType).HasMaxLength(ShortTextLength); + builder.Property(e => e.MessageId).HasMaxLength(ColumnLengths.ShortTextLength); + builder.Property(e => e.ConversationId).HasMaxLength(ColumnLengths.ShortTextLength); + builder.Property(e => e.QueueAddress).HasMaxLength(ColumnLengths.ShortTextLength); + builder.Property(e => e.SendingEndpointName).HasMaxLength(ColumnLengths.ShortTextLength); + builder.Property(e => e.SendingEndpointHost).HasMaxLength(ColumnLengths.ShortTextLength); + builder.Property(e => e.ReceivingEndpointName).HasMaxLength(ColumnLengths.ShortTextLength); + builder.Property(e => e.ReceivingEndpointHost).HasMaxLength(ColumnLengths.ShortTextLength); + builder.Property(e => e.BodyContentType).HasMaxLength(ColumnLengths.ShortTextLength); builder.Property(e => e.IsSystemMessage).IsRequired(); builder.Property(e => e.HeadersJson).IsRequired(); diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageRetryConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageRetryConfiguration.cs index ea98f8240f..3cc7007040 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageRetryConfiguration.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageRetryConfiguration.cs @@ -10,6 +10,6 @@ public void Configure(EntityTypeBuilder builder) { builder.HasKey(e => e.UniqueMessageId); builder.Property(e => e.UniqueMessageId).ValueGeneratedNever(); - builder.Property(e => e.RetryId).HasMaxLength(FailedMessageConfiguration.ShortTextLength); + builder.Property(e => e.RetryId).HasMaxLength(ColumnLengths.ShortTextLength); } } diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/KnownEndpointConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/KnownEndpointConfiguration.cs index 4bcb26d5e2..8dfcee88a0 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/KnownEndpointConfiguration.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/KnownEndpointConfiguration.cs @@ -10,9 +10,9 @@ public void Configure(EntityTypeBuilder builder) { builder.HasKey(e => e.Id); builder.Property(e => e.Id).ValueGeneratedNever(); - builder.Property(e => e.Name).IsRequired(); + builder.Property(e => e.Name).IsRequired().HasMaxLength(ColumnLengths.ShortTextLength); builder.Property(e => e.HostId).IsRequired(); - builder.Property(e => e.Host).IsRequired(); + builder.Property(e => e.Host).IsRequired().HasMaxLength(ColumnLengths.ShortTextLength); builder.Property(e => e.Monitored).IsRequired(); } }