From 9c41d58249e50aa525e15514a4aa95466abcef39 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Wed, 28 Oct 2026 16:25:04 +0300 Subject: [PATCH 1/2] feat: suggest retry policies with AI --- SW.Bitween.Api/Domain/OperatorAction.cs | 1 + .../Resources/RetrySuggestions/Suggest.cs | 156 +++++++++++ .../Services/Ai/RetryPolicyPrompt.cs | 227 ++++++++++++++++ SW.Bitween.HttpTests/RetrySuggestionTests.cs | 129 +++++++++ .../Ai/RetryPolicyPromptTests.cs | 103 ++++++++ SW.Bitween.Web/ClientApp/src/api/client.ts | 9 + .../ClientApp/src/api/http/retryPolicies.ts | 10 + SW.Bitween.Web/ClientApp/src/api/types.ts | 25 ++ .../__tests__/suggestRetryPolicy.test.tsx | 99 +++++++ .../subscriptions/studio/LaneAndRetry.tsx | 24 ++ .../pages/subscriptions/studio/Overview.tsx | 1 + .../studio/SuggestRetryPolicy.tsx | 246 ++++++++++++++++++ docs/admin-ui.md | 2 +- docs/api-reference.md | 1 + docs/configuration.md | 2 +- docs/retries-and-alerts.md | 12 + 16 files changed, 1045 insertions(+), 2 deletions(-) create mode 100644 SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs create mode 100644 SW.Bitween.Api/Services/Ai/RetryPolicyPrompt.cs create mode 100644 SW.Bitween.HttpTests/RetrySuggestionTests.cs create mode 100644 SW.Bitween.UnitTests/Ai/RetryPolicyPromptTests.cs create mode 100644 SW.Bitween.Web/ClientApp/src/pages/subscriptions/__tests__/suggestRetryPolicy.test.tsx create mode 100644 SW.Bitween.Web/ClientApp/src/pages/subscriptions/studio/SuggestRetryPolicy.tsx diff --git a/SW.Bitween.Api/Domain/OperatorAction.cs b/SW.Bitween.Api/Domain/OperatorAction.cs index 488ceb6a..76536b44 100644 --- a/SW.Bitween.Api/Domain/OperatorAction.cs +++ b/SW.Bitween.Api/Domain/OperatorAction.cs @@ -20,6 +20,7 @@ public class OperatorAction /// A failed exchange sent, masked, to the AI provider under Settings, AI for an explanation. public const string ExplainFailure = "explain-failure"; + public const string SuggestRetryPolicy = "suggest-retry-policy"; private OperatorAction() { diff --git a/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs b/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs new file mode 100644 index 00000000..692856a2 --- /dev/null +++ b/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs @@ -0,0 +1,156 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.Bitween.Services.Ai; +using SW.Bitween.Services.Metrics; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.RetrySuggestions; + +public class RetrySuggestionRequest +{ + public int SubscriptionId { get; set; } + + /// How far back to look, 1 to 90 days. + public int Days { get; set; } = 30; +} + +/// One kind of failure, and what the current and the suggested policy each do with it. +public class RetrySuggestionCoverage +{ + /// The error's first line, masked as it was sent. + public string Error { get; set; } = ""; + public int Count { get; set; } + public int Retried { get; set; } + public int FixedByRetry { get; set; } + public string Now { get; set; } = ""; + public string Suggested { get; set; } = ""; +} + +public class RetrySuggestionResponse +{ + public string Name { get; set; } = ""; + public string Summary { get; set; } = ""; + + /// The groups, ready for POST /api/retrypolicies; nothing is saved until then. + public List Groups { get; set; } = new(); + + public List Coverage { get; set; } = new(); + public int Failures { get; set; } + public int Days { get; set; } + public string? Model { get; set; } + + /// Exactly what went to the provider. + public string Sent { get; set; } = ""; +} + +/// +/// Proposes a retry policy for a subscription from its own recent failures. +/// +/// +/// +/// The provider sees each kind of failure as a masked line with counts and how often a retry fixed +/// it, and the current policy's rules. Never a document or a response body. The suggestion is then +/// checked as a saved policy would be and run against the real failures here, so the coverage +/// shows what changes for each before anyone saves it. +/// +/// +/// It saves nothing: saving the suggestion is creating a policy, so asking needs that right too, +/// and a question nobody could act on isn't sent. Each one is recorded as an operator action, +/// since it sends something about the subscription to a third party. +/// +/// +public class Suggest(RequestContext requestContext, BitweenDbContext dbContext, AiModels models) + : ICommandHandler +{ + public async Task Handle(RetrySuggestionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); + await requestContext.EnsurePermission(dbContext, Model.Permissions.RetryPolicies.Create); + var model = models.Current(); + + var subscription = await dbContext.Set().AsNoTracking() + .Include(s => s.RetryPolicy) + .FirstOrDefaultAsync(s => s.Id == request.SubscriptionId) + ?? throw new SWNotFoundException(request.SubscriptionId.ToString()); + var current = subscription.CustomRetryPolicy?.Groups ?? subscription.RetryPolicy?.Groups; + var days = Math.Clamp(request.Days, 1, 90); + var since = DateTime.UtcNow.AddDays(-days); + + // The newest failures, by the (SubscriptionId, FinishedOn) index; only the start of each + // error, which is where its type and message are. + var failed = await dbContext.Set().AsNoTracking() + .Where(r => r.SubscriptionId == subscription.Id && r.FinishedOn >= since && (!r.Success || r.ResponseBad)) + .OrderByDescending(r => r.FinishedOn) + .Take(RetryPolicyPrompt.MaxFailures) + .Select(r => new + { + r.Id, r.Success, r.FinishedOn, + Exception = r.Exception == null ? null : r.Exception.Substring(0, RetryPolicyPrompt.SampleLength), + }) + .ToListAsync(); + var errors = failed.Where(f => !f.Success).ToList(); + if (errors.Count == 0) + throw new SWValidationException("NO_FAILURES", + $"This subscription has no failed exchanges in the last {days} days, so there is nothing to learn a policy from."); + + // Whether each one's retry succeeded: the attempt made from it, by the RetryFor index. + var ids = errors.Select(e => e.Id).ToList(); + var retries = await dbContext.Set().AsNoTracking() + .Where(x => x.RetryFor != null && ids.Contains(x.RetryFor)) + .Join(dbContext.Set(), x => x.Id, r => r.Id, (x, r) => new { x.RetryFor, Fixed = r.Success && !r.ResponseBad }) + .ToListAsync(); + var retriedFrom = retries.GroupBy(r => r.RetryFor!).ToDictionary(g => g.Key, g => g.Any(r => r.Fixed)); + + var kinds = errors + .GroupBy(e => ErrorFingerprint.Of(e.Exception ?? "")) + .Select(g => + { + var newest = g.First(); + var sample = newest.Exception ?? ""; + return new FailureKind(sample, RetryPolicyPrompt.Mask(ErrorFingerprint.Sample(sample)), g.Count(), + g.Count(e => retriedFrom.ContainsKey(e.Id)), g.Count(e => retriedFrom.TryGetValue(e.Id, out var ok) && ok), + g.Min(e => e.FinishedOn), g.Max(e => e.FinishedOn)); + }) + .OrderByDescending(k => k.Count) + .Take(RetryPolicyPrompt.MaxKinds) + .ToList(); + + var describe = $"Subscription kind: {subscription.Type}; mapper: {subscription.MapperId ?? "none"}; handler: {subscription.HandlerId ?? "none"}"; + var message = RetryPolicyPrompt.Message(describe, days, errors.Count, failed.Count - errors.Count, kinds, current); + // Recorded before it goes, so a call that fails or times out after sending is on record too. + await dbContext.RecordNowAsync(requestContext, OperatorAction.SuggestRetryPolicy, $"subscription {subscription.Id}", + $"Sent to {models.ModelName}: {errors.Count} failures, errors masked."); + var suggestion = RetryPolicyPrompt.Parse( + await model.Complete(new AiPrompt(RetryPolicyPrompt.Instructions, message, Json: true, MaxTokens: 4000))); + + var coverage = new List(); + foreach (var kind in kinds) + coverage.Add(new RetrySuggestionCoverage + { + Error = kind.Masked, + Count = kind.Count, + Retried = kind.Retried, + FixedByRetry = kind.FixedByRetry, + Now = await RetryPolicyPrompt.Outcome(current, kind.Sample), + Suggested = await RetryPolicyPrompt.Outcome(suggestion.Groups, kind.Sample), + }); + + return new RetrySuggestionResponse + { + Name = suggestion.Name, + Summary = suggestion.Summary, + Groups = suggestion.Groups, + Coverage = coverage, + Failures = errors.Count, + Days = days, + Model = models.ModelName, + Sent = message, + }; + } +} diff --git a/SW.Bitween.Api/Services/Ai/RetryPolicyPrompt.cs b/SW.Bitween.Api/Services/Ai/RetryPolicyPrompt.cs new file mode 100644 index 00000000..d01fec6d --- /dev/null +++ b/SW.Bitween.Api/Services/Ai/RetryPolicyPrompt.cs @@ -0,0 +1,227 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Serialization; +using SW.Bitween.JsonConverters; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Services.Ai; + +/// One kind of failure a subscription had: the same first line, give or take its details. +/// The real first line and a little more - matched against here, never sent. +/// What the model sees of it. +/// How many of these were retried. +/// How many of those retries succeeded - the mark of a passing glitch. +public record FailureKind(string Sample, string Masked, int Count, int Retried, int FixedByRetry, DateTime FirstOn, DateTime LastOn); + +/// What the model proposes: a named policy, why, and its groups. +public record RetrySuggestion(string Name, string Summary, List Groups); + +/// +/// Proposes a retry policy from a subscription's real failures. +/// +/// +/// +/// The model sees each kind of failure as one masked line - the wording kept, values, addresses and +/// quoted text taken out - with how often it happened and how often a retry fixed it. Never a +/// document or a response body. The current policy's rules go too: they are configuration. +/// +/// +/// What comes back is checked as a saved policy is, then run against the real failures here, so +/// the person reviewing it sees what each one would do under it before anything is saved. +/// +/// +public static class RetryPolicyPrompt +{ + /// How many of the newest failures are looked at. + public const int MaxFailures = 2_000; + + /// How many kinds of failure are sent, most frequent first. + public const int MaxKinds = 15; + + /// How much of each error is kept to match against: enough for its exception types and message. + public const int SampleLength = 4_000; + + public const string Instructions = """ + You design retry policies for Bitween, an integration platform. When an exchange fails, + its subscription's retry policy decides whether to try it again, how often and how soon. + + A policy is a list of groups, checked in priority order (lowest first); the first group with + a matching matcher decides. A failure no group matches is not retried. + + A group, as JSON: + { + "name": "short name", + "priority": 0, + "appliesTo": ["Error"], + "matchers": [ ... at least one ... ], + "action": "Allow" | "Block", + "budget": { // required when action is Allow; null for Block + "maxAttemptsPerError": 3, // retries of one failing exchange + "maxAttemptsTotal": 50, // retries across all exchanges until the subscription next succeeds; null for no ceiling + "delayStrategy": { "type": "exponential", "initialDelayMs": 30000, "multiplier": 2, "maxDelayMs": 600000 } + // or { "type": "fixed", "delayMs": 60000 } + // or { "type": "linear", "initialDelayMs": 30000, "incrementMs": 30000 } + }, + "notes": "one sentence: which failures this is for and why it retries or stops them" + } + Matchers (any one matching is enough): + - { "type": "contains", "value": "text", "caseSensitive": false } + - { "type": "regex", "pattern": "a .NET regex", "flags": "i" } + - { "type": "exceptionType", "value": "System.Net.Http.HttpRequestException", "includeInner": true } + Matchers are checked against the whole error text: exception types, message and stack. + + You get the subscription's kinds of failure from recent weeks, most frequent first, each + with a count and how often a retry fixed it, plus its current policy. Each error is masked: + values became , , , . Match only on wording that survived + masking - never on a , which is not in the real text. + + Guidance: + - Retry what passes by itself: timeouts, refused or reset connections, 429, 502, 503, 504, + failures that a retry has often fixed. + - Don't retry what can't pass unchanged: mapping errors, validation errors, 400, 401, 403, + 404, malformed configuration such as an invalid URL. Leave them unmatched, or add a Block + group when a broader Allow group would otherwise catch them. + - Back off: exponential delays, a few attempts per exchange, a total that stops a long + outage from hammering the partner. + - Keep it small: two to four groups. + + Answer with one JSON object and nothing else: + { "name": "policy name", "summary": "two sentences on what it does and why", "groups": [ ... ] } + """; + + /// The message for one subscription. + public static string Message(string subscription, int days, int failures, int badResponses, + IReadOnlyList kinds, IReadOnlyList? current) + { + var message = new StringBuilder(); + message.AppendLine(subscription); + message.AppendLine($"Looked at: the last {days} days, {failures} failures" + + (failures >= MaxFailures ? $" (the newest {MaxFailures})" : "") + "."); + if (badResponses > 0) message.AppendLine($"Also {badResponses} bad responses; their bodies are not sent."); + message.AppendLine(); + message.AppendLine(""); + // The rules only: a group's alert settings name channels and, on old groups, carry handler properties. + message.AppendLine(current is { Count: > 0 } + ? JsonConvert.SerializeObject(current.Select(g => new { g.Name, g.Priority, g.Enabled, g.AppliesTo, g.Matchers, g.Action, g.Budget }), Settings) + : "(none: nothing is retried)"); + message.AppendLine(""); + message.AppendLine(); + message.AppendLine(""); + var n = 0; + foreach (var kind in kinds) + { + n++; + message.AppendLine($"{n}. {kind.Count} times, {kind.FirstOn:yyyy-MM-dd} to {kind.LastOn:yyyy-MM-dd}. " + + (kind.Retried == 0 ? "Never retried." : $"Retried {kind.Retried} times; the retry succeeded {kind.FixedByRetry} times.")); + message.AppendLine($" {kind.Masked}"); + } + if (kinds.Count == 0) message.AppendLine("(none)"); + message.AppendLine(""); + return message.ToString(); + } + + static readonly Regex Url = new(@"[a-z][a-z0-9+.-]*://[^\s'""<>]+", RegexOptions.Compiled | RegexOptions.IgnoreCase); + + /// An error's first line as the model may see it: masked, with addresses taken out too. + public static string Mask(string firstLine) => Url.Replace(new Masking().Error(firstLine), ""); + + /// + /// The model's answer as a policy that could be saved, or a validation error saying why not. + /// Alerts are never taken from it: where a policy's alerts go is a person's choice. + /// + public static RetrySuggestion Parse(string answer) + { + var start = answer.IndexOf('{'); + var end = answer.LastIndexOf('}'); + RawSuggestion? raw = null; + if (start >= 0 && end > start) + { + try + { + raw = JObject.Parse(answer[start..(end + 1)]).ToObject(JsonSerializer.Create(Settings)); + } + catch (Exception e) when (e is JsonException or ArgumentException or InvalidOperationException) + { + } + } + if (raw?.Groups is not { Count: > 0 }) + throw new SWValidationException("AI_SUGGESTION", "The AI's answer wasn't a retry policy. Try again."); + + var groups = raw.Groups.Select((g, i) => new RetryGroup + { + Name = string.IsNullOrWhiteSpace(g.Name) ? $"Group {i + 1}" : g.Name.Trim(), + Priority = g.Priority ?? i, + AppliesTo = g.AppliesTo?.Where(t => t != XchangeResultType.Success).Distinct().ToList() ?? [], + Matchers = g.Matchers?.Where(m => m is not null).ToList() ?? [], + Action = g.Action ?? RetryAction.Allow, + Budget = g.Action == RetryAction.Block ? null : g.Budget, + Notes = g.Notes, + }).ToList(); + + try + { + SW.Bitween.Resources.RetryPolicies.RetryGroupValidation.EnsureCanFire(groups); + } + catch (SWValidationException e) + { + throw new SWValidationException("AI_SUGGESTION", $"The AI suggested a policy that can't be saved: {e.Message} Try again."); + } + + return new RetrySuggestion( + string.IsNullOrWhiteSpace(raw.Name) ? "Suggested retry policy" : raw.Name.Trim(), + raw.Summary?.Trim() ?? "", groups); + } + + /// What a policy does with one failure: how many times it retries it, by which group, or why it stops. + public static async Task Outcome(IReadOnlyList? groups, string content) + { + if (groups is not { Count: > 0 }) return "Not retried: no retry policy"; + var evaluator = new RetryPolicyEvaluator(new CustomRetryPolicy { Groups = groups.ToList() }, new InMemoryRetryGroupBudget()); + var retries = 0; + string? by = null; + for (var attempt = 0; attempt < 20; attempt++) + { + var decision = await evaluator.Evaluate(XchangeResultType.Error, content, attempt); + if (!decision.ShouldRetry) + return retries == 0 ? $"Not retried: {decision.Reason}" : $"Retried up to {retries} times by \"{by}\""; + retries++; + by = decision.MatchedGroupName; + } + return $"Retried {retries}+ times by \"{by}\""; + } + + /// The wire's own settings: camelCase, enums as names, matchers and delays by their type. + static readonly JsonSerializerSettings Settings = new() + { + Converters = { new MatcherJsonConverter(), new DelayStrategyJsonConverter(), new Newtonsoft.Json.Converters.StringEnumConverter() }, + ContractResolver = new CamelCasePropertyNamesContractResolver(), + NullValueHandling = NullValueHandling.Ignore, + }; + + // Loose on purpose: whatever the model leaves out gets a default or is caught by validation. + sealed class RawSuggestion + { + public string? Name { get; set; } + public string? Summary { get; set; } + public List? Groups { get; set; } + } + + sealed class RawGroup + { + public string? Name { get; set; } + public int? Priority { get; set; } + public List? AppliesTo { get; set; } + public List? Matchers { get; set; } + public RetryAction? Action { get; set; } + public RetryBudget? Budget { get; set; } + public string? Notes { get; set; } + } +} diff --git a/SW.Bitween.HttpTests/RetrySuggestionTests.cs b/SW.Bitween.HttpTests/RetrySuggestionTests.cs new file mode 100644 index 00000000..b4a1aa44 --- /dev/null +++ b/SW.Bitween.HttpTests/RetrySuggestionTests.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Services.Ai; +using Xunit; + +namespace SW.Bitween.HttpTests; + +/// +/// Retry suggestions: a subscription's real failures go to the model masked, a policy comes back +/// that could be saved, tried on those failures, and the request is in the audit trail. +/// +[Collection("Http")] +public class RetrySuggestionTests(HttpFixture fixture) : IDisposable +{ + public void Dispose() + { + AiModels.ModelOverride = null; + using var admin = fixture.AdminAsync().GetAwaiter().GetResult(); + admin.DeleteAsync("/api/settings/Bitween.Ai.ApiKey").GetAwaiter().GetResult(); + } + + sealed class FakeModel(string answer) : IAiModel + { + public List Asked { get; } = new(); + + public Task Complete(AiPrompt prompt) + { + Asked.Add(prompt); + return Task.FromResult(answer); + } + } + + /// A subscription whose exchanges fail in mapping: "maybe" can't become a yes/no value. + async Task<(HttpClient Admin, int SubscriptionId)> FailingSubscriptionAsync(int failures) + { + var (admin, urlName, _, partnerId, subscriptionId) = await Api.GatewayAsync(fixture); + await Api.Json(await admin.PostAsJsonAsync($"/api/subscriptions/{subscriptionId}/savemapper", new + { + mapperId = "NativeMapper", + mapperProperties = new[] + { + new + { + key = "MappingRules", + value = """{ "version": 1, "fields": [ { "target": ["flag"], "from": { "kind": "path", "path": "order.flag" }, "type": "boolean" } ] }""", + }, + }, + })); + var key = await Api.NewKeyAsync(admin); + await Api.SetKeysAsync(admin, partnerId, ("default", key)); + for (var i = 0; i < failures; i++) + { + using var _ = await Api.CallSyncAsync(fixture, urlName, key, """{ "order": { "flag": "maybe", "owner": "lina@acme.example" } }"""); + } + return (admin, subscriptionId); + } + + async Task ConfigureAsync(HttpClient admin) + { + await Api.Json(await admin.PostAsJsonAsync("/api/settings/Bitween.Ai.ApiKey", new { value = "sk-test-key" })); + var live = fixture.App.Services.GetService(typeof(BitweenOptions)) as BitweenOptions; + for (var waited = 0; live!.Ai.ApiKey != "sk-test-key"; waited += 50) + { + Assert.True(waited < 10_000, "the AI key never took effect"); + await Task.Delay(50); + } + } + + [Fact] + public async Task A_policy_is_suggested_from_masked_failures_tried_on_them_and_audited() + { + var (admin, subscriptionId) = await FailingSubscriptionAsync(2); + await ConfigureAsync(admin); + var model = new FakeModel(""" + { "name": "Mapping stays failed", "summary": "S", "groups": [ + { "name": "Bad data", "appliesTo": ["Error"], "action": "Block", + "matchers": [ { "type": "contains", "value": "boolean" } ] }, + { "name": "Outages", "appliesTo": ["Error"], "action": "Allow", + "matchers": [ { "type": "contains", "value": "timed out" } ], + "budget": { "maxAttemptsPerError": 3, "maxAttemptsTotal": 20, "delayStrategy": { "type": "fixed", "delayMs": 60000 } } } ] } + """); + AiModels.ModelOverride = _ => model; + + var result = await Api.Json(await admin.PostAsJsonAsync("/api/retrysuggestions", new { subscriptionId })); + + Assert.Equal("Mapping stays failed", (string)result["name"]!); + Assert.Equal(2, result["groups"]!.AsArray().Count); + Assert.Equal(2, (int)result["failures"]!); + var row = Assert.Single(result["coverage"]!.AsArray()); + Assert.Equal(2, (int)row!["count"]!); + Assert.Equal("Not retried: no retry policy", (string)row["now"]!); + Assert.Equal("Not retried: Group 'Bad data' explicitly blocks this error", (string)row["suggested"]!); + + var sent = Assert.Single(model.Asked).Message; + Assert.Equal(sent, (string)result["sent"]!); + Assert.DoesNotContain("maybe", sent); + Assert.DoesNotContain("lina@acme.example", sent); + + // What came back saves as it is. + var saved = await admin.PostAsJsonAsync("/api/retrypolicies", new { name = Api.Unique("Suggested"), groups = result["groups"] }); + Assert.Equal(HttpStatusCode.OK, saved.StatusCode); + + var audited = await fixture.InDbAsync(db => db.Set().AsNoTracking() + .AnyAsync(a => a.Action == OperatorAction.SuggestRetryPolicy && a.Target == $"subscription {subscriptionId}")); + Assert.True(audited, "the suggestion wasn't recorded"); + } + + [Fact] + public async Task A_subscription_without_failures_has_nothing_to_learn_from() + { + var (admin, _, _, _, subscriptionId) = await Api.GatewayAsync(fixture); + await ConfigureAsync(admin); + var model = new FakeModel("{}"); + AiModels.ModelOverride = _ => model; + + var response = await admin.PostAsJsonAsync("/api/retrysuggestions", new { subscriptionId }); + + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + Assert.Contains("nothing to learn a policy from", await response.Content.ReadAsStringAsync()); + Assert.Empty(model.Asked); + } +} diff --git a/SW.Bitween.UnitTests/Ai/RetryPolicyPromptTests.cs b/SW.Bitween.UnitTests/Ai/RetryPolicyPromptTests.cs new file mode 100644 index 00000000..5ec86301 --- /dev/null +++ b/SW.Bitween.UnitTests/Ai/RetryPolicyPromptTests.cs @@ -0,0 +1,103 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Model; +using SW.Bitween.Services.Ai; +using SW.PrimitiveTypes; + +namespace SW.Bitween.UnitTests.Ai; + +/// +/// Retry suggestions: what the model proposes is a policy that could be saved, it is tried on the +/// real errors here, and what is sent of them is masked. +/// +[TestClass] +public class RetryPolicyPromptTests +{ + const string Suggestion = """ + ```json + { "name": "Retry outages", "summary": "S", + "groups": [ + { "name": "Connection problems", "appliesTo": ["Error"], "action": "Allow", + "matchers": [ { "type": "exceptionType", "value": "System.Net.Http.HttpRequestException" } ], + "budget": { "maxAttemptsPerError": 3, "maxAttemptsTotal": 50, + "delayStrategy": { "type": "exponential", "initialDelayMs": 30000, "multiplier": 2, "maxDelayMs": 600000 } }, + "alertMode": "Send", "alertChannelId": 7, "notes": "Passing outages." }, + { "name": "Bad URL", "appliesTo": ["Error"], "action": "Block", + "matchers": [ { "type": "contains", "value": "Invalid URI" } ], + "budget": { "maxAttemptsPerError": 9, "delayStrategy": { "type": "fixed", "delayMs": 1000 } } } + ] } + ``` + """; + + [TestMethod] + public void A_suggestion_is_a_policy_that_could_be_saved_without_its_alerts() + { + var suggestion = RetryPolicyPrompt.Parse(Suggestion); + + Assert.AreEqual("Retry outages", suggestion.Name); + Assert.AreEqual(2, suggestion.Groups.Count); + var outages = suggestion.Groups[0]; + Assert.AreEqual(0, outages.Priority); + Assert.IsInstanceOfType(outages.Matchers[0]); + Assert.IsInstanceOfType(outages.Budget!.DelayStrategy); + Assert.AreEqual(RetryAlertMode.Inherit, outages.AlertMode); + Assert.IsNull(outages.AlertChannelId); + Assert.IsNull(suggestion.Groups[1].Budget, "a blocking group has no budget"); + } + + [TestMethod] + public void A_suggestion_that_couldnt_be_saved_is_refused() + { + var e = Assert.ThrowsException(() => RetryPolicyPrompt.Parse(""" + { "groups": [ { "name": "No budget", "appliesTo": ["Error"], "action": "Allow", + "matchers": [ { "type": "contains", "value": "timeout" } ] } ] } + """)); + StringAssert.Contains(e.Message, "can't be saved"); + + Assert.ThrowsException(() => RetryPolicyPrompt.Parse("I would retry timeouts.")); + } + + [TestMethod] + public async Task Each_failure_is_tried_on_the_policy_here() + { + var groups = RetryPolicyPrompt.Parse(Suggestion).Groups; + + Assert.AreEqual("Retried up to 3 times by \"Connection problems\"", await RetryPolicyPrompt.Outcome(groups, + "System.Net.Http.HttpRequestException: Connection refused (skynet.example:443)\n at Send()")); + Assert.AreEqual("Not retried: Group 'Bad URL' explicitly blocks this error", await RetryPolicyPrompt.Outcome(groups, + "System.UriFormatException: Invalid URI: The format of the URI could not be determined.")); + Assert.AreEqual("Not retried: no retry policy", await RetryPolicyPrompt.Outcome(null, "anything")); + } + + [TestMethod] + public void An_error_line_loses_its_values_and_addresses() + { + var masked = RetryPolicyPrompt.Mask("Response 503 from https://api.skynet.example/v2/orders for 'Lina Haddad' ref 88213"); + + StringAssert.Contains(masked, "Response 503 from for '' ref "); + } + + [TestMethod] + public void The_current_policy_goes_without_its_alert_settings() + { + var current = new List + { + new() + { + Name = "Old", AppliesTo = [XchangeResultType.Error], Matchers = [new ContainsMatcher { Value = "timeout" }], + Budget = new RetryBudget { MaxAttemptsPerError = 2, DelayStrategy = new FixedDelayStrategy { DelayMs = 1000 } }, + AlertHandlerId = "legacy", AlertHandlerProperties = new() { ["Password"] = "hunter2" }, AlertChannelId = 4, + }, + }; + + var message = RetryPolicyPrompt.Message("Subscription kind: Internal", 30, 12, 0, + [new FailureKind("real", "System.TimeoutException: ", 12, 10, 9, DateTime.UtcNow, DateTime.UtcNow)], current); + + StringAssert.Contains(message, "\"value\":\"timeout\""); + StringAssert.Contains(message, "Retried 10 times; the retry succeeded 9 times."); + Assert.IsFalse(message.Contains("hunter2") || message.Contains("legacy") || message.Contains("alertChannelId"), message); + Assert.IsFalse(message.Contains("real"), "the real error text is never sent"); + } +} diff --git a/SW.Bitween.Web/ClientApp/src/api/client.ts b/SW.Bitween.Web/ClientApp/src/api/client.ts index b232b221..603ae9e2 100644 --- a/SW.Bitween.Web/ClientApp/src/api/client.ts +++ b/SW.Bitween.Web/ClientApp/src/api/client.ts @@ -86,6 +86,7 @@ import type { RetryAlertConfig, RetryAttempts, RetryPolicy, + RetrySuggestion, RetryPolicyDetail, RetryPolicyListRow, RetryUsageRow, @@ -633,6 +634,14 @@ export interface ApiClient { }): Promise>; getRetryPolicy(id: number): Promise; createRetryPolicy(input: { name: string }): Promise; + /** + * Asks the AI provider under Settings, AI for a retry policy from a subscription's recent + * failures. Only masked error lines, their counts and how often a retry fixed them go, with the + * current policy's rules; never documents. Saves nothing. + */ + suggestRetryPolicy(subscriptionId: number, signal?: AbortSignal): Promise; + /** Creates a policy from a suggestion's groups; returns its id. */ + saveSuggestedRetryPolicy(name: string, groups: RetrySuggestion["groups"]): Promise; updateRetryPolicy( id: number, changes: { diff --git a/SW.Bitween.Web/ClientApp/src/api/http/retryPolicies.ts b/SW.Bitween.Web/ClientApp/src/api/http/retryPolicies.ts index 5234d721..61d905b4 100644 --- a/SW.Bitween.Web/ClientApp/src/api/http/retryPolicies.ts +++ b/SW.Bitween.Web/ClientApp/src/api/http/retryPolicies.ts @@ -16,6 +16,7 @@ import { type Paged, } from "../types"; import { get, post, request } from "./request"; +import type { RetrySuggestion } from "../types"; import { buildListQuery, SEARCHY_RULE } from "./searchQuery"; interface SearchyResponse { @@ -267,6 +268,15 @@ export const retryPolicyMethods = { return { id, name, groups: [], createdOn: "", alertChannelId: null }; }, + async suggestRetryPolicy(subscriptionId: number, signal?: AbortSignal): Promise { + return request("/retrysuggestions", { method: "POST", body: { subscriptionId }, signal }); + }, + + /** Saves a suggestion's groups exactly as the server wrote them, so no field is lost on the way. */ + async saveSuggestedRetryPolicy(name: string, groups: RetrySuggestion["groups"]): Promise { + return post("/retrypolicies", { name, groups }); + }, + async updateRetryPolicy( id: number, changes: { diff --git a/SW.Bitween.Web/ClientApp/src/api/types.ts b/SW.Bitween.Web/ClientApp/src/api/types.ts index e69370c7..4135594a 100644 --- a/SW.Bitween.Web/ClientApp/src/api/types.ts +++ b/SW.Bitween.Web/ClientApp/src/api/types.ts @@ -2608,3 +2608,28 @@ export interface MirrorBring { needsYou: MirrorNote[]; failed: { section: string; noun: string; key: string; error: string }[]; } + +/** A retry policy the AI proposed from a subscription's failures. Nothing is saved until it's created. */ +export interface RetrySuggestion { + name: string; + summary: string; + /** In the server's own shape (delays in ms), passed back untouched to save them. */ + groups: { + name: string; + priority: number; + appliesTo: string[]; + action: "Allow" | "Block"; + matchers: { type: string; value?: string; pattern?: string; path?: string }[]; + budget: { + maxAttemptsPerError: number; + maxAttemptsTotal: number | null; + delayStrategy: { type: string; delayMs?: number; initialDelayMs?: number; incrementMs?: number; multiplier?: number; maxDelayMs?: number }; + } | null; + notes?: string | null; + }[]; + coverage: { error: string; count: number; retried: number; fixedByRetry: number; now: string; suggested: string }[]; + failures: number; + days: number; + model: string | null; + sent: string; +} diff --git a/SW.Bitween.Web/ClientApp/src/pages/subscriptions/__tests__/suggestRetryPolicy.test.tsx b/SW.Bitween.Web/ClientApp/src/pages/subscriptions/__tests__/suggestRetryPolicy.test.tsx new file mode 100644 index 00000000..db7dc647 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/subscriptions/__tests__/suggestRetryPolicy.test.tsx @@ -0,0 +1,99 @@ +import { screen, waitFor, within } from "@testing-library/react"; +import { http, HttpResponse } from "msw"; +import { describe, expect, it } from "vitest"; +import { apiPath, renderApp } from "../../../__tests__/support/renderApp"; +import { backend, sub } from "./subscriptionBackend"; + +/** + * Suggest a retry policy with AI, from the subscription page. What is sent and how the suggestion + * is checked is the C# suite's business (RetryPolicyPromptTests, RetrySuggestionTests); this is the + * dialog, saving the suggestion as a policy, and picking it. + */ + +globalThis.ResizeObserver ??= class { + observe() {} + unobserve() {} + disconnect() {} +} as unknown as typeof ResizeObserver; + +const GROUPS = [ + { + name: "Outages", + priority: 0, + appliesTo: ["Error"], + action: "Allow", + matchers: [{ type: "exceptionType", value: "System.Net.Http.HttpRequestException" }], + budget: { + maxAttemptsPerError: 3, + maxAttemptsTotal: 50, + whenExhausted: "Hold", + delayStrategy: { type: "exponential", initialDelayMs: 30000, multiplier: 2, maxDelayMs: 600000 }, + }, + notes: "Connection failures pass by themselves.", + }, +]; + +describe("suggesting a retry policy", { timeout: 20000 }, () => { + it("shows what would change, saves the suggestion as a new policy and picks it", async () => { + const created: unknown[] = []; + const { handlers, saves } = backend([sub()]); + const { user } = renderApp("/subscriptions/10", { + config: { aiConfigured: true }, + handlers: [ + http.post(apiPath("/retrysuggestions"), async ({ request }) => { + expect(await request.json()).toEqual({ subscriptionId: 10 }); + return HttpResponse.json({ + name: "Retry outages", + summary: "Retries connection failures, stops the rest.", + groups: GROUPS, + coverage: [ + { + error: "System.Net.Http.HttpRequestException: Connection refused ()", + count: 40, retried: 10, fixedByRetry: 9, + now: "Not retried: no retry policy", + suggested: 'Retried up to 3 times by "Outages"', + }, + ], + failures: 40, days: 30, model: "claude-sonnet-5-5", sent: "...", + }); + }), + http.post(apiPath("/retrypolicies"), async ({ request }) => { + created.push(await request.json()); + return HttpResponse.json(9); + }), + http.get(apiPath("/retrypolicies"), () => + HttpResponse.json({ result: created.length ? [{ id: 9, name: "Retry outages", groupCount: 1, usedByCount: 0 }] : [], totalCount: created.length }), + ), + ...handlers, + ], + }); + + await user.click(await screen.findByRole("button", { name: /Suggest with AI/ })); + const dialog = await screen.findByRole("dialog", { name: "Suggest a retry policy" }); + await user.click(within(dialog).getByRole("button", { name: "Suggest" })); + + expect(await within(dialog).findByText("Retries connection failures, stops the rest.")).toBeVisible(); + expect(within(dialog).getByText(/is a System\.Net\.Http\.HttpRequestException · up to 3 per exchange, 50 in all/)).toBeVisible(); + expect(within(dialog).getByText("9 of 10")).toBeVisible(); + expect(within(dialog).getByText('Retried up to 3 times by "Outages"')).toBeVisible(); + + await user.click(within(dialog).getByRole("button", { name: "Save as a new policy" })); + + // Saved exactly as the server wrote it: nothing, like the hold, is lost on the way. + await waitFor(() => expect(created).toEqual([{ name: "Retry outages", groups: GROUPS }])); + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + await user.click(screen.getByRole("button", { name: "Save changes" })); + await waitFor(() => expect(saves).toHaveLength(1)); + expect(saves[0].retryPolicyId).toBe(9); + }); + + it("says where to turn it on until Settings has a provider key", async () => { + const { handlers } = backend([sub()]); + const { user } = renderApp("/subscriptions/10", { config: { aiConfigured: false }, handlers }); + + await user.click(await screen.findByRole("button", { name: /Suggest with AI/ })); + + const dialog = await screen.findByRole("dialog", { name: "Suggest a retry policy" }); + expect(within(dialog).getByRole("link", { name: "Open Settings, AI in a new tab" })).toHaveAttribute("target", "_blank"); + }); +}); diff --git a/SW.Bitween.Web/ClientApp/src/pages/subscriptions/studio/LaneAndRetry.tsx b/SW.Bitween.Web/ClientApp/src/pages/subscriptions/studio/LaneAndRetry.tsx index 7b7c8a92..5e94034d 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/subscriptions/studio/LaneAndRetry.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/subscriptions/studio/LaneAndRetry.tsx @@ -1,5 +1,6 @@ import { useState } from "react"; import { Link } from "react-router"; +import { Sparkles } from "lucide-react"; import { useQuery } from "@tanstack/react-query"; import { api } from "../../../api"; import { useSessionCan } from "../../../auth/useSessionCan"; @@ -9,6 +10,7 @@ import { CreateRetryPolicyDialog } from "../../../components/config/RetryPolicyD import { keys } from "../../../api/queryKeys"; import { workGroupQueueName } from "../../../lib/busMessageName"; import { Fact } from "./Fact"; +import { SuggestRetryPolicy } from "./SuggestRetryPolicy"; /** * Which lane a subscription runs in, and what happens when it fails. @@ -30,6 +32,7 @@ export function LaneAndRetry({ idPrefix, inlineRetryGroups = null, inlineReplaced = false, + subscriptionId, }: { workGroupId: number | null; retryPolicyId: number | null; @@ -42,6 +45,8 @@ export function LaneAndRetry({ canEdit: boolean; /** Ids have to be unique per page - the create pages render this beside their own fields. */ idPrefix: string; + /** A saved subscription, whose own failures a retry policy can be suggested from. */ + subscriptionId?: number; }) { const canCreateWorkGroup = useSessionCan("workgroups.create"); const canCreateRetryPolicy = useSessionCan("retry-policies.create"); @@ -53,6 +58,7 @@ export function LaneAndRetry({ /** undefined = closed, null = creating, number = editing that group. */ const [groupDialog, setGroupDialog] = useState(undefined); const [creatingPolicy, setCreatingPolicy] = useState(false); + const [suggesting, setSuggesting] = useState(false); return ( <> @@ -136,6 +142,17 @@ export function LaneAndRetry({ + New )} + {canCreateRetryPolicy && canEdit && subscriptionId !== undefined && ( + + )} {groupDialog !== undefined && ( @@ -145,6 +162,13 @@ export function LaneAndRetry({ onSaved={onWorkGroupChange} /> )} + {suggesting && subscriptionId !== undefined && ( + setSuggesting(false)} + onCreated={onRetryPolicyChange} + /> + )} {creatingPolicy && ( // Selects the new policy rather than navigating to it - the rest of the policy is // configured behind "View", and leaving here would abandon the subscription edit. diff --git a/SW.Bitween.Web/ClientApp/src/pages/subscriptions/studio/Overview.tsx b/SW.Bitween.Web/ClientApp/src/pages/subscriptions/studio/Overview.tsx index 55b4a8e4..26cdafc2 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/subscriptions/studio/Overview.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/subscriptions/studio/Overview.tsx @@ -279,6 +279,7 @@ export function Overview({ idPrefix="in" inlineRetryGroups={s.customRetryGroups ?? null} inlineReplaced={draft.retryPolicyId !== s.retryPolicyId} + subscriptionId={s.id} /> { + const s = Math.round((ms ?? 0) / 1000); + return s >= 120 ? `${Math.round(s / 60)} min` : `${s}s`; +}; + +function delayLabel(d: NonNullable["delayStrategy"]) { + if (d.type === "fixed") return `every ${seconds(d.delayMs)}`; + if (d.type === "linear") return `${seconds(d.initialDelayMs)}, then ${seconds(d.incrementMs)} longer each time`; + return `${seconds(d.initialDelayMs)}, doubling${d.multiplier && d.multiplier !== 2 ? ` (x${d.multiplier})` : ""} up to ${seconds(d.maxDelayMs)}`; +} + +const matcherLabel = (m: Group["matchers"][number]) => + m.type === "contains" + ? `contains "${m.value}"` + : m.type === "regex" + ? `matches /${m.pattern}/` + : m.type === "exceptionType" + ? `is a ${m.value}` + : `${m.path} ${m.value ?? ""}`.trim(); + +/** + * Suggest a retry policy with AI, from this subscription's own failures. Nothing is saved until + * "Save as a new policy", and then the new policy is only picked here: the subscription saves as + * it always does. + */ +export function SuggestRetryPolicy({ + subscriptionId, + onClose, + onCreated, +}: { + subscriptionId: number; + onClose: () => void; + onCreated: (id: number) => void; +}) { + const config = useQuery({ queryKey: keys.appConfig, queryFn: getAppConfig }); + const canSetUp = useSessionCan("settings.edit"); + const queryClient = useQueryClient(); + const [busy, setBusy] = useState(false); + const [elapsed, setElapsed] = useState(0); + const [error, setError] = useState(null); + const [suggestion, setSuggestion] = useState(null); + const [name, setName] = useState(""); + const [showSent, setShowSent] = useState(false); + const [saving, setSaving] = useState(false); + const controller = useRef(null); + + useEffect(() => () => controller.current?.abort(), []); + useEffect(() => { + if (!busy) return; + setElapsed(0); + const timer = setInterval(() => setElapsed((s) => s + 1), 1000); + return () => clearInterval(timer); + }, [busy]); + + async function suggest() { + controller.current = new AbortController(); + setBusy(true); + setError(null); + try { + const res = await api.suggestRetryPolicy(subscriptionId, controller.current.signal); + setSuggestion(res); + setName(res.name); + } catch (e) { + if (!controller.current.signal.aborted) setError(e instanceof Error ? e.message : String(e)); + } finally { + setBusy(false); + } + } + + async function save() { + if (!suggestion || !name.trim()) return; + setSaving(true); + setError(null); + try { + const id = await api.saveSuggestedRetryPolicy(name.trim(), suggestion.groups); + await queryClient.invalidateQueries({ queryKey: keys.retryPolicies.list }); + onCreated(id); + onClose(); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } finally { + setSaving(false); + } + } + + if (config.data && !config.data.aiConfigured) + return ( + +
+

AI isn't set up on this Bitween yet: it needs an AI provider and its API key under Settings, AI.

+ {canSetUp ? ( + + Open Settings, AI in a new tab + + ) : ( +

Ask an administrator to set it up.

+ )} +
+ + ); + + return ( + + {!suggestion ? ( +
+

+ Proposes which of this subscription's failures to retry, how often and how soon, from its failures in the + last 30 days. Nothing changes until you save the suggestion as a new policy. +

+

+ Sent to the AI provider: each kind of error as one line with every value masked, how often it happened and how + often a retry fixed it, and the current policy's rules. Never a document or a response. +

+ {error && {error}} +
+ {busy ? ( + <> + + + Suggesting… {elapsed}s · usually 10-30 seconds + + + + ) : ( + + )} +
+
+ ) : ( +
+ {suggestion.summary &&

{suggestion.summary}

} + +
+ {suggestion.groups.map((g, i) => ( +
+
+ {g.name} + + {g.action === "Block" ? "Stops" : "Retries"} + +
+

+ When the error {g.matchers.map(matcherLabel).join(" or ")} + {g.budget && + ` · up to ${g.budget.maxAttemptsPerError} per exchange` + + (g.budget.maxAttemptsTotal === null ? "" : `, ${g.budget.maxAttemptsTotal} in all until it next succeeds`) + + ` · waits ${delayLabel(g.budget.delayStrategy)}`} +

+ {g.notes &&

{g.notes}

} +
+ ))} +
+ +
+

+ What changes, on its {suggestion.failures.toLocaleString()} failures in the last {suggestion.days} days +

+
+ + + + + + + + + + + + {suggestion.coverage.map((c, i) => ( + + + + + + + + ))} + +
ErrorCount + Fixed by retry + Now + Suggested +
{c.error}{c.count.toLocaleString()} + {c.retried === 0 ? "-" : `${c.fixedByRetry} of ${c.retried}`} + {c.now}{c.suggested}
+
+
+ +

+ {suggestion.model ? `Written by ${suggestion.model}. ` : ""} + +

+ {showSent && ( + 
+              {suggestion.sent}
+
+ )} + + {error && {error}} +
+ + + +
+

+ It's then picked for this subscription; save the subscription to start using it. +

+
+ )} + + ); +} diff --git a/docs/admin-ui.md b/docs/admin-ui.md index 7e64b5ac..c567e168 100644 --- a/docs/admin-ui.md +++ b/docs/admin-ui.md @@ -45,7 +45,7 @@ Selecting a stage opens its editor. Adapter stages pick an adapter and fill in i When a stage uses a database or broker adapter, it also binds the subscription to a data source and picks the statement and operation, the receive mode and batch size, or where to publish. See [Databases](databases.md) and [External brokers](external-brokers.md). -With no stage selected, the overview shows health, next and last run, work group, retry policy, a retry budget banner once a budget is spent, receive attempts or recent runs, recent exchanges, notifications and change history. The **Notifications** card picks channels and when each sends; a new channel can be made from there without losing the page's edits. Header buttons pause or resume, receive or roll up now, create an aggregation of this subscription, and delete it. +With no stage selected, the overview shows health, next and last run, work group, retry policy, a retry budget banner once a budget is spent, receive attempts or recent runs, recent exchanges, notifications and change history. The **Notifications** card picks channels and when each sends; a new channel can be made from there without losing the page's edits. **Suggest with AI**, beside the retry policy, proposes one from the subscription's own failures ([Suggesting a policy with AI](retries-and-alerts.md#suggesting-a-policy-with-ai)). Header buttons pause or resume, receive or roll up now, create an aggregation of this subscription, and delete it. Edits stay a draft until saved from the bar at the bottom of the page. diff --git a/docs/api-reference.md b/docs/api-reference.md index 97156ddf..6ca83e8b 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -296,6 +296,7 @@ See [Data sources](data-sources.md) and [Databases](databases.md). | `POST /api/workgroups/{id}/delete` | `workgroups.delete`. Refused while subscriptions use it. | | `GET /api/retrypolicies`, `GET /api/retrypolicies/{id}` | `retry-policies.view` | | `POST /api/retrypolicies` | `retry-policies.create`. `{ name, groups, alertChannelId }`; see [Retry policies](retries-and-alerts.md#retry-policies). An alert names a notification channel; `alertHandlerId` is refused. | +| `POST /api/retrysuggestions` | `subscriptions.view` and `retry-policies.create` | Suggest a retry policy with AI from a subscription's failures: `{ subscriptionId, days }` (1-90, default 30), replying with `name`, `summary`, `groups` (ready for `POST /api/retrypolicies`), `coverage`, `model` and `sent`. Saves nothing. See [Suggesting a policy with AI](retries-and-alerts.md#suggesting-a-policy-with-ai). | | `POST /api/retrypolicies/{id}` | `retry-policies.edit` | | `DELETE /api/retrypolicies/{id}` | `retry-policies.delete` | | `POST /api/retrypolicies/test` | `retry-policies.view` | diff --git a/docs/configuration.md b/docs/configuration.md index 3c432098..2c9bd2aa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -216,7 +216,7 @@ The **Mail server** section is the one SMTP server every Email notification chan ### AI -The **AI** section is the provider every AI feature calls; they are [Generate with AI](mapping.md#generate-with-ai) in the mapping editor and [Explain this failure](#explaining-a-failure) on a failed exchange. It is read on each call, and AI features stay off until an API key is set. +The **AI** section is the provider every AI feature calls; they are [Generate with AI](mapping.md#generate-with-ai) in the mapping editor and [Explain this failure](#explaining-a-failure) on a failed exchange, and [Suggest with AI](retries-and-alerts.md#suggesting-a-policy-with-ai) beside a subscription's retry policy. It is read on each call, and AI features stay off until an API key is set. - **Provider** is `Anthropic` or `OpenAI`. Any other value, such as a misspelling in configuration, leaves AI features off. `OpenAI` also covers anything that speaks its chat completions API at **API address**: an Azure OpenAI resource (`https://your-resource.openai.azure.com/openai/v1/`), a local Ollama (`http://localhost:11434/v1/`) or a proxy. - **Model** empty uses `claude-opus-5-5` with Anthropic. OpenAI needs one named, or the Azure deployment's name. diff --git a/docs/retries-and-alerts.md b/docs/retries-and-alerts.md index 19be7445..c4a13e32 100644 --- a/docs/retries-and-alerts.md +++ b/docs/retries-and-alerts.md @@ -175,6 +175,18 @@ Deleting a policy is refused while a subscription uses it. Removing a group dele It returns each simulated attempt with the matched group, whether it would retry, the delay and the reason, and stops at the first refusal. The policy page's test panel runs the unsaved draft. +### Suggesting a policy with AI + +**Suggest with AI**, beside the retry policy on a subscription's page, proposes a policy from that subscription's own failures in the last 30 days, using the provider under [Settings, AI](configuration.md#ai). It sends the provider only: + +- each kind of error (the newest 2,000 failures, grouped by their first line, the 15 most frequent) as one line with its values masked: quoted text, numbers of four or more digits, email addresses and addresses (``) are replaced; +- how often each happened, how often it was retried and how often the retry succeeded; +- the subscription's kind and adapters, and the current policy's groups without their alert settings. + +Documents and response bodies are never sent; bad responses are only counted. Failures from before the upgrade that started recording each result's subscription (2026-10-14) aren't seen, so on an upgraded install the first suggestions learn only from failures since. The suggestion is checked as a saved policy would be, alerts are left to you, and it is then run on the real errors on the server: **What changes** shows, for each kind of error, what the current policy does with it and what the suggestion would. **See what was sent** shows exactly what went. + +Nothing changes until **Save as a new policy**, which creates the policy and picks it for the subscription; saving the subscription starts using it. Asking needs `subscriptions.view` and `retry-policies.create`, and is recorded as a `suggest-retry-policy` operator action before anything is sent. `POST /api/retrysuggestions { subscriptionId, days }` is the call behind it. + ## Scheduled retries An automatic retry waits as a *delayed retry* until its time comes. From 0fccde5e9d16982be5439fbc9a74631442b049ac Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Thu, 29 Oct 2026 12:54:39 +0300 Subject: [PATCH 2/2] fix: retry suggestion review findings --- .../Resources/RetrySuggestions/Suggest.cs | 37 +++++++++++++------ SW.Bitween.Api/Services/Ai/FailurePrompt.cs | 2 +- SW.Bitween.Api/Services/Ai/Masking.cs | 12 ++++++ .../Services/Ai/RetryPolicyPrompt.cs | 8 ++-- SW.Bitween.UnitTests/Ai/MaskingTests.cs | 13 +++++++ .../Ai/RetryPolicyPromptTests.cs | 1 + .../studio/SuggestRetryPolicy.tsx | 13 +++++-- docs/api-reference.md | 2 +- docs/configuration.md | 6 +-- docs/retries-and-alerts.md | 4 +- 10 files changed, 72 insertions(+), 26 deletions(-) diff --git a/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs b/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs index 692856a2..3e038dd7 100644 --- a/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs +++ b/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs @@ -45,7 +45,7 @@ public class RetrySuggestionResponse public int Days { get; set; } public string? Model { get; set; } - /// Exactly what went to the provider. + /// The message that went to the provider, after the fixed . public string Sent { get; set; } = ""; } @@ -83,18 +83,20 @@ public async Task Handle(RetrySuggestionRequest request var since = DateTime.UtcNow.AddDays(-days); // The newest failures, by the (SubscriptionId, FinishedOn) index; only the start of each - // error, which is where its type and message are. - var failed = await dbContext.Set().AsNoTracking() - .Where(r => r.SubscriptionId == subscription.Id && r.FinishedOn >= since && (!r.Success || r.ResponseBad)) + // error, which is where its type and message are. Bad responses are counted apart, so many + // of them can't crowd the failures out. + var inWindow = dbContext.Set().AsNoTracking() + .Where(r => r.SubscriptionId == subscription.Id && r.FinishedOn >= since); + var errors = await inWindow + .Where(r => !r.Success) .OrderByDescending(r => r.FinishedOn) .Take(RetryPolicyPrompt.MaxFailures) .Select(r => new { - r.Id, r.Success, r.FinishedOn, + r.Id, r.FinishedOn, Exception = r.Exception == null ? null : r.Exception.Substring(0, RetryPolicyPrompt.SampleLength), }) .ToListAsync(); - var errors = failed.Where(f => !f.Success).ToList(); if (errors.Count == 0) throw new SWValidationException("NO_FAILURES", $"This subscription has no failed exchanges in the last {days} days, so there is nothing to learn a policy from."); @@ -107,22 +109,33 @@ public async Task Handle(RetrySuggestionRequest request .ToListAsync(); var retriedFrom = retries.GroupBy(r => r.RetryFor!).ToDictionary(g => g.Key, g => g.Any(r => r.Fixed)); - var kinds = errors + var groups = errors .GroupBy(e => ErrorFingerprint.Of(e.Exception ?? "")) + .OrderByDescending(g => g.Count()) + .Take(RetryPolicyPrompt.MaxKinds) + .ToList(); + + // Each kind's newest error in full, as the live retry sees it: a matcher may hit deep in the stack. + var newestIds = groups.Select(g => g.First().Id).ToList(); + var full = await dbContext.Set().AsNoTracking() + .Where(r => newestIds.Contains(r.Id)) + .Select(r => new { r.Id, r.Exception }) + .ToDictionaryAsync(r => r.Id, r => r.Exception); + + var kinds = groups .Select(g => { var newest = g.First(); - var sample = newest.Exception ?? ""; - return new FailureKind(sample, RetryPolicyPrompt.Mask(ErrorFingerprint.Sample(sample)), g.Count(), + return new FailureKind(full.GetValueOrDefault(newest.Id) ?? newest.Exception ?? "", + RetryPolicyPrompt.Mask(ErrorFingerprint.Sample(newest.Exception ?? "")), g.Count(), g.Count(e => retriedFrom.ContainsKey(e.Id)), g.Count(e => retriedFrom.TryGetValue(e.Id, out var ok) && ok), g.Min(e => e.FinishedOn), g.Max(e => e.FinishedOn)); }) - .OrderByDescending(k => k.Count) - .Take(RetryPolicyPrompt.MaxKinds) .ToList(); + var badResponses = await inWindow.CountAsync(r => r.Success && r.ResponseBad); var describe = $"Subscription kind: {subscription.Type}; mapper: {subscription.MapperId ?? "none"}; handler: {subscription.HandlerId ?? "none"}"; - var message = RetryPolicyPrompt.Message(describe, days, errors.Count, failed.Count - errors.Count, kinds, current); + var message = RetryPolicyPrompt.Message(describe, days, errors.Count, badResponses, kinds, current); // Recorded before it goes, so a call that fails or times out after sending is on record too. await dbContext.RecordNowAsync(requestContext, OperatorAction.SuggestRetryPolicy, $"subscription {subscription.Id}", $"Sent to {models.ModelName}: {errors.Count} failures, errors masked."); diff --git a/SW.Bitween.Api/Services/Ai/FailurePrompt.cs b/SW.Bitween.Api/Services/Ai/FailurePrompt.cs index c1423c92..297a24c7 100644 --- a/SW.Bitween.Api/Services/Ai/FailurePrompt.cs +++ b/SW.Bitween.Api/Services/Ai/FailurePrompt.cs @@ -21,7 +21,7 @@ public static class FailurePrompt exchanges can be retried by a retry policy, or by hand. Every value in the documents and the error has been masked before you see it: text is , - numbers are , booleans are , and long lists show their first entries. Keys, + numbers are , booleans are , credentials and tokens are , and long lists show their first entries. Keys, structure and the error's wording are real. Never guess at a masked value; reason from structure, names, types and the error. diff --git a/SW.Bitween.Api/Services/Ai/Masking.cs b/SW.Bitween.Api/Services/Ai/Masking.cs index bab6f41e..e3eeb346 100644 --- a/SW.Bitween.Api/Services/Ai/Masking.cs +++ b/SW.Bitween.Api/Services/Ai/Masking.cs @@ -30,6 +30,7 @@ public sealed class Masking { public const string Text = ""; public const string Number = ""; + public const string Secret = ""; public const string Boolean = ""; /// A long list shows this many entries, then how many more there were. @@ -92,6 +93,10 @@ string Words(string text) text = Email.Replace(text, ""); text = Quoted.Replace(text, m => $"{m.Groups["q"].Value}{Text}{m.Groups["q"].Value}"); text = QueryString.Replace(text, "?"); + // Credentials a partner's error can echo unquoted: "Bearer eyJ...", "api_key=...", or a bare token. + text = Scheme.Replace(text, m => $"{m.Groups["scheme"].Value} {Secret}"); + text = Labelled.Replace(text, m => $"{m.Groups["label"].Value}{m.Groups["sep"].Value}{Secret}"); + text = TokenLike.Replace(text, Secret); return LongNumber.Replace(text, Number); } @@ -182,4 +187,11 @@ static string Cut(string text) => static readonly Regex Quoted = new(@"(?['""])(?!<)[^'""\r\n]{1,200}\k", RegexOptions.Compiled); static readonly Regex QueryString = new(@"\?[^\s'""<>]+", RegexOptions.Compiled); static readonly Regex LongNumber = new(@"(?bearer|basic)\s+(?!<)[A-Za-z0-9._~+/=-]{8,}", RegexOptions.Compiled | RegexOptions.IgnoreCase); + static readonly Regex Labelled = new( + @"(?