diff --git a/SW.Bitween.Api/Domain/OperatorAction.cs b/SW.Bitween.Api/Domain/OperatorAction.cs index 7b2685e8..41c8e1ec 100644 --- a/SW.Bitween.Api/Domain/OperatorAction.cs +++ b/SW.Bitween.Api/Domain/OperatorAction.cs @@ -21,6 +21,7 @@ public class OperatorAction /// A failed exchange sent, masked, to the AI provider under Settings, AI for an explanation. public const string ExplainFailure = "explain-failure"; + public const string SuggestRetryPolicy = "suggest-retry-policy"; private OperatorAction() { diff --git a/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs b/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs new file mode 100644 index 00000000..3e038dd7 --- /dev/null +++ b/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs @@ -0,0 +1,169 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.Bitween.Services.Ai; +using SW.Bitween.Services.Metrics; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.RetrySuggestions; + +public class RetrySuggestionRequest +{ + public int SubscriptionId { get; set; } + + /// How far back to look, 1 to 90 days. + public int Days { get; set; } = 30; +} + +/// One kind of failure, and what the current and the suggested policy each do with it. +public class RetrySuggestionCoverage +{ + /// The error's first line, masked as it was sent. + public string Error { get; set; } = ""; + public int Count { get; set; } + public int Retried { get; set; } + public int FixedByRetry { get; set; } + public string Now { get; set; } = ""; + public string Suggested { get; set; } = ""; +} + +public class RetrySuggestionResponse +{ + public string Name { get; set; } = ""; + public string Summary { get; set; } = ""; + + /// The groups, ready for POST /api/retrypolicies; nothing is saved until then. + public List Groups { get; set; } = new(); + + public List Coverage { get; set; } = new(); + public int Failures { get; set; } + public int Days { get; set; } + public string? Model { get; set; } + + /// The message that went to the provider, after the fixed . + public string Sent { get; set; } = ""; +} + +/// +/// Proposes a retry policy for a subscription from its own recent failures. +/// +/// +/// +/// The provider sees each kind of failure as a masked line with counts and how often a retry fixed +/// it, and the current policy's rules. Never a document or a response body. The suggestion is then +/// checked as a saved policy would be and run against the real failures here, so the coverage +/// shows what changes for each before anyone saves it. +/// +/// +/// It saves nothing: saving the suggestion is creating a policy, so asking needs that right too, +/// and a question nobody could act on isn't sent. Each one is recorded as an operator action, +/// since it sends something about the subscription to a third party. +/// +/// +public class Suggest(RequestContext requestContext, BitweenDbContext dbContext, AiModels models) + : ICommandHandler +{ + public async Task Handle(RetrySuggestionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); + await requestContext.EnsurePermission(dbContext, Model.Permissions.RetryPolicies.Create); + var model = models.Current(); + + var subscription = await dbContext.Set().AsNoTracking() + .Include(s => s.RetryPolicy) + .FirstOrDefaultAsync(s => s.Id == request.SubscriptionId) + ?? throw new SWNotFoundException(request.SubscriptionId.ToString()); + var current = subscription.CustomRetryPolicy?.Groups ?? subscription.RetryPolicy?.Groups; + var days = Math.Clamp(request.Days, 1, 90); + var since = DateTime.UtcNow.AddDays(-days); + + // The newest failures, by the (SubscriptionId, FinishedOn) index; only the start of each + // error, which is where its type and message are. Bad responses are counted apart, so many + // of them can't crowd the failures out. + var inWindow = dbContext.Set().AsNoTracking() + .Where(r => r.SubscriptionId == subscription.Id && r.FinishedOn >= since); + var errors = await inWindow + .Where(r => !r.Success) + .OrderByDescending(r => r.FinishedOn) + .Take(RetryPolicyPrompt.MaxFailures) + .Select(r => new + { + r.Id, r.FinishedOn, + Exception = r.Exception == null ? null : r.Exception.Substring(0, RetryPolicyPrompt.SampleLength), + }) + .ToListAsync(); + if (errors.Count == 0) + throw new SWValidationException("NO_FAILURES", + $"This subscription has no failed exchanges in the last {days} days, so there is nothing to learn a policy from."); + + // Whether each one's retry succeeded: the attempt made from it, by the RetryFor index. + var ids = errors.Select(e => e.Id).ToList(); + var retries = await dbContext.Set().AsNoTracking() + .Where(x => x.RetryFor != null && ids.Contains(x.RetryFor)) + .Join(dbContext.Set(), x => x.Id, r => r.Id, (x, r) => new { x.RetryFor, Fixed = r.Success && !r.ResponseBad }) + .ToListAsync(); + var retriedFrom = retries.GroupBy(r => r.RetryFor!).ToDictionary(g => g.Key, g => g.Any(r => r.Fixed)); + + var groups = errors + .GroupBy(e => ErrorFingerprint.Of(e.Exception ?? "")) + .OrderByDescending(g => g.Count()) + .Take(RetryPolicyPrompt.MaxKinds) + .ToList(); + + // Each kind's newest error in full, as the live retry sees it: a matcher may hit deep in the stack. + var newestIds = groups.Select(g => g.First().Id).ToList(); + var full = await dbContext.Set().AsNoTracking() + .Where(r => newestIds.Contains(r.Id)) + .Select(r => new { r.Id, r.Exception }) + .ToDictionaryAsync(r => r.Id, r => r.Exception); + + var kinds = groups + .Select(g => + { + var newest = g.First(); + return new FailureKind(full.GetValueOrDefault(newest.Id) ?? newest.Exception ?? "", + RetryPolicyPrompt.Mask(ErrorFingerprint.Sample(newest.Exception ?? "")), g.Count(), + g.Count(e => retriedFrom.ContainsKey(e.Id)), g.Count(e => retriedFrom.TryGetValue(e.Id, out var ok) && ok), + g.Min(e => e.FinishedOn), g.Max(e => e.FinishedOn)); + }) + .ToList(); + var badResponses = await inWindow.CountAsync(r => r.Success && r.ResponseBad); + + var describe = $"Subscription kind: {subscription.Type}; mapper: {subscription.MapperId ?? "none"}; handler: {subscription.HandlerId ?? "none"}"; + var message = RetryPolicyPrompt.Message(describe, days, errors.Count, badResponses, kinds, current); + // Recorded before it goes, so a call that fails or times out after sending is on record too. + await dbContext.RecordNowAsync(requestContext, OperatorAction.SuggestRetryPolicy, $"subscription {subscription.Id}", + $"Sent to {models.ModelName}: {errors.Count} failures, errors masked."); + var suggestion = RetryPolicyPrompt.Parse( + await model.Complete(new AiPrompt(RetryPolicyPrompt.Instructions, message, Json: true, MaxTokens: 4000))); + + var coverage = new List(); + foreach (var kind in kinds) + coverage.Add(new RetrySuggestionCoverage + { + Error = kind.Masked, + Count = kind.Count, + Retried = kind.Retried, + FixedByRetry = kind.FixedByRetry, + Now = await RetryPolicyPrompt.Outcome(current, kind.Sample), + Suggested = await RetryPolicyPrompt.Outcome(suggestion.Groups, kind.Sample), + }); + + return new RetrySuggestionResponse + { + Name = suggestion.Name, + Summary = suggestion.Summary, + Groups = suggestion.Groups, + Coverage = coverage, + Failures = errors.Count, + Days = days, + Model = models.ModelName, + Sent = message, + }; + } +} diff --git a/SW.Bitween.Api/Services/Ai/FailurePrompt.cs b/SW.Bitween.Api/Services/Ai/FailurePrompt.cs index c1423c92..297a24c7 100644 --- a/SW.Bitween.Api/Services/Ai/FailurePrompt.cs +++ b/SW.Bitween.Api/Services/Ai/FailurePrompt.cs @@ -21,7 +21,7 @@ public static class FailurePrompt exchanges can be retried by a retry policy, or by hand. Every value in the documents and the error has been masked before you see it: text is , - numbers are , booleans are , and long lists show their first entries. Keys, + numbers are , booleans are , credentials and tokens are , and long lists show their first entries. Keys, structure and the error's wording are real. Never guess at a masked value; reason from structure, names, types and the error. diff --git a/SW.Bitween.Api/Services/Ai/Masking.cs b/SW.Bitween.Api/Services/Ai/Masking.cs index bab6f41e..e3eeb346 100644 --- a/SW.Bitween.Api/Services/Ai/Masking.cs +++ b/SW.Bitween.Api/Services/Ai/Masking.cs @@ -30,6 +30,7 @@ public sealed class Masking { public const string Text = ""; public const string Number = ""; + public const string Secret = ""; public const string Boolean = ""; /// A long list shows this many entries, then how many more there were. @@ -92,6 +93,10 @@ string Words(string text) text = Email.Replace(text, ""); text = Quoted.Replace(text, m => $"{m.Groups["q"].Value}{Text}{m.Groups["q"].Value}"); text = QueryString.Replace(text, "?"); + // Credentials a partner's error can echo unquoted: "Bearer eyJ...", "api_key=...", or a bare token. + text = Scheme.Replace(text, m => $"{m.Groups["scheme"].Value} {Secret}"); + text = Labelled.Replace(text, m => $"{m.Groups["label"].Value}{m.Groups["sep"].Value}{Secret}"); + text = TokenLike.Replace(text, Secret); return LongNumber.Replace(text, Number); } @@ -182,4 +187,11 @@ static string Cut(string text) => static readonly Regex Quoted = new(@"(?['""])(?!<)[^'""\r\n]{1,200}\k", RegexOptions.Compiled); static readonly Regex QueryString = new(@"\?[^\s'""<>]+", RegexOptions.Compiled); static readonly Regex LongNumber = new(@"(?bearer|basic)\s+(?!<)[A-Za-z0-9._~+/=-]{8,}", RegexOptions.Compiled | RegexOptions.IgnoreCase); + static readonly Regex Labelled = new( + @"(?