diff --git a/SW.Bitween.Api/Domain/OperatorAction.cs b/SW.Bitween.Api/Domain/OperatorAction.cs
index 7b2685e8..41c8e1ec 100644
--- a/SW.Bitween.Api/Domain/OperatorAction.cs
+++ b/SW.Bitween.Api/Domain/OperatorAction.cs
@@ -21,6 +21,7 @@ public class OperatorAction
/// A failed exchange sent, masked, to the AI provider under Settings, AI for an explanation.
public const string ExplainFailure = "explain-failure";
+ public const string SuggestRetryPolicy = "suggest-retry-policy";
private OperatorAction()
{
diff --git a/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs b/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs
new file mode 100644
index 00000000..3e038dd7
--- /dev/null
+++ b/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs
@@ -0,0 +1,169 @@
+#nullable enable
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+using SW.Bitween.Domain;
+using SW.Bitween.Model;
+using SW.Bitween.Services.Ai;
+using SW.Bitween.Services.Metrics;
+using SW.PrimitiveTypes;
+
+namespace SW.Bitween.Resources.RetrySuggestions;
+
+public class RetrySuggestionRequest
+{
+ public int SubscriptionId { get; set; }
+
+ /// How far back to look, 1 to 90 days.
+ public int Days { get; set; } = 30;
+}
+
+/// One kind of failure, and what the current and the suggested policy each do with it.
+public class RetrySuggestionCoverage
+{
+ /// The error's first line, masked as it was sent.
+ public string Error { get; set; } = "";
+ public int Count { get; set; }
+ public int Retried { get; set; }
+ public int FixedByRetry { get; set; }
+ public string Now { get; set; } = "";
+ public string Suggested { get; set; } = "";
+}
+
+public class RetrySuggestionResponse
+{
+ public string Name { get; set; } = "";
+ public string Summary { get; set; } = "";
+
+ /// The groups, ready for POST /api/retrypolicies; nothing is saved until then.
+ public List Groups { get; set; } = new();
+
+ public List Coverage { get; set; } = new();
+ public int Failures { get; set; }
+ public int Days { get; set; }
+ public string? Model { get; set; }
+
+ /// The message that went to the provider, after the fixed .
+ public string Sent { get; set; } = "";
+}
+
+///
+/// Proposes a retry policy for a subscription from its own recent failures.
+///
+///
+///
+/// The provider sees each kind of failure as a masked line with counts and how often a retry fixed
+/// it, and the current policy's rules. Never a document or a response body. The suggestion is then
+/// checked as a saved policy would be and run against the real failures here, so the coverage
+/// shows what changes for each before anyone saves it.
+///
+///
+/// It saves nothing: saving the suggestion is creating a policy, so asking needs that right too,
+/// and a question nobody could act on isn't sent. Each one is recorded as an operator action,
+/// since it sends something about the subscription to a third party.
+///
+///
+public class Suggest(RequestContext requestContext, BitweenDbContext dbContext, AiModels models)
+ : ICommandHandler
+{
+ public async Task Handle(RetrySuggestionRequest request)
+ {
+ await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View);
+ await requestContext.EnsurePermission(dbContext, Model.Permissions.RetryPolicies.Create);
+ var model = models.Current();
+
+ var subscription = await dbContext.Set().AsNoTracking()
+ .Include(s => s.RetryPolicy)
+ .FirstOrDefaultAsync(s => s.Id == request.SubscriptionId)
+ ?? throw new SWNotFoundException(request.SubscriptionId.ToString());
+ var current = subscription.CustomRetryPolicy?.Groups ?? subscription.RetryPolicy?.Groups;
+ var days = Math.Clamp(request.Days, 1, 90);
+ var since = DateTime.UtcNow.AddDays(-days);
+
+ // The newest failures, by the (SubscriptionId, FinishedOn) index; only the start of each
+ // error, which is where its type and message are. Bad responses are counted apart, so many
+ // of them can't crowd the failures out.
+ var inWindow = dbContext.Set().AsNoTracking()
+ .Where(r => r.SubscriptionId == subscription.Id && r.FinishedOn >= since);
+ var errors = await inWindow
+ .Where(r => !r.Success)
+ .OrderByDescending(r => r.FinishedOn)
+ .Take(RetryPolicyPrompt.MaxFailures)
+ .Select(r => new
+ {
+ r.Id, r.FinishedOn,
+ Exception = r.Exception == null ? null : r.Exception.Substring(0, RetryPolicyPrompt.SampleLength),
+ })
+ .ToListAsync();
+ if (errors.Count == 0)
+ throw new SWValidationException("NO_FAILURES",
+ $"This subscription has no failed exchanges in the last {days} days, so there is nothing to learn a policy from.");
+
+ // Whether each one's retry succeeded: the attempt made from it, by the RetryFor index.
+ var ids = errors.Select(e => e.Id).ToList();
+ var retries = await dbContext.Set().AsNoTracking()
+ .Where(x => x.RetryFor != null && ids.Contains(x.RetryFor))
+ .Join(dbContext.Set(), x => x.Id, r => r.Id, (x, r) => new { x.RetryFor, Fixed = r.Success && !r.ResponseBad })
+ .ToListAsync();
+ var retriedFrom = retries.GroupBy(r => r.RetryFor!).ToDictionary(g => g.Key, g => g.Any(r => r.Fixed));
+
+ var groups = errors
+ .GroupBy(e => ErrorFingerprint.Of(e.Exception ?? ""))
+ .OrderByDescending(g => g.Count())
+ .Take(RetryPolicyPrompt.MaxKinds)
+ .ToList();
+
+ // Each kind's newest error in full, as the live retry sees it: a matcher may hit deep in the stack.
+ var newestIds = groups.Select(g => g.First().Id).ToList();
+ var full = await dbContext.Set().AsNoTracking()
+ .Where(r => newestIds.Contains(r.Id))
+ .Select(r => new { r.Id, r.Exception })
+ .ToDictionaryAsync(r => r.Id, r => r.Exception);
+
+ var kinds = groups
+ .Select(g =>
+ {
+ var newest = g.First();
+ return new FailureKind(full.GetValueOrDefault(newest.Id) ?? newest.Exception ?? "",
+ RetryPolicyPrompt.Mask(ErrorFingerprint.Sample(newest.Exception ?? "")), g.Count(),
+ g.Count(e => retriedFrom.ContainsKey(e.Id)), g.Count(e => retriedFrom.TryGetValue(e.Id, out var ok) && ok),
+ g.Min(e => e.FinishedOn), g.Max(e => e.FinishedOn));
+ })
+ .ToList();
+ var badResponses = await inWindow.CountAsync(r => r.Success && r.ResponseBad);
+
+ var describe = $"Subscription kind: {subscription.Type}; mapper: {subscription.MapperId ?? "none"}; handler: {subscription.HandlerId ?? "none"}";
+ var message = RetryPolicyPrompt.Message(describe, days, errors.Count, badResponses, kinds, current);
+ // Recorded before it goes, so a call that fails or times out after sending is on record too.
+ await dbContext.RecordNowAsync(requestContext, OperatorAction.SuggestRetryPolicy, $"subscription {subscription.Id}",
+ $"Sent to {models.ModelName}: {errors.Count} failures, errors masked.");
+ var suggestion = RetryPolicyPrompt.Parse(
+ await model.Complete(new AiPrompt(RetryPolicyPrompt.Instructions, message, Json: true, MaxTokens: 4000)));
+
+ var coverage = new List();
+ foreach (var kind in kinds)
+ coverage.Add(new RetrySuggestionCoverage
+ {
+ Error = kind.Masked,
+ Count = kind.Count,
+ Retried = kind.Retried,
+ FixedByRetry = kind.FixedByRetry,
+ Now = await RetryPolicyPrompt.Outcome(current, kind.Sample),
+ Suggested = await RetryPolicyPrompt.Outcome(suggestion.Groups, kind.Sample),
+ });
+
+ return new RetrySuggestionResponse
+ {
+ Name = suggestion.Name,
+ Summary = suggestion.Summary,
+ Groups = suggestion.Groups,
+ Coverage = coverage,
+ Failures = errors.Count,
+ Days = days,
+ Model = models.ModelName,
+ Sent = message,
+ };
+ }
+}
diff --git a/SW.Bitween.Api/Services/Ai/FailurePrompt.cs b/SW.Bitween.Api/Services/Ai/FailurePrompt.cs
index c1423c92..297a24c7 100644
--- a/SW.Bitween.Api/Services/Ai/FailurePrompt.cs
+++ b/SW.Bitween.Api/Services/Ai/FailurePrompt.cs
@@ -21,7 +21,7 @@ public static class FailurePrompt
exchanges can be retried by a retry policy, or by hand.
Every value in the documents and the error has been masked before you see it: text is ,
- numbers are , booleans are , and long lists show their first entries. Keys,
+ numbers are , booleans are , credentials and tokens are , and long lists show their first entries. Keys,
structure and the error's wording are real. Never guess at a masked value; reason from structure,
names, types and the error.
diff --git a/SW.Bitween.Api/Services/Ai/Masking.cs b/SW.Bitween.Api/Services/Ai/Masking.cs
index bab6f41e..e3eeb346 100644
--- a/SW.Bitween.Api/Services/Ai/Masking.cs
+++ b/SW.Bitween.Api/Services/Ai/Masking.cs
@@ -30,6 +30,7 @@ public sealed class Masking
{
public const string Text = "";
public const string Number = "";
+ public const string Secret = "";
public const string Boolean = "";
/// A long list shows this many entries, then how many more there were.
@@ -92,6 +93,10 @@ string Words(string text)
text = Email.Replace(text, "");
text = Quoted.Replace(text, m => $"{m.Groups["q"].Value}{Text}{m.Groups["q"].Value}");
text = QueryString.Replace(text, "?");
+ // Credentials a partner's error can echo unquoted: "Bearer eyJ...", "api_key=...", or a bare token.
+ text = Scheme.Replace(text, m => $"{m.Groups["scheme"].Value} {Secret}");
+ text = Labelled.Replace(text, m => $"{m.Groups["label"].Value}{m.Groups["sep"].Value}{Secret}");
+ text = TokenLike.Replace(text, Secret);
return LongNumber.Replace(text, Number);
}
@@ -182,4 +187,11 @@ static string Cut(string text) =>
static readonly Regex Quoted = new(@"(?['""])(?!<)[^'""\r\n]{1,200}\k", RegexOptions.Compiled);
static readonly Regex QueryString = new(@"\?[^\s'""<>]+", RegexOptions.Compiled);
static readonly Regex LongNumber = new(@"(?bearer|basic)\s+(?!<)[A-Za-z0-9._~+/=-]{8,}", RegexOptions.Compiled | RegexOptions.IgnoreCase);
+ static readonly Regex Labelled = new(
+ @"(?