diff --git a/SW.Bitween.Api/Domain/OperatorAction.cs b/SW.Bitween.Api/Domain/OperatorAction.cs index 969c09ac..488ceb6a 100644 --- a/SW.Bitween.Api/Domain/OperatorAction.cs +++ b/SW.Bitween.Api/Domain/OperatorAction.cs @@ -18,6 +18,9 @@ public class OperatorAction public const string ForgetBookmark = "forget-bookmark"; public const string InstallKit = "install-kit"; + /// A failed exchange sent, masked, to the AI provider under Settings, AI for an explanation. + public const string ExplainFailure = "explain-failure"; + private OperatorAction() { } diff --git a/SW.Bitween.Api/Resources/XchangeExplanations/Explain.cs b/SW.Bitween.Api/Resources/XchangeExplanations/Explain.cs new file mode 100644 index 00000000..306f72af --- /dev/null +++ b/SW.Bitween.Api/Resources/XchangeExplanations/Explain.cs @@ -0,0 +1,230 @@ +#nullable enable +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.Bitween.Services.Ai; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.XchangeExplanations; + +public class XchangeExplanationRequest +{ + public string XchangeId { get; set; } = ""; +} + +public class XchangeExplanationResponse +{ + public string Summary { get; set; } = ""; + public string Cause { get; set; } = ""; + public string Fix { get; set; } = ""; + + /// The model that wrote it. + public string? Model { get; set; } + + /// Exactly what went to the provider, masked, so anyone can see what left Bitween. + public string Sent { get; set; } = ""; +} + +/// +/// Explains a failed exchange in plain words, from its error, its setup and its documents - with +/// every value in them masked before anything is sent. +/// +/// +/// +/// What is sent, and nothing else: the error; which step failed; the subscription's kind and +/// adapters, and the names - never the values - of their settings; the retry policy's shape and the +/// retry state; the last attempts' errors; and the documents with their values replaced by types. +/// Adapter settings are names only because the exchange holds them with partner and global values +/// already filled in, and some of those are credentials. +/// +/// +/// A view right, like reading the exchange: an explanation changes nothing. Each one is recorded as +/// an operator action all the same, since it sends something about the exchange to a third party. +/// +/// +public class Explain(RequestContext requestContext, BitweenDbContext dbContext, XchangeService xchangeService, + AiModels models) + : ICommandHandler +{ + /// Earlier attempts whose errors are worth showing beside this one. + const int EarlierAttempts = 3; + + /// The largest document masked and sent; a few records are enough to explain a failure. + const int MaxMaskedFileBytes = 1_000_000; + + public async Task Handle(XchangeExplanationRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Exchanges.View); + var model = models.Current(); + + var xchange = await dbContext.Set().AsNoTracking().FirstOrDefaultAsync(x => x.Id == request.XchangeId) + ?? throw new SWNotFoundException(request.XchangeId); + var result = await dbContext.Set().AsNoTracking().FirstOrDefaultAsync(r => r.Id == xchange.Id); + if (result is null) + throw new SWValidationException("NOT_FINISHED", "This exchange hasn't finished yet, so there's nothing to explain."); + if (result.Success && !result.ResponseBad) + throw new SWValidationException("NOT_FAILED", "This exchange didn't fail."); + + var subscription = xchange.SubscriptionId is { } subscriptionId + ? await dbContext.Subscriptions().AsNoTracking().FirstOrDefaultAsync(s => s.Id == subscriptionId) + : null; + var document = await dbContext.Set().AsNoTracking().FirstOrDefaultAsync(d => d.Id == xchange.DocumentId); + var delayed = await dbContext.Set().AsNoTracking().FirstOrDefaultAsync(d => d.Id == xchange.Id); + + var masking = new Masking(); + var sent = new StringBuilder(); + + // Documents first: masking them is what tells the error masking which values to take out. + var files = new List<(string Name, string Text)>(); + await AddFile(files, masking, xchange, XchangeFileType.Input, xchange.InputSize, xchange.InputContentType ?? ContentTypeOf(document)); + await AddFile(files, masking, xchange, XchangeFileType.Output, result.OutputSize, result.OutputContentType); + await AddFile(files, masking, xchange, XchangeFileType.Response, result.ResponseSize, result.ResponseContentType); + + sent.AppendLine("Failed step: " + FailedStep(xchange, result)); + if (subscription is not null) + { + sent.AppendLine($"Subscription kind: {subscription.Type}"); + sent.AppendLine($"Mapper: {subscription.MapperId ?? "none"}; handler: {subscription.HandlerId ?? "none"}"); + } + else + { + sent.AppendLine($"Mapper: {xchange.MapperId ?? "none"}; handler: {xchange.HandlerId ?? "none"} (the subscription no longer exists)"); + } + sent.AppendLine("Handler settings (names only): " + Names(xchange.HandlerProperties)); + sent.AppendLine("Document format: " + (document?.DocumentFormat.ToString() ?? "unknown")); + if (result.ResponseBad) sent.AppendLine("The partner's response was marked as bad data."); + var earlier = await EarlierErrors(xchange); + sent.AppendLine($"Attempt {earlier.Chain + 1} of its retry chain"); + if (!string.IsNullOrWhiteSpace(result.RetryBlockedReason)) + sent.AppendLine("Why it won't be retried: " + masking.Error(result.RetryBlockedReason)); + if (delayed is not null) sent.AppendLine($"A retry is scheduled for {delayed.On:u}."); + sent.AppendLine(RetryPolicy(subscription)); + + sent.AppendLine(); + sent.AppendLine(""); + sent.AppendLine(string.IsNullOrWhiteSpace(result.Exception) ? "(no error text; see the response)" : masking.Error(result.Exception)); + sent.AppendLine(""); + + // Only each earlier error's first line: the rest is a stack trace, nearly always the same one. + var thisError = FirstLine(result.Exception); + foreach (var (attempt, error) in earlier.Errors) + { + var line = FirstLine(error); + sent.AppendLine(line == thisError + ? $"Earlier attempt {attempt}: the same error." + : $"Earlier attempt {attempt}: {masking.Error(line)}"); + } + + foreach (var (name, text) in files) + { + sent.AppendLine(); + sent.AppendLine($"<{name}_document>"); + sent.AppendLine(text); + sent.AppendLine($""); + } + + var message = sent.ToString(); + + // Recorded before it goes, so a call that fails or times out after sending is on record too. + await dbContext.RecordNowAsync(requestContext, OperatorAction.ExplainFailure, xchange.Id, + $"Sent to {models.ModelName}; documents and error masked."); + var explanation = FailurePrompt.Parse(await model.Complete(new AiPrompt(FailurePrompt.Instructions, message, Json: true, MaxTokens: 4000))); + + return new XchangeExplanationResponse + { + Summary = explanation.Summary, + Cause = explanation.Cause, + Fix = explanation.Fix, + Model = models.ModelName, + Sent = message, + }; + } + + async Task AddFile(List<(string, string)> files, Masking masking, Xchange xchange, XchangeFileType type, int size, + string? contentType) + { + if (size <= 0) return; + var name = type.ToString().ToLowerInvariant(); + // Masking reads the whole document before cutting what it sends; past this, it isn't read. + if (size > MaxMaskedFileBytes) + { + files.Add((name, "(not sent: too large to mask)")); + return; + } + string text; + try + { + text = await xchangeService.GetFile(xchange, type); + } + catch (SWValidationException) + { + files.Add((name, "(no longer in storage)")); + return; + } + files.Add((name, masking.Document(text, contentType) + ?? "(not sent: only JSON, XML and CSV documents can be masked)")); + } + + /// Not stored, so worked out: no output from a mapping exchange means mapping failed. + static string FailedStep(Xchange xchange, XchangeResult result) => + result.Success && result.ResponseBad ? "delivered, but the partner's response was bad" + : xchange.MapperId is not null && result.OutputSize <= 0 ? "mapping" + : "delivery (the handler)"; + + static string Names(IReadOnlyDictionary? properties) => + properties is null || properties.Count == 0 + ? "none" + : string.Join(", ", properties.OrderBy(p => p.Key).Select(p => $"{p.Key} ({(string.IsNullOrEmpty(p.Value) ? "empty" : "set")})")); + + static string RetryPolicy(Subscription? subscription) + { + var groups = subscription?.RetryPolicy?.Groups ?? subscription?.CustomRetryPolicy?.Groups; + if (groups is null || groups.Count == 0) return "Retry policy: none"; + return "Retry policy" + (subscription?.RetryPolicy?.Name is { } name ? $" \"{name}\"" : "") + ": " + string.Join("; ", + groups.Select(g => $"{g.Name} ({(g.Enabled ? "on" : "off")}, applies to {string.Join("/", g.AppliesTo)}, {g.Action}" + + (g.Budget is { } b ? $", up to {b.MaxAttemptsTotal} attempts" : "") + ")")); + } + + /// How many attempts came before this one, and the errors of the last few. + /// + /// Counted along the chain rather than read from the result's attempt number, which counts only + /// the retries a policy made - five attempts by hand all say 1. + /// + async Task<(int Chain, List<(string Id, string Error)> Errors)> EarlierErrors(Xchange xchange) + { + var errors = new List<(string, string)>(); + var chain = 0; + var id = xchange.RetryFor; + while (id is not null && chain < MaxChain) + { + var previous = await dbContext.Set().AsNoTracking() + .Where(x => x.Id == id).Select(x => new { x.Id, x.RetryFor }).FirstOrDefaultAsync(); + if (previous is null) break; + chain++; + if (errors.Count < EarlierAttempts) + { + var error = await dbContext.Set().AsNoTracking() + .Where(r => r.Id == previous.Id).Select(r => r.Exception).FirstOrDefaultAsync(); + if (!string.IsNullOrWhiteSpace(error)) errors.Add((previous.Id, error)); + } + id = previous.RetryFor; + } + return (chain, errors); + } + + /// A chain is walked this far at most; a longer one says so as "50+" would, by stopping. + const int MaxChain = 50; + + static string FirstLine(string? text) => (text ?? "").Split('\n')[0].Trim(); + + static string? ContentTypeOf(Document? document) => document?.DocumentFormat switch + { + DocumentFormat.Json => "application/json", + DocumentFormat.Xml => "application/xml", + _ => null, + }; +} diff --git a/SW.Bitween.Api/Services/Ai/FailurePrompt.cs b/SW.Bitween.Api/Services/Ai/FailurePrompt.cs new file mode 100644 index 00000000..c1423c92 --- /dev/null +++ b/SW.Bitween.Api/Services/Ai/FailurePrompt.cs @@ -0,0 +1,68 @@ +#nullable enable +using System; +using System.Linq; +using Newtonsoft.Json.Linq; + +namespace SW.Bitween.Services.Ai; + +/// An exchange's failure in plain words: what happened, why, and what to change. +public sealed record FailureExplanation(string Summary, string Cause, string Fix); + +/// What a model is told to explain a failed exchange, and how its answer is read. +public static class FailurePrompt +{ + public const string Instructions = """ + You explain why an exchange failed in Bitween, an integration platform, to the person running it. + They know their partners and subscriptions; they may not be developers. + + How Bitween works: a partner or a receiver delivers a document; a subscription picks it up; its + mapper turns the document into what the destination expects; its handler delivers that, often + over HTTP, and may get a response back. A failure is either in mapping or in delivery. Failed + exchanges can be retried by a retry policy, or by hand. + + Every value in the documents and the error has been masked before you see it: text is , + numbers are , booleans are , and long lists show their first entries. Keys, + structure and the error's wording are real. Never guess at a masked value; reason from structure, + names, types and the error. + + Answer with exactly one JSON object and nothing else: + { "summary": "...", "cause": "...", "fix": "..." } + - summary: one sentence, what went wrong, in plain words. + - cause: the most likely reason, two to four sentences, pointing at the field, setting or step + involved. Say how sure you are when the evidence is thin. + - fix: the change most likely to make the next attempt succeed, concrete and short - which setting, + rule or field to change and to what, or who to ask (the partner, an administrator). If a retry + alone would help (a timeout, a 5xx, a rate limit), say so. + """; + + /// The model's answer as an explanation; a plain-text answer becomes the summary. + public static FailureExplanation Parse(string answer) + { + var start = answer.IndexOf('{'); + var end = answer.LastIndexOf('}'); + if (start >= 0 && end > start) + { + try + { + var json = JObject.Parse(answer[start..(end + 1)]); + var summary = Text(json["summary"]); + if (!string.IsNullOrWhiteSpace(summary)) + return new(summary.Trim(), (Text(json["cause"]) ?? "").Trim(), (Text(json["fix"]) ?? "").Trim()); + } + catch (Newtonsoft.Json.JsonException) + { + // Read as prose below. + } + } + return new(answer.Trim(), "", ""); + } + + /// A field as text, whatever the model made it: a fix often comes back as a list of steps. + static string? Text(JToken? token) => token switch + { + null or { Type: JTokenType.Null } => null, + JValue value => value.ToString(), + JArray list => string.Join("\n", list.Select(i => i.Type == JTokenType.String ? (string?)i : i.ToString(Newtonsoft.Json.Formatting.None))), + _ => token.ToString(Newtonsoft.Json.Formatting.None), + }; +} diff --git a/SW.Bitween.Api/Services/Ai/Masking.cs b/SW.Bitween.Api/Services/Ai/Masking.cs new file mode 100644 index 00000000..bab6f41e --- /dev/null +++ b/SW.Bitween.Api/Services/Ai/Masking.cs @@ -0,0 +1,185 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using SW.Bitween.NativeAdapters.Mapper; +using SW.Bitween.NativeAdapters.Mapper.Formats; + +namespace SW.Bitween.Services.Ai; + +/// +/// Documents and error text with every value taken out, so an AI provider sees how an exchange is +/// shaped and where it broke but none of what it carried. +/// +/// +/// +/// A document keeps its keys, nesting and the type of each value: "name": "Lina" becomes +/// "name": "<text>". That is nearly always enough to explain a failure - a missing field, +/// a number where text was wanted, an empty list - and it is the only thing sent. +/// +/// +/// Error text is free text, so it is masked three ways: a document inside it (an HTTP adapter puts +/// the partner's response body there) is masked as a document; every value the exchange's own +/// documents hold is replaced wherever it appears; and quoted text, long numbers and email +/// addresses are replaced whatever they are. What survives is the error's wording and the code's +/// stack, which is where an explanation comes from. +/// +/// +public sealed class Masking +{ + public const string Text = ""; + public const string Number = ""; + public const string Boolean = ""; + + /// A long list shows this many entries, then how many more there were. + const int ListEntries = 3; + + /// How much of one masked document or error is sent. + public const int MaxLength = 12_000; + + // Every value any masked document held, so error text naming one can have it taken out. + readonly HashSet seen = new(StringComparer.OrdinalIgnoreCase); + + /// + /// A document with its values replaced by their types, or null when it can't be read as JSON, XML + /// or CSV - in which case nothing of it is sent. + /// + /// The file's content type, when known; it picks the reader. + public string? Document(string text, string? contentType) + { + foreach (var format in FormatsFor(contentType)) + { + ValueNode tree; + try + { + tree = format.Read(text); + } + catch (DocumentFormatException) + { + continue; + } + + try + { + return Cut(format.Write(Mask(tree))); + } + catch (DocumentFormatException) + { + return null; + } + } + return null; + } + + /// Error text with documents, values and anything that looks like data taken out. + public string Error(string text) + { + // A document inside is masked as a document; the words around it as free text. Free-text + // rules run on the words only, or they would take a masked document's keys as quoted text. + var (before, document, after) = EmbeddedDocument(text); + return Cut(Words(before) + document + Words(after)); + } + + string Words(string text) + { + // Longest first, so "Lina Haddad" goes before "Lina" can leave " Haddad" behind. + foreach (var value in seen.Where(v => v.Length >= 3).OrderByDescending(v => v.Length)) + text = text.Replace(value, Text, StringComparison.OrdinalIgnoreCase); + + // A stack frame's file keeps its name: the folders above it describe the server, not the failure. + text = SourcePath.Replace(text, " in ${file}:line"); + text = Email.Replace(text, ""); + text = Quoted.Replace(text, m => $"{m.Groups["q"].Value}{Text}{m.Groups["q"].Value}"); + text = QueryString.Replace(text, "?"); + return LongNumber.Replace(text, Number); + } + + ValueNode Mask(ValueNode node) + { + switch (node) + { + case ScalarNode scalar: + switch (scalar.Value) + { + case null: + return scalar; + case string s: + if (s.Trim().Length > 0) seen.Add(s.Trim()); + return ValueNode.Value(s.Length == 0 ? "" : Text); + case bool: + return ValueNode.Value(Boolean); + default: + seen.Add(Convert.ToString(scalar.Value, System.Globalization.CultureInfo.InvariantCulture)!); + return ValueNode.Value(Number); + } + + case ObjectNode obj: + var copy = ValueNode.Object(); + foreach (var (key, child) in obj.Children()) copy.Set(key, Mask(child)); + return copy; + + case ListNode list: + var masked = ValueNode.List(); + foreach (var item in list.Items.Take(ListEntries)) masked.Add(Mask(item)); + // Still walked, so their values are known to be taken out of the error text. + foreach (var item in list.Items.Skip(ListEntries)) Mask(item); + if (list.Items.Count > ListEntries) + masked.Add(ValueNode.Value($"<{list.Items.Count - ListEntries} more entries like these>")); + return masked; + + default: + return node; + } + } + + /// The text before the first JSON or XML document inside it, that document masked, and the text after. + (string Before, string Document, string After) EmbeddedDocument(string text) + { + foreach (var (open, close, format) in new (char, char, IDocumentFormat)[] + { ('{', '}', new JsonFormat()), ('[', ']', new JsonFormat()), ('<', '>', new XmlFormat()) }) + { + var start = text.IndexOf(open); + var end = text.LastIndexOf(close); + if (start < 0 || end <= start) continue; + + var inner = text[start..(end + 1)]; + try + { + var masked = format.Write(Mask(format.Read(inner))); + return (text[..start], masked, text[(end + 1)..]); + } + catch (DocumentFormatException) + { + // Not a document after all - a stack frame's [brackets], say. + } + } + return (text, "", ""); + } + + static IEnumerable FormatsFor(string? contentType) + { + var type = contentType?.ToLowerInvariant() ?? ""; + if (type.Contains("json")) yield return new JsonFormat(); + else if (type.Contains("xml")) yield return new XmlFormat(); + // Read without a header: a header's names would be kept as keys, and a file without one + // would send its first row as them. Every row is masked, a real header row included. + else if (type.Contains("csv")) yield return new CsvFormat(new CsvOptions { HasHeader = false }); + else + { + // Unknown: the two formats that say so of themselves. CSV is never guessed - nearly any + // text reads as CSV, and its values would then be the whole of it. + yield return new JsonFormat(); + yield return new XmlFormat(); + } + } + + static string Cut(string text) => + text.Length <= MaxLength ? text : text[..MaxLength] + $"\n<{text.Length - MaxLength} more characters not sent>"; + + static readonly Regex SourcePath = new(@" in [^\s]*?[\\/](?[^\\/\s]+):line", RegexOptions.Compiled); + static readonly Regex Email = new(@"[\w.+-]+@[\w-]+(\.[\w-]+)+", RegexOptions.Compiled); + static readonly Regex Quoted = new(@"(?['""])(?!<)[^'""\r\n]{1,200}\k", RegexOptions.Compiled); + static readonly Regex QueryString = new(@"\?[^\s'""<>]+", RegexOptions.Compiled); + static readonly Regex LongNumber = new(@"(? +/// Explain this failure: a real failed exchange goes to the model with its values masked, the +/// answer comes back in three parts, and the request is in the audit trail. +/// +[Collection("Http")] +public class FailureExplanationTests(HttpFixture fixture) : IDisposable +{ + public void Dispose() + { + AiModels.ModelOverride = null; + using var admin = fixture.AdminAsync().GetAwaiter().GetResult(); + admin.DeleteAsync("/api/settings/Bitween.Ai.ApiKey").GetAwaiter().GetResult(); + } + + sealed class FakeModel(string answer) : IAiModel + { + public List Asked { get; } = new(); + + public Task Complete(AiPrompt prompt) + { + Asked.Add(prompt); + return Task.FromResult(answer); + } + } + + /// An exchange that fails in mapping: "maybe" can't become a yes/no value. + async Task<(HttpClient Admin, string XchangeId)> FailedExchangeAsync() + { + var (admin, urlName, _, partnerId, subscriptionId) = await Api.GatewayAsync(fixture); + await Api.Json(await admin.PostAsJsonAsync($"/api/subscriptions/{subscriptionId}/savemapper", new + { + mapperId = "NativeMapper", + mapperProperties = new[] + { + new + { + key = "MappingRules", + value = """{ "version": 1, "fields": [ { "target": ["flag"], "from": { "kind": "path", "path": "order.flag" }, "type": "boolean" } ] }""", + }, + }, + })); + var key = await Api.NewKeyAsync(admin); + await Api.SetKeysAsync(admin, partnerId, ("default", key)); + using var _ = await Api.CallSyncAsync(fixture, urlName, key, """{ "order": { "flag": "maybe", "owner": "lina@acme.example" } }"""); + + var found = await Api.Json(await admin.GetAsync( + $"/api/xchanges?filter={Uri.EscapeDataString($"SubscriptionId:1:{subscriptionId}")}&size=1")); + return (admin, (string)found["result"]![0]!["id"]!); + } + + async Task ConfigureAsync(HttpClient admin) + { + await Api.Json(await admin.PostAsJsonAsync("/api/settings/Bitween.Ai.ApiKey", new { value = "sk-test-key" })); + var live = fixture.App.Services.GetService(typeof(BitweenOptions)) as BitweenOptions; + for (var waited = 0; live!.Ai.ApiKey != "sk-test-key"; waited += 50) + { + Assert.True(waited < 10_000, "the AI key never took effect"); + await Task.Delay(50); + } + } + + [Fact] + public async Task A_failed_exchange_is_explained_from_masked_data_and_audited() + { + var (admin, xchangeId) = await FailedExchangeAsync(); + await ConfigureAsync(admin); + var model = new FakeModel("""{ "summary": "S", "cause": "C", "fix": "F" }"""); + AiModels.ModelOverride = _ => model; + + var result = await Api.Json(await admin.PostAsJsonAsync("/api/xchangeexplanations", new { xchangeId })); + + Assert.Equal("S", (string)result["summary"]!); + Assert.Equal("F", (string)result["fix"]!); + var sent = Assert.Single(model.Asked).Message; + Assert.Equal(sent, (string)result["sent"]!); + Assert.Contains("Failed step: mapping", sent); + Assert.Contains("\"flag\": \"\"", sent); + Assert.DoesNotContain("maybe", sent); + Assert.DoesNotContain("lina@acme.example", sent); + + var audited = await fixture.InDbAsync(db => db.Set().AsNoTracking() + .AnyAsync(a => a.Action == OperatorAction.ExplainFailure && a.Target == xchangeId)); + Assert.True(audited, "the explanation wasn't recorded"); + } + + [Fact] + public async Task Without_AI_set_up_nothing_is_read_or_sent() + { + var (admin, xchangeId) = await FailedExchangeAsync(); + + var response = await admin.PostAsJsonAsync("/api/xchangeexplanations", new { xchangeId }); + + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + Assert.Contains("Settings, AI", await response.Content.ReadAsStringAsync()); + } +} diff --git a/SW.Bitween.UnitTests/Ai/MaskingTests.cs b/SW.Bitween.UnitTests/Ai/MaskingTests.cs new file mode 100644 index 00000000..89f066ad --- /dev/null +++ b/SW.Bitween.UnitTests/Ai/MaskingTests.cs @@ -0,0 +1,131 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Services.Ai; + +namespace SW.Bitween.UnitTests.Ai; + +/// +/// What "Explain this failure" sends: structure and wording, never a value. Each test names a value +/// and checks it can't be found in what would go to the provider. +/// +[TestClass] +public class MaskingTests +{ + [TestMethod] + public void A_json_document_keeps_its_keys_and_types_and_loses_its_values() + { + var masked = new Masking().Document(""" + { "customer": { "name": "Lina Haddad", "vip": true, "balance": 1200.5, "note": "", "fax": null } } + """, "application/json")!; + + StringAssert.Contains(masked, "\"name\": \"\""); + StringAssert.Contains(masked, "\"vip\": \"\""); + StringAssert.Contains(masked, "\"balance\": \"\""); + StringAssert.Contains(masked, "\"note\": \"\""); + StringAssert.Contains(masked, "\"fax\": null"); + Assert.IsFalse(masked.Contains("Lina") || masked.Contains("1200")); + } + + [TestMethod] + public void An_xml_document_is_masked_attributes_and_all() + { + var masked = new Masking().Document("""Atlas Trading""", "text/xml")!; + + StringAssert.Contains(masked, "<text>"); + Assert.IsFalse(masked.Contains("SO-77") || masked.Contains("Atlas")); + } + + [TestMethod] + public void A_long_list_shows_its_first_entries_and_how_many_more() + { + var masked = new Masking().Document("""{ "lines": [ {"sku":"A1"}, {"sku":"A2"}, {"sku":"A3"}, {"sku":"A4"}, {"sku":"A5"} ] }""", null)!; + + StringAssert.Contains(masked, "<2 more entries like these>"); + Assert.IsFalse(masked.Contains("A5")); + } + + /// A file without a header row must not send its first record as column names. + [TestMethod] + public void A_csv_without_a_header_keeps_none_of_its_first_row() + { + var masked = new Masking().Document("Lina Haddad,lina@acme.example,SO-77\nOmar Said,omar@acme.example,SO-78", "text/csv")!; + + foreach (var leaked in new[] { "Lina", "lina@acme.example", "SO-77", "Omar", "SO-78" }) + Assert.IsFalse(masked.Contains(leaked), $"'{leaked}' was sent: {masked}"); + StringAssert.Contains(masked, ""); + } + + [TestMethod] + public void A_csv_with_a_header_has_every_row_masked() + { + var masked = new Masking().Document("name,email\nLina Haddad,lina@acme.example", "text/csv")!; + + Assert.IsFalse(masked.Contains("Lina") || masked.Contains("lina@acme.example"), masked); + StringAssert.Contains(masked, ""); + } + + /// Nothing is sent of a document that can't be masked; guessing would send it whole. + [TestMethod] + public void A_document_that_isnt_json_xml_or_csv_isnt_sent_at_all() + { + Assert.IsNull(new Masking().Document("Dear Lina, your parcel 4471 is late.", "text/plain")); + } + + /// An HTTP adapter puts the partner's response body in the error text. + [TestMethod] + public void A_response_body_inside_an_error_is_masked_as_a_document() + { + var masked = new Masking().Error(""" + 400: BadRequest: {"error":"Customer Lina Haddad has no account","code":"ACC_MISSING"} + at SW.Bitween.NativeAdapters.NativeHttpHandler.Handle() in NativeHttpHandler.cs:line 170 + """); + + StringAssert.Contains(masked, "400: BadRequest:"); + StringAssert.Contains(masked, "\"error\": \"\""); + StringAssert.Contains(masked, "NativeHttpHandler.Handle()"); + Assert.IsFalse(masked.Contains("Lina") || masked.Contains("ACC_MISSING")); + } + + [TestMethod] + public void An_error_loses_document_values_quotes_emails_and_long_numbers() + { + var masking = new Masking(); + masking.Document("""{ "ref": "SO-5001", "city": "Amman" }""", "application/json"); + + var masked = masking.Error( + "Could not deliver SO-5001 to Amman for ops@acme.example: value 'blue-77' is not allowed, card 4111222233334444, see https://p.example/x?token=abc"); + + foreach (var leaked in new[] { "SO-5001", "Amman", "ops@acme.example", "blue-77", "4111222233334444", "token=abc" }) + Assert.IsFalse(masked.Contains(leaked), $"'{leaked}' was not masked: {masked}"); + StringAssert.Contains(masked, "Could not deliver"); + StringAssert.Contains(masked, "is not allowed"); + } + + /// A frame keeps its file and line; the folders above it describe the server, not the failure. + [TestMethod] + public void Stack_frames_keep_their_file_but_not_the_servers_folders() + { + var masked = new Masking().Error(""" + System.UriFormatException: Invalid URI + at SW.Bitween.NativeAdapters.DynamicHttpProxy.GetClient(String fullUrl) in /Users/someone/Work/Bitween-api/SW.Bitween.NativeAdapters/HttpHandler/DynamicHttpProxy.cs:line 22 + at SW.Bitween.XchangeService.Process(XchangeMessage message) in C:\agent\_work\1\s\SW.Bitween.Api\Services\XchangeService.cs:line 703 + """); + + StringAssert.Contains(masked, "GetClient(String fullUrl) in DynamicHttpProxy.cs:line 22"); + StringAssert.Contains(masked, "in XchangeService.cs:line 703"); + Assert.IsFalse(masked.Contains("/Users/someone") || masked.Contains("_work"), masked); + } + + [TestMethod] + public void The_answer_is_read_as_json_or_kept_as_prose() + { + var json = FailurePrompt.Parse("""```json {"summary":"S","cause":"C","fix":"F"} ```"""); + Assert.AreEqual(("S", "C", "F"), (json.Summary, json.Cause, json.Fix)); + + var steps = FailurePrompt.Parse("""{"summary":"S","cause":{"step":"mapping"},"fix":["Fix the Url.","Retry it."]}"""); + Assert.AreEqual("Fix the Url.\nRetry it.", steps.Fix); + Assert.AreEqual("""{"step":"mapping"}""", steps.Cause); + + var prose = FailurePrompt.Parse("The partner timed out; retry."); + Assert.AreEqual(("The partner timed out; retry.", "", ""), (prose.Summary, prose.Cause, prose.Fix)); + } +} diff --git a/SW.Bitween.Web/ClientApp/src/api/client.ts b/SW.Bitween.Web/ClientApp/src/api/client.ts index 0bbe0eba..92701281 100644 --- a/SW.Bitween.Web/ClientApp/src/api/client.ts +++ b/SW.Bitween.Web/ClientApp/src/api/client.ts @@ -753,6 +753,15 @@ export interface ApiClient { * AUTO_RETRY_SCHEDULED when an auto-retry is already pending. */ retryExchange(id: string, opts: { reset: boolean }): Promise<{ id: string }>; + /** + * Explains a failed exchange in plain words through the AI provider under Settings, AI. The + * error and documents go masked - every value replaced by its type - and `sent` is exactly what + * went. Recorded in the audit log. + */ + explainExchange( + xchangeId: string, + signal?: AbortSignal, + ): Promise<{ summary: string; cause: string; fix: string; model: string | null; sent: string }>; /** Retries many; exchanges with a pending auto-retry are skipped, not failed. */ bulkRetryExchanges(selection: BulkRetrySelection, opts: { reset: boolean }): Promise; previewBulkRetry(selection: BulkRetrySelection, opts: { reset: boolean }): Promise; diff --git a/SW.Bitween.Web/ClientApp/src/api/http/exchanges.ts b/SW.Bitween.Web/ClientApp/src/api/http/exchanges.ts index 16b82ffc..9cea8e6e 100644 --- a/SW.Bitween.Web/ClientApp/src/api/http/exchanges.ts +++ b/SW.Bitween.Web/ClientApp/src/api/http/exchanges.ts @@ -14,7 +14,7 @@ import type { ScheduledRetryRow, } from "../types"; import { partnerMethods } from "./partners"; -import { download, get, post } from "./request"; +import { download, get, post, request } from "./request"; import { searchyQueryString } from "./searchQuery"; // --- backend shapes (camelCase over the wire) --- @@ -326,6 +326,13 @@ export const exchangeMethods = { }; }, + explainExchange( + xchangeId: string, + signal?: AbortSignal, + ): Promise<{ summary: string; cause: string; fix: string; model: string | null; sent: string }> { + return request("/xchangeexplanations", { method: "POST", body: { xchangeId }, signal }); + }, + async retryExchange(id: string, { reset }: { reset: boolean }): Promise<{ id: string }> { await post(`/xchanges/${id}/retry`, { reason: "Manual retry", reset }); // Retry.cs returns null - look up the retry it just created (the newest diff --git a/SW.Bitween.Web/ClientApp/src/pages/exchanges/ExchangeDrawer.tsx b/SW.Bitween.Web/ClientApp/src/pages/exchanges/ExchangeDrawer.tsx index 11898d8e..b8aa0777 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/exchanges/ExchangeDrawer.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/exchanges/ExchangeDrawer.tsx @@ -15,6 +15,7 @@ import { useSubscriptionsCache } from "../../components/config/lookups"; import { RetryDialog, type JourneyStage } from "./shared"; import { journeyStages } from "./journey"; import { RetryChain } from "./RetryChain"; +import { ExplainFailure } from "./ExplainFailure"; import { hasRetryChain, newestAttempt, retryTreeQuery } from "./retryTree"; import { keys } from "../../api/queryKeys"; @@ -479,6 +480,13 @@ export function ExchangeDrawer({ x }: { x: ExchangeRow }) { )} + {/* Outside the operate gate: an explanation changes nothing, so seeing the exchange is enough. */} + {(x.status === "failed" || x.status === "badResponse") && ( +
+ +
+ )} + {confirming && ( >; + +/** + * "Explain this failure": the AI provider under Settings, AI reads the failed exchange and says + * what happened, why, and what to change. + * + * Offered to anyone who can see the exchange - it changes nothing. What goes to the provider is + * masked on the server, every value replaced by its type, and the dialog shows exactly what went. + */ +export function ExplainFailure({ xchangeId }: { xchangeId: string }) { + const [open, setOpen] = useState(false); + return ( + <> + + {open && setOpen(false)} />} + + ); +} + +function ExplainDialog({ xchangeId, onClose }: { xchangeId: string; onClose: () => void }) { + const config = useQuery({ queryKey: keys.appConfig, queryFn: getAppConfig }); + const canSetUp = useSessionCan("settings.edit"); + const [result, setResult] = useState(null); + const [error, setError] = useState(null); + const [pending, setPending] = useState(false); + const [seconds, setSeconds] = useState(0); + const [showSent, setShowSent] = useState(false); + const running = useRef(null); + + useEffect(() => { + if (!pending) return; + const started = Date.now(); + setSeconds(0); + const tick = setInterval(() => setSeconds(Math.floor((Date.now() - started) / 1000)), 1000); + return () => clearInterval(tick); + }, [pending]); + + // Stops waiting when the dialog closes, so a late answer has nowhere to land. + useEffect(() => () => running.current?.abort(), []); + + const explain = async () => { + const abort = new AbortController(); + running.current = abort; + setPending(true); + setError(null); + try { + const answer = await api.explainExchange(xchangeId, abort.signal); + if (!abort.signal.aborted) setResult(answer); + } catch (e) { + if (!abort.signal.aborted) setError((e as Error).message); + } finally { + if (running.current === abort) { + running.current = null; + setPending(false); + } + } + }; + + const cancel = () => { + running.current?.abort(); + running.current = null; + setPending(false); + }; + + if (config.data && !config.data.aiConfigured) + return ( + +
+

AI isn't set up on this Bitween yet: it needs an AI provider and its API key under Settings, AI.

+ {canSetUp ? ( + + Open Settings, AI in a new tab + + ) : ( +

Ask an administrator to set it up.

+ )} +
+ + ); + + return ( + +
+ {result ? ( + <> +

{result.summary}

+ {result.cause &&
{result.cause}
} + {result.fix &&
{result.fix}
} +

+ Written by {result.model ?? "the AI provider"} from the error, the setup and the documents, with + every value masked. It can be wrong; check before you change anything. +

+ + {showSent && ( + 
+                {result.sent}
+
+ )} + + ) : ( +

+ Sends this exchange's error, which step failed, the subscription's adapters and the names of their + settings, its retry state, and its documents to the AI provider under Settings, AI. Every value is + masked first: "name": "Lina" goes as{" "} + "name": "<text>". The request is recorded in the + audit log. +

+ )} + + {error} + +
+ {pending ? ( + <> + + Explaining… {seconds}s + · usually 5-20 seconds + + + + ) : ( + <> + + {!result && ( + + )} + + )} +
+
+ + ); +} + +function Section({ title, children }: { title: string; children: string }) { + return ( +
+

{title}

+

{children}

+
+ ); +} diff --git a/SW.Bitween.Web/ClientApp/src/pages/exchanges/__tests__/explainFailure.test.tsx b/SW.Bitween.Web/ClientApp/src/pages/exchanges/__tests__/explainFailure.test.tsx new file mode 100644 index 00000000..b1bd64c4 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/exchanges/__tests__/explainFailure.test.tsx @@ -0,0 +1,81 @@ +import { screen, within } from "@testing-library/react"; +import { http, HttpResponse } from "msw"; +import { describe, expect, it } from "vitest"; +import { apiPath, renderApp } from "../../../__tests__/support/renderApp"; + +/** + * Explain this failure, from the exchange panel. What is masked and sent is the C# suite's + * business (MaskingTests, FailureExplanationTests); this is the button, the dialog and the answer. + */ + +const EXCHANGE_ID = "3f2a9c1e-0000-4000-8000-000000000002"; +const NONE = { result: [], totalCount: 0 }; + +/** GET /xchanges row (`RawXchangeRow`): finished, not successful - failed. */ +const FAILED = { + id: EXCHANGE_ID, subscriptionId: 12, subscriptionName: "Orders in", documentId: 3, documentName: "SHIPMENT_ORDER", + mapperId: "NativeMapper", status: false, exception: "flag: can't read as a boolean", + finishedOn: "2026-09-24T08:00:02Z", startedOn: "2026-09-24T08:00:00Z", + inputFileName: "input.json", outputFileName: null, responseFileName: null, + inputFileSize: 20, outputFileSize: 0, responseFileSize: 0, inputKey: "in-1", outputKey: null, responseKey: null, + promotedProperties: {}, retryFor: null, aggregationXchangeId: null, responseBad: null, correlationId: null, + partnerId: null, scheduledRetryOn: null, hasRetry: false, +}; + +function openFailed(aiConfigured: boolean, onExplain = () => {}) { + return renderApp(`/exchanges?ids=${EXCHANGE_ID}`, { + config: { aiConfigured }, + handlers: [ + http.get(apiPath("/xchanges"), () => HttpResponse.json({ result: [FAILED], totalCount: 1 })), + http.get(apiPath("/bitweendocs"), () => HttpResponse.json({ key: "in-1", data: '{"order":{"flag":"maybe"}}' })), + http.get(apiPath("/subscriptions"), () => HttpResponse.json(NONE)), + http.get(apiPath("/partners"), () => HttpResponse.json(NONE)), + http.get(apiPath("/documents"), () => HttpResponse.json(NONE)), + http.get(apiPath("/xchanges/summary"), () => + HttpResponse.json({ counts: { processing: 0, success: 0, badResponse: 0, failed: 1 }, buckets: [], bucketMinutes: 0 }), + ), + http.post(apiPath("/xchangeexplanations"), async ({ request }) => { + expect(await request.json()).toEqual({ xchangeId: EXCHANGE_ID }); + onExplain(); + return HttpResponse.json({ + summary: "The flag can't be read as yes or no.", + cause: "The rule for flag converts to boolean, and the partner sent text.", + fix: "Add a lookup from the partner's words to true and false.", + model: "claude-sonnet-5-5", + sent: 'Failed step: mapping\n{ "order": { "flag": "" } }', + }); + }), + ], + }); +} + +describe("Explain this failure", { timeout: 20000 }, () => { + it("explains a failed exchange in three parts and shows exactly what was sent", async () => { + let asked = 0; + const { user } = openFailed(true, () => asked++); + await screen.findByRole("complementary", { name: `Exchange ${EXCHANGE_ID}` }); + + await user.click(screen.getByRole("button", { name: /Explain this failure/ })); + const dialog = await screen.findByRole("dialog", { name: "Explain this failure" }); + // Nothing goes until asked, and the dialog says what will. + expect(asked).toBe(0); + expect(within(dialog).getByText(/Every value is/)).toBeInTheDocument(); + await user.click(within(dialog).getByRole("button", { name: /^Explain$/ })); + + expect(await within(dialog).findByText("The flag can't be read as yes or no.")).toBeInTheDocument(); + expect(within(dialog).getByText(/Add a lookup/)).toBeInTheDocument(); + expect(within(dialog).getByText(/claude-sonnet-5-5/)).toBeInTheDocument(); + await user.click(within(dialog).getByRole("button", { name: "See what was sent" })); + expect(within(dialog).getByLabelText("What was sent to the AI provider")).toHaveTextContent('"flag": ""'); + }); + + it("says where to turn it on until Settings has a provider key", async () => { + const { user } = openFailed(false); + await screen.findByRole("complementary", { name: `Exchange ${EXCHANGE_ID}` }); + + await user.click(screen.getByRole("button", { name: /Explain this failure/ })); + + const dialog = await screen.findByRole("dialog", { name: "Explain this failure" }); + expect(within(dialog).getByRole("link", { name: "Open Settings, AI in a new tab" })).toHaveAttribute("target", "_blank"); + }); +}); diff --git a/docs/admin-ui.md b/docs/admin-ui.md index 4fdf6e39..7e64b5ac 100644 --- a/docs/admin-ui.md +++ b/docs/admin-ui.md @@ -12,7 +12,7 @@ The sign-in page offers email and password, Microsoft, an [OpenID Connect provid | Page | What it does | |---|---| -| **Exchanges** | Search and filter every exchange with live refresh, optionally showing only the latest attempt of each retry chain. A filter's operator is picked on its chip: *is* or *is not*, *contains* or *does not contain*; subscription, partner and information type can pick several (*is any of*, *is none of*), and any number of promoted properties can be filtered on at once. Expand a row to see each stage's file, formatted or raw, plus the exception, promoted properties and the retry chain. Retry one exchange, a selection, or every match of the current filters after previewing the plan. Create an exchange by hand. | +| **Exchanges** | Search and filter every exchange with live refresh, optionally showing only the latest attempt of each retry chain. A filter's operator is picked on its chip: *is* or *is not*, *contains* or *does not contain*; subscription, partner and information type can pick several (*is any of*, *is none of*), and any number of promoted properties can be filtered on at once. Expand a row to see each stage's file, formatted or raw, plus the exception, promoted properties and the retry chain. On a failed exchange, **Explain this failure** asks the AI provider for what went wrong and what to change ([Explaining a failure](configuration.md#explaining-a-failure)). Retry one exchange, a selection, or every match of the current filters after previewing the plan. Create an exchange by hand. | | **Scheduled retries** | Automatic retries waiting to run, with the policy that scheduled each one. Run one now. | | **Queue health** | Live RabbitMQ lanes, backlogs, dead letters, alerts and orphaned queues. | diff --git a/docs/api-reference.md b/docs/api-reference.md index 4f6ea40f..d974cbd3 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -84,6 +84,7 @@ All need the global values permissions: `global-values.view` to read, `.create`, | `GET /api/xchanges/retrytree?id=` | `exchanges.view` | The retry chain any attempt belongs to | | `POST /api/xchanges` | signed in, no permission checked | Create an exchange by hand | | `POST /api/xchanges/{id}/retry` | `exchanges.operate` | `{ reason, reset }` | +| `POST /api/xchangeexplanations` | `exchanges.view` | Explain a failed exchange with AI: `{ xchangeId }`, replying with `summary`, `cause`, `fix`, `model`, and `sent` (exactly what went to the provider, masked). Recorded as an `explain-failure` operator action. See [Explaining a failure](configuration.md#explaining-a-failure). | | `POST /api/xchanges/bulkretrypreview` | `exchanges.operate` | Plan a bulk retry without running it | | `POST /api/xchanges/bulkretry` | `exchanges.operate` | Run a bulk retry | | `POST /api/xchanges/export` | `exchanges.view` | The selected exchanges' files as one zip | diff --git a/docs/configuration.md b/docs/configuration.md index c4b03756..3c432098 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -216,7 +216,7 @@ The **Mail server** section is the one SMTP server every Email notification chan ### AI -The **AI** section is the provider every AI feature calls; the first is [Generate with AI](mapping.md#generate-with-ai) in the mapping editor. It is read on each call, and AI features stay off until an API key is set. +The **AI** section is the provider every AI feature calls; they are [Generate with AI](mapping.md#generate-with-ai) in the mapping editor and [Explain this failure](#explaining-a-failure) on a failed exchange. It is read on each call, and AI features stay off until an API key is set. - **Provider** is `Anthropic` or `OpenAI`. Any other value, such as a misspelling in configuration, leaves AI features off. `OpenAI` also covers anything that speaks its chat completions API at **API address**: an Azure OpenAI resource (`https://your-resource.openai.azure.com/openai/v1/`), a local Ollama (`http://localhost:11434/v1/`) or a proxy. - **Model** empty uses `claude-opus-5-5` with Anthropic. OpenAI needs one named, or the Azure deployment's name. @@ -227,6 +227,18 @@ When the provider refuses or fails a call, the message gives its HTTP status, no Sample documents an AI feature sends go to this provider, with the names - never the values - of partner properties and global values. Secret ones are not named. +#### Explaining a failure + +**Explain this failure** on a failed exchange sends this, and only this, to the provider: + +- the error, which step failed, the subscription's kind and adapters, and the names of the handler's settings, never their values (they hold partner and global values already filled in, some of them credentials); +- the retry policy's groups, the attempt's place in its retry chain, why it won't be retried and any scheduled retry, and the first line of up to three earlier attempts' errors; +- the input, mapped and response documents, with every value replaced by its type: `"name": "Lina"` goes as `"name": ""`. Long lists show their first three entries. A CSV's every row is masked, a header row included, so a file without one can't send its first record as column names. A document that isn't JSON, XML or CSV, or is over 1 MB, isn't sent at all. + +Error text is masked too: a document inside it, such as a partner's response body, is masked as a document; every value the exchange's documents hold is taken out wherever it appears; quoted text, numbers of four or more digits, email addresses and URL query strings are replaced; and a stack frame keeps its file name and line but not the server's folders. + +The answer says what happened, the most likely cause and what to change. **See what was sent** shows exactly the masked text that went. Anyone who can view the exchange can ask, and each request is recorded as an `explain-failure` operator action in the audit trail before anything is sent, so a call that fails afterwards is on record too. + The page also has a **Notification channels** section. Channels are records rather than settings: they have their own table, each opens on a page of its own, and the `notifiers` permissions cover them rather than `settings`. See [Notification channels](retries-and-alerts.md#notification-channels). `GET /api/retention` and `POST /api/retention/preview` describe what the retention settings do, as saved or as proposed. The Settings page uses them. `POST /api/retention/aggregation` says whether exchanges would be removed before an aggregation on the given schedules rolls them up; the aggregation page uses it.