From 52a732523f96f4654fd2dcf727085df9cc5f25dc Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Tue, 27 Oct 2026 16:06:34 +0300 Subject: [PATCH 1/4] fix: open the prod picker without starting every adapter --- SW.Bitween.Api/Resources/Adapters/Catalog.cs | 7 ++- .../Services/Mirror/MirrorService.cs | 49 +++++++++++++------ .../Tests/AdapterCatalogTests.cs | 17 ++++++- SW.Bitween.Sdk/Model/Adapter.cs | 6 +++ 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/SW.Bitween.Api/Resources/Adapters/Catalog.cs b/SW.Bitween.Api/Resources/Adapters/Catalog.cs index a87fb39e..6c192631 100644 --- a/SW.Bitween.Api/Resources/Adapters/Catalog.cs +++ b/SW.Bitween.Api/Resources/Adapters/Catalog.cs @@ -41,7 +41,10 @@ public async Task Handle(AdapterSearchRequest request) // reflection and should not be made to queue, and how many published adapters may be // running at a time is capped in ServerlessAdapterDescriber - process-wide, which is the // only place it can be, since this handler knows nothing of the other requests in flight. - var described = await Task.WhenAll(adapters.Select(async a => (a.Key, Values: await Describe(a.Key)))); + // Not at all for a caller after versions alone: a published adapter is described by starting + // it, which on a cold cache takes half a minute for a kind. + var described = request.StartupValues == false ? [] + : await Task.WhenAll(adapters.Select(async a => (a.Key, Values: await Describe(a.Key)))); var byKey = described.ToDictionary(d => d.Key, d => d.Values); return adapters.Select(a => new @@ -58,7 +61,7 @@ public async Task Handle(AdapterSearchRequest request) .Distinct(StringComparer.OrdinalIgnoreCase) .OrderBy(v => System.Version.TryParse(v.Split('-')[0], out var parsed) ? parsed : new System.Version()) .ToList(), - StartupValues = byKey[a.Key], + StartupValues = byKey.GetValueOrDefault(a.Key), // Additive: a client that predates the catalog ignores all of these. Every one is null // or empty for an adapter published before manifests existed. diff --git a/SW.Bitween.Api/Services/Mirror/MirrorService.cs b/SW.Bitween.Api/Services/Mirror/MirrorService.cs index 2f6a2fa3..b99100f7 100644 --- a/SW.Bitween.Api/Services/Mirror/MirrorService.cs +++ b/SW.Bitween.Api/Services/Mirror/MirrorService.cs @@ -134,7 +134,7 @@ async Task CompareAsync(BitweenApi prod, Func openSelf, var skipped = await ReadAsync(prodCtx, renamed); var (copy, _) = CopyOf(prodCtx); var used = AdapterUses(copy.Sections); - var prodAdapters = await AdaptersAsync(prod, used.Keys); + var prodAdapters = await AdaptersAsync(prod, used.Keys, isProd: true); // ---- staging, as it is var self = openSelf(); @@ -347,28 +347,47 @@ internal static IEnumerable Distinguish(Context ctx, Kind kind, List<(st sealed record AdapterInfo(string Id, bool Native, string Current, IReadOnlyCollection Versions, bool FromRegistry); + /// + /// Prod's listed adapters as last read, by its address. A prod whose release can't list them without + /// starting each published one to describe it takes half a minute and more, every picker and + /// comparison asks, and what prod has installed seldom changes. + /// + static readonly System.Collections.Concurrent.ConcurrentDictionary Listed)> ProdAdapters = + new(StringComparer.Ordinal); + /// /// The adapters a Bitween lists, and for any one its items use that the lists leave out - a - /// database's, say - what versions it has of it. + /// database's, say - what versions it has of it. Prod's lists are read again once ten minutes old; + /// staging's every time, since a copy installs adapters. /// - static async Task> AdaptersAsync(BitweenApi api, IEnumerable used) + static async Task> AdaptersAsync(BitweenApi api, IEnumerable used, bool isProd = false) { - var found = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var prefix in AdapterPrefixes) + Dictionary listed; + if (isProd && ProdAdapters.TryGetValue(api.Url, out var read) && DateTimeOffset.UtcNow - read.At < TimeSpan.FromMinutes(10)) + listed = read.Listed; + else { - JsonNode answer; - try { answer = await api.GetAsync($"api/adapters/catalog?prefix={prefix}"); } - catch (BitweenApiException ex) when (ex.Status is HttpStatusCode.NotFound or HttpStatusCode.Forbidden) { continue; } - foreach (var row in (answer as JsonArray ?? []).OfType()) + listed = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var prefix in AdapterPrefixes) { - var id = (string)row["key"]; - if (string.IsNullOrEmpty(id)) continue; - var history = (row["versionHistory"] as JsonArray ?? []).OfType().ToList(); - found[id] = new AdapterInfo(id, (bool?)row["native"] ?? false, (string)row["currentVersion"], - (row["versions"] as JsonArray ?? []).Select(v => v?.ToString()).Where(v => v != null).ToList(), - history.Any(h => ((string)h["publishedBy"])?.StartsWith("registry:", StringComparison.Ordinal) == true)); + JsonNode answer; + // Without their startup properties, which nothing here reads: describing a published + // adapter starts it. A release from before the option ignores it and describes them. + try { answer = await api.GetAsync($"api/adapters/catalog?prefix={prefix}&startupValues=false"); } + catch (BitweenApiException ex) when (ex.Status is HttpStatusCode.NotFound or HttpStatusCode.Forbidden) { continue; } + foreach (var row in (answer as JsonArray ?? []).OfType()) + { + var id = (string)row["key"]; + if (string.IsNullOrEmpty(id)) continue; + var history = (row["versionHistory"] as JsonArray ?? []).OfType().ToList(); + listed[id] = new AdapterInfo(id, (bool?)row["native"] ?? false, (string)row["currentVersion"], + (row["versions"] as JsonArray ?? []).Select(v => v?.ToString()).Where(v => v != null).ToList(), + history.Any(h => ((string)h["publishedBy"])?.StartsWith("registry:", StringComparison.Ordinal) == true)); + } } + if (isProd) ProdAdapters[api.Url] = (DateTimeOffset.UtcNow, listed); } + var found = new Dictionary(listed, StringComparer.OrdinalIgnoreCase); foreach (var id in used.Where(id => !found.ContainsKey(id) && !NativeAdapterDiscoveryService.IsNative(id))) { JsonNode listing; diff --git a/SW.Bitween.IntegrationTests/Tests/AdapterCatalogTests.cs b/SW.Bitween.IntegrationTests/Tests/AdapterCatalogTests.cs index 0ee87a22..f94a4d77 100644 --- a/SW.Bitween.IntegrationTests/Tests/AdapterCatalogTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/AdapterCatalogTests.cs @@ -121,12 +121,12 @@ async Task Stored(int id) .Set().AsNoTracking().SingleAsync(s => s.Id == id); } - async Task CatalogOf(string prefix) + async Task CatalogOf(string prefix, bool? startupValues = null) { await using var scope = fixture.CreateScope(); scope.Superuser(); var handler = ActivatorUtilities.CreateInstance(scope.ServiceProvider); - return JArray.FromObject(await handler.Handle(new AdapterSearchRequest { Prefix = prefix }), + return JArray.FromObject(await handler.Handle(new AdapterSearchRequest { Prefix = prefix, StartupValues = startupValues }), // Camel-cased property names, as the API sends them, with dictionary keys - property // names of the adapter - left as they are. JsonSerializer.Create(new JsonSerializerSettings @@ -233,6 +233,19 @@ public async Task A_manifest_that_lists_properties_is_described_from_them() Assert.Null(values["ContentType"]); // from the manifest, not by asking the adapter } + [Fact] + public async Task Asked_without_startup_values_the_catalog_lists_versions_and_starts_nothing() + { + var id = await PublishHandler(versions: ["1.0.0", "1.1.0"]); + + var listed = (await CatalogOf("handlers", startupValues: false)).Single(a => (string)a["key"] == id); + + Assert.Equal("1.1.0", (string)listed["currentVersion"]); + Assert.Equal(["1.0.0", "1.1.0"], ((JArray)listed["versionHistory"]).Select(v => (string)v["version"])); + // Null rather than empty: a describe that fails answers empty, so null means none was tried. + Assert.Equal(JTokenType.Null, listed["startupValues"]!.Type); + } + [Fact] public async Task Native_adapters_list_with_their_manifest() { diff --git a/SW.Bitween.Sdk/Model/Adapter.cs b/SW.Bitween.Sdk/Model/Adapter.cs index e8458614..0ca47eed 100644 --- a/SW.Bitween.Sdk/Model/Adapter.cs +++ b/SW.Bitween.Sdk/Model/Adapter.cs @@ -9,6 +9,12 @@ public class AdapterSearchRequest { /// Optional filter; null lists every adapter. public string? Prefix { get; set; } + + /// + /// False lists the adapters without their startup properties, which for a published adapter + /// means starting it: for a caller after its versions alone. Null or true describes them. + /// + public bool? StartupValues { get; set; } } public class AdapterRow From e84091d22d0c9c9815ec286e59778cff7b66c60a Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Tue, 27 Oct 2026 16:22:33 +0300 Subject: [PATCH 2/4] fix: bring a subscription whose adapter needs a secret --- SW.Bitween.Api/Resources/Mirror/Mirror.cs | 24 ++++++ .../Services/Mirror/MirrorService.Bring.cs | 84 ++++++++++++++++++- .../Services/Mirror/MirrorService.cs | 11 ++- SW.Bitween.Api/Services/Mirror/MirrorSetup.cs | 6 +- SW.Bitween.UnitTests/MirrorBringTests.cs | 79 +++++++++++++++++ SW.Bitween.Web/ClientApp/src/api/client.ts | 1 + .../ClientApp/src/api/http/mirror.ts | 5 ++ SW.Bitween.Web/ClientApp/src/api/types.ts | 5 +- .../ClientApp/src/pages/mirror/shared.tsx | 31 +++++-- docs/api-reference.md | 5 +- docs/prod-mirror.md | 4 +- 11 files changed, 240 insertions(+), 15 deletions(-) diff --git a/SW.Bitween.Api/Resources/Mirror/Mirror.cs b/SW.Bitween.Api/Resources/Mirror/Mirror.cs index 738672bc..d0861a27 100644 --- a/SW.Bitween.Api/Resources/Mirror/Mirror.cs +++ b/SW.Bitween.Api/Resources/Mirror/Mirror.cs @@ -290,6 +290,30 @@ public async Task Handle(MirrorTestManyRequest request) } } +public class MirrorFilledInRequest +{ + /// The item's kind: subscriptions. + public string Section { get; set; } + + /// The item, by its name. + public string Key { get; set; } +} + +/// +/// Says a person has put staging's own value in each secret a bring had to save as a placeholder on the +/// item: Bitween hides a saved secret, so staging can't see it change. Its to-dos close and its tests run. +/// +[HandlerName("filledin")] +public class FilledIn(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror) + : ICommandHandler +{ + public async Task Handle(MirrorFilledInRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + return new { Closed = await mirror.FilledInAsync(request?.Section, request?.Key) }; + } +} + public class MirrorTestStopRequest; /// Drops the tests waiting their turn; the one under way finishes. How many were dropped. diff --git a/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs b/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs index f8a1dc23..8837e62a 100644 --- a/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs +++ b/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs @@ -3,9 +3,11 @@ using System.Linq; using System.Net; using System.Net.Http; +using System.Text.Json; using System.Text.Json.Nodes; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using SW.Bitween.Cli; using SW.Bitween.Cli.Configuration; using SW.PrimitiveTypes; @@ -175,9 +177,19 @@ public async Task BringAsync(string section, IReadOnlyCollection stoodIn; + await using (var scope = scopes.CreateAsyncScope()) + stoodIn = await StandInAsync(bringing.Plan.Changes, comparison.Copy.Blanks, + scope.ServiceProvider.GetRequiredService().MissingFor); var failed = await comparison.Planner.ApplyAllAsync(bringing.Plan); foreach (var change in bringing.Plan.Changes) LastFailures.TryRemove((change.Section, change.Key), out _); foreach (var (change, error) in failed) LastFailures[(change.Section, change.Key)] = error; + if (stoodIn.Where(s => failed.All(f => !ReferenceEquals(f.Change, s.Change))).Select(s => s.Id).ToList() is { Count: > 0 } saved) + { + var standIns = await ReadStandInsAsync(); + standIns.UnionWith(saved); + await SaveStandInsAsync(standIns); + } var made = bringing.Pieces.Select(p => p.State == "new" ? p with { StagingId = IdOn(comparison.Staging, p.Section, p.Key, bringing.Index) } : p).ToList(); return new MirrorBring(true, made, bringing.NeedsYou, failed.Select(f => new MirrorFailure(f.Change.Section, f.Change.Noun, f.Change.Key, f.Error)).ToList()); } @@ -239,14 +251,84 @@ static IEnumerable FillIns(Comparison comparison, IEnumerable<(strin var alerts = OnlyAlerts(blank.Section); yield return new MirrorNote($"fill:{blank.Section}:{blank.Key}:{string.Join("/", blank.Path)}", "fill", blank.Kind switch { + _ when blank.StoodIn => $"Fill in {what} with staging's own secret, then tick Filled in: it holds {MirrorFaker.PlaceholderText}, " + + "which its adapter can't be saved without, and Bitween hides a saved secret, so staging can't see it change.", "secret" => $"Fill in {what} with staging's own secret: it is empty.", "login" => $"Fill in {what} with your test login, in place of {MirrorFaker.PlaceholderText}.", _ => $"Fill in {what} with your test system's, in place of the placeholder, which leads nowhere.", } + (alerts ? " Tests run without it: it only sends alerts." : ""), - blank.Section, blank.Key, alerts ? "alerts" : null, row?.StagingId, row?.ProdId, row?.Type, blank.Field, Kinds.Of(blank.Section).Noun); + blank.Section, blank.Key, alerts ? "alerts" : blank.StoodIn ? "filledIn" : null, row?.StagingId, row?.ProdId, row?.Type, + blank.Field, Kinds.Of(blank.Section).Noun); + } + } + + // ------------------------------------------------------------------ secrets saved as placeholders + + /// The secrets a bring saved as placeholders, each by , until a person says each was filled in. + const string StandInsKey = "mirror/standins.json"; + + internal static string StandInId(MirrorBlank blank) => $"{blank.Section}/{blank.Key}/{string.Join("/", blank.Path)}"; + + /// + /// The secrets of subscriptions about to be made that their adapters can't be saved without, and that come + /// empty because the copy never brings one: each gets a placeholder, so the subscription is made, and is + /// said for staging to remember, since once saved Bitween hides it like a filled one. + /// is Bitween's own check, what saving it would refuse for (see + /// ); an adapter it can't describe yet - one this bring + /// installs - is left for saving to refuse, as before. + /// + internal static async Task> StandInAsync(IEnumerable changes, IReadOnlyList blanks, + Func, Task>> missingFor) + { + var stoodIn = new List<(Change, string)>(); + foreach (var change in changes.Where(c => c.Op == Op.Create && c.Section == "subscriptions")) + foreach (var role in new[] { "receiver", "validator", "mapper", "handler" }) + { + if (change.Want[role] is not JsonObject block || (string)block["adapter"] is not { Length: > 0 } adapter) continue; + var properties = block["properties"] as JsonObject ?? []; + IReadOnlyCollection missing; + try + { + missing = await missingFor(SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(adapter, (string)block["version"]), + properties.Select(p => new KeyAndValue { Key = p.Key, Value = p.Value?.ToString() }).ToList()); + } + catch (Exception) { continue; } + foreach (var blank in blanks.Where(b => b.Section == change.Section && b.Key == change.Key && b.Kind == "secret" + && b.Path.Count == 3 && b.Path[0] == role && missing.Contains(b.Path[2], StringComparer.OrdinalIgnoreCase))) + { + properties[blank.Path[2]] = MirrorFaker.PlaceholderText; + stoodIn.Add((change, StandInId(blank))); + } + } + return stoodIn; + } + + async Task> ReadStandInsAsync() + { + try + { + await using var stream = await cloudFiles.OpenReadAsync(StandInsKey); + return await JsonSerializer.DeserializeAsync>(stream, ManifestJson) ?? []; + } + catch (Exception ex) when (StorageErrors.IsNotFound(ex)) + { + return []; } } + Task SaveStandInsAsync(HashSet standIns) => + cloudFiles.WriteTextAsync(JsonSerializer.Serialize(standIns, ManifestJson), + new WriteFileSettings { Key = StandInsKey, ContentType = "application/json" }); + + /// A person has filled in an item's secrets a bring saved as placeholders: how many it closes. + public async Task FilledInAsync(string section, string key) + { + var standIns = await ReadStandInsAsync(); + var closed = standIns.RemoveWhere(id => id.StartsWith($"{section}/{key}/", StringComparison.Ordinal)); + if (closed > 0) await SaveStandInsAsync(standIns); + return closed; + } + /// Kinds that only send alerts: an exchange runs the same whether they are filled in or not. internal static bool OnlyAlerts(string section) => section is "notificationChannels" or "notifiers"; diff --git a/SW.Bitween.Api/Services/Mirror/MirrorService.cs b/SW.Bitween.Api/Services/Mirror/MirrorService.cs index b99100f7..483cf1f8 100644 --- a/SW.Bitween.Api/Services/Mirror/MirrorService.cs +++ b/SW.Bitween.Api/Services/Mirror/MirrorService.cs @@ -146,7 +146,7 @@ async Task CompareAsync(BitweenApi prod, Func openSelf, config.Rename(ConfigurationName); var planner = new Planner(selfCtx, config); await planner.ReadAsync(); - var open = TakeStagingsOwn(copy, selfCtx); + var open = TakeStagingsOwn(copy, selfCtx, await ReadStandInsAsync()); // A subscription staging doesn't have yet, whose adapter it lacks and can't get from the // marketplace, would only fail: it waits, and says for which adapter. One staging has already @@ -180,9 +180,11 @@ async Task CompareAsync(BitweenApi prod, Func openSelf, /// /// Every address and secret the copy didn't bring takes staging's own where staging has one, so - /// staging's never reads as a difference from prod's. Those still empty or a placeholder are left to do. + /// staging's never reads as a difference from prod's. Those still empty or a placeholder are left to do, + /// as is a secret a bring saved as a placeholder (, see ), + /// which reads back hidden like a filled one. /// - static List TakeStagingsOwn(MirrorSetupCopy copy, Context staging) + internal static List TakeStagingsOwn(MirrorSetupCopy copy, Context staging, ISet standIns) { var open = new List(); foreach (var blank in copy.Blanks) @@ -191,7 +193,10 @@ static List TakeStagingsOwn(MirrorSetupCopy copy, Context staging) var item = copy.Sections.GetValueOrDefault(blank.Section)?.FirstOrDefault(i => Kinds.Of(blank.Section).KeyOf(i) == blank.Key); if (At(mine, blank.Path) is JsonValue v && v.TryGetValue(out var value) && !MirrorFaker.IsPlaceholder(value) && At(item, blank.Path.Take(blank.Path.Count - 1)) is JsonObject owner) + { owner[blank.Path[^1]] = value; + if (standIns.Contains(StandInId(blank))) open.Add(blank with { StoodIn = true }); + } else open.Add(blank); } diff --git a/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs b/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs index e1e0dc07..2c3ed81c 100644 --- a/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs +++ b/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs @@ -15,7 +15,11 @@ namespace SW.Bitween.Services.Mirror; /// Where it is in the item, as its configuration file says it: handler, properties, Url. /// Where it is, in words: handler · Url. /// address, secret or login: what staging's own value is. -public record MirrorBlank(string Section, string Key, IReadOnlyList Path, string Field, string Kind); +/// +/// A secret a bring saved as a placeholder, which Bitween hides like any saved secret: open until a person +/// says it was filled in. +/// +public record MirrorBlank(string Section, string Key, IReadOnlyList Path, string Field, string Kind, bool StoodIn = false); /// A partner property or global value that is faked, unless it is kept because it is a setting. public record MirrorFakedKey(string Id, string Label, int Count, bool Kept); diff --git a/SW.Bitween.UnitTests/MirrorBringTests.cs b/SW.Bitween.UnitTests/MirrorBringTests.cs index 72c8cfd9..fbccee32 100644 --- a/SW.Bitween.UnitTests/MirrorBringTests.cs +++ b/SW.Bitween.UnitTests/MirrorBringTests.cs @@ -1,8 +1,13 @@ +using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Text.Json.Nodes; +using System.Threading.Tasks; using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Cli.Configuration; using SW.Bitween.Services.Mirror; +using SW.PrimitiveTypes; namespace SW.Bitween.UnitTests; @@ -109,4 +114,78 @@ public void Says_in_one_line_whose_data_comes_faked_however_many() MirrorService.FakedNote(index, [("partners", "P2"), ("partners", "P1"), ("partners", "Bare")]).Text); Assert.IsNull(MirrorService.FakedNote(index, [("partners", "Bare")])); } + + // ------------------------------------------------------------------ secrets saved as placeholders + + static readonly MirrorFaker Faker = new(Encoding.UTF8.GetBytes("a key only these tests use")); + + /// Prod's "Agent tracing", whose adapter can't be saved without its Password; its ApiKey it can. + static MirrorSetupCopy Tracing() => MirrorSetup.Copy(new Dictionary> + { + ["subscriptions"] = [new Existing("1", "Agent tracing", (JsonObject)JsonNode.Parse(""" + { + "name": "Agent tracing", "type": "Receiving", + "receiver": { "adapter": "infolink5.receivers.cassini", "properties": { + "Url": "https://cassini.example.com", "Password": "__private__", "ApiKey": "__private__" } } + } + """)!, new JsonObject())], + }, Faker, new HashSet()); + + /// Bitween's check: what saving would refuse for, of Url and Password. + static Task> Requires(string adapter, ICollection provided) => + Task.FromResult>(new[] { "Url", "Password" } + .Where(n => string.IsNullOrEmpty(provided.FirstOrDefault(p => p.Key == n)?.Value)).ToList()); + + [TestMethod] + public async Task A_secret_its_adapter_cannot_be_saved_without_is_saved_as_a_placeholder() + { + var copy = Tracing(); + var item = copy.Sections["subscriptions"].Single(); + var creating = new Change(Op.Create, "subscriptions", "Agent tracing", item); + + var stoodIn = await MirrorService.StandInAsync([creating], copy.Blanks, Requires); + + var properties = item["receiver"]!["properties"]!; + Assert.AreEqual("set-me", (string)properties["Password"]!); + Assert.AreEqual("", (string)properties["ApiKey"]!); // it can do without: left empty, as before + Assert.AreEqual("https://set-me.invalid", (string)properties["Url"]!); + CollectionAssert.AreEqual(new[] { "subscriptions/Agent tracing/receiver/properties/Password" }, stoodIn.Select(s => s.Id).ToList()); + Assert.AreSame(creating, stoodIn.Single().Change); + } + + [TestMethod] + public async Task Only_a_subscription_being_made_whose_adapter_can_be_described_gets_one() + { + var copy = Tracing(); + var item = copy.Sections["subscriptions"].Single(); + + // Staging's own is never touched; one whose adapter this bring installs is left for saving to refuse. + Assert.AreEqual(0, (await MirrorService.StandInAsync([new Change(Op.Update, "subscriptions", "Agent tracing", item)], copy.Blanks, Requires)).Count); + Assert.AreEqual(0, (await MirrorService.StandInAsync([new Change(Op.Create, "subscriptions", "Agent tracing", item)], copy.Blanks, + (_, _) => throw new InvalidOperationException("not installed"))).Count); + Assert.AreEqual("", (string)item["receiver"]!["properties"]!["Password"]!); + } + + [TestMethod] + public void A_secret_saved_as_a_placeholder_stays_to_do_until_a_person_says_it_is_filled_in() + { + var copy = Tracing(); + var staging = new Context(null); + staging.Current["subscriptions"] = [new Existing("7", "Agent tracing", (JsonObject)JsonNode.Parse(""" + { "name": "Agent tracing", "receiver": { "adapter": "infolink5.receivers.cassini", "properties": { + "Url": "https://test.cassini.example.com", "Password": "__private__", "ApiKey": "" } } } + """)!, new JsonObject())]; + var password = "subscriptions/Agent tracing/receiver/properties/Password"; + + var open = MirrorService.TakeStagingsOwn(copy, staging, new HashSet { password }); + + // Hidden like a filled one, so it reads as no difference; but it still stops tests until ticked. + Assert.AreEqual("__private__", (string)copy.Sections["subscriptions"].Single()["receiver"]!["properties"]!["Password"]!); + CollectionAssert.AreEquivalent(new[] { "receiver · Password True", "receiver · ApiKey False" }, + open.Select(b => $"{b.Field} {b.StoodIn}").ToList()); + + // Ticked: only the empty ApiKey is left. + CollectionAssert.AreEqual(new[] { "receiver · ApiKey" }, + MirrorService.TakeStagingsOwn(Tracing(), staging, new HashSet()).Select(b => b.Field).ToList()); + } } diff --git a/SW.Bitween.Web/ClientApp/src/api/client.ts b/SW.Bitween.Web/ClientApp/src/api/client.ts index 0bbe0eba..f0ccdf86 100644 --- a/SW.Bitween.Web/ClientApp/src/api/client.ts +++ b/SW.Bitween.Web/ClientApp/src/api/client.ts @@ -683,6 +683,7 @@ export interface ApiClient { getMirrorListed(section: MirrorBringSection): Promise; testMirrorMany(section: MirrorBringSection, ids: number[]): Promise<{ subscriptions: string[] }>; stopMirrorTests(): Promise<{ dropped: number }>; + mirrorFilledIn(section: string, key: string): Promise<{ closed: number }>; // - settings - listSettings(): Promise; diff --git a/SW.Bitween.Web/ClientApp/src/api/http/mirror.ts b/SW.Bitween.Web/ClientApp/src/api/http/mirror.ts index 9a8e85da..112b8ef4 100644 --- a/SW.Bitween.Web/ClientApp/src/api/http/mirror.ts +++ b/SW.Bitween.Web/ClientApp/src/api/http/mirror.ts @@ -65,6 +65,11 @@ export const mirrorMethods = { return post<{ dropped: number }>("/mirror/teststop", {}); }, + /** Says staging's own secrets are in where a bring saved placeholders on the item, which staging can't see. */ + mirrorFilledIn(section: string, key: string): Promise<{ closed: number }> { + return post<{ closed: number }>("/mirror/filledin", { section, key }); + }, + /** What bringing the picked items from prod does; with `apply`, brings them. */ bringFromProd(section: MirrorBringSection, keys: string[], apply: boolean): Promise { return post("/mirror/bring", { section, keys, apply }); diff --git a/SW.Bitween.Web/ClientApp/src/api/types.ts b/SW.Bitween.Web/ClientApp/src/api/types.ts index d13ea297..e69370c7 100644 --- a/SW.Bitween.Web/ClientApp/src/api/types.ts +++ b/SW.Bitween.Web/ClientApp/src/api/types.ts @@ -2297,7 +2297,10 @@ export interface MirrorNote { /** The item it is about, to open it. */ section: string | null; key: string | null; - /** Where it is done: adapters-installed or adapters-marketplace. */ + /** + * Where it is done: adapters-installed or adapters-marketplace. For a fill-in: alerts (tests run + * without it), or filledIn (a secret saved as a placeholder, which a person ticks once filled in). + */ action: string | null; /** The item on staging and on prod, to open it on each; null where a side doesn't have it yet. */ stagingId?: string | null; diff --git a/SW.Bitween.Web/ClientApp/src/pages/mirror/shared.tsx b/SW.Bitween.Web/ClientApp/src/pages/mirror/shared.tsx index 93b73574..a53f92b7 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/mirror/shared.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/mirror/shared.tsx @@ -1,10 +1,10 @@ -import { useQuery } from "@tanstack/react-query"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { ExternalLink } from "lucide-react"; import { useMemo, useState } from "react"; import { Link } from "react-router"; import { api, type MirrorItem, type MirrorNote, type MirrorSample, type MirrorSampleFile, type SubscriptionType } from "../../api"; import { keys } from "../../api/queryKeys"; -import { Badge } from "../../components/ui/basics"; +import { Badge, Button } from "../../components/ui/basics"; import { useSessionCan } from "../../auth/useSessionCan"; import { whole } from "../../lib/numbers"; import { subscriptionPath } from "../../lib/subscriptionPaths"; @@ -76,12 +76,14 @@ const FEW = 5; */ export function FillIns({ notes, links = false }: { notes: MirrorNote[]; links?: boolean }) { const [all, setAll] = useState(false); + const { canOperate } = useMirror(); const items = useMemo(() => { - const byItem = new Map(); + const byItem = new Map(); for (const n of notes) { const id = `${n.section}/${n.key}`; - const item = byItem.get(id) ?? { note: n, fields: [], alerts: n.action === "alerts" }; + const item = byItem.get(id) ?? { note: n, fields: [], alerts: n.action === "alerts", standIn: false }; if (n.field) item.fields.push(n.field); + if (n.action === "filledIn") item.standIn = true; byItem.set(id, item); } return [...byItem.values()].sort((a, b) => Number(a.alerts) - Number(b.alerts)); @@ -90,7 +92,7 @@ export function FillIns({ notes, links = false }: { notes: MirrorNote[]; links?: return (
    - {shown.map(({ note, fields, alerts }) => ( + {shown.map(({ note, fields, alerts, standIn }) => (
  • {note.key} {note.noun} @@ -104,6 +106,7 @@ export function FillIns({ notes, links = false }: { notes: MirrorNote[]; links?: Stops tests )} {links && } + {links && standIn && canOperate && }
    • ))}
@@ -116,6 +119,24 @@ export function FillIns({ notes, links = false }: { notes: MirrorNote[]; links?: ); } +/** + * For a secret a bring saved as a placeholder so its subscription could be made: Bitween hides it once + * saved, so only a person can say staging's own is in. + */ +function FilledIn({ section, itemKey }: { section: string; itemKey: string }) { + const queryClient = useQueryClient(); + const tick = useMutation({ + mutationFn: () => api.mirrorFilledIn(section, itemKey), + onSuccess: () => queryClient.invalidateQueries({ queryKey: keys.mirror.all }), + }); + return ( + + ); +} + /** "On staging" and "On prod" links to one item, as far as each side has it. */ export function ItemLinks({ section, item, prodUrl }: { section: string; item: MirrorItem; prodUrl: string }) { const staging = itemPath(section, item.stagingId, item.type); diff --git a/docs/api-reference.md b/docs/api-reference.md index 4f6ea40f..78385301 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -193,7 +193,7 @@ A value that breaks a rule is refused with a validation error whose message says | Method and path | Permission | Description | |---|---|---| -| `GET /api/adapters/Catalog?prefix=` | `subscriptions.view` | Every adapter of one kind with its properties. `prefix` is `receivers`, `handlers`, `mappers` or `validators`. Includes adapters published only to the catalog. Each entry of `versionHistory` has `hasSource` and `runtime`. | +| `GET /api/adapters/Catalog?prefix=&startupValues=` | `subscriptions.view` | Every adapter of one kind with its properties. `prefix` is `receivers`, `handlers`, `mappers` or `validators`. Includes adapters published only to the catalog. Each entry of `versionHistory` has `hasSource` and `runtime`. `startupValues=false` leaves the properties out (`startupValues` is null): reading them starts each published adapter, which takes seconds each the first time. | | `GET /api/adapters?prefix=` | `subscriptions.view` | Adapter ids of one kind | | `GET /api/adapters/Versioned?prefix=` | `subscriptions.view` | Adapter ids with versions | | `GET /api/adapters/{id}/GetStartupValues` | `subscriptions.view` | One adapter's properties | @@ -350,13 +350,14 @@ See [Data sources](data-sources.md) and [Databases](databases.md). | `GET /api/mirror/prodkinds` | `mirror.view`. How many of prod's items there are of each kind a list page offers to bring, read from prod's lists alone: `subscriptions` by type, with `all`, `apiGateways` and `busGateways`. `error` says why prod couldn't be read. | | `GET /api/mirror/prod?section=&type=` | `mirror.view`. Prod's items of one kind (`subscriptions`, `apiGateways` or `busGateways`, and of subscriptions only one `type` when it is given), busiest first: each with its `traffic` over the last `days` (`total`, `failed`, `lastOn`, `daily`), a `detail`, its `state` on the staging (`inSync`, `differs`, `missing` or `waiting`) and its id on both. A prod whose release doesn't count traffic answers `days: null` with a `trafficNote`. A prod that can't be reached or is older than r10.0 answers `{ error }`. | | `POST /api/mirror/bring` | `mirror.operate`. `{ section, keys, apply }`. Brings prod's picked items with what they need to run, as [Prod mirror](prod-mirror.md#bringing-items-from-prod) describes: `pieces`, each with its `state` (`new`, `kept`, `adds`, `waits` or `install`), `why` and `reason`, `needsYou`, and with `apply`, what `failed`. Without `apply` it changes nothing. Refused with `MIRROR_NOT_TARGET` unless the deployment sets `Bitween:Mirror:Target`. | -| `GET /api/mirror/subscription?id=` | `mirror.view`. One of this staging's subscriptions beside prod's of the same name: `onProd`, `prodId`, `used` (the subscription and each item it needs, each with its `part`, `level` (`same`, `faked`, `info`, `check` or `different`), prod's and the staging's side, a `note` and `fields`, then its adapters), `toBring` (what bringing it would bring, as `POST /api/mirror/bring` says it), `needsYou` (each with its `group`: `needsYou`, `fill` for an address, secret or login still to fill in, with the item's `stagingId` and `prodId`, `action` `alerts` when it only sends alerts, or `faked`) and `customAdapters`, the adapters it runs that don't come with Bitween. A prod that can't be read answers `{ error }`. | +| `GET /api/mirror/subscription?id=` | `mirror.view`. One of this staging's subscriptions beside prod's of the same name: `onProd`, `prodId`, `used` (the subscription and each item it needs, each with its `part`, `level` (`same`, `faked`, `info`, `check` or `different`), prod's and the staging's side, a `note` and `fields`, then its adapters), `toBring` (what bringing it would bring, as `POST /api/mirror/bring` says it), `needsYou` (each with its `group`: `needsYou`, `fill` for an address, secret or login still to fill in, with the item's `stagingId` and `prodId`, `action` `alerts` when it only sends alerts or `filledIn` for a secret saved as `set-me`, to tick with `POST /api/mirror/filledin` once filled in, or `faked`) and `customAdapters`, the adapters it runs that don't come with Bitween. A prod that can't be read answers `{ error }`. | | `GET /api/mirror/test?id=` | `mirror.view`. A subscription's test: `state` (`idle`, `taking`, `running`, `queued` while it waits its turn, or `failed` with an `error`), `busy` (other work of the mirror under way), `done` of `total`, prod's arrivals (`families`, `takenOn`, and a `reason` when there are none), how they last ran (`runs`, `ranOn`, `setupError`), `exchanges`, how many exchanges its runs left on staging, and `last`, how the last run went (`verdict`: `same`, `check` or `different`, `ranOn`, and how many arrivals were `same`, `check`, `different` or `notRun`), which a clear keeps. | | `POST /api/mirror/testtake` | `mirror.operate`. `{ id }`. Starts taking prod's latest arrivals of the subscription in the background, and answers as `GET /api/mirror/test` does. Refused with `MIRROR_NOT_TARGET` unless the deployment sets `Bitween:Mirror:Target`, and with `MIRROR_BUSY` while other work of the mirror runs. | | `POST /api/mirror/testrun` | `mirror.operate`. `{ id }`. Starts the subscription's test in the background: each arrival sent in on staging the way it came in on prod and compared with prod's. Answers and is refused as `POST /api/mirror/testtake` is. The test stops with an `error` while an address, secret or login it needs is still to fill in. | | `POST /api/mirror/testclear` | `mirror.operate`. `{ id }`. Removes every exchange the subscription's test made on staging, with its files, and each arrival's results; prod's arrivals stay, and so does `last`. Refused with `MIRROR_BUSY` while the test runs. | | `GET /api/mirror/listed?section=` | `mirror.view`. This staging's items of one kind (`subscriptions`, `apiGateways` or `busGateways`) that prod has too, by name, read from prod's list: each with its `id`, `key`, a subscription's `type`, and its test's `verdict` (`same`, `check`, `different`, `noTraffic` when prod had none of it to take, or null when not tested) and `ranOn`; a gateway's are those of the subscriptions it leads to that prod has too, with how many were `tested` `of` them; and `custom` when it runs an adapter that doesn't come with Bitween. Also `testing`, the subscription whose test is under way, and `waiting`, those waiting their turn. `error` says why prod couldn't be read. | | `POST /api/mirror/testmany` | `mirror.operate`. `{ section, ids }`. Starts testing the items one subscription after another in the background (a gateway through the subscriptions it leads to), taking prod's arrivals first where none are, or none were found last time; answers the `subscriptions` it tests. Only those prod has too. Refused with `MIRROR_NOT_TARGET` unless the deployment sets `Bitween:Mirror:Target`, with `MIRROR_NONE` when none of them is on prod, and with `MIRROR_BUSY` while other work of the mirror runs. | +| `POST /api/mirror/filledin` | `mirror.operate`. `{ section, key }`. Says staging's own secrets are in where a bring saved `set-me` on the item, because its adapter can't be saved without them; staging can't see a saved secret. Their to-dos (`action` `filledIn`) close and its tests can run. Answers how many it `closed`. | | `POST /api/mirror/teststop` | `mirror.operate`. Drops the tests waiting their turn; the one under way finishes. Answers how many were `dropped`. | ## Monitoring diff --git a/docs/prod-mirror.md b/docs/prod-mirror.md index bf934632..dc4e5d97 100644 --- a/docs/prod-mirror.md +++ b/docs/prod-mirror.md @@ -24,12 +24,12 @@ Values are handled the moment they arrive, before anything is saved, so the stag |---|---|---| | How a flow behaves | formats, schedules, filters and match expressions, mapping rules, methods, delimiters, file name patterns, `{{globals.…}}` tokens | Copied as it is | | Addresses | hosts, URLs, email addresses, connection strings, an API gateway's issuer, a connection's login URLs | Never copied: a placeholder that leads nowhere, `https://set-me.invalid` or `set-me@set-me.invalid`, until you fill in your test system's | -| Secrets | anything prod shows as `__private__`, values named like secrets (`apiKey`, `password`, `token`) | Never copied: left empty until you fill in staging's own | +| Secrets | anything prod shows as `__private__`, values named like secrets (`apiKey`, `password`, `token`) | Never copied: left empty until you fill in staging's own. One a subscription's adapter can't be saved without is saved as `set-me` | | Logins | a connection's client id and username | Never copied: `set-me` until you fill in your test login | | Partner properties and global values | account numbers, codes, names | A fake of the same shape: digits stay digits, letters keep their case, dates stay dates in their format, GUIDs stay GUIDs, JSON keeps its keys | | Partner API keys | | Keys of staging's own, the same on every copy | -Each address, secret and login left this way is something to fill in. Bringing lists them, and each subscription's **Against prod** section lists those it needs, with a link to the item on staging and one to the same item on prod, to look the value up. A value staging already has, one your team typed in, is kept and never reads as a difference from prod's. A placeholder is used rather than an empty value because Bitween won't save some items without one, such as an HTTP delivery without its URL. +Each address, secret and login left this way is something to fill in. Bringing lists them, and each subscription's **Against prod** section lists those it needs, with a link to the item on staging and one to the same item on prod, to look the value up. A value staging already has, one your team typed in, is kept and never reads as a difference from prod's. A placeholder is used rather than an empty value because Bitween won't save some items without one, such as an HTTP delivery without its URL. The same goes for a secret a subscription's adapter can't do without, such as a password: it is saved as `set-me`. Bitween hides a saved secret, so staging can't see when you put yours in: once you have, click **Filled in** on its row. Until then its tests don't run. The same value always becomes the same fake, so a code named in two places still matches, and copying again changes nothing. A partner property or global value that is a setting rather than data, such as a time zone or a date format, can be kept as it is: list it in **Settings, Prod mirror, Values copied as they are**, one per line, as `partner:timeZone` or `global:formats/date`. Addresses and secrets are never kept. From 6ec34a2a87f44f8973501043790e92cb98ed7cd2 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Tue, 27 Oct 2026 16:25:20 +0300 Subject: [PATCH 3/4] fix: never bring prod's user names to staging --- SW.Bitween.Api/Services/Mirror/MirrorFaker.cs | 22 +++++++++++++++---- SW.Bitween.Api/Services/Mirror/MirrorSetup.cs | 4 ++-- SW.Bitween.UnitTests/MirrorFakerTests.cs | 14 ++++++++++++ SW.Bitween.UnitTests/MirrorSetupTests.cs | 18 +++++++++------ docs/prod-mirror.md | 2 +- 5 files changed, 46 insertions(+), 14 deletions(-) diff --git a/SW.Bitween.Api/Services/Mirror/MirrorFaker.cs b/SW.Bitween.Api/Services/Mirror/MirrorFaker.cs index 6c17660a..f058d068 100644 --- a/SW.Bitween.Api/Services/Mirror/MirrorFaker.cs +++ b/SW.Bitween.Api/Services/Mirror/MirrorFaker.cs @@ -39,15 +39,17 @@ public sealed class MirrorFaker(byte[] key, IEnumerable keep = null) /// /// What a setting or a value of prod's is that staging never holds, so a copy leaves it empty for - /// staging's own: an address (a URL, a host, an email, a connection string) or a secret (one prod - /// masks, or one named like a secret). Null for anything else. A host built from a {{token}} isn't - /// one yet, nor is a secret that names one: the token is what is copied. Hosts every system names - /// alike, such as an XML namespace's www.w3.org, are nobody's. + /// staging's own: an address (a URL, a host, an email, a connection string), a secret (one prod + /// masks, or one named like a secret) or a login (one named like a user name). Null for anything + /// else. A host built from a {{token}} isn't one yet, nor is a secret or a login that names one: the + /// token is what is copied. Hosts every system names alike, such as an XML namespace's www.w3.org, + /// are nobody's. /// public static string BlankKind(string key, string value) { if (string.IsNullOrEmpty(value)) return null; if (value == Masked || IsSecretLike(key) && !TokenPattern.IsMatch(value)) return "secret"; + if (IsLoginKey(key) && !TokenPattern.IsMatch(value)) return "login"; if (IsConnectionStringKey(key) || LooksLikeConnectionString(value) || HostsIn(key, value).Any(h => !Untouched(h))) return "address"; return null; } @@ -97,6 +99,16 @@ public static bool IsSecretLike(string key) return k is "pass" or "key" or "pin" || SecretWords.Any(k.Contains); } + /// + /// A name a login goes by - a user name, a client id - which prod's never is on + /// staging: the team's test login is. SftpUsername and ftpUser are; UserAgent isn't. + /// + public static bool IsLoginKey(string key) + { + var k = (key ?? "").ToLowerInvariant().Replace("_", "").Replace("-", "").Replace(".", ""); + return LoginEndings.Any(k.EndsWith); + } + /// How every stand-in secret starts, which tells one from a secret set on staging. public const string SecretPrefix = "mirror-"; @@ -536,6 +548,8 @@ static byte[] Block(HMACSHA256 hmac, int block, byte[] data) static readonly string[] SecretWords = ["password", "passwd", "pwd", "secret", "token", "credential", "apikey", "accesskey", "privatekey", "passphrase", "certificate", "signingkey"]; + static readonly string[] LoginEndings = ["user", "username", "userid", "login", "loginname", "loginid", "clientid"]; + static readonly string[] AddressWords = ["url", "uri", "host", "server", "endpoint", "address", "domain", "webhook", "smtp", "imap", "ftp"]; static readonly Regex TokenPattern = new(@"\{\{[^}]*\}\}", RegexOptions.Compiled); diff --git a/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs b/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs index 2c3ed81c..def788e3 100644 --- a/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs +++ b/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs @@ -196,7 +196,7 @@ static void AddWords(HashSet words, string text) /// /// Every partner property and global value that gets faked, by key, and whether it is kept. A - /// secret or an address is never offered: neither can be kept, whatever the list says. + /// secret, an address or a login is never offered: none can be kept, whatever the list says. /// static List FakedKeys(IReadOnlyDictionary> prod, ISet keep) { @@ -220,7 +220,7 @@ static IEnumerable Offered(JsonObject values, JsonNode secretNames) { var text = value is JsonValue v && v.TryGetValue(out var s) ? s : null; if (string.IsNullOrEmpty(text) || text == MirrorFaker.Masked || secret.Contains(name)) continue; - if (MirrorFaker.IsAddressKey(name) || MirrorFaker.IsSecretLike(name) || name.EndsWith("port", StringComparison.OrdinalIgnoreCase)) continue; + if (MirrorFaker.IsAddressKey(name) || MirrorFaker.IsSecretLike(name) || MirrorFaker.IsLoginKey(name) || name.EndsWith("port", StringComparison.OrdinalIgnoreCase)) continue; yield return name; } } diff --git a/SW.Bitween.UnitTests/MirrorFakerTests.cs b/SW.Bitween.UnitTests/MirrorFakerTests.cs index 57c0b820..3d360186 100644 --- a/SW.Bitween.UnitTests/MirrorFakerTests.cs +++ b/SW.Bitween.UnitTests/MirrorFakerTests.cs @@ -82,6 +82,20 @@ public void Addresses_are_left_for_stagings_own() Assert.AreEqual("address", MirrorFaker.BlankKind("note", "mail ops@acmecargo.net")); } + [TestMethod] + public void Logins_are_left_for_the_teams_test_login() + { + Assert.AreEqual("login", MirrorFaker.BlankKind("Username", "acme")); + Assert.AreEqual("login", MirrorFaker.BlankKind("SftpUser", "ops")); + Assert.AreEqual("login", MirrorFaker.BlankKind("client_id", "acme-prod")); + // An access key's id goes by a secret's name, so it stays a secret. + Assert.AreEqual("secret", MirrorFaker.BlankKind("AccessKeyId", "AKIAEXAMPLE7Q2")); + // An address named like a login is still a login: the team's own goes there. + Assert.AreEqual("login", MirrorFaker.BlankKind("LoginUser", "ops@acmecargo.net")); + Assert.IsNull(MirrorFaker.BlankKind("UserAgent", "Bitween/10")); + Assert.IsNull(MirrorFaker.BlankKind("Username", "{{partner.ftpUser}}")); + } + [TestMethod] public void A_placeholder_stands_in_and_reads_as_still_to_fill_in() { diff --git a/SW.Bitween.UnitTests/MirrorSetupTests.cs b/SW.Bitween.UnitTests/MirrorSetupTests.cs index 82ad184d..967d50ea 100644 --- a/SW.Bitween.UnitTests/MirrorSetupTests.cs +++ b/SW.Bitween.UnitTests/MirrorSetupTests.cs @@ -46,7 +46,8 @@ public void A_subscription_keeps_how_it_behaves_and_arrives_paused() "name": "Push labels", "type": "Internal", "paused": false, "matchExpression": { "type": "one_of", "path": "$.country", "values": ["FR"] }, "handler": { "adapter": "NativeHttpHandler", "properties": { - "Url": "https://api.carrier.com/v2/labels", "Method": "POST", "Password": "__private__", "Token": "{{globals.carrier.token}}" } } + "Url": "https://api.carrier.com/v2/labels", "Method": "POST", "Username": "acme", "Password": "__private__", + "Token": "{{globals.carrier.token}}" } } } """)), Faker, new HashSet()); @@ -59,8 +60,10 @@ public void A_subscription_keeps_how_it_behaves_and_arrives_paused() // Bitween won't save a handler without its Url: a placeholder that leads nowhere stands in. Assert.AreEqual("https://set-me.invalid", (string)p["Url"]!); Assert.AreEqual("", (string)p["Password"]!); + // Prod's login never comes: the team's test login goes there. + Assert.AreEqual("set-me", (string)p["Username"]!); - CollectionAssert.AreEquivalent(new[] { "handler · Url address", "handler · Password secret" }, + CollectionAssert.AreEquivalent(new[] { "handler · Url address", "handler · Username login", "handler · Password secret" }, copy.Blanks.Select(b => $"{b.Field} {b.Kind}").ToList()); var url = copy.Blanks.Single(b => b.Kind == "address"); Assert.AreEqual(("subscriptions", "Push labels"), (url.Section, url.Key)); @@ -74,10 +77,10 @@ public void A_partner_gets_fakes_of_its_data_and_keys_of_its_own() { "name": "Acme", "secretProperties": ["sftpPassword"], "loginIdentity": "acme-client-778", "properties": { "accountNumber": "AC-1001", "timeZone": "Europe/Paris", "sftpPassword": "__private__", - "apiKey": "live_9f8e7d", "webhookUrl": "https://hooks.acme.com/in" }, + "apiKey": "live_9f8e7d", "webhookUrl": "https://hooks.acme.com/in", "sftpUser": "acme-ftp" }, "keys": { "main": "abcde...(hidden)" } } - """)), Faker, new HashSet { "partner:timeZone", "partner:apiKey", "partner:webhookUrl" }); + """)), Faker, new HashSet { "partner:timeZone", "partner:apiKey", "partner:webhookUrl", "partner:sftpUser" }); var a = Only(copy, "partners"); var p = a["properties"]!; @@ -85,15 +88,16 @@ public void A_partner_gets_fakes_of_its_data_and_keys_of_its_own() Assert.AreEqual("AC-1001".Length, ((string)p["accountNumber"]!).Length); // Kept, because it was asked for and it is a setting. Assert.AreEqual("Europe/Paris", (string)p["timeZone"]!); - // Asked for, and never kept: a name a secret goes by, and an address. They are staging's own to fill in. + // Asked for, and never kept: a name a secret goes by, an address and a login. They are staging's own to fill in. Assert.AreEqual("", (string)p["apiKey"]!); Assert.AreEqual("https://set-me.invalid", (string)p["webhookUrl"]!); Assert.AreEqual("", (string)p["sftpPassword"]!); - CollectionAssert.AreEquivalent(new[] { "apiKey", "webhookUrl", "sftpPassword" }, copy.Blanks.Select(b => b.Field).ToList()); + Assert.AreEqual("set-me", (string)p["sftpUser"]!); + CollectionAssert.AreEquivalent(new[] { "apiKey", "webhookUrl", "sftpPassword", "sftpUser" }, copy.Blanks.Select(b => b.Field).ToList()); Assert.AreNotEqual("acme-client-778", (string)a["loginIdentity"]!); StringAssert.StartsWith((string)a["keys"]!["main"]!, "mirror-"); - // Offered for keeping: data that could be a setting, never a secret, an address or a secret's name. + // Offered for keeping: data that could be a setting, never a secret, an address, a login or a secret's name. CollectionAssert.AreEquivalent(new[] { "partner:accountNumber", "partner:timeZone" }, copy.FakedKeys.Select(k => k.Id).ToList()); Assert.IsTrue(copy.FakedKeys.Single(k => k.Id == "partner:timeZone").Kept); } diff --git a/docs/prod-mirror.md b/docs/prod-mirror.md index dc4e5d97..83bfe4be 100644 --- a/docs/prod-mirror.md +++ b/docs/prod-mirror.md @@ -25,7 +25,7 @@ Values are handled the moment they arrive, before anything is saved, so the stag | How a flow behaves | formats, schedules, filters and match expressions, mapping rules, methods, delimiters, file name patterns, `{{globals.…}}` tokens | Copied as it is | | Addresses | hosts, URLs, email addresses, connection strings, an API gateway's issuer, a connection's login URLs | Never copied: a placeholder that leads nowhere, `https://set-me.invalid` or `set-me@set-me.invalid`, until you fill in your test system's | | Secrets | anything prod shows as `__private__`, values named like secrets (`apiKey`, `password`, `token`) | Never copied: left empty until you fill in staging's own. One a subscription's adapter can't be saved without is saved as `set-me` | -| Logins | a connection's client id and username | Never copied: `set-me` until you fill in your test login | +| Logins | a connection's client id and username; any setting, partner property or global value named like a user name or client id (`Username`, `SftpUser`, `ClientId`) | Never copied: `set-me` until you fill in your test login | | Partner properties and global values | account numbers, codes, names | A fake of the same shape: digits stay digits, letters keep their case, dates stay dates in their format, GUIDs stay GUIDs, JSON keeps its keys | | Partner API keys | | Keys of staging's own, the same on every copy | From 83f8290ce15449855db10205014eb0e4b04ba43e Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Tue, 27 Oct 2026 16:42:53 +0300 Subject: [PATCH 4/4] fix: install adapters before placeholders, and keep placeholder updates safe --- .../Services/Mirror/MirrorService.Bring.cs | 72 +++++++++++++------ .../Configuration/Planner.cs | 7 +- SW.Bitween.UnitTests/MirrorBringTests.cs | 44 ++++++++++-- 3 files changed, 96 insertions(+), 27 deletions(-) diff --git a/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs b/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs index 8837e62a..ccb25f60 100644 --- a/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs +++ b/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs @@ -6,6 +6,7 @@ using System.Text.Json; using System.Text.Json.Nodes; using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using SW.Bitween.Cli; @@ -177,19 +178,14 @@ public async Task BringAsync(string section, IReadOnlyCollection stoodIn; - await using (var scope = scopes.CreateAsyncScope()) - stoodIn = await StandInAsync(bringing.Plan.Changes, comparison.Copy.Blanks, - scope.ServiceProvider.GetRequiredService().MissingFor); - var failed = await comparison.Planner.ApplyAllAsync(bringing.Plan); + await using var scope = scopes.CreateAsyncScope(); + var requirements = scope.ServiceProvider.GetRequiredService(); + var (failed, saved) = await ApplyBringingAsync(bringing.Plan.Changes, + part => comparison.Planner.ApplyAllAsync(bringing.Plan, part), + all => StandInAsync(all, comparison.Copy.Blanks, requirements.MissingFor)); foreach (var change in bringing.Plan.Changes) LastFailures.TryRemove((change.Section, change.Key), out _); foreach (var (change, error) in failed) LastFailures[(change.Section, change.Key)] = error; - if (stoodIn.Where(s => failed.All(f => !ReferenceEquals(f.Change, s.Change))).Select(s => s.Id).ToList() is { Count: > 0 } saved) - { - var standIns = await ReadStandInsAsync(); - standIns.UnionWith(saved); - await SaveStandInsAsync(standIns); - } + if (saved.Count > 0) await UpdateStandInsAsync(standIns => { standIns.UnionWith(saved); return true; }); var made = bringing.Pieces.Select(p => p.State == "new" ? p with { StagingId = IdOn(comparison.Staging, p.Section, p.Key, bringing.Index) } : p).ToList(); return new MirrorBring(true, made, bringing.NeedsYou, failed.Select(f => new MirrorFailure(f.Change.Section, f.Change.Noun, f.Change.Key, f.Error)).ToList()); } @@ -267,15 +263,35 @@ static IEnumerable FillIns(Comparison comparison, IEnumerable<(strin /// The secrets a bring saved as placeholders, each by , until a person says each was filled in. const string StandInsKey = "mirror/standins.json"; - internal static string StandInId(MirrorBlank blank) => $"{blank.Section}/{blank.Key}/{string.Join("/", blank.Path)}"; + /// One read-change-write of at a time: a bring and a Filled in can come together. One staging runs one node. + static readonly SemaphoreSlim StandInsLock = new(1, 1); + + /// A secret's place, as its kind, its item's name and its path, which a name with a slash in it can't run into another's. + internal static string StandInId(MirrorBlank blank) => JsonSerializer.Serialize([blank.Section, blank.Key, .. blank.Path]); + + /// + /// Carries out a bring's changes: adapters first, so the placeholders (see ) can ask + /// what an adapter the bring installs can't do without, then the rest. What failed and why, and the placeholders + /// saved with what was made. + /// + internal static async Task<(List<(Change Change, string Error)> Failed, List Saved)> ApplyBringingAsync(IReadOnlyList changes, + Func, Task>> apply, + Func, Task>> standIn) + { + var adapters = changes.Where(c => c.Section == Kinds.Adapters).ToList(); + var failed = adapters.Count > 0 ? await apply(adapters) : []; + var stoodIn = await standIn(changes); + failed.AddRange(await apply(changes.Where(c => c.Section != Kinds.Adapters).ToList())); + return (failed, stoodIn.Where(s => failed.All(f => !ReferenceEquals(f.Change, s.Change))).Select(s => s.Id).ToList()); + } /// /// The secrets of subscriptions about to be made that their adapters can't be saved without, and that come /// empty because the copy never brings one: each gets a placeholder, so the subscription is made, and is /// said for staging to remember, since once saved Bitween hides it like a filled one. /// is Bitween's own check, what saving it would refuse for (see - /// ); an adapter it can't describe yet - one this bring - /// installs - is left for saving to refuse, as before. + /// ); an adapter it can't describe - one that failed to + /// install - is left for saving to refuse. /// internal static async Task> StandInAsync(IEnumerable changes, IReadOnlyList blanks, Func, Task>> missingFor) @@ -316,19 +332,35 @@ async Task> ReadStandInsAsync() } } - Task SaveStandInsAsync(HashSet standIns) => - cloudFiles.WriteTextAsync(JsonSerializer.Serialize(standIns, ManifestJson), - new WriteFileSettings { Key = StandInsKey, ContentType = "application/json" }); + /// Changes the placeholders kept, written back when says it changed them. + async Task UpdateStandInsAsync(Func, bool> change) + { + await StandInsLock.WaitAsync(); + try + { + var standIns = await ReadStandInsAsync(); + if (change(standIns)) + await cloudFiles.WriteTextAsync(JsonSerializer.Serialize(standIns, ManifestJson), + new WriteFileSettings { Key = StandInsKey, ContentType = "application/json" }); + } + finally + { + StandInsLock.Release(); + } + } /// A person has filled in an item's secrets a bring saved as placeholders: how many it closes. public async Task FilledInAsync(string section, string key) { - var standIns = await ReadStandInsAsync(); - var closed = standIns.RemoveWhere(id => id.StartsWith($"{section}/{key}/", StringComparison.Ordinal)); - if (closed > 0) await SaveStandInsAsync(standIns); + var closed = 0; + await UpdateStandInsAsync(standIns => (closed = CloseStandIns(standIns, section, key)) > 0); return closed; } + /// Takes out the item's placeholders, and only that item's: how many. + internal static int CloseStandIns(HashSet standIns, string section, string key) => + standIns.RemoveWhere(id => JsonSerializer.Deserialize(id) is [var s, var k, ..] && s == section && k == key); + /// Kinds that only send alerts: an exchange runs the same whether they are filled in or not. internal static bool OnlyAlerts(string section) => section is "notificationChannels" or "notifiers"; diff --git a/SW.Bitween.Configuration/Configuration/Planner.cs b/SW.Bitween.Configuration/Configuration/Planner.cs index 7261a10d..eb14dc66 100644 --- a/SW.Bitween.Configuration/Configuration/Planner.cs +++ b/SW.Bitween.Configuration/Configuration/Planner.cs @@ -246,14 +246,15 @@ public async Task ApplyAsync(Plan plan, TextWriter output) /// /// Every change it can make, carrying on past one that fails, and why each that failed did: for a /// copy of a whole Bitween, where one subscription waiting for its adapter shouldn't hold up the - /// rest. A change that names what failed before it fails too, and says so. + /// rest. A change that names what failed before it fails too, and says so. With , + /// just those of the plan's changes, so a caller can do something between one part and the next. /// - public async Task> ApplyAllAsync(Plan plan) + public async Task> ApplyAllAsync(Plan plan, IReadOnlyList only = null) { var failed = new List<(Change, string)>(); try { - foreach (var change in plan.Changes) + foreach (var change in only ?? plan.Changes) { try { await CarryOutAsync(change); } // A timed-out call surfaces as TaskCanceledException: one slow item, not a reason to stop. diff --git a/SW.Bitween.UnitTests/MirrorBringTests.cs b/SW.Bitween.UnitTests/MirrorBringTests.cs index fbccee32..76b63dc9 100644 --- a/SW.Bitween.UnitTests/MirrorBringTests.cs +++ b/SW.Bitween.UnitTests/MirrorBringTests.cs @@ -131,6 +131,8 @@ static MirrorSetupCopy Tracing() => MirrorSetup.Copy(new Dictionary()); + const string Password = """["subscriptions","Agent tracing","receiver","properties","Password"]"""; + /// Bitween's check: what saving would refuse for, of Url and Password. static Task> Requires(string adapter, ICollection provided) => Task.FromResult>(new[] { "Url", "Password" } @@ -149,7 +151,7 @@ public async Task A_secret_its_adapter_cannot_be_saved_without_is_saved_as_a_pla Assert.AreEqual("set-me", (string)properties["Password"]!); Assert.AreEqual("", (string)properties["ApiKey"]!); // it can do without: left empty, as before Assert.AreEqual("https://set-me.invalid", (string)properties["Url"]!); - CollectionAssert.AreEqual(new[] { "subscriptions/Agent tracing/receiver/properties/Password" }, stoodIn.Select(s => s.Id).ToList()); + CollectionAssert.AreEqual(new[] { Password }, stoodIn.Select(s => s.Id).ToList()); Assert.AreSame(creating, stoodIn.Single().Change); } @@ -175,9 +177,7 @@ public void A_secret_saved_as_a_placeholder_stays_to_do_until_a_person_says_it_i { "name": "Agent tracing", "receiver": { "adapter": "infolink5.receivers.cassini", "properties": { "Url": "https://test.cassini.example.com", "Password": "__private__", "ApiKey": "" } } } """)!, new JsonObject())]; - var password = "subscriptions/Agent tracing/receiver/properties/Password"; - - var open = MirrorService.TakeStagingsOwn(copy, staging, new HashSet { password }); + var open = MirrorService.TakeStagingsOwn(copy, staging, new HashSet { Password }); // Hidden like a filled one, so it reads as no difference; but it still stops tests until ticked. Assert.AreEqual("__private__", (string)copy.Sections["subscriptions"].Single()["receiver"]!["properties"]!["Password"]!); @@ -188,4 +188,40 @@ public void A_secret_saved_as_a_placeholder_stays_to_do_until_a_person_says_it_i CollectionAssert.AreEqual(new[] { "receiver · ApiKey" }, MirrorService.TakeStagingsOwn(Tracing(), staging, new HashSet()).Select(b => b.Field).ToList()); } + + [TestMethod] + public async Task Adapters_go_in_before_the_placeholders_are_worked_out_and_only_whats_made_keeps_one() + { + var install = new Change(Op.Install, Kinds.Adapters, "infolink5.receivers.cassini"); + var made = new Change(Op.Create, "subscriptions", "Agent tracing"); + var refused = new Change(Op.Create, "subscriptions", "Agent export"); + var steps = new List(); + + var (failed, saved) = await MirrorService.ApplyBringingAsync([install, made, refused], + part => + { + steps.Add("apply " + string.Join(", ", part.Select(c => c.Key))); + return Task.FromResult(part.Contains(refused) ? new List<(Change Change, string Error)> { (refused, "Missing: Token") } : []); + }, + all => + { + steps.Add("placeholders"); + return Task.FromResult(new List<(Change Change, string Id)> { (made, "made"), (refused, "refused") }); + }); + + CollectionAssert.AreEqual(new[] { "apply infolink5.receivers.cassini", "placeholders", "apply Agent tracing, Agent export" }, steps); + Assert.AreSame(refused, failed.Single().Change); + CollectionAssert.AreEqual(new[] { "made" }, saved); + } + + [TestMethod] + public void Filled_in_closes_that_items_placeholders_and_no_other_whatever_its_name() + { + static string Id(string key, string field) => MirrorService.StandInId(new MirrorBlank("subscriptions", key, ["handler", "properties", field], field, "secret")); + var standIns = new HashSet { Id("A", "Password"), Id("A", "Token"), Id("A/B", "Password"), Id("A/handler", "Password") }; + + Assert.AreEqual(2, MirrorService.CloseStandIns(standIns, "subscriptions", "A")); + + CollectionAssert.AreEquivalent(new[] { Id("A/B", "Password"), Id("A/handler", "Password") }, standIns.ToList()); + } }