fix: prod mirror picker speed, required secrets and user names

SW.Bitween.Api/Resources/Adapters/Catalog.cs

@@ -41,7 +41,10 @@ public async Task<object> Handle(AdapterSearchRequest request)
         // reflection and should not be made to queue, and how many published adapters may be
         // running at a time is capped in ServerlessAdapterDescriber - process-wide, which is the
         // only place it can be, since this handler knows nothing of the other requests in flight.
-        var described = await Task.WhenAll(adapters.Select(async a => (a.Key, Values: await Describe(a.Key))));
+        // Not at all for a caller after versions alone: a published adapter is described by starting
+        // it, which on a cold cache takes half a minute for a kind.
+        var described = request.StartupValues == false ? []
+            : await Task.WhenAll(adapters.Select(async a => (a.Key, Values: await Describe(a.Key))));
         var byKey = described.ToDictionary(d => d.Key, d => d.Values);
 
         return adapters.Select(a => new
@@ -58,7 +61,7 @@ public async Task<object> Handle(AdapterSearchRequest request)
                 .Distinct(StringComparer.OrdinalIgnoreCase)
                 .OrderBy(v => System.Version.TryParse(v.Split('-')[0], out var parsed) ? parsed : new System.Version())
                 .ToList(),
-            StartupValues = byKey[a.Key],
+            StartupValues = byKey.GetValueOrDefault(a.Key),
 
             // Additive: a client that predates the catalog ignores all of these. Every one is null
             // or empty for an adapter published before manifests existed.

SW.Bitween.Api/Resources/Mirror/Mirror.cs

@@ -290,6 +290,30 @@ public async Task<object> Handle(MirrorTestManyRequest request)
     }
 }
 
+public class MirrorFilledInRequest
+{
+    /// <summary>The item's kind: subscriptions.</summary>
+    public string Section { get; set; }
+
+    /// <summary>The item, by its name.</summary>
+    public string Key { get; set; }
+}
+
+/// <summary>
+/// Says a person has put staging's own value in each secret a bring had to save as a placeholder on the
+/// item: Bitween hides a saved secret, so staging can't see it change. Its to-dos close and its tests run.
+/// </summary>
+[HandlerName("filledin")]
+public class FilledIn(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror)
+    : ICommandHandler<MirrorFilledInRequest, object>
+{
+    public async Task<object> Handle(MirrorFilledInRequest request)
+    {
+        await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate);
+        return new { Closed = await mirror.FilledInAsync(request?.Section, request?.Key) };
+    }
+}
+
 public class MirrorTestStopRequest;
 
 /// <summary>Drops the tests waiting their turn; the one under way finishes. How many were dropped.</summary>

SW.Bitween.Api/Services/Mirror/MirrorFaker.cs

@@ -39,15 +39,17 @@ public sealed class MirrorFaker(byte[] key, IEnumerable<string> keep = null)
 
     /// <summary>
     /// What a setting or a value of prod's is that staging never holds, so a copy leaves it empty for
-    /// staging's own: an address (a URL, a host, an email, a connection string) or a secret (one prod
-    /// masks, or one named like a secret). Null for anything else. A host built from a {{token}} isn't
-    /// one yet, nor is a secret that names one: the token is what is copied. Hosts every system names
-    /// alike, such as an XML namespace's www.w3.org, are nobody's.
+    /// staging's own: an address (a URL, a host, an email, a connection string), a secret (one prod
+    /// masks, or one named like a secret) or a login (one named like a user name). Null for anything
+    /// else. A host built from a {{token}} isn't one yet, nor is a secret or a login that names one: the
+    /// token is what is copied. Hosts every system names alike, such as an XML namespace's www.w3.org,
+    /// are nobody's.
     /// </summary>
     public static string BlankKind(string key, string value)
     {
         if (string.IsNullOrEmpty(value)) return null;
         if (value == Masked || IsSecretLike(key) && !TokenPattern.IsMatch(value)) return "secret";
+        if (IsLoginKey(key) && !TokenPattern.IsMatch(value)) return "login";
         if (IsConnectionStringKey(key) || LooksLikeConnectionString(value) || HostsIn(key, value).Any(h => !Untouched(h))) return "address";
         return null;
     }
@@ -97,6 +99,16 @@ public static bool IsSecretLike(string key)
         return k is "pass" or "key" or "pin" || SecretWords.Any(k.Contains);
     }
 
+    /// <summary>
+    /// A name a login goes by - a user name, a client id - which prod's never is on
+    /// staging: the team's test login is. SftpUsername and ftpUser are; UserAgent isn't.
+    /// </summary>
+    public static bool IsLoginKey(string key)
+    {
+        var k = (key ?? "").ToLowerInvariant().Replace("_", "").Replace("-", "").Replace(".", "");
+        return LoginEndings.Any(k.EndsWith);
+    }
+
     /// <summary>How every stand-in secret starts, which tells one from a secret set on staging.</summary>
     public const string SecretPrefix = "mirror-";
 
@@ -536,6 +548,8 @@ static byte[] Block(HMACSHA256 hmac, int block, byte[] data)
     static readonly string[] SecretWords =
         ["password", "passwd", "pwd", "secret", "token", "credential", "apikey", "accesskey", "privatekey", "passphrase", "certificate", "signingkey"];
 
+    static readonly string[] LoginEndings = ["user", "username", "userid", "login", "loginname", "loginid", "clientid"];
+
     static readonly string[] AddressWords = ["url", "uri", "host", "server", "endpoint", "address", "domain", "webhook", "smtp", "imap", "ftp"];
 
     static readonly Regex TokenPattern = new(@"\{\{[^}]*\}\}", RegexOptions.Compiled);

SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs

@@ -3,9 +3,12 @@
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Text.Json;
 using System.Text.Json.Nodes;
 using System.Text.RegularExpressions;
+using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
 using SW.Bitween.Cli;
 using SW.Bitween.Cli.Configuration;
 using SW.PrimitiveTypes;
@@ -175,9 +178,14 @@ public async Task<MirrorBring> BringAsync(string section, IReadOnlyCollection<st
             var bringing = await PlanBringingAsync(comparison, kind, keys);
             if (!apply) return new MirrorBring(false, bringing.Pieces, bringing.NeedsYou, []);
 
-            var failed = await comparison.Planner.ApplyAllAsync(bringing.Plan);
+            await using var scope = scopes.CreateAsyncScope();
+            var requirements = scope.ServiceProvider.GetRequiredService<AdapterRequirements>();
+            var (failed, saved) = await ApplyBringingAsync(bringing.Plan.Changes,
+                part => comparison.Planner.ApplyAllAsync(bringing.Plan, part),
+                all => StandInAsync(all, comparison.Copy.Blanks, requirements.MissingFor));
             foreach (var change in bringing.Plan.Changes) LastFailures.TryRemove((change.Section, change.Key), out _);
             foreach (var (change, error) in failed) LastFailures[(change.Section, change.Key)] = error;
+            if (saved.Count > 0) await UpdateStandInsAsync(standIns => { standIns.UnionWith(saved); return true; });
             var made = bringing.Pieces.Select(p => p.State == "new" ? p with { StagingId = IdOn(comparison.Staging, p.Section, p.Key, bringing.Index) } : p).ToList();
             return new MirrorBring(true, made, bringing.NeedsYou, failed.Select(f => new MirrorFailure(f.Change.Section, f.Change.Noun, f.Change.Key, f.Error)).ToList());
         }
@@ -239,14 +247,120 @@ static IEnumerable<MirrorNote> FillIns(Comparison comparison, IEnumerable<(strin
             var alerts = OnlyAlerts(blank.Section);
             yield return new MirrorNote($"fill:{blank.Section}:{blank.Key}:{string.Join("/", blank.Path)}", "fill", blank.Kind switch
                 {
+                    _ when blank.StoodIn => $"Fill in {what} with staging's own secret, then tick Filled in: it holds {MirrorFaker.PlaceholderText}, " +
+                        "which its adapter can't be saved without, and Bitween hides a saved secret, so staging can't see it change.",
                     "secret" => $"Fill in {what} with staging's own secret: it is empty.",
                     "login" => $"Fill in {what} with your test login, in place of {MirrorFaker.PlaceholderText}.",
                     _ => $"Fill in {what} with your test system's, in place of the placeholder, which leads nowhere.",
                 } + (alerts ? " Tests run without it: it only sends alerts." : ""),
-                blank.Section, blank.Key, alerts ? "alerts" : null, row?.StagingId, row?.ProdId, row?.Type, blank.Field, Kinds.Of(blank.Section).Noun);
+                blank.Section, blank.Key, alerts ? "alerts" : blank.StoodIn ? "filledIn" : null, row?.StagingId, row?.ProdId, row?.Type,
+                blank.Field, Kinds.Of(blank.Section).Noun);
         }
     }
 
+    // ------------------------------------------------------------------ secrets saved as placeholders
+
+    /// <summary>The secrets a bring saved as placeholders, each by <see cref="StandInId"/>, until a person says each was filled in.</summary>
+    const string StandInsKey = "mirror/standins.json";
+
+    /// <summary>One read-change-write of <see cref="StandInsKey"/> at a time: a bring and a Filled in can come together. One staging runs one node.</summary>
+    static readonly SemaphoreSlim StandInsLock = new(1, 1);
+
+    /// <summary>A secret's place, as its kind, its item's name and its path, which a name with a slash in it can't run into another's.</summary>
+    internal static string StandInId(MirrorBlank blank) => JsonSerializer.Serialize<string[]>([blank.Section, blank.Key, .. blank.Path]);
+
+    /// <summary>
+    /// Carries out a bring's changes: adapters first, so the placeholders (see <see cref="StandInAsync"/>) can ask
+    /// what an adapter the bring installs can't do without, then the rest. What failed and why, and the placeholders
+    /// saved with what was made.
+    /// </summary>
+    internal static async Task<(List<(Change Change, string Error)> Failed, List<string> Saved)> ApplyBringingAsync(IReadOnlyList<Change> changes,
+        Func<IReadOnlyList<Change>, Task<List<(Change Change, string Error)>>> apply,
+        Func<IReadOnlyList<Change>, Task<List<(Change Change, string Id)>>> standIn)
+    {
+        var adapters = changes.Where(c => c.Section == Kinds.Adapters).ToList();
+        var failed = adapters.Count > 0 ? await apply(adapters) : [];
+        var stoodIn = await standIn(changes);
+        failed.AddRange(await apply(changes.Where(c => c.Section != Kinds.Adapters).ToList()));
+        return (failed, stoodIn.Where(s => failed.All(f => !ReferenceEquals(f.Change, s.Change))).Select(s => s.Id).ToList());
+    }
+
+    /// <summary>
+    /// The secrets of subscriptions about to be made that their adapters can't be saved without, and that come
+    /// empty because the copy never brings one: each gets a placeholder, so the subscription is made, and is
+    /// said for staging to remember, since once saved Bitween hides it like a filled one.
+    /// <paramref name="missingFor"/> is Bitween's own check, what saving it would refuse for (see
+    /// <see cref="AdapterRequirements.MissingFor"/>); an adapter it can't describe - one that failed to
+    /// install - is left for saving to refuse.
+    /// </summary>
+    internal static async Task<List<(Change Change, string Id)>> StandInAsync(IEnumerable<Change> changes, IReadOnlyList<MirrorBlank> blanks,
+        Func<string, ICollection<KeyAndValue>, Task<IReadOnlyCollection<string>>> missingFor)
+    {
+        var stoodIn = new List<(Change, string)>();
+        foreach (var change in changes.Where(c => c.Op == Op.Create && c.Section == "subscriptions"))
+        foreach (var role in new[] { "receiver", "validator", "mapper", "handler" })
+        {
+            if (change.Want[role] is not JsonObject block || (string)block["adapter"] is not { Length: > 0 } adapter) continue;
+            var properties = block["properties"] as JsonObject ?? [];
+            IReadOnlyCollection<string> missing;
+            try
+            {
+                missing = await missingFor(SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(adapter, (string)block["version"]),
+                    properties.Select(p => new KeyAndValue { Key = p.Key, Value = p.Value?.ToString() }).ToList());
+            }
+            catch (Exception) { continue; }
+            foreach (var blank in blanks.Where(b => b.Section == change.Section && b.Key == change.Key && b.Kind == "secret"
+                         && b.Path.Count == 3 && b.Path[0] == role && missing.Contains(b.Path[2], StringComparer.OrdinalIgnoreCase)))
+            {
+                properties[blank.Path[2]] = MirrorFaker.PlaceholderText;
+                stoodIn.Add((change, StandInId(blank)));
+            }
+        }
+        return stoodIn;
+    }
+
+    async Task<HashSet<string>> ReadStandInsAsync()
+    {
+        try
+        {
+            await using var stream = await cloudFiles.OpenReadAsync(StandInsKey);
+            return await JsonSerializer.DeserializeAsync<HashSet<string>>(stream, ManifestJson) ?? [];
+        }
+        catch (Exception ex) when (StorageErrors.IsNotFound(ex))
+        {
+            return [];
+        }
+    }
+
+    /// <summary>Changes the placeholders kept, written back when <paramref name="change"/> says it changed them.</summary>
+    async Task UpdateStandInsAsync(Func<HashSet<string>, bool> change)
+    {
+        await StandInsLock.WaitAsync();
+        try
+        {
+            var standIns = await ReadStandInsAsync();
+            if (change(standIns))
+                await cloudFiles.WriteTextAsync(JsonSerializer.Serialize(standIns, ManifestJson),
+                    new WriteFileSettings { Key = StandInsKey, ContentType = "application/json" });
+        }
+        finally
+        {
+            StandInsLock.Release();
+        }
+    }
+
+    /// <summary>A person has filled in an item's secrets a bring saved as placeholders: how many it closes.</summary>
+    public async Task<int> FilledInAsync(string section, string key)
+    {
+        var closed = 0;
+        await UpdateStandInsAsync(standIns => (closed = CloseStandIns(standIns, section, key)) > 0);
+        return closed;
+    }
+
+    /// <summary>Takes out the item's placeholders, and only that item's: how many.</summary>
+    internal static int CloseStandIns(HashSet<string> standIns, string section, string key) =>
+        standIns.RemoveWhere(id => JsonSerializer.Deserialize<string[]>(id) is [var s, var k, ..] && s == section && k == key);
+
     /// <summary>Kinds that only send alerts: an exchange runs the same whether they are filled in or not.</summary>
     internal static bool OnlyAlerts(string section) => section is "notificationChannels" or "notifiers";