diff --git a/SW.Bitween.Api/Services/AggregationJob.cs b/SW.Bitween.Api/Services/AggregationJob.cs index bf57a4a7..eeb1dcaf 100644 --- a/SW.Bitween.Api/Services/AggregationJob.cs +++ b/SW.Bitween.Api/Services/AggregationJob.cs @@ -23,7 +23,8 @@ public class AggregationJob( BitweenDbContext dbContext, RunFlagUpdater runFlagUpdater, XchangeService xchangeService, - ILogger logger) : IScheduledJob + ILogger logger, + BitweenOptions options = null) : IScheduledJob { public async Task Execute(AggregationJobParams jobParams) { @@ -31,6 +32,13 @@ public async Task Execute(AggregationJobParams jobParams) .FirstOrDefaultAsync(s => s.Id == jobParams.SubscriptionId && !s.Inactive); if (aggSub == null) return; + if (Services.Mirror.MirrorTraffic.HoldsScheduledRun(options, aggSub, jobParams.CronExpression)) + { + // Held, but its next run still moves on, so its page doesn't show one in the past. + try { aggSub.SetSchedules(); await dbContext.SaveChangesAsync(); } + catch (BitweenException) { } + return; + } // Quartz runs each fire once across the cluster, but each schedule of a subscription and each // "aggregate now" is a job of its own, so two runs of one aggregation can still overlap, on one diff --git a/SW.Bitween.Api/Services/Mirror/MirrorFaker.cs b/SW.Bitween.Api/Services/Mirror/MirrorFaker.cs index f058d068..08c49481 100644 --- a/SW.Bitween.Api/Services/Mirror/MirrorFaker.cs +++ b/SW.Bitween.Api/Services/Mirror/MirrorFaker.cs @@ -40,7 +40,8 @@ public sealed class MirrorFaker(byte[] key, IEnumerable keep = null) /// /// What a setting or a value of prod's is that staging never holds, so a copy leaves it empty for /// staging's own: an address (a URL, a host, an email, a connection string), a secret (one prod - /// masks, or one named like a secret) or a login (one named like a user name). Null for anything + /// masks, one named like a secret, or one that reads as a credential whatever its name) or a login + /// (one named like a user name). Null for anything /// else. A host built from a {{token}} isn't one yet, nor is a secret or a login that names one: the /// token is what is copied. Hosts every system names alike, such as an XML namespace's www.w3.org, /// are nobody's. @@ -48,7 +49,7 @@ public sealed class MirrorFaker(byte[] key, IEnumerable keep = null) public static string BlankKind(string key, string value) { if (string.IsNullOrEmpty(value)) return null; - if (value == Masked || IsSecretLike(key) && !TokenPattern.IsMatch(value)) return "secret"; + if (value == Masked || IsSecretLike(key) && !TokenPattern.IsMatch(value) || LooksLikeCredential(value)) return "secret"; if (IsLoginKey(key) && !TokenPattern.IsMatch(value)) return "login"; if (IsConnectionStringKey(key) || LooksLikeConnectionString(value) || HostsIn(key, value).Any(h => !Untouched(h))) return "address"; return null; @@ -99,6 +100,28 @@ public static bool IsSecretLike(string key) return k is "pass" or "key" or "pin" || SecretWords.Any(k.Contains); } + /// + /// A value that reads as a credential whatever its name: a JWT, a bearer or basic header, or a long run + /// of letters and digits with no word in it, such as a license or access key. Prod shows a secret as + /// it is when its adapter doesn't say it is one, and a name like TraxisJwt or LicenseKey says nothing. + /// A GUID or a path isn't one. + /// + public static bool LooksLikeCredential(string value) + { + var v = (value ?? "").Trim(); + if (JwtPattern.IsMatch(v) || AuthHeaderPattern.IsMatch(v)) return true; + if (v.Length < 20 || v.StartsWith('/') || System.Guid.TryParse(v, out _) || !KeyPattern.IsMatch(v)) return false; + // A long run of hex, of digits and letters both, is a key or a hash, which reads as low as words do. One of + // 32 is a GUID's shape, and isn't. + if (v.Length is >= 40 and <= 512 && HexPattern.IsMatch(v) && v.Any(char.IsDigit) && v.Any(char.IsLetter)) return true; + var kinds = (v.Any(char.IsUpper) ? 1 : 0) + (v.Any(char.IsLower) ? 1 : 0) + (v.Any(char.IsDigit) ? 1 : 0); + return kinds >= 2 && BitsPerChar(v) >= 4.0; + } + + /// How unpredictable a value's characters are: about 4.3 for a random key of 20, under 4 for words and paths. + static double BitsPerChar(string v) => + -v.GroupBy(c => c).Sum(g => (double)g.Count() / v.Length * Math.Log2((double)g.Count() / v.Length)); + /// /// A name a login goes by - a user name, a client id - which prod's never is on /// staging: the team's test login is. SftpUsername and ftpUser are; UserAgent isn't. @@ -546,13 +569,18 @@ static byte[] Block(HMACSHA256 hmac, int block, byte[] data) static readonly HashSet KeptWords = new(StringComparer.OrdinalIgnoreCase) { "true", "false", "null", "none" }; static readonly string[] SecretWords = - ["password", "passwd", "pwd", "secret", "token", "credential", "apikey", "accesskey", "privatekey", "passphrase", "certificate", "signingkey"]; + ["password", "passwd", "pwd", "secret", "token", "credential", "apikey", "accesskey", "privatekey", "passphrase", "certificate", "signingkey", + "jwt", "bearer", "licensekey"]; static readonly string[] LoginEndings = ["user", "username", "userid", "login", "loginname", "loginid", "clientid"]; static readonly string[] AddressWords = ["url", "uri", "host", "server", "endpoint", "address", "domain", "webhook", "smtp", "imap", "ftp"]; static readonly Regex TokenPattern = new(@"\{\{[^}]*\}\}", RegexOptions.Compiled); + static readonly Regex JwtPattern = new(@"^eyJ[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]*$", RegexOptions.Compiled); + static readonly Regex AuthHeaderPattern = new(@"^(Bearer|Basic)\s+[A-Za-z0-9+/=_\-.]{4,}$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + static readonly Regex HexPattern = new(@"^[0-9A-Fa-f]+$", RegexOptions.Compiled); + static readonly Regex KeyPattern = new(@"^[A-Za-z0-9+/=_\-]+$", RegexOptions.Compiled); // Scheme and local part are bounded, as real ones are: unbounded, a long value with neither :// nor @, // such as a file's encoded attachment, took a search from every position to its end. static readonly Regex UrlPattern = new(@"(?[A-Za-z][A-Za-z0-9+.\-]{0,31})://(?[^\s/?#""'<>\\]+)(?[^\s""'<>\\]*)", RegexOptions.Compiled); diff --git a/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs b/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs index ccb25f60..33d1db93 100644 --- a/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs +++ b/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs @@ -227,6 +227,7 @@ async Task PlanBringingAsync(Comparison comparison, Kind kind, IReadOn var needsYou = AdapterNotes(adapters); needsYou.AddRange(plan.Problems.Distinct().Select(p => new MirrorNote($"problem:{p}", "needsYou", $"On staging: {p}"))); needsYou.AddRange(FillIns(comparison, needs.Keys)); + needsYou.AddRange(AsIsNotes(comparison.Copy, pieces.Where(p => p.State == "new").Select(p => (p.Section, p.Key)))); if (FakedNote(index, needs.Keys) is { } faked) needsYou.Add(faked); return new Bringing(pieces, needsYou, plan, needs, index, adapters); } @@ -361,6 +362,18 @@ public async Task FilledInAsync(string section, string key) internal static int CloseStandIns(HashSet standIns, string section, string key) => standIns.RemoveWhere(id => JsonSerializer.Deserialize(id) is [var s, var k, ..] && s == section && k == key); + /// + /// The adapters' settings of the items a bring makes that come as prod has them, each with its value, so a + /// person sees them before they come: what decides a secret is its name and its value, and a name can say + /// nothing. Each whole: the page shortens a long one until it is asked for. + /// + internal static IEnumerable AsIsNotes(MirrorSetupCopy copy, IEnumerable<(string Section, string Key)> items) + { + var making = items.ToHashSet(); + foreach (var s in (copy.AsIs ?? []).Where(s => making.Contains((s.Section, s.Key)))) + yield return new MirrorNote($"asIs:{s.Section}:{s.Key}:{s.Field}", "asIs", s.Value, s.Section, s.Key, Field: s.Field, Noun: Kinds.Of(s.Section).Noun); + } + /// Kinds that only send alerts: an exchange runs the same whether they are filled in or not. internal static bool OnlyAlerts(string section) => section is "notificationChannels" or "notifiers"; diff --git a/SW.Bitween.Api/Services/Mirror/MirrorService.cs b/SW.Bitween.Api/Services/Mirror/MirrorService.cs index 483cf1f8..9f97dc17 100644 --- a/SW.Bitween.Api/Services/Mirror/MirrorService.cs +++ b/SW.Bitween.Api/Services/Mirror/MirrorService.cs @@ -277,10 +277,40 @@ JsonValue v when v.TryGetValue(out var s) && Version.TryParse(s, out var }; /// - /// Every kind prod has, each read on its own: a kind prod's release doesn't have, or the login - /// can't read, is left out and says why, rather than stopping the rest. + /// What prod's API last answered for each kind, by its address, with the kinds it didn't and why. Its + /// setup is a request an item - nearly 200 on Traxis's - and the picker, its preview, the bring and each + /// test's take all want it within minutes, against the 600 a minute prod allows staging's login. /// - static async Task> ReadAsync(Context ctx, List renamed) + static readonly System.Collections.Concurrent.ConcurrentDictionary> Read)> ProdSetups = + new(StringComparer.Ordinal); + + sealed record ProdSetup(Dictionary> Raw, Dictionary Skipped); + + /// + /// Every kind prod has, each read on its own, read again once five minutes old: a kind prod's release + /// doesn't have, or the login can't read, is left out and says why, rather than stopping the rest. One + /// read at a time per prod - whoever asks while it runs waits on it - and a copy each, since a comparison + /// renames in it. A read that fails isn't kept. + /// + static async Task<(Dictionary> Raw, Dictionary Skipped)> AnsweredAsync(Context ctx) + { + var url = ctx.Api.Url; + var entry = ProdSetups.AddOrUpdate(url, + _ => (DateTimeOffset.UtcNow, new Lazy>(() => ReadSetupAsync(ctx))), + (_, old) => DateTimeOffset.UtcNow - old.At < TimeSpan.FromMinutes(5) ? old + : (DateTimeOffset.UtcNow, new Lazy>(() => ReadSetupAsync(ctx)))); + ProdSetup read; + try { read = await entry.Read.Value; } + catch + { + ProdSetups.TryRemove(new KeyValuePair>)>(url, entry)); + throw; + } + return (read.Raw.ToDictionary(k => k.Key, k => k.Value.Select(x => (x.Id, (JsonObject)x.Api.DeepClone())).ToList(), StringComparer.Ordinal), + new Dictionary(read.Skipped, StringComparer.Ordinal)); + } + + static async Task ReadSetupAsync(Context ctx) { var skipped = new Dictionary(StringComparer.Ordinal); var raw = new Dictionary>(StringComparer.Ordinal); @@ -291,14 +321,25 @@ static async Task> ReadAsync(Context ctx, List + /// Every kind prod has, as the copy reads it (see ): several of one name are + /// told apart, and a kind in a form this release doesn't read is left out and says why. + /// + static async Task> ReadAsync(Context ctx, List renamed) + { + var (raw, skipped) = await AnsweredAsync(ctx); + foreach (var kind in Kinds.All.Where(k => !skipped.ContainsKey(k.Section))) + { renamed.AddRange(Distinguish(ctx, kind, raw[kind.Section])); foreach (var (id, api) in raw[kind.Section]) ctx.Know(kind.Section, kind.KeyOfApi(ctx, api), id); } @@ -373,13 +414,19 @@ static async Task> AdaptersAsync(BitweenApi api, else { listed = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var prefix in AdapterPrefixes) + async Task ListOf(string prefix) { - JsonNode answer; // Without their startup properties, which nothing here reads: describing a published // adapter starts it. A release from before the option ignores it and describes them. - try { answer = await api.GetAsync($"api/adapters/catalog?prefix={prefix}&startupValues=false"); } - catch (BitweenApiException ex) when (ex.Status is HttpStatusCode.NotFound or HttpStatusCode.Forbidden) { continue; } + try { return await api.GetAsync($"api/adapters/catalog?prefix={prefix}&startupValues=false"); } + catch (BitweenApiException ex) when (ex.Status is HttpStatusCode.NotFound or HttpStatusCode.Forbidden) { return null; } + } + // Staging's kinds side by side, each a read of its storage; prod's one after another, to go easy on it. + var answers = new List(); + if (isProd) foreach (var prefix in AdapterPrefixes) answers.Add(await ListOf(prefix)); + else answers.AddRange(await Task.WhenAll(AdapterPrefixes.Select(ListOf))); + foreach (var answer in answers) + { foreach (var row in (answer as JsonArray ?? []).OfType()) { var id = (string)row["key"]; diff --git a/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs b/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs index def788e3..27fee7b0 100644 --- a/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs +++ b/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs @@ -21,11 +21,19 @@ namespace SW.Bitween.Services.Mirror; /// public record MirrorBlank(string Section, string Key, IReadOnlyList Path, string Field, string Kind, bool StoodIn = false); +/// +/// An adapter's setting the copy brings as prod has it, since it says how a flow behaves - neither an +/// address, a secret nor a login by its name or its value - for a person to look over before it comes. +/// +/// Where it is, in words: handler · Protocol. +public record MirrorAsIs(string Section, string Key, string Field, string Value); + /// A partner property or global value that is faked, unless it is kept because it is a setting. public record MirrorFakedKey(string Id, string Label, int Count, bool Kept); /// Prod's setup as staging gets it: every value handled, and what that turned up. /// Every address and secret of prod's the copy left empty. +/// Adapters' settings it brings as prod has them, that could say anything of prod's. /// /// The data sources and bus gateways prod has on and the copy switched off, as section/key: a test's /// traffic still goes through them, as prod's does. @@ -34,7 +42,8 @@ public record MirrorSetupCopy( Dictionary> Sections, IReadOnlyList Blanks, IReadOnlyList FakedKeys, - IReadOnlySet SwitchedOff = null); + IReadOnlySet SwitchedOff = null, + IReadOnlyList AsIs = null); /// /// Prod's setup, read as a configuration file would say it, with what staging must not hold taken out: @@ -54,17 +63,19 @@ public static MirrorSetupCopy Copy(IReadOnlyDictionary> p { var sections = new Dictionary>(StringComparer.Ordinal); var blanks = new List(); + var asIs = new List(); foreach (var (section, items) in prod) sections[section] = items.Where(e => e.Key != null).OrderBy(e => e.Key, StringComparer.Ordinal) - .Select(e => Handle(section, e.Key, (JsonObject)e.File.DeepClone(), faker, keep, blanks)).ToList(); + .Select(e => Handle(section, e.Key, (JsonObject)e.File.DeepClone(), faker, keep, blanks, asIs)).ToList(); var switchedOff = prod.Where(p => p.Key is "dataSources" or "busGateways") .SelectMany(p => p.Value.Where(e => e.Key != null && !Values.Same(e.File["inactive"], true)).Select(e => $"{p.Key}/{e.Key}")) .ToHashSet(StringComparer.Ordinal); - return new MirrorSetupCopy(sections, blanks, FakedKeys(prod, keep), switchedOff); + return new MirrorSetupCopy(sections, blanks, FakedKeys(prod, keep), switchedOff, asIs); } - static JsonObject Handle(string section, string key, JsonObject item, MirrorFaker faker, ISet keep, List blanks) + static JsonObject Handle(string section, string key, JsonObject item, MirrorFaker faker, ISet keep, List blanks, + List asIs) { var where = $"{section}/{key}"; void Blank(JsonObject owner, string name, string kind, params string[] path) @@ -72,6 +83,8 @@ void Blank(JsonObject owner, string name, string kind, params string[] path) owner[name] = MirrorFaker.Placeholder(kind, name, (string)owner[name] ?? ""); blanks.Add(new MirrorBlank(section, key, path, string.Join(" · ", path.Where(p => p != "properties")), kind)); } + void AsIs(string value, params string[] path) => + asIs.Add(new MirrorAsIs(section, key, string.Join(" · ", path.Where(p => p != "properties")), value)); switch (section) { @@ -100,23 +113,24 @@ void Blank(JsonObject owner, string name, string kind, params string[] path) case "notificationChannels": case "notifiers": case "dataSources": - Settings(item, Blank, "properties"); + Settings(item, Blank, AsIs, "properties"); if (section == "dataSources") item["inactive"] = true; break; case "busGateways": - Settings(item, Blank, "endpointProperties"); + Settings(item, Blank, AsIs, "endpointProperties"); item["inactive"] = true; break; case "apiGateways": // Its issuer is where partner logins are checked: an address like any other. - Settings(item, Blank, "authentication"); + Settings(item, Blank, AsIs, "authentication"); break; case "subscriptions": foreach (var role in new[] { "receiver", "validator", "mapper", "handler" }) - Settings(item[role] as JsonObject, (owner, name, kind, path) => Blank(owner, name, kind, [role, .. path]), "properties"); + Settings(item[role] as JsonObject, (owner, name, kind, path) => Blank(owner, name, kind, [role, .. path]), + (value, path) => AsIs(value, [role, .. path]), "properties"); item["paused"] = true; break; } @@ -125,15 +139,32 @@ void Blank(JsonObject owner, string name, string kind, params string[] path) delegate void Blanker(JsonObject owner, string name, string kind, params string[] path); - /// How an adapter behaves: copied, except addresses and secrets, which are left empty. - static void Settings(JsonObject item, Blanker blank, string field) + delegate void Copied(string value, params string[] path); + + /// + /// How an adapter behaves: copied, except addresses, secrets and logins, which are left for staging's own. + /// What is copied and could say anything of prod's is told (see ). + /// + static void Settings(JsonObject item, Blanker blank, Copied asIs, string field) { if (item?[field] is not JsonObject properties) return; foreach (var name in properties.Select(p => p.Key).ToList()) - if (properties[name] is JsonValue v && v.TryGetValue(out var value) && MirrorFaker.BlankKind(name, value) is { } kind) - blank(properties, name, kind, field, name); + if (properties[name] is JsonValue v && v.TryGetValue(out var value)) + { + if (MirrorFaker.BlankKind(name, value) is { } kind) blank(properties, name, kind, field, name); + else if (Telling(value)) asIs(value, field, name); + } + } + + /// Whether a value copied as it is could say anything of prod's: not a yes or no, a short number or {{tokens}} alone. + static bool Telling(string value) + { + var v = value?.Trim() ?? ""; + return v.Length > 0 && !bool.TryParse(v, out _) && !(v.Length <= 4 && v.All(char.IsDigit)) && !TokensOnly.IsMatch(v); } + static readonly Regex TokensOnly = new(@"^(\{\{[^}]*\}\}\s*)+$", RegexOptions.Compiled); + /// Someone's data: faked, unless kept; an address or a secret in it is left empty, as anywhere. static void Data(JsonObject item, string field, Func kept, MirrorFaker faker, Blanker blank) { diff --git a/SW.Bitween.Api/Services/Mirror/MirrorTraffic.cs b/SW.Bitween.Api/Services/Mirror/MirrorTraffic.cs index fdc21d00..97f440c0 100644 --- a/SW.Bitween.Api/Services/Mirror/MirrorTraffic.cs +++ b/SW.Bitween.Api/Services/Mirror/MirrorTraffic.cs @@ -36,6 +36,14 @@ public static class MirrorTraffic public static bool IsRehearsal(BitweenOptions options, string correlationId) => options.Mirror.Target && correlationId?.StartsWith(Prefix, StringComparison.Ordinal) == true; + /// + /// A paused scheduled job's or aggregation's run on its schedule, on a mirror, which holds it back. + /// Bitween otherwise runs a paused one's schedule, so a brought receiver would poll the team's test + /// system on its own. Receive now, Roll up now and a test have no schedule, and run. + /// + public static bool HoldsScheduledRun(BitweenOptions options, Subscription subscription, string cronExpression) => + options?.Mirror.Target == true && subscription.PausedOn != null && cronExpression != null; + /// A key for one rehearsal's calls through a door; close it when the rehearsal ends. public static string OpenKey() { diff --git a/SW.Bitween.Api/Services/ReceivingJob.cs b/SW.Bitween.Api/Services/ReceivingJob.cs index 602780db..862f3fe0 100644 --- a/SW.Bitween.Api/Services/ReceivingJob.cs +++ b/SW.Bitween.Api/Services/ReceivingJob.cs @@ -21,7 +21,8 @@ public class ReceivingJob( IAdapterInvoker adapterInvoker, XchangeService xchangeService, ILogger logger, - Services.ReceiverBookmarks bookmarks = null) : IScheduledJob + Services.ReceiverBookmarks bookmarks = null, + BitweenOptions options = null) : IScheduledJob { public async Task Execute(ReceivingJobParams jobParams) { @@ -29,6 +30,13 @@ public async Task Execute(ReceivingJobParams jobParams) .FirstOrDefaultAsync(s => s.Id == jobParams.SubscriptionId && !s.Inactive); if (rec == null) return; + if (Services.Mirror.MirrorTraffic.HoldsScheduledRun(options, rec, jobParams.CronExpression)) + { + // Held, but its next run still moves on, so its page doesn't show one in the past. + try { rec.SetSchedules(); await dbContext.SaveChangesAsync(); } + catch (BitweenException) { } + return; + } // Atomic DB-level guard: returns false if another execution is already running. var isIdle = await runFlagUpdater.MarkAsRunning(rec.Id); diff --git a/SW.Bitween.IntegrationTests/Tests/AggregationTests.cs b/SW.Bitween.IntegrationTests/Tests/AggregationTests.cs index 6d490ab1..0510cb09 100644 --- a/SW.Bitween.IntegrationTests/Tests/AggregationTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/AggregationTests.cs @@ -53,6 +53,43 @@ public async Task A_mirror_test_rolls_up_only_what_tests_ran() Assert.Null(await db.Set().AsNoTracking().SingleOrDefaultAsync(a => a.Id == ids[1])); } + [Fact] + public async Task On_a_mirror_a_paused_aggregation_waits_out_its_schedule_but_rolls_up_by_hand() + { + await using var scope = fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var mirror = new BitweenOptions(); + mirror.Mirror.Target = true; + var job = ActivatorUtilities.CreateInstance(scope.ServiceProvider, mirror); + + var doc = new Document(null, "Agg Paused Doc", DocumentFormat.Json); + db.Set().Add(doc); + await db.SaveChangesAsync(); + var source = new Subscription("Agg Paused Source", doc.Id) { Inactive = false }; + db.Set().Add(source); + await db.SaveChangesAsync(); + var xchange = new Xchange(source, new XchangeFile("{}")); + db.Set().Add(xchange); + await db.SaveChangesAsync(); + db.Set().Add(new XchangeResult(xchange.Id, null, null)); + var aggregation = new Subscription("Agg Paused", source.Id, Partner.SystemId) { Inactive = false, AggregationTarget = XchangeFileType.Input }; + aggregation.SetSchedules(new[] { new Schedule(Recurrence.Hourly, TimeSpan.FromMinutes(30)) }); + aggregation.Pause(); + db.Set().Add(aggregation); + await db.SaveChangesAsync(); + fixture.App.Services.GetRequiredService().Revoke(); + + // Its schedule fires: held back, as a brought one arrives paused, but its next run moves on. + var next = await db.Subscriptions().AsNoTracking().Where(s => s.Id == aggregation.Id).Select(s => s.AggregateOn).SingleAsync(); + await job.Execute(new AggregationJobParams(aggregation.Id, "0 0 * * * ?")); + Assert.Null(await db.Set().AsNoTracking().SingleOrDefaultAsync(a => a.Id == xchange.Id)); + Assert.True(await db.Subscriptions().AsNoTracking().Where(s => s.Id == aggregation.Id).Select(s => s.AggregateOn).SingleAsync() >= next); + + // Roll up now, which has no schedule, runs. + await job.Execute(new AggregationJobParams(aggregation.Id, null)); + Assert.NotNull(await db.Set().AsNoTracking().SingleOrDefaultAsync(a => a.Id == xchange.Id)); + } + [Fact] public async Task Aggregation_job_creates_one_xchange_from_successful_source_xchanges() { diff --git a/SW.Bitween.UnitTests/MirrorBringTests.cs b/SW.Bitween.UnitTests/MirrorBringTests.cs index 76b63dc9..f4d9baca 100644 --- a/SW.Bitween.UnitTests/MirrorBringTests.cs +++ b/SW.Bitween.UnitTests/MirrorBringTests.cs @@ -224,4 +224,34 @@ public void Filled_in_closes_that_items_placeholders_and_no_other_whatever_its_n CollectionAssert.AreEquivalent(new[] { Id("A/B", "Password"), Id("A/handler", "Password") }, standIns.ToList()); } + + [TestMethod] + public void The_preview_shows_each_setting_that_comes_as_it_is_with_its_value() + { + var copy = MirrorSetup.Copy(new Dictionary> + { + ["subscriptions"] = + [ + new Existing("1", "Agent tracing", (JsonObject)JsonNode.Parse($$$""" + { + "name": "Agent tracing", "type": "Receiving", + "receiver": { "adapter": "infolink5.receivers.ftp", "properties": { + "Host": "sftp.acmecargo.net", "Protocol": "sftp", "Port": "22", "CheckFileExistence": "false", + "Token": "{{globals.carrier.token}}", "TargetPath": "/acme/prod/out/event", "Rules": "{{{new string('x', 60)}}}" } } + } + """)!, new JsonObject()), + new Existing("2", "Already there", (JsonObject)JsonNode.Parse(""" + { "name": "Already there", "handler": { "adapter": "x", "properties": { "Agent": "SODEXI" } } } + """)!, new JsonObject()), + ], + }, Faker, new HashSet()); + + var notes = MirrorService.AsIsNotes(copy, [("subscriptions", "Agent tracing")]).ToList(); + + // Yes or no, a short number, a token alone and what is left for staging's own aren't listed; nor is what isn't made. + CollectionAssert.AreEqual(new[] { "receiver · Protocol", "receiver · TargetPath", "receiver · Rules" }, notes.Select(n => n.Field).ToList()); + Assert.AreEqual("sftp", notes[0].Text); + Assert.AreEqual(new string('x', 60), notes[2].Text); // whole: the page shortens it until asked + Assert.IsTrue(notes.All(n => n.Group == "asIs" && n.Noun == "subscription")); + } } diff --git a/SW.Bitween.UnitTests/MirrorFakerTests.cs b/SW.Bitween.UnitTests/MirrorFakerTests.cs index 3d360186..66fa3267 100644 --- a/SW.Bitween.UnitTests/MirrorFakerTests.cs +++ b/SW.Bitween.UnitTests/MirrorFakerTests.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Security.Cryptography; using System.Text; using System.Text.Json.Nodes; using System.Text.RegularExpressions; @@ -82,6 +83,33 @@ public void Addresses_are_left_for_stagings_own() Assert.AreEqual("address", MirrorFaker.BlankKind("note", "mail ops@acmecargo.net")); } + /// + /// Prod shows a secret as it is when its adapter doesn't say it is one, under names like TraxisJwt or + /// LicenseKey. Made up here, as it runs, so nothing in the code reads as a real one. + /// + [TestMethod] + public void A_value_that_reads_as_a_credential_is_a_secret_whatever_its_name() + { + static string Part(string text) => Convert.ToBase64String(Encoding.UTF8.GetBytes(text)).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + var jwt = $"{Part("""{"alg":"HS256","typ":"JWT"}""")}.{Part("""{"nameid":"made-up","exp":1}""")}.{Part("a signature made up for this test")}"; + var key = "==" + Convert.ToBase64String(Encoding.UTF8.GetBytes("A license key Made Up for these 2 tests only")); + + Assert.AreEqual("secret", MirrorFaker.BlankKind("TraxisJwt", jwt)); + Assert.AreEqual("secret", MirrorFaker.BlankKind("Header", "Bearer " + Part("made up bearer value 42"))); + Assert.AreEqual("secret", MirrorFaker.BlankKind("Adapter", key)); + Assert.AreEqual("secret", MirrorFaker.BlankKind("LicenseKey", "short")); + Assert.AreEqual("secret", MirrorFaker.BlankKind("Header", "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes("made:up")))); + Assert.AreEqual("secret", MirrorFaker.BlankKind("Signature", Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes("made up"))).ToLowerInvariant())); + // 32 hex digits is a GUID's shape: an id, not a key. + Assert.IsNull(MirrorFaker.BlankKind("Reference", Guid.NewGuid().ToString("N"))); + // Ids, paths, names and words aren't. + Assert.IsNull(MirrorFaker.BlankKind("AccountId", Guid.NewGuid().ToString())); + Assert.IsNull(MirrorFaker.BlankKind("TargetPath", "/traxis/Orders2024/Out/Archive")); + Assert.IsNull(MirrorFaker.BlankKind("Handler", "infolink6.handlers.traxis.importtrackingupdates")); + Assert.IsNull(MirrorFaker.BlankKind("Agent", "SODEXI")); + Assert.IsNull(MirrorFaker.BlankKind("Subject", "Daily shipments report for 2026")); + } + [TestMethod] public void Logins_are_left_for_the_teams_test_login() { diff --git a/SW.Bitween.Web/ClientApp/src/api/types.ts b/SW.Bitween.Web/ClientApp/src/api/types.ts index e69370c7..ada032db 100644 --- a/SW.Bitween.Web/ClientApp/src/api/types.ts +++ b/SW.Bitween.Web/ClientApp/src/api/types.ts @@ -2292,7 +2292,7 @@ export interface MirrorItem { */ export interface MirrorNote { id: string; - group: "needsYou" | "fill" | "faked" | "copy" | "notCovered" | "onPurpose"; + group: "needsYou" | "fill" | "faked" | "asIs" | "copy" | "notCovered" | "onPurpose"; text: string; /** The item it is about, to open it. */ section: string | null; diff --git a/SW.Bitween.Web/ClientApp/src/pages/mirror/BringFromProd.tsx b/SW.Bitween.Web/ClientApp/src/pages/mirror/BringFromProd.tsx index aa5802d4..49330bdb 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/mirror/BringFromProd.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/mirror/BringFromProd.tsx @@ -7,6 +7,7 @@ import { type MirrorBring, type MirrorBringSection, type MirrorItemState, + type MirrorNote, type MirrorPiece, type MirrorPieceState, type MirrorProdCounts, @@ -294,6 +295,7 @@ function Preview({ const needsYou = result.needsYou.filter((n) => n.group === "needsYou"); const fill = result.needsYou.filter((n) => n.group === "fill"); const faked = result.needsYou.find((n) => n.group === "faked"); + const asIs = result.needsYou.filter((n) => n.group === "asIs"); return (
{doing === 0 && !result.pieces.some((p) => p.state === "waits") && ( @@ -330,6 +332,21 @@ function Preview({ )} {faked &&

{faked.text}

} + {asIs.length > 0 && ( +
+

+ Copied as-is from prod{" "} + + ({plural(asIs.length, "setting", "settings")} in {plural(new Set(asIs.map((n) => `${n.section}/${n.key}`)).size, "item", "items")}) + +

+

Check none of these is a secret or someone's data.

+
+
+ )}

Subscriptions arrive paused, and bus gateways and data sources switched off. Nothing on staging is overwritten or removed. @@ -345,6 +362,60 @@ function Preview({ ); } +/** How many items' settings show before "Show all". */ +const FEW = 5; + +/** The settings each item brings as prod has them, one row an item, every setting with its value. */ +function AsIs({ notes }: { notes: MirrorNote[] }) { + const [all, setAll] = useState(false); + const items = useMemo(() => { + const byItem = new Map(); + for (const n of notes) { + const id = `${n.section}/${n.key}`; + byItem.set(id, { note: byItem.get(id)?.note ?? n, settings: [...(byItem.get(id)?.settings ?? []), n] }); + } + return [...byItem.values()]; + }, [notes]); + return ( +

+
    + {(all ? items : items.slice(0, FEW)).map(({ note, settings }) => ( +
  • + {note.key} {note.noun} +
    + {settings.map((s) => ( +
    +
    {s.field}
    + +
    + ))} +
    +
    • + ))} +
+ {items.length > FEW && ( + + )} +
+ ); +} + +/** A setting's value, a long one shortened until it is clicked, to read all of it before bringing. */ +function AsIsValue({ text }: { text: string }) { + const [open, setOpen] = useState(false); + if (text.length <= 48) return
{text}
; + return ( +
+ +
+ ); +} + /** "retry policy" as "retry policies", "partner" as "partners", as the server says them. */ const nouns = (noun: string) => /[^aeiou]y$/.test(noun) ? `${noun.slice(0, -1)}ies` : /(s|x|ch|sh)$/.test(noun) ? `${noun}es` : `${noun}s`; diff --git a/docs/exchange-pipeline.md b/docs/exchange-pipeline.md index 604768c2..3dea1a14 100644 --- a/docs/exchange-pipeline.md +++ b/docs/exchange-pipeline.md @@ -150,7 +150,7 @@ Saving the result publishes its id on the work group's `-Result` lane. There, ea While a subscription is paused, messages that reach it by content are stored as on-hold exchanges. That covers documents matched to internal subscriptions and bus gateway routes. Resuming the subscription publishes an event that turns every on-hold exchange into a real exchange and deletes the holds. -API gateway calls, legacy API calls, scheduled job runs, aggregations and exchanges created by hand ignore the pause and still create exchanges. +API gateway calls, legacy API calls, scheduled job runs, aggregations and exchanges created by hand ignore the pause and still create exchanges. On a staging marked as a [prod mirror](prod-mirror.md), a paused scheduled job or aggregation doesn't run on its schedule; **Receive now** and **Roll up now** still run it. ## Aggregation diff --git a/docs/prod-mirror.md b/docs/prod-mirror.md index 83bfe4be..0000564b 100644 --- a/docs/prod-mirror.md +++ b/docs/prod-mirror.md @@ -14,6 +14,8 @@ Prod is only read, through its own API, with a view-only login. Nothing is insta Enter prod's address and the login in **Settings, Prod mirror**. The token and password are stored encrypted, which needs `Bitween:SettingsEncryptionKey`. The address is only an address, with no login, `?` or `#` in it; use `https`, and plain `http` only for an address inside the same cluster. Changing the address empties the saved token and password, so a login is never sent to an address it wasn't entered for: enter it again. +Reading prod's setup takes a request an item, and prod allows each login 600 requests a minute, so the staging keeps what it read for five minutes: the picker, bringing and the tests share one read, and a change made on prod shows on the staging within five minutes. + ## What is copied, and how Everything a configuration file can hold is copied by name: subscription categories, work groups, information types, partners, global values, connections, notification channels, retry policies, data sources and their SQL statements, subscriptions, API gateways, bus gateways and legacy notifiers. A kind prod's release doesn't have is left out and says so. The copy goes through the same engine as [`bitween apply`](cli.md). @@ -24,7 +26,7 @@ Values are handled the moment they arrive, before anything is saved, so the stag |---|---|---| | How a flow behaves | formats, schedules, filters and match expressions, mapping rules, methods, delimiters, file name patterns, `{{globals.…}}` tokens | Copied as it is | | Addresses | hosts, URLs, email addresses, connection strings, an API gateway's issuer, a connection's login URLs | Never copied: a placeholder that leads nowhere, `https://set-me.invalid` or `set-me@set-me.invalid`, until you fill in your test system's | -| Secrets | anything prod shows as `__private__`, values named like secrets (`apiKey`, `password`, `token`) | Never copied: left empty until you fill in staging's own. One a subscription's adapter can't be saved without is saved as `set-me` | +| Secrets | anything prod shows as `__private__`, values named like secrets (`apiKey`, `password`, `token`, `jwt`, `licenseKey`), and values that read as a credential whatever their name: a JWT, a `Bearer` or `Basic` header, a long random key | Never copied: left empty until you fill in staging's own. One a subscription's adapter can't be saved without is saved as `set-me` | | Logins | a connection's client id and username; any setting, partner property or global value named like a user name or client id (`Username`, `SftpUser`, `ClientId`) | Never copied: `set-me` until you fill in your test login | | Partner properties and global values | account numbers, codes, names | A fake of the same shape: digits stay digits, letters keep their case, dates stay dates in their format, GUIDs stay GUIDs, JSON keeps its keys | | Partner API keys | | Keys of staging's own, the same on every copy | @@ -33,7 +35,7 @@ Each address, secret and login left this way is something to fill in. Bringing l The same value always becomes the same fake, so a code named in two places still matches, and copying again changes nothing. A partner property or global value that is a setting rather than data, such as a time zone or a date format, can be kept as it is: list it in **Settings, Prod mirror, Values copied as they are**, one per line, as `partner:timeZone` or `global:formats/date`. Addresses and secrets are never kept. -Copied subscriptions arrive paused, and data sources and bus gateways switched off, so nothing on the staging runs on its own. +Copied subscriptions arrive paused, and data sources and bus gateways switched off, so nothing on the staging runs on its own. On a mirror, a paused scheduled job or aggregation doesn't run on its schedule either, as it would elsewhere: **Receive now**, **Roll up now** and tests still run it. Legacy setups are copied as prod has them: Internal and ApiCall subscriptions, legacy notifiers, and a response handed to an Internal or ApiCall subscription. Bitween keeps such a response link on an item that already has it but refuses a new one, and a copy is a new save. So the copy alone may recreate it: its calls carry a key it opens for the length of the copy, on a Bitween marked as a mirror. Someone editing on the staging never carries one and meets the same rule as on prod. @@ -56,7 +58,7 @@ Tick the ones to bring, or all of them. **See what comes** then lists everything A picked gateway brings every subscription it leads to. -What the staging doesn't have comes as prod has it, handled as above, so subscriptions arrive paused and data sources and bus gateways switched off. The preview counts what comes by kind, saying how many you picked and how many they need, with each item behind **Show them**, lists the items you'll fill in on staging after bringing, one row each with its fields, and says which partners' and global values' data comes faked. What the staging has stays as it is, so edits made on the staging are never overwritten: a gateway the staging has only gets the partner or route that leads to what comes. Nothing is removed. An item that can't come yet, such as a subscription waiting for an adapter, stays behind and says why, and so does anything that needs it. +What the staging doesn't have comes as prod has it, handled as above, so subscriptions arrive paused and data sources and bus gateways switched off. The preview counts what comes by kind, saying how many you picked and how many they need, with each item behind **Show them**, lists the items you'll fill in on staging after bringing, one row each with its fields, and says which partners' and global values' data comes faked. Under **Copied as-is from prod** it lists every adapter setting that comes as prod has it, with its value: a secret is told by its name and its value, so check none of these is one, or someone's data, and clear any that is on the staging after bringing. What the staging has stays as it is, so edits made on the staging are never overwritten: a gateway the staging has only gets the partner or route that leads to what comes. Nothing is removed. An item that can't come yet, such as a subscription waiting for an adapter, stays behind and says why, and so does anything that needs it. **Bring** then brings them. It carries on past an item that fails, and lists each failure with Bitween's reason.