From 02de4baaff4f40e9c3fa3b4e73fec1fcd81094dc Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 09:03:39 +0300 Subject: [PATCH 01/14] Describe adapters in the protocol without .NET in the picture Hello gains optional fields an SDK in any language can fill: its language, the settings it reads (the neutral form of Runner.Expect), the kinds and contracts it implements, and JSON Schemas for each command's argument and result beside the existing .NET type names. Every field is new and optional, so adapters and hosts built before them keep talking as they did. The host records them on the instance and its health view; the .NET SDK now says it is dotnet. Tested both ways: a current adapter reports its language, and an adapter on the published 10.0.0 SDK attaches and runs as before with the new fields simply empty. --- .../NewHostTests.cs | 8 +++++ SW.Serverless.Contract/Protos/adapter.proto | 35 +++++++++++++++++++ SW.Serverless.Sdk/Resident/ResidentRunner.cs | 1 + .../CommandDiscoveryTests.cs | 7 ++++ SW.Serverless/Resident/AdapterCommand.cs | 23 ++++++++++++ SW.Serverless/Resident/AdapterHostService.cs | 15 +++++++- SW.Serverless/Resident/InstanceHealth.cs | 4 +++ SW.Serverless/Resident/ResidentAdapterHost.cs | 4 +++ .../Resident/ResidentAdapterInstance.cs | 12 +++++++ 9 files changed, 108 insertions(+), 1 deletion(-) diff --git a/SW.Serverless.CompatibilityTests/NewHostTests.cs b/SW.Serverless.CompatibilityTests/NewHostTests.cs index f1ce3fe..4ac3108 100644 --- a/SW.Serverless.CompatibilityTests/NewHostTests.cs +++ b/SW.Serverless.CompatibilityTests/NewHostTests.cs @@ -141,6 +141,14 @@ public async Task A_resident_adapter_on_sdk_10_0_0_runs_on_the_current_host() Assert.AreEqual(InstanceState.Ready, instance.State); Assert.AreEqual("v10:ping", await instance.InvokeAsync("Echo", "ping")); + // The 10.0.0 SDK knows nothing of the language-neutral Hello fields: they're simply + // empty, and nothing depends on them being there. + var described = adapters.Describe().Single(h => h.InstanceKey == "compat"); + Assert.AreEqual("", described.SdkLanguage); + Assert.AreEqual(0, described.Settings.Count); + Assert.AreEqual(0, described.Kinds.Count); + Assert.AreEqual(0, described.Contracts.Count); + await adapters.StopAsync("sdk10.resident", "compat", drain: false); } finally diff --git a/SW.Serverless.Contract/Protos/adapter.proto b/SW.Serverless.Contract/Protos/adapter.proto index 681a408..f692739 100644 --- a/SW.Serverless.Contract/Protos/adapter.proto +++ b/SW.Serverless.Contract/Protos/adapter.proto @@ -118,6 +118,36 @@ message Hello { // adapters and hosts built before this field existed; a host that understands this one gets // the shape as well as the name. repeated CommandInfo commands = 7; + + // Fields 8–11 describe the adapter without .NET in the picture, for SDKs in any language. All + // optional: a host reads them when present and an adapter built before them leaves them out. + + // The SDK's language — dotnet, python, node, go — beside sdk_version. + string sdk_language = 8; + + // The settings the adapter reads, the language-neutral form of Runner.Expect: what a host or a + // UI needs to ask for before running it. + repeated SettingInfo settings = 9; + + // The kinds of adapter it implements for a contract, as that contract names them — for + // Bitween: handler, mapper, validator, receiver. + repeated string kinds = 10; + + // The contracts it implements and their versions, e.g. bitween -> 1. + map contracts = 11; +} + +// One setting an adapter reads. +message SettingInfo { + string name = 1; + string description = 2; + bool required = 3; + // Secret settings are masked wherever they're shown and never logged. + bool secret = 4; + // Empty when the setting has no default. + string default_value = 5; + // text, multiline, number, boolean, select or json, as adapter manifests type properties. + string type = 6; } // One callable command on an adapter. @@ -140,6 +170,11 @@ message CommandInfo { // From [AdapterCommand(Description = "…")] on the method. Optional, and worth setting: the // difference between a list of names and a list an operator can act on. string description = 5; + + // JSON Schemas of the argument and the result, for SDKs in any language: parameter_type and + // parameter_schema above are .NET's view and stay as they are. Empty when not given. + string input_schema = 6; + string output_schema = 7; } message InvokeResult { diff --git a/SW.Serverless.Sdk/Resident/ResidentRunner.cs b/SW.Serverless.Sdk/Resident/ResidentRunner.cs index 5ff31dd..242cded 100644 --- a/SW.Serverless.Sdk/Resident/ResidentRunner.cs +++ b/SW.Serverless.Sdk/Resident/ResidentRunner.cs @@ -161,6 +161,7 @@ await call.RequestStream.WriteAsync(new AdapterFrame InstanceKey = handshake.InstanceKey ?? "", ProtocolVersion = protocol, SdkVersion = typeof(ResidentRunner).Assembly.GetName().Version?.ToString() ?? "0.0.0", + SdkLanguage = "dotnet", Capabilities = { Capabilities() }, Commands = { CommandInfos() } } diff --git a/SW.Serverless.UnitTests/CommandDiscoveryTests.cs b/SW.Serverless.UnitTests/CommandDiscoveryTests.cs index 6fc2826..fd6b406 100644 --- a/SW.Serverless.UnitTests/CommandDiscoveryTests.cs +++ b/SW.Serverless.UnitTests/CommandDiscoveryTests.cs @@ -83,6 +83,13 @@ public static async Task ClassCleanup() static IReadOnlyCollection Commands() => adapters.Describe().Single(h => h.InstanceKey == "discovery").CommandDetails; + [TestMethod] + public void The_adapter_says_which_language_its_SDK_is() + { + var described = adapters.Describe().Single(h => h.InstanceKey == "discovery"); + Assert.AreEqual("dotnet", described.SdkLanguage); + } + [TestMethod] public void Every_public_command_is_discovered_without_being_declared_anywhere() { diff --git a/SW.Serverless/Resident/AdapterCommand.cs b/SW.Serverless/Resident/AdapterCommand.cs index b180f0c..c78fc1b 100644 --- a/SW.Serverless/Resident/AdapterCommand.cs +++ b/SW.Serverless/Resident/AdapterCommand.cs @@ -24,6 +24,12 @@ public class AdapterCommand /// public string ParameterSchema { get; set; } + /// The argument's JSON Schema, from an SDK in any language. Empty when not given. + public string InputSchema { get; set; } + + /// The result's JSON Schema, from an SDK in any language. Empty when not given. + public string OutputSchema { get; set; } + /// False for a command returning Task rather than Task<T>. public bool ReturnsValue { get; set; } @@ -35,4 +41,21 @@ public class AdapterCommand public override string ToString() => $"{Name}({ParameterType}){(ReturnsValue ? "" : " -> void")}"; } + + /// One setting an adapter says it reads, in any language. + public class AdapterSetting + { + public string Name { get; set; } + public string Description { get; set; } + public bool Required { get; set; } + + /// Masked wherever it's shown, never logged. + public bool Secret { get; set; } + + /// Empty when the setting has no default. + public string DefaultValue { get; set; } + + /// text, multiline, number, boolean, select or json. + public string Type { get; set; } + } } diff --git a/SW.Serverless/Resident/AdapterHostService.cs b/SW.Serverless/Resident/AdapterHostService.cs index fae5a83..3f91d51 100644 --- a/SW.Serverless/Resident/AdapterHostService.cs +++ b/SW.Serverless/Resident/AdapterHostService.cs @@ -63,12 +63,25 @@ public override async Task Attach(IAsyncStreamReader requestStream ParameterType = c.ParameterType, ParameterSchema = c.ParameterSchema, ReturnsValue = c.ReturnsValue, - Description = c.Description + Description = c.Description, + InputSchema = c.InputSchema, + OutputSchema = c.OutputSchema }) .OrderBy(c => c.Name) .ToArray(); instance.SdkVersion = first.Hello.SdkVersion; + // Empty from an SDK that predates them, which is fine: they describe, they don't decide. + instance.SdkLanguage = first.Hello.SdkLanguage; + instance.Settings = first.Hello.Settings + .Select(x => new AdapterSetting + { + Name = x.Name, Description = x.Description, Required = x.Required, Secret = x.Secret, + DefaultValue = x.DefaultValue, Type = x.Type + }) + .ToArray(); + instance.Kinds = first.Hello.Kinds.ToArray(); + instance.Contracts = first.Hello.Contracts.ToDictionary(kv => kv.Key, kv => kv.Value); instance.ProtocolVersion = first.Hello.ProtocolVersion; logger.LogInformation( diff --git a/SW.Serverless/Resident/InstanceHealth.cs b/SW.Serverless/Resident/InstanceHealth.cs index 5900d51..7d00a19 100644 --- a/SW.Serverless/Resident/InstanceHealth.cs +++ b/SW.Serverless/Resident/InstanceHealth.cs @@ -55,6 +55,10 @@ public class InstanceHealth public IReadOnlyCollection CommandDetails { get; set; } = Array.Empty(); public string SdkVersion { get; set; } + public string SdkLanguage { get; set; } + public IReadOnlyCollection Settings { get; set; } + public IReadOnlyCollection Kinds { get; set; } + public IReadOnlyDictionary Contracts { get; set; } public int ProtocolVersion { get; set; } public IReadOnlyDictionary StartupValues { get; set; } diff --git a/SW.Serverless/Resident/ResidentAdapterHost.cs b/SW.Serverless/Resident/ResidentAdapterHost.cs index 8d543a0..81174b2 100644 --- a/SW.Serverless/Resident/ResidentAdapterHost.cs +++ b/SW.Serverless/Resident/ResidentAdapterHost.cs @@ -609,6 +609,10 @@ static InstanceHealth Describe(Supervised supervised) Commands = instance.Commands, CommandDetails = instance.CommandDetails, SdkVersion = instance.SdkVersion, + SdkLanguage = instance.SdkLanguage, + Settings = instance.Settings, + Kinds = instance.Kinds, + Contracts = instance.Contracts, ProtocolVersion = instance.ProtocolVersion, // Names only. These are connection strings and passwords, and health is what gets // shown on screens and serialised into logs. diff --git a/SW.Serverless/Resident/ResidentAdapterInstance.cs b/SW.Serverless/Resident/ResidentAdapterInstance.cs index 8268b20..91c16af 100644 --- a/SW.Serverless/Resident/ResidentAdapterInstance.cs +++ b/SW.Serverless/Resident/ResidentAdapterInstance.cs @@ -111,6 +111,18 @@ internal ResidentAdapterInstance(string adapterId, string instanceKey, string to internal IDisposable DirectoryLease { get; set; } public string SdkVersion { get; internal set; } + + /// The SDK's language — dotnet, python, node, go — or empty from an SDK that predates it. + public string SdkLanguage { get; internal set; } + + /// The settings the adapter says it reads; empty from an SDK that predates them. + public IReadOnlyCollection Settings { get; internal set; } = Array.Empty(); + + /// The kinds it implements for a contract, e.g. handler or receiver for Bitween. + public IReadOnlyCollection Kinds { get; internal set; } = Array.Empty(); + + /// The contracts it implements and their versions, e.g. bitween → 1. + public IReadOnlyDictionary Contracts { get; internal set; } = new Dictionary(); public int ProtocolVersion { get; internal set; } public IReadOnlyDictionary AdapterValues { get; internal set; } From 757198a10417de5e3ddbc3ac9e4abda263b35e94 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 09:05:25 +0300 Subject: [PATCH 02/14] Add the manifest fields multi-language adapters need AdapterManifest gains runtimeVersion, platforms, per-platform entries beside the default entry, the contracts an adapter implements, and the source the package carries (its folder, each file's SHA-256, the build command and the lockfiles). Named runtimes are dotnet, exec, python and node; a runtime is a name, never a path. Validation names a wrong value in any of them. Every field is optional and the default entry stays what older readers use, so a manifest written today still means the same to them. Tested against the manifest parser actually released (SimplyWorks.Serverless.Contract 10.0.2, which released Bitween reads with): it keeps every new field through a round trip, finds nothing wrong, and sees the entry and runtime as before. --- SW.Serverless.Compat.ManifestV1002/Program.cs | 12 ++ .../SW.Serverless.Compat.ManifestV1002.csproj | 16 +++ SW.Serverless.CompatibilityTests/Compat.cs | 22 ++++ .../MultiLanguageManifestTests.cs | 106 ++++++++++++++++++ .../SW.Serverless.CompatibilityTests.csproj | 2 + .../Catalog/AdapterManifest.cs | 89 +++++++++++++++ SW.Serverless.sln | 14 +++ 7 files changed, 261 insertions(+) create mode 100644 SW.Serverless.Compat.ManifestV1002/Program.cs create mode 100644 SW.Serverless.Compat.ManifestV1002/SW.Serverless.Compat.ManifestV1002.csproj create mode 100644 SW.Serverless.CompatibilityTests/MultiLanguageManifestTests.cs diff --git a/SW.Serverless.Compat.ManifestV1002/Program.cs b/SW.Serverless.Compat.ManifestV1002/Program.cs new file mode 100644 index 0000000..be6b3cd --- /dev/null +++ b/SW.Serverless.Compat.ManifestV1002/Program.cs @@ -0,0 +1,12 @@ +using System; +using System.IO; +using SW.Serverless.Contract.Catalog; + +// Reads the manifest at args[0] with the 10.0.2 parser and writes it back: what a released tool +// does to a manifest a newer one wrote. Prints the problems 10.0.2 finds, then the JSON. +var manifest = AdapterManifest.Parse(File.ReadAllText(args[0])); +foreach (var problem in manifest.Validate()) + Console.WriteLine("PROBLEM:" + problem); +Console.WriteLine("ENTRY:" + manifest.Entry); +Console.WriteLine("RUNTIME:" + manifest.Runtime); +Console.WriteLine("JSON:" + manifest.ToJson().ReplaceLineEndings(" ")); diff --git a/SW.Serverless.Compat.ManifestV1002/SW.Serverless.Compat.ManifestV1002.csproj b/SW.Serverless.Compat.ManifestV1002/SW.Serverless.Compat.ManifestV1002.csproj new file mode 100644 index 0000000..f498991 --- /dev/null +++ b/SW.Serverless.Compat.ManifestV1002/SW.Serverless.Compat.ManifestV1002.csproj @@ -0,0 +1,16 @@ + + + + + Exe + net10.0 + false + + + + + + + diff --git a/SW.Serverless.CompatibilityTests/Compat.cs b/SW.Serverless.CompatibilityTests/Compat.cs index 01a2cc7..dc3e54e 100644 --- a/SW.Serverless.CompatibilityTests/Compat.cs +++ b/SW.Serverless.CompatibilityTests/Compat.cs @@ -146,6 +146,28 @@ public record OldHostRun(int ExitCode, List Lines) } /// Runs the host built on the PUBLISHED SimplyWorks.Serverless 10.0.0 as its own process. + /// Reads a manifest file with the parser published in SimplyWorks.Serverless.Contract 10.0.2. + public static async Task ReadWithManifestV1002Async(string manifestPath) + { + var entry = System.IO.Path.Combine(BuildOutput("SW.Serverless.Compat.ManifestV1002"), "SW.Serverless.Compat.ManifestV1002.dll"); + var start = new ProcessStartInfo("dotnet") { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false }; + start.ArgumentList.Add(entry); + start.ArgumentList.Add(manifestPath); + + using var process = Process.Start(start)!; + var stdout = process.StandardOutput.ReadToEndAsync(); + var stderr = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(60_000)) + { + process.Kill(true); + Assert.Fail("the 10.0.2 manifest reader did not finish within a minute"); + } + + var lines = (await stdout).Split('\n', StringSplitOptions.RemoveEmptyEntries).Select(l => l.TrimEnd('\r')).ToList(); + lines.AddRange((await stderr).Split('\n', StringSplitOptions.RemoveEmptyEntries).Select(l => "stderr: " + l.TrimEnd('\r'))); + return new OldHostRun(process.ExitCode, lines); + } + public static async Task RunOldHostAsync(Bucket bucket, string adapter, params string[] args) { var entry = System.IO.Path.Combine(BuildOutput("SW.Serverless.Compat.OldHost"), "SW.Serverless.Compat.OldHost.dll"); diff --git a/SW.Serverless.CompatibilityTests/MultiLanguageManifestTests.cs b/SW.Serverless.CompatibilityTests/MultiLanguageManifestTests.cs new file mode 100644 index 0000000..7dee375 --- /dev/null +++ b/SW.Serverless.CompatibilityTests/MultiLanguageManifestTests.cs @@ -0,0 +1,106 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Serverless.Contract.Catalog; + +namespace SW.Serverless.CompatibilityTests; + +/// +/// The manifest fields multi-language adapters add — runtime version, platforms and their entries, +/// contracts, the source the package carries — written by the current tools and read by the +/// manifest parser released Bitween uses (SimplyWorks.Serverless.Contract 10.0.2). That parser +/// must keep every one of them, find nothing wrong, and still see the fields it knows as before. +/// +[TestClass] +public class MultiLanguageManifestTests +{ + static AdapterManifest Written() => new() + { + Id = "acme.orders", + Version = "2.1.0", + DisplayName = "Acme orders", + Kinds = { "handler" }, + Runtime = AdapterManifest.PythonRuntime, + RuntimeVersion = ">=3.12", + Language = "python", + Entry = "main.py", + Lifecycle = AdapterManifest.ClassicLifecycle, + Platforms = new() { "linux-x64", "linux-arm64" }, + Entries = new() { ["linux-x64"] = "x64/main.py", ["linux-arm64"] = "arm64/main.py" }, + Contracts = new() { ["bitween"] = 1 }, + Source = new AdapterSource + { + Files = new() { ["main.py"] = new string('a', 64), ["requirements.lock"] = new string('b', 64) }, + BuildCommand = "serverless build", + Lockfiles = new() { "requirements.lock" }, + }, + }; + + [TestMethod] + public async Task The_released_manifest_parser_keeps_every_new_field_and_finds_nothing_wrong() + { + var written = Written(); + Assert.AreEqual(0, written.Validate().Count, string.Join("; ", written.Validate())); + var path = Path.Combine(Path.GetTempPath(), $"manifest-{Guid.NewGuid():N}.json"); + await File.WriteAllTextAsync(path, written.ToJson()); + try + { + var read = await Compat.ReadWithManifestV1002Async(path); + + Assert.AreEqual(0, read.ExitCode, read.ToString()); + Assert.IsNull(read.Value("PROBLEM"), read.ToString()); + Assert.AreEqual("main.py", read.Value("ENTRY"), "the default entry is what released readers use"); + Assert.AreEqual("python", read.Value("RUNTIME"), "a released host sees the runtime, and refuses one it can't run"); + + var before = JsonNode.Parse(written.ToJson())!; + var after = JsonNode.Parse(read.Value("JSON")!)!; + foreach (var field in new[] { "runtimeVersion", "platforms", "entries", "contracts", "source" }) + Assert.IsTrue(JsonNode.DeepEquals(before[field], after[field]), $"{field} changed in a round trip through 10.0.2:\n{after[field]}"); + } + finally + { + File.Delete(path); + } + } + + [TestMethod] + public void The_new_fields_round_trip_through_the_current_parser() + { + var again = AdapterManifest.Parse(Written().ToJson()); + + Assert.AreEqual(">=3.12", again.RuntimeVersion); + CollectionAssert.AreEqual(new[] { "linux-x64", "linux-arm64" }, again.Platforms); + Assert.AreEqual("arm64/main.py", again.EntryFor("linux-arm64")); + Assert.AreEqual("main.py", again.EntryFor("osx-arm64"), "a platform without its own entry uses the default"); + Assert.AreEqual(1, again.Contracts!["bitween"]); + Assert.AreEqual(AdapterSource.DefaultPath, again.Source!.Path); + Assert.AreEqual(2, again.Source.Files.Count); + Assert.IsNull(again.Extensions, "nothing unknown to the current parser"); + } + + [TestMethod] + public void A_manifest_without_the_new_fields_is_read_as_a_dotnet_adapter_as_before() + { + var old = AdapterManifest.Parse("""{ "id": "legacy", "entry": "Legacy.dll" }"""); + + Assert.AreEqual(AdapterManifest.DotnetRuntime, old.Runtime); + Assert.AreEqual("Legacy.dll", old.EntryFor("linux-x64")); + Assert.IsNull(old.Platforms); + Assert.IsNull(old.Source); + Assert.AreEqual(0, old.Validate().Count); + } + + [DataTestMethod] + [DataRow("""{ "runtime": "../bin/sh" }""", "runtime")] + [DataRow("""{ "platforms": ["Linux x64"] }""", "platform")] + [DataRow("""{ "platforms": ["linux-x64"], "entries": { "osx-arm64": "a" } }""", "doesn't list")] + [DataRow("""{ "platforms": ["linux-x64"], "entries": { "linux-x64": "../../etc/passwd" } }""", "inside the package")] + [DataRow("""{ "contracts": { "bitween": 0 } }""", "contract")] + [DataRow("""{ "source": { "files": { "main.py": "not-a-hash" } } }""", "SHA-256")] + [DataRow("""{ "source": { "path": "/abs", "files": {} } }""", "source.path")] + public void A_wrong_new_field_is_named_by_validation(string json, string expected) + { + var problems = AdapterManifest.Parse(json).Validate(); + Assert.IsTrue(problems.Any(p => p.Contains(expected)), $"expected a problem about {expected}: {string.Join("; ", problems)}"); + } +} diff --git a/SW.Serverless.CompatibilityTests/SW.Serverless.CompatibilityTests.csproj b/SW.Serverless.CompatibilityTests/SW.Serverless.CompatibilityTests.csproj index c93e8aa..8910b0a 100644 --- a/SW.Serverless.CompatibilityTests/SW.Serverless.CompatibilityTests.csproj +++ b/SW.Serverless.CompatibilityTests/SW.Serverless.CompatibilityTests.csproj @@ -30,6 +30,8 @@ + A self-contained executable — Go, Rust, a native-compiled .NET app. + public const string ExecRuntime = "exec"; + public const string PythonRuntime = "python"; + public const string NodeRuntime = "node"; public int ManifestVersion { get; set; } = CurrentManifestVersion; @@ -78,6 +82,31 @@ public class AdapterManifest /// The file the runtime starts, relative to the package root. public string Entry { get; set; } + /// + /// The runtime versions it needs, e.g. ">=3.12" for Python. A host refuses to install it + /// when the runtime it has is outside. Absent means any. + /// + public string RuntimeVersion { get; set; } + + /// + /// The platforms it runs on — linux-x64, linux-arm64, osx-arm64, win-x64 — for a package + /// with platform-specific contents, such as an exec binary or Python wheels with native code. + /// Absent means it runs anywhere its runtime does. + /// + public List Platforms { get; set; } + + /// + /// A per-platform , for a package carrying one build per platform. + /// stays the default, and stays what readers before this field use. + /// + public Dictionary Entries { get; set; } + + /// The contracts it implements and their versions, e.g. bitween → 1. + public Dictionary Contracts { get; set; } + + /// The source the package was built from, when it carries it. + public AdapterSource Source { get; set; } + /// or . public string Lifecycle { get; set; } = ClassicLifecycle; @@ -116,6 +145,10 @@ public static AdapterManifest Parse(string json) => public string ToJson() => JsonSerializer.Serialize(this, JsonOptions); + /// What the runtime starts on : its own entry, or the default. + public string EntryFor(string platform) => + platform != null && Entries != null && Entries.TryGetValue(platform, out var entry) ? entry : Entry; + public bool IsResident => string.Equals(Lifecycle, ResidentLifecycle, StringComparison.OrdinalIgnoreCase); @@ -123,6 +156,9 @@ public static AdapterManifest Parse(string json) => static readonly Regex IdPattern = new(@"^[a-z0-9][a-z0-9._-]*$"); static readonly Regex VersionPattern = new(@"^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?$"); + static readonly Regex RuntimePattern = new(@"^[a-z][a-z0-9.-]*$"); + static readonly Regex PlatformPattern = new(@"^[a-z]+-[a-z0-9]+$"); + static readonly Regex Sha256Pattern = new(@"^[0-9a-fA-F]{64}$"); static readonly Regex PropertyNamePattern = new(@"^[A-Za-z_][A-Za-z0-9_.:-]*$"); /// What is wrong with this manifest, empty when nothing is. @@ -140,6 +176,33 @@ public IReadOnlyList Validate() problems.Add($"entry '{Entry}' must be a path inside the package."); if (!string.IsNullOrEmpty(Icon) && !IsPackagePath(Icon)) problems.Add($"icon '{Icon}' must be a path inside the package."); + if (!string.IsNullOrEmpty(Runtime) && !RuntimePattern.IsMatch(Runtime)) + problems.Add($"runtime '{Runtime}' must be a name such as dotnet, exec, python or node."); + foreach (var platform in Platforms ?? new List()) + if (!PlatformPattern.IsMatch(platform ?? "")) + problems.Add($"platform '{platform}' must look like linux-x64 or osx-arm64."); + foreach (var (platform, entry) in Entries ?? new Dictionary()) + { + if (Platforms == null || !Platforms.Contains(platform)) + problems.Add($"entries names platform '{platform}', which platforms doesn't list."); + if (!IsPackagePath(entry)) + problems.Add($"entries.{platform} '{entry}' must be a path inside the package."); + } + foreach (var (contract, version) in Contracts ?? new Dictionary()) + if (string.IsNullOrWhiteSpace(contract) || version < 1) + problems.Add($"contract '{contract}' needs a name and a version of 1 or more."); + if (Source != null) + { + if (!IsPackagePath(Source.Path ?? "")) + problems.Add($"source.path '{Source.Path}' must be a folder inside the package."); + foreach (var (file, hash) in Source.Files ?? new Dictionary()) + { + if (!IsPackagePath(file)) + problems.Add($"source file '{file}' must be a path inside the source folder."); + if (!Sha256Pattern.IsMatch(hash ?? "")) + problems.Add($"source file '{file}' needs its SHA-256 as 64 hex digits."); + } + } if (!string.IsNullOrEmpty(Lifecycle) && Lifecycle is not (ClassicLifecycle or ResidentLifecycle)) problems.Add($"lifecycle '{Lifecycle}' must be '{ClassicLifecycle}' or '{ResidentLifecycle}'."); if (Protocol != null && Protocol.Min > Protocol.Max) @@ -192,6 +255,32 @@ public class AdapterProtocolRange public Dictionary Extensions { get; set; } } + /// + /// The source a package carries, so a published version can be read, compared, audited and + /// rebuilt. The files themselves sit in the package under ; hosts don't + /// extract them. + /// + public class AdapterSource + { + public const string DefaultPath = "source"; + + /// The folder inside the package that holds the source. + public string Path { get; set; } = DefaultPath; + + /// Every source file, relative to , with its SHA-256 in hex. + public Dictionary Files { get; set; } = new(); + + /// How the package was built from the source, e.g. "dotnet publish -c Release". + public string BuildCommand { get; set; } + + /// The lockfiles the build resolved dependencies from, relative to . + public List Lockfiles { get; set; } + + /// Fields written by a newer tool, kept so they are not lost on a round trip. + [JsonExtensionData] + public Dictionary Extensions { get; set; } + } + /// The oldest host each side may run on. Absent means no constraint. public class AdapterCompatibility { diff --git a/SW.Serverless.sln b/SW.Serverless.sln index cd717b4..3764d8a 100644 --- a/SW.Serverless.sln +++ b/SW.Serverless.sln @@ -55,6 +55,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.Compat.Reside EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.CompatibilityTests", "SW.Serverless.CompatibilityTests\SW.Serverless.CompatibilityTests.csproj", "{8532437C-5983-48D0-AD79-666B5EA07617}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.Compat.ManifestV1002", "SW.Serverless.Compat.ManifestV1002\SW.Serverless.Compat.ManifestV1002.csproj", "{0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -341,6 +343,18 @@ Global {8532437C-5983-48D0-AD79-666B5EA07617}.Release|x64.Build.0 = Release|Any CPU {8532437C-5983-48D0-AD79-666B5EA07617}.Release|x86.ActiveCfg = Release|Any CPU {8532437C-5983-48D0-AD79-666B5EA07617}.Release|x86.Build.0 = Release|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Debug|x64.ActiveCfg = Debug|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Debug|x64.Build.0 = Debug|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Debug|x86.ActiveCfg = Debug|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Debug|x86.Build.0 = Debug|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Release|Any CPU.Build.0 = Release|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Release|x64.ActiveCfg = Release|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Release|x64.Build.0 = Release|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Release|x86.ActiveCfg = Release|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 67da374d685bb8b3253fd09218788d96a14ebadd Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 09:11:52 +0300 Subject: [PATCH 03/14] Keep source out of hosts and other runtimes out of old hosts' sight Packages that declare their source in the manifest carry it in a folder hosts now leave packed: it is for reading and rebuilding, never for running. Only a declared source folder is skipped, so a package from before manifests with a folder called source of its own unpacks as it always did. Hosts before manifests run adapters/{id}, always with dotnet, and released Bitween lists only adapters/. An adapter in another runtime therefore never goes there: publishing and promoting it record its current version in the catalog alone, and the host now falls back to the catalog's current version when an id has no adapters/{id}. Such an adapter must be published with a version, and an id published as .NET can't move to another runtime, which old hosts would keep running under it. PackageLayoutTests: a .NET package carrying its source runs on a host built on the released 10.0.0 and on the current one, which leaves the source packed; an undeclared source folder is still unpacked; a Python adapter leaves nothing under adapters/, an old host can't find it, and the current host resolves its current version through publish and promote; and both refusals. --- .../PackageLayoutTests.cs | 160 ++++++++++++++++++ .../Shared/AdapterRepository.cs | 37 +++- SW.Serverless/Services/AdapterInstaller.cs | 49 +++++- 3 files changed, 243 insertions(+), 3 deletions(-) create mode 100644 SW.Serverless.CompatibilityTests/PackageLayoutTests.cs diff --git a/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs b/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs new file mode 100644 index 0000000..5efc697 --- /dev/null +++ b/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs @@ -0,0 +1,160 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.PrimitiveTypes; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Installer.Shared; + +namespace SW.Serverless.CompatibilityTests; + +/// +/// The package layout multi-language adapters bring, against the hosts already deployed: +/// a package that carries its source runs on old and new hosts alike, and only new hosts leave +/// the source unpacked; an adapter in another runtime is kept out of old hosts' sight entirely — +/// they only ever run adapters/{id}, and always with dotnet — while new hosts find it in the catalog. +/// +[TestClass] +public class PackageLayoutTests +{ + const string Sample = "SW.Serverless.Samples.Classic"; + + /// The sample's build output as a zip, with an adapter.json and any extra entries. + static string Package(AdapterManifest manifest, IDictionary? extra = null) + { + var output = Compat.BuildOutput(Sample); + var zip = Path.Combine(Path.GetTempPath(), "swsl-compat", $"layout-{Guid.NewGuid():N}.zip"); + Directory.CreateDirectory(Path.GetDirectoryName(zip)!); + using (var archive = ZipFile.Open(zip, ZipArchiveMode.Create)) + { + foreach (var file in Directory.EnumerateFiles(output, "*", SearchOption.AllDirectories)) + archive.CreateEntryFromFile(file, Path.GetRelativePath(output, file).Replace('\\', '/'), CompressionLevel.Fastest); + using (var writer = new StreamWriter(archive.CreateEntry(AdapterManifest.FileName).Open())) + writer.Write(manifest.ToJson()); + foreach (var (name, content) in extra ?? new Dictionary()) + using (var writer = new StreamWriter(archive.CreateEntry(name).Open())) + writer.Write(content); + } + return zip; + } + + static string Sha256(string text) => Convert.ToHexString(SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(text))).ToLowerInvariant(); + + static AdapterManifest DotnetWithSource(string id) => new() + { + Id = id, + Entry = Sample + ".dll", + Source = new AdapterSource { Files = { ["Program.cs"] = Sha256("// the source"), ["Sample.csproj"] = Sha256("") } }, + }; + + static AdapterManifest Python(string id) => new() + { + Id = id, + Runtime = AdapterManifest.PythonRuntime, + Language = "python", + Entry = "main.py", + Kinds = { "handler" }, + }; + + static AdapterRepository Repository(Bucket bucket) => new(bucket.Files, _ => { }); + + [TestMethod] + public async Task A_package_carrying_its_source_runs_on_old_and_new_hosts_and_new_ones_leave_the_source_packed() + { + using var bucket = new Bucket(); + const string id = "layout.withsource"; + var zip = Package(DotnetWithSource(id), new Dictionary + { + ["source/Program.cs"] = "// the source", + ["source/Sample.csproj"] = "", + }); + await Repository(bucket).PublishVersionAsync(id, "1.0.0", zip, + new PackageInfo { EntryAssembly = Sample + ".dll", Manifest = DotnetWithSource(id) }, promote: true, "compat"); + + var old = await Compat.RunOldHostAsync(bucket, id, "--command", "Echo", "--input", "carried"); + Assert.AreEqual(0, old.ExitCode, old.ToString()); + Assert.AreEqual("carried", old.Value("RESULT"), "a host on the released 10.0.0 runs it as before"); + + using var host = Compat.NewHost(bucket); + using (var service = (ServerlessService)host.GetRequiredService()) + { + await service.StartAsync(id, "correlation"); + Assert.AreEqual("carried", await service.InvokeAsync("Echo", "carried")); + } + + var installed = await host.GetRequiredService().InstallAsync(id); + Assert.IsTrue(File.Exists(installed.LocalPath)); + Assert.IsFalse(Directory.Exists(Path.Combine(installed.Directory, "source")), "the source was unpacked onto the host"); + } + + [TestMethod] + public async Task A_folder_called_source_in_a_package_without_declared_source_is_unpacked_as_before() + { + using var bucket = new Bucket(); + const string id = "layout.ownsource"; + var manifest = new AdapterManifest { Id = id, Entry = Sample + ".dll" }; + await Repository(bucket).PublishVersionAsync(id, "1.0.0", + Package(manifest, new Dictionary { ["source/needed-at-runtime.txt"] = "keep me" }), + new PackageInfo { EntryAssembly = Sample + ".dll", Manifest = manifest }, promote: true, "compat"); + + using var host = Compat.NewHost(bucket); + var installed = await host.GetRequiredService().InstallAsync(id); + + Assert.AreEqual("keep me", await File.ReadAllTextAsync(Path.Combine(installed.Directory, "source", "needed-at-runtime.txt"))); + } + + [TestMethod] + public async Task An_adapter_in_another_runtime_is_out_of_old_hosts_sight_and_new_hosts_find_its_current_version() + { + using var bucket = new Bucket(); + const string id = "layout.python"; + var repository = Repository(bucket); + var package = new PackageInfo { EntryAssembly = "main.py", Manifest = Python(id) }; + await repository.PublishVersionAsync(id, "1.0.0", Package(Python(id)), package, promote: true, "compat"); + await repository.PublishVersionAsync(id, "1.1.0", Package(Python(id)), package, promote: false, "compat"); + + Assert.AreEqual(0, (await bucket.KeysAsync("adapters/")).Count, "nothing under adapters/, where old hosts and old Bitween look"); + Assert.AreEqual("1.0.0", (await repository.LoadEntryAsync(id)).Current); + + var old = await Compat.RunOldHostAsync(bucket, id, "--command", "Echo", "--input", "x"); + Assert.AreNotEqual(0, old.ExitCode, "an old host must not find it, let alone start it with dotnet"); + + using (var host = Compat.NewHost(bucket)) + Assert.AreEqual($"adapters-versions/{id}/1.0.0", + (await host.GetRequiredService().GetMetadataAsync(id)).RemoteKey); + + await repository.PromoteAsync(id, "1.1.0", Path.Combine(bucket.Path, "..", "work-" + Guid.NewGuid().ToString("N"))); + + Assert.AreEqual(0, (await bucket.KeysAsync("adapters/")).Count, "promote doesn't put it there either"); + Assert.AreEqual("1.1.0", (await repository.LoadEntryAsync(id)).Current); + using (var host = Compat.NewHost(bucket)) + Assert.AreEqual($"adapters-versions/{id}/1.1.0", + (await host.GetRequiredService().GetMetadataAsync(id)).RemoteKey); + } + + [TestMethod] + public async Task An_id_published_as_dotnet_cannot_move_to_another_runtime() + { + using var bucket = new Bucket(); + const string id = "layout.switch"; + var repository = Repository(bucket); + await repository.PublishVersionAsync(id, "1.0.0", Package(new AdapterManifest { Id = id, Entry = Sample + ".dll" }), + new PackageInfo { EntryAssembly = Sample + ".dll" }, promote: true, "compat"); + + var refused = await Assert.ThrowsExceptionAsync(() => repository.PublishVersionAsync(id, "2.0.0", + Package(Python(id)), new PackageInfo { EntryAssembly = "main.py", Manifest = Python(id) }, promote: true, "compat")); + StringAssert.Contains(refused.Message, "new id"); + Assert.AreEqual("1.0.0", (await repository.LoadEntryAsync(id)).Current); + } + + [TestMethod] + public async Task An_adapter_in_another_runtime_must_be_published_with_a_version() + { + using var bucket = new Bucket(); + var refused = await Assert.ThrowsExceptionAsync(() => Repository(bucket).PublishUnversionedAsync("layout.unversioned", + Package(Python("layout.unversioned")), new PackageInfo { EntryAssembly = "main.py", Manifest = Python("layout.unversioned") })); + + StringAssert.Contains(refused.Message, "with a version"); + Assert.AreEqual(0, (await bucket.KeysAsync()).Count); + } +} diff --git a/SW.Serverless.Installer/Shared/AdapterRepository.cs b/SW.Serverless.Installer/Shared/AdapterRepository.cs index c433ac0..450e85a 100644 --- a/SW.Serverless.Installer/Shared/AdapterRepository.cs +++ b/SW.Serverless.Installer/Shared/AdapterRepository.cs @@ -212,13 +212,24 @@ public async Task PublishVersionAsync(string adapterId, string version, string z if ((await LocateVersionsAsync(adapterId)).ContainsKey(version)) throw new SWException($"Version {version} of '{adapterId}' has already been published."); + // An id keeps its runtime: hosts that predate manifests run adapters/{id} with dotnet, + // and would go on running the old .NET package under an id that had moved on. + var runsOnDotnet = RunsOnDotnet(package.Manifest); + if (!runsOnDotnet && await ExistsAsync(CurrentKey(adapterId))) + throw new SWException( + $"'{adapterId}' is a .NET adapter, and older hosts would keep running that under its id. " + + $"Publish the {package.Manifest!.Runtime} adapter under a new id."); + // Loaded before uploading, so the new version is not mistaken for one the catalog missed. var entry = await LoadEntryAsync(adapterId); var sha256 = InstallerLogic.Sha256Of(zipPath); var metadata = LegacyMetadata(package.EntryAssembly, package.Lifecycle, package.Kind, sha256, version); await UploadAsync(versionKey, zipPath, metadata); - if (promote) await UploadAsync(CurrentKey(adapterId), zipPath, metadata); + // adapters/{id} is what hosts before manifests list and run, always with dotnet. An + // adapter in another runtime never goes there; newer hosts find its current version in + // the catalog. + if (promote && runsOnDotnet) await UploadAsync(CurrentKey(adapterId), zipPath, metadata); entry.Versions.Add(new AdapterVersionRecord { @@ -245,6 +256,12 @@ public async Task PublishVersionAsync(string adapterId, string version, string z /// public async Task PublishUnversionedAsync(string adapterId, string zipPath, PackageInfo package) { + // An unversioned package lives only at adapters/{id}, where older hosts would start it + // with dotnet whatever it is written in. + if (!RunsOnDotnet(package.Manifest)) + throw new SWException( + $"A {package.Manifest!.Runtime} adapter must be published with a version (-v), so older hosts never see it."); + var entry = await LoadEntryAsync(adapterId); var sha256 = InstallerLogic.Sha256Of(zipPath); @@ -255,6 +272,11 @@ await UploadAsync(CurrentKey(adapterId), zipPath, await catalog.SaveAsync(entry); } + /// A package with no manifest, or no runtime in it, is .NET — as every package before manifests was. + static bool RunsOnDotnet(AdapterManifest manifest) => + string.IsNullOrWhiteSpace(manifest?.Runtime) || + string.Equals(manifest.Runtime, AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase); + static void MakeCurrent(AdapterCatalogEntry entry, string version, AdapterManifest manifest, string sha256, string iconDataUri) { @@ -306,6 +328,19 @@ public async Task PromoteAsync(string adapterId, string version, string workDire $"({Short(expected)} recorded, {Short(sha256)} in storage). Nothing was changed."); var manifest = ReadManifest(zipPath) ?? record.Manifest; + + // Another runtime is never copied to adapters/{id}, where hosts before manifests + // would start it with dotnet: being current is the catalog's alone. + if (!RunsOnDotnet(manifest)) + { + record.Sha256 ??= sha256; + record.Manifest ??= manifest; + MakeCurrent(entry, version, manifest, sha256, IconFromPackage(zipPath, manifest)); + await catalog.SaveAsync(entry); + log($"Version {version} of '{adapterId}' is now current."); + return; + } + var entryAssembly = Value(versionMetadata, "EntryAssembly") ?? manifest?.Entry; if (string.IsNullOrWhiteSpace(entryAssembly)) throw new SWException( diff --git a/SW.Serverless/Services/AdapterInstaller.cs b/SW.Serverless/Services/AdapterInstaller.cs index 0de886a..42ac927 100644 --- a/SW.Serverless/Services/AdapterInstaller.cs +++ b/SW.Serverless/Services/AdapterInstaller.cs @@ -145,11 +145,18 @@ async Task ExtractAsync(string adapterId, string remoteKey, string directory) FileAccess.Read, FileShare.Read)) using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read)) { + var sourcePrefix = SourcePrefixOf(archive); long total = 0; foreach (var entry in archive.Entries) { if (string.IsNullOrEmpty(entry.Name)) continue; + // The source a package carries is for reading and rebuilding, never for + // running: left in storage rather than unpacked onto every host. + if (sourcePrefix != null && + entry.FullName.Replace("\\", "/").StartsWith(sourcePrefix, StringComparison.Ordinal)) + continue; + // An entry's name is the package author's to choose, and "../../app/x.dll" // would otherwise be written over the host's own files before any adapter // code ran. Every entry must land inside the extraction directory. @@ -190,6 +197,30 @@ async Task ExtractAsync(string adapterId, string remoteKey, string directory) } } + /// + /// The folder a package's manifest says holds its source, as a "folder/" prefix, or null + /// when the manifest declares none. Only a declared source folder is skipped: a package + /// from before manifests had source may well have a folder called "source" it needs to run. + /// + static string SourcePrefixOf(ZipArchive archive) + { + var manifestEntry = archive.GetEntry(AdapterManifest.FileName); + if (manifestEntry == null) return null; + try + { + using var reader = new StreamReader(manifestEntry.Open()); + var source = AdapterManifest.Parse(reader.ReadToEnd()).Source; + if (source == null || !AdapterManifest.IsPackagePath(source.Path ?? "")) return null; + return source.Path.Replace("\\", "/").TrimEnd('/') + "/"; + } + catch (System.Text.Json.JsonException) + { + // A manifest that can't be read is reported, with its reason, once the package is + // unpacked; here it only means there's no source folder to leave out. + return null; + } + } + /// /// under , or null when it would resolve /// outside it — an absolute path, or one climbing out with "..". @@ -264,14 +295,28 @@ async Task RemoteKeyOf(string adapterRef) { var root = options.AdapterRemotePath; var (adapterId, version) = AdapterCatalogPaths.Split(adapterRef); - if (version == null) return AdapterCatalogPaths.Current(root, adapterRef); + if (version == null) + { + var current = AdapterCatalogPaths.Current(root, adapterRef); + if (await ExistsAsync(current)) return current; + + // An adapter in a runtime older hosts can't run has no adapters/{id} — it is kept + // out of their sight on purpose — so its current version is the catalog's. + var entry = await new AdapterCatalogStore(cloudFilesService, root).GetAsync(adapterRef); + if (entry?.Current is { } currentVersion) + return AdapterCatalogPaths.Version(root, adapterRef, currentVersion); + return current; + } var key = AdapterCatalogPaths.Version(root, adapterId, version); - if ((await cloudFilesService.ListAsync(key)).Any(f => string.Equals(f.Key, key, StringComparison.Ordinal))) + if (await ExistsAsync(key)) return key; return AdapterCatalogPaths.LegacyVersion(root, adapterId, version); } + async Task ExistsAsync(string key) => + (await cloudFilesService.ListAsync(key)).Any(f => string.Equals(f.Key, key, StringComparison.Ordinal)); + public async Task GetMetadataAsync(string adapterId) { if (memoryCache.TryGetValue($"{NamingPrefix}.{adapterId}", out InstalledAdapter cached)) From 3e253004341c5443469e70dfd5843a4a6c54de5e Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 09:37:57 +0300 Subject: [PATCH 04/14] Run adapters on the runtime their manifest names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A runtime is a launcher the host knows by name — dotnet, exec, python, node — never a path a package chooses. AdapterRuntimes holds them, detects which this host actually has (once per process) and names its platform; each launcher applies its own memory ceiling where the runtime has one: the .NET heap limit, V8's --max-old-space-size, nothing in-process for Python, where the watchdog holds it. AddAdapterRuntimes sets where python3, node and dotnet are. Installing refuses, saying why, a runtime this host has no launcher for or doesn't have, a runtimeVersion outside the manifest's range (">=3.12", ">=22, <24", "3.12"), and a package built for other platforms; it picks the platform's own entry and makes an exec entry executable after unpacking. .NET adapters are still started without checking dotnet is there — production images carry the runtime without the SDK — unless the manifest asks for a version. Resident adapters launch through their runtime; an Executable set by the caller or the package's metadata is honoured exactly as before. Tested: an unknown runtime, a missing interpreter, a version out of range and in it, another platform's package, a per-platform entry made executable, a .NET adapter installing with no dotnet to ask, version ranges, and end to end a self-contained single-file binary packaged with an exec manifest, started as itself and answering over gRPC. --- SW.Serverless.UnitTests/ExecRuntimeTests.cs | 152 ++++++++++++ .../InstallerSafetyTests.cs | 102 +++++++- .../RuntimeVersionRangeTests.cs | 28 +++ .../IServiceCollectionExtensions.cs | 17 ++ .../Resident/AdapterProcessLauncher.cs | 59 +++-- SW.Serverless/Resident/AdapterSpec.cs | 12 +- .../Resident/IResidentAdapterLocator.cs | 6 + SW.Serverless/Resident/ResidentAdapterHost.cs | 6 +- SW.Serverless/Runtimes/AdapterRuntimes.cs | 226 ++++++++++++++++++ SW.Serverless/Runtimes/RuntimeVersionRange.cs | 58 +++++ SW.Serverless/Services/AdapterInstaller.cs | 59 ++++- 11 files changed, 685 insertions(+), 40 deletions(-) create mode 100644 SW.Serverless.UnitTests/ExecRuntimeTests.cs create mode 100644 SW.Serverless.UnitTests/RuntimeVersionRangeTests.cs create mode 100644 SW.Serverless/Runtimes/AdapterRuntimes.cs create mode 100644 SW.Serverless/Runtimes/RuntimeVersionRange.cs diff --git a/SW.Serverless.UnitTests/ExecRuntimeTests.cs b/SW.Serverless.UnitTests/ExecRuntimeTests.cs new file mode 100644 index 0000000..bf29553 --- /dev/null +++ b/SW.Serverless.UnitTests/ExecRuntimeTests.cs @@ -0,0 +1,152 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; +using SW.CloudFiles.Extensions; +using SW.PrimitiveTypes; +using SW.Serverless.Resident; +using SW.Serverless.Runtimes; +using SW.Serverless.UnitTests.Fixtures; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; + +namespace SW.Serverless.UnitTests +{ + /// + /// The exec runtime: an adapter that is one self-contained executable, started as itself rather + /// than through dotnet or an interpreter. Built here as the Greedy sample published single-file + /// for this machine, packaged with an exec manifest the way any language's binary would be, and + /// run by the resident host over gRPC. + /// + [TestClass] + public class ExecRuntimeTests + { + const string Project = "SW.Serverless.Samples.Greedy"; + const string AdapterId = "test.exec"; + + static IHost host; + static IResidentAdapterHost adapters; + static string workDirectory; + + [ClassInitialize] + public static async Task ClassInitialize(TestContext context) + { + workDirectory = Path.Combine(Path.GetTempPath(), "swsl-exectests", Guid.NewGuid().ToString("N")); + var publishDirectory = Path.Combine(workDirectory, "publish"); + var platform = AdapterRuntimes.CurrentPlatform; + + var project = Path.Combine(RepositoryRoot(), Project, Project + ".csproj"); + using (var publish = Process.Start(new ProcessStartInfo("dotnet", + $"publish \"{project}\" -c Release -r {platform} --self-contained true -p:PublishSingleFile=true -o \"{publishDirectory}\" -v q") + { RedirectStandardOutput = true, RedirectStandardError = true })) + { + var output = await publish!.StandardOutput.ReadToEndAsync() + await publish.StandardError.ReadToEndAsync(); + await publish.WaitForExitAsync(); + Assert.AreEqual(0, publish.ExitCode, output); + } + + var binary = Path.Combine(publishDirectory, OperatingSystem.IsWindows() ? Project + ".exe" : Project); + Assert.IsTrue(File.Exists(binary), $"no single-file binary at {binary}"); + + host = Host.CreateDefaultBuilder() + .ConfigureLogging(l => l.ClearProviders()) + .ConfigureServices(s => + { + s.AddLocalTestsCloudFiles(o => o.BucketName = TestStore.BucketName + "-exec"); + s.AddServerless(o => + { + o.AdapterRemotePath = "adapters"; + o.AdapterLocalPath = Path.Combine(workDirectory, "installed"); + o.AdapterMetadataCacheDuration = 1; + }); + s.AddSingleton(); + s.AddSingleton(sp => sp.GetRequiredService()); + s.AddResidentAdapters(o => + { + o.SocketPath = $"/tmp/swsl-x{Environment.ProcessId}.sock"; + o.PipeName = $"swsl-x{Environment.ProcessId}"; + o.HeartbeatInterval = TimeSpan.FromSeconds(2); + o.HandshakeTimeout = TimeSpan.FromSeconds(60); + }); + }) + .Build(); + await host.StartAsync(); + adapters = host.Services.GetRequiredService(); + + // Packaged with only the binary and its manifest — no .dll anywhere for dotnet to start — + // and stored as a version, as a non-.NET adapter is published. + var zip = Path.Combine(workDirectory, "package.zip"); + using (var archive = ZipFile.Open(zip, ZipArchiveMode.Create)) + { + archive.CreateEntryFromFile(binary, Path.GetFileName(binary)); + using var writer = new StreamWriter(archive.CreateEntry("adapter.json").Open()); + writer.Write(new JObject + { + ["id"] = AdapterId, + ["runtime"] = "exec", + ["lifecycle"] = "resident", + ["entry"] = Path.GetFileName(binary), + ["platforms"] = new JArray(platform), + }.ToString()); + } + + var files = host.Services.GetRequiredService(); + await using (var stream = File.OpenRead(zip)) + await files.WriteAsync(stream, new WriteFileSettings + { + Key = $"adapters-versions/{AdapterId}/1.0.0", + ContentType = "application/zip", + Metadata = new Dictionary + { + ["EntryAssembly"] = Path.GetFileName(binary), + ["Hash"] = "exec-" + Guid.NewGuid().ToString("N")[..12], + ["Protocol"] = "2", + ["Lifecycle"] = "resident", + } + }); + } + + [ClassCleanup] + public static async Task ClassCleanup() + { + if (host != null) await host.StopAsync(); + host?.Dispose(); + try { Directory.Delete(workDirectory, true); } catch { } + } + + static string RepositoryRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null && !File.Exists(Path.Combine(dir.FullName, "SW.Serverless.sln"))) dir = dir.Parent; + return dir!.FullName; + } + + [TestMethod] + public async Task A_self_contained_binary_runs_as_itself_and_answers_over_the_protocol() + { + var instance = await adapters.StartExclusiveAsync(new AdapterSpec + { + AdapterId = $"{AdapterId}/1.0.0", + InstanceKey = "exec", + }); + + Assert.AreEqual(InstanceState.Ready, instance.State); + // Started as itself: the process is the binary, not dotnet hosting a dll. + Assert.AreEqual(OperatingSystem.IsWindows() ? Project + ".exe" : Project, Path.GetFileName(instance.Process!.MainModule!.FileName)); + + await instance.InvokeAsync("Allocate", 3, timeoutSeconds: 30); + var stats = await instance.InvokeAsync("GetStats", timeoutSeconds: 15); + Assert.AreEqual(3, stats!.Value("allocatedMb")); + + Assert.AreEqual("dotnet", adapters.Describe().Single(h => h.InstanceKey == "exec").SdkLanguage, + "a .NET adapter compiled to a binary still says which SDK it was built with"); + await adapters.StopAsync($"{AdapterId}/1.0.0", "exec", drain: false); + } + } +} diff --git a/SW.Serverless.UnitTests/InstallerSafetyTests.cs b/SW.Serverless.UnitTests/InstallerSafetyTests.cs index f941ad8..51ffc61 100644 --- a/SW.Serverless.UnitTests/InstallerSafetyTests.cs +++ b/SW.Serverless.UnitTests/InstallerSafetyTests.cs @@ -3,6 +3,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using SW.CloudFiles.Extensions; using SW.PrimitiveTypes; +using SW.Serverless.Runtimes; using System; using System.Collections.Generic; using System.IO; @@ -54,10 +55,27 @@ public static void ClassCleanup() try { Directory.Delete(Path.GetDirectoryName(localRoot)!, true); } catch { } } - static AdapterInstaller Installer() => new( + static AdapterInstaller Installer(AdapterRuntimeOptions runtimes = null) => new( services.GetRequiredService(), new MemoryCache(new MemoryCacheOptions()), - cloudFiles); + cloudFiles, + new AdapterRuntimes(runtimes)); + + /// A host whose interpreters are at paths that don't exist, whatever this machine has. + static AdapterRuntimeOptions NoInterpreters => new() + { + PythonExecutable = "/nonexistent/python3", + NodeExecutable = "/nonexistent/node", + }; + + /// A host whose "python" answers with a fixed version, so version checks don't depend on this machine. + static AdapterRuntimeOptions PythonReporting(string version) + { + var script = Path.Combine(Path.GetTempPath(), $"fake-python-{Guid.NewGuid():N}.sh"); + File.WriteAllText(script, $"#!/bin/sh\necho \"Python {version}\"\n"); + File.SetUnixFileMode(script, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + return new AdapterRuntimeOptions { PythonExecutable = script }; + } static async Task Publish(string adapterId, IDictionary entries, string hash, string entryAssembly = "Adapter.dll", string key = null) @@ -207,6 +225,19 @@ public async Task An_adapter_needing_a_newer_host_is_refused_with_both_versions( [TestMethod] public async Task A_runtime_the_host_cannot_launch_is_refused_by_name() + { + await Publish("needs-ruby", new Dictionary + { + ["main.rb"] = "puts", + ["adapter.json"] = """{ "runtime": "ruby", "entry": "main.rb" }""" + }, hash: "rb1", entryAssembly: "main.rb"); + + var error = await Assert.ThrowsExceptionAsync(() => Installer().InstallAsync("needs-ruby")); + StringAssert.Contains(error.Message, "'ruby'"); + } + + [TestMethod] + public async Task A_known_runtime_missing_from_the_host_is_refused_saying_so() { await Publish("needs-python", new Dictionary { @@ -214,8 +245,71 @@ public async Task A_runtime_the_host_cannot_launch_is_refused_by_name() ["adapter.json"] = """{ "runtime": "python", "entry": "main.py" }""" }, hash: "py1", entryAssembly: "main.py"); - var error = await Assert.ThrowsExceptionAsync(() => Installer().InstallAsync("needs-python")); - StringAssert.Contains(error.Message, "python"); + var error = await Assert.ThrowsExceptionAsync(() => + Installer(NoInterpreters).InstallAsync("needs-python")); + StringAssert.Contains(error.Message, "'python'"); + StringAssert.Contains(error.Message, "not installed"); + } + + [TestMethod] + public async Task A_runtime_version_outside_the_manifest_s_range_is_refused_and_one_inside_installs() + { + await Publish("needs-python312", new Dictionary + { + ["main.py"] = "print()", + ["adapter.json"] = """{ "runtime": "python", "runtimeVersion": ">=3.12", "entry": "main.py" }""" + }, hash: "py312", entryAssembly: "main.py"); + + var error = await Assert.ThrowsExceptionAsync(() => + Installer(PythonReporting("3.11.9")).InstallAsync("needs-python312")); + StringAssert.Contains(error.Message, ">=3.12"); + StringAssert.Contains(error.Message, "3.11.9"); + + var installed = await Installer(PythonReporting("3.12.4")).InstallAsync("needs-python312"); + Assert.AreEqual("python", installed.Runtime); + Assert.IsTrue(installed.LocalPath.EndsWith("main.py")); + } + + [TestMethod] + public async Task A_package_for_other_platforms_is_refused_and_one_for_this_platform_runs_its_own_entry() + { + await Publish("elsewhere", new Dictionary + { + ["adapter"] = "binary", + ["adapter.json"] = """{ "runtime": "exec", "entry": "adapter", "platforms": ["plan9-mips"] }""" + }, hash: "ex1", entryAssembly: "adapter"); + var error = await Assert.ThrowsExceptionAsync(() => Installer().InstallAsync("elsewhere")); + StringAssert.Contains(error.Message, "plan9-mips"); + + var here = AdapterRuntimes.CurrentPlatform; + await Publish("everywhere", new Dictionary + { + ["default/adapter"] = "default build", + [$"{here}/adapter"] = "this platform's build", + ["adapter.json"] = $$"""{ "runtime": "exec", "entry": "default/adapter", "platforms": ["{{here}}", "plan9-mips"], "entries": { "{{here}}": "{{here}}/adapter" } }""" + }, hash: "ex2", entryAssembly: "default/adapter"); + + var installed = await Installer().InstallAsync("everywhere"); + Assert.AreEqual("this platform's build", File.ReadAllText(installed.LocalPath)); + if (!OperatingSystem.IsWindows()) + Assert.IsTrue(File.GetUnixFileMode(installed.LocalPath).HasFlag(UnixFileMode.UserExecute), + "an exec entry must be executable after unpacking"); + } + + [TestMethod] + public async Task A_dotnet_adapter_installs_without_asking_whether_dotnet_is_there() + { + // As before runtimes: production images carry the runtime without the SDK, and a + // .NET adapter was never refused for want of a check. + await Publish("plain-dotnet", new Dictionary + { + ["Adapter.dll"] = "assembly", + ["adapter.json"] = """{ "runtime": "dotnet", "entry": "Adapter.dll" }""" + }, hash: "dn1"); + + var installed = await Installer(new AdapterRuntimeOptions { DotnetExecutable = "/nonexistent/dotnet" }) + .InstallAsync("plain-dotnet"); + Assert.AreEqual("dotnet", installed.Runtime); } [TestMethod] diff --git a/SW.Serverless.UnitTests/RuntimeVersionRangeTests.cs b/SW.Serverless.UnitTests/RuntimeVersionRangeTests.cs new file mode 100644 index 0000000..61e1eef --- /dev/null +++ b/SW.Serverless.UnitTests/RuntimeVersionRangeTests.cs @@ -0,0 +1,28 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Serverless.Runtimes; + +namespace SW.Serverless.UnitTests +{ + [TestClass] + public class RuntimeVersionRangeTests + { + [DataTestMethod] + [DataRow(null, "3.11.0", true)] + [DataRow(">=3.12", "3.12.0", true)] + [DataRow(">=3.12", "3.13.1", true)] + [DataRow(">=3.12", "3.11.9", false)] + [DataRow(">=22, <24", "22.9.0", true)] + [DataRow(">=22, <24", "24.0.0", false)] + [DataRow(">=22 <24", "23.1.0", true)] + [DataRow("3.12", "3.12.7", true)] + [DataRow("3.12", "3.13.0", false)] + [DataRow("=22", "22.11.0", true)] + [DataRow(">3.12.0", "3.12.0", false)] + [DataRow("<=3.12", "3.12.0", true)] + [DataRow("v22", "22.1.0", true)] + [DataRow("lots", "3.12.0", false)] + [DataRow(">=3.12", "not-a-version", false)] + public void A_runtime_version_is_in_or_out_of_range(string range, string version, bool expected) => + Assert.AreEqual(expected, RuntimeVersionRange.Satisfies(range, version), $"{version} in '{range}'"); + } +} diff --git a/SW.Serverless/Extensions/IServiceCollectionExtensions.cs b/SW.Serverless/Extensions/IServiceCollectionExtensions.cs index 7924dd7..9524c56 100644 --- a/SW.Serverless/Extensions/IServiceCollectionExtensions.cs +++ b/SW.Serverless/Extensions/IServiceCollectionExtensions.cs @@ -17,11 +17,26 @@ public static IServiceCollection AddServerless(this IServiceCollection services, services.AddSingleton(serverlessOptions); services.AddTransient(); services.AddMemoryCache(); + services.TryAddSingleton(new Runtimes.AdapterRuntimeOptions()); + services.TryAddSingleton(); services.TryAddSingleton(); return services; } + /// + /// Where the host finds the interpreters adapters run on — python3, node and dotnet on the + /// PATH unless set. Call before AddServerless or AddResidentAdapters to take effect. + /// + public static IServiceCollection AddAdapterRuntimes(this IServiceCollection services, + Action configure) + { + var options = new Runtimes.AdapterRuntimeOptions(); + configure?.Invoke(options); + services.AddSingleton(options); + return services; + } + /// /// Adds the resident adapter runtime: a Kestrel endpoint on a Unix domain socket or named /// pipe that adapters dial, plus the supervisor that owns their processes. @@ -43,6 +58,8 @@ public static IServiceCollection AddResidentAdapters(this IServiceCollect // process, so a cursor saved on one node is invisible to the next one to run the // adapter. TryAdd, so a host that registered its own keeps it. services.TryAddSingleton(); + services.TryAddSingleton(new Runtimes.AdapterRuntimeOptions()); + services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.AddSingleton(); diff --git a/SW.Serverless/Resident/AdapterProcessLauncher.cs b/SW.Serverless/Resident/AdapterProcessLauncher.cs index 1009e28..09b1d88 100644 --- a/SW.Serverless/Resident/AdapterProcessLauncher.cs +++ b/SW.Serverless/Resident/AdapterProcessLauncher.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.Logging; +using SW.Serverless.Runtimes; using SW.Serverless.Sdk.Resident; using System; using System.Diagnostics; @@ -15,11 +16,13 @@ internal class AdapterProcessLauncher { readonly ResidentOptions options; readonly ILogger logger; + readonly AdapterRuntimes runtimes; - public AdapterProcessLauncher(ResidentOptions options, ILogger logger) + public AdapterProcessLauncher(ResidentOptions options, ILogger logger, AdapterRuntimes runtimes) { this.options = options; this.logger = logger; + this.runtimes = runtimes; } public Process Launch(AdapterSpec spec, ResidentAdapterInstance instance) @@ -31,28 +34,42 @@ public Process Launch(AdapterSpec spec, ResidentAdapterInstance instance) if (!File.Exists(assembly)) throw new FileNotFoundException($"Adapter entry assembly not found.", assembly); - var executable = spec.Executable ?? "dotnet"; - var arguments = executable == "dotnet" ? $"\"{assembly}\"" : ""; - - var psi = new ProcessStartInfo(executable) - { - Arguments = arguments, - WorkingDirectory = Path.GetDirectoryName(assembly), - UseShellExecute = false, - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true - }; - - // Workstation GC: server GC allocates a heap and a GC thread per core, which is the - // wrong trade for an adapter that mostly waits on a socket. - if (options.UseWorkstationGc) psi.Environment["DOTNET_gcServer"] = "0"; - var hard = spec.HardMemoryLimitBytes > 0 ? spec.HardMemoryLimitBytes : options.HardMemoryLimitBytes; - if (hard > 0) + ProcessStartInfo psi; + if (!string.IsNullOrWhiteSpace(spec.Executable)) + { + // An executable set by the caller or the package's metadata, as before runtimes: + // dotnet with the assembly, anything else started bare. + var arguments = spec.Executable == "dotnet" ? $"\"{assembly}\"" : ""; + psi = new ProcessStartInfo(spec.Executable) + { + Arguments = arguments, + WorkingDirectory = Path.GetDirectoryName(assembly), + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + // Workstation GC: server GC allocates a heap and a GC thread per core, which is the + // wrong trade for an adapter that mostly waits on a socket. + if (options.UseWorkstationGc) psi.Environment["DOTNET_gcServer"] = "0"; + if (hard > 0) + { + psi.Environment["DOTNET_GCHeapHardLimit"] = hard.ToString("X"); + psi.Environment["DOTNET_GCConserveMemory"] = "5"; + } + } + else { - psi.Environment["DOTNET_GCHeapHardLimit"] = hard.ToString("X"); - psi.Environment["DOTNET_GCConserveMemory"] = "5"; + var runtime = runtimes.Find(spec.Runtime) + ?? throw new NotSupportedException( + $"Adapter '{spec.AdapterId}' needs the '{spec.Runtime}' runtime, which this host does not have."); + psi = runtime.StartInfo(assembly, new RuntimeLaunch + { + HardMemoryLimitBytes = hard, + UseWorkstationGc = options.UseWorkstationGc, + }); } var process = new Process { StartInfo = psi, EnableRaisingEvents = true }; diff --git a/SW.Serverless/Resident/AdapterSpec.cs b/SW.Serverless/Resident/AdapterSpec.cs index 8c1e5ca..1626615 100644 --- a/SW.Serverless/Resident/AdapterSpec.cs +++ b/SW.Serverless/Resident/AdapterSpec.cs @@ -19,9 +19,19 @@ public class AdapterSpec /// public string EntryAssemblyPath { get; set; } - /// Executable to launch. Defaults to `dotnet`; set for self-contained or non-.NET adapters. + /// + /// Executable to launch, overriding the runtime. Kept for callers that set it before + /// runtimes existed; an adapter installed from storage gets its launcher from + /// instead. + /// public string Executable { get; set; } + /// + /// The runtime it runs on — dotnet, exec, python, node — as its manifest names it. Filled + /// in when the adapter is resolved; empty means dotnet. + /// + public string Runtime { get; set; } + /// /// Which warm pool a POOLED rental belongs to. Left null the host derives one from the /// adapter id and a hash of , so that two configurations of the diff --git a/SW.Serverless/Resident/IResidentAdapterLocator.cs b/SW.Serverless/Resident/IResidentAdapterLocator.cs index c0de9be..4d9bb0a 100644 --- a/SW.Serverless/Resident/IResidentAdapterLocator.cs +++ b/SW.Serverless/Resident/IResidentAdapterLocator.cs @@ -17,6 +17,10 @@ public class ResolvedAdapter { public string EntryAssemblyPath { get; set; } public string Executable { get; set; } + + /// The runtime it runs on, from its manifest; empty means dotnet. + public string Runtime { get; set; } + public System.Collections.Generic.IDictionary AdapterValues { get; set; } /// The extraction directory, when installed from storage. Held while the adapter runs. @@ -36,6 +40,7 @@ public async Task ResolveAsync(AdapterSpec spec, CancellationTo { EntryAssemblyPath = spec.EntryAssemblyPath, Executable = spec.Executable, + Runtime = spec.Runtime, AdapterValues = spec.AdapterValues }; @@ -53,6 +58,7 @@ public async Task ResolveAsync(AdapterSpec spec, CancellationTo EntryAssemblyPath = installed.LocalPath, Directory = installed.Directory, Executable = spec.Executable ?? (values.TryGetValue("Executable", out var exe) ? exe : null), + Runtime = installed.Runtime, AdapterValues = values }; } diff --git a/SW.Serverless/Resident/ResidentAdapterHost.cs b/SW.Serverless/Resident/ResidentAdapterHost.cs index 81174b2..04740fb 100644 --- a/SW.Serverless/Resident/ResidentAdapterHost.cs +++ b/SW.Serverless/Resident/ResidentAdapterHost.cs @@ -44,7 +44,8 @@ public class ResidentAdapterHost : IResidentAdapterHost, IHostedService, IAsyncD readonly IResidentAdapterLocator locator; public ResidentAdapterHost(ResidentOptions options, IAdapterEventSink sink, - IAdapterStateStore stateStore, IResidentAdapterLocator locator, ILoggerFactory loggerFactory) + IAdapterStateStore stateStore, IResidentAdapterLocator locator, ILoggerFactory loggerFactory, + Runtimes.AdapterRuntimes runtimes = null) { this.options = options; this.sink = sink; @@ -52,7 +53,7 @@ public ResidentAdapterHost(ResidentOptions options, IAdapterEventSink sink, this.locator = locator; this.loggerFactory = loggerFactory; logger = loggerFactory.CreateLogger(); - launcher = new AdapterProcessLauncher(options, logger); + launcher = new AdapterProcessLauncher(options, logger, runtimes ?? new Runtimes.AdapterRuntimes(null)); } // ------------------------------------------------------------------ transport @@ -197,6 +198,7 @@ async Task SpawnAsync(Supervised supervised, CancellationToken cancellationToken var resolved = await locator.ResolveAsync(spec, cancellationToken); spec.EntryAssemblyPath = resolved.EntryAssemblyPath; spec.Executable = resolved.Executable ?? spec.Executable; + spec.Runtime = resolved.Runtime ?? spec.Runtime; if (resolved.AdapterValues != null) spec.AdapterValues = resolved.AdapterValues; var instance = new ResidentAdapterInstance( diff --git a/SW.Serverless/Runtimes/AdapterRuntimes.cs b/SW.Serverless/Runtimes/AdapterRuntimes.cs new file mode 100644 index 0000000..1e75b8b --- /dev/null +++ b/SW.Serverless/Runtimes/AdapterRuntimes.cs @@ -0,0 +1,226 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using SW.Serverless.Contract.Catalog; + +namespace SW.Serverless.Runtimes +{ + /// Where the host finds the interpreters its adapters run on. + public class AdapterRuntimeOptions + { + /// The Python interpreter. "python3" on the PATH unless set. + public string PythonExecutable { get; set; } = "python3"; + + /// The Node.js executable. "node" on the PATH unless set. + public string NodeExecutable { get; set; } = "node"; + + /// The .NET host. "dotnet" on the PATH unless set. + public string DotnetExecutable { get; set; } = "dotnet"; + } + + /// Whether a runtime is present on this host, and which version. + public sealed record RuntimeStatus(bool Available, string Version, string Reason) + { + public static RuntimeStatus Present(string version) => new(true, version, null); + public static RuntimeStatus Missing(string reason) => new(false, null, reason); + } + + /// What a launch is asked to apply, beside the entry it starts. + public sealed class RuntimeLaunch + { + /// A memory ceiling the runtime enforces itself, where it can. 0 for none. + public long HardMemoryLimitBytes { get; set; } + + /// .NET only: workstation GC, the right trade for an adapter that mostly waits. + public bool UseWorkstationGc { get; set; } + } + + /// + /// One runtime adapters can be written for: how to tell it's here, and how to start an adapter + /// on it. Named in an adapter's manifest; never a path a package can choose. + /// + public interface IAdapterRuntime + { + /// As manifests name it: dotnet, exec, python, node. + string Name { get; } + + Task DetectAsync(); + + /// How to start on this runtime, with its stdio redirected. + ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch); + } + + /// + /// The runtimes this host knows, and which of them it actually has. Detection runs each + /// runtime's version check once and remembers the answer for the life of the process. + /// + public class AdapterRuntimes + { + readonly Dictionary runtimes; + readonly ConcurrentDictionary>> detected = new(StringComparer.OrdinalIgnoreCase); + + public AdapterRuntimes(AdapterRuntimeOptions options) + { + options ??= new AdapterRuntimeOptions(); + runtimes = new IAdapterRuntime[] + { + new DotnetRuntime(options.DotnetExecutable), + new ExecRuntime(), + new PythonRuntime(options.PythonExecutable), + new NodeRuntime(options.NodeExecutable), + } + .ToDictionary(r => r.Name, StringComparer.OrdinalIgnoreCase); + } + + public IReadOnlyCollection Known => runtimes.Keys; + + /// The runtime named, or null when this host has no launcher for it. + public IAdapterRuntime Find(string name) => + runtimes.TryGetValue(string.IsNullOrWhiteSpace(name) ? AdapterManifest.DotnetRuntime : name, out var runtime) + ? runtime + : null; + + public Task StatusAsync(string name) + { + var runtime = Find(name); + if (runtime == null) return Task.FromResult(RuntimeStatus.Missing($"this host has no launcher for '{name}'")); + return detected.GetOrAdd(runtime.Name, _ => new Lazy>(runtime.DetectAsync)).Value; + } + + /// This host's platform as manifests name platforms: linux-x64, osx-arm64, win-x64… + public static string CurrentPlatform + { + get + { + var os = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "win" + : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? "osx" + : "linux"; + var arch = RuntimeInformation.OSArchitecture switch + { + Architecture.X64 => "x64", + Architecture.Arm64 => "arm64", + Architecture.X86 => "x86", + Architecture.Arm => "arm", + var other => other.ToString().ToLowerInvariant(), + }; + return $"{os}-{arch}"; + } + } + + /// Runs with and reads the first version in what it prints. + internal static async Task VersionOf(string executable, string arguments) + { + try + { + using var process = Process.Start(new ProcessStartInfo(executable, arguments) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }); + if (process == null) return RuntimeStatus.Missing($"'{executable}' could not be started"); + var output = await process.StandardOutput.ReadToEndAsync() + await process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(15_000)) + { + try { process.Kill(true); } catch { } + return RuntimeStatus.Missing($"'{executable} {arguments}' did not answer"); + } + var version = Regex.Match(output, @"\d+\.\d+(\.\d+)?"); + return process.ExitCode == 0 && version.Success + ? RuntimeStatus.Present(version.Value) + : RuntimeStatus.Missing($"'{executable} {arguments}' answered: {output.Trim()}"); + } + catch (Exception ex) when (ex is System.ComponentModel.Win32Exception or FileNotFoundException) + { + return RuntimeStatus.Missing($"'{executable}' is not installed on this host"); + } + } + + static ProcessStartInfo Redirected(string executable, string workingDirectory) => new(executable) + { + WorkingDirectory = workingDirectory, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + + sealed class DotnetRuntime(string executable) : IAdapterRuntime + { + public string Name => AdapterManifest.DotnetRuntime; + + // --list-runtimes, not --version: production images carry the runtime without the SDK, + // and there --version fails. The first line names the oldest runtime installed. + public Task DetectAsync() => VersionOf(executable, "--list-runtimes"); + + public ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch) + { + var psi = Redirected(executable, Path.GetDirectoryName(entryPath)); + psi.ArgumentList.Add(entryPath); + if (launch.UseWorkstationGc) psi.Environment["DOTNET_gcServer"] = "0"; + if (launch.HardMemoryLimitBytes > 0) + { + psi.Environment["DOTNET_GCHeapHardLimit"] = launch.HardMemoryLimitBytes.ToString("X"); + psi.Environment["DOTNET_GCConserveMemory"] = "5"; + } + return psi; + } + } + + /// A self-contained executable: the entry itself is started. + sealed class ExecRuntime : IAdapterRuntime + { + public string Name => AdapterManifest.ExecRuntime; + + // Nothing to install: the package carries everything it needs. + public Task DetectAsync() => Task.FromResult(RuntimeStatus.Present("native")); + + public ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch) => + Redirected(entryPath, Path.GetDirectoryName(entryPath)); + } + + sealed class PythonRuntime(string executable) : IAdapterRuntime + { + public string Name => AdapterManifest.PythonRuntime; + + public Task DetectAsync() => VersionOf(executable, "--version"); + + public ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch) + { + var psi = Redirected(executable, Path.GetDirectoryName(entryPath)); + // Unbuffered, so what it writes reaches the host's diagnostics when it's written. + psi.ArgumentList.Add("-u"); + psi.ArgumentList.Add(entryPath); + psi.Environment["PYTHONUNBUFFERED"] = "1"; + psi.Environment["PYTHONDONTWRITEBYTECODE"] = "1"; + // Python has no heap ceiling of its own; the watchdog and, where the container + // allows them, cgroups are what hold it. + return psi; + } + } + + sealed class NodeRuntime(string executable) : IAdapterRuntime + { + public string Name => AdapterManifest.NodeRuntime; + + public Task DetectAsync() => VersionOf(executable, "--version"); + + public ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch) + { + var psi = Redirected(executable, Path.GetDirectoryName(entryPath)); + if (launch.HardMemoryLimitBytes > 0) + // V8's own heap ceiling, in megabytes: an allocation past it fails inside the + // adapter, as DOTNET_GCHeapHardLimit does for .NET. + psi.ArgumentList.Add($"--max-old-space-size={Math.Max(16, launch.HardMemoryLimitBytes / (1024 * 1024))}"); + psi.ArgumentList.Add(entryPath); + return psi; + } + } + } +} diff --git a/SW.Serverless/Runtimes/RuntimeVersionRange.cs b/SW.Serverless/Runtimes/RuntimeVersionRange.cs new file mode 100644 index 0000000..dcb21e0 --- /dev/null +++ b/SW.Serverless/Runtimes/RuntimeVersionRange.cs @@ -0,0 +1,58 @@ +using System; +using System.Linq; +using System.Text.RegularExpressions; + +namespace SW.Serverless.Runtimes +{ + /// + /// A manifest's runtimeVersion: comparisons joined by commas or spaces — ">=3.12", + /// ">=22, <24", "3.12" for exactly that minor line. Empty means any version. + /// + public static class RuntimeVersionRange + { + static readonly Regex Part = new(@"^(>=|<=|>|<|=|==)?\s*v?(\d+(?:\.\d+){0,2})$"); + + /// Whether is in ; a range that can't be read is not satisfied. + public static bool Satisfies(string range, string version) + { + if (string.IsNullOrWhiteSpace(range)) return true; + if (!TryVersion(version, out var actual)) return false; + + foreach (var raw in Regex.Split(range.Trim(), @"\s*,\s*|\s+(?=[<>=])")) + { + var match = Part.Match(raw.Trim()); + if (!match.Success || !TryVersion(match.Groups[2].Value, out var bound)) return false; + var length = match.Groups[2].Value.Split('.').Length; + var ok = match.Groups[1].Value switch + { + ">=" => actual >= bound, + ">" => actual > bound, + "<=" => actual <= bound, + "<" => actual < bound, + // A bare or "=" version matches every release that starts with it: 3.12 is any 3.12.x. + _ => Prefix(actual, length) == Prefix(bound, length), + }; + if (!ok) return false; + } + return true; + } + + public static bool IsReadable(string range) => + string.IsNullOrWhiteSpace(range) || + Regex.Split(range.Trim(), @"\s*,\s*|\s+(?=[<>=])").All(p => Part.IsMatch(p.Trim())); + + static bool TryVersion(string text, out Version version) + { + var parts = (text ?? "").Split('.'); + while (parts.Length < 3) parts = parts.Append("0").ToArray(); + return Version.TryParse(string.Join('.', parts.Take(3)), out version); + } + + static Version Prefix(Version v, int length) => length switch + { + 1 => new Version(v.Major, 0, 0), + 2 => new Version(v.Major, v.Minor, 0), + _ => new Version(v.Major, v.Minor, Math.Max(0, v.Build)), + }; + } +} diff --git a/SW.Serverless/Services/AdapterInstaller.cs b/SW.Serverless/Services/AdapterInstaller.cs index 42ac927..b8d19f0 100644 --- a/SW.Serverless/Services/AdapterInstaller.cs +++ b/SW.Serverless/Services/AdapterInstaller.cs @@ -1,3 +1,4 @@ +using SW.Serverless.Runtimes; using Microsoft.Extensions.Caching.Memory; using SW.PrimitiveTypes; using SW.Serverless.Contract.Catalog; @@ -25,13 +26,15 @@ public class AdapterInstaller readonly ServerlessOptions options; readonly IMemoryCache memoryCache; readonly ICloudFilesService cloudFilesService; + readonly AdapterRuntimes runtimes; public AdapterInstaller(ServerlessOptions options, IMemoryCache memoryCache, - ICloudFilesService cloudFilesService) + ICloudFilesService cloudFilesService, AdapterRuntimes runtimes = null) { this.options = options; this.memoryCache = memoryCache; this.cloudFilesService = cloudFilesService; + this.runtimes = runtimes ?? new AdapterRuntimes(null); } /// @@ -61,7 +64,7 @@ public async Task InstallAsync(string adapterId) PruneSupersededVersions(adapterId, directory); - ApplyManifest(metadata); + await ApplyManifestAsync(metadata); return metadata; } @@ -71,7 +74,7 @@ public async Task InstallAsync(string adapterId) /// the package, while metadata can be changed without touching it. A package without a /// manifest — every package published before them — is left exactly as its metadata says. /// - void ApplyManifest(InstalledAdapter installed) + async Task ApplyManifestAsync(InstalledAdapter installed) { if (installed.Manifest != null || string.IsNullOrEmpty(installed.Directory)) return; @@ -89,25 +92,54 @@ void ApplyManifest(InstalledAdapter installed) $"Adapter '{installed.AdapterId}' has an {AdapterManifest.FileName} that cannot be read: {ex.Message}", ex); } - // A runtime this host has no launcher for is refused now, with its name, rather than - // started with dotnet and failing in a way that says nothing about why. - if (!string.IsNullOrWhiteSpace(manifest.Runtime) && - !string.Equals(manifest.Runtime, AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase)) + // Everything that would stop it running is refused now, saying why, rather than found + // out by a process that fails to start, or starts on the wrong runtime. + var runtimeName = string.IsNullOrWhiteSpace(manifest.Runtime) ? AdapterManifest.DotnetRuntime : manifest.Runtime; + if (runtimes.Find(runtimeName) == null) throw new NotSupportedException( - $"Adapter '{installed.AdapterId}' needs the '{manifest.Runtime}' runtime, which this host does not have."); + $"Adapter '{installed.AdapterId}' needs the '{runtimeName}' runtime, which this host does not have."); + + // .NET adapters have always been started without asking whether dotnet is there, and + // still are: only a version the manifest asks for is checked. Every other runtime is + // checked before anything is started. + var isDotnet = string.Equals(runtimeName, AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase); + if (!isDotnet || !string.IsNullOrWhiteSpace(manifest.RuntimeVersion)) + { + var status = await runtimes.StatusAsync(runtimeName); + if (!status.Available) + throw new NotSupportedException( + $"Adapter '{installed.AdapterId}' needs the '{runtimeName}' runtime, which this host does not have: {status.Reason}."); + if (!RuntimeVersionRange.Satisfies(manifest.RuntimeVersion, status.Version)) + throw new NotSupportedException( + $"Adapter '{installed.AdapterId}' needs {runtimeName} {manifest.RuntimeVersion}; this host has {status.Version}."); + } + + var platform = AdapterRuntimes.CurrentPlatform; + if (manifest.Platforms is { Count: > 0 } platforms && + !platforms.Contains(platform, StringComparer.OrdinalIgnoreCase)) + throw new NotSupportedException( + $"Adapter '{installed.AdapterId}' is built for {string.Join(", ", platforms)}; this host is {platform}."); if (!HostInfo.Satisfies(manifest.Compatibility?.MinHostVersion)) throw new InvalidOperationException( $"Adapter '{installed.AdapterId}' needs SW.Serverless {manifest.Compatibility.MinHostVersion} or later; this host is {HostInfo.Version}."); - if (!string.IsNullOrWhiteSpace(manifest.Entry)) + var entry = manifest.EntryFor(platform); + if (!string.IsNullOrWhiteSpace(entry)) { - installed.LocalPath = ContainedPath(installed.Directory, manifest.Entry.Replace("\\", "/")) + installed.LocalPath = ContainedPath(installed.Directory, entry.Replace("\\", "/")) ?? throw new InvalidOperationException( - $"Adapter '{installed.AdapterId}' declares an entry outside its package: '{manifest.Entry}'."); - installed.EntryAssembly = manifest.Entry; + $"Adapter '{installed.AdapterId}' declares an entry outside its package: '{entry}'."); + installed.EntryAssembly = entry; } + // A zip doesn't reliably carry the execute bit, and an exec adapter's entry is the + // program itself. + if (string.Equals(runtimeName, AdapterManifest.ExecRuntime, StringComparison.OrdinalIgnoreCase) && + !OperatingSystem.IsWindows() && File.Exists(installed.LocalPath)) + File.SetUnixFileMode(installed.LocalPath, File.GetUnixFileMode(installed.LocalPath) | + UnixFileMode.UserExecute | UnixFileMode.GroupExecute); + installed.Manifest = manifest; } @@ -418,6 +450,9 @@ public class InstalledAdapter /// The package's own description, once installed. Null for a package without one. public AdapterManifest Manifest { get; set; } + /// The runtime it runs on: the manifest's, or dotnet for a package without one. + public string Runtime => string.IsNullOrWhiteSpace(Manifest?.Runtime) ? AdapterManifest.DotnetRuntime : Manifest.Runtime; + public IDictionary AdapterValues { get; set; } = new Dictionary(); } } From 338bdd39c880656a8e8d241a1124f712e0be7a43 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 09:48:14 +0300 Subject: [PATCH 05/14] Run classic sessions over gRPC for adapters that speak it IServerlessService keeps its API. An adapter in a runtime other than .NET, or a .NET adapter whose manifest opts in with protocol 2, is no longer started with the classic text protocol: its session runs on a resident instance of its own, under a one-off key, started by StartAsync, called command by command and stopped on dispose. Its expected startup values are the settings it declared in its handshake. Every other .NET adapter takes the text protocol path, unchanged. Without the resident host registered, the error says to register AddResidentAdapters. The .NET SDK now sends what Runner.Expect declared in its handshake, for a handler constructed before it is handed to the runner, the usual way. GrpcClassicTests runs a .NET adapter that speaks gRPC but is run classically: startup values and the correlation id reach it, declared defaults apply, objects go in and come back, its settings are the expected startup values, an error fails the call with its message, disposing stops its process. And a classic session with the self-contained exec adapter runs over gRPC too. The exec test now builds under its own folder, so the sample's usual build output other tests read stays as it was. --- SW.Serverless.Sdk/Resident/ResidentRunner.cs | 14 ++ SW.Serverless.Sdk/Runner.cs | 4 + .../Program.cs | 32 +++ ...erless.UnitTests.GrpcClassicAdapter.csproj | 15 ++ SW.Serverless.UnitTests/ExecRuntimeTests.cs | 23 +- SW.Serverless.UnitTests/GrpcClassicTests.cs | 207 ++++++++++++++++++ .../SW.Serverless.UnitTests.csproj | 2 + SW.Serverless.sln | 14 ++ SW.Serverless/Services/ServerlessService.cs | 124 ++++++++++- 9 files changed, 432 insertions(+), 3 deletions(-) create mode 100644 SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs create mode 100644 SW.Serverless.UnitTests.GrpcClassicAdapter/SW.Serverless.UnitTests.GrpcClassicAdapter.csproj create mode 100644 SW.Serverless.UnitTests/GrpcClassicTests.cs diff --git a/SW.Serverless.Sdk/Resident/ResidentRunner.cs b/SW.Serverless.Sdk/Resident/ResidentRunner.cs index 242cded..28203e1 100644 --- a/SW.Serverless.Sdk/Resident/ResidentRunner.cs +++ b/SW.Serverless.Sdk/Resident/ResidentRunner.cs @@ -162,6 +162,20 @@ await call.RequestStream.WriteAsync(new AdapterFrame ProtocolVersion = protocol, SdkVersion = typeof(ResidentRunner).Assembly.GetName().Version?.ToString() ?? "0.0.0", SdkLanguage = "dotnet", + // Declared with Runner.Expect by the time the handshake goes: so in a handler + // that was constructed before it was handed to the runner, which is the usual way. + Settings = + { + Runner.DeclaredStartupValues.Select(kv => new SettingInfo + { + Name = kv.Key, + Description = kv.Value.Description ?? "", + Required = !kv.Value.Optional, + Secret = kv.Value.Private, + DefaultValue = kv.Value.Default ?? "", + Type = kv.Value.Type ?? "text", + }) + }, Capabilities = { Capabilities() }, Commands = { CommandInfos() } } diff --git a/SW.Serverless.Sdk/Runner.cs b/SW.Serverless.Sdk/Runner.cs index 19f413b..1f54d69 100644 --- a/SW.Serverless.Sdk/Runner.cs +++ b/SW.Serverless.Sdk/Runner.cs @@ -289,6 +289,10 @@ static Dictionary BuildMethodsDictionary(object comma return methodsDictionary; } + /// What has been declared with so far. + internal static IReadOnlyDictionary DeclaredStartupValues => + new Dictionary(expectedStartupValues); + public static void Expect(string name, bool optional = false, bool isPrivate = false, string description = null) { Expect(name, null, optional, isPrivate, description); diff --git a/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs b/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs new file mode 100644 index 0000000..02aa77b --- /dev/null +++ b/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs @@ -0,0 +1,32 @@ +using System.Threading.Tasks; +using SW.Serverless.Sdk; + +namespace SW.Serverless.UnitTests.GrpcClassicAdapter; + +public class Handler +{ + public Handler() + { + Runner.Expect("Prefix", "hello "); + Runner.Expect("ApiKey", isPrivate: true, description: "The partner's key."); + } + + public Task Greet(string name) => Task.FromResult(Runner.StartupValueOf("Prefix") + name); + + public Task Correlation() => Task.FromResult(Runner.StartupValueOf("CorrelationId")); + + public Task Add(Numbers numbers) => Task.FromResult(numbers.A + numbers.B); + + public Task Fail() => throw new System.InvalidOperationException("asked to fail"); +} + +public class Numbers +{ + public int A { get; set; } + public int B { get; set; } +} + +static class Program +{ + static Task Main() => Runner.RunResident(new Handler()); +} diff --git a/SW.Serverless.UnitTests.GrpcClassicAdapter/SW.Serverless.UnitTests.GrpcClassicAdapter.csproj b/SW.Serverless.UnitTests.GrpcClassicAdapter/SW.Serverless.UnitTests.GrpcClassicAdapter.csproj new file mode 100644 index 0000000..c3076cd --- /dev/null +++ b/SW.Serverless.UnitTests.GrpcClassicAdapter/SW.Serverless.UnitTests.GrpcClassicAdapter.csproj @@ -0,0 +1,15 @@ + + + + + Exe + net10.0 + false + + + + + + + diff --git a/SW.Serverless.UnitTests/ExecRuntimeTests.cs b/SW.Serverless.UnitTests/ExecRuntimeTests.cs index bf29553..5294580 100644 --- a/SW.Serverless.UnitTests/ExecRuntimeTests.cs +++ b/SW.Serverless.UnitTests/ExecRuntimeTests.cs @@ -43,7 +43,10 @@ public static async Task ClassInitialize(TestContext context) var project = Path.Combine(RepositoryRoot(), Project, Project + ".csproj"); using (var publish = Process.Start(new ProcessStartInfo("dotnet", - $"publish \"{project}\" -c Release -r {platform} --self-contained true -p:PublishSingleFile=true -o \"{publishDirectory}\" -v q") + $"publish \"{project}\" -c Release -r {platform} --self-contained true -p:PublishSingleFile=true -o \"{publishDirectory}\" " + + // Built under the test's own folder, never beside the sample's usual build output, + // which other tests read. + $"--artifacts-path \"{Path.Combine(workDirectory, "artifacts")}\" -v q") { RedirectStandardOutput = true, RedirectStandardError = true })) { var output = await publish!.StandardOutput.ReadToEndAsync() + await publish.StandardError.ReadToEndAsync(); @@ -148,5 +151,23 @@ public async Task A_self_contained_binary_runs_as_itself_and_answers_over_the_pr "a .NET adapter compiled to a binary still says which SDK it was built with"); await adapters.StopAsync($"{AdapterId}/1.0.0", "exec", drain: false); } + + /// A classic session with an adapter in another runtime goes over gRPC, unchanged for the caller. + [TestMethod] + public async Task A_classic_session_with_an_exec_adapter_runs_over_grpc() + { + var service = host.Services.GetRequiredService(); + await service.StartAsync($"{AdapterId}/1.0.0", "corr-exec"); + try + { + await service.InvokeAsync("Allocate", 2); + var stats = await service.InvokeAsync("GetStats", null); + Assert.AreEqual(2, stats.Value("allocatedMb")); + } + finally + { + ((IDisposable)service).Dispose(); + } + } } } diff --git a/SW.Serverless.UnitTests/GrpcClassicTests.cs b/SW.Serverless.UnitTests/GrpcClassicTests.cs new file mode 100644 index 0000000..d8400cb --- /dev/null +++ b/SW.Serverless.UnitTests/GrpcClassicTests.cs @@ -0,0 +1,207 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; +using SW.CloudFiles.Extensions; +using SW.PrimitiveTypes; +using SW.Serverless.Resident; +using SW.Serverless.UnitTests.Fixtures; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; + +namespace SW.Serverless.UnitTests +{ + /// + /// A classic session — start, call, dispose, as Bitween runs handlers — with an adapter that + /// speaks gRPC: what every adapter in a language other than .NET does, and a .NET adapter + /// whose manifest opts in with protocol 2. IServerlessService is unchanged for the caller; the + /// session runs on a resident instance of its own. + /// + [TestClass] + public class GrpcClassicTests + { + const string Project = "SW.Serverless.UnitTests.GrpcClassicAdapter"; + const string AdapterId = "test.grpcclassic"; + + static IHost host; + static string workDirectory; + + [ClassInitialize] + public static async Task ClassInitialize(TestContext context) + { + workDirectory = Path.Combine(Path.GetTempPath(), "swsl-grpcclassic", Guid.NewGuid().ToString("N")); + host = Host.CreateDefaultBuilder() + .ConfigureLogging(l => l.ClearProviders()) + .ConfigureServices(s => + { + s.AddLocalTestsCloudFiles(o => o.BucketName = TestStore.BucketName + "-grpcclassic"); + s.AddServerless(o => + { + o.AdapterRemotePath = "adapters"; + o.AdapterLocalPath = Path.Combine(workDirectory, "installed"); + o.AdapterMetadataCacheDuration = 1; + o.CommandTimeout = 30; + }); + s.AddSingleton(); + s.AddSingleton(sp => sp.GetRequiredService()); + s.AddResidentAdapters(o => + { + o.SocketPath = $"/tmp/swsl-gc{Environment.ProcessId}.sock"; + o.PipeName = $"swsl-gc{Environment.ProcessId}"; + o.HeartbeatInterval = TimeSpan.FromSeconds(2); + o.HandshakeTimeout = TimeSpan.FromSeconds(60); + }); + }) + .Build(); + await host.StartAsync(); + + var output = TestStore.LocateBuildOutput(Project); + using var buffer = new MemoryStream(); + using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var file in Directory.EnumerateFiles(output, "*", SearchOption.AllDirectories)) + archive.CreateEntryFromFile(file, Path.GetRelativePath(output, file), CompressionLevel.Fastest); + using var writer = new StreamWriter(archive.CreateEntry("adapter.json").Open()); + writer.Write(new JObject + { + ["id"] = AdapterId, + ["entry"] = Project + ".dll", + ["lifecycle"] = "classic", + ["protocol"] = new JObject { ["min"] = 2, ["max"] = 2 }, + }.ToString()); + } + buffer.Position = 0; + await host.Services.GetRequiredService().WriteAsync(buffer, new WriteFileSettings + { + Key = $"adapters/{AdapterId}", + ContentType = "application/zip", + Metadata = new Dictionary + { + ["EntryAssembly"] = Project + ".dll", + ["Hash"] = "gc-" + Guid.NewGuid().ToString("N")[..12], + } + }); + } + + [ClassCleanup] + public static async Task ClassCleanup() + { + if (host != null) await host.StopAsync(); + host?.Dispose(); + try { Directory.Delete(workDirectory, true); } catch { } + } + + static IServerlessService Service() => host.Services.GetRequiredService(); + + static IEnumerable RunningKeys() => + host.Services.GetRequiredService().Describe().Select(h => h.InstanceKey); + + [TestMethod] + public async Task A_call_returns_the_adapter_s_answer_using_the_startup_values_it_was_given() + { + var service = Service(); + await service.StartAsync(AdapterId, "corr-1", new Dictionary { ["Prefix"] = "hi " }); + try + { + Assert.AreEqual("hi world", await service.InvokeAsync("Greet", "world")); + Assert.AreEqual("corr-1", await service.InvokeAsync("Correlation", null)); + Assert.AreEqual(5, await service.InvokeAsync("Add", new { A = 2, B = 3 })); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + + [TestMethod] + public async Task A_declared_default_applies_when_no_value_is_given() + { + var service = Service(); + await service.StartAsync(AdapterId, "corr-2"); + try + { + Assert.AreEqual("hello world", await service.InvokeAsync("Greet", "world")); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + + [TestMethod] + public async Task The_expected_startup_values_are_the_settings_it_declared() + { + var service = Service(); + await service.StartAsync(AdapterId, "corr-3"); + try + { + var expected = await service.GetExpectedStartupValues(); + + Assert.IsTrue(expected["Prefix"].Optional); + Assert.AreEqual("hello ", expected["Prefix"].Default); + Assert.IsFalse(expected["ApiKey"].Optional); + Assert.IsTrue(expected["ApiKey"].Private); + Assert.AreEqual("The partner's key.", expected["ApiKey"].Description); + Assert.IsNull(expected["ApiKey"].Default); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + + [TestMethod] + public async Task An_adapter_error_fails_the_call_with_its_message() + { + var service = Service(); + await service.StartAsync(AdapterId, "corr-4"); + try + { + var error = await Assert.ThrowsExceptionAsync(() => service.InvokeAsync("Fail", null)); + StringAssert.Contains(error.Message, "asked to fail"); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + + [TestMethod] + public async Task Disposing_the_session_stops_its_process() + { + var before = RunningKeys().ToHashSet(); + var service = Service(); + await service.StartAsync(AdapterId, "corr-5"); + var mine = RunningKeys().Except(before).Single(); + StringAssert.StartsWith(mine, "classic-"); + + ((IDisposable)service).Dispose(); + + CollectionAssert.DoesNotContain(RunningKeys().ToList(), mine); + } + + [TestMethod] + public async Task Without_the_resident_host_registered_the_error_says_what_to_register() + { + using var bare = new ServiceCollection() + .AddLogging(l => l.ClearProviders()) + // The same storage, so the package and its metadata are the ones the host published. + .AddSingleton(host.Services.GetRequiredService()) + .AddServerless(o => + { + o.AdapterRemotePath = "adapters"; + o.AdapterLocalPath = Path.Combine(workDirectory, "installed-bare"); + }) + .BuildServiceProvider(); + + var error = await Assert.ThrowsExceptionAsync(() => + bare.GetRequiredService().StartAsync(AdapterId, "corr-6")); + StringAssert.Contains(error.Message, "AddResidentAdapters"); + } + } +} diff --git a/SW.Serverless.UnitTests/SW.Serverless.UnitTests.csproj b/SW.Serverless.UnitTests/SW.Serverless.UnitTests.csproj index 583f3b4..a8e183c 100644 --- a/SW.Serverless.UnitTests/SW.Serverless.UnitTests.csproj +++ b/SW.Serverless.UnitTests/SW.Serverless.UnitTests.csproj @@ -28,6 +28,8 @@ + /// Additive overload for hosts that never install adapters from cloud storage — the /// StartAsync(adapterId, correlationId, adapterPath, ...) path needs no ICloudFilesService. @@ -53,7 +60,9 @@ public ServerlessService(ServerlessOptions serverlessOptions, IMemoryCache memor this.memoryCache = memoryCache; this.loggerFactory = loggerFactory; this.cloudFilesService = cloudFilesService; - installer = new AdapterInstaller(serverlessOptions, memoryCache, cloudFilesService); + this.serviceProvider = serviceProvider; + installer = new AdapterInstaller(serverlessOptions, memoryCache, cloudFilesService, + serviceProvider?.GetService()); logger = loggerFactory.CreateLogger(); @@ -73,6 +82,16 @@ async public Task StartAsync(string adapterId, string correlationId, IDictionary } var adapterMetadata = await Install(adapterId); + + if (adapterMetadata.UsesGrpc) + { + if (processStarted || grpc != null) + throw new Exception("Already started."); + grpc = await GrpcSession.StartAsync(serviceProvider, adapterId, correlationId, startupValues, + adapterMetadata.AdapterValues); + return; + } + // Held while this process runs, so publishing a newer version can't prune its files. directoryLease = AdapterDirectoryLeases.Hold(adapterMetadata.Directory); @@ -150,11 +169,17 @@ Task StartAsync(string adapterId, AdapterMetadata adapterMetadata, string correl public Task> GetExpectedStartupValues() { + if (grpc != null) return Task.FromResult(grpc.ExpectedStartupValues()); return InvokeAsync>(Constants.ExpectedCommand, null); } async public Task InvokeAsync(string command, object input, int commandTimeout = 0) { + if (grpc != null) + { + await grpc.InvokeAsync(command, input, commandTimeout == 0 ? serverlessOptions.CommandTimeout : commandTimeout); + return; + } await InvokeAsync(command, input, commandTimeout); } @@ -167,6 +192,9 @@ async public Task InvokeAsync(string command, object input, in throw new ArgumentException("Invalid name.", nameof(command)); } + if (grpc != null) + return await grpc.InvokeAsync(command, input, commandTimeout); + if (!processStarted || process.HasExited || timedOut) throw new Exception("Process not started or terminated."); @@ -318,12 +346,36 @@ async Task Install(string adapterId) EntryAssembly = installed.EntryAssembly, LocalPath = installed.LocalPath, Directory = installed.Directory, - AdapterValues = installed.AdapterValues + AdapterValues = installed.AdapterValues, + UsesGrpc = UsesGrpc(installed) }; } + /// + /// Whether a classic call to this adapter goes over gRPC: always for a runtime other than + /// .NET, whose SDKs speak only that, and for a .NET adapter whose manifest opts in with a + /// protocol of 2 or more. Every other .NET adapter keeps the text protocol it was built for. + /// + internal static bool UsesGrpc(InstalledAdapter installed) => + !string.Equals(installed.Runtime, Contract.Catalog.AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase) || + installed.Manifest?.Protocol is { Min: >= 2 }; + public void Dispose() { + if (grpc != null) + { + try + { + grpc.StopAsync().GetAwaiter().GetResult(); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Service did not dispose properly."); + } + grpc = null; + return; + } + try { if (processStarted) @@ -363,6 +415,74 @@ private class AdapterMetadata public string LocalPath { get; set; } public string Directory { get; set; } public IDictionary AdapterValues { get; set; } = new Dictionary(); + public bool UsesGrpc { get; set; } + } + + /// + /// A classic session over gRPC: a resident instance under a key of its own, started for + /// the session and stopped when it ends, called one command at a time as the text protocol + /// is. The resident host does the launching, handshake, timeouts and logs. + /// + private sealed class GrpcSession + { + readonly Resident.IResidentAdapterHost host; + readonly string adapterId; + readonly string instanceKey; + readonly Resident.ResidentAdapterInstance instance; + + GrpcSession(Resident.IResidentAdapterHost host, string adapterId, string instanceKey, + Resident.ResidentAdapterInstance instance) + { + this.host = host; + this.adapterId = adapterId; + this.instanceKey = instanceKey; + this.instance = instance; + } + + public static async Task StartAsync(IServiceProvider services, string adapterId, + string correlationId, IDictionary startupValues, IDictionary adapterValues) + { + var host = services?.GetService() + ?? throw new InvalidOperationException( + $"Adapter '{adapterId}' speaks the gRPC protocol, which runs on the resident adapter host. " + + "Register it with AddResidentAdapters."); + + var values = startupValues == null + ? new Dictionary() + : new Dictionary(startupValues); + values[Constants.CorrelationIdName] = correlationId; + + var instanceKey = $"classic-{Guid.NewGuid():N}"; + var instance = await host.StartExclusiveAsync(new Resident.AdapterSpec + { + AdapterId = adapterId, + InstanceKey = instanceKey, + StartupValues = values, + AdapterValues = adapterValues ?? new Dictionary(), + }); + return new GrpcSession(host, adapterId, instanceKey, instance); + } + + public Task InvokeAsync(string command, object input, int timeoutSeconds) => + instance.InvokeAsync(command, input, timeoutSeconds); + + public Task InvokeAsync(string command, object input, int timeoutSeconds) => + instance.InvokeAsync(command, input, timeoutSeconds); + + /// The settings it declared in its handshake, as the classic protocol has always described them. + public IDictionary ExpectedStartupValues() => + instance.Settings.ToDictionary( + s => s.Name, + s => new StartupValue + { + Optional = !s.Required, + Default = string.IsNullOrEmpty(s.DefaultValue) ? null : s.DefaultValue, + Type = s.Type, + Private = s.Secret, + Description = s.Description, + }); + + public Task StopAsync() => host.StopAsync(adapterId, instanceKey, drain: false); } } } From f93bb1faad18c14c95468216e66d4804b73e25e4 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 09:55:47 +0300 Subject: [PATCH 06/14] Send classic adapters their startup values on stdin, off the command line Startup values hold passwords and keys, and the classic protocol passed them as base64 arguments, which any process on the machine can read. The SDK now also accepts --values-on-stdin and reads the three values from stdin before the first command. The host sends them that way only to an adapter whose manifest records SDK 10.1.0 or later; every other adapter, including every package from before manifests, gets them as arguments exactly as before. HostInfo's baseline is 10.1.0, the release these host features ship in. StartupValuesOnStdinTests runs one adapter both ways and reads the machine's process list while it runs: built on the newer SDK, its command line is dotnet, the assembly and the flag, nothing more, and the password still reaches it; from before, it still gets the three base64 values as arguments. --- SW.Serverless.Sdk/Constants.cs | 11 ++ SW.Serverless.Sdk/Runner.cs | 12 ++ SW.Serverless.UnitTests.Adapter/Handler.cs | 2 + .../StartupValuesOnStdinTests.cs | 142 ++++++++++++++++++ SW.Serverless/Services/HostInfo.cs | 2 +- SW.Serverless/Services/ServerlessService.cs | 26 +++- 6 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 SW.Serverless.UnitTests/StartupValuesOnStdinTests.cs diff --git a/SW.Serverless.Sdk/Constants.cs b/SW.Serverless.Sdk/Constants.cs index 2a51eb5..8994cdc 100644 --- a/SW.Serverless.Sdk/Constants.cs +++ b/SW.Serverless.Sdk/Constants.cs @@ -14,5 +14,16 @@ public static class Constants public const string LogErrorIdentifier = "{{log.error}}"; public const string CorrelationIdName = "CorrelationId"; + /// + /// Passed instead of the three base64 arguments when the host sends them on stdin: the + /// options, the startup values and the adapter values, one line each, before the first + /// command. Startup values hold passwords and keys, and anything on the command line can + /// be read by every process on the machine. + /// + public const string ValuesOnStdinFlag = "--values-on-stdin"; + + /// The first SDK release that reads . + public const string ValuesOnStdinSince = "10.1.0"; + } } diff --git a/SW.Serverless.Sdk/Runner.cs b/SW.Serverless.Sdk/Runner.cs index 1f54d69..dcaeb5c 100644 --- a/SW.Serverless.Sdk/Runner.cs +++ b/SW.Serverless.Sdk/Runner.cs @@ -81,6 +81,18 @@ async public static Task Run(object commandHandler) Console.OutputEncoding = Encoding.UTF8; var commandLineArgs = Environment.GetCommandLineArgs(); + // A host that knows this SDK sends the values on stdin, off the command line where + // any process on the machine could read them. Older hosts still pass them as + // arguments, which is still understood. + if (commandLineArgs.Length > 1 && commandLineArgs[1] == Constants.ValuesOnStdinFlag) + commandLineArgs = new[] + { + commandLineArgs[0], + await Console.In.ReadLineAsync(), + await Console.In.ReadLineAsync(), + await Console.In.ReadLineAsync(), + }; + try { ServerlessOptions = diff --git a/SW.Serverless.UnitTests.Adapter/Handler.cs b/SW.Serverless.UnitTests.Adapter/Handler.cs index 6f08b33..92f46b7 100644 --- a/SW.Serverless.UnitTests.Adapter/Handler.cs +++ b/SW.Serverless.UnitTests.Adapter/Handler.cs @@ -39,6 +39,8 @@ public Task TestTypedReturn() }); } + public Task TestStartupValue(string name) => Task.FromResult(Runner.StartupValueOf(name)); + public Task TestString(string value) { return Task.FromResult((object)value); diff --git a/SW.Serverless.UnitTests/StartupValuesOnStdinTests.cs b/SW.Serverless.UnitTests/StartupValuesOnStdinTests.cs new file mode 100644 index 0000000..ec563f7 --- /dev/null +++ b/SW.Serverless.UnitTests/StartupValuesOnStdinTests.cs @@ -0,0 +1,142 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; +using SW.CloudFiles.Extensions; +using SW.PrimitiveTypes; +using SW.Serverless.UnitTests.Fixtures; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; + +namespace SW.Serverless.UnitTests +{ + /// + /// A classic adapter's startup values hold passwords and keys. On the command line, as they + /// have always gone, any process on the machine can read them. An adapter built on an SDK + /// that reads them from stdin gets them there; one built before still gets them as arguments. + /// + [TestClass] + public class StartupValuesOnStdinTests + { + const string Project = "SW.Serverless.UnitTests.Adapter"; + const string Secret = "s3cret-on-stdin-only"; + + static IHost host; + static string workDirectory; + + [ClassInitialize] + public static async Task ClassInitialize(TestContext context) + { + workDirectory = Path.Combine(Path.GetTempPath(), "swsl-stdin", Guid.NewGuid().ToString("N")); + host = Host.CreateDefaultBuilder() + .ConfigureLogging(l => l.ClearProviders()) + .ConfigureServices(s => + { + s.AddLocalTestsCloudFiles(o => o.BucketName = TestStore.BucketName + "-stdin"); + s.AddServerless(o => + { + o.AdapterRemotePath = "adapters"; + o.AdapterLocalPath = Path.Combine(workDirectory, "installed"); + o.AdapterMetadataCacheDuration = 1; + o.CommandTimeout = 30; + }); + }) + .Build(); + await host.StartAsync(); + + await PublishAsync("test.stdin.new", sdkVersion: "10.1.0"); + await PublishAsync("test.stdin.old", sdkVersion: null); + } + + static async Task PublishAsync(string adapterId, string sdkVersion) + { + var output = TestStore.LocateBuildOutput(Project); + using var buffer = new MemoryStream(); + using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var file in Directory.EnumerateFiles(output, "*", SearchOption.AllDirectories)) + archive.CreateEntryFromFile(file, Path.GetRelativePath(output, file), CompressionLevel.Fastest); + if (sdkVersion != null) + using (var writer = new StreamWriter(archive.CreateEntry("adapter.json").Open())) + writer.Write(new JObject { ["id"] = adapterId, ["entry"] = Project + ".dll", ["sdkVersion"] = sdkVersion }.ToString()); + } + buffer.Position = 0; + await host.Services.GetRequiredService().WriteAsync(buffer, new WriteFileSettings + { + Key = $"adapters/{adapterId}", + ContentType = "application/zip", + Metadata = new Dictionary + { + ["EntryAssembly"] = Project + ".dll", + ["Hash"] = adapterId.Replace('.', '-') + "-" + Guid.NewGuid().ToString("N")[..8], + } + }); + } + + [ClassCleanup] + public static async Task ClassCleanup() + { + if (host != null) await host.StopAsync(); + host?.Dispose(); + try { Directory.Delete(workDirectory, true); } catch { } + } + + /// The command lines of this machine's running adapter processes, as any process can read them. + static List AdapterCommandLines() + { + using var ps = Process.Start(new ProcessStartInfo("ps", "-axo args") { RedirectStandardOutput = true })!; + var lines = ps.StandardOutput.ReadToEnd().Split('\n'); + ps.WaitForExit(); + return lines.Where(l => l.Contains(Project + ".dll") && l.Contains(workDirectory)).ToList(); + } + + static async Task<(string Password, List CommandLines)> RunAsync(string adapterId) + { + var service = host.Services.GetRequiredService(); + await service.StartAsync(adapterId, "corr", new Dictionary { ["Password"] = Secret }); + try + { + var password = await service.InvokeAsync("TestStartupValue", "Password"); + return (password, AdapterCommandLines()); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + + [TestMethod] + public async Task An_adapter_on_the_newer_SDK_gets_its_values_on_stdin_and_none_on_its_command_line() + { + if (OperatingSystem.IsWindows()) Assert.Inconclusive("reads command lines with ps"); + + var (password, commandLines) = await RunAsync("test.stdin.new"); + + Assert.AreEqual(Secret, password, "the value reached the adapter"); + var line = commandLines.Single(); + StringAssert.Contains(line, "--values-on-stdin"); + Assert.AreEqual(3, line.Split(' ', StringSplitOptions.RemoveEmptyEntries).Length, + $"nothing but dotnet, the assembly and the flag on the command line: {line}"); + } + + [TestMethod] + public async Task An_adapter_from_before_still_gets_its_values_as_arguments() + { + if (OperatingSystem.IsWindows()) Assert.Inconclusive("reads command lines with ps"); + + var (password, commandLines) = await RunAsync("test.stdin.old"); + + Assert.AreEqual(Secret, password); + var line = commandLines.Single(); + Assert.IsFalse(line.Contains("--values-on-stdin"), line); + Assert.AreEqual(5, line.Split(' ', StringSplitOptions.RemoveEmptyEntries).Length, + $"dotnet, the assembly and the three base64 values, as before: {line}"); + } + } +} diff --git a/SW.Serverless/Services/HostInfo.cs b/SW.Serverless/Services/HostInfo.cs index 5a8412a..9f9da0b 100644 --- a/SW.Serverless/Services/HostInfo.cs +++ b/SW.Serverless/Services/HostInfo.cs @@ -12,7 +12,7 @@ public static class HostInfo /// assembly cannot say which release it is. Raise it with every release that adds something /// an adapter could depend on. /// - public const string Baseline = "10.0.2"; + public const string Baseline = "10.1.0"; public static Version Version { get; } = Resolve(); diff --git a/SW.Serverless/Services/ServerlessService.cs b/SW.Serverless/Services/ServerlessService.cs index 9090002..c0b2625 100644 --- a/SW.Serverless/Services/ServerlessService.cs +++ b/SW.Serverless/Services/ServerlessService.cs @@ -135,11 +135,15 @@ Task StartAsync(string adapterId, AdapterMetadata adapterMetadata, string correl var serverlessOptionsBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(serverlessOptions))); var adapterValuesBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(adapterMetadata.AdapterValues))); + // On stdin for an adapter whose SDK reads them there, as arguments for every other. + var valuesOnStdin = adapterMetadata.ValuesOnStdin; process = new Process { StartInfo = new ProcessStartInfo("dotnet") { - Arguments = $"\"{adapterMetadata.LocalPath}\" {serverlessOptionsBase64} {startupValuesBase64} {adapterValuesBase64}", + Arguments = valuesOnStdin + ? $"\"{adapterMetadata.LocalPath}\" {Constants.ValuesOnStdinFlag}" + : $"\"{adapterMetadata.LocalPath}\" {serverlessOptionsBase64} {startupValuesBase64} {adapterValuesBase64}", WorkingDirectory = Path.GetDirectoryName(adapterMetadata.LocalPath), UseShellExecute = false, @@ -164,6 +168,14 @@ Task StartAsync(string adapterId, AdapterMetadata adapterMetadata, string correl process.BeginOutputReadLine(); process.BeginErrorReadLine(); + if (valuesOnStdin) + { + process.StandardInput.WriteLine(serverlessOptionsBase64); + process.StandardInput.WriteLine(startupValuesBase64); + process.StandardInput.WriteLine(adapterValuesBase64); + process.StandardInput.Flush(); + } + return Task.CompletedTask; } @@ -347,7 +359,8 @@ async Task Install(string adapterId) LocalPath = installed.LocalPath, Directory = installed.Directory, AdapterValues = installed.AdapterValues, - UsesGrpc = UsesGrpc(installed) + UsesGrpc = UsesGrpc(installed), + ValuesOnStdin = ReadsValuesOnStdin(installed) }; } @@ -356,6 +369,14 @@ async Task Install(string adapterId) /// .NET, whose SDKs speak only that, and for a .NET adapter whose manifest opts in with a /// protocol of 2 or more. Every other .NET adapter keeps the text protocol it was built for. /// + /// + /// Whether the adapter's SDK reads its values from stdin: known only from the SDK version + /// its manifest records. A package without one is from before, and gets them as arguments. + /// + internal static bool ReadsValuesOnStdin(InstalledAdapter installed) => + Version.TryParse((installed.Manifest?.SdkVersion ?? "").Split('-', '+')[0], out var sdk) && + sdk >= Version.Parse(Constants.ValuesOnStdinSince); + internal static bool UsesGrpc(InstalledAdapter installed) => !string.Equals(installed.Runtime, Contract.Catalog.AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase) || installed.Manifest?.Protocol is { Min: >= 2 }; @@ -416,6 +437,7 @@ private class AdapterMetadata public string Directory { get; set; } public IDictionary AdapterValues { get; set; } = new Dictionary(); public bool UsesGrpc { get; set; } + public bool ValuesOnStdin { get; set; } } /// From 754e8b44f23ef98d34394ec9f1253393601bcfee Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 10:22:17 +0300 Subject: [PATCH 07/14] Move what the serverless CLI does into a library SW.Serverless.Tooling (package SimplyWorks.Serverless.Tooling) now holds the describing, manifest-building, probing, repository and publishing logic the CLI ran, so Bitween's server, a future code editor or anything else calls the same code rather than shelling out to the CLI. The serverless project is the command line over it. Nothing behaves differently: the same commands, flags and output, and the installer and compatibility suites pass unchanged apart from the namespace (SW.Serverless.Tooling). --- SW.Serverless.CompatibilityTests/Compat.cs | 2 +- .../ListingAndManifestTests.cs | 2 +- .../PackageLayoutTests.cs | 2 +- SW.Serverless.Desktop/MainWindow.xaml.cs | 2 +- .../AdapterDescriberTests.cs | 2 +- .../CatalogPublishingTests.cs | 2 +- .../InstallerLogicTests.cs | 2 +- .../ObjectStore.cs | 2 +- .../SemverTests.cs | 2 +- SW.Serverless.Installer/Options.cs | 44 ----------------- SW.Serverless.Installer/Program.cs | 2 +- .../SW.Serverless.Installer.csproj | 12 +---- .../AdapterDescriber.cs | 2 +- .../AdapterRepository.cs | 2 +- .../CloudFilesFactory.cs | 3 +- .../ExpectedValuesProbe.cs | 2 +- .../InstallerLogic.cs | 2 +- .../ManifestBuilder.cs | 2 +- .../PackagePublisher.cs | 2 +- .../SW.Serverless.Tooling.csproj | 37 ++++++++++++++ .../Semver.cs | 2 +- .../ServerlessUploadOptions.cs | 48 +++++++++++++++++++ SW.Serverless.sln | 14 ++++++ 23 files changed, 120 insertions(+), 72 deletions(-) rename {SW.Serverless.Installer/Shared => SW.Serverless.Tooling}/AdapterDescriber.cs (99%) rename {SW.Serverless.Installer/Shared => SW.Serverless.Tooling}/AdapterRepository.cs (99%) rename {SW.Serverless.Installer/Shared => SW.Serverless.Tooling}/CloudFilesFactory.cs (98%) rename {SW.Serverless.Installer/Shared => SW.Serverless.Tooling}/ExpectedValuesProbe.cs (99%) rename {SW.Serverless.Installer/Shared => SW.Serverless.Tooling}/InstallerLogic.cs (99%) rename {SW.Serverless.Installer/Shared => SW.Serverless.Tooling}/ManifestBuilder.cs (99%) rename {SW.Serverless.Installer/Shared => SW.Serverless.Tooling}/PackagePublisher.cs (99%) create mode 100644 SW.Serverless.Tooling/SW.Serverless.Tooling.csproj rename {SW.Serverless.Installer/Shared => SW.Serverless.Tooling}/Semver.cs (98%) create mode 100644 SW.Serverless.Tooling/ServerlessUploadOptions.cs diff --git a/SW.Serverless.CompatibilityTests/Compat.cs b/SW.Serverless.CompatibilityTests/Compat.cs index dc3e54e..f6c9640 100644 --- a/SW.Serverless.CompatibilityTests/Compat.cs +++ b/SW.Serverless.CompatibilityTests/Compat.cs @@ -8,7 +8,7 @@ using SW.CloudFiles.Extensions; using SW.PrimitiveTypes; using SW.Serverless.Installer; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; namespace SW.Serverless.CompatibilityTests; diff --git a/SW.Serverless.CompatibilityTests/ListingAndManifestTests.cs b/SW.Serverless.CompatibilityTests/ListingAndManifestTests.cs index bf5aa0c..edd35ae 100644 --- a/SW.Serverless.CompatibilityTests/ListingAndManifestTests.cs +++ b/SW.Serverless.CompatibilityTests/ListingAndManifestTests.cs @@ -3,7 +3,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using SW.PrimitiveTypes; using SW.Serverless.Contract.Catalog; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; namespace SW.Serverless.CompatibilityTests; diff --git a/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs b/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs index 5efc697..e3d5815 100644 --- a/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs +++ b/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs @@ -4,7 +4,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using SW.PrimitiveTypes; using SW.Serverless.Contract.Catalog; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; namespace SW.Serverless.CompatibilityTests; diff --git a/SW.Serverless.Desktop/MainWindow.xaml.cs b/SW.Serverless.Desktop/MainWindow.xaml.cs index ffa21ed..c79eaa1 100644 --- a/SW.Serverless.Desktop/MainWindow.xaml.cs +++ b/SW.Serverless.Desktop/MainWindow.xaml.cs @@ -1,6 +1,6 @@ using Microsoft.Win32; using Newtonsoft.Json; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; using System; using System.Collections.Generic; using System.IO; diff --git a/SW.Serverless.Installer.UnitTests/AdapterDescriberTests.cs b/SW.Serverless.Installer.UnitTests/AdapterDescriberTests.cs index c51c20a..96e3832 100644 --- a/SW.Serverless.Installer.UnitTests/AdapterDescriberTests.cs +++ b/SW.Serverless.Installer.UnitTests/AdapterDescriberTests.cs @@ -2,7 +2,7 @@ using System.IO; using System.Linq; using Microsoft.VisualStudio.TestTools.UnitTesting; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; namespace SW.Serverless.Installer.UnitTests; diff --git a/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs b/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs index 3902b29..35bd4b6 100644 --- a/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs +++ b/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs @@ -7,7 +7,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using SW.PrimitiveTypes; using SW.Serverless.Contract.Catalog; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; namespace SW.Serverless.Installer.UnitTests; diff --git a/SW.Serverless.Installer.UnitTests/InstallerLogicTests.cs b/SW.Serverless.Installer.UnitTests/InstallerLogicTests.cs index 470fdc2..382ab96 100644 --- a/SW.Serverless.Installer.UnitTests/InstallerLogicTests.cs +++ b/SW.Serverless.Installer.UnitTests/InstallerLogicTests.cs @@ -5,7 +5,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.VisualStudio.TestTools.UnitTesting; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; namespace SW.Serverless.Installer.UnitTests; diff --git a/SW.Serverless.Installer.UnitTests/ObjectStore.cs b/SW.Serverless.Installer.UnitTests/ObjectStore.cs index 346944f..b7e2336 100644 --- a/SW.Serverless.Installer.UnitTests/ObjectStore.cs +++ b/SW.Serverless.Installer.UnitTests/ObjectStore.cs @@ -28,7 +28,7 @@ record Item(byte[] Content, Dictionary Metadata); public string Register() { var name = "objectstore-" + Guid.NewGuid().ToString("N"); - Shared.CloudFilesFactory.Register(name, _ => this); + SW.Serverless.Tooling.CloudFilesFactory.Register(name, _ => this); return name; } diff --git a/SW.Serverless.Installer.UnitTests/SemverTests.cs b/SW.Serverless.Installer.UnitTests/SemverTests.cs index 5c8581d..65159a9 100644 --- a/SW.Serverless.Installer.UnitTests/SemverTests.cs +++ b/SW.Serverless.Installer.UnitTests/SemverTests.cs @@ -1,5 +1,5 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; namespace SW.Serverless.UnitTests; diff --git a/SW.Serverless.Installer/Options.cs b/SW.Serverless.Installer/Options.cs index 02e99dd..4f533de 100644 --- a/SW.Serverless.Installer/Options.cs +++ b/SW.Serverless.Installer/Options.cs @@ -103,48 +103,4 @@ public class WithdrawCliOptions : StorageCliOptions public string Version { get; set; } } - public class ServerlessUploadOptions - { - public string Provider { get; set; } - - public string AccessKeyId { get; set; } - - public string SecretAccessKey { get; set; } - - public string BucketName { get; set; } - - public string ServiceUrl { get; set; } - - - public string RSAKey { get; set; } - - public string UserId { get; set; } - - public string FingerPrint { get; set; } - - public string TenantId { get; set; } - - public string Region { get; set; } - - - public string Version { get; set; } - - - public string AdapterId { get; set; } - public string NamespaceName { get; set; } - - // ——— Google Cloud service account ——— - public string ProjectId { get; set; } - public string PrivateKeyId { get; set; } - public string PrivateKey { get; set; } - public string ClientEmail { get; set; } - public string ClientId { get; set; } - public string ClientX509CertUrl { get; set; } - - /// - /// Roles this adapter serves, comma separated. Overrides what the assembly declared, for - /// an adapter whose author has not added [AdapterKind] to it. - /// - public string Kind { get; set; } - } } diff --git a/SW.Serverless.Installer/Program.cs b/SW.Serverless.Installer/Program.cs index 95d4f48..174a814 100644 --- a/SW.Serverless.Installer/Program.cs +++ b/SW.Serverless.Installer/Program.cs @@ -1,7 +1,7 @@ using CommandLine; using SW.PrimitiveTypes; using SW.Serverless.Contract.Catalog; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; using System; using System.Collections.Generic; using System.IO; diff --git a/SW.Serverless.Installer/SW.Serverless.Installer.csproj b/SW.Serverless.Installer/SW.Serverless.Installer.csproj index 5ba43b1..3f95b76 100644 --- a/SW.Serverless.Installer/SW.Serverless.Installer.csproj +++ b/SW.Serverless.Installer/SW.Serverless.Installer.csproj @@ -9,16 +9,6 @@ - - - - - - - - - - @@ -26,6 +16,8 @@ + + diff --git a/SW.Serverless.Installer/Shared/AdapterDescriber.cs b/SW.Serverless.Tooling/AdapterDescriber.cs similarity index 99% rename from SW.Serverless.Installer/Shared/AdapterDescriber.cs rename to SW.Serverless.Tooling/AdapterDescriber.cs index 4cb621d..8c5394e 100644 --- a/SW.Serverless.Installer/Shared/AdapterDescriber.cs +++ b/SW.Serverless.Tooling/AdapterDescriber.cs @@ -4,7 +4,7 @@ using System.Linq; using System.Reflection; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { /// /// What an adapter IS, read from the assembly at publish time. diff --git a/SW.Serverless.Installer/Shared/AdapterRepository.cs b/SW.Serverless.Tooling/AdapterRepository.cs similarity index 99% rename from SW.Serverless.Installer/Shared/AdapterRepository.cs rename to SW.Serverless.Tooling/AdapterRepository.cs index 450e85a..c4cef5d 100644 --- a/SW.Serverless.Installer/Shared/AdapterRepository.cs +++ b/SW.Serverless.Tooling/AdapterRepository.cs @@ -8,7 +8,7 @@ using System.Linq; using System.Threading.Tasks; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { /// What goes into storage for one package, beside the zip itself. public class PackageInfo diff --git a/SW.Serverless.Installer/Shared/CloudFilesFactory.cs b/SW.Serverless.Tooling/CloudFilesFactory.cs similarity index 98% rename from SW.Serverless.Installer/Shared/CloudFilesFactory.cs rename to SW.Serverless.Tooling/CloudFilesFactory.cs index f3cdca3..910014f 100644 --- a/SW.Serverless.Installer/Shared/CloudFilesFactory.cs +++ b/SW.Serverless.Tooling/CloudFilesFactory.cs @@ -8,8 +8,9 @@ // GC, OC, S3 and LocalTests all publish their registration extensions into this one namespace. using SW.CloudFiles.Extensions; using SW.PrimitiveTypes; +using SW.Serverless.Installer; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { /// /// Builds the storage provider an adapter is published to. diff --git a/SW.Serverless.Installer/Shared/ExpectedValuesProbe.cs b/SW.Serverless.Tooling/ExpectedValuesProbe.cs similarity index 99% rename from SW.Serverless.Installer/Shared/ExpectedValuesProbe.cs rename to SW.Serverless.Tooling/ExpectedValuesProbe.cs index 5b3d47b..a835a2c 100644 --- a/SW.Serverless.Installer/Shared/ExpectedValuesProbe.cs +++ b/SW.Serverless.Tooling/ExpectedValuesProbe.cs @@ -8,7 +8,7 @@ using System.Text; using System.Threading.Tasks; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { /// /// Asks a built CLASSIC adapter which startup values it expects, the way every host already diff --git a/SW.Serverless.Installer/Shared/InstallerLogic.cs b/SW.Serverless.Tooling/InstallerLogic.cs similarity index 99% rename from SW.Serverless.Installer/Shared/InstallerLogic.cs rename to SW.Serverless.Tooling/InstallerLogic.cs index a741082..c720733 100644 --- a/SW.Serverless.Installer/Shared/InstallerLogic.cs +++ b/SW.Serverless.Tooling/InstallerLogic.cs @@ -10,7 +10,7 @@ using System.Threading.Tasks; using System.Xml.Linq; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { public class InstallerLogic { diff --git a/SW.Serverless.Installer/Shared/ManifestBuilder.cs b/SW.Serverless.Tooling/ManifestBuilder.cs similarity index 99% rename from SW.Serverless.Installer/Shared/ManifestBuilder.cs rename to SW.Serverless.Tooling/ManifestBuilder.cs index 540d577..456667c 100644 --- a/SW.Serverless.Installer/Shared/ManifestBuilder.cs +++ b/SW.Serverless.Tooling/ManifestBuilder.cs @@ -8,7 +8,7 @@ using System.Reflection; using System.Text.Json; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { /// What the installer knows about a build, and fills into the manifest over whatever the author wrote. public class GeneratedManifestFields diff --git a/SW.Serverless.Installer/Shared/PackagePublisher.cs b/SW.Serverless.Tooling/PackagePublisher.cs similarity index 99% rename from SW.Serverless.Installer/Shared/PackagePublisher.cs rename to SW.Serverless.Tooling/PackagePublisher.cs index fdb547a..d698548 100644 --- a/SW.Serverless.Installer/Shared/PackagePublisher.cs +++ b/SW.Serverless.Tooling/PackagePublisher.cs @@ -4,7 +4,7 @@ using System.IO; using System.Threading.Tasks; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { public class PublishRequest { diff --git a/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj new file mode 100644 index 0000000..391b979 --- /dev/null +++ b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj @@ -0,0 +1,37 @@ + + + + + net10.0 + latest + SimplyWorks.Serverless.Tooling + true + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SW.Serverless.Installer/Shared/Semver.cs b/SW.Serverless.Tooling/Semver.cs similarity index 98% rename from SW.Serverless.Installer/Shared/Semver.cs rename to SW.Serverless.Tooling/Semver.cs index 01b1772..feb526f 100644 --- a/SW.Serverless.Installer/Shared/Semver.cs +++ b/SW.Serverless.Tooling/Semver.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Text.RegularExpressions; -namespace SW.Serverless.Installer.Shared; +namespace SW.Serverless.Tooling; public static class Semver { diff --git a/SW.Serverless.Tooling/ServerlessUploadOptions.cs b/SW.Serverless.Tooling/ServerlessUploadOptions.cs new file mode 100644 index 0000000..f1989ed --- /dev/null +++ b/SW.Serverless.Tooling/ServerlessUploadOptions.cs @@ -0,0 +1,48 @@ +namespace SW.Serverless.Installer +{ + /// Where adapters are published: the storage provider and how to reach it. + public class ServerlessUploadOptions + { + public string Provider { get; set; } + + public string AccessKeyId { get; set; } + + public string SecretAccessKey { get; set; } + + public string BucketName { get; set; } + + public string ServiceUrl { get; set; } + + + public string RSAKey { get; set; } + + public string UserId { get; set; } + + public string FingerPrint { get; set; } + + public string TenantId { get; set; } + + public string Region { get; set; } + + + public string Version { get; set; } + + + public string AdapterId { get; set; } + public string NamespaceName { get; set; } + + // ——— Google Cloud service account ——— + public string ProjectId { get; set; } + public string PrivateKeyId { get; set; } + public string PrivateKey { get; set; } + public string ClientEmail { get; set; } + public string ClientId { get; set; } + public string ClientX509CertUrl { get; set; } + + /// + /// Roles this adapter serves, comma separated. Overrides what the assembly declared, for + /// an adapter whose author has not added [AdapterKind] to it. + /// + public string Kind { get; set; } + } +} diff --git a/SW.Serverless.sln b/SW.Serverless.sln index 74e3f56..dc01398 100644 --- a/SW.Serverless.sln +++ b/SW.Serverless.sln @@ -59,6 +59,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.Compat.Manife EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.UnitTests.GrpcClassicAdapter", "SW.Serverless.UnitTests.GrpcClassicAdapter\SW.Serverless.UnitTests.GrpcClassicAdapter.csproj", "{D77684BD-D24F-45AE-9B49-30278D6CCA3C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.Tooling", "SW.Serverless.Tooling\SW.Serverless.Tooling.csproj", "{C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -369,6 +371,18 @@ Global {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Release|x64.Build.0 = Release|Any CPU {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Release|x86.ActiveCfg = Release|Any CPU {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Release|x86.Build.0 = Release|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Debug|x64.ActiveCfg = Debug|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Debug|x64.Build.0 = Debug|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Debug|x86.ActiveCfg = Debug|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Debug|x86.Build.0 = Debug|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|Any CPU.Build.0 = Release|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|x64.ActiveCfg = Release|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|x64.Build.0 = Release|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|x86.ActiveCfg = Release|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From fe154d00399109d6c5ec99df5d283b990a82014b Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 10:26:10 +0300 Subject: [PATCH 08/14] Let an adapter describe itself with --describe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Started with --describe, an adapter prints an AdapterSelfDescription — its SDK language and version, lifecycle and protocol, the settings it declares, its commands with JSON Schemas of their argument and result, its kinds and the contracts it implements — and exits without any host. The document is defined in SW.Serverless.Contract for SDKs in every language to produce, so the tools learn about an adapter by running it rather than by reading .NET assemblies. A handler that can't be built without its settings is still described, with a warning saying what may be missing. The .NET SDK answers it from both runners, takes kinds from [AdapterKind] and the new [AdapterContract(name, version)], and sends kinds and contracts in its handshake too. It reports its own version from SdkInfo — the release, 10.1.0 — rather than the 1.0.0.0 the build stamps on every assembly. DescribeTests runs a classic and a resident sample and a contract-declaring adapter with --describe: settings with secrets and their defaults withheld, commands with schemas whose properties are named as they travel, lifecycle methods left out, kinds and contracts, and a round trip through the parser. GrpcClassicTests checks the handshake carries language, kinds, contracts and version. --- .../Catalog/AdapterSelfDescription.cs | 80 +++++++++++ SW.Serverless.Sdk/AdapterContractAttribute.cs | 21 +++ SW.Serverless.Sdk/Describer.cs | 130 ++++++++++++++++++ SW.Serverless.Sdk/Resident/ResidentRunner.cs | 20 ++- SW.Serverless.Sdk/Runner.cs | 20 +++ SW.Serverless.Sdk/SdkInfo.cs | 27 ++++ .../Program.cs | 2 + SW.Serverless.UnitTests/DescribeTests.cs | 102 ++++++++++++++ SW.Serverless.UnitTests/GrpcClassicTests.cs | 21 +++ 9 files changed, 422 insertions(+), 1 deletion(-) create mode 100644 SW.Serverless.Contract/Catalog/AdapterSelfDescription.cs create mode 100644 SW.Serverless.Sdk/AdapterContractAttribute.cs create mode 100644 SW.Serverless.Sdk/Describer.cs create mode 100644 SW.Serverless.Sdk/SdkInfo.cs create mode 100644 SW.Serverless.UnitTests/DescribeTests.cs diff --git a/SW.Serverless.Contract/Catalog/AdapterSelfDescription.cs b/SW.Serverless.Contract/Catalog/AdapterSelfDescription.cs new file mode 100644 index 0000000..b7f09cc --- /dev/null +++ b/SW.Serverless.Contract/Catalog/AdapterSelfDescription.cs @@ -0,0 +1,80 @@ +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace SW.Serverless.Contract.Catalog +{ + /// + /// What an adapter says about itself when started with : printed as one JSON + /// document on stdout, after which it exits. Every SDK, in every language, produces this, so the + /// tools learn an adapter's settings, commands, kinds and contracts by running it — not by + /// reading .NET assemblies — and write them into its manifest. + /// + public class AdapterSelfDescription + { + public const string Flag = "--describe"; + public const int CurrentDescribeVersion = 1; + + public int DescribeVersion { get; set; } = CurrentDescribeVersion; + + /// dotnet, python, node, go… + public string SdkLanguage { get; set; } + public string SdkVersion { get; set; } + + /// classic or resident: how the adapter's entry point runs it. + public string Lifecycle { get; set; } + + /// The protocol versions it speaks: 1 is the classic text protocol, 2 is gRPC. + public AdapterProtocolRange Protocol { get; set; } + + public List Settings { get; set; } = new(); + public List Commands { get; set; } = new(); + public List Kinds { get; set; } = new(); + public Dictionary Contracts { get; set; } = new(); + + /// Anything that kept the description from being complete, e.g. a handler that couldn't be built without its settings. + public List Warnings { get; set; } = new(); + + [JsonExtensionData] + public Dictionary Extensions { get; set; } + + public static AdapterSelfDescription Parse(string json) => + JsonSerializer.Deserialize(json, AdapterManifest.JsonOptions) + ?? throw new JsonException("The adapter's description is empty."); + + public string ToJson() => JsonSerializer.Serialize(this, AdapterManifest.JsonOptions); + } + + public class DescribedSetting + { + public string Name { get; set; } + public string Description { get; set; } + + /// text, multiline, number, boolean, select or json, as manifest properties are typed. + public string Type { get; set; } = AdapterProperty.TextType; + + public bool Required { get; set; } + public bool Secret { get; set; } + public string Default { get; set; } + + [JsonExtensionData] + public Dictionary Extensions { get; set; } + } + + public class DescribedCommand + { + public string Name { get; set; } + public string Description { get; set; } + + /// JSON Schema of the argument; null when it takes none. + public JsonElement? InputSchema { get; set; } + + /// JSON Schema of the result; null when it returns nothing. + public JsonElement? OutputSchema { get; set; } + + public bool ReturnsValue { get; set; } + + [JsonExtensionData] + public Dictionary Extensions { get; set; } + } +} diff --git a/SW.Serverless.Sdk/AdapterContractAttribute.cs b/SW.Serverless.Sdk/AdapterContractAttribute.cs new file mode 100644 index 0000000..c5b729b --- /dev/null +++ b/SW.Serverless.Sdk/AdapterContractAttribute.cs @@ -0,0 +1,21 @@ +using System; + +namespace SW.Serverless.Sdk +{ + /// + /// A contract this adapter implements, and its version — for Bitween, "bitween" 1. Reported + /// in the adapter's description and handshake, and checked by serverless test. + /// + [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] + public class AdapterContractAttribute : Attribute + { + public AdapterContractAttribute(string name, int version) + { + Name = name; + Version = version; + } + + public string Name { get; } + public int Version { get; } + } +} diff --git a/SW.Serverless.Sdk/Describer.cs b/SW.Serverless.Sdk/Describer.cs new file mode 100644 index 0000000..8cf889e --- /dev/null +++ b/SW.Serverless.Sdk/Describer.cs @@ -0,0 +1,130 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Schema; +using System.Threading.Tasks; +using SW.Serverless.Contract.Catalog; + +namespace SW.Serverless.Sdk +{ + /// + /// Answers --describe: the adapter's settings, commands, kinds and contracts as an + /// on stdout, without connecting to any host. + /// + internal static class Describer + { + public static bool Requested(string[] args) => + args.Length > 1 && args[1] == AdapterSelfDescription.Flag; + + // Payloads travel as Newtonsoft writes them — property names as declared — so the schema + // names them the same way. + static readonly JsonSerializerOptions SchemaOptions = new(JsonSerializerDefaults.General) + { + TypeInfoResolver = new System.Text.Json.Serialization.Metadata.DefaultJsonTypeInfoResolver(), + }; + + /// + /// Builds the handler, which is what runs its Runner.Expect declarations. Without its + /// settings a handler may refuse to be built; the description is then completed without + /// them and says why. + /// + public static AdapterSelfDescription Describe(Type handlerType, Func buildHandler, string lifecycle, + Func> commandsOf) + { + var description = new AdapterSelfDescription + { + SdkLanguage = "dotnet", + SdkVersion = SdkInfo.Version, + Lifecycle = lifecycle, + Protocol = lifecycle == AdapterManifest.ResidentLifecycle + ? new AdapterProtocolRange { Min = 2, Max = 2 } + : new AdapterProtocolRange { Min = 1, Max = 1 }, + }; + + object handler = null; + try + { + handler = buildHandler?.Invoke(); + } + catch (Exception ex) + { + description.Warnings.Add( + $"The handler could not be built without its settings, so settings it declares when built may be missing: {ex.GetBaseException().Message}"); + } + handlerType ??= handler?.GetType(); + var commands = commandsOf(handler).ToList(); + + foreach (var (name, value) in Runner.DeclaredStartupValues) + description.Settings.Add(new DescribedSetting + { + Name = name, + Description = value.Description, + Type = string.IsNullOrEmpty(value.Type) ? AdapterProperty.TextType : value.Type, + Required = !value.Optional, + Secret = value.Private, + Default = value.Private ? null : value.Default, + }); + + foreach (var (name, parameter, result, text) in commands.OrderBy(c => c.Name, StringComparer.Ordinal)) + description.Commands.Add(new DescribedCommand + { + Name = name, + Description = string.IsNullOrEmpty(text) ? null : text, + InputSchema = SchemaOf(parameter), + OutputSchema = SchemaOf(result), + ReturnsValue = result != null, + }); + + if (handlerType != null) + { + description.Kinds = handlerType.GetCustomAttributes(true) + .Select(k => k.Kind).Distinct().ToList(); + description.Contracts = handlerType.GetCustomAttributes(true) + .GroupBy(c => c.Name).ToDictionary(g => g.Key, g => g.Max(c => c.Version)); + } + + return description; + } + + /// The command's result type: what its Task carries, or null for a bare Task. + public static Type ResultOf(MethodInfo method) + { + var type = method.ReturnType; + if (type == typeof(Task) || type == typeof(void)) return null; + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Task<>)) + { + var result = type.GetGenericArguments()[0]; + // Task says nothing about its shape; describe it as anything. + return result; + } + return type; + } + + static JsonElement? SchemaOf(Type type) + { + if (type == null) return null; + try + { + JsonNode schema = type == typeof(object) + ? new JsonObject() + : SchemaOptions.GetJsonSchemaAsNode(type); + return JsonSerializer.SerializeToElement(schema); + } + catch + { + // Describing an argument must never stop the description. + return JsonSerializer.SerializeToElement(new JsonObject()); + } + } + + public static Task Print(AdapterSelfDescription description) + { + Console.Out.WriteLine(description.ToJson()); + Console.Out.Flush(); + return Task.FromResult(0); + } + } +} diff --git a/SW.Serverless.Sdk/Resident/ResidentRunner.cs b/SW.Serverless.Sdk/Resident/ResidentRunner.cs index 28203e1..8e4133f 100644 --- a/SW.Serverless.Sdk/Resident/ResidentRunner.cs +++ b/SW.Serverless.Sdk/Resident/ResidentRunner.cs @@ -110,6 +110,18 @@ public static Task RunAsync(object handler) => /// public static async Task RunAsync(Type handlerType, Func handlerFactory) { + // --describe: say what this adapter is, without a host, and exit. + if (Describer.Requested(Environment.GetCommandLineArgs())) + { + // Commands come from the type, as they do for Hello, built or not. + await Describer.Print(Describer.Describe(handlerType, () => handlerFactory(null), + Contract.Catalog.AdapterManifest.ResidentLifecycle, + _ => BuildCommands(handlerType).Select(c => (c.Key, c.Value.ParameterType, + c.Value.Void ? null : Describer.ResultOf(c.Value.MethodInfo), + c.Value.MethodInfo.GetCustomAttribute()?.Description)))); + return; + } + var runner = new ResidentRunner(handlerType, handlerFactory); try { @@ -160,8 +172,14 @@ await call.RequestStream.WriteAsync(new AdapterFrame AdapterId = handshake.AdapterId ?? "", InstanceKey = handshake.InstanceKey ?? "", ProtocolVersion = protocol, - SdkVersion = typeof(ResidentRunner).Assembly.GetName().Version?.ToString() ?? "0.0.0", + SdkVersion = SdkInfo.Version, SdkLanguage = "dotnet", + Kinds = { handlerType.GetCustomAttributes(true).Select(k => k.Kind).Distinct() }, + Contracts = + { + handlerType.GetCustomAttributes(true) + .GroupBy(c => c.Name).ToDictionary(g => g.Key, g => g.Max(c => c.Version)) + }, // Declared with Runner.Expect by the time the handshake goes: so in a handler // that was constructed before it was handed to the runner, which is the usual way. Settings = diff --git a/SW.Serverless.Sdk/Runner.cs b/SW.Serverless.Sdk/Runner.cs index dcaeb5c..d97d711 100644 --- a/SW.Serverless.Sdk/Runner.cs +++ b/SW.Serverless.Sdk/Runner.cs @@ -73,6 +73,17 @@ public static Task RunResident(object commandHandler) => async public static Task Run(object commandHandler) { + // --describe: say what this adapter is, without a host, and exit. + if (Describer.Requested(Environment.GetCommandLineArgs())) + { + await Describer.Print(Describer.Describe( + commandHandler is Func ? null : commandHandler.GetType(), + () => commandHandler is Func factory ? factory() : commandHandler, + Contract.Catalog.AdapterManifest.ClassicLifecycle, + CommandsOf)); + return; + } + Timer idleTimer = null; try @@ -257,6 +268,15 @@ static bool IsPrimitive(this Type type) return type.IsPrimitive; } + /// The classic commands of a built handler, for --describe. + static IEnumerable<(string, Type, Type, string)> CommandsOf(object built) + { + if (built == null) yield break; + foreach (var (name, info) in BuildMethodsDictionary(built)) + yield return (name, info.ParameterType, info.Void ? null : Describer.ResultOf(info.MethodInfo), + info.MethodInfo.GetCustomAttribute()?.Description); + } + static Dictionary BuildMethodsDictionary(object commandHandler) { var methodsDictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); diff --git a/SW.Serverless.Sdk/SdkInfo.cs b/SW.Serverless.Sdk/SdkInfo.cs new file mode 100644 index 0000000..a3e43d4 --- /dev/null +++ b/SW.Serverless.Sdk/SdkInfo.cs @@ -0,0 +1,27 @@ +using System; +using System.Reflection; + +namespace SW.Serverless.Sdk +{ + /// The version of this SDK, as an adapter reports it in its description and handshake. + public static class SdkInfo + { + /// + /// The release this code is, kept by hand as the host's HostInfo.Baseline is: the publish + /// workflow stamps the version on the package, and the assembly itself says 1.0.0.0. + /// + public const string Baseline = "10.1.0"; + + public static string Version { get; } = Resolve(); + + static string Resolve() + { + var informational = typeof(SdkInfo).Assembly + .GetCustomAttribute()?.InformationalVersion; + var core = informational?.Split('+', '-')[0]; + return System.Version.TryParse(core, out var stamped) && stamped > System.Version.Parse(Baseline) + ? stamped.ToString(3) + : Baseline; + } + } +} diff --git a/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs b/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs index 02aa77b..ff4694f 100644 --- a/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs +++ b/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs @@ -3,6 +3,8 @@ namespace SW.Serverless.UnitTests.GrpcClassicAdapter; +[AdapterKind("handler")] +[AdapterContract("bitween", 1)] public class Handler { public Handler() diff --git a/SW.Serverless.UnitTests/DescribeTests.cs b/SW.Serverless.UnitTests/DescribeTests.cs new file mode 100644 index 0000000..77273c1 --- /dev/null +++ b/SW.Serverless.UnitTests/DescribeTests.cs @@ -0,0 +1,102 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.UnitTests.Fixtures; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text.Json; + +namespace SW.Serverless.UnitTests +{ + /// + /// An adapter started with --describe prints what it is — settings, commands with JSON Schemas, + /// kinds, contracts — and exits, without any host. This is how the tools learn about an adapter + /// in any language, rather than by reading .NET assemblies. + /// + [TestClass] + public class DescribeTests + { + static AdapterSelfDescription Describe(string project) + { + var entry = Path.Combine(TestStore.LocateBuildOutput(project), project + ".dll"); + using var process = Process.Start(new ProcessStartInfo("dotnet") + { + ArgumentList = { entry, AdapterSelfDescription.Flag }, + RedirectStandardOutput = true, + RedirectStandardError = true, + RedirectStandardInput = true, + })!; + // Nothing is ever written to it: a describe never waits on stdin. + process.StandardInput.Close(); + var output = process.StandardOutput.ReadToEnd(); + var error = process.StandardError.ReadToEnd(); + Assert.IsTrue(process.WaitForExit(30_000), "--describe did not exit"); + Assert.AreEqual(0, process.ExitCode, error); + return AdapterSelfDescription.Parse(output); + } + + [TestMethod] + public void A_classic_adapter_describes_its_settings_commands_and_kinds() + { + var d = Describe("SW.Serverless.Samples.Classic"); + + Assert.AreEqual("dotnet", d.SdkLanguage); + Assert.AreEqual("10.1.0", d.SdkVersion); + Assert.AreEqual(AdapterManifest.ClassicLifecycle, d.Lifecycle); + Assert.AreEqual(1, d.Protocol.Max, "the classic text protocol"); + CollectionAssert.AreEquivalent(new[] { "handler", "mapper" }, d.Kinds); + + var apiKey = d.Settings.Single(s => s.Name == "ApiKey"); + Assert.IsTrue(apiKey.Required); + Assert.IsTrue(apiKey.Secret); + Assert.IsNull(apiKey.Default, "a secret's default is never published"); + Assert.AreEqual("https://api.example.test", d.Settings.Single(s => s.Name == "BaseUrl").Default); + + var echo = d.Commands.Single(c => c.Name == "Echo"); + StringAssert.Contains(echo.InputSchema!.Value.GetRawText(), "string"); + Assert.IsTrue(echo.ReturnsValue); + Assert.IsFalse(d.Commands.Single(c => c.Name == "Fail").ReturnsValue); + + // An object argument is described by its properties, named as they travel. + var summarize = d.Commands.Single(c => c.Name == "Summarize").InputSchema!.Value; + Assert.IsTrue(summarize.GetProperty("properties").TryGetProperty("Lines", out _)); + } + + [TestMethod] + public void A_resident_adapter_describes_its_commands_and_leaves_out_its_lifecycle_methods() + { + var d = Describe("SW.Serverless.Samples.Greedy"); + + Assert.AreEqual(AdapterManifest.ResidentLifecycle, d.Lifecycle); + Assert.AreEqual(2, d.Protocol.Min, "gRPC"); + var names = d.Commands.Select(c => c.Name).ToList(); + CollectionAssert.IsSubsetOf(new[] { "Allocate", "GetStats", "Release" }, names); + CollectionAssert.DoesNotContain(names, "StartAsync"); + CollectionAssert.DoesNotContain(names, "StopAsync"); + } + + [TestMethod] + public void An_adapter_describes_the_contracts_it_implements() + { + var d = Describe("SW.Serverless.UnitTests.GrpcClassicAdapter"); + + Assert.AreEqual(1, d.Contracts["bitween"]); + CollectionAssert.AreEqual(new[] { "handler" }, d.Kinds); + Assert.AreEqual("hello ", d.Settings.Single(s => s.Name == "Prefix").Default); + Assert.AreEqual(0, d.Warnings.Count, string.Join("; ", d.Warnings)); + } + + [TestMethod] + public void The_description_round_trips_through_its_own_parser() + { + var d = Describe("SW.Serverless.Samples.Classic"); + var again = AdapterSelfDescription.Parse(d.ToJson()); + + Assert.AreEqual(d.Commands.Count, again.Commands.Count); + Assert.AreEqual(d.Settings.Count, again.Settings.Count); + Assert.IsTrue(JsonElement.DeepEquals( + d.Commands.Single(c => c.Name == "Summarize").InputSchema!.Value, + again.Commands.Single(c => c.Name == "Summarize").InputSchema!.Value)); + } + } +} diff --git a/SW.Serverless.UnitTests/GrpcClassicTests.cs b/SW.Serverless.UnitTests/GrpcClassicTests.cs index d8400cb..544ce33 100644 --- a/SW.Serverless.UnitTests/GrpcClassicTests.cs +++ b/SW.Serverless.UnitTests/GrpcClassicTests.cs @@ -155,6 +155,27 @@ public async Task The_expected_startup_values_are_the_settings_it_declared() } } + [TestMethod] + public async Task Its_handshake_names_its_language_kinds_and_contracts() + { + var before = RunningKeys().ToHashSet(); + var service = Service(); + await service.StartAsync(AdapterId, "corr-7"); + try + { + var described = host.Services.GetRequiredService().Describe() + .Single(h => !before.Contains(h.InstanceKey)); + Assert.AreEqual("dotnet", described.SdkLanguage); + CollectionAssert.AreEqual(new[] { "handler" }, described.Kinds.ToList()); + Assert.AreEqual(1, described.Contracts["bitween"]); + Assert.AreEqual("10.1.0", described.SdkVersion); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + [TestMethod] public async Task An_adapter_error_fails_the_call_with_its_message() { From e0a4855016d777b85601912a1001f5edc6a5b168 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 10:33:42 +0300 Subject: [PATCH 09/14] Add the conformance kit: check an adapter against its contracts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ConformanceRunner (in SW.Serverless.Tooling, what serverless test will run) takes an unpacked adapter package and settings, and checks: - its manifest is valid and its entry is in the package; - it answers --describe, and the settings it declares are the manifest's properties — names, required, secret, defaults — so the two can't drift; - it starts the way a host starts it: installed from storage into a temporary host, on its runtime, classic or resident; - for each contract it declares, each of its kinds has the contract's methods, each method answers the contract's example inputs with output that satisfies the output type's schema, and a session kind such as a receiver runs through in the contract's order, with a destructive method such as DeleteFile called only when allowed; - an unknown command is refused with an error and it keeps answering. Contracts are read from their machine-readable JSON, so the kit knows no contract in particular; it carries a copy of Bitween's and takes others from a file. Runtime launches can now pass arguments after the entry, which is how --describe reaches an adapter in any language. ConformanceTests runs it against real adapters: a conforming Bitween handler passes every check; one answering without Data fails with the schema's reason; a setting the manifest leaves out is named; a receiver over gRPC runs its session and leaves the source alone unless deleting is allowed, then deletes the first file listed; and an adapter on the 10.0.0 SDK is told it can't describe itself while still running. --- .../ConformanceTests.cs | 169 ++++++ .../SW.Serverless.Installer.UnitTests.csproj | 6 + .../Conformance/ConformanceOptions.cs | 35 ++ .../Conformance/ConformanceReport.cs | 21 + .../Conformance/ConformanceRunner.cs | 494 ++++++++++++++++++ .../Conformance/ContractDocument.cs | 93 ++++ .../bitween/bitween-adapter-contract.v1.json | 121 +++++ .../bitween/exchange-file.schema.json | 16 + .../bitween/validation-result.schema.json | 24 + .../SW.Serverless.Tooling.csproj | 12 +- .../Program.cs | 39 ++ ...Serverless.UnitTests.BitweenHandler.csproj | 13 + .../Program.cs | 47 ++ ...erverless.UnitTests.BitweenReceiver.csproj | 13 + SW.Serverless.sln | 28 + SW.Serverless/Runtimes/AdapterRuntimes.cs | 14 +- 16 files changed, 1142 insertions(+), 3 deletions(-) create mode 100644 SW.Serverless.Installer.UnitTests/ConformanceTests.cs create mode 100644 SW.Serverless.Tooling/Conformance/ConformanceOptions.cs create mode 100644 SW.Serverless.Tooling/Conformance/ConformanceReport.cs create mode 100644 SW.Serverless.Tooling/Conformance/ConformanceRunner.cs create mode 100644 SW.Serverless.Tooling/Conformance/ContractDocument.cs create mode 100644 SW.Serverless.Tooling/Contracts/bitween/bitween-adapter-contract.v1.json create mode 100644 SW.Serverless.Tooling/Contracts/bitween/exchange-file.schema.json create mode 100644 SW.Serverless.Tooling/Contracts/bitween/validation-result.schema.json create mode 100644 SW.Serverless.UnitTests.BitweenHandler/Program.cs create mode 100644 SW.Serverless.UnitTests.BitweenHandler/SW.Serverless.UnitTests.BitweenHandler.csproj create mode 100644 SW.Serverless.UnitTests.BitweenReceiver/Program.cs create mode 100644 SW.Serverless.UnitTests.BitweenReceiver/SW.Serverless.UnitTests.BitweenReceiver.csproj diff --git a/SW.Serverless.Installer.UnitTests/ConformanceTests.cs b/SW.Serverless.Installer.UnitTests/ConformanceTests.cs new file mode 100644 index 0000000..a080e48 --- /dev/null +++ b/SW.Serverless.Installer.UnitTests/ConformanceTests.cs @@ -0,0 +1,169 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Tooling.Conformance; + +namespace SW.Serverless.Installer.UnitTests; + +/// +/// The conformance kit — what serverless test runs — against real adapters built beside the tests: +/// started as a host starts them, described, and called with the Bitween contract's examples. +/// +[TestClass] +public class ConformanceTests +{ + static string BuildOutput(string project) + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null && !File.Exists(Path.Combine(dir.FullName, "SW.Serverless.sln"))) dir = dir.Parent; + var bin = Path.Combine(dir!.FullName, project, "bin"); + return Directory.GetDirectories(bin, "*", SearchOption.AllDirectories) + .Where(d => File.Exists(Path.Combine(d, project + ".dll"))) + .OrderByDescending(d => File.GetLastWriteTimeUtc(Path.Combine(d, project + ".dll"))) + .First(); + } + + /// A package: the project's build output with this manifest at its root. + static string Package(string project, AdapterManifest manifest) + { + var target = Path.Combine(Path.GetTempPath(), "swsl-conformance-tests", Guid.NewGuid().ToString("N")); + var source = BuildOutput(project); + foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories)) + { + var to = Path.Combine(target, Path.GetRelativePath(source, file)); + Directory.CreateDirectory(Path.GetDirectoryName(to)!); + File.Copy(file, to); + } + manifest.Entry ??= project + ".dll"; + File.WriteAllText(Path.Combine(target, AdapterManifest.FileName), manifest.ToJson()); + return target; + } + + static AdapterManifest HandlerManifest(bool listApiKey = true) + { + var manifest = new AdapterManifest + { + Id = "test.bitween.handler", + Kinds = { "handler" }, + Contracts = new() { ["bitween"] = 1 }, + Properties = + { + new AdapterProperty { Name = "Endpoint", Default = "https://partner.example.test/orders", Description = "Where orders go." }, + new AdapterProperty { Name = "Mode", Default = "working" }, + }, + }; + if (listApiKey) manifest.Properties.Add(new AdapterProperty { Name = "ApiKey", Required = true, Secret = true }); + return manifest; + } + + static async Task RunAsync(string packageDirectory, IDictionary settings = null, + bool allowDelete = false) + { + var report = await new ConformanceRunner().RunAsync(new ConformanceOptions + { + PackageDirectory = packageDirectory, + Settings = settings ?? new Dictionary { ["ApiKey"] = "test-key" }, + AllowDelete = allowDelete, + CommandTimeoutSeconds = 30, + }); + Console.WriteLine(string.Join(Environment.NewLine, report.Checks.Select(c => $"{c.Outcome,-7} {c.Name} {c.Detail}"))); + return report; + } + + static ConformanceCheck Check(ConformanceReport report, string name) => + report.Checks.Single(c => c.Name == name); + + [TestMethod] + public async Task A_conforming_handler_passes_every_check() + { + var report = await RunAsync(Package("SW.Serverless.UnitTests.BitweenHandler", HandlerManifest())); + + Assert.IsTrue(report.Passed, string.Join("; ", report.Checks.Where(c => c.Outcome == CheckOutcome.Failed))); + foreach (var name in new[] + { + "manifest", "describe", "settings match the manifest", "starts", + "bitween handler: methods", "bitween handler: Handle answers example 1", "an unknown command is refused", + }) + Assert.AreEqual(CheckOutcome.Passed, Check(report, name).Outcome, name); + } + + [TestMethod] + public async Task A_handler_answering_without_data_fails_the_contract() + { + var report = await RunAsync(Package("SW.Serverless.UnitTests.BitweenHandler", HandlerManifest()), + new Dictionary { ["ApiKey"] = "k", ["Mode"] = "broken" }); + + var check = Check(report, "bitween handler: Handle answers example 1"); + Assert.AreEqual(CheckOutcome.Failed, check.Outcome); + StringAssert.Contains(check.Detail, "isn't a valid ExchangeFile"); + Assert.IsFalse(report.Passed); + } + + [TestMethod] + public async Task Settings_the_manifest_leaves_out_are_named() + { + var report = await RunAsync(Package("SW.Serverless.UnitTests.BitweenHandler", HandlerManifest(listApiKey: false))); + + var check = Check(report, "settings match the manifest"); + Assert.AreEqual(CheckOutcome.Failed, check.Outcome); + StringAssert.Contains(check.Detail, "'ApiKey' is declared by the adapter but missing from the manifest"); + } + + static (string Package, string Folder) Receiver() + { + var folder = Path.Combine(Path.GetTempPath(), "swsl-conformance-tests", "source-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(folder); + File.WriteAllText(Path.Combine(folder, "a.json"), "{\"n\":1}"); + File.WriteAllText(Path.Combine(folder, "b.json"), "{\"n\":2}"); + + var package = Package("SW.Serverless.UnitTests.BitweenReceiver", new AdapterManifest + { + Id = "test.bitween.receiver", + Kinds = { "receiver" }, + Contracts = new() { ["bitween"] = 1 }, + // Run as Bitween runs receivers, classically, over gRPC. + Protocol = new AdapterProtocolRange { Min = 2, Max = 2 }, + Properties = { new AdapterProperty { Name = "Folder", Required = true } }, + }); + return (package, folder); + } + + [TestMethod] + public async Task A_receiver_runs_its_session_in_order_and_leaves_the_source_alone_unless_allowed() + { + var (package, folder) = Receiver(); + var report = await RunAsync(package, new Dictionary { ["Folder"] = folder }); + + Assert.IsTrue(report.Passed, string.Join("; ", report.Checks.Where(c => c.Outcome == CheckOutcome.Failed))); + Assert.AreEqual(CheckOutcome.Passed, Check(report, "bitween receiver: ListFiles").Outcome); + Assert.AreEqual(CheckOutcome.Passed, Check(report, "bitween receiver: GetFile").Outcome); + Assert.AreEqual(CheckOutcome.Skipped, Check(report, "bitween receiver: DeleteFile").Outcome); + Assert.AreEqual(2, Directory.GetFiles(folder).Length, "nothing was deleted"); + } + + [TestMethod] + public async Task A_receiver_s_delete_runs_when_allowed() + { + var (package, folder) = Receiver(); + var report = await RunAsync(package, new Dictionary { ["Folder"] = folder }, allowDelete: true); + + Assert.AreEqual(CheckOutcome.Passed, Check(report, "bitween receiver: DeleteFile").Outcome); + CollectionAssert.AreEqual(new[] { "b.json" }, Directory.GetFiles(folder).Select(Path.GetFileName).ToArray(), + "the first file listed was deleted"); + } + + [TestMethod] + public async Task An_adapter_on_the_old_SDK_is_told_it_can_t_describe_itself() + { + var report = await RunAsync(Package("SW.Serverless.Compat.ClassicV10", new AdapterManifest { Id = "test.old" })); + + var describe = Check(report, "describe"); + Assert.AreEqual(CheckOutcome.Failed, describe.Outcome); + StringAssert.Contains(describe.Detail, "older than 10.1.0"); + Assert.AreEqual(CheckOutcome.Passed, Check(report, "starts").Outcome, "it still runs"); + } +} diff --git a/SW.Serverless.Installer.UnitTests/SW.Serverless.Installer.UnitTests.csproj b/SW.Serverless.Installer.UnitTests/SW.Serverless.Installer.UnitTests.csproj index d0ff5e5..74d418e 100644 --- a/SW.Serverless.Installer.UnitTests/SW.Serverless.Installer.UnitTests.csproj +++ b/SW.Serverless.Installer.UnitTests/SW.Serverless.Installer.UnitTests.csproj @@ -22,6 +22,12 @@ + + + The adapter's package, unpacked: its runnable files with adapter.json at the root. + public string PackageDirectory { get; set; } + + /// The settings to run it with — the adapter calls whatever these point at. + public IDictionary Settings { get; set; } = new Dictionary(); + + /// + /// Let a receiver's DeleteFile run. Off by default: it removes or moves a real file at the + /// source the settings point at. + /// + public bool AllowDelete { get; set; } + + /// Contracts beyond those the kit carries, e.g. a newer version from a file. + public IList Contracts { get; set; } = new List(); + + /// How long one call may take. + public int CommandTimeoutSeconds { get; set; } = 60; + + /// Where the kit's temporary host keeps its files. + public string WorkDirectory { get; set; } + + /// Where the host finds python3, node and dotnet. + public Runtimes.AdapterRuntimeOptions Runtimes { get; set; } = new(); + + /// Progress, one line at a time. + public Action Log { get; set; } = _ => { }; + } +} diff --git a/SW.Serverless.Tooling/Conformance/ConformanceReport.cs b/SW.Serverless.Tooling/Conformance/ConformanceReport.cs new file mode 100644 index 0000000..1ee07b4 --- /dev/null +++ b/SW.Serverless.Tooling/Conformance/ConformanceReport.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; + +namespace SW.Serverless.Tooling.Conformance +{ + public enum CheckOutcome { Passed, Failed, Skipped } + + /// One thing the kit checked, and what it found. + public record ConformanceCheck(string Name, CheckOutcome Outcome, string Detail = null); + + public class ConformanceReport + { + public List Checks { get; } = new(); + + public bool Passed => Checks.All(c => c.Outcome != CheckOutcome.Failed); + + internal void Pass(string name, string detail = null) => Checks.Add(new(name, CheckOutcome.Passed, detail)); + internal void Fail(string name, string detail) => Checks.Add(new(name, CheckOutcome.Failed, detail)); + internal void Skip(string name, string detail) => Checks.Add(new(name, CheckOutcome.Skipped, detail)); + } +} diff --git a/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs b/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs new file mode 100644 index 0000000..3f5a596 --- /dev/null +++ b/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs @@ -0,0 +1,494 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json.Linq; +using SW.CloudFiles.Extensions; +using SW.PrimitiveTypes; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Resident; +using SW.Serverless.Runtimes; + +namespace SW.Serverless.Tooling.Conformance +{ + /// + /// Runs an adapter package the way a host does — installed from storage, started on its runtime, + /// called by name — and checks it against its manifest, its own description and every contract + /// it declares. What serverless test runs, for an adapter in any language. + /// + public class ConformanceRunner + { + const string Version = "0.0.0"; + const string UnknownCommand = "__ConformanceNoSuchCommand__"; + + public async Task RunAsync(ConformanceOptions options) + { + var report = new ConformanceReport(); + var work = options.WorkDirectory ?? Path.Combine(Path.GetTempPath(), "swsl-conformance", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(work); + try + { + await RunCoreAsync(options, report, work); + } + finally + { + if (options.WorkDirectory == null) + try { Directory.Delete(work, true); } catch { /* best effort */ } + } + return report; + } + + async Task RunCoreAsync(ConformanceOptions options, ConformanceReport report, string work) + { + // ------------------------------------------------------------ the manifest + var manifestPath = Path.Combine(options.PackageDirectory, AdapterManifest.FileName); + if (!File.Exists(manifestPath)) + { + report.Fail("manifest", $"there is no {AdapterManifest.FileName} in {options.PackageDirectory}; serverless build writes it"); + return; + } + + AdapterManifest manifest; + try + { + manifest = AdapterManifest.Parse(File.ReadAllText(manifestPath)); + } + catch (Exception ex) + { + report.Fail("manifest", $"{AdapterManifest.FileName} can't be read: {ex.Message}"); + return; + } + + var problems = manifest.Validate(); + if (problems.Count > 0) report.Fail("manifest", string.Join("; ", problems)); + else report.Pass("manifest"); + + var entry = manifest.EntryFor(AdapterRuntimes.CurrentPlatform); + if (string.IsNullOrWhiteSpace(entry) || !File.Exists(Path.Combine(options.PackageDirectory, entry))) + { + report.Fail("entry", $"the manifest's entry '{entry}' is not in the package"); + return; + } + + // ------------------------------------------------------------ its own description + var runtimes = new AdapterRuntimes(options.Runtimes); + options.Log("Asking the adapter to describe itself..."); + var description = await DescribeAsync(runtimes, manifest, Path.Combine(options.PackageDirectory, entry), report); + if (description != null) CheckSettings(manifest, description, report); + + // ------------------------------------------------------------ running it + options.Log("Starting it the way a host does..."); + using var host = BuildHost(options, work); + await host.StartAsync(); + try + { + var adapterId = string.IsNullOrWhiteSpace(manifest.Id) ? "conformance.adapter" : manifest.Id; + await UploadAsync(host.Services.GetRequiredService(), options.PackageDirectory, + adapterId, entry); + + await using var session = await StartAsync(host.Services, manifest, $"{adapterId}/{Version}", options, report); + if (session == null) return; + + foreach (var (contract, kinds) in ContractsToCheck(manifest, description, options, report)) + foreach (var kind in kinds) + await CheckKindAsync(session, contract, kind, description, options, report); + + await CheckUnknownCommandAsync(session, report); + } + finally + { + await host.StopAsync(); + } + } + + // ---------------------------------------------------------------- describe + + static async Task DescribeAsync(AdapterRuntimes runtimes, AdapterManifest manifest, + string entryPath, ConformanceReport report) + { + var runtime = runtimes.Find(manifest.Runtime); + if (runtime == null) + { + report.Fail("describe", $"no launcher for the '{manifest.Runtime}' runtime"); + return null; + } + + var startInfo = runtime.StartInfo(entryPath, new RuntimeLaunch { Arguments = new[] { AdapterSelfDescription.Flag } }); + try + { + using var process = Process.Start(startInfo)!; + process.StandardInput.Close(); + var output = process.StandardOutput.ReadToEndAsync(); + var errors = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(30_000)) + { + try { process.Kill(true); } catch { } + report.Fail("describe", $"it didn't answer {AdapterSelfDescription.Flag} within 30 seconds — an SDK older than 10.1.0 doesn't know it"); + return null; + } + + var description = AdapterSelfDescription.Parse(await output); + if (process.ExitCode != 0) + report.Fail("describe", $"{AdapterSelfDescription.Flag} exited with {process.ExitCode}: {await errors}"); + else if (string.IsNullOrWhiteSpace(description.SdkLanguage)) + report.Fail("describe", "the description doesn't name its SDK's language"); + else + report.Pass("describe", $"{description.SdkLanguage} SDK {description.SdkVersion}, {description.Commands.Count} commands" + + (description.Warnings.Count > 0 ? $"; warnings: {string.Join("; ", description.Warnings)}" : "")); + return description; + } + catch (Exception ex) + { + report.Fail("describe", $"its description couldn't be read: {ex.Message}"); + return null; + } + } + + /// The manifest's properties must be the settings the adapter itself declares. + static void CheckSettings(AdapterManifest manifest, AdapterSelfDescription description, ConformanceReport report) + { + var declared = description.Settings.ToDictionary(s => s.Name, StringComparer.OrdinalIgnoreCase); + var listed = (manifest.Properties ?? new List()).ToDictionary(p => p.Name, StringComparer.OrdinalIgnoreCase); + var differences = new List(); + + foreach (var name in declared.Keys.Except(listed.Keys, StringComparer.OrdinalIgnoreCase)) + differences.Add($"'{name}' is declared by the adapter but missing from the manifest"); + foreach (var name in listed.Keys.Except(declared.Keys, StringComparer.OrdinalIgnoreCase)) + differences.Add($"'{name}' is in the manifest but the adapter doesn't declare it"); + foreach (var name in declared.Keys.Intersect(listed.Keys, StringComparer.OrdinalIgnoreCase)) + { + var (code, file) = (declared[name], listed[name]); + if (code.Required != file.Required) differences.Add($"'{name}' is {(code.Required ? "" : "not ")}required in the adapter but {(file.Required ? "" : "not ")}in the manifest"); + if (code.Secret != file.Secret) differences.Add($"'{name}' is {(code.Secret ? "" : "not ")}secret in the adapter but {(file.Secret ? "" : "not ")}in the manifest"); + if (!code.Secret && (code.Default ?? "") != (file.Default ?? "")) differences.Add($"'{name}' defaults to '{code.Default}' in the adapter but '{file.Default}' in the manifest"); + } + + if (differences.Count > 0) report.Fail("settings match the manifest", string.Join("; ", differences) + " — serverless build rewrites the manifest from the adapter"); + else report.Pass("settings match the manifest", $"{declared.Count} settings"); + } + + // ---------------------------------------------------------------- the host + + static IHost BuildHost(ConformanceOptions options, string work) + { + var tag = Guid.NewGuid().ToString("N")[..10]; + return Host.CreateDefaultBuilder() + .ConfigureLogging(l => l.ClearProviders()) + .ConfigureAppConfiguration(c => c.Sources.Clear()) + .ConfigureServices(s => + { + s.AddLocalTestsCloudFiles(o => + { + o.BucketName = "conformance-" + tag; + o.StoragePath = Path.Combine(work, "store"); + }); + s.AddAdapterRuntimes(o => + { + o.PythonExecutable = options.Runtimes.PythonExecutable; + o.NodeExecutable = options.Runtimes.NodeExecutable; + o.DotnetExecutable = options.Runtimes.DotnetExecutable; + }); + s.AddServerless(o => + { + o.AdapterRemotePath = "adapters"; + o.AdapterLocalPath = Path.Combine(work, "installed"); + o.AdapterMetadataCacheDuration = 1; + o.CommandTimeout = options.CommandTimeoutSeconds; + }); + s.AddResidentAdapters(o => + { + o.SocketPath = Path.Combine("/tmp", $"swsl-conf-{tag}.sock"); + o.PipeName = $"swsl-conf-{tag}"; + o.HandshakeTimeout = TimeSpan.FromSeconds(60); + }); + }) + .Build(); + } + + static async Task UploadAsync(ICloudFilesService files, string directory, string adapterId, string entry) + { + using var buffer = new MemoryStream(); + using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories)) + archive.CreateEntryFromFile(file, Path.GetRelativePath(directory, file).Replace('\\', '/'), CompressionLevel.Fastest); + + buffer.Position = 0; + var hash = Convert.ToHexString(await SHA256.HashDataAsync(buffer))[..16].ToLowerInvariant(); + buffer.Position = 0; + await files.WriteAsync(buffer, new WriteFileSettings + { + Key = AdapterCatalogPaths.Version("adapters", adapterId, Version), + ContentType = "application/zip", + Metadata = new Dictionary { ["EntryAssembly"] = entry, ["Hash"] = "conformance-" + hash }, + }); + } + + // ---------------------------------------------------------------- sessions + + /// A running adapter, called by command name with raw payloads. + interface ISession : IAsyncDisposable + { + Task CallAsync(string command, object input); + Task CallVoidAsync(string command, object input); + } + + static async Task StartAsync(IServiceProvider services, AdapterManifest manifest, string adapterRef, + ConformanceOptions options, ConformanceReport report) + { + try + { + ISession session = manifest.IsResident + ? await ResidentSession.StartAsync(services, adapterRef, options) + : await ClassicSession.StartAsync(services, adapterRef, options); + report.Pass("starts", manifest.IsResident ? "resident, attached over gRPC" : "classic session"); + return session; + } + catch (Exception ex) + { + report.Fail("starts", ex.GetBaseException().Message); + return null; + } + } + + sealed class ClassicSession(IServiceScope scope, IServerlessService service, int timeout) : ISession + { + public static async Task StartAsync(IServiceProvider services, string adapterRef, ConformanceOptions options) + { + var scope = services.CreateScope(); + var service = scope.ServiceProvider.GetRequiredService(); + await service.StartAsync(adapterRef, "conformance", new Dictionary(options.Settings)); + return new ClassicSession(scope, service, options.CommandTimeoutSeconds); + } + + public Task CallAsync(string command, object input) => service.InvokeAsync(command, input, timeout); + public Task CallVoidAsync(string command, object input) => service.InvokeAsync(command, input, timeout); + + public ValueTask DisposeAsync() + { + (service as IDisposable)?.Dispose(); + scope.Dispose(); + return ValueTask.CompletedTask; + } + } + + sealed class ResidentSession(IResidentAdapterHost host, ResidentAdapterInstance instance, string adapterRef, int timeout) : ISession + { + public static async Task StartAsync(IServiceProvider services, string adapterRef, ConformanceOptions options) + { + var host = services.GetRequiredService(); + var instance = await host.StartExclusiveAsync(new AdapterSpec + { + AdapterId = adapterRef, + InstanceKey = "conformance", + StartupValues = new Dictionary(options.Settings), + }); + return new ResidentSession(host, instance, adapterRef, options.CommandTimeoutSeconds); + } + + public Task CallAsync(string command, object input) => instance.InvokeAsync(command, input, timeoutSeconds: timeout); + public Task CallVoidAsync(string command, object input) => instance.InvokeAsync(command, input, timeoutSeconds: timeout); + + public async ValueTask DisposeAsync() => await host.StopAsync(adapterRef, "conformance", drain: false); + } + + sealed class NoEvents : IAdapterEventSink + { + public Task OnEventAsync(InboundEvent inboundEvent, CancellationToken cancellationToken) => + Task.FromResult(EventOutcome.Ok("conformance")); + } + + // ---------------------------------------------------------------- contracts + + static IEnumerable<(ContractDocument Contract, List Kinds)> ContractsToCheck(AdapterManifest manifest, + AdapterSelfDescription description, ConformanceOptions options, ConformanceReport report) + { + var declared = new Dictionary(manifest.Contracts ?? new Dictionary(), StringComparer.OrdinalIgnoreCase); + foreach (var (name, version) in description?.Contracts ?? new Dictionary()) + declared.TryAdd(name, version); + + if (declared.Count == 0) + report.Skip("contracts", "it declares no contract, so only what every adapter must do is checked"); + + var kinds = (manifest.Kinds ?? new List()).Union(description?.Kinds ?? new List(), StringComparer.OrdinalIgnoreCase).ToList(); + foreach (var (name, version) in declared) + { + var contract = options.Contracts.FirstOrDefault(c => string.Equals(c.Name, name, StringComparison.OrdinalIgnoreCase) && c.Version == version) + ?? ContractDocument.Known(name, version); + if (contract == null) + { + report.Fail($"contract {name} v{version}", "the kit doesn't know this contract; pass it with --contract"); + continue; + } + + var checkedKinds = kinds.Where(k => contract.Kinds.Contains(k, StringComparer.OrdinalIgnoreCase)).ToList(); + if (checkedKinds.Count == 0) + report.Fail($"contract {name} v{version}", $"it implements none of the contract's kinds ({string.Join(", ", contract.Kinds)})"); + yield return (contract, checkedKinds); + } + } + + static async Task CheckKindAsync(ISession session, ContractDocument contract, string kind, + AdapterSelfDescription description, ConformanceOptions options, ConformanceReport report) + { + var prefix = $"{contract.Name} {kind}"; + var methods = contract.MethodsOf(kind); + + if (description != null) + { + var commands = description.Commands.Select(c => c.Name).ToHashSet(StringComparer.Ordinal); + var missing = methods.Where(m => !commands.Contains(m.Name)).Select(m => m.Name).ToList(); + if (missing.Count > 0) report.Fail($"{prefix}: methods", $"missing {string.Join(", ", missing)} (names are case-sensitive)"); + else report.Pass($"{prefix}: methods", string.Join(", ", methods.Select(m => m.Name))); + } + + if (contract.IsSession(kind)) + { + await CheckSessionAsync(session, contract, prefix, methods, options, report); + return; + } + + foreach (var method in methods) + { + if (method.Examples.Count == 0) + { + report.Skip($"{prefix}: {method.Name}", "the contract gives no example to call it with"); + continue; + } + + for (var i = 0; i < method.Examples.Count; i++) + { + var name = $"{prefix}: {method.Name} answers example {i + 1}"; + try + { + var input = contract.IsText(method.Input) ? (object)(string)method.Examples[i] : method.Examples[i]; + if (method.Output == null) + { + await session.CallVoidAsync(method.Name, input); + report.Pass(name); + } + else + await CheckOutputAsync(contract, method.Output, await session.CallAsync(method.Name, input), name, report); + } + catch (Exception ex) + { + report.Fail(name, $"the call failed: {ex.GetBaseException().Message}"); + } + } + } + } + + /// + /// A session kind, such as a receiver, called once through in the contract's order: methods + /// without input as they come, a text input fed the first id the session listed, and a + /// destructive method only when allowed. + /// + static async Task CheckSessionAsync(ISession session, ContractDocument contract, string prefix, + IReadOnlyList methods, ConformanceOptions options, ConformanceReport report) + { + string firstId = null; + foreach (var method in methods) + { + var name = $"{prefix}: {method.Name}"; + if (method.Destructive && !options.AllowDelete) + { + report.Skip(name, "it changes the source the settings point at; allow it with --allow-delete"); + continue; + } + + object input = null; + if (method.Input != null) + { + if (firstId == null) + { + report.Skip(name, "nothing was listed to call it with"); + continue; + } + input = firstId; + } + + try + { + if (method.Output == null) + { + await session.CallVoidAsync(method.Name, input); + report.Pass(name); + continue; + } + + var output = await session.CallAsync(method.Name, input); + await CheckOutputAsync(contract, method.Output, output, name, report); + if (firstId == null && TryParse(output) is JArray ids && ids.Count > 0 && ids[0].Type == JTokenType.String) + firstId = (string)ids[0]; + } + catch (Exception ex) + { + report.Fail(name, $"the call failed: {ex.GetBaseException().Message}"); + } + } + } + + static async Task CheckOutputAsync(ContractDocument contract, string type, string output, string name, ConformanceReport report) + { + var schema = await contract.SchemaOfAsync(type); + if (schema == null) + { + report.Pass(name); + return; + } + + if (TryParse(output) is not { } json) + { + report.Fail(name, $"the answer isn't JSON: {Shorten(output)}"); + return; + } + + var errors = schema.Validate(json); + if (errors.Count > 0) report.Fail(name, $"the answer isn't a valid {type}: {string.Join("; ", errors.Select(e => e.ToString()))}"); + else report.Pass(name, $"a valid {type}"); + } + + /// An unknown command is refused with an error, and the adapter keeps answering. + static async Task CheckUnknownCommandAsync(ISession session, ConformanceReport report) + { + for (var attempt = 1; attempt <= 2; attempt++) + { + try + { + await session.CallAsync(UnknownCommand, null); + report.Fail("an unknown command is refused", "it answered a command it doesn't have"); + return; + } + catch (TimeoutException) + { + report.Fail("an unknown command is refused", "it didn't answer a command it doesn't have"); + return; + } + catch (Exception) when (attempt == 1) + { + // Refused; the second attempt shows it is still answering after that. + } + catch (Exception) + { + report.Pass("an unknown command is refused", "with an error, and it kept answering"); + } + } + } + + static JToken TryParse(string text) + { + try { return string.IsNullOrWhiteSpace(text) ? null : JToken.Parse(text); } + catch { return null; } + } + + static string Shorten(string text) => text == null ? "nothing" : text.Length <= 200 ? text : text[..200] + "…"; + } +} diff --git a/SW.Serverless.Tooling/Conformance/ContractDocument.cs b/SW.Serverless.Tooling/Conformance/ContractDocument.cs new file mode 100644 index 0000000..3088d57 --- /dev/null +++ b/SW.Serverless.Tooling/Conformance/ContractDocument.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NJsonSchema; + +namespace SW.Serverless.Tooling.Conformance +{ + /// + /// A contract an adapter can implement, as its machine-readable JSON: the kinds, the methods a + /// host calls on each, their payload types and schemas, and example inputs. Nothing here knows + /// any particular contract; the kit checks an adapter against whichever one it declares. + /// + public class ContractDocument + { + readonly JObject json; + readonly Func readSibling; + + ContractDocument(JObject json, Func readSibling) + { + this.json = json; + this.readSibling = readSibling; + } + + public string Name => (string)json["contract"]; + public int Version => (int?)json["version"] ?? 0; + + public IEnumerable Kinds => ((JObject)json["kinds"])?.Properties().Select(p => p.Name) ?? Enumerable.Empty(); + + public IReadOnlyList MethodsOf(string kind) => + json["kinds"]?[kind]?["methods"]?.Select(m => new ContractMethod( + (string)m["name"], + (string)m["input"], + (string)m["output"], + m["examples"]?.ToList() ?? new List(), + (bool?)m["destructive"] ?? false)) + .ToList() + ?? new List(); + + public bool IsSession(string kind) => json["kinds"]?[kind]?["session"] != null; + + /// Whether a type travels as raw text rather than JSON. + public bool IsText(string type) => (string)json["types"]?[type]?["encoding"] == "string"; + + /// The JSON Schema of a payload type, or null for one with none (raw text). + public async Task SchemaOfAsync(string type) + { + var declared = json["types"]?[type]?["schema"]; + return declared switch + { + null => null, + JValue file => await JsonSchema.FromJsonAsync(readSibling((string)file)), + JObject inline => await JsonSchema.FromJsonAsync(inline.ToString()), + _ => null, + }; + } + + /// The contracts the kit knows by name, carried with it. + public static ContractDocument Known(string name, int version) + { + // Found by file name: the folder in a resource's name is written with whichever + // separator the machine that built it uses. + var file = $"{name}-adapter-contract.v{version}.json"; + var resource = typeof(ContractDocument).Assembly.GetManifestResourceNames() + .FirstOrDefault(r => r.StartsWith("SW.Serverless.Tooling.Contracts.", StringComparison.Ordinal) && + r.EndsWith(file, StringComparison.Ordinal)); + if (resource == null) return null; + + var prefix = resource[..^file.Length]; + return new ContractDocument(JObject.Parse(ReadResource(resource)), sibling => ReadResource(prefix + sibling)); + } + + /// A contract from a file, with its schemas beside it. + public static ContractDocument FromFile(string path) + { + var directory = Path.GetDirectoryName(Path.GetFullPath(path)); + return new ContractDocument(JObject.Parse(File.ReadAllText(path)), + file => File.ReadAllText(Path.Combine(directory, file))); + } + + static string ReadResource(string name) + { + using var stream = typeof(ContractDocument).Assembly.GetManifestResourceStream(name) + ?? throw new FileNotFoundException(name); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } + } + + public record ContractMethod(string Name, string Input, string Output, IReadOnlyList Examples, bool Destructive); +} diff --git a/SW.Serverless.Tooling/Contracts/bitween/bitween-adapter-contract.v1.json b/SW.Serverless.Tooling/Contracts/bitween/bitween-adapter-contract.v1.json new file mode 100644 index 0000000..a56d1b4 --- /dev/null +++ b/SW.Serverless.Tooling/Contracts/bitween/bitween-adapter-contract.v1.json @@ -0,0 +1,121 @@ +{ + "contract": "bitween", + "version": 1, + "description": "What Bitween calls on each kind of adapter, and what it passes. Bitween calls adapters by method name over the SW-Serverless protocol; the names below are exact and case-sensitive.", + "encoding": { + "string": "A string argument or result is the raw UTF-8 text, not a JSON string.", + "object": "Any other argument or result is JSON, with property names exactly as its schema gives them.", + "none": "A method with no argument receives an empty payload; one with no result returns an empty payload." + }, + "types": { + "ExchangeFile": { + "schema": "exchange-file.schema.json" + }, + "ValidationResult": { + "schema": "validation-result.schema.json" + }, + "FileId": { + "encoding": "string", + "description": "An id a receiver gave in ListFiles, passed back exactly as given." + }, + "FileIdList": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "errors": "A call fails when the adapter raises an error; Bitween records the error's type and message on the exchange, and retry policies decide what happens next. A handler whose partner rejected the message does not fail: it returns the partner's answer with BadData true.", + "kinds": { + "handler": { + "description": "Delivers a message and returns the partner's response.", + "methods": [ + { + "name": "Handle", + "input": "ExchangeFile", + "output": "ExchangeFile", + "examples": [ + { + "Data": "{\"orderId\":\"SO-1001\",\"lines\":2}", + "Filename": "order-SO-1001.json", + "BadData": false, + "ContentType": "application/json" + } + ] + } + ] + }, + "mapper": { + "description": "Transforms a message into the shape the next step expects.", + "methods": [ + { + "name": "Handle", + "input": "ExchangeFile", + "output": "ExchangeFile", + "examples": [ + { + "Data": "{\"orderId\":\"SO-1001\",\"lines\":2}", + "Filename": "order-SO-1001.json", + "BadData": false, + "ContentType": "application/json" + } + ] + } + ] + }, + "validator": { + "description": "Checks a message before it is accepted.", + "methods": [ + { + "name": "Validate", + "input": "ExchangeFile", + "output": "ValidationResult", + "examples": [ + { + "Data": "{\"orderId\":\"SO-1001\",\"lines\":2}", + "Filename": "order-SO-1001.json", + "BadData": false, + "ContentType": "application/json" + } + ] + } + ] + }, + "receiver": { + "description": "Fetches files from a source on a schedule.", + "session": "One session per run. Calls come in this order: Initialize, ListFiles, then for each file GetFile and, once it is safely taken in, DeleteFile, and finally Finalize, which is also called after a failure.", + "methods": [ + { + "name": "Initialize", + "input": null, + "output": null + }, + { + "name": "ListFiles", + "input": null, + "output": "FileIdList" + }, + { + "name": "GetFile", + "input": "FileId", + "output": "ExchangeFile" + }, + { + "name": "DeleteFile", + "input": "FileId", + "output": null, + "destructive": true + }, + { + "name": "Finalize", + "input": null, + "output": null + } + ] + } + }, + "examples": "Example inputs for each method that takes one. serverless test calls the adapter with them, using the settings it is given, and checks each answer against the method's output type.", + "destructive": "A destructive method changes the source an adapter reads: serverless test calls it only when told it may." +} diff --git a/SW.Serverless.Tooling/Contracts/bitween/exchange-file.schema.json b/SW.Serverless.Tooling/Contracts/bitween/exchange-file.schema.json new file mode 100644 index 0000000..52a5fa0 --- /dev/null +++ b/SW.Serverless.Tooling/Contracts/bitween/exchange-file.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://bitween.systems/contract/v1/exchange-file.schema.json", + "title": "ExchangeFile", + "description": "A file as Bitween passes it to and from adapters. Property names are exactly as shown; unknown properties are allowed and ignored.", + "type": "object", + "required": ["Data"], + "properties": { + "Filename": { "type": ["string", "null"], "description": "The file's name, when it has one." }, + "Data": { "type": "string", "description": "The content: text, or base64 for binary content." }, + "Hash": { "type": ["string", "null"], "description": "SHA-1 of Data as lower-case hex. Written by .NET adapters; always recomputed from Data when read, never trusted." }, + "BadData": { "type": "boolean", "default": false, "description": "Whether this is a bad response: a delivery the partner rejected." }, + "ContentType": { "type": ["string", "null"], "description": "The content's media type, when known." } + }, + "additionalProperties": true +} diff --git a/SW.Serverless.Tooling/Contracts/bitween/validation-result.schema.json b/SW.Serverless.Tooling/Contracts/bitween/validation-result.schema.json new file mode 100644 index 0000000..97d1a83 --- /dev/null +++ b/SW.Serverless.Tooling/Contracts/bitween/validation-result.schema.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://bitween.systems/contract/v1/validation-result.schema.json", + "title": "ValidationResult", + "description": "What a validator found. Property names are exactly as shown; unknown properties are allowed and ignored.", + "type": "object", + "required": ["Validations"], + "properties": { + "Success": { "type": "boolean", "description": "True when Validations is empty. Bitween derives it from Validations." }, + "Validations": { + "type": "array", + "description": "Each failure: a code, often the field it concerns, and a message.", + "items": { + "type": "object", + "required": ["Key", "Value"], + "properties": { + "Key": { "type": "string" }, + "Value": { "type": "string" } + } + } + } + }, + "additionalProperties": true +} diff --git a/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj index 391b979..71b7936 100644 --- a/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj +++ b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj @@ -11,7 +11,7 @@ - + @@ -21,11 +21,21 @@ + + + + + + + + diff --git a/SW.Serverless.UnitTests.BitweenHandler/Program.cs b/SW.Serverless.UnitTests.BitweenHandler/Program.cs new file mode 100644 index 0000000..8396520 --- /dev/null +++ b/SW.Serverless.UnitTests.BitweenHandler/Program.cs @@ -0,0 +1,39 @@ +using System.Threading.Tasks; +using SW.Serverless.Sdk; + +namespace SW.Serverless.UnitTests.BitweenHandler; + +/// An exchange file as Bitween sends it. +public class ExchangeFile +{ + public string Data { get; set; } + public string Filename { get; set; } + public bool BadData { get; set; } + public string ContentType { get; set; } +} + +/// +/// A Bitween handler on the classic text protocol: delivers by echoing the order back. With Mode +/// set to "broken" it answers without the Data every exchange file must carry. +/// +[AdapterKind("handler")] +[AdapterContract("bitween", 1)] +public class Handler +{ + public Handler() + { + Runner.Expect("Endpoint", "https://partner.example.test/orders", description: "Where orders go."); + Runner.Expect("Mode", "working"); + Runner.Expect("ApiKey", isPrivate: true); + } + + public Task Handle(ExchangeFile file) => + Task.FromResult(Runner.StartupValueOf("Mode") == "broken" + ? new { Filename = "answer.json" } + : new ExchangeFile { Data = "{\"accepted\":true}", Filename = "answer.json", ContentType = "application/json" }); +} + +static class Program +{ + static Task Main() => Runner.Run(new Handler()); +} diff --git a/SW.Serverless.UnitTests.BitweenHandler/SW.Serverless.UnitTests.BitweenHandler.csproj b/SW.Serverless.UnitTests.BitweenHandler/SW.Serverless.UnitTests.BitweenHandler.csproj new file mode 100644 index 0000000..033b739 --- /dev/null +++ b/SW.Serverless.UnitTests.BitweenHandler/SW.Serverless.UnitTests.BitweenHandler.csproj @@ -0,0 +1,13 @@ + + + + Exe + net10.0 + false + + + + + + + diff --git a/SW.Serverless.UnitTests.BitweenReceiver/Program.cs b/SW.Serverless.UnitTests.BitweenReceiver/Program.cs new file mode 100644 index 0000000..6e15511 --- /dev/null +++ b/SW.Serverless.UnitTests.BitweenReceiver/Program.cs @@ -0,0 +1,47 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using SW.Serverless.Sdk; + +namespace SW.Serverless.UnitTests.BitweenReceiver; + +public class ExchangeFile +{ + public string Data { get; set; } + public string Filename { get; set; } +} + +/// +/// A Bitween receiver over gRPC that reads the files in a folder and deletes one when told to — +/// so a test can see whether DeleteFile was called. +/// +[AdapterKind("receiver")] +[AdapterContract("bitween", 1)] +public class Handler +{ + public Handler() => Runner.Expect("Folder"); + + string Folder => Runner.StartupValueOf("Folder"); + + public Task Initialize() => Task.CompletedTask; + + public Task> ListFiles() => + Task.FromResult>(Directory.GetFiles(Folder).Select(Path.GetFileName).OrderBy(n => n).ToList()); + + public Task GetFile(string fileId) => + Task.FromResult(new ExchangeFile { Data = File.ReadAllText(Path.Combine(Folder, fileId)), Filename = fileId }); + + public Task DeleteFile(string fileId) + { + File.Delete(Path.Combine(Folder, fileId)); + return Task.CompletedTask; + } + + public Task Finalize() => Task.CompletedTask; +} + +static class Program +{ + static Task Main() => Runner.RunResident(new Handler()); +} diff --git a/SW.Serverless.UnitTests.BitweenReceiver/SW.Serverless.UnitTests.BitweenReceiver.csproj b/SW.Serverless.UnitTests.BitweenReceiver/SW.Serverless.UnitTests.BitweenReceiver.csproj new file mode 100644 index 0000000..033b739 --- /dev/null +++ b/SW.Serverless.UnitTests.BitweenReceiver/SW.Serverless.UnitTests.BitweenReceiver.csproj @@ -0,0 +1,13 @@ + + + + Exe + net10.0 + false + + + + + + + diff --git a/SW.Serverless.sln b/SW.Serverless.sln index dc01398..8f2b7f4 100644 --- a/SW.Serverless.sln +++ b/SW.Serverless.sln @@ -61,6 +61,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.UnitTests.Grp EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.Tooling", "SW.Serverless.Tooling\SW.Serverless.Tooling.csproj", "{C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.UnitTests.BitweenHandler", "SW.Serverless.UnitTests.BitweenHandler\SW.Serverless.UnitTests.BitweenHandler.csproj", "{DCF81A88-37CA-4746-A4D8-EADEF00422FD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.UnitTests.BitweenReceiver", "SW.Serverless.UnitTests.BitweenReceiver\SW.Serverless.UnitTests.BitweenReceiver.csproj", "{4D824901-1889-4BB2-9F72-2D33E5CFA7B3}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -383,6 +387,30 @@ Global {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|x64.Build.0 = Release|Any CPU {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|x86.ActiveCfg = Release|Any CPU {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|x86.Build.0 = Release|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Debug|x64.ActiveCfg = Debug|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Debug|x64.Build.0 = Debug|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Debug|x86.ActiveCfg = Debug|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Debug|x86.Build.0 = Debug|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Release|Any CPU.Build.0 = Release|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Release|x64.ActiveCfg = Release|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Release|x64.Build.0 = Release|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Release|x86.ActiveCfg = Release|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Release|x86.Build.0 = Release|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Debug|x64.ActiveCfg = Debug|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Debug|x64.Build.0 = Debug|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Debug|x86.ActiveCfg = Debug|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Debug|x86.Build.0 = Debug|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Release|Any CPU.Build.0 = Release|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Release|x64.ActiveCfg = Release|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Release|x64.Build.0 = Release|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Release|x86.ActiveCfg = Release|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/SW.Serverless/Runtimes/AdapterRuntimes.cs b/SW.Serverless/Runtimes/AdapterRuntimes.cs index 1e75b8b..bdbec37 100644 --- a/SW.Serverless/Runtimes/AdapterRuntimes.cs +++ b/SW.Serverless/Runtimes/AdapterRuntimes.cs @@ -39,6 +39,9 @@ public sealed class RuntimeLaunch /// .NET only: workstation GC, the right trade for an adapter that mostly waits. public bool UseWorkstationGc { get; set; } + + /// Arguments after the entry, such as --describe. + public System.Collections.Generic.IReadOnlyList Arguments { get; set; } = Array.Empty(); } /// @@ -163,6 +166,7 @@ public ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch) { var psi = Redirected(executable, Path.GetDirectoryName(entryPath)); psi.ArgumentList.Add(entryPath); + foreach (var argument in launch.Arguments) psi.ArgumentList.Add(argument); if (launch.UseWorkstationGc) psi.Environment["DOTNET_gcServer"] = "0"; if (launch.HardMemoryLimitBytes > 0) { @@ -181,8 +185,12 @@ sealed class ExecRuntime : IAdapterRuntime // Nothing to install: the package carries everything it needs. public Task DetectAsync() => Task.FromResult(RuntimeStatus.Present("native")); - public ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch) => - Redirected(entryPath, Path.GetDirectoryName(entryPath)); + public ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch) + { + var psi = Redirected(entryPath, Path.GetDirectoryName(entryPath)); + foreach (var argument in launch.Arguments) psi.ArgumentList.Add(argument); + return psi; + } } sealed class PythonRuntime(string executable) : IAdapterRuntime @@ -197,6 +205,7 @@ public ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch) // Unbuffered, so what it writes reaches the host's diagnostics when it's written. psi.ArgumentList.Add("-u"); psi.ArgumentList.Add(entryPath); + foreach (var argument in launch.Arguments) psi.ArgumentList.Add(argument); psi.Environment["PYTHONUNBUFFERED"] = "1"; psi.Environment["PYTHONDONTWRITEBYTECODE"] = "1"; // Python has no heap ceiling of its own; the watchdog and, where the container @@ -219,6 +228,7 @@ public ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch) // adapter, as DOTNET_GCHeapHardLimit does for .NET. psi.ArgumentList.Add($"--max-old-space-size={Math.Max(16, launch.HardMemoryLimitBytes / (1024 * 1024))}"); psi.ArgumentList.Add(entryPath); + foreach (var argument in launch.Arguments) psi.ArgumentList.Add(argument); return psi; } } From 45dfccdd175a72498c8b8abb7aadbb4e8f6ae798 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 10:40:01 +0300 Subject: [PATCH 10/14] Build adapter packages that describe themselves and carry their source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PackageBuilder (what serverless build will run) takes an adapter project with its adapter.json and makes a package any host runs: it builds it, asks it to describe itself, and writes the manifest from both — the author's id, names and presentation of settings; the adapter's own settings (required, secret, default), kinds, contracts, lifecycle and SDK version, which win. A gRPC adapter the author marks classic is built to run classically over gRPC. The package carries the project's source in source/, with every file's SHA-256, the build command and lockfiles in the manifest. Carried: the project and the local projects it references, so it rebuilds. Left out: build output and dependencies, editor and git folders, files that hold secrets, and whatever .gitignore and .serverlessignore say. Everything carried is scanned for private keys, cloud and service tokens, passwords in connection strings and secrets assigned in code; a finding stops the build naming file and line, until that file is allowed. Source can be left out entirely, and a dry run reports what would be carried without building. LocalAdapterHost, taken out of the conformance kit, runs a package on this machine as a host does, for serverless run and test alike. Tested on a real author project: the manifest it writes, the source rules (.env, a .serverlessignore'd log, bin and obj left out; the referenced SDK project carried), the zip's layout, that what it builds passes the conformance kit, a planted key stopping the build until allowed, no source, a dry run, the gRPC-classic opt-in; plus the ignore rules and which lines the scanner does and doesn't report. --- .gitignore | 1 + .../BuildTests.cs | 244 +++++++++++++ SW.Serverless.Tooling/Building/IgnoreRules.cs | 92 +++++ .../Building/PackageBuilder.cs | 322 ++++++++++++++++++ .../Building/SecretScanner.cs | 58 ++++ .../Conformance/ConformanceRunner.cs | 222 ++---------- SW.Serverless.Tooling/LocalAdapterHost.cs | 235 +++++++++++++ 7 files changed, 978 insertions(+), 196 deletions(-) create mode 100644 SW.Serverless.Installer.UnitTests/BuildTests.cs create mode 100644 SW.Serverless.Tooling/Building/IgnoreRules.cs create mode 100644 SW.Serverless.Tooling/Building/PackageBuilder.cs create mode 100644 SW.Serverless.Tooling/Building/SecretScanner.cs create mode 100644 SW.Serverless.Tooling/LocalAdapterHost.cs diff --git a/.gitignore b/.gitignore index 1139e7b..3e29084 100644 --- a/.gitignore +++ b/.gitignore @@ -337,3 +337,4 @@ ASALocalRun/ binaries out +.test-work/ diff --git a/SW.Serverless.Installer.UnitTests/BuildTests.cs b/SW.Serverless.Installer.UnitTests/BuildTests.cs new file mode 100644 index 0000000..27fe5e0 --- /dev/null +++ b/SW.Serverless.Installer.UnitTests/BuildTests.cs @@ -0,0 +1,244 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Tooling.Building; +using SW.Serverless.Tooling.Conformance; + +namespace SW.Serverless.Installer.UnitTests; + +[TestClass] +public class IgnoreRuleTests +{ + [DataTestMethod] + [DataRow("bin/Debug/net10.0/a.dll", true)] + [DataRow("src/obj/project.assets.json", true)] + [DataRow("node_modules/x/index.js", true)] + [DataRow(".env", true)] + [DataRow("config/.env.production", true)] + [DataRow("keys/partner.pem", true)] + [DataRow("appsettings.Production.json", true)] + [DataRow("appsettings.json", false)] + [DataRow("Handler.cs", false)] + [DataRow("src/binder/Thing.cs", false)] + public void Built_in_rules(string path, bool ignored) => + Assert.AreEqual(ignored, new IgnoreRules(IgnoreRules.BuiltIn).Ignores(path), path); + + [TestMethod] + public void A_project_s_own_patterns_add_to_them_and_can_bring_a_file_back() + { + var rules = new IgnoreRules(IgnoreRules.BuiltIn.Concat(new[] { "fixtures/**/*.zip", "/local.txt", "*.log", "!keep.log", "!keys/public.pem" })); + + Assert.IsTrue(rules.Ignores("fixtures/a/b/big.zip")); + Assert.IsTrue(rules.Ignores("fixtures/big.zip")); + Assert.IsTrue(rules.Ignores("local.txt")); + Assert.IsFalse(rules.Ignores("sub/local.txt"), "a leading slash anchors to the root"); + Assert.IsTrue(rules.Ignores("logs/run.log")); + Assert.IsFalse(rules.Ignores("keep.log")); + Assert.IsFalse(rules.Ignores("keys/public.pem")); + } +} + +[TestClass] +public class SecretScannerTests +{ + [DataTestMethod] + [DataRow("-----BEGIN RSA PRIVATE KEY-----", "a private key")] + [DataRow("-----BEGIN OPENSSH PRIVATE KEY-----", "a private key")] + [DataRow("var key = \"AKIAIOSFODNN7EXAMPLE\";", "an AWS access key")] + [DataRow("\"Db\": \"Server=x;User Id=sa;Password=Sup3rS3cret!;\"", "a password in a connection string")] + [DataRow("const apiKey = \"9f8e7d6c5b4a3f2e1d0c\";", "a secret assigned in code")] + [DataRow("client_secret: 'abcdefgh12345678'", "a secret assigned in code")] + [DataRow("token = ghp_abcdefghijklmnopqrstuvwxyz0123456789AB", "a GitHub token")] + public void A_secret_is_found_and_named(string line, string kind) + { + var finding = SecretScanner.Scan("Handler.cs", "first line\n" + line + "\n").Single(); + Assert.AreEqual(2, finding.Line); + Assert.AreEqual(kind, finding.Kind); + } + + [DataTestMethod] + [DataRow("Runner.Expect(\"Password\", isPrivate: true);")] + [DataRow("var password = Runner.StartupValueOf(\"Password\");")] + [DataRow("\"Password\": \"********\"")] + [DataRow("apiKey = \"your-api-key-here\"")] + [DataRow("secret: \"${SECRET}\"")] + [DataRow("Password={password};")] + public void Code_that_only_names_or_reads_a_secret_is_not_reported(string line) => + Assert.AreEqual(0, SecretScanner.Scan("Handler.cs", line).Count(), line); +} + +/// +/// serverless build on a real adapter project: built, described, its manifest written from the +/// adapter itself, its source carried under the rules — and the package it makes is what the +/// conformance kit and any host run. +/// +[TestClass] +public class BuildTests +{ + static string RepositoryRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null && !File.Exists(Path.Combine(dir.FullName, "SW.Serverless.sln"))) dir = dir.Parent; + return dir!.FullName; + } + + /// + /// A copy of the BitweenHandler test adapter as an author's project: inside the repository, so + /// its reference to the SDK project still resolves, with an adapter.json and a few files the + /// source rules must leave out. + /// + static string AuthorProject(string authorJson = null, IDictionary extraFiles = null) + { + var root = RepositoryRoot(); + var project = Path.Combine(root, ".test-work", Guid.NewGuid().ToString("N"), "AcmeOrders"); + Directory.CreateDirectory(project); + var sdk = Path.GetRelativePath(project, Path.Combine(root, "SW.Serverless.Sdk", "SW.Serverless.Sdk.csproj")); + File.WriteAllText(Path.Combine(project, "AcmeOrders.csproj"), $""" + + + Exe + net10.0 + + + + + + """); + File.Copy(Path.Combine(root, "SW.Serverless.UnitTests.BitweenHandler", "Program.cs"), Path.Combine(project, "Program.cs")); + File.WriteAllText(Path.Combine(project, AdapterManifest.FileName), authorJson ?? + """{ "id": "acme.orders", "version": "1.2.0", "displayName": "Acme orders", "properties": [ { "name": "Mode", "type": "select", "options": ["working", "broken"] } ] }"""); + File.WriteAllText(Path.Combine(project, ".env"), "API_KEY=never-carried"); + File.WriteAllText(Path.Combine(project, "notes.log"), "left out by .serverlessignore"); + File.WriteAllText(Path.Combine(project, ".serverlessignore"), "*.log\n"); + foreach (var (name, content) in extraFiles ?? new Dictionary()) + File.WriteAllText(Path.Combine(project, name), content); + return project; + } + + [ClassCleanup] + public static void Cleanup() + { + try { Directory.Delete(Path.Combine(RepositoryRoot(), ".test-work"), true); } catch { } + } + + static Task Build(string project, Action configure = null) + { + var request = new BuildRequest { ProjectDirectory = project, Log = Console.WriteLine }; + configure?.Invoke(request); + return PackageBuilder.BuildAsync(request); + } + + [TestMethod] + public async Task A_build_writes_the_manifest_from_the_adapter_and_carries_its_source_under_the_rules() + { + var project = AuthorProject(); + var result = await Build(project); + Assert.IsTrue(result.Succeeded, string.Join("; ", result.Problems)); + + var manifest = result.Manifest; + Assert.AreEqual("acme.orders", manifest.Id); + Assert.AreEqual("1.2.0", manifest.Version); + Assert.AreEqual("Acme orders", manifest.DisplayName, "the author's text is kept"); + Assert.AreEqual("dotnet", manifest.Runtime); + Assert.AreEqual("AcmeOrders.dll", manifest.Entry); + Assert.AreEqual("10.1.0", manifest.SdkVersion); + Assert.AreEqual(AdapterManifest.ClassicLifecycle, manifest.Lifecycle); + CollectionAssert.AreEqual(new[] { "handler" }, manifest.Kinds); + Assert.AreEqual(1, manifest.Contracts!["bitween"]); + + var mode = manifest.Properties.Single(p => p.Name == "Mode"); + Assert.AreEqual("select", mode.Type, "the author's presentation is kept"); + Assert.AreEqual("working", mode.Default, "the adapter's default wins"); + var apiKey = manifest.Properties.Single(p => p.Name == "ApiKey"); + Assert.IsTrue(apiKey.Required); + Assert.IsTrue(apiKey.Secret); + Assert.IsNull(apiKey.Default); + + var carried = manifest.Source!.Files.Keys.ToList(); + Assert.IsTrue(carried.Any(f => f.EndsWith("AcmeOrders/Program.cs")), string.Join(", ", carried)); + Assert.IsTrue(carried.Any(f => f.EndsWith("AcmeOrders/AcmeOrders.csproj"))); + Assert.IsTrue(carried.Any(f => f.EndsWith("SW.Serverless.Sdk/Runner.cs")), "the local project it references, so it rebuilds"); + Assert.IsFalse(carried.Any(f => f.EndsWith(".env") || f.EndsWith("notes.log") || f.Contains("/bin/") || f.Contains("/obj/")), + string.Join(", ", carried.Where(f => f.EndsWith(".env") || f.EndsWith(".log") || f.Contains("/bin/") || f.Contains("/obj/")))); + + using var zip = ZipFile.OpenRead(result.ZipPath); + var names = zip.Entries.Select(e => e.FullName.Replace('\\', '/')).ToList(); + CollectionAssert.Contains(names, "adapter.json"); + CollectionAssert.Contains(names, "AcmeOrders.dll"); + Assert.IsTrue(names.Any(n => n.StartsWith("source/") && n.EndsWith("AcmeOrders/Program.cs"))); + Assert.IsTrue(result.ZipPath.EndsWith("acme.orders-1.2.0.zip")); + } + + [TestMethod] + public async Task What_a_build_makes_passes_the_conformance_kit() + { + var result = await Build(AuthorProject()); + Assert.IsTrue(result.Succeeded, string.Join("; ", result.Problems)); + + var report = await new ConformanceRunner().RunAsync(new ConformanceOptions + { + PackageDirectory = result.PackageDirectory, + Settings = new Dictionary { ["ApiKey"] = "k" }, + }); + Assert.IsTrue(report.Passed, string.Join("; ", report.Checks.Where(c => c.Outcome == CheckOutcome.Failed))); + } + + [TestMethod] + public async Task A_secret_in_the_source_stops_the_build_until_it_is_allowed() + { + var project = AuthorProject(extraFiles: new Dictionary + { + ["Defaults.cs"] = "class Defaults { const string ApiKey = \"live-9f8e7d6c5b4a3f2e\"; }", + }); + + var stopped = await Build(project); + Assert.IsFalse(stopped.Succeeded); + var problem = stopped.Problems.Single(); + StringAssert.Contains(problem, "Defaults.cs:1"); + StringAssert.Contains(problem, "--allow"); + Assert.IsNull(stopped.ZipPath, "nothing was built"); + + var allowed = await Build(project, r => r.AllowedFiles.Add("Defaults.cs")); + Assert.IsTrue(allowed.Succeeded, string.Join("; ", allowed.Problems)); + } + + [TestMethod] + public async Task Without_source_the_package_carries_none() + { + var result = await Build(AuthorProject(), r => r.IncludeSource = false); + + Assert.IsTrue(result.Succeeded, string.Join("; ", result.Problems)); + Assert.IsNull(result.Manifest.Source); + Assert.IsFalse(Directory.Exists(Path.Combine(result.PackageDirectory, "source"))); + } + + [TestMethod] + public async Task A_dry_run_reports_the_source_and_builds_nothing() + { + var project = AuthorProject(); + var result = await Build(project, r => r.DryRun = true); + + Assert.IsTrue(result.Succeeded); + Assert.IsTrue(result.SourceFiles.Any(f => f.Path.EndsWith("Program.cs"))); + Assert.IsNull(result.ZipPath); + Assert.IsFalse(Directory.Exists(Path.Combine(project, "bin", "serverless"))); + } + + [TestMethod] + public async Task A_gRPC_adapter_the_author_marks_classic_is_built_to_run_classically_over_gRPC() + { + var root = RepositoryRoot(); + var project = AuthorProject("""{ "id": "acme.grpc", "lifecycle": "classic" }"""); + File.Copy(Path.Combine(root, "SW.Serverless.UnitTests.GrpcClassicAdapter", "Program.cs"), Path.Combine(project, "Program.cs"), overwrite: true); + + var result = await Build(project); + Assert.IsTrue(result.Succeeded, string.Join("; ", result.Problems)); + Assert.AreEqual(AdapterManifest.ClassicLifecycle, result.Manifest.Lifecycle); + Assert.AreEqual(2, result.Manifest.Protocol!.Min); + } +} diff --git a/SW.Serverless.Tooling/Building/IgnoreRules.cs b/SW.Serverless.Tooling/Building/IgnoreRules.cs new file mode 100644 index 0000000..cf64dde --- /dev/null +++ b/SW.Serverless.Tooling/Building/IgnoreRules.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; + +namespace SW.Serverless.Tooling.Building +{ + /// + /// Which files of a project stay out of the source a package carries: a built-in list — build + /// output, dependencies, editor folders and the kinds of file that hold secrets — then the + /// project's .gitignore and .serverlessignore, read with gitignore's rules (*, **, a trailing / + /// for folders, a leading / to anchor, ! to bring a file back). + /// + public class IgnoreRules + { + /// Always left out, whatever the project's ignore files say. + public static readonly IReadOnlyList BuiltIn = new[] + { + // Build output and downloaded dependencies. + "bin/", "obj/", "dist/", "build/", "out/", "target/", "__pycache__/", "*.pyc", ".venv/", "venv/", + "node_modules/", ".pytest_cache/", ".mypy_cache/", "packages/", "*.nupkg", + // Version control and editors. + ".git/", ".vs/", ".idea/", ".vscode/", "*.user", "*.suo", ".DS_Store", + // What holds secrets. + ".env", ".env.*", "*.pem", "*.key", "*.pfx", "*.p12", "*.jks", "id_rsa*", "id_ed25519*", "id_ecdsa*", + "secrets.json", "*.publishsettings", "*.pubxml", "*.pubxml.user", "appsettings.*.json", + }; + + readonly List<(Regex Pattern, bool Negated, bool DirectoryOnly)> rules = new(); + + public IgnoreRules(IEnumerable lines) + { + foreach (var raw in lines) + { + var line = raw.TrimEnd(); + if (line.Length == 0 || line.StartsWith('#')) continue; + + var negated = line.StartsWith('!'); + if (negated) line = line[1..]; + var directoryOnly = line.EndsWith('/'); + line = line.TrimEnd('/'); + var anchored = line.StartsWith('/') || line.Contains('/'); + line = line.TrimStart('/'); + + rules.Add((new Regex("^" + (anchored ? "" : "(.*/)?") + Translate(line) + (directoryOnly ? "(/.*)?$" : "(/.*)?$"), + RegexOptions.CultureInvariant), negated, directoryOnly)); + } + } + + /// The built-in list plus the project's own .gitignore and .serverlessignore. + public static IgnoreRules For(string projectDirectory) + { + var lines = new List(BuiltIn); + foreach (var file in new[] { ".gitignore", ".serverlessignore" }) + { + var path = Path.Combine(projectDirectory, file); + if (File.Exists(path)) lines.AddRange(File.ReadAllLines(path)); + } + return new IgnoreRules(lines); + } + + /// Whether a path, relative to the project and with forward slashes, is left out. + public bool Ignores(string relativePath) + { + var path = relativePath.Replace('\\', '/').TrimStart('/'); + var ignored = false; + foreach (var (pattern, negated, _) in rules) + if (pattern.IsMatch(path)) ignored = !negated; + return ignored; + } + + static string Translate(string glob) + { + var result = new System.Text.StringBuilder(); + for (var i = 0; i < glob.Length; i++) + { + var c = glob[i]; + if (c == '*' && i + 1 < glob.Length && glob[i + 1] == '*') + { + // ** crosses folders; "**/" may also match nothing. + if (i + 2 < glob.Length && glob[i + 2] == '/') { result.Append("(.*/)?"); i += 2; } + else { result.Append(".*"); i++; } + } + else if (c == '*') result.Append("[^/]*"); + else if (c == '?') result.Append("[^/]"); + else result.Append(Regex.Escape(c.ToString())); + } + return result.ToString(); + } + } +} diff --git a/SW.Serverless.Tooling/Building/PackageBuilder.cs b/SW.Serverless.Tooling/Building/PackageBuilder.cs new file mode 100644 index 0000000..01c7ce5 --- /dev/null +++ b/SW.Serverless.Tooling/Building/PackageBuilder.cs @@ -0,0 +1,322 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Threading.Tasks; +using System.Xml.Linq; +using SW.Serverless.Contract.Catalog; + +namespace SW.Serverless.Tooling.Building +{ + public class BuildRequest + { + /// The adapter's project folder, with its adapter.json. + public string ProjectDirectory { get; set; } + + /// Where the package folder and zip go; bin/serverless in the project unless set, which the source rules leave out. + public string OutputDirectory { get; set; } + + /// Carry the source in the package. On unless a partner won't hand it over. + public bool IncludeSource { get; set; } = true; + + /// Source files whose secret-scan findings are false positives, relative to the project. + public ISet AllowedFiles { get; set; } = new HashSet(StringComparer.OrdinalIgnoreCase); + + /// Report what would be carried without building anything. + public bool DryRun { get; set; } + + public Runtimes.AdapterRuntimeOptions Runtimes { get; set; } = new(); + public Action Log { get; set; } = _ => { }; + } + + public class BuildResult + { + public bool Succeeded => Problems.Count == 0; + public List Problems { get; } = new(); + public List Warnings { get; } = new(); + public AdapterManifest Manifest { get; set; } + public string PackageDirectory { get; set; } + public string ZipPath { get; set; } + + /// The source carried, relative to its root, with sizes. + public List<(string Path, long Bytes)> SourceFiles { get; } = new(); + public List Findings { get; } = new(); + } + + /// + /// What serverless build does, for an adapter in any language its SDK supports: builds it, + /// asks it to describe itself, writes its manifest from that and the author's adapter.json, + /// carries its source under the source rules, and zips the result into a package any host runs. + /// + public static class PackageBuilder + { + /// The source folder size past which the build warns: usually build output slipped in. + public const long SourceWarningBytes = 10 * 1024 * 1024; + + static readonly string[] Lockfiles = + { "packages.lock.json", "poetry.lock", "requirements.lock", "requirements.txt", "uv.lock", "Pipfile.lock", + "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "go.sum" }; + + public static async Task BuildAsync(BuildRequest request) + { + var result = new BuildResult(); + var project = Path.GetFullPath(request.ProjectDirectory); + + var authorPath = Path.Combine(project, AdapterManifest.FileName); + if (!File.Exists(authorPath)) + { + result.Problems.Add($"there is no {AdapterManifest.FileName} in {project}; serverless init writes one"); + return result; + } + var authorJson = await File.ReadAllTextAsync(authorPath); + var author = AdapterManifest.Parse(authorJson); + if (string.IsNullOrWhiteSpace(author.Id)) + { + result.Problems.Add($"{AdapterManifest.FileName} needs an id"); + return result; + } + + var runtime = string.IsNullOrWhiteSpace(author.Runtime) ? AdapterManifest.DotnetRuntime : author.Runtime; + if (!string.Equals(runtime, AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase)) + { + result.Problems.Add($"building a '{runtime}' adapter arrives with that language's SDK; this build does .NET"); + return result; + } + + var projectFiles = Directory.GetFiles(project, "*.*proj").Where(f => f.EndsWith(".csproj") || f.EndsWith(".fsproj") || f.EndsWith(".vbproj")).ToList(); + if (projectFiles.Count != 1) + { + result.Problems.Add(projectFiles.Count == 0 + ? $"there is no .NET project file in {project}" + : $"there are {projectFiles.Count} project files in {project}; keep one adapter per folder"); + return result; + } + var projectFile = projectFiles[0]; + + // Source first: a secret found here stops the build before anything is produced. + var source = request.IncludeSource ? CollectSource(project, projectFile, request, result) : null; + if (!result.Succeeded || request.DryRun) return result; + + var output = Path.GetFullPath(request.OutputDirectory ?? Path.Combine(project, "bin", "serverless")); + var packageDirectory = Path.Combine(output, "package"); + if (Directory.Exists(packageDirectory)) Directory.Delete(packageDirectory, true); + Directory.CreateDirectory(packageDirectory); + + request.Log("Building..."); + if (!new InstallerLogic().BuildPublish(projectFile, packageDirectory)) + { + result.Problems.Add("dotnet publish failed; its output above says why"); + return result; + } + + var entry = InstallerLogic.ResolveEntryAssembly(packageDirectory, projectFile); + if (entry == null) + { + result.Problems.Add("the built output has no entry assembly"); + return result; + } + + request.Log("Asking the adapter to describe itself..."); + var (description, problem) = await LocalAdapterHost.DescribeAsync(Path.Combine(packageDirectory, entry), + AdapterManifest.DotnetRuntime, request.Runtimes); + if (description == null) + { + result.Problems.Add($"{problem}. serverless build needs SimplyWorks.Serverless.Sdk 10.1.0 or later; " + + "an adapter on an older SDK is published with serverless as before"); + return result; + } + result.Warnings.AddRange(description.Warnings); + + var manifest = ManifestFrom(author, AuthorChoseClassic(authorJson), description, entry, ManifestBuilder.LanguageOf(projectFile)); + + if (source != null) + { + var sourceDirectory = Path.Combine(packageDirectory, AdapterSource.DefaultPath); + manifest.Source = new AdapterSource + { + BuildCommand = "dotnet publish -c Release", + Lockfiles = source.Keys.Where(k => Lockfiles.Contains(Path.GetFileName(k), StringComparer.OrdinalIgnoreCase)).OrderBy(k => k).ToList(), + }; + foreach (var (relative, absolute) in source.OrderBy(s => s.Key, StringComparer.Ordinal)) + { + var target = Path.Combine(sourceDirectory, relative); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(absolute, target); + manifest.Source.Files[relative] = Convert.ToHexString(SHA256.HashData(await File.ReadAllBytesAsync(absolute))).ToLowerInvariant(); + } + if (manifest.Source.Lockfiles.Count == 0) manifest.Source.Lockfiles = null; + } + + var problems = manifest.Validate(); + if (problems.Count > 0) + { + result.Problems.AddRange(problems); + return result; + } + + await File.WriteAllTextAsync(Path.Combine(packageDirectory, AdapterManifest.FileName), manifest.ToJson()); + + var zip = Path.Combine(output, $"{manifest.Id}{(string.IsNullOrWhiteSpace(manifest.Version) ? "" : "-" + manifest.Version)}.zip"); + if (File.Exists(zip)) File.Delete(zip); + ZipFile.CreateFromDirectory(packageDirectory, zip, CompressionLevel.Optimal, includeBaseDirectory: false); + + result.Manifest = manifest; + result.PackageDirectory = packageDirectory; + result.ZipPath = zip; + return result; + } + + /// + /// The manifest: the author's file — id, names, marketplace text, presentation of settings — + /// with what the adapter says about itself, which wins wherever the two speak to the same thing. + /// + internal static AdapterManifest ManifestFrom(AdapterManifest author, bool authorChoseClassic, + AdapterSelfDescription description, string entry, string language) + { + var manifest = AdapterManifest.Parse(author.ToJson()); + manifest.Entry = entry; + manifest.Runtime = AdapterManifest.DotnetRuntime; + manifest.Language ??= language; + manifest.SdkVersion = description.SdkVersion; + + if (description.Lifecycle == AdapterManifest.ResidentLifecycle) + { + // It speaks gRPC. Run classically only when the author says so — the opt-in for a + // .NET adapter on the gRPC protocol — and as a resident otherwise. + manifest.Lifecycle = authorChoseClassic ? AdapterManifest.ClassicLifecycle : AdapterManifest.ResidentLifecycle; + manifest.Protocol = new AdapterProtocolRange { Min = 2, Max = 2 }; + } + else + { + manifest.Lifecycle = AdapterManifest.ClassicLifecycle; + manifest.Protocol = null; + } + + manifest.Kinds = (author.Kinds ?? new List()).Union(description.Kinds, StringComparer.OrdinalIgnoreCase).ToList(); + var contracts = new Dictionary(author.Contracts ?? new Dictionary(), StringComparer.OrdinalIgnoreCase); + foreach (var (name, version) in description.Contracts) contracts[name] = version; + manifest.Contracts = contracts.Count == 0 ? null : contracts; + + var authored = (author.Properties ?? new List()).ToDictionary(p => p.Name, StringComparer.OrdinalIgnoreCase); + manifest.Properties = description.Settings.Select(setting => + { + var property = authored.TryGetValue(setting.Name, out var a) ? a : new AdapterProperty { Name = setting.Name }; + property.Name = setting.Name; + property.Required = setting.Required; + property.Secret = setting.Secret; + property.Default = setting.Secret ? null : setting.Default; + if (!string.IsNullOrWhiteSpace(setting.Description)) property.Description = setting.Description; + if (string.IsNullOrWhiteSpace(property.Type) || property.Type == AdapterProperty.TextType) + property.Type = string.IsNullOrWhiteSpace(setting.Type) ? AdapterProperty.TextType : setting.Type; + return property; + }).ToList(); + + return manifest; + } + + /// + /// Whether the author's own adapter.json says "lifecycle": "classic" — read from the file, + /// since the model fills in classic when nothing is said. + /// + internal static bool AuthorChoseClassic(string authorJson) + { + using var document = System.Text.Json.JsonDocument.Parse(authorJson, new System.Text.Json.JsonDocumentOptions + { + CommentHandling = System.Text.Json.JsonCommentHandling.Skip, + AllowTrailingCommas = true, + }); + return document.RootElement.TryGetProperty("lifecycle", out var lifecycle) && + string.Equals(lifecycle.GetString(), AdapterManifest.ClassicLifecycle, StringComparison.OrdinalIgnoreCase); + } + + /// + /// The source to carry, keyed by its path in the package's source folder: the project and + /// the local projects it references, under the ignore rules, scanned for secrets. + /// + static Dictionary CollectSource(string project, string projectFile, BuildRequest request, BuildResult result) + { + var roots = new List { project }; + foreach (var referenced in LocalReferences(projectFile, new HashSet(StringComparer.Ordinal))) + if (!roots.Contains(referenced)) roots.Add(referenced); + + var common = CommonAncestor(roots); + var repository = RepositoryRoot(project); + if (repository != null && !common.StartsWith(repository, StringComparison.Ordinal)) + { + result.Warnings.Add($"a referenced project is outside the repository ({common}); the source carried may not rebuild on another machine"); + } + + var files = new Dictionary(StringComparer.Ordinal); + long total = 0; + foreach (var root in roots) + { + var rules = IgnoreRules.For(root); + foreach (var file in Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories)) + { + var withinRoot = Path.GetRelativePath(root, file).Replace('\\', '/'); + if (rules.Ignores(withinRoot)) continue; + var relative = Path.GetRelativePath(common, file).Replace('\\', '/'); + if (files.ContainsKey(relative)) continue; + + var bytes = File.ReadAllBytes(file); + if (SecretScanner.IsText(bytes)) + { + var withinProject = Path.GetRelativePath(project, file).Replace('\\', '/'); + if (!request.AllowedFiles.Contains(withinProject) && !request.AllowedFiles.Contains(relative)) + result.Findings.AddRange(SecretScanner.Scan(relative, System.Text.Encoding.UTF8.GetString(bytes))); + } + + files[relative] = file; + total += bytes.Length; + result.SourceFiles.Add((relative, bytes.Length)); + } + } + + foreach (var finding in result.Findings) + result.Problems.Add($"{finding.File}:{finding.Line} looks like {finding.Kind}. Remove it, or pass --allow {finding.File} if it isn't one"); + + if (total > SourceWarningBytes) + result.Warnings.Add($"the source is {total / (1024 * 1024)} MB; build output may have slipped in — add it to .serverlessignore"); + + return files; + } + + /// The folders of the projects a project references, all the way down. + static IEnumerable LocalReferences(string projectFile, HashSet seen) + { + if (!seen.Add(projectFile)) yield break; + XDocument document; + try { document = XDocument.Load(projectFile); } + catch { yield break; } + + foreach (var include in document.Descendants().Where(e => e.Name.LocalName == "ProjectReference") + .Select(e => (string)e.Attribute("Include")).Where(i => !string.IsNullOrWhiteSpace(i))) + { + var path = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(projectFile)!, include.Replace('\\', Path.DirectorySeparatorChar))); + if (!File.Exists(path)) continue; + yield return Path.GetDirectoryName(path)!; + foreach (var deeper in LocalReferences(path, seen)) yield return deeper; + } + } + + static string CommonAncestor(IEnumerable folders) + { + var split = folders.Select(f => Path.GetFullPath(f).TrimEnd(Path.DirectorySeparatorChar).Split(Path.DirectorySeparatorChar)).ToList(); + var length = split.Min(s => s.Length); + var i = 0; + while (i < length && split.All(s => s[i] == split[0][i])) i++; + var common = string.Join(Path.DirectorySeparatorChar, split[0].Take(i)); + return string.IsNullOrEmpty(common) ? Path.DirectorySeparatorChar.ToString() : common; + } + + static string RepositoryRoot(string folder) + { + for (var dir = new DirectoryInfo(folder); dir != null; dir = dir.Parent) + if (Directory.Exists(Path.Combine(dir.FullName, ".git")) || File.Exists(Path.Combine(dir.FullName, ".git"))) + return dir.FullName; + return null; + } + } +} diff --git a/SW.Serverless.Tooling/Building/SecretScanner.cs b/SW.Serverless.Tooling/Building/SecretScanner.cs new file mode 100644 index 0000000..49b4fca --- /dev/null +++ b/SW.Serverless.Tooling/Building/SecretScanner.cs @@ -0,0 +1,58 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; + +namespace SW.Serverless.Tooling.Building +{ + /// Something in a source file that looks like a secret. + public record SecretFinding(string File, int Line, string Kind); + + /// + /// Looks through the source a package is about to carry for what shouldn't travel with it: + /// private keys, cloud and service tokens, and passwords or keys written into code or + /// configuration. Errs towards reporting; a false positive is let through with --allow. + /// + public static class SecretScanner + { + static readonly (string Kind, Regex Pattern)[] Patterns = + { + ("a private key", new Regex(@"-----BEGIN ([A-Z ]+ )?PRIVATE KEY-----")), + ("an AWS access key", new Regex(@"\b(AKIA|ASIA)[0-9A-Z]{16}\b")), + ("a GitHub token", new Regex(@"\bgh[pousr]_[A-Za-z0-9]{36,}\b")), + ("a Slack token", new Regex(@"\bxox[abprs]-[A-Za-z0-9-]{10,}")), + ("a Google API key", new Regex(@"\bAIza[0-9A-Za-z_-]{35}\b")), + ("a Stripe key", new Regex(@"\b(sk|rk)_live_[0-9A-Za-z]{20,}\b")), + ("an Azure storage key", new Regex(@"AccountKey=[A-Za-z0-9+/=]{40,}", RegexOptions.IgnoreCase)), + // Written as connection strings write it — Password=value, no spaces — which tells it + // apart from code assigning a variable called password. + ("a password in a connection string", new Regex(@"(?Values that are plainly placeholders, never reported. + static readonly Regex Placeholder = new( + @"(?i)^(\*+|x+|changeme|change[-_]?me|your[-_].*|example.*|placeholder|dummy|test|password|secret|<.*>|\$\{.*\}|\{\{.*\}\})$"); + + public static IEnumerable Scan(string relativePath, string text) + { + var lines = text.Split('\n'); + for (var i = 0; i < lines.Length; i++) + foreach (var (kind, pattern) in Patterns) + { + var match = pattern.Match(lines[i]); + if (!match.Success) continue; + var value = match.Groups.Count > 2 ? match.Groups[match.Groups.Count - 1].Value : match.Value; + if (Placeholder.IsMatch(value.Trim())) continue; + yield return new SecretFinding(relativePath, i + 1, kind); + break; + } + } + + /// Whether a file is text worth scanning: small enough, and no NUL bytes in its start. + public static bool IsText(byte[] bytes) => + bytes.Length <= 2 * 1024 * 1024 && !bytes.Take(8000).Contains((byte)0); + } +} diff --git a/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs b/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs index 3f5a596..87203ae 100644 --- a/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs +++ b/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs @@ -1,21 +1,10 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; -using System.IO.Compression; using System.Linq; -using System.Security.Cryptography; -using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; -using SW.CloudFiles.Extensions; -using SW.PrimitiveTypes; using SW.Serverless.Contract.Catalog; -using SW.Serverless.Resident; using SW.Serverless.Runtimes; namespace SW.Serverless.Tooling.Conformance @@ -27,7 +16,6 @@ namespace SW.Serverless.Tooling.Conformance /// public class ConformanceRunner { - const string Version = "0.0.0"; const string UnknownCommand = "__ConformanceNoSuchCommand__"; public async Task RunAsync(ConformanceOptions options) @@ -80,77 +68,49 @@ async Task RunCoreAsync(ConformanceOptions options, ConformanceReport report, st } // ------------------------------------------------------------ its own description - var runtimes = new AdapterRuntimes(options.Runtimes); options.Log("Asking the adapter to describe itself..."); - var description = await DescribeAsync(runtimes, manifest, Path.Combine(options.PackageDirectory, entry), report); + var description = await DescribeAsync(options, manifest, Path.Combine(options.PackageDirectory, entry), report); if (description != null) CheckSettings(manifest, description, report); // ------------------------------------------------------------ running it options.Log("Starting it the way a host does..."); - using var host = BuildHost(options, work); - await host.StartAsync(); + LocalAdapterHost session; try { - var adapterId = string.IsNullOrWhiteSpace(manifest.Id) ? "conformance.adapter" : manifest.Id; - await UploadAsync(host.Services.GetRequiredService(), options.PackageDirectory, - adapterId, entry); - - await using var session = await StartAsync(host.Services, manifest, $"{adapterId}/{Version}", options, report); - if (session == null) return; + session = await LocalAdapterHost.StartAsync(options.PackageDirectory, options.Settings, options.Runtimes, + options.CommandTimeoutSeconds, work); + report.Pass("starts", manifest.IsResident ? "resident, attached over gRPC" : "classic session"); + } + catch (Exception ex) + { + report.Fail("starts", ex.GetBaseException().Message); + return; + } + await using (session) + { foreach (var (contract, kinds) in ContractsToCheck(manifest, description, options, report)) foreach (var kind in kinds) await CheckKindAsync(session, contract, kind, description, options, report); await CheckUnknownCommandAsync(session, report); } - finally - { - await host.StopAsync(); - } } // ---------------------------------------------------------------- describe - static async Task DescribeAsync(AdapterRuntimes runtimes, AdapterManifest manifest, + static async Task DescribeAsync(ConformanceOptions options, AdapterManifest manifest, string entryPath, ConformanceReport report) { - var runtime = runtimes.Find(manifest.Runtime); - if (runtime == null) - { - report.Fail("describe", $"no launcher for the '{manifest.Runtime}' runtime"); - return null; - } - - var startInfo = runtime.StartInfo(entryPath, new RuntimeLaunch { Arguments = new[] { AdapterSelfDescription.Flag } }); - try - { - using var process = Process.Start(startInfo)!; - process.StandardInput.Close(); - var output = process.StandardOutput.ReadToEndAsync(); - var errors = process.StandardError.ReadToEndAsync(); - if (!process.WaitForExit(30_000)) - { - try { process.Kill(true); } catch { } - report.Fail("describe", $"it didn't answer {AdapterSelfDescription.Flag} within 30 seconds — an SDK older than 10.1.0 doesn't know it"); - return null; - } - - var description = AdapterSelfDescription.Parse(await output); - if (process.ExitCode != 0) - report.Fail("describe", $"{AdapterSelfDescription.Flag} exited with {process.ExitCode}: {await errors}"); - else if (string.IsNullOrWhiteSpace(description.SdkLanguage)) - report.Fail("describe", "the description doesn't name its SDK's language"); - else - report.Pass("describe", $"{description.SdkLanguage} SDK {description.SdkVersion}, {description.Commands.Count} commands" + - (description.Warnings.Count > 0 ? $"; warnings: {string.Join("; ", description.Warnings)}" : "")); - return description; - } - catch (Exception ex) - { - report.Fail("describe", $"its description couldn't be read: {ex.Message}"); - return null; - } + var (description, problem) = await LocalAdapterHost.DescribeAsync(entryPath, manifest.Runtime, options.Runtimes); + if (description == null) + report.Fail("describe", problem); + else if (string.IsNullOrWhiteSpace(description.SdkLanguage)) + report.Fail("describe", "the description doesn't name its SDK's language"); + else + report.Pass("describe", $"{description.SdkLanguage} SDK {description.SdkVersion}, {description.Commands.Count} commands" + + (description.Warnings.Count > 0 ? $"; warnings: {string.Join("; ", description.Warnings)}" : "")); + return description; } /// The manifest's properties must be the settings the adapter itself declares. @@ -176,136 +136,6 @@ static void CheckSettings(AdapterManifest manifest, AdapterSelfDescription descr else report.Pass("settings match the manifest", $"{declared.Count} settings"); } - // ---------------------------------------------------------------- the host - - static IHost BuildHost(ConformanceOptions options, string work) - { - var tag = Guid.NewGuid().ToString("N")[..10]; - return Host.CreateDefaultBuilder() - .ConfigureLogging(l => l.ClearProviders()) - .ConfigureAppConfiguration(c => c.Sources.Clear()) - .ConfigureServices(s => - { - s.AddLocalTestsCloudFiles(o => - { - o.BucketName = "conformance-" + tag; - o.StoragePath = Path.Combine(work, "store"); - }); - s.AddAdapterRuntimes(o => - { - o.PythonExecutable = options.Runtimes.PythonExecutable; - o.NodeExecutable = options.Runtimes.NodeExecutable; - o.DotnetExecutable = options.Runtimes.DotnetExecutable; - }); - s.AddServerless(o => - { - o.AdapterRemotePath = "adapters"; - o.AdapterLocalPath = Path.Combine(work, "installed"); - o.AdapterMetadataCacheDuration = 1; - o.CommandTimeout = options.CommandTimeoutSeconds; - }); - s.AddResidentAdapters(o => - { - o.SocketPath = Path.Combine("/tmp", $"swsl-conf-{tag}.sock"); - o.PipeName = $"swsl-conf-{tag}"; - o.HandshakeTimeout = TimeSpan.FromSeconds(60); - }); - }) - .Build(); - } - - static async Task UploadAsync(ICloudFilesService files, string directory, string adapterId, string entry) - { - using var buffer = new MemoryStream(); - using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) - foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories)) - archive.CreateEntryFromFile(file, Path.GetRelativePath(directory, file).Replace('\\', '/'), CompressionLevel.Fastest); - - buffer.Position = 0; - var hash = Convert.ToHexString(await SHA256.HashDataAsync(buffer))[..16].ToLowerInvariant(); - buffer.Position = 0; - await files.WriteAsync(buffer, new WriteFileSettings - { - Key = AdapterCatalogPaths.Version("adapters", adapterId, Version), - ContentType = "application/zip", - Metadata = new Dictionary { ["EntryAssembly"] = entry, ["Hash"] = "conformance-" + hash }, - }); - } - - // ---------------------------------------------------------------- sessions - - /// A running adapter, called by command name with raw payloads. - interface ISession : IAsyncDisposable - { - Task CallAsync(string command, object input); - Task CallVoidAsync(string command, object input); - } - - static async Task StartAsync(IServiceProvider services, AdapterManifest manifest, string adapterRef, - ConformanceOptions options, ConformanceReport report) - { - try - { - ISession session = manifest.IsResident - ? await ResidentSession.StartAsync(services, adapterRef, options) - : await ClassicSession.StartAsync(services, adapterRef, options); - report.Pass("starts", manifest.IsResident ? "resident, attached over gRPC" : "classic session"); - return session; - } - catch (Exception ex) - { - report.Fail("starts", ex.GetBaseException().Message); - return null; - } - } - - sealed class ClassicSession(IServiceScope scope, IServerlessService service, int timeout) : ISession - { - public static async Task StartAsync(IServiceProvider services, string adapterRef, ConformanceOptions options) - { - var scope = services.CreateScope(); - var service = scope.ServiceProvider.GetRequiredService(); - await service.StartAsync(adapterRef, "conformance", new Dictionary(options.Settings)); - return new ClassicSession(scope, service, options.CommandTimeoutSeconds); - } - - public Task CallAsync(string command, object input) => service.InvokeAsync(command, input, timeout); - public Task CallVoidAsync(string command, object input) => service.InvokeAsync(command, input, timeout); - - public ValueTask DisposeAsync() - { - (service as IDisposable)?.Dispose(); - scope.Dispose(); - return ValueTask.CompletedTask; - } - } - - sealed class ResidentSession(IResidentAdapterHost host, ResidentAdapterInstance instance, string adapterRef, int timeout) : ISession - { - public static async Task StartAsync(IServiceProvider services, string adapterRef, ConformanceOptions options) - { - var host = services.GetRequiredService(); - var instance = await host.StartExclusiveAsync(new AdapterSpec - { - AdapterId = adapterRef, - InstanceKey = "conformance", - StartupValues = new Dictionary(options.Settings), - }); - return new ResidentSession(host, instance, adapterRef, options.CommandTimeoutSeconds); - } - - public Task CallAsync(string command, object input) => instance.InvokeAsync(command, input, timeoutSeconds: timeout); - public Task CallVoidAsync(string command, object input) => instance.InvokeAsync(command, input, timeoutSeconds: timeout); - - public async ValueTask DisposeAsync() => await host.StopAsync(adapterRef, "conformance", drain: false); - } - - sealed class NoEvents : IAdapterEventSink - { - public Task OnEventAsync(InboundEvent inboundEvent, CancellationToken cancellationToken) => - Task.FromResult(EventOutcome.Ok("conformance")); - } - // ---------------------------------------------------------------- contracts static IEnumerable<(ContractDocument Contract, List Kinds)> ContractsToCheck(AdapterManifest manifest, @@ -336,7 +166,7 @@ public Task OnEventAsync(InboundEvent inboundEvent, CancellationTo } } - static async Task CheckKindAsync(ISession session, ContractDocument contract, string kind, + static async Task CheckKindAsync(LocalAdapterHost session, ContractDocument contract, string kind, AdapterSelfDescription description, ConformanceOptions options, ConformanceReport report) { var prefix = $"{contract.Name} {kind}"; @@ -391,7 +221,7 @@ static async Task CheckKindAsync(ISession session, ContractDocument contract, st /// without input as they come, a text input fed the first id the session listed, and a /// destructive method only when allowed. /// - static async Task CheckSessionAsync(ISession session, ContractDocument contract, string prefix, + static async Task CheckSessionAsync(LocalAdapterHost session, ContractDocument contract, string prefix, IReadOnlyList methods, ConformanceOptions options, ConformanceReport report) { string firstId = null; @@ -457,7 +287,7 @@ static async Task CheckOutputAsync(ContractDocument contract, string type, strin } /// An unknown command is refused with an error, and the adapter keeps answering. - static async Task CheckUnknownCommandAsync(ISession session, ConformanceReport report) + static async Task CheckUnknownCommandAsync(LocalAdapterHost session, ConformanceReport report) { for (var attempt = 1; attempt <= 2; attempt++) { diff --git a/SW.Serverless.Tooling/LocalAdapterHost.cs b/SW.Serverless.Tooling/LocalAdapterHost.cs new file mode 100644 index 0000000..511ff3e --- /dev/null +++ b/SW.Serverless.Tooling/LocalAdapterHost.cs @@ -0,0 +1,235 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using SW.CloudFiles.Extensions; +using SW.PrimitiveTypes; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Resident; +using SW.Serverless.Runtimes; + +namespace SW.Serverless.Tooling +{ + /// + /// An adapter package run on this machine exactly as a host runs one: put in a temporary store, + /// installed from it, started on its runtime, classic or resident, and called by command name. + /// What serverless run and serverless test use. + /// + public sealed class LocalAdapterHost : IAsyncDisposable + { + const string Version = "0.0.0"; + + readonly IHost host; + readonly ISession session; + readonly string work; + readonly bool ownsWork; + + LocalAdapterHost(IHost host, ISession session, string work, bool ownsWork) + { + this.host = host; + this.session = session; + this.work = work; + this.ownsWork = ownsWork; + } + + public static async Task StartAsync(string packageDirectory, IDictionary settings, + AdapterRuntimeOptions runtimes = null, int commandTimeoutSeconds = 60, string workDirectory = null) + { + var manifest = AdapterManifest.Parse(File.ReadAllText(Path.Combine(packageDirectory, AdapterManifest.FileName))); + var work = workDirectory ?? Path.Combine(Path.GetTempPath(), "swsl-local", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(work); + + var host = Build(runtimes ?? new AdapterRuntimeOptions(), commandTimeoutSeconds, work); + await host.StartAsync(); + try + { + var adapterId = string.IsNullOrWhiteSpace(manifest.Id) ? "local.adapter" : manifest.Id; + await UploadAsync(host.Services.GetRequiredService(), packageDirectory, adapterId, + manifest.EntryFor(AdapterRuntimes.CurrentPlatform)); + + var adapterRef = $"{adapterId}/{Version}"; + ISession session = manifest.IsResident + ? await ResidentSession.StartAsync(host.Services, adapterRef, settings, commandTimeoutSeconds) + : await ClassicSession.StartAsync(host.Services, adapterRef, settings, commandTimeoutSeconds); + return new LocalAdapterHost(host, session, work, workDirectory == null); + } + catch + { + await host.StopAsync(); + host.Dispose(); + if (workDirectory == null) try { Directory.Delete(work, true); } catch { } + throw; + } + } + + /// Calls a command and returns its answer as text: raw for a string, JSON for anything else. + public Task CallAsync(string command, object input) => session.CallAsync(command, input); + + /// Calls a command that answers nothing. + public Task CallVoidAsync(string command, object input) => session.CallVoidAsync(command, input); + + public async ValueTask DisposeAsync() + { + try { await session.DisposeAsync(); } catch { /* stopping only */ } + await host.StopAsync(); + host.Dispose(); + if (ownsWork) try { Directory.Delete(work, true); } catch { } + } + + static IHost Build(AdapterRuntimeOptions runtimes, int commandTimeoutSeconds, string work) + { + var tag = Guid.NewGuid().ToString("N")[..10]; + return Host.CreateDefaultBuilder() + .ConfigureLogging(l => l.ClearProviders()) + .ConfigureAppConfiguration(c => c.Sources.Clear()) + .ConfigureServices(s => + { + s.AddLocalTestsCloudFiles(o => + { + o.BucketName = "local-" + tag; + o.StoragePath = Path.Combine(work, "store"); + }); + s.AddAdapterRuntimes(o => + { + o.PythonExecutable = runtimes.PythonExecutable; + o.NodeExecutable = runtimes.NodeExecutable; + o.DotnetExecutable = runtimes.DotnetExecutable; + }); + s.AddServerless(o => + { + o.AdapterRemotePath = "adapters"; + o.AdapterLocalPath = Path.Combine(work, "installed"); + o.AdapterMetadataCacheDuration = 1; + o.CommandTimeout = commandTimeoutSeconds; + }); + s.AddResidentAdapters(o => + { + // Unix socket paths are short-limited, so under /tmp rather than the work folder. + o.SocketPath = Path.Combine("/tmp", $"swsl-local-{tag}.sock"); + o.PipeName = $"swsl-local-{tag}"; + o.HandshakeTimeout = TimeSpan.FromSeconds(60); + }); + }) + .Build(); + } + + static async Task UploadAsync(ICloudFilesService files, string directory, string adapterId, string entry) + { + using var buffer = new MemoryStream(); + using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories)) + archive.CreateEntryFromFile(file, Path.GetRelativePath(directory, file).Replace('\\', '/'), CompressionLevel.Fastest); + + buffer.Position = 0; + var hash = Convert.ToHexString(await SHA256.HashDataAsync(buffer))[..16].ToLowerInvariant(); + buffer.Position = 0; + await files.WriteAsync(buffer, new WriteFileSettings + { + Key = AdapterCatalogPaths.Version("adapters", adapterId, Version), + ContentType = "application/zip", + Metadata = new Dictionary { ["EntryAssembly"] = entry, ["Hash"] = "local-" + hash }, + }); + } + + interface ISession : IAsyncDisposable + { + Task CallAsync(string command, object input); + Task CallVoidAsync(string command, object input); + } + + sealed class ClassicSession(IServiceScope scope, IServerlessService service, int timeout) : ISession + { + public static async Task StartAsync(IServiceProvider services, string adapterRef, + IDictionary settings, int timeout) + { + var scope = services.CreateScope(); + var service = scope.ServiceProvider.GetRequiredService(); + try + { + await service.StartAsync(adapterRef, "local", new Dictionary(settings ?? new Dictionary())); + } + catch + { + (service as IDisposable)?.Dispose(); + scope.Dispose(); + throw; + } + return new ClassicSession(scope, service, timeout); + } + + public Task CallAsync(string command, object input) => service.InvokeAsync(command, input, timeout); + public Task CallVoidAsync(string command, object input) => service.InvokeAsync(command, input, timeout); + + public ValueTask DisposeAsync() + { + (service as IDisposable)?.Dispose(); + scope.Dispose(); + return ValueTask.CompletedTask; + } + } + + sealed class ResidentSession(IResidentAdapterHost host, ResidentAdapterInstance instance, string adapterRef, int timeout) : ISession + { + public static async Task StartAsync(IServiceProvider services, string adapterRef, + IDictionary settings, int timeout) + { + var host = services.GetRequiredService(); + var instance = await host.StartExclusiveAsync(new AdapterSpec + { + AdapterId = adapterRef, + InstanceKey = "local", + StartupValues = new Dictionary(settings ?? new Dictionary()), + }); + return new ResidentSession(host, instance, adapterRef, timeout); + } + + public Task CallAsync(string command, object input) => instance.InvokeAsync(command, input, timeoutSeconds: timeout); + public Task CallVoidAsync(string command, object input) => instance.InvokeAsync(command, input, timeoutSeconds: timeout); + + public async ValueTask DisposeAsync() => await host.StopAsync(adapterRef, "local", drain: false); + } + + sealed class NoEvents : IAdapterEventSink + { + public Task OnEventAsync(InboundEvent inboundEvent, CancellationToken cancellationToken) => + Task.FromResult(EventOutcome.Ok("local")); + } + + /// Runs an adapter with --describe and reads its answer; null and a reason when it gives none. + public static async Task<(AdapterSelfDescription Description, string Problem)> DescribeAsync(string entryPath, + string runtime, AdapterRuntimeOptions runtimes = null, int timeoutSeconds = 30) + { + var launcher = new AdapterRuntimes(runtimes).Find(runtime); + if (launcher == null) return (null, $"no launcher for the '{runtime}' runtime"); + + var startInfo = launcher.StartInfo(entryPath, new RuntimeLaunch { Arguments = new[] { AdapterSelfDescription.Flag } }); + try + { + using var process = Process.Start(startInfo)!; + process.StandardInput.Close(); + var output = process.StandardOutput.ReadToEndAsync(); + var errors = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(timeoutSeconds * 1000)) + { + try { process.Kill(true); } catch { } + return (null, $"it didn't answer {AdapterSelfDescription.Flag} within {timeoutSeconds} seconds — an SDK older than 10.1.0 doesn't know it"); + } + if (process.ExitCode != 0) + return (null, $"{AdapterSelfDescription.Flag} exited with {process.ExitCode}: {await errors}"); + return (AdapterSelfDescription.Parse(await output), null); + } + catch (Exception ex) + { + return (null, $"its description couldn't be read: {ex.Message}"); + } + } + } +} From e427370179fc00e891ffddb0e17fbfd61ec638d5 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 10:57:23 +0300 Subject: [PATCH 11/14] Add serverless init, build, test, run, manifest validate and publish Commands for adapters in any language, each a thin layer over SW.Serverless.Tooling, beside the existing ones, which parse and behave exactly as before: - init [--kind --id --dir]: a working adapter of a Bitween kind, referencing the SDK and the Bitween contract package, with adapter.json, example settings, a .gitignore and a README. The same templates a code editor will start from. - build [project] [--out --no-source --allow --dry-run]: the package, with what source it carries listed. - test [package|folder|project] [--settings --allow-delete --contract]: the conformance kit, one line per check. - run [package] --call Command [--input json|text|@file]: one call, its answer. - manifest validate [path] - publish [-v version|bump --no-promote --notes]: a built package, any language, versioned into the catalog. dotnet builds the tools start no longer leave MSBuild worker nodes or the build server running. They inherit the build's redirected output, so waiting for that output to end waited on processes that never exit: the first build on a machine could hang for good. Reproduced from cold build servers; fixed. CliCommandTests runs the commands through the CLI's entry point: init and its refusals, manifest validate, build then test then run then publish into storage and publish again with a bump, and what init writes for a handler, a receiver and a validator building and conforming. --- .../CliCommandTests.cs | 189 ++++++++++ SW.Serverless.Installer/AdapterCommands.cs | 325 ++++++++++++++++++ SW.Serverless.Installer/Program.cs | 41 ++- SW.Serverless.Tooling/DotnetBuilds.cs | 23 ++ SW.Serverless.Tooling/InstallerLogic.cs | 4 +- SW.Serverless.Tooling/PackagePublisher.cs | 78 +++++ .../Scaffolding/Scaffolder.cs | 250 ++++++++++++++ SW.Serverless.UnitTests/ExecRuntimeTests.cs | 7 +- 8 files changed, 913 insertions(+), 4 deletions(-) create mode 100644 SW.Serverless.Installer.UnitTests/CliCommandTests.cs create mode 100644 SW.Serverless.Installer/AdapterCommands.cs create mode 100644 SW.Serverless.Tooling/DotnetBuilds.cs create mode 100644 SW.Serverless.Tooling/Scaffolding/Scaffolder.cs diff --git a/SW.Serverless.Installer.UnitTests/CliCommandTests.cs b/SW.Serverless.Installer.UnitTests/CliCommandTests.cs new file mode 100644 index 0000000..f894bb2 --- /dev/null +++ b/SW.Serverless.Installer.UnitTests/CliCommandTests.cs @@ -0,0 +1,189 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Tooling; + +namespace SW.Serverless.Installer.UnitTests; + +/// +/// The commands for adapters in any language, run through the CLI's own entry point as a person or +/// a CI job runs them: init, build, test, run, manifest validate and publish. +/// +[TestClass] +public class CliCommandTests +{ + static string RepositoryRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null && !File.Exists(Path.Combine(dir.FullName, "SW.Serverless.sln"))) dir = dir.Parent; + return dir!.FullName; + } + + static string WorkFolder() + { + var folder = Path.Combine(RepositoryRoot(), ".test-work", "cli-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(folder); + return folder; + } + + [ClassCleanup] + public static void Cleanup() + { + try { Directory.Delete(Path.Combine(RepositoryRoot(), ".test-work"), true); } catch { } + } + + /// Runs the CLI and returns its exit code and what it printed. + static async Task<(int Exit, string Output)> Cli(params string[] args) + { + var original = Console.Out; + using var output = new StringWriter(); + Console.SetOut(output); + try + { + var exit = await Program.RunAsync(args, _ => null); + return (exit, output.ToString()); + } + finally + { + Console.SetOut(original); + } + } + + [TestMethod] + public async Task Init_makes_a_project_and_refuses_what_it_can_t_make() + { + var work = WorkFolder(); + + var (exit, output) = await Cli("init", "AcmeOrders", "--kind", "validator", "--dir", work); + Assert.AreEqual(Program.Success, exit, output); + var project = Path.Combine(work, "AcmeOrders"); + foreach (var file in new[] { "AcmeOrders.csproj", "Program.cs", "adapter.json", "settings.example.json", ".gitignore", "README.md" }) + Assert.IsTrue(File.Exists(Path.Combine(project, file)), file); + Assert.AreEqual("acme.orders", AdapterManifest.Parse(File.ReadAllText(Path.Combine(project, "adapter.json"))).Id); + StringAssert.Contains(File.ReadAllText(Path.Combine(project, "Program.cs")), "IBitweenValidator"); + + Assert.AreEqual(Program.Failure, (await Cli("init", "AcmeOrders", "--dir", work)).Exit, "an existing project isn't overwritten"); + var (pythonExit, pythonOutput) = await Cli("init", "PyOrders", "--lang", "python", "--dir", work); + Assert.AreEqual(Program.Failure, pythonExit); + StringAssert.Contains(pythonOutput, "arrives with that language's SDK"); + } + + [TestMethod] + public async Task Manifest_validate_passes_a_good_file_and_names_what_is_wrong_with_a_bad_one() + { + var work = WorkFolder(); + File.WriteAllText(Path.Combine(work, "adapter.json"), """{ "id": "good.adapter", "version": "1.0.0" }"""); + Assert.AreEqual(Program.Success, (await Cli("manifest", "validate", work)).Exit); + + File.WriteAllText(Path.Combine(work, "adapter.json"), """{ "id": "Bad Id", "runtime": "../sh" }"""); + var (exit, output) = await Cli("manifest", "validate", work); + Assert.AreEqual(Program.Failure, exit); + StringAssert.Contains(output, "id 'Bad Id'"); + StringAssert.Contains(output, "runtime '../sh'"); + } + + /// An author's project, as BuildTests makes one: the BitweenHandler adapter with an adapter.json. + static string AuthorProject() + { + var root = RepositoryRoot(); + var project = Path.Combine(WorkFolder(), "AcmeOrders"); + Directory.CreateDirectory(project); + var sdk = Path.GetRelativePath(project, Path.Combine(root, "SW.Serverless.Sdk", "SW.Serverless.Sdk.csproj")); + File.WriteAllText(Path.Combine(project, "AcmeOrders.csproj"), $""" + + + Exe + net10.0 + + + + + + """); + File.Copy(Path.Combine(root, "SW.Serverless.UnitTests.BitweenHandler", "Program.cs"), Path.Combine(project, "Program.cs")); + File.WriteAllText(Path.Combine(project, "adapter.json"), """{ "id": "acme.orders", "version": "1.2.0", "displayName": "Acme orders" }"""); + return project; + } + + [TestMethod] + public async Task Build_test_run_and_publish_take_a_project_into_storage() + { + var project = AuthorProject(); + var settings = Path.Combine(Path.GetDirectoryName(project)!, "settings.json"); + File.WriteAllText(settings, """{ "ApiKey": "test-key" }"""); + + var build = await Cli("build", project); + Assert.AreEqual(Program.Success, build.Exit, build.Output); + StringAssert.Contains(build.Output, "Source carried"); + var zip = Path.Combine(project, "bin", "serverless", "acme.orders-1.2.0.zip"); + Assert.IsTrue(File.Exists(zip), build.Output); + + var test = await Cli("test", zip, "--settings", settings); + Assert.AreEqual(Program.Success, test.Exit, test.Output); + StringAssert.Contains(test.Output, "PASS bitween handler: Handle answers example 1"); + StringAssert.Contains(test.Output, "Conforms."); + + var run = await Cli("run", zip, "--settings", settings, "--call", "Handle", "--input", """{"Data":"{\"order\":1}"}"""); + Assert.AreEqual(Program.Success, run.Exit, run.Output); + StringAssert.Contains(run.Output, "accepted"); + + var store = Path.Combine(Path.GetDirectoryName(project)!, "store"); + var publish = await Cli("publish", zip, "-p", "local", "-b", "cli-tests", "-u", store); + Assert.AreEqual(Program.Success, publish.Exit, publish.Output); + StringAssert.Contains(publish.Output, "Published acme.orders 1.2.0"); + + var repository = new AdapterRepository(CloudFilesFactory.Create(new ServerlessUploadOptions + { + Provider = "local", BucketName = "cli-tests", ServiceUrl = store, + }), _ => { }); + var entry = await repository.LoadEntryAsync("acme.orders"); + Assert.AreEqual("1.2.0", entry.Current); + Assert.IsNotNull(entry.Manifest!.PublishedOn); + Assert.IsNotNull(entry.Manifest.Source, "it carries its source into storage"); + + // Published again with a bump: the version moves on, the old one stays. + var again = await Cli("publish", zip, "-v", "minor", "-p", "local", "-b", "cli-tests", "-u", store); + Assert.AreEqual(Program.Success, again.Exit, again.Output); + Assert.AreEqual("1.3.0", (await repository.LoadEntryAsync("acme.orders")).Current); + } + + /// + /// What init writes builds and conforms. The templates reference the published SDK and Bitween + /// contract packages; until those are on NuGet the project is pointed at the projects themselves, + /// with Bitween-api beside this repository. + /// + [DataTestMethod] + [DataRow("handler")] + [DataRow("receiver")] + [DataRow("validator")] + public async Task What_init_writes_builds_and_conforms(string kind) + { + var root = RepositoryRoot(); + var contracts = Path.GetFullPath(Path.Combine(root, "..", "Bitween-api", "SW.Bitween.Adapters", "SW.Bitween.Adapters.csproj")); + if (!File.Exists(contracts)) Assert.Inconclusive($"Bitween-api isn't beside this repository ({contracts})"); + + var work = WorkFolder(); + Assert.AreEqual(Program.Success, (await Cli("init", "Acme" + char.ToUpper(kind[0]) + kind[1..], "--kind", kind, "--dir", work)).Exit); + var project = Directory.GetDirectories(work).Single(); + var csproj = Directory.GetFiles(project, "*.csproj").Single(); + var text = File.ReadAllText(csproj) + .Replace($"", + $"") + .Replace($"", + $"") + .Replace("true", ""); + File.WriteAllText(csproj, text); + + var build = await Cli("build", project, "--no-source"); + Assert.AreEqual(Program.Success, build.Exit, build.Output); + + var settings = Path.Combine(work, "settings.json"); + File.WriteAllText(settings, """{ "ApiKey": "k" }"""); + var test = await Cli("test", Path.Combine(project, "bin", "serverless", "package"), "--settings", settings); + Assert.AreEqual(Program.Success, test.Exit, test.Output); + } +} diff --git a/SW.Serverless.Installer/AdapterCommands.cs b/SW.Serverless.Installer/AdapterCommands.cs new file mode 100644 index 0000000..b2be99f --- /dev/null +++ b/SW.Serverless.Installer/AdapterCommands.cs @@ -0,0 +1,325 @@ +using CommandLine; +using SW.PrimitiveTypes; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Tooling; +using SW.Serverless.Tooling.Building; +using SW.Serverless.Tooling.Conformance; +using SW.Serverless.Tooling.Scaffolding; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; + +namespace SW.Serverless.Installer +{ + /// serverless init <name> + public class InitCliOptions + { + [Value(0, Required = true, MetaName = "name", HelpText = "The adapter's name, e.g. AcmeOrders: its folder, project and class.")] + public string Name { get; set; } + + [Option("lang", Default = "dotnet", HelpText = "Its language: dotnet. Python, Node and Go arrive with their SDKs.")] + public string Language { get; set; } + + [Option("kind", Default = "handler", HelpText = "handler, mapper, validator or receiver.")] + public string Kind { get; set; } + + [Option("id", HelpText = "Its id in storage; derived from the name when not given (AcmeOrders -> acme.orders).")] + public string Id { get; set; } + + [Option("dir", Default = ".", HelpText = "The folder to make it in.")] + public string Directory { get; set; } + } + + /// serverless build [project] + public class BuildCliOptions + { + [Value(0, MetaName = "project", Default = ".", HelpText = "The adapter's project folder, with its adapter.json.")] + public string Project { get; set; } + + [Option('o', "out", HelpText = "Where the package goes; bin/serverless in the project unless set.")] + public string Output { get; set; } + + [Option("no-source", HelpText = "Leave the source out of the package.")] + public bool NoSource { get; set; } + + [Option("allow", HelpText = "A source file whose secret-scan finding is a false positive. Repeat for more.")] + public IEnumerable Allow { get; set; } + + [Option("dry-run", HelpText = "Show the source that would be carried, and build nothing.")] + public bool DryRun { get; set; } + } + + /// serverless test [package] + public class TestCliOptions + { + [Value(0, MetaName = "package", Default = ".", + HelpText = "A package zip, a package folder, or a project folder (built first).")] + public string Package { get; set; } + + [Option("settings", HelpText = "A JSON file of settings to run it with: { \"Name\": \"value\" }. The adapter calls whatever they point at.")] + public string Settings { get; set; } + + [Option("allow-delete", HelpText = "Let a receiver's DeleteFile run: it removes or moves a real file at the source.")] + public bool AllowDelete { get; set; } + + [Option("contract", HelpText = "A contract file to check against, beyond those the CLI carries. Repeat for more.")] + public IEnumerable Contracts { get; set; } + + [Option("timeout", Default = 60, HelpText = "Seconds one call may take.")] + public int Timeout { get; set; } + } + + /// serverless run [package] --call Command + public class RunCliOptions + { + [Value(0, MetaName = "package", Default = ".", HelpText = "A package zip, a package folder, or a project folder (built first).")] + public string Package { get; set; } + + [Option("settings", HelpText = "A JSON file of settings to run it with.")] + public string Settings { get; set; } + + [Option("call", Required = true, HelpText = "The command to call, e.g. Handle.")] + public string Command { get; set; } + + [Option("input", HelpText = "Its argument: JSON, plain text, or @file to read it from a file.")] + public string Input { get; set; } + + [Option("timeout", Default = 60, HelpText = "Seconds the call may take.")] + public int Timeout { get; set; } + } + + /// serverless manifest validate [path] + public class ManifestCliOptions + { + [Value(0, Required = true, MetaName = "action", HelpText = "validate")] + public string Action { get; set; } + + [Value(1, MetaName = "path", Default = ".", HelpText = "An adapter.json, or the folder holding one.")] + public string Path { get; set; } + } + + /// serverless publish <package> + public class PublishPackageCliOptions : StorageCliOptions + { + [Value(0, Required = true, MetaName = "package", HelpText = "A package zip from serverless build.")] + public string Package { get; set; } + + [Option('v', "version", HelpText = "The version: explicit, or major, minor or patch to bump. The manifest's version unless set.")] + public string Version { get; set; } + + [Option("no-promote", HelpText = "Upload the version without making it the one that runs.")] + public bool NoPromote { get; set; } + + [Option("notes", HelpText = "Release notes for this version (Markdown).")] + public string Notes { get; set; } + + [Option("published-by", HelpText = "Who is publishing, for the catalog.")] + public string PublishedBy { get; set; } + } + + /// The commands that work on adapters in any language, over SW.Serverless.Tooling. + static class AdapterCommands + { + const int Success = Program.Success; + const int Failure = Program.Failure; + + public static Task Init(InitCliOptions opts) + { + var result = Scaffolder.Scaffold(new ScaffoldRequest + { + Name = opts.Name, + Id = opts.Id, + Language = opts.Language, + Kind = opts.Kind, + ParentDirectory = opts.Directory, + }); + if (!result.Succeeded) + { + foreach (var problem in result.Problems) Console.WriteLine(problem); + return Task.FromResult(Failure); + } + + Console.WriteLine($"Made a {opts.Kind} adapter in {result.ProjectDirectory}:"); + foreach (var file in result.Files) Console.WriteLine($" {file}"); + Console.WriteLine("Next: serverless build, then serverless test --settings settings.json"); + return Task.FromResult(Success); + } + + public static async Task Build(BuildCliOptions opts) + { + var result = await PackageBuilder.BuildAsync(new BuildRequest + { + ProjectDirectory = opts.Project, + OutputDirectory = opts.Output, + IncludeSource = !opts.NoSource, + AllowedFiles = new HashSet(opts.Allow ?? Enumerable.Empty(), StringComparer.OrdinalIgnoreCase), + DryRun = opts.DryRun, + Log = Console.WriteLine, + }); + + if (result.SourceFiles.Count > 0) + { + Console.WriteLine($"Source carried ({result.SourceFiles.Count} files, {result.SourceFiles.Sum(f => f.Bytes) / 1024} KB):"); + foreach (var (path, bytes) in result.SourceFiles.OrderBy(f => f.Path, StringComparer.Ordinal)) + Console.WriteLine($" {path} ({bytes} B)"); + } + foreach (var warning in result.Warnings) Console.WriteLine($"Warning: {warning}"); + foreach (var problem in result.Problems) Console.WriteLine(problem); + if (!result.Succeeded) return Failure; + + if (!opts.DryRun) Console.WriteLine($"Built {result.ZipPath}"); + return Success; + } + + public static async Task Test(TestCliOptions opts) + { + var (package, cleanup) = await PackageFolderAsync(opts.Package); + if (package == null) return Failure; + try + { + var report = await new ConformanceRunner().RunAsync(new ConformanceOptions + { + PackageDirectory = package, + Settings = ReadSettings(opts.Settings), + AllowDelete = opts.AllowDelete, + Contracts = (opts.Contracts ?? Enumerable.Empty()).Select(ContractDocument.FromFile).ToList(), + CommandTimeoutSeconds = opts.Timeout, + Log = Console.WriteLine, + }); + + foreach (var check in report.Checks) + Console.WriteLine($"{Mark(check.Outcome)} {check.Name}{(string.IsNullOrEmpty(check.Detail) ? "" : " — " + check.Detail)}"); + var failed = report.Checks.Count(c => c.Outcome == CheckOutcome.Failed); + Console.WriteLine(failed == 0 ? "Conforms." : $"{failed} check{(failed == 1 ? "" : "s")} failed."); + return report.Passed ? Success : Failure; + } + finally + { + cleanup(); + } + } + + public static async Task Run(RunCliOptions opts) + { + var (package, cleanup) = await PackageFolderAsync(opts.Package); + if (package == null) return Failure; + try + { + await using var host = await LocalAdapterHost.StartAsync(package, ReadSettings(opts.Settings), commandTimeoutSeconds: opts.Timeout); + Console.WriteLine(await host.CallAsync(opts.Command, ReadInput(opts.Input))); + return Success; + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + Console.WriteLine(ex.GetBaseException().Message); + return Failure; + } + finally + { + cleanup(); + } + } + + public static Task Manifest(ManifestCliOptions opts) + { + if (opts.Action != "validate") + { + Console.WriteLine($"'{opts.Action}' isn't a manifest action: use validate"); + return Task.FromResult(Failure); + } + + var path = Directory.Exists(opts.Path) ? System.IO.Path.Combine(opts.Path, AdapterManifest.FileName) : opts.Path; + if (!File.Exists(path)) + { + Console.WriteLine($"There is no {path}"); + return Task.FromResult(Failure); + } + + try + { + var problems = AdapterManifest.Parse(File.ReadAllText(path)).Validate(); + foreach (var problem in problems) Console.WriteLine(problem); + Console.WriteLine(problems.Count == 0 ? $"{path} is valid." : $"{problems.Count} problem{(problems.Count == 1 ? "" : "s")} in {path}."); + return Task.FromResult(problems.Count == 0 ? Success : Failure); + } + catch (JsonException ex) + { + Console.WriteLine($"{path} isn't valid JSON: {ex.Message}"); + return Task.FromResult(Failure); + } + } + + public static Task Publish(PublishPackageCliOptions opts, Func environment) => + Program.WithStorage(opts, environment, async files => + { + var result = await PackagePublisher.PublishPackageAsync(files, new PublishPackageRequest + { + PackagePath = opts.Package, + Version = opts.Version, + Promote = !opts.NoPromote, + ReleaseNotes = opts.Notes, + PublishedBy = UploadOptionsResolver.ResolvePublishedBy(opts.PublishedBy, environment), + }, Console.WriteLine); + Console.WriteLine($"Published {result.Manifest.Id} {result.Version}."); + }); + + static string Mark(CheckOutcome outcome) => outcome switch + { + CheckOutcome.Passed => "PASS", + CheckOutcome.Failed => "FAIL", + _ => "SKIP", + }; + + /// A package folder for whatever was given: a folder with adapter.json, a zip, or a project to build. + static async Task<(string Folder, Action Cleanup)> PackageFolderAsync(string given) + { + var path = System.IO.Path.GetFullPath(given); + if (File.Exists(path) && path.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + { + var folder = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "swsl-cli", Guid.NewGuid().ToString("N")); + ZipFile.ExtractToDirectory(path, folder); + return (folder, () => TryDelete(folder)); + } + + if (Directory.Exists(path) && Directory.GetFiles(path, "*.*proj").Length > 0) + { + var output = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "swsl-cli", Guid.NewGuid().ToString("N")); + var built = await PackageBuilder.BuildAsync(new BuildRequest { ProjectDirectory = path, OutputDirectory = output, Log = Console.WriteLine }); + foreach (var problem in built.Problems) Console.WriteLine(problem); + return built.Succeeded ? (built.PackageDirectory, () => TryDelete(output)) : (null, () => TryDelete(output)); + } + + if (Directory.Exists(path) && File.Exists(System.IO.Path.Combine(path, AdapterManifest.FileName))) + return (path, () => { }); + + Console.WriteLine($"{given} is neither a package zip, a package folder nor a project folder"); + return (null, () => { }); + } + + static IDictionary ReadSettings(string path) + { + if (string.IsNullOrWhiteSpace(path)) return new Dictionary(); + using var document = JsonDocument.Parse(File.ReadAllText(path)); + return document.RootElement.EnumerateObject().ToDictionary(p => p.Name, + p => p.Value.ValueKind == JsonValueKind.String ? p.Value.GetString() : p.Value.GetRawText()); + } + + static object ReadInput(string input) + { + if (input == null) return null; + if (input.StartsWith('@')) input = File.ReadAllText(input[1..]); + // JSON goes as JSON; anything else as the plain text a string argument is. + try { return Newtonsoft.Json.Linq.JToken.Parse(input); } + catch (Newtonsoft.Json.JsonReaderException) { return input; } + } + + static void TryDelete(string folder) + { + try { Directory.Delete(folder, true); } catch { } + } + } +} diff --git a/SW.Serverless.Installer/Program.cs b/SW.Serverless.Installer/Program.cs index 174a814..cb31f36 100644 --- a/SW.Serverless.Installer/Program.cs +++ b/SW.Serverless.Installer/Program.cs @@ -16,7 +16,12 @@ public static class Program public const int Failure = 1; /// The commands besides publishing. Matched on the first argument only. - static readonly HashSet Commands = new(StringComparer.Ordinal) { "promote", "versions", "withdraw" }; + static readonly HashSet Commands = new(StringComparer.Ordinal) + { + "promote", "versions", "withdraw", + // For adapters in any language, over SW.Serverless.Tooling. + "init", "build", "test", "run", "manifest", "publish", + }; private static Task Main(string[] args) => RunAsync(args); @@ -47,6 +52,13 @@ public static async Task RunAsync(string[] args, Func envir o => Promote(o, environment)), "versions" => await Run(parser.ParseArguments(rest), o => Versions(o, environment)), + "init" => await Run(parser.ParseArguments(rest), AdapterCommands.Init), + "build" => await Run(parser.ParseArguments(rest), AdapterCommands.Build), + "test" => await Run(parser.ParseArguments(rest), AdapterCommands.Test), + "run" => await Run(parser.ParseArguments(rest), AdapterCommands.Run), + "manifest" => await Run(parser.ParseArguments(rest), AdapterCommands.Manifest), + "publish" => await Run(parser.ParseArguments(rest), + o => AdapterCommands.Publish(o, environment)), _ => await Run(parser.ParseArguments(rest), o => Withdraw(o, environment)), }; @@ -221,6 +233,33 @@ static async Task WithRepository(StorageCliOptions opts, string adapterId, } } + /// Runs against the storage the options name, reporting a failure as a message. + internal static async Task WithStorage(StorageCliOptions opts, Func environment, + Func action) + { + try + { + await action(CloudFilesFactory.Create(await GetServerlessUploadOptions(opts, environment))); + return Success; + } + catch (SWException ex) + { + Console.WriteLine(ex.Message); + return Failure; + } + catch (ArgumentException ex) + { + // Semver's verdicts: an existing or lower version, or an unknown bump. + Console.WriteLine(ex.Message); + return Failure; + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + return Failure; + } + } + public static IEnumerable FormatVersions(string adapterId, VersionListing listing) { if (listing.Versions.Count == 0) diff --git a/SW.Serverless.Tooling/DotnetBuilds.cs b/SW.Serverless.Tooling/DotnetBuilds.cs new file mode 100644 index 0000000..a6b3e18 --- /dev/null +++ b/SW.Serverless.Tooling/DotnetBuilds.cs @@ -0,0 +1,23 @@ +using System.Diagnostics; + +namespace SW.Serverless.Tooling +{ + /// + /// How the tools run dotnet builds. MSBuild leaves worker nodes and a build server running for + /// the next build, and they inherit the build's redirected output: a caller waiting for that + /// output to end — as WaitForExit and ReadToEnd do — then waits for processes that never exit, + /// and a build that finished in seconds hangs for good. So nothing is left running. + /// + public static class DotnetBuilds + { + public static ProcessStartInfo WithoutLingeringNodes(this ProcessStartInfo startInfo) + { + startInfo.Environment["MSBUILDDISABLENODEREUSE"] = "1"; + startInfo.Environment["DOTNET_CLI_USE_MSBUILD_SERVER"] = "0"; + startInfo.Environment["UseSharedCompilation"] = "false"; + if (!startInfo.Arguments.Contains("-nodeReuse") && startInfo.ArgumentList.Count == 0) + startInfo.Arguments += " -nodeReuse:false"; + return startInfo; + } + } +} diff --git a/SW.Serverless.Tooling/InstallerLogic.cs b/SW.Serverless.Tooling/InstallerLogic.cs index c720733..14b4252 100644 --- a/SW.Serverless.Tooling/InstallerLogic.cs +++ b/SW.Serverless.Tooling/InstallerLogic.cs @@ -36,7 +36,7 @@ public bool BuildPublish(string projectPath, string outputPath) Arguments = $"publish \"{projectPath}\" -c Release -o \"{outputPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, - } + }.WithoutLingeringNodes() }; process.OutputDataReceived += OutputDataReceived; @@ -99,7 +99,7 @@ private static string AssemblyNameFromMsBuild(string projectPath) Arguments = $"msbuild \"{projectPath}\" -getProperty:AssemblyName -p:Configuration=Release", RedirectStandardOutput = true, RedirectStandardError = true, - }); + }.WithoutLingeringNodes()); if (process == null) return null; var stderr = process.StandardError.ReadToEndAsync(); diff --git a/SW.Serverless.Tooling/PackagePublisher.cs b/SW.Serverless.Tooling/PackagePublisher.cs index d698548..d5b2b45 100644 --- a/SW.Serverless.Tooling/PackagePublisher.cs +++ b/SW.Serverless.Tooling/PackagePublisher.cs @@ -32,6 +32,19 @@ public class PublishRequest public string PublishedBy { get; set; } } + /// A package serverless build made — in any language — to publish as it is. + public class PublishPackageRequest + { + public string PackagePath { get; set; } + + /// Explicit, or major, minor or patch to bump; the manifest's own version when empty. + public string Version { get; set; } + + public bool Promote { get; set; } = true; + public string ReleaseNotes { get; set; } + public string PublishedBy { get; set; } + } + public class PublishResult { /// The version published, or null for an unversioned upload. @@ -143,5 +156,70 @@ await repository.PublishVersionAsync(adapterId, version, zipPath, package, reque return new PublishResult { Version = version, Sha256 = sha256, Manifest = manifest, IconDataUri = iconDataUri }; } + + /// + /// Publishes a package serverless build made: its manifest is already complete, so this only + /// settles the version, stamps it into the manifest and uploads. Always versioned; an adapter + /// in another runtime goes only where hosts that can run it look. + /// + public static async Task PublishPackageAsync(ICloudFilesService files, PublishPackageRequest request, + Action log = null) + { + log ??= Console.WriteLine; + void Warn(string message) => log($"Warning: {message}"); + + if (!File.Exists(request.PackagePath)) + throw new SWException($"There is no package at {request.PackagePath}."); + + var work = Path.Combine(Path.GetTempPath(), "swsl-publish", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(work); + try + { + var folder = Path.Combine(work, "package"); + System.IO.Compression.ZipFile.ExtractToDirectory(request.PackagePath, folder); + var manifestPath = Path.Combine(folder, AdapterManifest.FileName); + if (!File.Exists(manifestPath)) + throw new SWException($"{request.PackagePath} has no {AdapterManifest.FileName}; build it with serverless build."); + + var manifest = AdapterManifest.Parse(await File.ReadAllTextAsync(manifestPath)); + var adapterId = manifest.Id?.ToLowerInvariant(); + if (!InstallerLogic.IsValidAdapterId(adapterId)) + throw new SWException($"The package's manifest has no valid id ('{manifest.Id}')."); + + var mode = string.IsNullOrWhiteSpace(request.Version) ? manifest.Version : request.Version; + if (string.IsNullOrWhiteSpace(mode)) + throw new SWException("The package has no version: give one with -v, or set version in adapter.json."); + + var repository = new AdapterRepository(files, log); + var version = await repository.ResolveVersionAsync(adapterId, mode); + manifest.Version = version; + manifest.PublishedOn = DateTimeOffset.UtcNow; + if (!string.IsNullOrWhiteSpace(request.ReleaseNotes)) manifest.ReleaseNotes = request.ReleaseNotes; + ManifestBuilder.Validate(manifest); + await File.WriteAllTextAsync(manifestPath, manifest.ToJson()); + + var zipPath = Path.Combine(work, "adapter.zip"); + if (!new InstallerLogic().Compress(folder, zipPath)) + throw new SWException("The package could not be rebuilt with its version, so nothing was uploaded."); + + var iconDataUri = ManifestBuilder.IconDataUri(manifest, folder, Warn); + var package = new PackageInfo + { + EntryAssembly = manifest.Entry, + Lifecycle = manifest.Lifecycle, + Kind = string.Join(",", manifest.Kinds), + Manifest = manifest, + IconDataUri = iconDataUri, + }; + + log($"Publishing version {version} of '{adapterId}' ({manifest.Runtime})."); + await repository.PublishVersionAsync(adapterId, version, zipPath, package, request.Promote, request.PublishedBy); + return new PublishResult { Version = version, Sha256 = InstallerLogic.Sha256Of(zipPath), Manifest = manifest, IconDataUri = iconDataUri }; + } + finally + { + try { Directory.Delete(work, true); } catch { } + } + } } } diff --git a/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs b/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs new file mode 100644 index 0000000..a008625 --- /dev/null +++ b/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs @@ -0,0 +1,250 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; + +namespace SW.Serverless.Tooling.Scaffolding +{ + public class ScaffoldRequest + { + /// The adapter's name, e.g. AcmeOrders: the folder, the project and the class names. + public string Name { get; set; } + + /// Its id in storage; derived from the name when not given. + public string Id { get; set; } + + public string Language { get; set; } = "dotnet"; + + /// handler, mapper, validator or receiver. + public string Kind { get; set; } = "handler"; + + /// The folder the project folder is made in. + public string ParentDirectory { get; set; } = "."; + } + + public class ScaffoldResult + { + public string ProjectDirectory { get; set; } + public List Files { get; } = new(); + public List Problems { get; } = new(); + public bool Succeeded => Problems.Count == 0; + } + + /// + /// What serverless init writes: a working adapter of one kind for the Bitween contract, ready to + /// build, test and publish. Also what a code editor starts a new adapter from. + /// + public static class Scaffolder + { + /// The SDK the templates reference; the first with --describe and the Bitween contract types. + public const string SdkPackageVersion = "10.1.0"; + + /// The Bitween contract package the templates reference. + public const string BitweenAdaptersPackageVersion = "1.0.0"; + + public static readonly IReadOnlyList Kinds = new[] { "handler", "mapper", "validator", "receiver" }; + public static readonly IReadOnlyList Languages = new[] { "dotnet" }; + + public static ScaffoldResult Scaffold(ScaffoldRequest request) + { + var result = new ScaffoldResult(); + var name = request.Name?.Trim(); + if (string.IsNullOrEmpty(name) || !Regex.IsMatch(name, "^[A-Za-z][A-Za-z0-9_]*$")) + result.Problems.Add($"'{request.Name}' isn't a name a project and a class can take: letters, digits and _, starting with a letter"); + if (!Kinds.Contains(request.Kind)) + result.Problems.Add($"'{request.Kind}' isn't a kind: use {string.Join(", ", Kinds)}"); + if (!Languages.Contains(request.Language)) + result.Problems.Add($"a '{request.Language}' template arrives with that language's SDK; this release scaffolds {string.Join(", ", Languages)}"); + + var id = string.IsNullOrWhiteSpace(request.Id) ? IdFrom(name ?? "") : request.Id.Trim(); + if (!InstallerLogic.IsValidAdapterId(id)) + result.Problems.Add($"'{id}' isn't an adapter id: lowercase letters, digits, '.', '_' and '-'"); + + var directory = Path.GetFullPath(Path.Combine(request.ParentDirectory ?? ".", name ?? "")); + if (Directory.Exists(directory) && Directory.EnumerateFileSystemEntries(directory).Any()) + result.Problems.Add($"{directory} already exists and isn't empty"); + if (!result.Succeeded) return result; + + Directory.CreateDirectory(directory); + result.ProjectDirectory = directory; + foreach (var (file, content) in DotnetFiles(name, id, request.Kind)) + { + var path = Path.Combine(directory, file); + File.WriteAllText(path, content.Replace("\r\n", "\n")); + result.Files.Add(file); + } + return result; + } + + /// AcmeOrders → acme.orders + public static string IdFrom(string name) => + Regex.Replace(name, "(?<=[a-z0-9])(?=[A-Z])", ".").Replace('_', '.').ToLowerInvariant(); + + static IEnumerable<(string File, string Content)> DotnetFiles(string name, string id, string kind) + { + yield return ($"{name}.csproj", $$""" + + + + Exe + net10.0 + enable + enable + + true + + + + + + + + + """); + + yield return ("adapter.json", $$""" + { + "id": "{{id}}", + "version": "0.1.0", + "displayName": "{{Spaced(name)}}", + "summary": "What this {{kind}} does, in one sentence, for the adapter list.", + "lifecycle": "classic" + } + """); + + yield return ("Program.cs", Program(name, kind)); + + yield return ("settings.example.json", """ + { + "BaseUrl": "https://partner.example.test", + "ApiKey": "put a test key here, and keep this file out of version control once it holds one" + } + """); + + yield return (".gitignore", """ + bin/ + obj/ + settings.json + """); + + yield return ("README.md", $$""" + # {{Spaced(name)}} + + A Bitween {{kind}} adapter. + + ```sh + serverless build # builds bin/serverless/{{id}}-0.1.0.zip + cp settings.example.json settings.json # then fill in real values + serverless test --settings settings.json # checks it against the Bitween contract + serverless publish bin/serverless/{{id}}-0.1.0.zip # with your storage flags + ``` + + Settings are declared in code with `Runner.Expect`; `serverless build` writes them into + the manifest Bitween reads. + """); + } + + static string Program(string name, string kind) => kind switch + { + "receiver" => $$""" + using SW.Bitween.Adapters; + using SW.Serverless.Sdk; + + namespace {{name}}; + + /// Fetches files on a schedule. Bitween calls Initialize, ListFiles, then GetFile and DeleteFile for each file, then Finalize. + [AdapterKind("receiver")] + [AdapterContract("bitween", 1)] + public class Receiver : IBitweenReceiver + { + public Receiver() + { + // Declare settings here; read them in the methods, never in the constructor. + Runner.Expect("BaseUrl", "https://partner.example.test", description: "Where files are fetched from."); + Runner.Expect("ApiKey", isPrivate: true, description: "The partner's key."); + } + + public Task Initialize() => Task.CompletedTask; + + public Task> ListFiles() => + Task.FromResult>(new[] { "example-1" }); + + public Task GetFile(string fileId) => + Task.FromResult(new ExchangeFile("{\"id\":\"" + fileId + "\"}", fileId + ".json")); + + public Task DeleteFile(string fileId) => Task.CompletedTask; + + public Task Finalize() => Task.CompletedTask; + } + + static class Program + { + static Task Main() => Runner.Run(new Receiver()); + } + """, + "validator" => $$""" + using SW.Bitween.Adapters; + using SW.Serverless.Sdk; + + namespace {{name}}; + + /// Checks a message before Bitween accepts it. + [AdapterKind("validator")] + [AdapterContract("bitween", 1)] + public class Validator : IBitweenValidator + { + public Validator() + { + Runner.Expect("MaxBytes", "1000000", description: "The largest message accepted."); + } + + public Task Validate(ExchangeFile file) + { + var result = new ValidationResult(); + if (file.Data.Length > Runner.StartupValueOf("MaxBytes")) + result.AddError("Data", "The message is larger than allowed."); + return Task.FromResult(result); + } + } + + static class Program + { + static Task Main() => Runner.Run(new Validator()); + } + """, + var handlerOrMapper => $$""" + using SW.Bitween.Adapters; + using SW.Serverless.Sdk; + + namespace {{name}}; + + /// {{(handlerOrMapper == "mapper" ? "Maps a message into the shape the next step expects." : "Delivers a message and returns the partner's response.")}} + [AdapterKind("{{handlerOrMapper}}")] + [AdapterContract("bitween", 1)] + public class {{(handlerOrMapper == "mapper" ? "Mapper : IBitweenMapper" : "Handler : IBitweenHandler")}} + { + public {{(handlerOrMapper == "mapper" ? "Mapper" : "Handler")}}() + { + // Declare settings here; read them in the methods, never in the constructor. + Runner.Expect("BaseUrl", "https://partner.example.test", description: "Where messages go."); + Runner.Expect("ApiKey", isPrivate: true, description: "The partner's key."); + } + + public Task Handle(ExchangeFile file) + { + // {{(handlerOrMapper == "mapper" ? "Return the message in its new shape." : "Send file.Data to the partner. A rejection is returned with BadData set, not thrown.")}} + return Task.FromResult(new ExchangeFile(file.Data, file.Filename)); + } + } + + static class Program + { + static Task Main() => Runner.Run(new {{(handlerOrMapper == "mapper" ? "Mapper" : "Handler")}}()); + } + """, + }; + + static string Spaced(string name) => Regex.Replace(name, "(?<=[a-z0-9])(?=[A-Z])", " "); + } +} diff --git a/SW.Serverless.UnitTests/ExecRuntimeTests.cs b/SW.Serverless.UnitTests/ExecRuntimeTests.cs index 5294580..28f2c68 100644 --- a/SW.Serverless.UnitTests/ExecRuntimeTests.cs +++ b/SW.Serverless.UnitTests/ExecRuntimeTests.cs @@ -47,7 +47,12 @@ public static async Task ClassInitialize(TestContext context) // Built under the test's own folder, never beside the sample's usual build output, // which other tests read. $"--artifacts-path \"{Path.Combine(workDirectory, "artifacts")}\" -v q") - { RedirectStandardOutput = true, RedirectStandardError = true })) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + // No MSBuild nodes left holding the output open: see DotnetBuilds. + Environment = { ["MSBUILDDISABLENODEREUSE"] = "1", ["DOTNET_CLI_USE_MSBUILD_SERVER"] = "0" }, + })) { var output = await publish!.StandardOutput.ReadToEndAsync() + await publish.StandardError.ReadToEndAsync(); await publish.WaitForExitAsync(); From ccb2575b150cb44410fba30d53953f5542cccab3 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 11:22:07 +0300 Subject: [PATCH 12/14] Publish the next release as 10.1 It adds runtimes, gRPC classic sessions, --describe, startup values on stdin and the tooling; HostInfo, SdkInfo and the stdin cut-off all say 10.1.0, so the published packages must too. --- .github/workflows/nuget-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nuget-publish.yml b/.github/workflows/nuget-publish.yml index 38c5993..0030ebd 100644 --- a/.github/workflows/nuget-publish.yml +++ b/.github/workflows/nuget-publish.yml @@ -24,7 +24,7 @@ jobs: SW.Serverless/SW.Serverless.csproj SW.Serverless.Sdk/SW.Serverless.Sdk.csproj major-version: '10' - minor-version: '0' + minor-version: '1' dotnet-version: '10.0.x' run-tests: true test-projects: '**/*UnitTests/*.csproj' From 1fdf461dae4463fecaec4153f3dc6086a78d83a2 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 11:22:16 +0300 Subject: [PATCH 13/14] Publish SimplyWorks.Serverless.Tooling with the other packages Bitween builds, tests and publishes adapters from its code editor with it, the same code the CLI runs. Published after the host and the contract it depends on. --- .github/workflows/nuget-publish.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nuget-publish.yml b/.github/workflows/nuget-publish.yml index 0030ebd..5e3b412 100644 --- a/.github/workflows/nuget-publish.yml +++ b/.github/workflows/nuget-publish.yml @@ -17,12 +17,14 @@ jobs: security-events: read # critical-vuln gate job uses: simplify9/.github/.github/workflows/reusable-nuget-publish.yml@0f33f79176152bc11acf0ff08ed970ae8b429b3d # main with: - # Contract FIRST: both of the others depend on it, and publishing them without it - # produces packages whose dependency cannot resolve — which is what 8.1.13 shipped as. + # Contract FIRST: the others depend on it, and publishing them without it produces + # packages whose dependency cannot resolve — which is what 8.1.13 shipped as. Tooling + # last: it depends on the host and the contract. nuget-projects: >- SW.Serverless.Contract/SW.Serverless.Contract.csproj SW.Serverless/SW.Serverless.csproj SW.Serverless.Sdk/SW.Serverless.Sdk.csproj + SW.Serverless.Tooling/SW.Serverless.Tooling.csproj major-version: '10' minor-version: '1' dotnet-version: '10.0.x' From 8d4e714552bada8b6098a26ec2484c63b102ab45 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 11:40:19 +0300 Subject: [PATCH 14/14] Reference the Bitween contract package as Bitween publishes it SimplyWorks.Bitween.Adapters is versioned with Bitween's SDK, so its first release is 10.0.59. NuGet reads the version as a minimum. --- SW.Serverless.Tooling/Scaffolding/Scaffolder.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs b/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs index a008625..8fcdb4a 100644 --- a/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs +++ b/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs @@ -40,8 +40,11 @@ public static class Scaffolder /// The SDK the templates reference; the first with --describe and the Bitween contract types. public const string SdkPackageVersion = "10.1.0"; - /// The Bitween contract package the templates reference. - public const string BitweenAdaptersPackageVersion = "1.0.0"; + /// + /// The Bitween contract package the templates reference: the first Bitween release that + /// publishes it. NuGet reads it as a minimum, so it resolves to the first one there is. + /// + public const string BitweenAdaptersPackageVersion = "10.0.59"; public static readonly IReadOnlyList Kinds = new[] { "handler", "mapper", "validator", "receiver" }; public static readonly IReadOnlyList Languages = new[] { "dotnet" };