diff --git a/.github/workflows/nuget-publish.yml b/.github/workflows/nuget-publish.yml index 38c5993..5e3b412 100644 --- a/.github/workflows/nuget-publish.yml +++ b/.github/workflows/nuget-publish.yml @@ -17,14 +17,16 @@ jobs: security-events: read # critical-vuln gate job uses: simplify9/.github/.github/workflows/reusable-nuget-publish.yml@0f33f79176152bc11acf0ff08ed970ae8b429b3d # main with: - # Contract FIRST: both of the others depend on it, and publishing them without it - # produces packages whose dependency cannot resolve — which is what 8.1.13 shipped as. + # Contract FIRST: the others depend on it, and publishing them without it produces + # packages whose dependency cannot resolve — which is what 8.1.13 shipped as. Tooling + # last: it depends on the host and the contract. nuget-projects: >- SW.Serverless.Contract/SW.Serverless.Contract.csproj SW.Serverless/SW.Serverless.csproj SW.Serverless.Sdk/SW.Serverless.Sdk.csproj + SW.Serverless.Tooling/SW.Serverless.Tooling.csproj major-version: '10' - minor-version: '0' + minor-version: '1' dotnet-version: '10.0.x' run-tests: true test-projects: '**/*UnitTests/*.csproj' diff --git a/.gitignore b/.gitignore index 1139e7b..3e29084 100644 --- a/.gitignore +++ b/.gitignore @@ -337,3 +337,4 @@ ASALocalRun/ binaries out +.test-work/ diff --git a/SW.Serverless.Compat.ManifestV1002/Program.cs b/SW.Serverless.Compat.ManifestV1002/Program.cs new file mode 100644 index 0000000..be6b3cd --- /dev/null +++ b/SW.Serverless.Compat.ManifestV1002/Program.cs @@ -0,0 +1,12 @@ +using System; +using System.IO; +using SW.Serverless.Contract.Catalog; + +// Reads the manifest at args[0] with the 10.0.2 parser and writes it back: what a released tool +// does to a manifest a newer one wrote. Prints the problems 10.0.2 finds, then the JSON. +var manifest = AdapterManifest.Parse(File.ReadAllText(args[0])); +foreach (var problem in manifest.Validate()) + Console.WriteLine("PROBLEM:" + problem); +Console.WriteLine("ENTRY:" + manifest.Entry); +Console.WriteLine("RUNTIME:" + manifest.Runtime); +Console.WriteLine("JSON:" + manifest.ToJson().ReplaceLineEndings(" ")); diff --git a/SW.Serverless.Compat.ManifestV1002/SW.Serverless.Compat.ManifestV1002.csproj b/SW.Serverless.Compat.ManifestV1002/SW.Serverless.Compat.ManifestV1002.csproj new file mode 100644 index 0000000..f498991 --- /dev/null +++ b/SW.Serverless.Compat.ManifestV1002/SW.Serverless.Compat.ManifestV1002.csproj @@ -0,0 +1,16 @@ + + + + + Exe + net10.0 + false + + + + + + + diff --git a/SW.Serverless.CompatibilityTests/Compat.cs b/SW.Serverless.CompatibilityTests/Compat.cs index 01a2cc7..f6c9640 100644 --- a/SW.Serverless.CompatibilityTests/Compat.cs +++ b/SW.Serverless.CompatibilityTests/Compat.cs @@ -8,7 +8,7 @@ using SW.CloudFiles.Extensions; using SW.PrimitiveTypes; using SW.Serverless.Installer; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; namespace SW.Serverless.CompatibilityTests; @@ -146,6 +146,28 @@ public record OldHostRun(int ExitCode, List Lines) } /// Runs the host built on the PUBLISHED SimplyWorks.Serverless 10.0.0 as its own process. + /// Reads a manifest file with the parser published in SimplyWorks.Serverless.Contract 10.0.2. + public static async Task ReadWithManifestV1002Async(string manifestPath) + { + var entry = System.IO.Path.Combine(BuildOutput("SW.Serverless.Compat.ManifestV1002"), "SW.Serverless.Compat.ManifestV1002.dll"); + var start = new ProcessStartInfo("dotnet") { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false }; + start.ArgumentList.Add(entry); + start.ArgumentList.Add(manifestPath); + + using var process = Process.Start(start)!; + var stdout = process.StandardOutput.ReadToEndAsync(); + var stderr = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(60_000)) + { + process.Kill(true); + Assert.Fail("the 10.0.2 manifest reader did not finish within a minute"); + } + + var lines = (await stdout).Split('\n', StringSplitOptions.RemoveEmptyEntries).Select(l => l.TrimEnd('\r')).ToList(); + lines.AddRange((await stderr).Split('\n', StringSplitOptions.RemoveEmptyEntries).Select(l => "stderr: " + l.TrimEnd('\r'))); + return new OldHostRun(process.ExitCode, lines); + } + public static async Task RunOldHostAsync(Bucket bucket, string adapter, params string[] args) { var entry = System.IO.Path.Combine(BuildOutput("SW.Serverless.Compat.OldHost"), "SW.Serverless.Compat.OldHost.dll"); diff --git a/SW.Serverless.CompatibilityTests/ListingAndManifestTests.cs b/SW.Serverless.CompatibilityTests/ListingAndManifestTests.cs index bf5aa0c..edd35ae 100644 --- a/SW.Serverless.CompatibilityTests/ListingAndManifestTests.cs +++ b/SW.Serverless.CompatibilityTests/ListingAndManifestTests.cs @@ -3,7 +3,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using SW.PrimitiveTypes; using SW.Serverless.Contract.Catalog; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; namespace SW.Serverless.CompatibilityTests; diff --git a/SW.Serverless.CompatibilityTests/MultiLanguageManifestTests.cs b/SW.Serverless.CompatibilityTests/MultiLanguageManifestTests.cs new file mode 100644 index 0000000..7dee375 --- /dev/null +++ b/SW.Serverless.CompatibilityTests/MultiLanguageManifestTests.cs @@ -0,0 +1,106 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Serverless.Contract.Catalog; + +namespace SW.Serverless.CompatibilityTests; + +/// +/// The manifest fields multi-language adapters add — runtime version, platforms and their entries, +/// contracts, the source the package carries — written by the current tools and read by the +/// manifest parser released Bitween uses (SimplyWorks.Serverless.Contract 10.0.2). That parser +/// must keep every one of them, find nothing wrong, and still see the fields it knows as before. +/// +[TestClass] +public class MultiLanguageManifestTests +{ + static AdapterManifest Written() => new() + { + Id = "acme.orders", + Version = "2.1.0", + DisplayName = "Acme orders", + Kinds = { "handler" }, + Runtime = AdapterManifest.PythonRuntime, + RuntimeVersion = ">=3.12", + Language = "python", + Entry = "main.py", + Lifecycle = AdapterManifest.ClassicLifecycle, + Platforms = new() { "linux-x64", "linux-arm64" }, + Entries = new() { ["linux-x64"] = "x64/main.py", ["linux-arm64"] = "arm64/main.py" }, + Contracts = new() { ["bitween"] = 1 }, + Source = new AdapterSource + { + Files = new() { ["main.py"] = new string('a', 64), ["requirements.lock"] = new string('b', 64) }, + BuildCommand = "serverless build", + Lockfiles = new() { "requirements.lock" }, + }, + }; + + [TestMethod] + public async Task The_released_manifest_parser_keeps_every_new_field_and_finds_nothing_wrong() + { + var written = Written(); + Assert.AreEqual(0, written.Validate().Count, string.Join("; ", written.Validate())); + var path = Path.Combine(Path.GetTempPath(), $"manifest-{Guid.NewGuid():N}.json"); + await File.WriteAllTextAsync(path, written.ToJson()); + try + { + var read = await Compat.ReadWithManifestV1002Async(path); + + Assert.AreEqual(0, read.ExitCode, read.ToString()); + Assert.IsNull(read.Value("PROBLEM"), read.ToString()); + Assert.AreEqual("main.py", read.Value("ENTRY"), "the default entry is what released readers use"); + Assert.AreEqual("python", read.Value("RUNTIME"), "a released host sees the runtime, and refuses one it can't run"); + + var before = JsonNode.Parse(written.ToJson())!; + var after = JsonNode.Parse(read.Value("JSON")!)!; + foreach (var field in new[] { "runtimeVersion", "platforms", "entries", "contracts", "source" }) + Assert.IsTrue(JsonNode.DeepEquals(before[field], after[field]), $"{field} changed in a round trip through 10.0.2:\n{after[field]}"); + } + finally + { + File.Delete(path); + } + } + + [TestMethod] + public void The_new_fields_round_trip_through_the_current_parser() + { + var again = AdapterManifest.Parse(Written().ToJson()); + + Assert.AreEqual(">=3.12", again.RuntimeVersion); + CollectionAssert.AreEqual(new[] { "linux-x64", "linux-arm64" }, again.Platforms); + Assert.AreEqual("arm64/main.py", again.EntryFor("linux-arm64")); + Assert.AreEqual("main.py", again.EntryFor("osx-arm64"), "a platform without its own entry uses the default"); + Assert.AreEqual(1, again.Contracts!["bitween"]); + Assert.AreEqual(AdapterSource.DefaultPath, again.Source!.Path); + Assert.AreEqual(2, again.Source.Files.Count); + Assert.IsNull(again.Extensions, "nothing unknown to the current parser"); + } + + [TestMethod] + public void A_manifest_without_the_new_fields_is_read_as_a_dotnet_adapter_as_before() + { + var old = AdapterManifest.Parse("""{ "id": "legacy", "entry": "Legacy.dll" }"""); + + Assert.AreEqual(AdapterManifest.DotnetRuntime, old.Runtime); + Assert.AreEqual("Legacy.dll", old.EntryFor("linux-x64")); + Assert.IsNull(old.Platforms); + Assert.IsNull(old.Source); + Assert.AreEqual(0, old.Validate().Count); + } + + [DataTestMethod] + [DataRow("""{ "runtime": "../bin/sh" }""", "runtime")] + [DataRow("""{ "platforms": ["Linux x64"] }""", "platform")] + [DataRow("""{ "platforms": ["linux-x64"], "entries": { "osx-arm64": "a" } }""", "doesn't list")] + [DataRow("""{ "platforms": ["linux-x64"], "entries": { "linux-x64": "../../etc/passwd" } }""", "inside the package")] + [DataRow("""{ "contracts": { "bitween": 0 } }""", "contract")] + [DataRow("""{ "source": { "files": { "main.py": "not-a-hash" } } }""", "SHA-256")] + [DataRow("""{ "source": { "path": "/abs", "files": {} } }""", "source.path")] + public void A_wrong_new_field_is_named_by_validation(string json, string expected) + { + var problems = AdapterManifest.Parse(json).Validate(); + Assert.IsTrue(problems.Any(p => p.Contains(expected)), $"expected a problem about {expected}: {string.Join("; ", problems)}"); + } +} diff --git a/SW.Serverless.CompatibilityTests/NewHostTests.cs b/SW.Serverless.CompatibilityTests/NewHostTests.cs index f1ce3fe..4ac3108 100644 --- a/SW.Serverless.CompatibilityTests/NewHostTests.cs +++ b/SW.Serverless.CompatibilityTests/NewHostTests.cs @@ -141,6 +141,14 @@ public async Task A_resident_adapter_on_sdk_10_0_0_runs_on_the_current_host() Assert.AreEqual(InstanceState.Ready, instance.State); Assert.AreEqual("v10:ping", await instance.InvokeAsync("Echo", "ping")); + // The 10.0.0 SDK knows nothing of the language-neutral Hello fields: they're simply + // empty, and nothing depends on them being there. + var described = adapters.Describe().Single(h => h.InstanceKey == "compat"); + Assert.AreEqual("", described.SdkLanguage); + Assert.AreEqual(0, described.Settings.Count); + Assert.AreEqual(0, described.Kinds.Count); + Assert.AreEqual(0, described.Contracts.Count); + await adapters.StopAsync("sdk10.resident", "compat", drain: false); } finally diff --git a/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs b/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs new file mode 100644 index 0000000..e3d5815 --- /dev/null +++ b/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs @@ -0,0 +1,160 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.PrimitiveTypes; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Tooling; + +namespace SW.Serverless.CompatibilityTests; + +/// +/// The package layout multi-language adapters bring, against the hosts already deployed: +/// a package that carries its source runs on old and new hosts alike, and only new hosts leave +/// the source unpacked; an adapter in another runtime is kept out of old hosts' sight entirely — +/// they only ever run adapters/{id}, and always with dotnet — while new hosts find it in the catalog. +/// +[TestClass] +public class PackageLayoutTests +{ + const string Sample = "SW.Serverless.Samples.Classic"; + + /// The sample's build output as a zip, with an adapter.json and any extra entries. + static string Package(AdapterManifest manifest, IDictionary? extra = null) + { + var output = Compat.BuildOutput(Sample); + var zip = Path.Combine(Path.GetTempPath(), "swsl-compat", $"layout-{Guid.NewGuid():N}.zip"); + Directory.CreateDirectory(Path.GetDirectoryName(zip)!); + using (var archive = ZipFile.Open(zip, ZipArchiveMode.Create)) + { + foreach (var file in Directory.EnumerateFiles(output, "*", SearchOption.AllDirectories)) + archive.CreateEntryFromFile(file, Path.GetRelativePath(output, file).Replace('\\', '/'), CompressionLevel.Fastest); + using (var writer = new StreamWriter(archive.CreateEntry(AdapterManifest.FileName).Open())) + writer.Write(manifest.ToJson()); + foreach (var (name, content) in extra ?? new Dictionary()) + using (var writer = new StreamWriter(archive.CreateEntry(name).Open())) + writer.Write(content); + } + return zip; + } + + static string Sha256(string text) => Convert.ToHexString(SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(text))).ToLowerInvariant(); + + static AdapterManifest DotnetWithSource(string id) => new() + { + Id = id, + Entry = Sample + ".dll", + Source = new AdapterSource { Files = { ["Program.cs"] = Sha256("// the source"), ["Sample.csproj"] = Sha256("") } }, + }; + + static AdapterManifest Python(string id) => new() + { + Id = id, + Runtime = AdapterManifest.PythonRuntime, + Language = "python", + Entry = "main.py", + Kinds = { "handler" }, + }; + + static AdapterRepository Repository(Bucket bucket) => new(bucket.Files, _ => { }); + + [TestMethod] + public async Task A_package_carrying_its_source_runs_on_old_and_new_hosts_and_new_ones_leave_the_source_packed() + { + using var bucket = new Bucket(); + const string id = "layout.withsource"; + var zip = Package(DotnetWithSource(id), new Dictionary + { + ["source/Program.cs"] = "// the source", + ["source/Sample.csproj"] = "", + }); + await Repository(bucket).PublishVersionAsync(id, "1.0.0", zip, + new PackageInfo { EntryAssembly = Sample + ".dll", Manifest = DotnetWithSource(id) }, promote: true, "compat"); + + var old = await Compat.RunOldHostAsync(bucket, id, "--command", "Echo", "--input", "carried"); + Assert.AreEqual(0, old.ExitCode, old.ToString()); + Assert.AreEqual("carried", old.Value("RESULT"), "a host on the released 10.0.0 runs it as before"); + + using var host = Compat.NewHost(bucket); + using (var service = (ServerlessService)host.GetRequiredService()) + { + await service.StartAsync(id, "correlation"); + Assert.AreEqual("carried", await service.InvokeAsync("Echo", "carried")); + } + + var installed = await host.GetRequiredService().InstallAsync(id); + Assert.IsTrue(File.Exists(installed.LocalPath)); + Assert.IsFalse(Directory.Exists(Path.Combine(installed.Directory, "source")), "the source was unpacked onto the host"); + } + + [TestMethod] + public async Task A_folder_called_source_in_a_package_without_declared_source_is_unpacked_as_before() + { + using var bucket = new Bucket(); + const string id = "layout.ownsource"; + var manifest = new AdapterManifest { Id = id, Entry = Sample + ".dll" }; + await Repository(bucket).PublishVersionAsync(id, "1.0.0", + Package(manifest, new Dictionary { ["source/needed-at-runtime.txt"] = "keep me" }), + new PackageInfo { EntryAssembly = Sample + ".dll", Manifest = manifest }, promote: true, "compat"); + + using var host = Compat.NewHost(bucket); + var installed = await host.GetRequiredService().InstallAsync(id); + + Assert.AreEqual("keep me", await File.ReadAllTextAsync(Path.Combine(installed.Directory, "source", "needed-at-runtime.txt"))); + } + + [TestMethod] + public async Task An_adapter_in_another_runtime_is_out_of_old_hosts_sight_and_new_hosts_find_its_current_version() + { + using var bucket = new Bucket(); + const string id = "layout.python"; + var repository = Repository(bucket); + var package = new PackageInfo { EntryAssembly = "main.py", Manifest = Python(id) }; + await repository.PublishVersionAsync(id, "1.0.0", Package(Python(id)), package, promote: true, "compat"); + await repository.PublishVersionAsync(id, "1.1.0", Package(Python(id)), package, promote: false, "compat"); + + Assert.AreEqual(0, (await bucket.KeysAsync("adapters/")).Count, "nothing under adapters/, where old hosts and old Bitween look"); + Assert.AreEqual("1.0.0", (await repository.LoadEntryAsync(id)).Current); + + var old = await Compat.RunOldHostAsync(bucket, id, "--command", "Echo", "--input", "x"); + Assert.AreNotEqual(0, old.ExitCode, "an old host must not find it, let alone start it with dotnet"); + + using (var host = Compat.NewHost(bucket)) + Assert.AreEqual($"adapters-versions/{id}/1.0.0", + (await host.GetRequiredService().GetMetadataAsync(id)).RemoteKey); + + await repository.PromoteAsync(id, "1.1.0", Path.Combine(bucket.Path, "..", "work-" + Guid.NewGuid().ToString("N"))); + + Assert.AreEqual(0, (await bucket.KeysAsync("adapters/")).Count, "promote doesn't put it there either"); + Assert.AreEqual("1.1.0", (await repository.LoadEntryAsync(id)).Current); + using (var host = Compat.NewHost(bucket)) + Assert.AreEqual($"adapters-versions/{id}/1.1.0", + (await host.GetRequiredService().GetMetadataAsync(id)).RemoteKey); + } + + [TestMethod] + public async Task An_id_published_as_dotnet_cannot_move_to_another_runtime() + { + using var bucket = new Bucket(); + const string id = "layout.switch"; + var repository = Repository(bucket); + await repository.PublishVersionAsync(id, "1.0.0", Package(new AdapterManifest { Id = id, Entry = Sample + ".dll" }), + new PackageInfo { EntryAssembly = Sample + ".dll" }, promote: true, "compat"); + + var refused = await Assert.ThrowsExceptionAsync(() => repository.PublishVersionAsync(id, "2.0.0", + Package(Python(id)), new PackageInfo { EntryAssembly = "main.py", Manifest = Python(id) }, promote: true, "compat")); + StringAssert.Contains(refused.Message, "new id"); + Assert.AreEqual("1.0.0", (await repository.LoadEntryAsync(id)).Current); + } + + [TestMethod] + public async Task An_adapter_in_another_runtime_must_be_published_with_a_version() + { + using var bucket = new Bucket(); + var refused = await Assert.ThrowsExceptionAsync(() => Repository(bucket).PublishUnversionedAsync("layout.unversioned", + Package(Python("layout.unversioned")), new PackageInfo { EntryAssembly = "main.py", Manifest = Python("layout.unversioned") })); + + StringAssert.Contains(refused.Message, "with a version"); + Assert.AreEqual(0, (await bucket.KeysAsync()).Count); + } +} diff --git a/SW.Serverless.CompatibilityTests/SW.Serverless.CompatibilityTests.csproj b/SW.Serverless.CompatibilityTests/SW.Serverless.CompatibilityTests.csproj index c93e8aa..8910b0a 100644 --- a/SW.Serverless.CompatibilityTests/SW.Serverless.CompatibilityTests.csproj +++ b/SW.Serverless.CompatibilityTests/SW.Serverless.CompatibilityTests.csproj @@ -30,6 +30,8 @@ + A self-contained executable — Go, Rust, a native-compiled .NET app. + public const string ExecRuntime = "exec"; + public const string PythonRuntime = "python"; + public const string NodeRuntime = "node"; public int ManifestVersion { get; set; } = CurrentManifestVersion; @@ -78,6 +82,31 @@ public class AdapterManifest /// The file the runtime starts, relative to the package root. public string Entry { get; set; } + /// + /// The runtime versions it needs, e.g. ">=3.12" for Python. A host refuses to install it + /// when the runtime it has is outside. Absent means any. + /// + public string RuntimeVersion { get; set; } + + /// + /// The platforms it runs on — linux-x64, linux-arm64, osx-arm64, win-x64 — for a package + /// with platform-specific contents, such as an exec binary or Python wheels with native code. + /// Absent means it runs anywhere its runtime does. + /// + public List Platforms { get; set; } + + /// + /// A per-platform , for a package carrying one build per platform. + /// stays the default, and stays what readers before this field use. + /// + public Dictionary Entries { get; set; } + + /// The contracts it implements and their versions, e.g. bitween → 1. + public Dictionary Contracts { get; set; } + + /// The source the package was built from, when it carries it. + public AdapterSource Source { get; set; } + /// or . public string Lifecycle { get; set; } = ClassicLifecycle; @@ -116,6 +145,10 @@ public static AdapterManifest Parse(string json) => public string ToJson() => JsonSerializer.Serialize(this, JsonOptions); + /// What the runtime starts on : its own entry, or the default. + public string EntryFor(string platform) => + platform != null && Entries != null && Entries.TryGetValue(platform, out var entry) ? entry : Entry; + public bool IsResident => string.Equals(Lifecycle, ResidentLifecycle, StringComparison.OrdinalIgnoreCase); @@ -123,6 +156,9 @@ public static AdapterManifest Parse(string json) => static readonly Regex IdPattern = new(@"^[a-z0-9][a-z0-9._-]*$"); static readonly Regex VersionPattern = new(@"^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?$"); + static readonly Regex RuntimePattern = new(@"^[a-z][a-z0-9.-]*$"); + static readonly Regex PlatformPattern = new(@"^[a-z]+-[a-z0-9]+$"); + static readonly Regex Sha256Pattern = new(@"^[0-9a-fA-F]{64}$"); static readonly Regex PropertyNamePattern = new(@"^[A-Za-z_][A-Za-z0-9_.:-]*$"); /// What is wrong with this manifest, empty when nothing is. @@ -140,6 +176,33 @@ public IReadOnlyList Validate() problems.Add($"entry '{Entry}' must be a path inside the package."); if (!string.IsNullOrEmpty(Icon) && !IsPackagePath(Icon)) problems.Add($"icon '{Icon}' must be a path inside the package."); + if (!string.IsNullOrEmpty(Runtime) && !RuntimePattern.IsMatch(Runtime)) + problems.Add($"runtime '{Runtime}' must be a name such as dotnet, exec, python or node."); + foreach (var platform in Platforms ?? new List()) + if (!PlatformPattern.IsMatch(platform ?? "")) + problems.Add($"platform '{platform}' must look like linux-x64 or osx-arm64."); + foreach (var (platform, entry) in Entries ?? new Dictionary()) + { + if (Platforms == null || !Platforms.Contains(platform)) + problems.Add($"entries names platform '{platform}', which platforms doesn't list."); + if (!IsPackagePath(entry)) + problems.Add($"entries.{platform} '{entry}' must be a path inside the package."); + } + foreach (var (contract, version) in Contracts ?? new Dictionary()) + if (string.IsNullOrWhiteSpace(contract) || version < 1) + problems.Add($"contract '{contract}' needs a name and a version of 1 or more."); + if (Source != null) + { + if (!IsPackagePath(Source.Path ?? "")) + problems.Add($"source.path '{Source.Path}' must be a folder inside the package."); + foreach (var (file, hash) in Source.Files ?? new Dictionary()) + { + if (!IsPackagePath(file)) + problems.Add($"source file '{file}' must be a path inside the source folder."); + if (!Sha256Pattern.IsMatch(hash ?? "")) + problems.Add($"source file '{file}' needs its SHA-256 as 64 hex digits."); + } + } if (!string.IsNullOrEmpty(Lifecycle) && Lifecycle is not (ClassicLifecycle or ResidentLifecycle)) problems.Add($"lifecycle '{Lifecycle}' must be '{ClassicLifecycle}' or '{ResidentLifecycle}'."); if (Protocol != null && Protocol.Min > Protocol.Max) @@ -192,6 +255,32 @@ public class AdapterProtocolRange public Dictionary Extensions { get; set; } } + /// + /// The source a package carries, so a published version can be read, compared, audited and + /// rebuilt. The files themselves sit in the package under ; hosts don't + /// extract them. + /// + public class AdapterSource + { + public const string DefaultPath = "source"; + + /// The folder inside the package that holds the source. + public string Path { get; set; } = DefaultPath; + + /// Every source file, relative to , with its SHA-256 in hex. + public Dictionary Files { get; set; } = new(); + + /// How the package was built from the source, e.g. "dotnet publish -c Release". + public string BuildCommand { get; set; } + + /// The lockfiles the build resolved dependencies from, relative to . + public List Lockfiles { get; set; } + + /// Fields written by a newer tool, kept so they are not lost on a round trip. + [JsonExtensionData] + public Dictionary Extensions { get; set; } + } + /// The oldest host each side may run on. Absent means no constraint. public class AdapterCompatibility { diff --git a/SW.Serverless.Contract/Catalog/AdapterSelfDescription.cs b/SW.Serverless.Contract/Catalog/AdapterSelfDescription.cs new file mode 100644 index 0000000..b7f09cc --- /dev/null +++ b/SW.Serverless.Contract/Catalog/AdapterSelfDescription.cs @@ -0,0 +1,80 @@ +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace SW.Serverless.Contract.Catalog +{ + /// + /// What an adapter says about itself when started with : printed as one JSON + /// document on stdout, after which it exits. Every SDK, in every language, produces this, so the + /// tools learn an adapter's settings, commands, kinds and contracts by running it — not by + /// reading .NET assemblies — and write them into its manifest. + /// + public class AdapterSelfDescription + { + public const string Flag = "--describe"; + public const int CurrentDescribeVersion = 1; + + public int DescribeVersion { get; set; } = CurrentDescribeVersion; + + /// dotnet, python, node, go… + public string SdkLanguage { get; set; } + public string SdkVersion { get; set; } + + /// classic or resident: how the adapter's entry point runs it. + public string Lifecycle { get; set; } + + /// The protocol versions it speaks: 1 is the classic text protocol, 2 is gRPC. + public AdapterProtocolRange Protocol { get; set; } + + public List Settings { get; set; } = new(); + public List Commands { get; set; } = new(); + public List Kinds { get; set; } = new(); + public Dictionary Contracts { get; set; } = new(); + + /// Anything that kept the description from being complete, e.g. a handler that couldn't be built without its settings. + public List Warnings { get; set; } = new(); + + [JsonExtensionData] + public Dictionary Extensions { get; set; } + + public static AdapterSelfDescription Parse(string json) => + JsonSerializer.Deserialize(json, AdapterManifest.JsonOptions) + ?? throw new JsonException("The adapter's description is empty."); + + public string ToJson() => JsonSerializer.Serialize(this, AdapterManifest.JsonOptions); + } + + public class DescribedSetting + { + public string Name { get; set; } + public string Description { get; set; } + + /// text, multiline, number, boolean, select or json, as manifest properties are typed. + public string Type { get; set; } = AdapterProperty.TextType; + + public bool Required { get; set; } + public bool Secret { get; set; } + public string Default { get; set; } + + [JsonExtensionData] + public Dictionary Extensions { get; set; } + } + + public class DescribedCommand + { + public string Name { get; set; } + public string Description { get; set; } + + /// JSON Schema of the argument; null when it takes none. + public JsonElement? InputSchema { get; set; } + + /// JSON Schema of the result; null when it returns nothing. + public JsonElement? OutputSchema { get; set; } + + public bool ReturnsValue { get; set; } + + [JsonExtensionData] + public Dictionary Extensions { get; set; } + } +} diff --git a/SW.Serverless.Contract/Protos/adapter.proto b/SW.Serverless.Contract/Protos/adapter.proto index 681a408..f692739 100644 --- a/SW.Serverless.Contract/Protos/adapter.proto +++ b/SW.Serverless.Contract/Protos/adapter.proto @@ -118,6 +118,36 @@ message Hello { // adapters and hosts built before this field existed; a host that understands this one gets // the shape as well as the name. repeated CommandInfo commands = 7; + + // Fields 8–11 describe the adapter without .NET in the picture, for SDKs in any language. All + // optional: a host reads them when present and an adapter built before them leaves them out. + + // The SDK's language — dotnet, python, node, go — beside sdk_version. + string sdk_language = 8; + + // The settings the adapter reads, the language-neutral form of Runner.Expect: what a host or a + // UI needs to ask for before running it. + repeated SettingInfo settings = 9; + + // The kinds of adapter it implements for a contract, as that contract names them — for + // Bitween: handler, mapper, validator, receiver. + repeated string kinds = 10; + + // The contracts it implements and their versions, e.g. bitween -> 1. + map contracts = 11; +} + +// One setting an adapter reads. +message SettingInfo { + string name = 1; + string description = 2; + bool required = 3; + // Secret settings are masked wherever they're shown and never logged. + bool secret = 4; + // Empty when the setting has no default. + string default_value = 5; + // text, multiline, number, boolean, select or json, as adapter manifests type properties. + string type = 6; } // One callable command on an adapter. @@ -140,6 +170,11 @@ message CommandInfo { // From [AdapterCommand(Description = "…")] on the method. Optional, and worth setting: the // difference between a list of names and a list an operator can act on. string description = 5; + + // JSON Schemas of the argument and the result, for SDKs in any language: parameter_type and + // parameter_schema above are .NET's view and stay as they are. Empty when not given. + string input_schema = 6; + string output_schema = 7; } message InvokeResult { diff --git a/SW.Serverless.Desktop/MainWindow.xaml.cs b/SW.Serverless.Desktop/MainWindow.xaml.cs index ffa21ed..c79eaa1 100644 --- a/SW.Serverless.Desktop/MainWindow.xaml.cs +++ b/SW.Serverless.Desktop/MainWindow.xaml.cs @@ -1,6 +1,6 @@ using Microsoft.Win32; using Newtonsoft.Json; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; using System; using System.Collections.Generic; using System.IO; diff --git a/SW.Serverless.Installer.UnitTests/AdapterDescriberTests.cs b/SW.Serverless.Installer.UnitTests/AdapterDescriberTests.cs index c51c20a..96e3832 100644 --- a/SW.Serverless.Installer.UnitTests/AdapterDescriberTests.cs +++ b/SW.Serverless.Installer.UnitTests/AdapterDescriberTests.cs @@ -2,7 +2,7 @@ using System.IO; using System.Linq; using Microsoft.VisualStudio.TestTools.UnitTesting; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; namespace SW.Serverless.Installer.UnitTests; diff --git a/SW.Serverless.Installer.UnitTests/BuildTests.cs b/SW.Serverless.Installer.UnitTests/BuildTests.cs new file mode 100644 index 0000000..27fe5e0 --- /dev/null +++ b/SW.Serverless.Installer.UnitTests/BuildTests.cs @@ -0,0 +1,244 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Tooling.Building; +using SW.Serverless.Tooling.Conformance; + +namespace SW.Serverless.Installer.UnitTests; + +[TestClass] +public class IgnoreRuleTests +{ + [DataTestMethod] + [DataRow("bin/Debug/net10.0/a.dll", true)] + [DataRow("src/obj/project.assets.json", true)] + [DataRow("node_modules/x/index.js", true)] + [DataRow(".env", true)] + [DataRow("config/.env.production", true)] + [DataRow("keys/partner.pem", true)] + [DataRow("appsettings.Production.json", true)] + [DataRow("appsettings.json", false)] + [DataRow("Handler.cs", false)] + [DataRow("src/binder/Thing.cs", false)] + public void Built_in_rules(string path, bool ignored) => + Assert.AreEqual(ignored, new IgnoreRules(IgnoreRules.BuiltIn).Ignores(path), path); + + [TestMethod] + public void A_project_s_own_patterns_add_to_them_and_can_bring_a_file_back() + { + var rules = new IgnoreRules(IgnoreRules.BuiltIn.Concat(new[] { "fixtures/**/*.zip", "/local.txt", "*.log", "!keep.log", "!keys/public.pem" })); + + Assert.IsTrue(rules.Ignores("fixtures/a/b/big.zip")); + Assert.IsTrue(rules.Ignores("fixtures/big.zip")); + Assert.IsTrue(rules.Ignores("local.txt")); + Assert.IsFalse(rules.Ignores("sub/local.txt"), "a leading slash anchors to the root"); + Assert.IsTrue(rules.Ignores("logs/run.log")); + Assert.IsFalse(rules.Ignores("keep.log")); + Assert.IsFalse(rules.Ignores("keys/public.pem")); + } +} + +[TestClass] +public class SecretScannerTests +{ + [DataTestMethod] + [DataRow("-----BEGIN RSA PRIVATE KEY-----", "a private key")] + [DataRow("-----BEGIN OPENSSH PRIVATE KEY-----", "a private key")] + [DataRow("var key = \"AKIAIOSFODNN7EXAMPLE\";", "an AWS access key")] + [DataRow("\"Db\": \"Server=x;User Id=sa;Password=Sup3rS3cret!;\"", "a password in a connection string")] + [DataRow("const apiKey = \"9f8e7d6c5b4a3f2e1d0c\";", "a secret assigned in code")] + [DataRow("client_secret: 'abcdefgh12345678'", "a secret assigned in code")] + [DataRow("token = ghp_abcdefghijklmnopqrstuvwxyz0123456789AB", "a GitHub token")] + public void A_secret_is_found_and_named(string line, string kind) + { + var finding = SecretScanner.Scan("Handler.cs", "first line\n" + line + "\n").Single(); + Assert.AreEqual(2, finding.Line); + Assert.AreEqual(kind, finding.Kind); + } + + [DataTestMethod] + [DataRow("Runner.Expect(\"Password\", isPrivate: true);")] + [DataRow("var password = Runner.StartupValueOf(\"Password\");")] + [DataRow("\"Password\": \"********\"")] + [DataRow("apiKey = \"your-api-key-here\"")] + [DataRow("secret: \"${SECRET}\"")] + [DataRow("Password={password};")] + public void Code_that_only_names_or_reads_a_secret_is_not_reported(string line) => + Assert.AreEqual(0, SecretScanner.Scan("Handler.cs", line).Count(), line); +} + +/// +/// serverless build on a real adapter project: built, described, its manifest written from the +/// adapter itself, its source carried under the rules — and the package it makes is what the +/// conformance kit and any host run. +/// +[TestClass] +public class BuildTests +{ + static string RepositoryRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null && !File.Exists(Path.Combine(dir.FullName, "SW.Serverless.sln"))) dir = dir.Parent; + return dir!.FullName; + } + + /// + /// A copy of the BitweenHandler test adapter as an author's project: inside the repository, so + /// its reference to the SDK project still resolves, with an adapter.json and a few files the + /// source rules must leave out. + /// + static string AuthorProject(string authorJson = null, IDictionary extraFiles = null) + { + var root = RepositoryRoot(); + var project = Path.Combine(root, ".test-work", Guid.NewGuid().ToString("N"), "AcmeOrders"); + Directory.CreateDirectory(project); + var sdk = Path.GetRelativePath(project, Path.Combine(root, "SW.Serverless.Sdk", "SW.Serverless.Sdk.csproj")); + File.WriteAllText(Path.Combine(project, "AcmeOrders.csproj"), $""" + + + Exe + net10.0 + + + + + + """); + File.Copy(Path.Combine(root, "SW.Serverless.UnitTests.BitweenHandler", "Program.cs"), Path.Combine(project, "Program.cs")); + File.WriteAllText(Path.Combine(project, AdapterManifest.FileName), authorJson ?? + """{ "id": "acme.orders", "version": "1.2.0", "displayName": "Acme orders", "properties": [ { "name": "Mode", "type": "select", "options": ["working", "broken"] } ] }"""); + File.WriteAllText(Path.Combine(project, ".env"), "API_KEY=never-carried"); + File.WriteAllText(Path.Combine(project, "notes.log"), "left out by .serverlessignore"); + File.WriteAllText(Path.Combine(project, ".serverlessignore"), "*.log\n"); + foreach (var (name, content) in extraFiles ?? new Dictionary()) + File.WriteAllText(Path.Combine(project, name), content); + return project; + } + + [ClassCleanup] + public static void Cleanup() + { + try { Directory.Delete(Path.Combine(RepositoryRoot(), ".test-work"), true); } catch { } + } + + static Task Build(string project, Action configure = null) + { + var request = new BuildRequest { ProjectDirectory = project, Log = Console.WriteLine }; + configure?.Invoke(request); + return PackageBuilder.BuildAsync(request); + } + + [TestMethod] + public async Task A_build_writes_the_manifest_from_the_adapter_and_carries_its_source_under_the_rules() + { + var project = AuthorProject(); + var result = await Build(project); + Assert.IsTrue(result.Succeeded, string.Join("; ", result.Problems)); + + var manifest = result.Manifest; + Assert.AreEqual("acme.orders", manifest.Id); + Assert.AreEqual("1.2.0", manifest.Version); + Assert.AreEqual("Acme orders", manifest.DisplayName, "the author's text is kept"); + Assert.AreEqual("dotnet", manifest.Runtime); + Assert.AreEqual("AcmeOrders.dll", manifest.Entry); + Assert.AreEqual("10.1.0", manifest.SdkVersion); + Assert.AreEqual(AdapterManifest.ClassicLifecycle, manifest.Lifecycle); + CollectionAssert.AreEqual(new[] { "handler" }, manifest.Kinds); + Assert.AreEqual(1, manifest.Contracts!["bitween"]); + + var mode = manifest.Properties.Single(p => p.Name == "Mode"); + Assert.AreEqual("select", mode.Type, "the author's presentation is kept"); + Assert.AreEqual("working", mode.Default, "the adapter's default wins"); + var apiKey = manifest.Properties.Single(p => p.Name == "ApiKey"); + Assert.IsTrue(apiKey.Required); + Assert.IsTrue(apiKey.Secret); + Assert.IsNull(apiKey.Default); + + var carried = manifest.Source!.Files.Keys.ToList(); + Assert.IsTrue(carried.Any(f => f.EndsWith("AcmeOrders/Program.cs")), string.Join(", ", carried)); + Assert.IsTrue(carried.Any(f => f.EndsWith("AcmeOrders/AcmeOrders.csproj"))); + Assert.IsTrue(carried.Any(f => f.EndsWith("SW.Serverless.Sdk/Runner.cs")), "the local project it references, so it rebuilds"); + Assert.IsFalse(carried.Any(f => f.EndsWith(".env") || f.EndsWith("notes.log") || f.Contains("/bin/") || f.Contains("/obj/")), + string.Join(", ", carried.Where(f => f.EndsWith(".env") || f.EndsWith(".log") || f.Contains("/bin/") || f.Contains("/obj/")))); + + using var zip = ZipFile.OpenRead(result.ZipPath); + var names = zip.Entries.Select(e => e.FullName.Replace('\\', '/')).ToList(); + CollectionAssert.Contains(names, "adapter.json"); + CollectionAssert.Contains(names, "AcmeOrders.dll"); + Assert.IsTrue(names.Any(n => n.StartsWith("source/") && n.EndsWith("AcmeOrders/Program.cs"))); + Assert.IsTrue(result.ZipPath.EndsWith("acme.orders-1.2.0.zip")); + } + + [TestMethod] + public async Task What_a_build_makes_passes_the_conformance_kit() + { + var result = await Build(AuthorProject()); + Assert.IsTrue(result.Succeeded, string.Join("; ", result.Problems)); + + var report = await new ConformanceRunner().RunAsync(new ConformanceOptions + { + PackageDirectory = result.PackageDirectory, + Settings = new Dictionary { ["ApiKey"] = "k" }, + }); + Assert.IsTrue(report.Passed, string.Join("; ", report.Checks.Where(c => c.Outcome == CheckOutcome.Failed))); + } + + [TestMethod] + public async Task A_secret_in_the_source_stops_the_build_until_it_is_allowed() + { + var project = AuthorProject(extraFiles: new Dictionary + { + ["Defaults.cs"] = "class Defaults { const string ApiKey = \"live-9f8e7d6c5b4a3f2e\"; }", + }); + + var stopped = await Build(project); + Assert.IsFalse(stopped.Succeeded); + var problem = stopped.Problems.Single(); + StringAssert.Contains(problem, "Defaults.cs:1"); + StringAssert.Contains(problem, "--allow"); + Assert.IsNull(stopped.ZipPath, "nothing was built"); + + var allowed = await Build(project, r => r.AllowedFiles.Add("Defaults.cs")); + Assert.IsTrue(allowed.Succeeded, string.Join("; ", allowed.Problems)); + } + + [TestMethod] + public async Task Without_source_the_package_carries_none() + { + var result = await Build(AuthorProject(), r => r.IncludeSource = false); + + Assert.IsTrue(result.Succeeded, string.Join("; ", result.Problems)); + Assert.IsNull(result.Manifest.Source); + Assert.IsFalse(Directory.Exists(Path.Combine(result.PackageDirectory, "source"))); + } + + [TestMethod] + public async Task A_dry_run_reports_the_source_and_builds_nothing() + { + var project = AuthorProject(); + var result = await Build(project, r => r.DryRun = true); + + Assert.IsTrue(result.Succeeded); + Assert.IsTrue(result.SourceFiles.Any(f => f.Path.EndsWith("Program.cs"))); + Assert.IsNull(result.ZipPath); + Assert.IsFalse(Directory.Exists(Path.Combine(project, "bin", "serverless"))); + } + + [TestMethod] + public async Task A_gRPC_adapter_the_author_marks_classic_is_built_to_run_classically_over_gRPC() + { + var root = RepositoryRoot(); + var project = AuthorProject("""{ "id": "acme.grpc", "lifecycle": "classic" }"""); + File.Copy(Path.Combine(root, "SW.Serverless.UnitTests.GrpcClassicAdapter", "Program.cs"), Path.Combine(project, "Program.cs"), overwrite: true); + + var result = await Build(project); + Assert.IsTrue(result.Succeeded, string.Join("; ", result.Problems)); + Assert.AreEqual(AdapterManifest.ClassicLifecycle, result.Manifest.Lifecycle); + Assert.AreEqual(2, result.Manifest.Protocol!.Min); + } +} diff --git a/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs b/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs index 3902b29..35bd4b6 100644 --- a/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs +++ b/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs @@ -7,7 +7,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using SW.PrimitiveTypes; using SW.Serverless.Contract.Catalog; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; namespace SW.Serverless.Installer.UnitTests; diff --git a/SW.Serverless.Installer.UnitTests/CliCommandTests.cs b/SW.Serverless.Installer.UnitTests/CliCommandTests.cs new file mode 100644 index 0000000..f894bb2 --- /dev/null +++ b/SW.Serverless.Installer.UnitTests/CliCommandTests.cs @@ -0,0 +1,189 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Tooling; + +namespace SW.Serverless.Installer.UnitTests; + +/// +/// The commands for adapters in any language, run through the CLI's own entry point as a person or +/// a CI job runs them: init, build, test, run, manifest validate and publish. +/// +[TestClass] +public class CliCommandTests +{ + static string RepositoryRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null && !File.Exists(Path.Combine(dir.FullName, "SW.Serverless.sln"))) dir = dir.Parent; + return dir!.FullName; + } + + static string WorkFolder() + { + var folder = Path.Combine(RepositoryRoot(), ".test-work", "cli-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(folder); + return folder; + } + + [ClassCleanup] + public static void Cleanup() + { + try { Directory.Delete(Path.Combine(RepositoryRoot(), ".test-work"), true); } catch { } + } + + /// Runs the CLI and returns its exit code and what it printed. + static async Task<(int Exit, string Output)> Cli(params string[] args) + { + var original = Console.Out; + using var output = new StringWriter(); + Console.SetOut(output); + try + { + var exit = await Program.RunAsync(args, _ => null); + return (exit, output.ToString()); + } + finally + { + Console.SetOut(original); + } + } + + [TestMethod] + public async Task Init_makes_a_project_and_refuses_what_it_can_t_make() + { + var work = WorkFolder(); + + var (exit, output) = await Cli("init", "AcmeOrders", "--kind", "validator", "--dir", work); + Assert.AreEqual(Program.Success, exit, output); + var project = Path.Combine(work, "AcmeOrders"); + foreach (var file in new[] { "AcmeOrders.csproj", "Program.cs", "adapter.json", "settings.example.json", ".gitignore", "README.md" }) + Assert.IsTrue(File.Exists(Path.Combine(project, file)), file); + Assert.AreEqual("acme.orders", AdapterManifest.Parse(File.ReadAllText(Path.Combine(project, "adapter.json"))).Id); + StringAssert.Contains(File.ReadAllText(Path.Combine(project, "Program.cs")), "IBitweenValidator"); + + Assert.AreEqual(Program.Failure, (await Cli("init", "AcmeOrders", "--dir", work)).Exit, "an existing project isn't overwritten"); + var (pythonExit, pythonOutput) = await Cli("init", "PyOrders", "--lang", "python", "--dir", work); + Assert.AreEqual(Program.Failure, pythonExit); + StringAssert.Contains(pythonOutput, "arrives with that language's SDK"); + } + + [TestMethod] + public async Task Manifest_validate_passes_a_good_file_and_names_what_is_wrong_with_a_bad_one() + { + var work = WorkFolder(); + File.WriteAllText(Path.Combine(work, "adapter.json"), """{ "id": "good.adapter", "version": "1.0.0" }"""); + Assert.AreEqual(Program.Success, (await Cli("manifest", "validate", work)).Exit); + + File.WriteAllText(Path.Combine(work, "adapter.json"), """{ "id": "Bad Id", "runtime": "../sh" }"""); + var (exit, output) = await Cli("manifest", "validate", work); + Assert.AreEqual(Program.Failure, exit); + StringAssert.Contains(output, "id 'Bad Id'"); + StringAssert.Contains(output, "runtime '../sh'"); + } + + /// An author's project, as BuildTests makes one: the BitweenHandler adapter with an adapter.json. + static string AuthorProject() + { + var root = RepositoryRoot(); + var project = Path.Combine(WorkFolder(), "AcmeOrders"); + Directory.CreateDirectory(project); + var sdk = Path.GetRelativePath(project, Path.Combine(root, "SW.Serverless.Sdk", "SW.Serverless.Sdk.csproj")); + File.WriteAllText(Path.Combine(project, "AcmeOrders.csproj"), $""" + + + Exe + net10.0 + + + + + + """); + File.Copy(Path.Combine(root, "SW.Serverless.UnitTests.BitweenHandler", "Program.cs"), Path.Combine(project, "Program.cs")); + File.WriteAllText(Path.Combine(project, "adapter.json"), """{ "id": "acme.orders", "version": "1.2.0", "displayName": "Acme orders" }"""); + return project; + } + + [TestMethod] + public async Task Build_test_run_and_publish_take_a_project_into_storage() + { + var project = AuthorProject(); + var settings = Path.Combine(Path.GetDirectoryName(project)!, "settings.json"); + File.WriteAllText(settings, """{ "ApiKey": "test-key" }"""); + + var build = await Cli("build", project); + Assert.AreEqual(Program.Success, build.Exit, build.Output); + StringAssert.Contains(build.Output, "Source carried"); + var zip = Path.Combine(project, "bin", "serverless", "acme.orders-1.2.0.zip"); + Assert.IsTrue(File.Exists(zip), build.Output); + + var test = await Cli("test", zip, "--settings", settings); + Assert.AreEqual(Program.Success, test.Exit, test.Output); + StringAssert.Contains(test.Output, "PASS bitween handler: Handle answers example 1"); + StringAssert.Contains(test.Output, "Conforms."); + + var run = await Cli("run", zip, "--settings", settings, "--call", "Handle", "--input", """{"Data":"{\"order\":1}"}"""); + Assert.AreEqual(Program.Success, run.Exit, run.Output); + StringAssert.Contains(run.Output, "accepted"); + + var store = Path.Combine(Path.GetDirectoryName(project)!, "store"); + var publish = await Cli("publish", zip, "-p", "local", "-b", "cli-tests", "-u", store); + Assert.AreEqual(Program.Success, publish.Exit, publish.Output); + StringAssert.Contains(publish.Output, "Published acme.orders 1.2.0"); + + var repository = new AdapterRepository(CloudFilesFactory.Create(new ServerlessUploadOptions + { + Provider = "local", BucketName = "cli-tests", ServiceUrl = store, + }), _ => { }); + var entry = await repository.LoadEntryAsync("acme.orders"); + Assert.AreEqual("1.2.0", entry.Current); + Assert.IsNotNull(entry.Manifest!.PublishedOn); + Assert.IsNotNull(entry.Manifest.Source, "it carries its source into storage"); + + // Published again with a bump: the version moves on, the old one stays. + var again = await Cli("publish", zip, "-v", "minor", "-p", "local", "-b", "cli-tests", "-u", store); + Assert.AreEqual(Program.Success, again.Exit, again.Output); + Assert.AreEqual("1.3.0", (await repository.LoadEntryAsync("acme.orders")).Current); + } + + /// + /// What init writes builds and conforms. The templates reference the published SDK and Bitween + /// contract packages; until those are on NuGet the project is pointed at the projects themselves, + /// with Bitween-api beside this repository. + /// + [DataTestMethod] + [DataRow("handler")] + [DataRow("receiver")] + [DataRow("validator")] + public async Task What_init_writes_builds_and_conforms(string kind) + { + var root = RepositoryRoot(); + var contracts = Path.GetFullPath(Path.Combine(root, "..", "Bitween-api", "SW.Bitween.Adapters", "SW.Bitween.Adapters.csproj")); + if (!File.Exists(contracts)) Assert.Inconclusive($"Bitween-api isn't beside this repository ({contracts})"); + + var work = WorkFolder(); + Assert.AreEqual(Program.Success, (await Cli("init", "Acme" + char.ToUpper(kind[0]) + kind[1..], "--kind", kind, "--dir", work)).Exit); + var project = Directory.GetDirectories(work).Single(); + var csproj = Directory.GetFiles(project, "*.csproj").Single(); + var text = File.ReadAllText(csproj) + .Replace($"", + $"") + .Replace($"", + $"") + .Replace("true", ""); + File.WriteAllText(csproj, text); + + var build = await Cli("build", project, "--no-source"); + Assert.AreEqual(Program.Success, build.Exit, build.Output); + + var settings = Path.Combine(work, "settings.json"); + File.WriteAllText(settings, """{ "ApiKey": "k" }"""); + var test = await Cli("test", Path.Combine(project, "bin", "serverless", "package"), "--settings", settings); + Assert.AreEqual(Program.Success, test.Exit, test.Output); + } +} diff --git a/SW.Serverless.Installer.UnitTests/ConformanceTests.cs b/SW.Serverless.Installer.UnitTests/ConformanceTests.cs new file mode 100644 index 0000000..a080e48 --- /dev/null +++ b/SW.Serverless.Installer.UnitTests/ConformanceTests.cs @@ -0,0 +1,169 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Tooling.Conformance; + +namespace SW.Serverless.Installer.UnitTests; + +/// +/// The conformance kit — what serverless test runs — against real adapters built beside the tests: +/// started as a host starts them, described, and called with the Bitween contract's examples. +/// +[TestClass] +public class ConformanceTests +{ + static string BuildOutput(string project) + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null && !File.Exists(Path.Combine(dir.FullName, "SW.Serverless.sln"))) dir = dir.Parent; + var bin = Path.Combine(dir!.FullName, project, "bin"); + return Directory.GetDirectories(bin, "*", SearchOption.AllDirectories) + .Where(d => File.Exists(Path.Combine(d, project + ".dll"))) + .OrderByDescending(d => File.GetLastWriteTimeUtc(Path.Combine(d, project + ".dll"))) + .First(); + } + + /// A package: the project's build output with this manifest at its root. + static string Package(string project, AdapterManifest manifest) + { + var target = Path.Combine(Path.GetTempPath(), "swsl-conformance-tests", Guid.NewGuid().ToString("N")); + var source = BuildOutput(project); + foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories)) + { + var to = Path.Combine(target, Path.GetRelativePath(source, file)); + Directory.CreateDirectory(Path.GetDirectoryName(to)!); + File.Copy(file, to); + } + manifest.Entry ??= project + ".dll"; + File.WriteAllText(Path.Combine(target, AdapterManifest.FileName), manifest.ToJson()); + return target; + } + + static AdapterManifest HandlerManifest(bool listApiKey = true) + { + var manifest = new AdapterManifest + { + Id = "test.bitween.handler", + Kinds = { "handler" }, + Contracts = new() { ["bitween"] = 1 }, + Properties = + { + new AdapterProperty { Name = "Endpoint", Default = "https://partner.example.test/orders", Description = "Where orders go." }, + new AdapterProperty { Name = "Mode", Default = "working" }, + }, + }; + if (listApiKey) manifest.Properties.Add(new AdapterProperty { Name = "ApiKey", Required = true, Secret = true }); + return manifest; + } + + static async Task RunAsync(string packageDirectory, IDictionary settings = null, + bool allowDelete = false) + { + var report = await new ConformanceRunner().RunAsync(new ConformanceOptions + { + PackageDirectory = packageDirectory, + Settings = settings ?? new Dictionary { ["ApiKey"] = "test-key" }, + AllowDelete = allowDelete, + CommandTimeoutSeconds = 30, + }); + Console.WriteLine(string.Join(Environment.NewLine, report.Checks.Select(c => $"{c.Outcome,-7} {c.Name} {c.Detail}"))); + return report; + } + + static ConformanceCheck Check(ConformanceReport report, string name) => + report.Checks.Single(c => c.Name == name); + + [TestMethod] + public async Task A_conforming_handler_passes_every_check() + { + var report = await RunAsync(Package("SW.Serverless.UnitTests.BitweenHandler", HandlerManifest())); + + Assert.IsTrue(report.Passed, string.Join("; ", report.Checks.Where(c => c.Outcome == CheckOutcome.Failed))); + foreach (var name in new[] + { + "manifest", "describe", "settings match the manifest", "starts", + "bitween handler: methods", "bitween handler: Handle answers example 1", "an unknown command is refused", + }) + Assert.AreEqual(CheckOutcome.Passed, Check(report, name).Outcome, name); + } + + [TestMethod] + public async Task A_handler_answering_without_data_fails_the_contract() + { + var report = await RunAsync(Package("SW.Serverless.UnitTests.BitweenHandler", HandlerManifest()), + new Dictionary { ["ApiKey"] = "k", ["Mode"] = "broken" }); + + var check = Check(report, "bitween handler: Handle answers example 1"); + Assert.AreEqual(CheckOutcome.Failed, check.Outcome); + StringAssert.Contains(check.Detail, "isn't a valid ExchangeFile"); + Assert.IsFalse(report.Passed); + } + + [TestMethod] + public async Task Settings_the_manifest_leaves_out_are_named() + { + var report = await RunAsync(Package("SW.Serverless.UnitTests.BitweenHandler", HandlerManifest(listApiKey: false))); + + var check = Check(report, "settings match the manifest"); + Assert.AreEqual(CheckOutcome.Failed, check.Outcome); + StringAssert.Contains(check.Detail, "'ApiKey' is declared by the adapter but missing from the manifest"); + } + + static (string Package, string Folder) Receiver() + { + var folder = Path.Combine(Path.GetTempPath(), "swsl-conformance-tests", "source-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(folder); + File.WriteAllText(Path.Combine(folder, "a.json"), "{\"n\":1}"); + File.WriteAllText(Path.Combine(folder, "b.json"), "{\"n\":2}"); + + var package = Package("SW.Serverless.UnitTests.BitweenReceiver", new AdapterManifest + { + Id = "test.bitween.receiver", + Kinds = { "receiver" }, + Contracts = new() { ["bitween"] = 1 }, + // Run as Bitween runs receivers, classically, over gRPC. + Protocol = new AdapterProtocolRange { Min = 2, Max = 2 }, + Properties = { new AdapterProperty { Name = "Folder", Required = true } }, + }); + return (package, folder); + } + + [TestMethod] + public async Task A_receiver_runs_its_session_in_order_and_leaves_the_source_alone_unless_allowed() + { + var (package, folder) = Receiver(); + var report = await RunAsync(package, new Dictionary { ["Folder"] = folder }); + + Assert.IsTrue(report.Passed, string.Join("; ", report.Checks.Where(c => c.Outcome == CheckOutcome.Failed))); + Assert.AreEqual(CheckOutcome.Passed, Check(report, "bitween receiver: ListFiles").Outcome); + Assert.AreEqual(CheckOutcome.Passed, Check(report, "bitween receiver: GetFile").Outcome); + Assert.AreEqual(CheckOutcome.Skipped, Check(report, "bitween receiver: DeleteFile").Outcome); + Assert.AreEqual(2, Directory.GetFiles(folder).Length, "nothing was deleted"); + } + + [TestMethod] + public async Task A_receiver_s_delete_runs_when_allowed() + { + var (package, folder) = Receiver(); + var report = await RunAsync(package, new Dictionary { ["Folder"] = folder }, allowDelete: true); + + Assert.AreEqual(CheckOutcome.Passed, Check(report, "bitween receiver: DeleteFile").Outcome); + CollectionAssert.AreEqual(new[] { "b.json" }, Directory.GetFiles(folder).Select(Path.GetFileName).ToArray(), + "the first file listed was deleted"); + } + + [TestMethod] + public async Task An_adapter_on_the_old_SDK_is_told_it_can_t_describe_itself() + { + var report = await RunAsync(Package("SW.Serverless.Compat.ClassicV10", new AdapterManifest { Id = "test.old" })); + + var describe = Check(report, "describe"); + Assert.AreEqual(CheckOutcome.Failed, describe.Outcome); + StringAssert.Contains(describe.Detail, "older than 10.1.0"); + Assert.AreEqual(CheckOutcome.Passed, Check(report, "starts").Outcome, "it still runs"); + } +} diff --git a/SW.Serverless.Installer.UnitTests/InstallerLogicTests.cs b/SW.Serverless.Installer.UnitTests/InstallerLogicTests.cs index 470fdc2..382ab96 100644 --- a/SW.Serverless.Installer.UnitTests/InstallerLogicTests.cs +++ b/SW.Serverless.Installer.UnitTests/InstallerLogicTests.cs @@ -5,7 +5,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.VisualStudio.TestTools.UnitTesting; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; namespace SW.Serverless.Installer.UnitTests; diff --git a/SW.Serverless.Installer.UnitTests/ObjectStore.cs b/SW.Serverless.Installer.UnitTests/ObjectStore.cs index 346944f..b7e2336 100644 --- a/SW.Serverless.Installer.UnitTests/ObjectStore.cs +++ b/SW.Serverless.Installer.UnitTests/ObjectStore.cs @@ -28,7 +28,7 @@ record Item(byte[] Content, Dictionary Metadata); public string Register() { var name = "objectstore-" + Guid.NewGuid().ToString("N"); - Shared.CloudFilesFactory.Register(name, _ => this); + SW.Serverless.Tooling.CloudFilesFactory.Register(name, _ => this); return name; } diff --git a/SW.Serverless.Installer.UnitTests/SW.Serverless.Installer.UnitTests.csproj b/SW.Serverless.Installer.UnitTests/SW.Serverless.Installer.UnitTests.csproj index d0ff5e5..74d418e 100644 --- a/SW.Serverless.Installer.UnitTests/SW.Serverless.Installer.UnitTests.csproj +++ b/SW.Serverless.Installer.UnitTests/SW.Serverless.Installer.UnitTests.csproj @@ -22,6 +22,12 @@ + + + serverless init <name> + public class InitCliOptions + { + [Value(0, Required = true, MetaName = "name", HelpText = "The adapter's name, e.g. AcmeOrders: its folder, project and class.")] + public string Name { get; set; } + + [Option("lang", Default = "dotnet", HelpText = "Its language: dotnet. Python, Node and Go arrive with their SDKs.")] + public string Language { get; set; } + + [Option("kind", Default = "handler", HelpText = "handler, mapper, validator or receiver.")] + public string Kind { get; set; } + + [Option("id", HelpText = "Its id in storage; derived from the name when not given (AcmeOrders -> acme.orders).")] + public string Id { get; set; } + + [Option("dir", Default = ".", HelpText = "The folder to make it in.")] + public string Directory { get; set; } + } + + /// serverless build [project] + public class BuildCliOptions + { + [Value(0, MetaName = "project", Default = ".", HelpText = "The adapter's project folder, with its adapter.json.")] + public string Project { get; set; } + + [Option('o', "out", HelpText = "Where the package goes; bin/serverless in the project unless set.")] + public string Output { get; set; } + + [Option("no-source", HelpText = "Leave the source out of the package.")] + public bool NoSource { get; set; } + + [Option("allow", HelpText = "A source file whose secret-scan finding is a false positive. Repeat for more.")] + public IEnumerable Allow { get; set; } + + [Option("dry-run", HelpText = "Show the source that would be carried, and build nothing.")] + public bool DryRun { get; set; } + } + + /// serverless test [package] + public class TestCliOptions + { + [Value(0, MetaName = "package", Default = ".", + HelpText = "A package zip, a package folder, or a project folder (built first).")] + public string Package { get; set; } + + [Option("settings", HelpText = "A JSON file of settings to run it with: { \"Name\": \"value\" }. The adapter calls whatever they point at.")] + public string Settings { get; set; } + + [Option("allow-delete", HelpText = "Let a receiver's DeleteFile run: it removes or moves a real file at the source.")] + public bool AllowDelete { get; set; } + + [Option("contract", HelpText = "A contract file to check against, beyond those the CLI carries. Repeat for more.")] + public IEnumerable Contracts { get; set; } + + [Option("timeout", Default = 60, HelpText = "Seconds one call may take.")] + public int Timeout { get; set; } + } + + /// serverless run [package] --call Command + public class RunCliOptions + { + [Value(0, MetaName = "package", Default = ".", HelpText = "A package zip, a package folder, or a project folder (built first).")] + public string Package { get; set; } + + [Option("settings", HelpText = "A JSON file of settings to run it with.")] + public string Settings { get; set; } + + [Option("call", Required = true, HelpText = "The command to call, e.g. Handle.")] + public string Command { get; set; } + + [Option("input", HelpText = "Its argument: JSON, plain text, or @file to read it from a file.")] + public string Input { get; set; } + + [Option("timeout", Default = 60, HelpText = "Seconds the call may take.")] + public int Timeout { get; set; } + } + + /// serverless manifest validate [path] + public class ManifestCliOptions + { + [Value(0, Required = true, MetaName = "action", HelpText = "validate")] + public string Action { get; set; } + + [Value(1, MetaName = "path", Default = ".", HelpText = "An adapter.json, or the folder holding one.")] + public string Path { get; set; } + } + + /// serverless publish <package> + public class PublishPackageCliOptions : StorageCliOptions + { + [Value(0, Required = true, MetaName = "package", HelpText = "A package zip from serverless build.")] + public string Package { get; set; } + + [Option('v', "version", HelpText = "The version: explicit, or major, minor or patch to bump. The manifest's version unless set.")] + public string Version { get; set; } + + [Option("no-promote", HelpText = "Upload the version without making it the one that runs.")] + public bool NoPromote { get; set; } + + [Option("notes", HelpText = "Release notes for this version (Markdown).")] + public string Notes { get; set; } + + [Option("published-by", HelpText = "Who is publishing, for the catalog.")] + public string PublishedBy { get; set; } + } + + /// The commands that work on adapters in any language, over SW.Serverless.Tooling. + static class AdapterCommands + { + const int Success = Program.Success; + const int Failure = Program.Failure; + + public static Task Init(InitCliOptions opts) + { + var result = Scaffolder.Scaffold(new ScaffoldRequest + { + Name = opts.Name, + Id = opts.Id, + Language = opts.Language, + Kind = opts.Kind, + ParentDirectory = opts.Directory, + }); + if (!result.Succeeded) + { + foreach (var problem in result.Problems) Console.WriteLine(problem); + return Task.FromResult(Failure); + } + + Console.WriteLine($"Made a {opts.Kind} adapter in {result.ProjectDirectory}:"); + foreach (var file in result.Files) Console.WriteLine($" {file}"); + Console.WriteLine("Next: serverless build, then serverless test --settings settings.json"); + return Task.FromResult(Success); + } + + public static async Task Build(BuildCliOptions opts) + { + var result = await PackageBuilder.BuildAsync(new BuildRequest + { + ProjectDirectory = opts.Project, + OutputDirectory = opts.Output, + IncludeSource = !opts.NoSource, + AllowedFiles = new HashSet(opts.Allow ?? Enumerable.Empty(), StringComparer.OrdinalIgnoreCase), + DryRun = opts.DryRun, + Log = Console.WriteLine, + }); + + if (result.SourceFiles.Count > 0) + { + Console.WriteLine($"Source carried ({result.SourceFiles.Count} files, {result.SourceFiles.Sum(f => f.Bytes) / 1024} KB):"); + foreach (var (path, bytes) in result.SourceFiles.OrderBy(f => f.Path, StringComparer.Ordinal)) + Console.WriteLine($" {path} ({bytes} B)"); + } + foreach (var warning in result.Warnings) Console.WriteLine($"Warning: {warning}"); + foreach (var problem in result.Problems) Console.WriteLine(problem); + if (!result.Succeeded) return Failure; + + if (!opts.DryRun) Console.WriteLine($"Built {result.ZipPath}"); + return Success; + } + + public static async Task Test(TestCliOptions opts) + { + var (package, cleanup) = await PackageFolderAsync(opts.Package); + if (package == null) return Failure; + try + { + var report = await new ConformanceRunner().RunAsync(new ConformanceOptions + { + PackageDirectory = package, + Settings = ReadSettings(opts.Settings), + AllowDelete = opts.AllowDelete, + Contracts = (opts.Contracts ?? Enumerable.Empty()).Select(ContractDocument.FromFile).ToList(), + CommandTimeoutSeconds = opts.Timeout, + Log = Console.WriteLine, + }); + + foreach (var check in report.Checks) + Console.WriteLine($"{Mark(check.Outcome)} {check.Name}{(string.IsNullOrEmpty(check.Detail) ? "" : " — " + check.Detail)}"); + var failed = report.Checks.Count(c => c.Outcome == CheckOutcome.Failed); + Console.WriteLine(failed == 0 ? "Conforms." : $"{failed} check{(failed == 1 ? "" : "s")} failed."); + return report.Passed ? Success : Failure; + } + finally + { + cleanup(); + } + } + + public static async Task Run(RunCliOptions opts) + { + var (package, cleanup) = await PackageFolderAsync(opts.Package); + if (package == null) return Failure; + try + { + await using var host = await LocalAdapterHost.StartAsync(package, ReadSettings(opts.Settings), commandTimeoutSeconds: opts.Timeout); + Console.WriteLine(await host.CallAsync(opts.Command, ReadInput(opts.Input))); + return Success; + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + Console.WriteLine(ex.GetBaseException().Message); + return Failure; + } + finally + { + cleanup(); + } + } + + public static Task Manifest(ManifestCliOptions opts) + { + if (opts.Action != "validate") + { + Console.WriteLine($"'{opts.Action}' isn't a manifest action: use validate"); + return Task.FromResult(Failure); + } + + var path = Directory.Exists(opts.Path) ? System.IO.Path.Combine(opts.Path, AdapterManifest.FileName) : opts.Path; + if (!File.Exists(path)) + { + Console.WriteLine($"There is no {path}"); + return Task.FromResult(Failure); + } + + try + { + var problems = AdapterManifest.Parse(File.ReadAllText(path)).Validate(); + foreach (var problem in problems) Console.WriteLine(problem); + Console.WriteLine(problems.Count == 0 ? $"{path} is valid." : $"{problems.Count} problem{(problems.Count == 1 ? "" : "s")} in {path}."); + return Task.FromResult(problems.Count == 0 ? Success : Failure); + } + catch (JsonException ex) + { + Console.WriteLine($"{path} isn't valid JSON: {ex.Message}"); + return Task.FromResult(Failure); + } + } + + public static Task Publish(PublishPackageCliOptions opts, Func environment) => + Program.WithStorage(opts, environment, async files => + { + var result = await PackagePublisher.PublishPackageAsync(files, new PublishPackageRequest + { + PackagePath = opts.Package, + Version = opts.Version, + Promote = !opts.NoPromote, + ReleaseNotes = opts.Notes, + PublishedBy = UploadOptionsResolver.ResolvePublishedBy(opts.PublishedBy, environment), + }, Console.WriteLine); + Console.WriteLine($"Published {result.Manifest.Id} {result.Version}."); + }); + + static string Mark(CheckOutcome outcome) => outcome switch + { + CheckOutcome.Passed => "PASS", + CheckOutcome.Failed => "FAIL", + _ => "SKIP", + }; + + /// A package folder for whatever was given: a folder with adapter.json, a zip, or a project to build. + static async Task<(string Folder, Action Cleanup)> PackageFolderAsync(string given) + { + var path = System.IO.Path.GetFullPath(given); + if (File.Exists(path) && path.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + { + var folder = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "swsl-cli", Guid.NewGuid().ToString("N")); + ZipFile.ExtractToDirectory(path, folder); + return (folder, () => TryDelete(folder)); + } + + if (Directory.Exists(path) && Directory.GetFiles(path, "*.*proj").Length > 0) + { + var output = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "swsl-cli", Guid.NewGuid().ToString("N")); + var built = await PackageBuilder.BuildAsync(new BuildRequest { ProjectDirectory = path, OutputDirectory = output, Log = Console.WriteLine }); + foreach (var problem in built.Problems) Console.WriteLine(problem); + return built.Succeeded ? (built.PackageDirectory, () => TryDelete(output)) : (null, () => TryDelete(output)); + } + + if (Directory.Exists(path) && File.Exists(System.IO.Path.Combine(path, AdapterManifest.FileName))) + return (path, () => { }); + + Console.WriteLine($"{given} is neither a package zip, a package folder nor a project folder"); + return (null, () => { }); + } + + static IDictionary ReadSettings(string path) + { + if (string.IsNullOrWhiteSpace(path)) return new Dictionary(); + using var document = JsonDocument.Parse(File.ReadAllText(path)); + return document.RootElement.EnumerateObject().ToDictionary(p => p.Name, + p => p.Value.ValueKind == JsonValueKind.String ? p.Value.GetString() : p.Value.GetRawText()); + } + + static object ReadInput(string input) + { + if (input == null) return null; + if (input.StartsWith('@')) input = File.ReadAllText(input[1..]); + // JSON goes as JSON; anything else as the plain text a string argument is. + try { return Newtonsoft.Json.Linq.JToken.Parse(input); } + catch (Newtonsoft.Json.JsonReaderException) { return input; } + } + + static void TryDelete(string folder) + { + try { Directory.Delete(folder, true); } catch { } + } + } +} diff --git a/SW.Serverless.Installer/Options.cs b/SW.Serverless.Installer/Options.cs index 02e99dd..4f533de 100644 --- a/SW.Serverless.Installer/Options.cs +++ b/SW.Serverless.Installer/Options.cs @@ -103,48 +103,4 @@ public class WithdrawCliOptions : StorageCliOptions public string Version { get; set; } } - public class ServerlessUploadOptions - { - public string Provider { get; set; } - - public string AccessKeyId { get; set; } - - public string SecretAccessKey { get; set; } - - public string BucketName { get; set; } - - public string ServiceUrl { get; set; } - - - public string RSAKey { get; set; } - - public string UserId { get; set; } - - public string FingerPrint { get; set; } - - public string TenantId { get; set; } - - public string Region { get; set; } - - - public string Version { get; set; } - - - public string AdapterId { get; set; } - public string NamespaceName { get; set; } - - // ——— Google Cloud service account ——— - public string ProjectId { get; set; } - public string PrivateKeyId { get; set; } - public string PrivateKey { get; set; } - public string ClientEmail { get; set; } - public string ClientId { get; set; } - public string ClientX509CertUrl { get; set; } - - /// - /// Roles this adapter serves, comma separated. Overrides what the assembly declared, for - /// an adapter whose author has not added [AdapterKind] to it. - /// - public string Kind { get; set; } - } } diff --git a/SW.Serverless.Installer/Program.cs b/SW.Serverless.Installer/Program.cs index 95d4f48..cb31f36 100644 --- a/SW.Serverless.Installer/Program.cs +++ b/SW.Serverless.Installer/Program.cs @@ -1,7 +1,7 @@ using CommandLine; using SW.PrimitiveTypes; using SW.Serverless.Contract.Catalog; -using SW.Serverless.Installer.Shared; +using SW.Serverless.Tooling; using System; using System.Collections.Generic; using System.IO; @@ -16,7 +16,12 @@ public static class Program public const int Failure = 1; /// The commands besides publishing. Matched on the first argument only. - static readonly HashSet Commands = new(StringComparer.Ordinal) { "promote", "versions", "withdraw" }; + static readonly HashSet Commands = new(StringComparer.Ordinal) + { + "promote", "versions", "withdraw", + // For adapters in any language, over SW.Serverless.Tooling. + "init", "build", "test", "run", "manifest", "publish", + }; private static Task Main(string[] args) => RunAsync(args); @@ -47,6 +52,13 @@ public static async Task RunAsync(string[] args, Func envir o => Promote(o, environment)), "versions" => await Run(parser.ParseArguments(rest), o => Versions(o, environment)), + "init" => await Run(parser.ParseArguments(rest), AdapterCommands.Init), + "build" => await Run(parser.ParseArguments(rest), AdapterCommands.Build), + "test" => await Run(parser.ParseArguments(rest), AdapterCommands.Test), + "run" => await Run(parser.ParseArguments(rest), AdapterCommands.Run), + "manifest" => await Run(parser.ParseArguments(rest), AdapterCommands.Manifest), + "publish" => await Run(parser.ParseArguments(rest), + o => AdapterCommands.Publish(o, environment)), _ => await Run(parser.ParseArguments(rest), o => Withdraw(o, environment)), }; @@ -221,6 +233,33 @@ static async Task WithRepository(StorageCliOptions opts, string adapterId, } } + /// Runs against the storage the options name, reporting a failure as a message. + internal static async Task WithStorage(StorageCliOptions opts, Func environment, + Func action) + { + try + { + await action(CloudFilesFactory.Create(await GetServerlessUploadOptions(opts, environment))); + return Success; + } + catch (SWException ex) + { + Console.WriteLine(ex.Message); + return Failure; + } + catch (ArgumentException ex) + { + // Semver's verdicts: an existing or lower version, or an unknown bump. + Console.WriteLine(ex.Message); + return Failure; + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + return Failure; + } + } + public static IEnumerable FormatVersions(string adapterId, VersionListing listing) { if (listing.Versions.Count == 0) diff --git a/SW.Serverless.Installer/SW.Serverless.Installer.csproj b/SW.Serverless.Installer/SW.Serverless.Installer.csproj index 5ba43b1..3f95b76 100644 --- a/SW.Serverless.Installer/SW.Serverless.Installer.csproj +++ b/SW.Serverless.Installer/SW.Serverless.Installer.csproj @@ -9,16 +9,6 @@ - - - - - - - - - - @@ -26,6 +16,8 @@ + + diff --git a/SW.Serverless.Sdk/AdapterContractAttribute.cs b/SW.Serverless.Sdk/AdapterContractAttribute.cs new file mode 100644 index 0000000..c5b729b --- /dev/null +++ b/SW.Serverless.Sdk/AdapterContractAttribute.cs @@ -0,0 +1,21 @@ +using System; + +namespace SW.Serverless.Sdk +{ + /// + /// A contract this adapter implements, and its version — for Bitween, "bitween" 1. Reported + /// in the adapter's description and handshake, and checked by serverless test. + /// + [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] + public class AdapterContractAttribute : Attribute + { + public AdapterContractAttribute(string name, int version) + { + Name = name; + Version = version; + } + + public string Name { get; } + public int Version { get; } + } +} diff --git a/SW.Serverless.Sdk/Constants.cs b/SW.Serverless.Sdk/Constants.cs index 2a51eb5..8994cdc 100644 --- a/SW.Serverless.Sdk/Constants.cs +++ b/SW.Serverless.Sdk/Constants.cs @@ -14,5 +14,16 @@ public static class Constants public const string LogErrorIdentifier = "{{log.error}}"; public const string CorrelationIdName = "CorrelationId"; + /// + /// Passed instead of the three base64 arguments when the host sends them on stdin: the + /// options, the startup values and the adapter values, one line each, before the first + /// command. Startup values hold passwords and keys, and anything on the command line can + /// be read by every process on the machine. + /// + public const string ValuesOnStdinFlag = "--values-on-stdin"; + + /// The first SDK release that reads . + public const string ValuesOnStdinSince = "10.1.0"; + } } diff --git a/SW.Serverless.Sdk/Describer.cs b/SW.Serverless.Sdk/Describer.cs new file mode 100644 index 0000000..8cf889e --- /dev/null +++ b/SW.Serverless.Sdk/Describer.cs @@ -0,0 +1,130 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Schema; +using System.Threading.Tasks; +using SW.Serverless.Contract.Catalog; + +namespace SW.Serverless.Sdk +{ + /// + /// Answers --describe: the adapter's settings, commands, kinds and contracts as an + /// on stdout, without connecting to any host. + /// + internal static class Describer + { + public static bool Requested(string[] args) => + args.Length > 1 && args[1] == AdapterSelfDescription.Flag; + + // Payloads travel as Newtonsoft writes them — property names as declared — so the schema + // names them the same way. + static readonly JsonSerializerOptions SchemaOptions = new(JsonSerializerDefaults.General) + { + TypeInfoResolver = new System.Text.Json.Serialization.Metadata.DefaultJsonTypeInfoResolver(), + }; + + /// + /// Builds the handler, which is what runs its Runner.Expect declarations. Without its + /// settings a handler may refuse to be built; the description is then completed without + /// them and says why. + /// + public static AdapterSelfDescription Describe(Type handlerType, Func buildHandler, string lifecycle, + Func> commandsOf) + { + var description = new AdapterSelfDescription + { + SdkLanguage = "dotnet", + SdkVersion = SdkInfo.Version, + Lifecycle = lifecycle, + Protocol = lifecycle == AdapterManifest.ResidentLifecycle + ? new AdapterProtocolRange { Min = 2, Max = 2 } + : new AdapterProtocolRange { Min = 1, Max = 1 }, + }; + + object handler = null; + try + { + handler = buildHandler?.Invoke(); + } + catch (Exception ex) + { + description.Warnings.Add( + $"The handler could not be built without its settings, so settings it declares when built may be missing: {ex.GetBaseException().Message}"); + } + handlerType ??= handler?.GetType(); + var commands = commandsOf(handler).ToList(); + + foreach (var (name, value) in Runner.DeclaredStartupValues) + description.Settings.Add(new DescribedSetting + { + Name = name, + Description = value.Description, + Type = string.IsNullOrEmpty(value.Type) ? AdapterProperty.TextType : value.Type, + Required = !value.Optional, + Secret = value.Private, + Default = value.Private ? null : value.Default, + }); + + foreach (var (name, parameter, result, text) in commands.OrderBy(c => c.Name, StringComparer.Ordinal)) + description.Commands.Add(new DescribedCommand + { + Name = name, + Description = string.IsNullOrEmpty(text) ? null : text, + InputSchema = SchemaOf(parameter), + OutputSchema = SchemaOf(result), + ReturnsValue = result != null, + }); + + if (handlerType != null) + { + description.Kinds = handlerType.GetCustomAttributes(true) + .Select(k => k.Kind).Distinct().ToList(); + description.Contracts = handlerType.GetCustomAttributes(true) + .GroupBy(c => c.Name).ToDictionary(g => g.Key, g => g.Max(c => c.Version)); + } + + return description; + } + + /// The command's result type: what its Task carries, or null for a bare Task. + public static Type ResultOf(MethodInfo method) + { + var type = method.ReturnType; + if (type == typeof(Task) || type == typeof(void)) return null; + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Task<>)) + { + var result = type.GetGenericArguments()[0]; + // Task says nothing about its shape; describe it as anything. + return result; + } + return type; + } + + static JsonElement? SchemaOf(Type type) + { + if (type == null) return null; + try + { + JsonNode schema = type == typeof(object) + ? new JsonObject() + : SchemaOptions.GetJsonSchemaAsNode(type); + return JsonSerializer.SerializeToElement(schema); + } + catch + { + // Describing an argument must never stop the description. + return JsonSerializer.SerializeToElement(new JsonObject()); + } + } + + public static Task Print(AdapterSelfDescription description) + { + Console.Out.WriteLine(description.ToJson()); + Console.Out.Flush(); + return Task.FromResult(0); + } + } +} diff --git a/SW.Serverless.Sdk/Resident/ResidentRunner.cs b/SW.Serverless.Sdk/Resident/ResidentRunner.cs index 5ff31dd..8e4133f 100644 --- a/SW.Serverless.Sdk/Resident/ResidentRunner.cs +++ b/SW.Serverless.Sdk/Resident/ResidentRunner.cs @@ -110,6 +110,18 @@ public static Task RunAsync(object handler) => /// public static async Task RunAsync(Type handlerType, Func handlerFactory) { + // --describe: say what this adapter is, without a host, and exit. + if (Describer.Requested(Environment.GetCommandLineArgs())) + { + // Commands come from the type, as they do for Hello, built or not. + await Describer.Print(Describer.Describe(handlerType, () => handlerFactory(null), + Contract.Catalog.AdapterManifest.ResidentLifecycle, + _ => BuildCommands(handlerType).Select(c => (c.Key, c.Value.ParameterType, + c.Value.Void ? null : Describer.ResultOf(c.Value.MethodInfo), + c.Value.MethodInfo.GetCustomAttribute()?.Description)))); + return; + } + var runner = new ResidentRunner(handlerType, handlerFactory); try { @@ -160,7 +172,28 @@ await call.RequestStream.WriteAsync(new AdapterFrame AdapterId = handshake.AdapterId ?? "", InstanceKey = handshake.InstanceKey ?? "", ProtocolVersion = protocol, - SdkVersion = typeof(ResidentRunner).Assembly.GetName().Version?.ToString() ?? "0.0.0", + SdkVersion = SdkInfo.Version, + SdkLanguage = "dotnet", + Kinds = { handlerType.GetCustomAttributes(true).Select(k => k.Kind).Distinct() }, + Contracts = + { + handlerType.GetCustomAttributes(true) + .GroupBy(c => c.Name).ToDictionary(g => g.Key, g => g.Max(c => c.Version)) + }, + // Declared with Runner.Expect by the time the handshake goes: so in a handler + // that was constructed before it was handed to the runner, which is the usual way. + Settings = + { + Runner.DeclaredStartupValues.Select(kv => new SettingInfo + { + Name = kv.Key, + Description = kv.Value.Description ?? "", + Required = !kv.Value.Optional, + Secret = kv.Value.Private, + DefaultValue = kv.Value.Default ?? "", + Type = kv.Value.Type ?? "text", + }) + }, Capabilities = { Capabilities() }, Commands = { CommandInfos() } } diff --git a/SW.Serverless.Sdk/Runner.cs b/SW.Serverless.Sdk/Runner.cs index 19f413b..d97d711 100644 --- a/SW.Serverless.Sdk/Runner.cs +++ b/SW.Serverless.Sdk/Runner.cs @@ -73,6 +73,17 @@ public static Task RunResident(object commandHandler) => async public static Task Run(object commandHandler) { + // --describe: say what this adapter is, without a host, and exit. + if (Describer.Requested(Environment.GetCommandLineArgs())) + { + await Describer.Print(Describer.Describe( + commandHandler is Func ? null : commandHandler.GetType(), + () => commandHandler is Func factory ? factory() : commandHandler, + Contract.Catalog.AdapterManifest.ClassicLifecycle, + CommandsOf)); + return; + } + Timer idleTimer = null; try @@ -81,6 +92,18 @@ async public static Task Run(object commandHandler) Console.OutputEncoding = Encoding.UTF8; var commandLineArgs = Environment.GetCommandLineArgs(); + // A host that knows this SDK sends the values on stdin, off the command line where + // any process on the machine could read them. Older hosts still pass them as + // arguments, which is still understood. + if (commandLineArgs.Length > 1 && commandLineArgs[1] == Constants.ValuesOnStdinFlag) + commandLineArgs = new[] + { + commandLineArgs[0], + await Console.In.ReadLineAsync(), + await Console.In.ReadLineAsync(), + await Console.In.ReadLineAsync(), + }; + try { ServerlessOptions = @@ -245,6 +268,15 @@ static bool IsPrimitive(this Type type) return type.IsPrimitive; } + /// The classic commands of a built handler, for --describe. + static IEnumerable<(string, Type, Type, string)> CommandsOf(object built) + { + if (built == null) yield break; + foreach (var (name, info) in BuildMethodsDictionary(built)) + yield return (name, info.ParameterType, info.Void ? null : Describer.ResultOf(info.MethodInfo), + info.MethodInfo.GetCustomAttribute()?.Description); + } + static Dictionary BuildMethodsDictionary(object commandHandler) { var methodsDictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -289,6 +321,10 @@ static Dictionary BuildMethodsDictionary(object comma return methodsDictionary; } + /// What has been declared with so far. + internal static IReadOnlyDictionary DeclaredStartupValues => + new Dictionary(expectedStartupValues); + public static void Expect(string name, bool optional = false, bool isPrivate = false, string description = null) { Expect(name, null, optional, isPrivate, description); diff --git a/SW.Serverless.Sdk/SdkInfo.cs b/SW.Serverless.Sdk/SdkInfo.cs new file mode 100644 index 0000000..a3e43d4 --- /dev/null +++ b/SW.Serverless.Sdk/SdkInfo.cs @@ -0,0 +1,27 @@ +using System; +using System.Reflection; + +namespace SW.Serverless.Sdk +{ + /// The version of this SDK, as an adapter reports it in its description and handshake. + public static class SdkInfo + { + /// + /// The release this code is, kept by hand as the host's HostInfo.Baseline is: the publish + /// workflow stamps the version on the package, and the assembly itself says 1.0.0.0. + /// + public const string Baseline = "10.1.0"; + + public static string Version { get; } = Resolve(); + + static string Resolve() + { + var informational = typeof(SdkInfo).Assembly + .GetCustomAttribute()?.InformationalVersion; + var core = informational?.Split('+', '-')[0]; + return System.Version.TryParse(core, out var stamped) && stamped > System.Version.Parse(Baseline) + ? stamped.ToString(3) + : Baseline; + } + } +} diff --git a/SW.Serverless.Installer/Shared/AdapterDescriber.cs b/SW.Serverless.Tooling/AdapterDescriber.cs similarity index 99% rename from SW.Serverless.Installer/Shared/AdapterDescriber.cs rename to SW.Serverless.Tooling/AdapterDescriber.cs index 4cb621d..8c5394e 100644 --- a/SW.Serverless.Installer/Shared/AdapterDescriber.cs +++ b/SW.Serverless.Tooling/AdapterDescriber.cs @@ -4,7 +4,7 @@ using System.Linq; using System.Reflection; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { /// /// What an adapter IS, read from the assembly at publish time. diff --git a/SW.Serverless.Installer/Shared/AdapterRepository.cs b/SW.Serverless.Tooling/AdapterRepository.cs similarity index 91% rename from SW.Serverless.Installer/Shared/AdapterRepository.cs rename to SW.Serverless.Tooling/AdapterRepository.cs index c433ac0..c4cef5d 100644 --- a/SW.Serverless.Installer/Shared/AdapterRepository.cs +++ b/SW.Serverless.Tooling/AdapterRepository.cs @@ -8,7 +8,7 @@ using System.Linq; using System.Threading.Tasks; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { /// What goes into storage for one package, beside the zip itself. public class PackageInfo @@ -212,13 +212,24 @@ public async Task PublishVersionAsync(string adapterId, string version, string z if ((await LocateVersionsAsync(adapterId)).ContainsKey(version)) throw new SWException($"Version {version} of '{adapterId}' has already been published."); + // An id keeps its runtime: hosts that predate manifests run adapters/{id} with dotnet, + // and would go on running the old .NET package under an id that had moved on. + var runsOnDotnet = RunsOnDotnet(package.Manifest); + if (!runsOnDotnet && await ExistsAsync(CurrentKey(adapterId))) + throw new SWException( + $"'{adapterId}' is a .NET adapter, and older hosts would keep running that under its id. " + + $"Publish the {package.Manifest!.Runtime} adapter under a new id."); + // Loaded before uploading, so the new version is not mistaken for one the catalog missed. var entry = await LoadEntryAsync(adapterId); var sha256 = InstallerLogic.Sha256Of(zipPath); var metadata = LegacyMetadata(package.EntryAssembly, package.Lifecycle, package.Kind, sha256, version); await UploadAsync(versionKey, zipPath, metadata); - if (promote) await UploadAsync(CurrentKey(adapterId), zipPath, metadata); + // adapters/{id} is what hosts before manifests list and run, always with dotnet. An + // adapter in another runtime never goes there; newer hosts find its current version in + // the catalog. + if (promote && runsOnDotnet) await UploadAsync(CurrentKey(adapterId), zipPath, metadata); entry.Versions.Add(new AdapterVersionRecord { @@ -245,6 +256,12 @@ public async Task PublishVersionAsync(string adapterId, string version, string z /// public async Task PublishUnversionedAsync(string adapterId, string zipPath, PackageInfo package) { + // An unversioned package lives only at adapters/{id}, where older hosts would start it + // with dotnet whatever it is written in. + if (!RunsOnDotnet(package.Manifest)) + throw new SWException( + $"A {package.Manifest!.Runtime} adapter must be published with a version (-v), so older hosts never see it."); + var entry = await LoadEntryAsync(adapterId); var sha256 = InstallerLogic.Sha256Of(zipPath); @@ -255,6 +272,11 @@ await UploadAsync(CurrentKey(adapterId), zipPath, await catalog.SaveAsync(entry); } + /// A package with no manifest, or no runtime in it, is .NET — as every package before manifests was. + static bool RunsOnDotnet(AdapterManifest manifest) => + string.IsNullOrWhiteSpace(manifest?.Runtime) || + string.Equals(manifest.Runtime, AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase); + static void MakeCurrent(AdapterCatalogEntry entry, string version, AdapterManifest manifest, string sha256, string iconDataUri) { @@ -306,6 +328,19 @@ public async Task PromoteAsync(string adapterId, string version, string workDire $"({Short(expected)} recorded, {Short(sha256)} in storage). Nothing was changed."); var manifest = ReadManifest(zipPath) ?? record.Manifest; + + // Another runtime is never copied to adapters/{id}, where hosts before manifests + // would start it with dotnet: being current is the catalog's alone. + if (!RunsOnDotnet(manifest)) + { + record.Sha256 ??= sha256; + record.Manifest ??= manifest; + MakeCurrent(entry, version, manifest, sha256, IconFromPackage(zipPath, manifest)); + await catalog.SaveAsync(entry); + log($"Version {version} of '{adapterId}' is now current."); + return; + } + var entryAssembly = Value(versionMetadata, "EntryAssembly") ?? manifest?.Entry; if (string.IsNullOrWhiteSpace(entryAssembly)) throw new SWException( diff --git a/SW.Serverless.Tooling/Building/IgnoreRules.cs b/SW.Serverless.Tooling/Building/IgnoreRules.cs new file mode 100644 index 0000000..cf64dde --- /dev/null +++ b/SW.Serverless.Tooling/Building/IgnoreRules.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; + +namespace SW.Serverless.Tooling.Building +{ + /// + /// Which files of a project stay out of the source a package carries: a built-in list — build + /// output, dependencies, editor folders and the kinds of file that hold secrets — then the + /// project's .gitignore and .serverlessignore, read with gitignore's rules (*, **, a trailing / + /// for folders, a leading / to anchor, ! to bring a file back). + /// + public class IgnoreRules + { + /// Always left out, whatever the project's ignore files say. + public static readonly IReadOnlyList BuiltIn = new[] + { + // Build output and downloaded dependencies. + "bin/", "obj/", "dist/", "build/", "out/", "target/", "__pycache__/", "*.pyc", ".venv/", "venv/", + "node_modules/", ".pytest_cache/", ".mypy_cache/", "packages/", "*.nupkg", + // Version control and editors. + ".git/", ".vs/", ".idea/", ".vscode/", "*.user", "*.suo", ".DS_Store", + // What holds secrets. + ".env", ".env.*", "*.pem", "*.key", "*.pfx", "*.p12", "*.jks", "id_rsa*", "id_ed25519*", "id_ecdsa*", + "secrets.json", "*.publishsettings", "*.pubxml", "*.pubxml.user", "appsettings.*.json", + }; + + readonly List<(Regex Pattern, bool Negated, bool DirectoryOnly)> rules = new(); + + public IgnoreRules(IEnumerable lines) + { + foreach (var raw in lines) + { + var line = raw.TrimEnd(); + if (line.Length == 0 || line.StartsWith('#')) continue; + + var negated = line.StartsWith('!'); + if (negated) line = line[1..]; + var directoryOnly = line.EndsWith('/'); + line = line.TrimEnd('/'); + var anchored = line.StartsWith('/') || line.Contains('/'); + line = line.TrimStart('/'); + + rules.Add((new Regex("^" + (anchored ? "" : "(.*/)?") + Translate(line) + (directoryOnly ? "(/.*)?$" : "(/.*)?$"), + RegexOptions.CultureInvariant), negated, directoryOnly)); + } + } + + /// The built-in list plus the project's own .gitignore and .serverlessignore. + public static IgnoreRules For(string projectDirectory) + { + var lines = new List(BuiltIn); + foreach (var file in new[] { ".gitignore", ".serverlessignore" }) + { + var path = Path.Combine(projectDirectory, file); + if (File.Exists(path)) lines.AddRange(File.ReadAllLines(path)); + } + return new IgnoreRules(lines); + } + + /// Whether a path, relative to the project and with forward slashes, is left out. + public bool Ignores(string relativePath) + { + var path = relativePath.Replace('\\', '/').TrimStart('/'); + var ignored = false; + foreach (var (pattern, negated, _) in rules) + if (pattern.IsMatch(path)) ignored = !negated; + return ignored; + } + + static string Translate(string glob) + { + var result = new System.Text.StringBuilder(); + for (var i = 0; i < glob.Length; i++) + { + var c = glob[i]; + if (c == '*' && i + 1 < glob.Length && glob[i + 1] == '*') + { + // ** crosses folders; "**/" may also match nothing. + if (i + 2 < glob.Length && glob[i + 2] == '/') { result.Append("(.*/)?"); i += 2; } + else { result.Append(".*"); i++; } + } + else if (c == '*') result.Append("[^/]*"); + else if (c == '?') result.Append("[^/]"); + else result.Append(Regex.Escape(c.ToString())); + } + return result.ToString(); + } + } +} diff --git a/SW.Serverless.Tooling/Building/PackageBuilder.cs b/SW.Serverless.Tooling/Building/PackageBuilder.cs new file mode 100644 index 0000000..01c7ce5 --- /dev/null +++ b/SW.Serverless.Tooling/Building/PackageBuilder.cs @@ -0,0 +1,322 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Threading.Tasks; +using System.Xml.Linq; +using SW.Serverless.Contract.Catalog; + +namespace SW.Serverless.Tooling.Building +{ + public class BuildRequest + { + /// The adapter's project folder, with its adapter.json. + public string ProjectDirectory { get; set; } + + /// Where the package folder and zip go; bin/serverless in the project unless set, which the source rules leave out. + public string OutputDirectory { get; set; } + + /// Carry the source in the package. On unless a partner won't hand it over. + public bool IncludeSource { get; set; } = true; + + /// Source files whose secret-scan findings are false positives, relative to the project. + public ISet AllowedFiles { get; set; } = new HashSet(StringComparer.OrdinalIgnoreCase); + + /// Report what would be carried without building anything. + public bool DryRun { get; set; } + + public Runtimes.AdapterRuntimeOptions Runtimes { get; set; } = new(); + public Action Log { get; set; } = _ => { }; + } + + public class BuildResult + { + public bool Succeeded => Problems.Count == 0; + public List Problems { get; } = new(); + public List Warnings { get; } = new(); + public AdapterManifest Manifest { get; set; } + public string PackageDirectory { get; set; } + public string ZipPath { get; set; } + + /// The source carried, relative to its root, with sizes. + public List<(string Path, long Bytes)> SourceFiles { get; } = new(); + public List Findings { get; } = new(); + } + + /// + /// What serverless build does, for an adapter in any language its SDK supports: builds it, + /// asks it to describe itself, writes its manifest from that and the author's adapter.json, + /// carries its source under the source rules, and zips the result into a package any host runs. + /// + public static class PackageBuilder + { + /// The source folder size past which the build warns: usually build output slipped in. + public const long SourceWarningBytes = 10 * 1024 * 1024; + + static readonly string[] Lockfiles = + { "packages.lock.json", "poetry.lock", "requirements.lock", "requirements.txt", "uv.lock", "Pipfile.lock", + "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "go.sum" }; + + public static async Task BuildAsync(BuildRequest request) + { + var result = new BuildResult(); + var project = Path.GetFullPath(request.ProjectDirectory); + + var authorPath = Path.Combine(project, AdapterManifest.FileName); + if (!File.Exists(authorPath)) + { + result.Problems.Add($"there is no {AdapterManifest.FileName} in {project}; serverless init writes one"); + return result; + } + var authorJson = await File.ReadAllTextAsync(authorPath); + var author = AdapterManifest.Parse(authorJson); + if (string.IsNullOrWhiteSpace(author.Id)) + { + result.Problems.Add($"{AdapterManifest.FileName} needs an id"); + return result; + } + + var runtime = string.IsNullOrWhiteSpace(author.Runtime) ? AdapterManifest.DotnetRuntime : author.Runtime; + if (!string.Equals(runtime, AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase)) + { + result.Problems.Add($"building a '{runtime}' adapter arrives with that language's SDK; this build does .NET"); + return result; + } + + var projectFiles = Directory.GetFiles(project, "*.*proj").Where(f => f.EndsWith(".csproj") || f.EndsWith(".fsproj") || f.EndsWith(".vbproj")).ToList(); + if (projectFiles.Count != 1) + { + result.Problems.Add(projectFiles.Count == 0 + ? $"there is no .NET project file in {project}" + : $"there are {projectFiles.Count} project files in {project}; keep one adapter per folder"); + return result; + } + var projectFile = projectFiles[0]; + + // Source first: a secret found here stops the build before anything is produced. + var source = request.IncludeSource ? CollectSource(project, projectFile, request, result) : null; + if (!result.Succeeded || request.DryRun) return result; + + var output = Path.GetFullPath(request.OutputDirectory ?? Path.Combine(project, "bin", "serverless")); + var packageDirectory = Path.Combine(output, "package"); + if (Directory.Exists(packageDirectory)) Directory.Delete(packageDirectory, true); + Directory.CreateDirectory(packageDirectory); + + request.Log("Building..."); + if (!new InstallerLogic().BuildPublish(projectFile, packageDirectory)) + { + result.Problems.Add("dotnet publish failed; its output above says why"); + return result; + } + + var entry = InstallerLogic.ResolveEntryAssembly(packageDirectory, projectFile); + if (entry == null) + { + result.Problems.Add("the built output has no entry assembly"); + return result; + } + + request.Log("Asking the adapter to describe itself..."); + var (description, problem) = await LocalAdapterHost.DescribeAsync(Path.Combine(packageDirectory, entry), + AdapterManifest.DotnetRuntime, request.Runtimes); + if (description == null) + { + result.Problems.Add($"{problem}. serverless build needs SimplyWorks.Serverless.Sdk 10.1.0 or later; " + + "an adapter on an older SDK is published with serverless as before"); + return result; + } + result.Warnings.AddRange(description.Warnings); + + var manifest = ManifestFrom(author, AuthorChoseClassic(authorJson), description, entry, ManifestBuilder.LanguageOf(projectFile)); + + if (source != null) + { + var sourceDirectory = Path.Combine(packageDirectory, AdapterSource.DefaultPath); + manifest.Source = new AdapterSource + { + BuildCommand = "dotnet publish -c Release", + Lockfiles = source.Keys.Where(k => Lockfiles.Contains(Path.GetFileName(k), StringComparer.OrdinalIgnoreCase)).OrderBy(k => k).ToList(), + }; + foreach (var (relative, absolute) in source.OrderBy(s => s.Key, StringComparer.Ordinal)) + { + var target = Path.Combine(sourceDirectory, relative); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(absolute, target); + manifest.Source.Files[relative] = Convert.ToHexString(SHA256.HashData(await File.ReadAllBytesAsync(absolute))).ToLowerInvariant(); + } + if (manifest.Source.Lockfiles.Count == 0) manifest.Source.Lockfiles = null; + } + + var problems = manifest.Validate(); + if (problems.Count > 0) + { + result.Problems.AddRange(problems); + return result; + } + + await File.WriteAllTextAsync(Path.Combine(packageDirectory, AdapterManifest.FileName), manifest.ToJson()); + + var zip = Path.Combine(output, $"{manifest.Id}{(string.IsNullOrWhiteSpace(manifest.Version) ? "" : "-" + manifest.Version)}.zip"); + if (File.Exists(zip)) File.Delete(zip); + ZipFile.CreateFromDirectory(packageDirectory, zip, CompressionLevel.Optimal, includeBaseDirectory: false); + + result.Manifest = manifest; + result.PackageDirectory = packageDirectory; + result.ZipPath = zip; + return result; + } + + /// + /// The manifest: the author's file — id, names, marketplace text, presentation of settings — + /// with what the adapter says about itself, which wins wherever the two speak to the same thing. + /// + internal static AdapterManifest ManifestFrom(AdapterManifest author, bool authorChoseClassic, + AdapterSelfDescription description, string entry, string language) + { + var manifest = AdapterManifest.Parse(author.ToJson()); + manifest.Entry = entry; + manifest.Runtime = AdapterManifest.DotnetRuntime; + manifest.Language ??= language; + manifest.SdkVersion = description.SdkVersion; + + if (description.Lifecycle == AdapterManifest.ResidentLifecycle) + { + // It speaks gRPC. Run classically only when the author says so — the opt-in for a + // .NET adapter on the gRPC protocol — and as a resident otherwise. + manifest.Lifecycle = authorChoseClassic ? AdapterManifest.ClassicLifecycle : AdapterManifest.ResidentLifecycle; + manifest.Protocol = new AdapterProtocolRange { Min = 2, Max = 2 }; + } + else + { + manifest.Lifecycle = AdapterManifest.ClassicLifecycle; + manifest.Protocol = null; + } + + manifest.Kinds = (author.Kinds ?? new List()).Union(description.Kinds, StringComparer.OrdinalIgnoreCase).ToList(); + var contracts = new Dictionary(author.Contracts ?? new Dictionary(), StringComparer.OrdinalIgnoreCase); + foreach (var (name, version) in description.Contracts) contracts[name] = version; + manifest.Contracts = contracts.Count == 0 ? null : contracts; + + var authored = (author.Properties ?? new List()).ToDictionary(p => p.Name, StringComparer.OrdinalIgnoreCase); + manifest.Properties = description.Settings.Select(setting => + { + var property = authored.TryGetValue(setting.Name, out var a) ? a : new AdapterProperty { Name = setting.Name }; + property.Name = setting.Name; + property.Required = setting.Required; + property.Secret = setting.Secret; + property.Default = setting.Secret ? null : setting.Default; + if (!string.IsNullOrWhiteSpace(setting.Description)) property.Description = setting.Description; + if (string.IsNullOrWhiteSpace(property.Type) || property.Type == AdapterProperty.TextType) + property.Type = string.IsNullOrWhiteSpace(setting.Type) ? AdapterProperty.TextType : setting.Type; + return property; + }).ToList(); + + return manifest; + } + + /// + /// Whether the author's own adapter.json says "lifecycle": "classic" — read from the file, + /// since the model fills in classic when nothing is said. + /// + internal static bool AuthorChoseClassic(string authorJson) + { + using var document = System.Text.Json.JsonDocument.Parse(authorJson, new System.Text.Json.JsonDocumentOptions + { + CommentHandling = System.Text.Json.JsonCommentHandling.Skip, + AllowTrailingCommas = true, + }); + return document.RootElement.TryGetProperty("lifecycle", out var lifecycle) && + string.Equals(lifecycle.GetString(), AdapterManifest.ClassicLifecycle, StringComparison.OrdinalIgnoreCase); + } + + /// + /// The source to carry, keyed by its path in the package's source folder: the project and + /// the local projects it references, under the ignore rules, scanned for secrets. + /// + static Dictionary CollectSource(string project, string projectFile, BuildRequest request, BuildResult result) + { + var roots = new List { project }; + foreach (var referenced in LocalReferences(projectFile, new HashSet(StringComparer.Ordinal))) + if (!roots.Contains(referenced)) roots.Add(referenced); + + var common = CommonAncestor(roots); + var repository = RepositoryRoot(project); + if (repository != null && !common.StartsWith(repository, StringComparison.Ordinal)) + { + result.Warnings.Add($"a referenced project is outside the repository ({common}); the source carried may not rebuild on another machine"); + } + + var files = new Dictionary(StringComparer.Ordinal); + long total = 0; + foreach (var root in roots) + { + var rules = IgnoreRules.For(root); + foreach (var file in Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories)) + { + var withinRoot = Path.GetRelativePath(root, file).Replace('\\', '/'); + if (rules.Ignores(withinRoot)) continue; + var relative = Path.GetRelativePath(common, file).Replace('\\', '/'); + if (files.ContainsKey(relative)) continue; + + var bytes = File.ReadAllBytes(file); + if (SecretScanner.IsText(bytes)) + { + var withinProject = Path.GetRelativePath(project, file).Replace('\\', '/'); + if (!request.AllowedFiles.Contains(withinProject) && !request.AllowedFiles.Contains(relative)) + result.Findings.AddRange(SecretScanner.Scan(relative, System.Text.Encoding.UTF8.GetString(bytes))); + } + + files[relative] = file; + total += bytes.Length; + result.SourceFiles.Add((relative, bytes.Length)); + } + } + + foreach (var finding in result.Findings) + result.Problems.Add($"{finding.File}:{finding.Line} looks like {finding.Kind}. Remove it, or pass --allow {finding.File} if it isn't one"); + + if (total > SourceWarningBytes) + result.Warnings.Add($"the source is {total / (1024 * 1024)} MB; build output may have slipped in — add it to .serverlessignore"); + + return files; + } + + /// The folders of the projects a project references, all the way down. + static IEnumerable LocalReferences(string projectFile, HashSet seen) + { + if (!seen.Add(projectFile)) yield break; + XDocument document; + try { document = XDocument.Load(projectFile); } + catch { yield break; } + + foreach (var include in document.Descendants().Where(e => e.Name.LocalName == "ProjectReference") + .Select(e => (string)e.Attribute("Include")).Where(i => !string.IsNullOrWhiteSpace(i))) + { + var path = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(projectFile)!, include.Replace('\\', Path.DirectorySeparatorChar))); + if (!File.Exists(path)) continue; + yield return Path.GetDirectoryName(path)!; + foreach (var deeper in LocalReferences(path, seen)) yield return deeper; + } + } + + static string CommonAncestor(IEnumerable folders) + { + var split = folders.Select(f => Path.GetFullPath(f).TrimEnd(Path.DirectorySeparatorChar).Split(Path.DirectorySeparatorChar)).ToList(); + var length = split.Min(s => s.Length); + var i = 0; + while (i < length && split.All(s => s[i] == split[0][i])) i++; + var common = string.Join(Path.DirectorySeparatorChar, split[0].Take(i)); + return string.IsNullOrEmpty(common) ? Path.DirectorySeparatorChar.ToString() : common; + } + + static string RepositoryRoot(string folder) + { + for (var dir = new DirectoryInfo(folder); dir != null; dir = dir.Parent) + if (Directory.Exists(Path.Combine(dir.FullName, ".git")) || File.Exists(Path.Combine(dir.FullName, ".git"))) + return dir.FullName; + return null; + } + } +} diff --git a/SW.Serverless.Tooling/Building/SecretScanner.cs b/SW.Serverless.Tooling/Building/SecretScanner.cs new file mode 100644 index 0000000..49b4fca --- /dev/null +++ b/SW.Serverless.Tooling/Building/SecretScanner.cs @@ -0,0 +1,58 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; + +namespace SW.Serverless.Tooling.Building +{ + /// Something in a source file that looks like a secret. + public record SecretFinding(string File, int Line, string Kind); + + /// + /// Looks through the source a package is about to carry for what shouldn't travel with it: + /// private keys, cloud and service tokens, and passwords or keys written into code or + /// configuration. Errs towards reporting; a false positive is let through with --allow. + /// + public static class SecretScanner + { + static readonly (string Kind, Regex Pattern)[] Patterns = + { + ("a private key", new Regex(@"-----BEGIN ([A-Z ]+ )?PRIVATE KEY-----")), + ("an AWS access key", new Regex(@"\b(AKIA|ASIA)[0-9A-Z]{16}\b")), + ("a GitHub token", new Regex(@"\bgh[pousr]_[A-Za-z0-9]{36,}\b")), + ("a Slack token", new Regex(@"\bxox[abprs]-[A-Za-z0-9-]{10,}")), + ("a Google API key", new Regex(@"\bAIza[0-9A-Za-z_-]{35}\b")), + ("a Stripe key", new Regex(@"\b(sk|rk)_live_[0-9A-Za-z]{20,}\b")), + ("an Azure storage key", new Regex(@"AccountKey=[A-Za-z0-9+/=]{40,}", RegexOptions.IgnoreCase)), + // Written as connection strings write it — Password=value, no spaces — which tells it + // apart from code assigning a variable called password. + ("a password in a connection string", new Regex(@"(?Values that are plainly placeholders, never reported. + static readonly Regex Placeholder = new( + @"(?i)^(\*+|x+|changeme|change[-_]?me|your[-_].*|example.*|placeholder|dummy|test|password|secret|<.*>|\$\{.*\}|\{\{.*\}\})$"); + + public static IEnumerable Scan(string relativePath, string text) + { + var lines = text.Split('\n'); + for (var i = 0; i < lines.Length; i++) + foreach (var (kind, pattern) in Patterns) + { + var match = pattern.Match(lines[i]); + if (!match.Success) continue; + var value = match.Groups.Count > 2 ? match.Groups[match.Groups.Count - 1].Value : match.Value; + if (Placeholder.IsMatch(value.Trim())) continue; + yield return new SecretFinding(relativePath, i + 1, kind); + break; + } + } + + /// Whether a file is text worth scanning: small enough, and no NUL bytes in its start. + public static bool IsText(byte[] bytes) => + bytes.Length <= 2 * 1024 * 1024 && !bytes.Take(8000).Contains((byte)0); + } +} diff --git a/SW.Serverless.Installer/Shared/CloudFilesFactory.cs b/SW.Serverless.Tooling/CloudFilesFactory.cs similarity index 98% rename from SW.Serverless.Installer/Shared/CloudFilesFactory.cs rename to SW.Serverless.Tooling/CloudFilesFactory.cs index f3cdca3..910014f 100644 --- a/SW.Serverless.Installer/Shared/CloudFilesFactory.cs +++ b/SW.Serverless.Tooling/CloudFilesFactory.cs @@ -8,8 +8,9 @@ // GC, OC, S3 and LocalTests all publish their registration extensions into this one namespace. using SW.CloudFiles.Extensions; using SW.PrimitiveTypes; +using SW.Serverless.Installer; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { /// /// Builds the storage provider an adapter is published to. diff --git a/SW.Serverless.Tooling/Conformance/ConformanceOptions.cs b/SW.Serverless.Tooling/Conformance/ConformanceOptions.cs new file mode 100644 index 0000000..54a4134 --- /dev/null +++ b/SW.Serverless.Tooling/Conformance/ConformanceOptions.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; + +namespace SW.Serverless.Tooling.Conformance +{ + public class ConformanceOptions + { + /// The adapter's package, unpacked: its runnable files with adapter.json at the root. + public string PackageDirectory { get; set; } + + /// The settings to run it with — the adapter calls whatever these point at. + public IDictionary Settings { get; set; } = new Dictionary(); + + /// + /// Let a receiver's DeleteFile run. Off by default: it removes or moves a real file at the + /// source the settings point at. + /// + public bool AllowDelete { get; set; } + + /// Contracts beyond those the kit carries, e.g. a newer version from a file. + public IList Contracts { get; set; } = new List(); + + /// How long one call may take. + public int CommandTimeoutSeconds { get; set; } = 60; + + /// Where the kit's temporary host keeps its files. + public string WorkDirectory { get; set; } + + /// Where the host finds python3, node and dotnet. + public Runtimes.AdapterRuntimeOptions Runtimes { get; set; } = new(); + + /// Progress, one line at a time. + public Action Log { get; set; } = _ => { }; + } +} diff --git a/SW.Serverless.Tooling/Conformance/ConformanceReport.cs b/SW.Serverless.Tooling/Conformance/ConformanceReport.cs new file mode 100644 index 0000000..1ee07b4 --- /dev/null +++ b/SW.Serverless.Tooling/Conformance/ConformanceReport.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; + +namespace SW.Serverless.Tooling.Conformance +{ + public enum CheckOutcome { Passed, Failed, Skipped } + + /// One thing the kit checked, and what it found. + public record ConformanceCheck(string Name, CheckOutcome Outcome, string Detail = null); + + public class ConformanceReport + { + public List Checks { get; } = new(); + + public bool Passed => Checks.All(c => c.Outcome != CheckOutcome.Failed); + + internal void Pass(string name, string detail = null) => Checks.Add(new(name, CheckOutcome.Passed, detail)); + internal void Fail(string name, string detail) => Checks.Add(new(name, CheckOutcome.Failed, detail)); + internal void Skip(string name, string detail) => Checks.Add(new(name, CheckOutcome.Skipped, detail)); + } +} diff --git a/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs b/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs new file mode 100644 index 0000000..87203ae --- /dev/null +++ b/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs @@ -0,0 +1,324 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Runtimes; + +namespace SW.Serverless.Tooling.Conformance +{ + /// + /// Runs an adapter package the way a host does — installed from storage, started on its runtime, + /// called by name — and checks it against its manifest, its own description and every contract + /// it declares. What serverless test runs, for an adapter in any language. + /// + public class ConformanceRunner + { + const string UnknownCommand = "__ConformanceNoSuchCommand__"; + + public async Task RunAsync(ConformanceOptions options) + { + var report = new ConformanceReport(); + var work = options.WorkDirectory ?? Path.Combine(Path.GetTempPath(), "swsl-conformance", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(work); + try + { + await RunCoreAsync(options, report, work); + } + finally + { + if (options.WorkDirectory == null) + try { Directory.Delete(work, true); } catch { /* best effort */ } + } + return report; + } + + async Task RunCoreAsync(ConformanceOptions options, ConformanceReport report, string work) + { + // ------------------------------------------------------------ the manifest + var manifestPath = Path.Combine(options.PackageDirectory, AdapterManifest.FileName); + if (!File.Exists(manifestPath)) + { + report.Fail("manifest", $"there is no {AdapterManifest.FileName} in {options.PackageDirectory}; serverless build writes it"); + return; + } + + AdapterManifest manifest; + try + { + manifest = AdapterManifest.Parse(File.ReadAllText(manifestPath)); + } + catch (Exception ex) + { + report.Fail("manifest", $"{AdapterManifest.FileName} can't be read: {ex.Message}"); + return; + } + + var problems = manifest.Validate(); + if (problems.Count > 0) report.Fail("manifest", string.Join("; ", problems)); + else report.Pass("manifest"); + + var entry = manifest.EntryFor(AdapterRuntimes.CurrentPlatform); + if (string.IsNullOrWhiteSpace(entry) || !File.Exists(Path.Combine(options.PackageDirectory, entry))) + { + report.Fail("entry", $"the manifest's entry '{entry}' is not in the package"); + return; + } + + // ------------------------------------------------------------ its own description + options.Log("Asking the adapter to describe itself..."); + var description = await DescribeAsync(options, manifest, Path.Combine(options.PackageDirectory, entry), report); + if (description != null) CheckSettings(manifest, description, report); + + // ------------------------------------------------------------ running it + options.Log("Starting it the way a host does..."); + LocalAdapterHost session; + try + { + session = await LocalAdapterHost.StartAsync(options.PackageDirectory, options.Settings, options.Runtimes, + options.CommandTimeoutSeconds, work); + report.Pass("starts", manifest.IsResident ? "resident, attached over gRPC" : "classic session"); + } + catch (Exception ex) + { + report.Fail("starts", ex.GetBaseException().Message); + return; + } + + await using (session) + { + foreach (var (contract, kinds) in ContractsToCheck(manifest, description, options, report)) + foreach (var kind in kinds) + await CheckKindAsync(session, contract, kind, description, options, report); + + await CheckUnknownCommandAsync(session, report); + } + } + + // ---------------------------------------------------------------- describe + + static async Task DescribeAsync(ConformanceOptions options, AdapterManifest manifest, + string entryPath, ConformanceReport report) + { + var (description, problem) = await LocalAdapterHost.DescribeAsync(entryPath, manifest.Runtime, options.Runtimes); + if (description == null) + report.Fail("describe", problem); + else if (string.IsNullOrWhiteSpace(description.SdkLanguage)) + report.Fail("describe", "the description doesn't name its SDK's language"); + else + report.Pass("describe", $"{description.SdkLanguage} SDK {description.SdkVersion}, {description.Commands.Count} commands" + + (description.Warnings.Count > 0 ? $"; warnings: {string.Join("; ", description.Warnings)}" : "")); + return description; + } + + /// The manifest's properties must be the settings the adapter itself declares. + static void CheckSettings(AdapterManifest manifest, AdapterSelfDescription description, ConformanceReport report) + { + var declared = description.Settings.ToDictionary(s => s.Name, StringComparer.OrdinalIgnoreCase); + var listed = (manifest.Properties ?? new List()).ToDictionary(p => p.Name, StringComparer.OrdinalIgnoreCase); + var differences = new List(); + + foreach (var name in declared.Keys.Except(listed.Keys, StringComparer.OrdinalIgnoreCase)) + differences.Add($"'{name}' is declared by the adapter but missing from the manifest"); + foreach (var name in listed.Keys.Except(declared.Keys, StringComparer.OrdinalIgnoreCase)) + differences.Add($"'{name}' is in the manifest but the adapter doesn't declare it"); + foreach (var name in declared.Keys.Intersect(listed.Keys, StringComparer.OrdinalIgnoreCase)) + { + var (code, file) = (declared[name], listed[name]); + if (code.Required != file.Required) differences.Add($"'{name}' is {(code.Required ? "" : "not ")}required in the adapter but {(file.Required ? "" : "not ")}in the manifest"); + if (code.Secret != file.Secret) differences.Add($"'{name}' is {(code.Secret ? "" : "not ")}secret in the adapter but {(file.Secret ? "" : "not ")}in the manifest"); + if (!code.Secret && (code.Default ?? "") != (file.Default ?? "")) differences.Add($"'{name}' defaults to '{code.Default}' in the adapter but '{file.Default}' in the manifest"); + } + + if (differences.Count > 0) report.Fail("settings match the manifest", string.Join("; ", differences) + " — serverless build rewrites the manifest from the adapter"); + else report.Pass("settings match the manifest", $"{declared.Count} settings"); + } + + // ---------------------------------------------------------------- contracts + + static IEnumerable<(ContractDocument Contract, List Kinds)> ContractsToCheck(AdapterManifest manifest, + AdapterSelfDescription description, ConformanceOptions options, ConformanceReport report) + { + var declared = new Dictionary(manifest.Contracts ?? new Dictionary(), StringComparer.OrdinalIgnoreCase); + foreach (var (name, version) in description?.Contracts ?? new Dictionary()) + declared.TryAdd(name, version); + + if (declared.Count == 0) + report.Skip("contracts", "it declares no contract, so only what every adapter must do is checked"); + + var kinds = (manifest.Kinds ?? new List()).Union(description?.Kinds ?? new List(), StringComparer.OrdinalIgnoreCase).ToList(); + foreach (var (name, version) in declared) + { + var contract = options.Contracts.FirstOrDefault(c => string.Equals(c.Name, name, StringComparison.OrdinalIgnoreCase) && c.Version == version) + ?? ContractDocument.Known(name, version); + if (contract == null) + { + report.Fail($"contract {name} v{version}", "the kit doesn't know this contract; pass it with --contract"); + continue; + } + + var checkedKinds = kinds.Where(k => contract.Kinds.Contains(k, StringComparer.OrdinalIgnoreCase)).ToList(); + if (checkedKinds.Count == 0) + report.Fail($"contract {name} v{version}", $"it implements none of the contract's kinds ({string.Join(", ", contract.Kinds)})"); + yield return (contract, checkedKinds); + } + } + + static async Task CheckKindAsync(LocalAdapterHost session, ContractDocument contract, string kind, + AdapterSelfDescription description, ConformanceOptions options, ConformanceReport report) + { + var prefix = $"{contract.Name} {kind}"; + var methods = contract.MethodsOf(kind); + + if (description != null) + { + var commands = description.Commands.Select(c => c.Name).ToHashSet(StringComparer.Ordinal); + var missing = methods.Where(m => !commands.Contains(m.Name)).Select(m => m.Name).ToList(); + if (missing.Count > 0) report.Fail($"{prefix}: methods", $"missing {string.Join(", ", missing)} (names are case-sensitive)"); + else report.Pass($"{prefix}: methods", string.Join(", ", methods.Select(m => m.Name))); + } + + if (contract.IsSession(kind)) + { + await CheckSessionAsync(session, contract, prefix, methods, options, report); + return; + } + + foreach (var method in methods) + { + if (method.Examples.Count == 0) + { + report.Skip($"{prefix}: {method.Name}", "the contract gives no example to call it with"); + continue; + } + + for (var i = 0; i < method.Examples.Count; i++) + { + var name = $"{prefix}: {method.Name} answers example {i + 1}"; + try + { + var input = contract.IsText(method.Input) ? (object)(string)method.Examples[i] : method.Examples[i]; + if (method.Output == null) + { + await session.CallVoidAsync(method.Name, input); + report.Pass(name); + } + else + await CheckOutputAsync(contract, method.Output, await session.CallAsync(method.Name, input), name, report); + } + catch (Exception ex) + { + report.Fail(name, $"the call failed: {ex.GetBaseException().Message}"); + } + } + } + } + + /// + /// A session kind, such as a receiver, called once through in the contract's order: methods + /// without input as they come, a text input fed the first id the session listed, and a + /// destructive method only when allowed. + /// + static async Task CheckSessionAsync(LocalAdapterHost session, ContractDocument contract, string prefix, + IReadOnlyList methods, ConformanceOptions options, ConformanceReport report) + { + string firstId = null; + foreach (var method in methods) + { + var name = $"{prefix}: {method.Name}"; + if (method.Destructive && !options.AllowDelete) + { + report.Skip(name, "it changes the source the settings point at; allow it with --allow-delete"); + continue; + } + + object input = null; + if (method.Input != null) + { + if (firstId == null) + { + report.Skip(name, "nothing was listed to call it with"); + continue; + } + input = firstId; + } + + try + { + if (method.Output == null) + { + await session.CallVoidAsync(method.Name, input); + report.Pass(name); + continue; + } + + var output = await session.CallAsync(method.Name, input); + await CheckOutputAsync(contract, method.Output, output, name, report); + if (firstId == null && TryParse(output) is JArray ids && ids.Count > 0 && ids[0].Type == JTokenType.String) + firstId = (string)ids[0]; + } + catch (Exception ex) + { + report.Fail(name, $"the call failed: {ex.GetBaseException().Message}"); + } + } + } + + static async Task CheckOutputAsync(ContractDocument contract, string type, string output, string name, ConformanceReport report) + { + var schema = await contract.SchemaOfAsync(type); + if (schema == null) + { + report.Pass(name); + return; + } + + if (TryParse(output) is not { } json) + { + report.Fail(name, $"the answer isn't JSON: {Shorten(output)}"); + return; + } + + var errors = schema.Validate(json); + if (errors.Count > 0) report.Fail(name, $"the answer isn't a valid {type}: {string.Join("; ", errors.Select(e => e.ToString()))}"); + else report.Pass(name, $"a valid {type}"); + } + + /// An unknown command is refused with an error, and the adapter keeps answering. + static async Task CheckUnknownCommandAsync(LocalAdapterHost session, ConformanceReport report) + { + for (var attempt = 1; attempt <= 2; attempt++) + { + try + { + await session.CallAsync(UnknownCommand, null); + report.Fail("an unknown command is refused", "it answered a command it doesn't have"); + return; + } + catch (TimeoutException) + { + report.Fail("an unknown command is refused", "it didn't answer a command it doesn't have"); + return; + } + catch (Exception) when (attempt == 1) + { + // Refused; the second attempt shows it is still answering after that. + } + catch (Exception) + { + report.Pass("an unknown command is refused", "with an error, and it kept answering"); + } + } + } + + static JToken TryParse(string text) + { + try { return string.IsNullOrWhiteSpace(text) ? null : JToken.Parse(text); } + catch { return null; } + } + + static string Shorten(string text) => text == null ? "nothing" : text.Length <= 200 ? text : text[..200] + "…"; + } +} diff --git a/SW.Serverless.Tooling/Conformance/ContractDocument.cs b/SW.Serverless.Tooling/Conformance/ContractDocument.cs new file mode 100644 index 0000000..3088d57 --- /dev/null +++ b/SW.Serverless.Tooling/Conformance/ContractDocument.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NJsonSchema; + +namespace SW.Serverless.Tooling.Conformance +{ + /// + /// A contract an adapter can implement, as its machine-readable JSON: the kinds, the methods a + /// host calls on each, their payload types and schemas, and example inputs. Nothing here knows + /// any particular contract; the kit checks an adapter against whichever one it declares. + /// + public class ContractDocument + { + readonly JObject json; + readonly Func readSibling; + + ContractDocument(JObject json, Func readSibling) + { + this.json = json; + this.readSibling = readSibling; + } + + public string Name => (string)json["contract"]; + public int Version => (int?)json["version"] ?? 0; + + public IEnumerable Kinds => ((JObject)json["kinds"])?.Properties().Select(p => p.Name) ?? Enumerable.Empty(); + + public IReadOnlyList MethodsOf(string kind) => + json["kinds"]?[kind]?["methods"]?.Select(m => new ContractMethod( + (string)m["name"], + (string)m["input"], + (string)m["output"], + m["examples"]?.ToList() ?? new List(), + (bool?)m["destructive"] ?? false)) + .ToList() + ?? new List(); + + public bool IsSession(string kind) => json["kinds"]?[kind]?["session"] != null; + + /// Whether a type travels as raw text rather than JSON. + public bool IsText(string type) => (string)json["types"]?[type]?["encoding"] == "string"; + + /// The JSON Schema of a payload type, or null for one with none (raw text). + public async Task SchemaOfAsync(string type) + { + var declared = json["types"]?[type]?["schema"]; + return declared switch + { + null => null, + JValue file => await JsonSchema.FromJsonAsync(readSibling((string)file)), + JObject inline => await JsonSchema.FromJsonAsync(inline.ToString()), + _ => null, + }; + } + + /// The contracts the kit knows by name, carried with it. + public static ContractDocument Known(string name, int version) + { + // Found by file name: the folder in a resource's name is written with whichever + // separator the machine that built it uses. + var file = $"{name}-adapter-contract.v{version}.json"; + var resource = typeof(ContractDocument).Assembly.GetManifestResourceNames() + .FirstOrDefault(r => r.StartsWith("SW.Serverless.Tooling.Contracts.", StringComparison.Ordinal) && + r.EndsWith(file, StringComparison.Ordinal)); + if (resource == null) return null; + + var prefix = resource[..^file.Length]; + return new ContractDocument(JObject.Parse(ReadResource(resource)), sibling => ReadResource(prefix + sibling)); + } + + /// A contract from a file, with its schemas beside it. + public static ContractDocument FromFile(string path) + { + var directory = Path.GetDirectoryName(Path.GetFullPath(path)); + return new ContractDocument(JObject.Parse(File.ReadAllText(path)), + file => File.ReadAllText(Path.Combine(directory, file))); + } + + static string ReadResource(string name) + { + using var stream = typeof(ContractDocument).Assembly.GetManifestResourceStream(name) + ?? throw new FileNotFoundException(name); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } + } + + public record ContractMethod(string Name, string Input, string Output, IReadOnlyList Examples, bool Destructive); +} diff --git a/SW.Serverless.Tooling/Contracts/bitween/bitween-adapter-contract.v1.json b/SW.Serverless.Tooling/Contracts/bitween/bitween-adapter-contract.v1.json new file mode 100644 index 0000000..a56d1b4 --- /dev/null +++ b/SW.Serverless.Tooling/Contracts/bitween/bitween-adapter-contract.v1.json @@ -0,0 +1,121 @@ +{ + "contract": "bitween", + "version": 1, + "description": "What Bitween calls on each kind of adapter, and what it passes. Bitween calls adapters by method name over the SW-Serverless protocol; the names below are exact and case-sensitive.", + "encoding": { + "string": "A string argument or result is the raw UTF-8 text, not a JSON string.", + "object": "Any other argument or result is JSON, with property names exactly as its schema gives them.", + "none": "A method with no argument receives an empty payload; one with no result returns an empty payload." + }, + "types": { + "ExchangeFile": { + "schema": "exchange-file.schema.json" + }, + "ValidationResult": { + "schema": "validation-result.schema.json" + }, + "FileId": { + "encoding": "string", + "description": "An id a receiver gave in ListFiles, passed back exactly as given." + }, + "FileIdList": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "errors": "A call fails when the adapter raises an error; Bitween records the error's type and message on the exchange, and retry policies decide what happens next. A handler whose partner rejected the message does not fail: it returns the partner's answer with BadData true.", + "kinds": { + "handler": { + "description": "Delivers a message and returns the partner's response.", + "methods": [ + { + "name": "Handle", + "input": "ExchangeFile", + "output": "ExchangeFile", + "examples": [ + { + "Data": "{\"orderId\":\"SO-1001\",\"lines\":2}", + "Filename": "order-SO-1001.json", + "BadData": false, + "ContentType": "application/json" + } + ] + } + ] + }, + "mapper": { + "description": "Transforms a message into the shape the next step expects.", + "methods": [ + { + "name": "Handle", + "input": "ExchangeFile", + "output": "ExchangeFile", + "examples": [ + { + "Data": "{\"orderId\":\"SO-1001\",\"lines\":2}", + "Filename": "order-SO-1001.json", + "BadData": false, + "ContentType": "application/json" + } + ] + } + ] + }, + "validator": { + "description": "Checks a message before it is accepted.", + "methods": [ + { + "name": "Validate", + "input": "ExchangeFile", + "output": "ValidationResult", + "examples": [ + { + "Data": "{\"orderId\":\"SO-1001\",\"lines\":2}", + "Filename": "order-SO-1001.json", + "BadData": false, + "ContentType": "application/json" + } + ] + } + ] + }, + "receiver": { + "description": "Fetches files from a source on a schedule.", + "session": "One session per run. Calls come in this order: Initialize, ListFiles, then for each file GetFile and, once it is safely taken in, DeleteFile, and finally Finalize, which is also called after a failure.", + "methods": [ + { + "name": "Initialize", + "input": null, + "output": null + }, + { + "name": "ListFiles", + "input": null, + "output": "FileIdList" + }, + { + "name": "GetFile", + "input": "FileId", + "output": "ExchangeFile" + }, + { + "name": "DeleteFile", + "input": "FileId", + "output": null, + "destructive": true + }, + { + "name": "Finalize", + "input": null, + "output": null + } + ] + } + }, + "examples": "Example inputs for each method that takes one. serverless test calls the adapter with them, using the settings it is given, and checks each answer against the method's output type.", + "destructive": "A destructive method changes the source an adapter reads: serverless test calls it only when told it may." +} diff --git a/SW.Serverless.Tooling/Contracts/bitween/exchange-file.schema.json b/SW.Serverless.Tooling/Contracts/bitween/exchange-file.schema.json new file mode 100644 index 0000000..52a5fa0 --- /dev/null +++ b/SW.Serverless.Tooling/Contracts/bitween/exchange-file.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://bitween.systems/contract/v1/exchange-file.schema.json", + "title": "ExchangeFile", + "description": "A file as Bitween passes it to and from adapters. Property names are exactly as shown; unknown properties are allowed and ignored.", + "type": "object", + "required": ["Data"], + "properties": { + "Filename": { "type": ["string", "null"], "description": "The file's name, when it has one." }, + "Data": { "type": "string", "description": "The content: text, or base64 for binary content." }, + "Hash": { "type": ["string", "null"], "description": "SHA-1 of Data as lower-case hex. Written by .NET adapters; always recomputed from Data when read, never trusted." }, + "BadData": { "type": "boolean", "default": false, "description": "Whether this is a bad response: a delivery the partner rejected." }, + "ContentType": { "type": ["string", "null"], "description": "The content's media type, when known." } + }, + "additionalProperties": true +} diff --git a/SW.Serverless.Tooling/Contracts/bitween/validation-result.schema.json b/SW.Serverless.Tooling/Contracts/bitween/validation-result.schema.json new file mode 100644 index 0000000..97d1a83 --- /dev/null +++ b/SW.Serverless.Tooling/Contracts/bitween/validation-result.schema.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://bitween.systems/contract/v1/validation-result.schema.json", + "title": "ValidationResult", + "description": "What a validator found. Property names are exactly as shown; unknown properties are allowed and ignored.", + "type": "object", + "required": ["Validations"], + "properties": { + "Success": { "type": "boolean", "description": "True when Validations is empty. Bitween derives it from Validations." }, + "Validations": { + "type": "array", + "description": "Each failure: a code, often the field it concerns, and a message.", + "items": { + "type": "object", + "required": ["Key", "Value"], + "properties": { + "Key": { "type": "string" }, + "Value": { "type": "string" } + } + } + } + }, + "additionalProperties": true +} diff --git a/SW.Serverless.Tooling/DotnetBuilds.cs b/SW.Serverless.Tooling/DotnetBuilds.cs new file mode 100644 index 0000000..a6b3e18 --- /dev/null +++ b/SW.Serverless.Tooling/DotnetBuilds.cs @@ -0,0 +1,23 @@ +using System.Diagnostics; + +namespace SW.Serverless.Tooling +{ + /// + /// How the tools run dotnet builds. MSBuild leaves worker nodes and a build server running for + /// the next build, and they inherit the build's redirected output: a caller waiting for that + /// output to end — as WaitForExit and ReadToEnd do — then waits for processes that never exit, + /// and a build that finished in seconds hangs for good. So nothing is left running. + /// + public static class DotnetBuilds + { + public static ProcessStartInfo WithoutLingeringNodes(this ProcessStartInfo startInfo) + { + startInfo.Environment["MSBUILDDISABLENODEREUSE"] = "1"; + startInfo.Environment["DOTNET_CLI_USE_MSBUILD_SERVER"] = "0"; + startInfo.Environment["UseSharedCompilation"] = "false"; + if (!startInfo.Arguments.Contains("-nodeReuse") && startInfo.ArgumentList.Count == 0) + startInfo.Arguments += " -nodeReuse:false"; + return startInfo; + } + } +} diff --git a/SW.Serverless.Installer/Shared/ExpectedValuesProbe.cs b/SW.Serverless.Tooling/ExpectedValuesProbe.cs similarity index 99% rename from SW.Serverless.Installer/Shared/ExpectedValuesProbe.cs rename to SW.Serverless.Tooling/ExpectedValuesProbe.cs index 5b3d47b..a835a2c 100644 --- a/SW.Serverless.Installer/Shared/ExpectedValuesProbe.cs +++ b/SW.Serverless.Tooling/ExpectedValuesProbe.cs @@ -8,7 +8,7 @@ using System.Text; using System.Threading.Tasks; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { /// /// Asks a built CLASSIC adapter which startup values it expects, the way every host already diff --git a/SW.Serverless.Installer/Shared/InstallerLogic.cs b/SW.Serverless.Tooling/InstallerLogic.cs similarity index 98% rename from SW.Serverless.Installer/Shared/InstallerLogic.cs rename to SW.Serverless.Tooling/InstallerLogic.cs index a741082..14b4252 100644 --- a/SW.Serverless.Installer/Shared/InstallerLogic.cs +++ b/SW.Serverless.Tooling/InstallerLogic.cs @@ -10,7 +10,7 @@ using System.Threading.Tasks; using System.Xml.Linq; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { public class InstallerLogic { @@ -36,7 +36,7 @@ public bool BuildPublish(string projectPath, string outputPath) Arguments = $"publish \"{projectPath}\" -c Release -o \"{outputPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, - } + }.WithoutLingeringNodes() }; process.OutputDataReceived += OutputDataReceived; @@ -99,7 +99,7 @@ private static string AssemblyNameFromMsBuild(string projectPath) Arguments = $"msbuild \"{projectPath}\" -getProperty:AssemblyName -p:Configuration=Release", RedirectStandardOutput = true, RedirectStandardError = true, - }); + }.WithoutLingeringNodes()); if (process == null) return null; var stderr = process.StandardError.ReadToEndAsync(); diff --git a/SW.Serverless.Tooling/LocalAdapterHost.cs b/SW.Serverless.Tooling/LocalAdapterHost.cs new file mode 100644 index 0000000..511ff3e --- /dev/null +++ b/SW.Serverless.Tooling/LocalAdapterHost.cs @@ -0,0 +1,235 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using SW.CloudFiles.Extensions; +using SW.PrimitiveTypes; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Resident; +using SW.Serverless.Runtimes; + +namespace SW.Serverless.Tooling +{ + /// + /// An adapter package run on this machine exactly as a host runs one: put in a temporary store, + /// installed from it, started on its runtime, classic or resident, and called by command name. + /// What serverless run and serverless test use. + /// + public sealed class LocalAdapterHost : IAsyncDisposable + { + const string Version = "0.0.0"; + + readonly IHost host; + readonly ISession session; + readonly string work; + readonly bool ownsWork; + + LocalAdapterHost(IHost host, ISession session, string work, bool ownsWork) + { + this.host = host; + this.session = session; + this.work = work; + this.ownsWork = ownsWork; + } + + public static async Task StartAsync(string packageDirectory, IDictionary settings, + AdapterRuntimeOptions runtimes = null, int commandTimeoutSeconds = 60, string workDirectory = null) + { + var manifest = AdapterManifest.Parse(File.ReadAllText(Path.Combine(packageDirectory, AdapterManifest.FileName))); + var work = workDirectory ?? Path.Combine(Path.GetTempPath(), "swsl-local", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(work); + + var host = Build(runtimes ?? new AdapterRuntimeOptions(), commandTimeoutSeconds, work); + await host.StartAsync(); + try + { + var adapterId = string.IsNullOrWhiteSpace(manifest.Id) ? "local.adapter" : manifest.Id; + await UploadAsync(host.Services.GetRequiredService(), packageDirectory, adapterId, + manifest.EntryFor(AdapterRuntimes.CurrentPlatform)); + + var adapterRef = $"{adapterId}/{Version}"; + ISession session = manifest.IsResident + ? await ResidentSession.StartAsync(host.Services, adapterRef, settings, commandTimeoutSeconds) + : await ClassicSession.StartAsync(host.Services, adapterRef, settings, commandTimeoutSeconds); + return new LocalAdapterHost(host, session, work, workDirectory == null); + } + catch + { + await host.StopAsync(); + host.Dispose(); + if (workDirectory == null) try { Directory.Delete(work, true); } catch { } + throw; + } + } + + /// Calls a command and returns its answer as text: raw for a string, JSON for anything else. + public Task CallAsync(string command, object input) => session.CallAsync(command, input); + + /// Calls a command that answers nothing. + public Task CallVoidAsync(string command, object input) => session.CallVoidAsync(command, input); + + public async ValueTask DisposeAsync() + { + try { await session.DisposeAsync(); } catch { /* stopping only */ } + await host.StopAsync(); + host.Dispose(); + if (ownsWork) try { Directory.Delete(work, true); } catch { } + } + + static IHost Build(AdapterRuntimeOptions runtimes, int commandTimeoutSeconds, string work) + { + var tag = Guid.NewGuid().ToString("N")[..10]; + return Host.CreateDefaultBuilder() + .ConfigureLogging(l => l.ClearProviders()) + .ConfigureAppConfiguration(c => c.Sources.Clear()) + .ConfigureServices(s => + { + s.AddLocalTestsCloudFiles(o => + { + o.BucketName = "local-" + tag; + o.StoragePath = Path.Combine(work, "store"); + }); + s.AddAdapterRuntimes(o => + { + o.PythonExecutable = runtimes.PythonExecutable; + o.NodeExecutable = runtimes.NodeExecutable; + o.DotnetExecutable = runtimes.DotnetExecutable; + }); + s.AddServerless(o => + { + o.AdapterRemotePath = "adapters"; + o.AdapterLocalPath = Path.Combine(work, "installed"); + o.AdapterMetadataCacheDuration = 1; + o.CommandTimeout = commandTimeoutSeconds; + }); + s.AddResidentAdapters(o => + { + // Unix socket paths are short-limited, so under /tmp rather than the work folder. + o.SocketPath = Path.Combine("/tmp", $"swsl-local-{tag}.sock"); + o.PipeName = $"swsl-local-{tag}"; + o.HandshakeTimeout = TimeSpan.FromSeconds(60); + }); + }) + .Build(); + } + + static async Task UploadAsync(ICloudFilesService files, string directory, string adapterId, string entry) + { + using var buffer = new MemoryStream(); + using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories)) + archive.CreateEntryFromFile(file, Path.GetRelativePath(directory, file).Replace('\\', '/'), CompressionLevel.Fastest); + + buffer.Position = 0; + var hash = Convert.ToHexString(await SHA256.HashDataAsync(buffer))[..16].ToLowerInvariant(); + buffer.Position = 0; + await files.WriteAsync(buffer, new WriteFileSettings + { + Key = AdapterCatalogPaths.Version("adapters", adapterId, Version), + ContentType = "application/zip", + Metadata = new Dictionary { ["EntryAssembly"] = entry, ["Hash"] = "local-" + hash }, + }); + } + + interface ISession : IAsyncDisposable + { + Task CallAsync(string command, object input); + Task CallVoidAsync(string command, object input); + } + + sealed class ClassicSession(IServiceScope scope, IServerlessService service, int timeout) : ISession + { + public static async Task StartAsync(IServiceProvider services, string adapterRef, + IDictionary settings, int timeout) + { + var scope = services.CreateScope(); + var service = scope.ServiceProvider.GetRequiredService(); + try + { + await service.StartAsync(adapterRef, "local", new Dictionary(settings ?? new Dictionary())); + } + catch + { + (service as IDisposable)?.Dispose(); + scope.Dispose(); + throw; + } + return new ClassicSession(scope, service, timeout); + } + + public Task CallAsync(string command, object input) => service.InvokeAsync(command, input, timeout); + public Task CallVoidAsync(string command, object input) => service.InvokeAsync(command, input, timeout); + + public ValueTask DisposeAsync() + { + (service as IDisposable)?.Dispose(); + scope.Dispose(); + return ValueTask.CompletedTask; + } + } + + sealed class ResidentSession(IResidentAdapterHost host, ResidentAdapterInstance instance, string adapterRef, int timeout) : ISession + { + public static async Task StartAsync(IServiceProvider services, string adapterRef, + IDictionary settings, int timeout) + { + var host = services.GetRequiredService(); + var instance = await host.StartExclusiveAsync(new AdapterSpec + { + AdapterId = adapterRef, + InstanceKey = "local", + StartupValues = new Dictionary(settings ?? new Dictionary()), + }); + return new ResidentSession(host, instance, adapterRef, timeout); + } + + public Task CallAsync(string command, object input) => instance.InvokeAsync(command, input, timeoutSeconds: timeout); + public Task CallVoidAsync(string command, object input) => instance.InvokeAsync(command, input, timeoutSeconds: timeout); + + public async ValueTask DisposeAsync() => await host.StopAsync(adapterRef, "local", drain: false); + } + + sealed class NoEvents : IAdapterEventSink + { + public Task OnEventAsync(InboundEvent inboundEvent, CancellationToken cancellationToken) => + Task.FromResult(EventOutcome.Ok("local")); + } + + /// Runs an adapter with --describe and reads its answer; null and a reason when it gives none. + public static async Task<(AdapterSelfDescription Description, string Problem)> DescribeAsync(string entryPath, + string runtime, AdapterRuntimeOptions runtimes = null, int timeoutSeconds = 30) + { + var launcher = new AdapterRuntimes(runtimes).Find(runtime); + if (launcher == null) return (null, $"no launcher for the '{runtime}' runtime"); + + var startInfo = launcher.StartInfo(entryPath, new RuntimeLaunch { Arguments = new[] { AdapterSelfDescription.Flag } }); + try + { + using var process = Process.Start(startInfo)!; + process.StandardInput.Close(); + var output = process.StandardOutput.ReadToEndAsync(); + var errors = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(timeoutSeconds * 1000)) + { + try { process.Kill(true); } catch { } + return (null, $"it didn't answer {AdapterSelfDescription.Flag} within {timeoutSeconds} seconds — an SDK older than 10.1.0 doesn't know it"); + } + if (process.ExitCode != 0) + return (null, $"{AdapterSelfDescription.Flag} exited with {process.ExitCode}: {await errors}"); + return (AdapterSelfDescription.Parse(await output), null); + } + catch (Exception ex) + { + return (null, $"its description couldn't be read: {ex.Message}"); + } + } + } +} diff --git a/SW.Serverless.Installer/Shared/ManifestBuilder.cs b/SW.Serverless.Tooling/ManifestBuilder.cs similarity index 99% rename from SW.Serverless.Installer/Shared/ManifestBuilder.cs rename to SW.Serverless.Tooling/ManifestBuilder.cs index 540d577..456667c 100644 --- a/SW.Serverless.Installer/Shared/ManifestBuilder.cs +++ b/SW.Serverless.Tooling/ManifestBuilder.cs @@ -8,7 +8,7 @@ using System.Reflection; using System.Text.Json; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { /// What the installer knows about a build, and fills into the manifest over whatever the author wrote. public class GeneratedManifestFields diff --git a/SW.Serverless.Installer/Shared/PackagePublisher.cs b/SW.Serverless.Tooling/PackagePublisher.cs similarity index 61% rename from SW.Serverless.Installer/Shared/PackagePublisher.cs rename to SW.Serverless.Tooling/PackagePublisher.cs index fdb547a..d5b2b45 100644 --- a/SW.Serverless.Installer/Shared/PackagePublisher.cs +++ b/SW.Serverless.Tooling/PackagePublisher.cs @@ -4,7 +4,7 @@ using System.IO; using System.Threading.Tasks; -namespace SW.Serverless.Installer.Shared +namespace SW.Serverless.Tooling { public class PublishRequest { @@ -32,6 +32,19 @@ public class PublishRequest public string PublishedBy { get; set; } } + /// A package serverless build made — in any language — to publish as it is. + public class PublishPackageRequest + { + public string PackagePath { get; set; } + + /// Explicit, or major, minor or patch to bump; the manifest's own version when empty. + public string Version { get; set; } + + public bool Promote { get; set; } = true; + public string ReleaseNotes { get; set; } + public string PublishedBy { get; set; } + } + public class PublishResult { /// The version published, or null for an unversioned upload. @@ -143,5 +156,70 @@ await repository.PublishVersionAsync(adapterId, version, zipPath, package, reque return new PublishResult { Version = version, Sha256 = sha256, Manifest = manifest, IconDataUri = iconDataUri }; } + + /// + /// Publishes a package serverless build made: its manifest is already complete, so this only + /// settles the version, stamps it into the manifest and uploads. Always versioned; an adapter + /// in another runtime goes only where hosts that can run it look. + /// + public static async Task PublishPackageAsync(ICloudFilesService files, PublishPackageRequest request, + Action log = null) + { + log ??= Console.WriteLine; + void Warn(string message) => log($"Warning: {message}"); + + if (!File.Exists(request.PackagePath)) + throw new SWException($"There is no package at {request.PackagePath}."); + + var work = Path.Combine(Path.GetTempPath(), "swsl-publish", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(work); + try + { + var folder = Path.Combine(work, "package"); + System.IO.Compression.ZipFile.ExtractToDirectory(request.PackagePath, folder); + var manifestPath = Path.Combine(folder, AdapterManifest.FileName); + if (!File.Exists(manifestPath)) + throw new SWException($"{request.PackagePath} has no {AdapterManifest.FileName}; build it with serverless build."); + + var manifest = AdapterManifest.Parse(await File.ReadAllTextAsync(manifestPath)); + var adapterId = manifest.Id?.ToLowerInvariant(); + if (!InstallerLogic.IsValidAdapterId(adapterId)) + throw new SWException($"The package's manifest has no valid id ('{manifest.Id}')."); + + var mode = string.IsNullOrWhiteSpace(request.Version) ? manifest.Version : request.Version; + if (string.IsNullOrWhiteSpace(mode)) + throw new SWException("The package has no version: give one with -v, or set version in adapter.json."); + + var repository = new AdapterRepository(files, log); + var version = await repository.ResolveVersionAsync(adapterId, mode); + manifest.Version = version; + manifest.PublishedOn = DateTimeOffset.UtcNow; + if (!string.IsNullOrWhiteSpace(request.ReleaseNotes)) manifest.ReleaseNotes = request.ReleaseNotes; + ManifestBuilder.Validate(manifest); + await File.WriteAllTextAsync(manifestPath, manifest.ToJson()); + + var zipPath = Path.Combine(work, "adapter.zip"); + if (!new InstallerLogic().Compress(folder, zipPath)) + throw new SWException("The package could not be rebuilt with its version, so nothing was uploaded."); + + var iconDataUri = ManifestBuilder.IconDataUri(manifest, folder, Warn); + var package = new PackageInfo + { + EntryAssembly = manifest.Entry, + Lifecycle = manifest.Lifecycle, + Kind = string.Join(",", manifest.Kinds), + Manifest = manifest, + IconDataUri = iconDataUri, + }; + + log($"Publishing version {version} of '{adapterId}' ({manifest.Runtime})."); + await repository.PublishVersionAsync(adapterId, version, zipPath, package, request.Promote, request.PublishedBy); + return new PublishResult { Version = version, Sha256 = InstallerLogic.Sha256Of(zipPath), Manifest = manifest, IconDataUri = iconDataUri }; + } + finally + { + try { Directory.Delete(work, true); } catch { } + } + } } } diff --git a/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj new file mode 100644 index 0000000..71b7936 --- /dev/null +++ b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj @@ -0,0 +1,47 @@ + + + + + net10.0 + latest + SimplyWorks.Serverless.Tooling + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs b/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs new file mode 100644 index 0000000..8fcdb4a --- /dev/null +++ b/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs @@ -0,0 +1,253 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; + +namespace SW.Serverless.Tooling.Scaffolding +{ + public class ScaffoldRequest + { + /// The adapter's name, e.g. AcmeOrders: the folder, the project and the class names. + public string Name { get; set; } + + /// Its id in storage; derived from the name when not given. + public string Id { get; set; } + + public string Language { get; set; } = "dotnet"; + + /// handler, mapper, validator or receiver. + public string Kind { get; set; } = "handler"; + + /// The folder the project folder is made in. + public string ParentDirectory { get; set; } = "."; + } + + public class ScaffoldResult + { + public string ProjectDirectory { get; set; } + public List Files { get; } = new(); + public List Problems { get; } = new(); + public bool Succeeded => Problems.Count == 0; + } + + /// + /// What serverless init writes: a working adapter of one kind for the Bitween contract, ready to + /// build, test and publish. Also what a code editor starts a new adapter from. + /// + public static class Scaffolder + { + /// The SDK the templates reference; the first with --describe and the Bitween contract types. + public const string SdkPackageVersion = "10.1.0"; + + /// + /// The Bitween contract package the templates reference: the first Bitween release that + /// publishes it. NuGet reads it as a minimum, so it resolves to the first one there is. + /// + public const string BitweenAdaptersPackageVersion = "10.0.59"; + + public static readonly IReadOnlyList Kinds = new[] { "handler", "mapper", "validator", "receiver" }; + public static readonly IReadOnlyList Languages = new[] { "dotnet" }; + + public static ScaffoldResult Scaffold(ScaffoldRequest request) + { + var result = new ScaffoldResult(); + var name = request.Name?.Trim(); + if (string.IsNullOrEmpty(name) || !Regex.IsMatch(name, "^[A-Za-z][A-Za-z0-9_]*$")) + result.Problems.Add($"'{request.Name}' isn't a name a project and a class can take: letters, digits and _, starting with a letter"); + if (!Kinds.Contains(request.Kind)) + result.Problems.Add($"'{request.Kind}' isn't a kind: use {string.Join(", ", Kinds)}"); + if (!Languages.Contains(request.Language)) + result.Problems.Add($"a '{request.Language}' template arrives with that language's SDK; this release scaffolds {string.Join(", ", Languages)}"); + + var id = string.IsNullOrWhiteSpace(request.Id) ? IdFrom(name ?? "") : request.Id.Trim(); + if (!InstallerLogic.IsValidAdapterId(id)) + result.Problems.Add($"'{id}' isn't an adapter id: lowercase letters, digits, '.', '_' and '-'"); + + var directory = Path.GetFullPath(Path.Combine(request.ParentDirectory ?? ".", name ?? "")); + if (Directory.Exists(directory) && Directory.EnumerateFileSystemEntries(directory).Any()) + result.Problems.Add($"{directory} already exists and isn't empty"); + if (!result.Succeeded) return result; + + Directory.CreateDirectory(directory); + result.ProjectDirectory = directory; + foreach (var (file, content) in DotnetFiles(name, id, request.Kind)) + { + var path = Path.Combine(directory, file); + File.WriteAllText(path, content.Replace("\r\n", "\n")); + result.Files.Add(file); + } + return result; + } + + /// AcmeOrders → acme.orders + public static string IdFrom(string name) => + Regex.Replace(name, "(?<=[a-z0-9])(?=[A-Z])", ".").Replace('_', '.').ToLowerInvariant(); + + static IEnumerable<(string File, string Content)> DotnetFiles(string name, string id, string kind) + { + yield return ($"{name}.csproj", $$""" + + + + Exe + net10.0 + enable + enable + + true + + + + + + + + + """); + + yield return ("adapter.json", $$""" + { + "id": "{{id}}", + "version": "0.1.0", + "displayName": "{{Spaced(name)}}", + "summary": "What this {{kind}} does, in one sentence, for the adapter list.", + "lifecycle": "classic" + } + """); + + yield return ("Program.cs", Program(name, kind)); + + yield return ("settings.example.json", """ + { + "BaseUrl": "https://partner.example.test", + "ApiKey": "put a test key here, and keep this file out of version control once it holds one" + } + """); + + yield return (".gitignore", """ + bin/ + obj/ + settings.json + """); + + yield return ("README.md", $$""" + # {{Spaced(name)}} + + A Bitween {{kind}} adapter. + + ```sh + serverless build # builds bin/serverless/{{id}}-0.1.0.zip + cp settings.example.json settings.json # then fill in real values + serverless test --settings settings.json # checks it against the Bitween contract + serverless publish bin/serverless/{{id}}-0.1.0.zip # with your storage flags + ``` + + Settings are declared in code with `Runner.Expect`; `serverless build` writes them into + the manifest Bitween reads. + """); + } + + static string Program(string name, string kind) => kind switch + { + "receiver" => $$""" + using SW.Bitween.Adapters; + using SW.Serverless.Sdk; + + namespace {{name}}; + + /// Fetches files on a schedule. Bitween calls Initialize, ListFiles, then GetFile and DeleteFile for each file, then Finalize. + [AdapterKind("receiver")] + [AdapterContract("bitween", 1)] + public class Receiver : IBitweenReceiver + { + public Receiver() + { + // Declare settings here; read them in the methods, never in the constructor. + Runner.Expect("BaseUrl", "https://partner.example.test", description: "Where files are fetched from."); + Runner.Expect("ApiKey", isPrivate: true, description: "The partner's key."); + } + + public Task Initialize() => Task.CompletedTask; + + public Task> ListFiles() => + Task.FromResult>(new[] { "example-1" }); + + public Task GetFile(string fileId) => + Task.FromResult(new ExchangeFile("{\"id\":\"" + fileId + "\"}", fileId + ".json")); + + public Task DeleteFile(string fileId) => Task.CompletedTask; + + public Task Finalize() => Task.CompletedTask; + } + + static class Program + { + static Task Main() => Runner.Run(new Receiver()); + } + """, + "validator" => $$""" + using SW.Bitween.Adapters; + using SW.Serverless.Sdk; + + namespace {{name}}; + + /// Checks a message before Bitween accepts it. + [AdapterKind("validator")] + [AdapterContract("bitween", 1)] + public class Validator : IBitweenValidator + { + public Validator() + { + Runner.Expect("MaxBytes", "1000000", description: "The largest message accepted."); + } + + public Task Validate(ExchangeFile file) + { + var result = new ValidationResult(); + if (file.Data.Length > Runner.StartupValueOf("MaxBytes")) + result.AddError("Data", "The message is larger than allowed."); + return Task.FromResult(result); + } + } + + static class Program + { + static Task Main() => Runner.Run(new Validator()); + } + """, + var handlerOrMapper => $$""" + using SW.Bitween.Adapters; + using SW.Serverless.Sdk; + + namespace {{name}}; + + /// {{(handlerOrMapper == "mapper" ? "Maps a message into the shape the next step expects." : "Delivers a message and returns the partner's response.")}} + [AdapterKind("{{handlerOrMapper}}")] + [AdapterContract("bitween", 1)] + public class {{(handlerOrMapper == "mapper" ? "Mapper : IBitweenMapper" : "Handler : IBitweenHandler")}} + { + public {{(handlerOrMapper == "mapper" ? "Mapper" : "Handler")}}() + { + // Declare settings here; read them in the methods, never in the constructor. + Runner.Expect("BaseUrl", "https://partner.example.test", description: "Where messages go."); + Runner.Expect("ApiKey", isPrivate: true, description: "The partner's key."); + } + + public Task Handle(ExchangeFile file) + { + // {{(handlerOrMapper == "mapper" ? "Return the message in its new shape." : "Send file.Data to the partner. A rejection is returned with BadData set, not thrown.")}} + return Task.FromResult(new ExchangeFile(file.Data, file.Filename)); + } + } + + static class Program + { + static Task Main() => Runner.Run(new {{(handlerOrMapper == "mapper" ? "Mapper" : "Handler")}}()); + } + """, + }; + + static string Spaced(string name) => Regex.Replace(name, "(?<=[a-z0-9])(?=[A-Z])", " "); + } +} diff --git a/SW.Serverless.Installer/Shared/Semver.cs b/SW.Serverless.Tooling/Semver.cs similarity index 98% rename from SW.Serverless.Installer/Shared/Semver.cs rename to SW.Serverless.Tooling/Semver.cs index 01b1772..feb526f 100644 --- a/SW.Serverless.Installer/Shared/Semver.cs +++ b/SW.Serverless.Tooling/Semver.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Text.RegularExpressions; -namespace SW.Serverless.Installer.Shared; +namespace SW.Serverless.Tooling; public static class Semver { diff --git a/SW.Serverless.Tooling/ServerlessUploadOptions.cs b/SW.Serverless.Tooling/ServerlessUploadOptions.cs new file mode 100644 index 0000000..f1989ed --- /dev/null +++ b/SW.Serverless.Tooling/ServerlessUploadOptions.cs @@ -0,0 +1,48 @@ +namespace SW.Serverless.Installer +{ + /// Where adapters are published: the storage provider and how to reach it. + public class ServerlessUploadOptions + { + public string Provider { get; set; } + + public string AccessKeyId { get; set; } + + public string SecretAccessKey { get; set; } + + public string BucketName { get; set; } + + public string ServiceUrl { get; set; } + + + public string RSAKey { get; set; } + + public string UserId { get; set; } + + public string FingerPrint { get; set; } + + public string TenantId { get; set; } + + public string Region { get; set; } + + + public string Version { get; set; } + + + public string AdapterId { get; set; } + public string NamespaceName { get; set; } + + // ——— Google Cloud service account ——— + public string ProjectId { get; set; } + public string PrivateKeyId { get; set; } + public string PrivateKey { get; set; } + public string ClientEmail { get; set; } + public string ClientId { get; set; } + public string ClientX509CertUrl { get; set; } + + /// + /// Roles this adapter serves, comma separated. Overrides what the assembly declared, for + /// an adapter whose author has not added [AdapterKind] to it. + /// + public string Kind { get; set; } + } +} diff --git a/SW.Serverless.UnitTests.Adapter/Handler.cs b/SW.Serverless.UnitTests.Adapter/Handler.cs index 6f08b33..92f46b7 100644 --- a/SW.Serverless.UnitTests.Adapter/Handler.cs +++ b/SW.Serverless.UnitTests.Adapter/Handler.cs @@ -39,6 +39,8 @@ public Task TestTypedReturn() }); } + public Task TestStartupValue(string name) => Task.FromResult(Runner.StartupValueOf(name)); + public Task TestString(string value) { return Task.FromResult((object)value); diff --git a/SW.Serverless.UnitTests.BitweenHandler/Program.cs b/SW.Serverless.UnitTests.BitweenHandler/Program.cs new file mode 100644 index 0000000..8396520 --- /dev/null +++ b/SW.Serverless.UnitTests.BitweenHandler/Program.cs @@ -0,0 +1,39 @@ +using System.Threading.Tasks; +using SW.Serverless.Sdk; + +namespace SW.Serverless.UnitTests.BitweenHandler; + +/// An exchange file as Bitween sends it. +public class ExchangeFile +{ + public string Data { get; set; } + public string Filename { get; set; } + public bool BadData { get; set; } + public string ContentType { get; set; } +} + +/// +/// A Bitween handler on the classic text protocol: delivers by echoing the order back. With Mode +/// set to "broken" it answers without the Data every exchange file must carry. +/// +[AdapterKind("handler")] +[AdapterContract("bitween", 1)] +public class Handler +{ + public Handler() + { + Runner.Expect("Endpoint", "https://partner.example.test/orders", description: "Where orders go."); + Runner.Expect("Mode", "working"); + Runner.Expect("ApiKey", isPrivate: true); + } + + public Task Handle(ExchangeFile file) => + Task.FromResult(Runner.StartupValueOf("Mode") == "broken" + ? new { Filename = "answer.json" } + : new ExchangeFile { Data = "{\"accepted\":true}", Filename = "answer.json", ContentType = "application/json" }); +} + +static class Program +{ + static Task Main() => Runner.Run(new Handler()); +} diff --git a/SW.Serverless.UnitTests.BitweenHandler/SW.Serverless.UnitTests.BitweenHandler.csproj b/SW.Serverless.UnitTests.BitweenHandler/SW.Serverless.UnitTests.BitweenHandler.csproj new file mode 100644 index 0000000..033b739 --- /dev/null +++ b/SW.Serverless.UnitTests.BitweenHandler/SW.Serverless.UnitTests.BitweenHandler.csproj @@ -0,0 +1,13 @@ + + + + Exe + net10.0 + false + + + + + + + diff --git a/SW.Serverless.UnitTests.BitweenReceiver/Program.cs b/SW.Serverless.UnitTests.BitweenReceiver/Program.cs new file mode 100644 index 0000000..6e15511 --- /dev/null +++ b/SW.Serverless.UnitTests.BitweenReceiver/Program.cs @@ -0,0 +1,47 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using SW.Serverless.Sdk; + +namespace SW.Serverless.UnitTests.BitweenReceiver; + +public class ExchangeFile +{ + public string Data { get; set; } + public string Filename { get; set; } +} + +/// +/// A Bitween receiver over gRPC that reads the files in a folder and deletes one when told to — +/// so a test can see whether DeleteFile was called. +/// +[AdapterKind("receiver")] +[AdapterContract("bitween", 1)] +public class Handler +{ + public Handler() => Runner.Expect("Folder"); + + string Folder => Runner.StartupValueOf("Folder"); + + public Task Initialize() => Task.CompletedTask; + + public Task> ListFiles() => + Task.FromResult>(Directory.GetFiles(Folder).Select(Path.GetFileName).OrderBy(n => n).ToList()); + + public Task GetFile(string fileId) => + Task.FromResult(new ExchangeFile { Data = File.ReadAllText(Path.Combine(Folder, fileId)), Filename = fileId }); + + public Task DeleteFile(string fileId) + { + File.Delete(Path.Combine(Folder, fileId)); + return Task.CompletedTask; + } + + public Task Finalize() => Task.CompletedTask; +} + +static class Program +{ + static Task Main() => Runner.RunResident(new Handler()); +} diff --git a/SW.Serverless.UnitTests.BitweenReceiver/SW.Serverless.UnitTests.BitweenReceiver.csproj b/SW.Serverless.UnitTests.BitweenReceiver/SW.Serverless.UnitTests.BitweenReceiver.csproj new file mode 100644 index 0000000..033b739 --- /dev/null +++ b/SW.Serverless.UnitTests.BitweenReceiver/SW.Serverless.UnitTests.BitweenReceiver.csproj @@ -0,0 +1,13 @@ + + + + Exe + net10.0 + false + + + + + + + diff --git a/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs b/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs new file mode 100644 index 0000000..ff4694f --- /dev/null +++ b/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs @@ -0,0 +1,34 @@ +using System.Threading.Tasks; +using SW.Serverless.Sdk; + +namespace SW.Serverless.UnitTests.GrpcClassicAdapter; + +[AdapterKind("handler")] +[AdapterContract("bitween", 1)] +public class Handler +{ + public Handler() + { + Runner.Expect("Prefix", "hello "); + Runner.Expect("ApiKey", isPrivate: true, description: "The partner's key."); + } + + public Task Greet(string name) => Task.FromResult(Runner.StartupValueOf("Prefix") + name); + + public Task Correlation() => Task.FromResult(Runner.StartupValueOf("CorrelationId")); + + public Task Add(Numbers numbers) => Task.FromResult(numbers.A + numbers.B); + + public Task Fail() => throw new System.InvalidOperationException("asked to fail"); +} + +public class Numbers +{ + public int A { get; set; } + public int B { get; set; } +} + +static class Program +{ + static Task Main() => Runner.RunResident(new Handler()); +} diff --git a/SW.Serverless.UnitTests.GrpcClassicAdapter/SW.Serverless.UnitTests.GrpcClassicAdapter.csproj b/SW.Serverless.UnitTests.GrpcClassicAdapter/SW.Serverless.UnitTests.GrpcClassicAdapter.csproj new file mode 100644 index 0000000..c3076cd --- /dev/null +++ b/SW.Serverless.UnitTests.GrpcClassicAdapter/SW.Serverless.UnitTests.GrpcClassicAdapter.csproj @@ -0,0 +1,15 @@ + + + + + Exe + net10.0 + false + + + + + + + diff --git a/SW.Serverless.UnitTests/CommandDiscoveryTests.cs b/SW.Serverless.UnitTests/CommandDiscoveryTests.cs index 6fc2826..fd6b406 100644 --- a/SW.Serverless.UnitTests/CommandDiscoveryTests.cs +++ b/SW.Serverless.UnitTests/CommandDiscoveryTests.cs @@ -83,6 +83,13 @@ public static async Task ClassCleanup() static IReadOnlyCollection Commands() => adapters.Describe().Single(h => h.InstanceKey == "discovery").CommandDetails; + [TestMethod] + public void The_adapter_says_which_language_its_SDK_is() + { + var described = adapters.Describe().Single(h => h.InstanceKey == "discovery"); + Assert.AreEqual("dotnet", described.SdkLanguage); + } + [TestMethod] public void Every_public_command_is_discovered_without_being_declared_anywhere() { diff --git a/SW.Serverless.UnitTests/DescribeTests.cs b/SW.Serverless.UnitTests/DescribeTests.cs new file mode 100644 index 0000000..77273c1 --- /dev/null +++ b/SW.Serverless.UnitTests/DescribeTests.cs @@ -0,0 +1,102 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.UnitTests.Fixtures; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text.Json; + +namespace SW.Serverless.UnitTests +{ + /// + /// An adapter started with --describe prints what it is — settings, commands with JSON Schemas, + /// kinds, contracts — and exits, without any host. This is how the tools learn about an adapter + /// in any language, rather than by reading .NET assemblies. + /// + [TestClass] + public class DescribeTests + { + static AdapterSelfDescription Describe(string project) + { + var entry = Path.Combine(TestStore.LocateBuildOutput(project), project + ".dll"); + using var process = Process.Start(new ProcessStartInfo("dotnet") + { + ArgumentList = { entry, AdapterSelfDescription.Flag }, + RedirectStandardOutput = true, + RedirectStandardError = true, + RedirectStandardInput = true, + })!; + // Nothing is ever written to it: a describe never waits on stdin. + process.StandardInput.Close(); + var output = process.StandardOutput.ReadToEnd(); + var error = process.StandardError.ReadToEnd(); + Assert.IsTrue(process.WaitForExit(30_000), "--describe did not exit"); + Assert.AreEqual(0, process.ExitCode, error); + return AdapterSelfDescription.Parse(output); + } + + [TestMethod] + public void A_classic_adapter_describes_its_settings_commands_and_kinds() + { + var d = Describe("SW.Serverless.Samples.Classic"); + + Assert.AreEqual("dotnet", d.SdkLanguage); + Assert.AreEqual("10.1.0", d.SdkVersion); + Assert.AreEqual(AdapterManifest.ClassicLifecycle, d.Lifecycle); + Assert.AreEqual(1, d.Protocol.Max, "the classic text protocol"); + CollectionAssert.AreEquivalent(new[] { "handler", "mapper" }, d.Kinds); + + var apiKey = d.Settings.Single(s => s.Name == "ApiKey"); + Assert.IsTrue(apiKey.Required); + Assert.IsTrue(apiKey.Secret); + Assert.IsNull(apiKey.Default, "a secret's default is never published"); + Assert.AreEqual("https://api.example.test", d.Settings.Single(s => s.Name == "BaseUrl").Default); + + var echo = d.Commands.Single(c => c.Name == "Echo"); + StringAssert.Contains(echo.InputSchema!.Value.GetRawText(), "string"); + Assert.IsTrue(echo.ReturnsValue); + Assert.IsFalse(d.Commands.Single(c => c.Name == "Fail").ReturnsValue); + + // An object argument is described by its properties, named as they travel. + var summarize = d.Commands.Single(c => c.Name == "Summarize").InputSchema!.Value; + Assert.IsTrue(summarize.GetProperty("properties").TryGetProperty("Lines", out _)); + } + + [TestMethod] + public void A_resident_adapter_describes_its_commands_and_leaves_out_its_lifecycle_methods() + { + var d = Describe("SW.Serverless.Samples.Greedy"); + + Assert.AreEqual(AdapterManifest.ResidentLifecycle, d.Lifecycle); + Assert.AreEqual(2, d.Protocol.Min, "gRPC"); + var names = d.Commands.Select(c => c.Name).ToList(); + CollectionAssert.IsSubsetOf(new[] { "Allocate", "GetStats", "Release" }, names); + CollectionAssert.DoesNotContain(names, "StartAsync"); + CollectionAssert.DoesNotContain(names, "StopAsync"); + } + + [TestMethod] + public void An_adapter_describes_the_contracts_it_implements() + { + var d = Describe("SW.Serverless.UnitTests.GrpcClassicAdapter"); + + Assert.AreEqual(1, d.Contracts["bitween"]); + CollectionAssert.AreEqual(new[] { "handler" }, d.Kinds); + Assert.AreEqual("hello ", d.Settings.Single(s => s.Name == "Prefix").Default); + Assert.AreEqual(0, d.Warnings.Count, string.Join("; ", d.Warnings)); + } + + [TestMethod] + public void The_description_round_trips_through_its_own_parser() + { + var d = Describe("SW.Serverless.Samples.Classic"); + var again = AdapterSelfDescription.Parse(d.ToJson()); + + Assert.AreEqual(d.Commands.Count, again.Commands.Count); + Assert.AreEqual(d.Settings.Count, again.Settings.Count); + Assert.IsTrue(JsonElement.DeepEquals( + d.Commands.Single(c => c.Name == "Summarize").InputSchema!.Value, + again.Commands.Single(c => c.Name == "Summarize").InputSchema!.Value)); + } + } +} diff --git a/SW.Serverless.UnitTests/ExecRuntimeTests.cs b/SW.Serverless.UnitTests/ExecRuntimeTests.cs new file mode 100644 index 0000000..28f2c68 --- /dev/null +++ b/SW.Serverless.UnitTests/ExecRuntimeTests.cs @@ -0,0 +1,178 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; +using SW.CloudFiles.Extensions; +using SW.PrimitiveTypes; +using SW.Serverless.Resident; +using SW.Serverless.Runtimes; +using SW.Serverless.UnitTests.Fixtures; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; + +namespace SW.Serverless.UnitTests +{ + /// + /// The exec runtime: an adapter that is one self-contained executable, started as itself rather + /// than through dotnet or an interpreter. Built here as the Greedy sample published single-file + /// for this machine, packaged with an exec manifest the way any language's binary would be, and + /// run by the resident host over gRPC. + /// + [TestClass] + public class ExecRuntimeTests + { + const string Project = "SW.Serverless.Samples.Greedy"; + const string AdapterId = "test.exec"; + + static IHost host; + static IResidentAdapterHost adapters; + static string workDirectory; + + [ClassInitialize] + public static async Task ClassInitialize(TestContext context) + { + workDirectory = Path.Combine(Path.GetTempPath(), "swsl-exectests", Guid.NewGuid().ToString("N")); + var publishDirectory = Path.Combine(workDirectory, "publish"); + var platform = AdapterRuntimes.CurrentPlatform; + + var project = Path.Combine(RepositoryRoot(), Project, Project + ".csproj"); + using (var publish = Process.Start(new ProcessStartInfo("dotnet", + $"publish \"{project}\" -c Release -r {platform} --self-contained true -p:PublishSingleFile=true -o \"{publishDirectory}\" " + + // Built under the test's own folder, never beside the sample's usual build output, + // which other tests read. + $"--artifacts-path \"{Path.Combine(workDirectory, "artifacts")}\" -v q") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + // No MSBuild nodes left holding the output open: see DotnetBuilds. + Environment = { ["MSBUILDDISABLENODEREUSE"] = "1", ["DOTNET_CLI_USE_MSBUILD_SERVER"] = "0" }, + })) + { + var output = await publish!.StandardOutput.ReadToEndAsync() + await publish.StandardError.ReadToEndAsync(); + await publish.WaitForExitAsync(); + Assert.AreEqual(0, publish.ExitCode, output); + } + + var binary = Path.Combine(publishDirectory, OperatingSystem.IsWindows() ? Project + ".exe" : Project); + Assert.IsTrue(File.Exists(binary), $"no single-file binary at {binary}"); + + host = Host.CreateDefaultBuilder() + .ConfigureLogging(l => l.ClearProviders()) + .ConfigureServices(s => + { + s.AddLocalTestsCloudFiles(o => o.BucketName = TestStore.BucketName + "-exec"); + s.AddServerless(o => + { + o.AdapterRemotePath = "adapters"; + o.AdapterLocalPath = Path.Combine(workDirectory, "installed"); + o.AdapterMetadataCacheDuration = 1; + }); + s.AddSingleton(); + s.AddSingleton(sp => sp.GetRequiredService()); + s.AddResidentAdapters(o => + { + o.SocketPath = $"/tmp/swsl-x{Environment.ProcessId}.sock"; + o.PipeName = $"swsl-x{Environment.ProcessId}"; + o.HeartbeatInterval = TimeSpan.FromSeconds(2); + o.HandshakeTimeout = TimeSpan.FromSeconds(60); + }); + }) + .Build(); + await host.StartAsync(); + adapters = host.Services.GetRequiredService(); + + // Packaged with only the binary and its manifest — no .dll anywhere for dotnet to start — + // and stored as a version, as a non-.NET adapter is published. + var zip = Path.Combine(workDirectory, "package.zip"); + using (var archive = ZipFile.Open(zip, ZipArchiveMode.Create)) + { + archive.CreateEntryFromFile(binary, Path.GetFileName(binary)); + using var writer = new StreamWriter(archive.CreateEntry("adapter.json").Open()); + writer.Write(new JObject + { + ["id"] = AdapterId, + ["runtime"] = "exec", + ["lifecycle"] = "resident", + ["entry"] = Path.GetFileName(binary), + ["platforms"] = new JArray(platform), + }.ToString()); + } + + var files = host.Services.GetRequiredService(); + await using (var stream = File.OpenRead(zip)) + await files.WriteAsync(stream, new WriteFileSettings + { + Key = $"adapters-versions/{AdapterId}/1.0.0", + ContentType = "application/zip", + Metadata = new Dictionary + { + ["EntryAssembly"] = Path.GetFileName(binary), + ["Hash"] = "exec-" + Guid.NewGuid().ToString("N")[..12], + ["Protocol"] = "2", + ["Lifecycle"] = "resident", + } + }); + } + + [ClassCleanup] + public static async Task ClassCleanup() + { + if (host != null) await host.StopAsync(); + host?.Dispose(); + try { Directory.Delete(workDirectory, true); } catch { } + } + + static string RepositoryRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null && !File.Exists(Path.Combine(dir.FullName, "SW.Serverless.sln"))) dir = dir.Parent; + return dir!.FullName; + } + + [TestMethod] + public async Task A_self_contained_binary_runs_as_itself_and_answers_over_the_protocol() + { + var instance = await adapters.StartExclusiveAsync(new AdapterSpec + { + AdapterId = $"{AdapterId}/1.0.0", + InstanceKey = "exec", + }); + + Assert.AreEqual(InstanceState.Ready, instance.State); + // Started as itself: the process is the binary, not dotnet hosting a dll. + Assert.AreEqual(OperatingSystem.IsWindows() ? Project + ".exe" : Project, Path.GetFileName(instance.Process!.MainModule!.FileName)); + + await instance.InvokeAsync("Allocate", 3, timeoutSeconds: 30); + var stats = await instance.InvokeAsync("GetStats", timeoutSeconds: 15); + Assert.AreEqual(3, stats!.Value("allocatedMb")); + + Assert.AreEqual("dotnet", adapters.Describe().Single(h => h.InstanceKey == "exec").SdkLanguage, + "a .NET adapter compiled to a binary still says which SDK it was built with"); + await adapters.StopAsync($"{AdapterId}/1.0.0", "exec", drain: false); + } + + /// A classic session with an adapter in another runtime goes over gRPC, unchanged for the caller. + [TestMethod] + public async Task A_classic_session_with_an_exec_adapter_runs_over_grpc() + { + var service = host.Services.GetRequiredService(); + await service.StartAsync($"{AdapterId}/1.0.0", "corr-exec"); + try + { + await service.InvokeAsync("Allocate", 2); + var stats = await service.InvokeAsync("GetStats", null); + Assert.AreEqual(2, stats.Value("allocatedMb")); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + } +} diff --git a/SW.Serverless.UnitTests/GrpcClassicTests.cs b/SW.Serverless.UnitTests/GrpcClassicTests.cs new file mode 100644 index 0000000..544ce33 --- /dev/null +++ b/SW.Serverless.UnitTests/GrpcClassicTests.cs @@ -0,0 +1,228 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; +using SW.CloudFiles.Extensions; +using SW.PrimitiveTypes; +using SW.Serverless.Resident; +using SW.Serverless.UnitTests.Fixtures; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; + +namespace SW.Serverless.UnitTests +{ + /// + /// A classic session — start, call, dispose, as Bitween runs handlers — with an adapter that + /// speaks gRPC: what every adapter in a language other than .NET does, and a .NET adapter + /// whose manifest opts in with protocol 2. IServerlessService is unchanged for the caller; the + /// session runs on a resident instance of its own. + /// + [TestClass] + public class GrpcClassicTests + { + const string Project = "SW.Serverless.UnitTests.GrpcClassicAdapter"; + const string AdapterId = "test.grpcclassic"; + + static IHost host; + static string workDirectory; + + [ClassInitialize] + public static async Task ClassInitialize(TestContext context) + { + workDirectory = Path.Combine(Path.GetTempPath(), "swsl-grpcclassic", Guid.NewGuid().ToString("N")); + host = Host.CreateDefaultBuilder() + .ConfigureLogging(l => l.ClearProviders()) + .ConfigureServices(s => + { + s.AddLocalTestsCloudFiles(o => o.BucketName = TestStore.BucketName + "-grpcclassic"); + s.AddServerless(o => + { + o.AdapterRemotePath = "adapters"; + o.AdapterLocalPath = Path.Combine(workDirectory, "installed"); + o.AdapterMetadataCacheDuration = 1; + o.CommandTimeout = 30; + }); + s.AddSingleton(); + s.AddSingleton(sp => sp.GetRequiredService()); + s.AddResidentAdapters(o => + { + o.SocketPath = $"/tmp/swsl-gc{Environment.ProcessId}.sock"; + o.PipeName = $"swsl-gc{Environment.ProcessId}"; + o.HeartbeatInterval = TimeSpan.FromSeconds(2); + o.HandshakeTimeout = TimeSpan.FromSeconds(60); + }); + }) + .Build(); + await host.StartAsync(); + + var output = TestStore.LocateBuildOutput(Project); + using var buffer = new MemoryStream(); + using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var file in Directory.EnumerateFiles(output, "*", SearchOption.AllDirectories)) + archive.CreateEntryFromFile(file, Path.GetRelativePath(output, file), CompressionLevel.Fastest); + using var writer = new StreamWriter(archive.CreateEntry("adapter.json").Open()); + writer.Write(new JObject + { + ["id"] = AdapterId, + ["entry"] = Project + ".dll", + ["lifecycle"] = "classic", + ["protocol"] = new JObject { ["min"] = 2, ["max"] = 2 }, + }.ToString()); + } + buffer.Position = 0; + await host.Services.GetRequiredService().WriteAsync(buffer, new WriteFileSettings + { + Key = $"adapters/{AdapterId}", + ContentType = "application/zip", + Metadata = new Dictionary + { + ["EntryAssembly"] = Project + ".dll", + ["Hash"] = "gc-" + Guid.NewGuid().ToString("N")[..12], + } + }); + } + + [ClassCleanup] + public static async Task ClassCleanup() + { + if (host != null) await host.StopAsync(); + host?.Dispose(); + try { Directory.Delete(workDirectory, true); } catch { } + } + + static IServerlessService Service() => host.Services.GetRequiredService(); + + static IEnumerable RunningKeys() => + host.Services.GetRequiredService().Describe().Select(h => h.InstanceKey); + + [TestMethod] + public async Task A_call_returns_the_adapter_s_answer_using_the_startup_values_it_was_given() + { + var service = Service(); + await service.StartAsync(AdapterId, "corr-1", new Dictionary { ["Prefix"] = "hi " }); + try + { + Assert.AreEqual("hi world", await service.InvokeAsync("Greet", "world")); + Assert.AreEqual("corr-1", await service.InvokeAsync("Correlation", null)); + Assert.AreEqual(5, await service.InvokeAsync("Add", new { A = 2, B = 3 })); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + + [TestMethod] + public async Task A_declared_default_applies_when_no_value_is_given() + { + var service = Service(); + await service.StartAsync(AdapterId, "corr-2"); + try + { + Assert.AreEqual("hello world", await service.InvokeAsync("Greet", "world")); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + + [TestMethod] + public async Task The_expected_startup_values_are_the_settings_it_declared() + { + var service = Service(); + await service.StartAsync(AdapterId, "corr-3"); + try + { + var expected = await service.GetExpectedStartupValues(); + + Assert.IsTrue(expected["Prefix"].Optional); + Assert.AreEqual("hello ", expected["Prefix"].Default); + Assert.IsFalse(expected["ApiKey"].Optional); + Assert.IsTrue(expected["ApiKey"].Private); + Assert.AreEqual("The partner's key.", expected["ApiKey"].Description); + Assert.IsNull(expected["ApiKey"].Default); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + + [TestMethod] + public async Task Its_handshake_names_its_language_kinds_and_contracts() + { + var before = RunningKeys().ToHashSet(); + var service = Service(); + await service.StartAsync(AdapterId, "corr-7"); + try + { + var described = host.Services.GetRequiredService().Describe() + .Single(h => !before.Contains(h.InstanceKey)); + Assert.AreEqual("dotnet", described.SdkLanguage); + CollectionAssert.AreEqual(new[] { "handler" }, described.Kinds.ToList()); + Assert.AreEqual(1, described.Contracts["bitween"]); + Assert.AreEqual("10.1.0", described.SdkVersion); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + + [TestMethod] + public async Task An_adapter_error_fails_the_call_with_its_message() + { + var service = Service(); + await service.StartAsync(AdapterId, "corr-4"); + try + { + var error = await Assert.ThrowsExceptionAsync(() => service.InvokeAsync("Fail", null)); + StringAssert.Contains(error.Message, "asked to fail"); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + + [TestMethod] + public async Task Disposing_the_session_stops_its_process() + { + var before = RunningKeys().ToHashSet(); + var service = Service(); + await service.StartAsync(AdapterId, "corr-5"); + var mine = RunningKeys().Except(before).Single(); + StringAssert.StartsWith(mine, "classic-"); + + ((IDisposable)service).Dispose(); + + CollectionAssert.DoesNotContain(RunningKeys().ToList(), mine); + } + + [TestMethod] + public async Task Without_the_resident_host_registered_the_error_says_what_to_register() + { + using var bare = new ServiceCollection() + .AddLogging(l => l.ClearProviders()) + // The same storage, so the package and its metadata are the ones the host published. + .AddSingleton(host.Services.GetRequiredService()) + .AddServerless(o => + { + o.AdapterRemotePath = "adapters"; + o.AdapterLocalPath = Path.Combine(workDirectory, "installed-bare"); + }) + .BuildServiceProvider(); + + var error = await Assert.ThrowsExceptionAsync(() => + bare.GetRequiredService().StartAsync(AdapterId, "corr-6")); + StringAssert.Contains(error.Message, "AddResidentAdapters"); + } + } +} diff --git a/SW.Serverless.UnitTests/InstallerSafetyTests.cs b/SW.Serverless.UnitTests/InstallerSafetyTests.cs index f941ad8..51ffc61 100644 --- a/SW.Serverless.UnitTests/InstallerSafetyTests.cs +++ b/SW.Serverless.UnitTests/InstallerSafetyTests.cs @@ -3,6 +3,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using SW.CloudFiles.Extensions; using SW.PrimitiveTypes; +using SW.Serverless.Runtimes; using System; using System.Collections.Generic; using System.IO; @@ -54,10 +55,27 @@ public static void ClassCleanup() try { Directory.Delete(Path.GetDirectoryName(localRoot)!, true); } catch { } } - static AdapterInstaller Installer() => new( + static AdapterInstaller Installer(AdapterRuntimeOptions runtimes = null) => new( services.GetRequiredService(), new MemoryCache(new MemoryCacheOptions()), - cloudFiles); + cloudFiles, + new AdapterRuntimes(runtimes)); + + /// A host whose interpreters are at paths that don't exist, whatever this machine has. + static AdapterRuntimeOptions NoInterpreters => new() + { + PythonExecutable = "/nonexistent/python3", + NodeExecutable = "/nonexistent/node", + }; + + /// A host whose "python" answers with a fixed version, so version checks don't depend on this machine. + static AdapterRuntimeOptions PythonReporting(string version) + { + var script = Path.Combine(Path.GetTempPath(), $"fake-python-{Guid.NewGuid():N}.sh"); + File.WriteAllText(script, $"#!/bin/sh\necho \"Python {version}\"\n"); + File.SetUnixFileMode(script, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + return new AdapterRuntimeOptions { PythonExecutable = script }; + } static async Task Publish(string adapterId, IDictionary entries, string hash, string entryAssembly = "Adapter.dll", string key = null) @@ -207,6 +225,19 @@ public async Task An_adapter_needing_a_newer_host_is_refused_with_both_versions( [TestMethod] public async Task A_runtime_the_host_cannot_launch_is_refused_by_name() + { + await Publish("needs-ruby", new Dictionary + { + ["main.rb"] = "puts", + ["adapter.json"] = """{ "runtime": "ruby", "entry": "main.rb" }""" + }, hash: "rb1", entryAssembly: "main.rb"); + + var error = await Assert.ThrowsExceptionAsync(() => Installer().InstallAsync("needs-ruby")); + StringAssert.Contains(error.Message, "'ruby'"); + } + + [TestMethod] + public async Task A_known_runtime_missing_from_the_host_is_refused_saying_so() { await Publish("needs-python", new Dictionary { @@ -214,8 +245,71 @@ public async Task A_runtime_the_host_cannot_launch_is_refused_by_name() ["adapter.json"] = """{ "runtime": "python", "entry": "main.py" }""" }, hash: "py1", entryAssembly: "main.py"); - var error = await Assert.ThrowsExceptionAsync(() => Installer().InstallAsync("needs-python")); - StringAssert.Contains(error.Message, "python"); + var error = await Assert.ThrowsExceptionAsync(() => + Installer(NoInterpreters).InstallAsync("needs-python")); + StringAssert.Contains(error.Message, "'python'"); + StringAssert.Contains(error.Message, "not installed"); + } + + [TestMethod] + public async Task A_runtime_version_outside_the_manifest_s_range_is_refused_and_one_inside_installs() + { + await Publish("needs-python312", new Dictionary + { + ["main.py"] = "print()", + ["adapter.json"] = """{ "runtime": "python", "runtimeVersion": ">=3.12", "entry": "main.py" }""" + }, hash: "py312", entryAssembly: "main.py"); + + var error = await Assert.ThrowsExceptionAsync(() => + Installer(PythonReporting("3.11.9")).InstallAsync("needs-python312")); + StringAssert.Contains(error.Message, ">=3.12"); + StringAssert.Contains(error.Message, "3.11.9"); + + var installed = await Installer(PythonReporting("3.12.4")).InstallAsync("needs-python312"); + Assert.AreEqual("python", installed.Runtime); + Assert.IsTrue(installed.LocalPath.EndsWith("main.py")); + } + + [TestMethod] + public async Task A_package_for_other_platforms_is_refused_and_one_for_this_platform_runs_its_own_entry() + { + await Publish("elsewhere", new Dictionary + { + ["adapter"] = "binary", + ["adapter.json"] = """{ "runtime": "exec", "entry": "adapter", "platforms": ["plan9-mips"] }""" + }, hash: "ex1", entryAssembly: "adapter"); + var error = await Assert.ThrowsExceptionAsync(() => Installer().InstallAsync("elsewhere")); + StringAssert.Contains(error.Message, "plan9-mips"); + + var here = AdapterRuntimes.CurrentPlatform; + await Publish("everywhere", new Dictionary + { + ["default/adapter"] = "default build", + [$"{here}/adapter"] = "this platform's build", + ["adapter.json"] = $$"""{ "runtime": "exec", "entry": "default/adapter", "platforms": ["{{here}}", "plan9-mips"], "entries": { "{{here}}": "{{here}}/adapter" } }""" + }, hash: "ex2", entryAssembly: "default/adapter"); + + var installed = await Installer().InstallAsync("everywhere"); + Assert.AreEqual("this platform's build", File.ReadAllText(installed.LocalPath)); + if (!OperatingSystem.IsWindows()) + Assert.IsTrue(File.GetUnixFileMode(installed.LocalPath).HasFlag(UnixFileMode.UserExecute), + "an exec entry must be executable after unpacking"); + } + + [TestMethod] + public async Task A_dotnet_adapter_installs_without_asking_whether_dotnet_is_there() + { + // As before runtimes: production images carry the runtime without the SDK, and a + // .NET adapter was never refused for want of a check. + await Publish("plain-dotnet", new Dictionary + { + ["Adapter.dll"] = "assembly", + ["adapter.json"] = """{ "runtime": "dotnet", "entry": "Adapter.dll" }""" + }, hash: "dn1"); + + var installed = await Installer(new AdapterRuntimeOptions { DotnetExecutable = "/nonexistent/dotnet" }) + .InstallAsync("plain-dotnet"); + Assert.AreEqual("dotnet", installed.Runtime); } [TestMethod] diff --git a/SW.Serverless.UnitTests/RuntimeVersionRangeTests.cs b/SW.Serverless.UnitTests/RuntimeVersionRangeTests.cs new file mode 100644 index 0000000..61e1eef --- /dev/null +++ b/SW.Serverless.UnitTests/RuntimeVersionRangeTests.cs @@ -0,0 +1,28 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Serverless.Runtimes; + +namespace SW.Serverless.UnitTests +{ + [TestClass] + public class RuntimeVersionRangeTests + { + [DataTestMethod] + [DataRow(null, "3.11.0", true)] + [DataRow(">=3.12", "3.12.0", true)] + [DataRow(">=3.12", "3.13.1", true)] + [DataRow(">=3.12", "3.11.9", false)] + [DataRow(">=22, <24", "22.9.0", true)] + [DataRow(">=22, <24", "24.0.0", false)] + [DataRow(">=22 <24", "23.1.0", true)] + [DataRow("3.12", "3.12.7", true)] + [DataRow("3.12", "3.13.0", false)] + [DataRow("=22", "22.11.0", true)] + [DataRow(">3.12.0", "3.12.0", false)] + [DataRow("<=3.12", "3.12.0", true)] + [DataRow("v22", "22.1.0", true)] + [DataRow("lots", "3.12.0", false)] + [DataRow(">=3.12", "not-a-version", false)] + public void A_runtime_version_is_in_or_out_of_range(string range, string version, bool expected) => + Assert.AreEqual(expected, RuntimeVersionRange.Satisfies(range, version), $"{version} in '{range}'"); + } +} diff --git a/SW.Serverless.UnitTests/SW.Serverless.UnitTests.csproj b/SW.Serverless.UnitTests/SW.Serverless.UnitTests.csproj index 583f3b4..a8e183c 100644 --- a/SW.Serverless.UnitTests/SW.Serverless.UnitTests.csproj +++ b/SW.Serverless.UnitTests/SW.Serverless.UnitTests.csproj @@ -28,6 +28,8 @@ + + /// A classic adapter's startup values hold passwords and keys. On the command line, as they + /// have always gone, any process on the machine can read them. An adapter built on an SDK + /// that reads them from stdin gets them there; one built before still gets them as arguments. + /// + [TestClass] + public class StartupValuesOnStdinTests + { + const string Project = "SW.Serverless.UnitTests.Adapter"; + const string Secret = "s3cret-on-stdin-only"; + + static IHost host; + static string workDirectory; + + [ClassInitialize] + public static async Task ClassInitialize(TestContext context) + { + workDirectory = Path.Combine(Path.GetTempPath(), "swsl-stdin", Guid.NewGuid().ToString("N")); + host = Host.CreateDefaultBuilder() + .ConfigureLogging(l => l.ClearProviders()) + .ConfigureServices(s => + { + s.AddLocalTestsCloudFiles(o => o.BucketName = TestStore.BucketName + "-stdin"); + s.AddServerless(o => + { + o.AdapterRemotePath = "adapters"; + o.AdapterLocalPath = Path.Combine(workDirectory, "installed"); + o.AdapterMetadataCacheDuration = 1; + o.CommandTimeout = 30; + }); + }) + .Build(); + await host.StartAsync(); + + await PublishAsync("test.stdin.new", sdkVersion: "10.1.0"); + await PublishAsync("test.stdin.old", sdkVersion: null); + } + + static async Task PublishAsync(string adapterId, string sdkVersion) + { + var output = TestStore.LocateBuildOutput(Project); + using var buffer = new MemoryStream(); + using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var file in Directory.EnumerateFiles(output, "*", SearchOption.AllDirectories)) + archive.CreateEntryFromFile(file, Path.GetRelativePath(output, file), CompressionLevel.Fastest); + if (sdkVersion != null) + using (var writer = new StreamWriter(archive.CreateEntry("adapter.json").Open())) + writer.Write(new JObject { ["id"] = adapterId, ["entry"] = Project + ".dll", ["sdkVersion"] = sdkVersion }.ToString()); + } + buffer.Position = 0; + await host.Services.GetRequiredService().WriteAsync(buffer, new WriteFileSettings + { + Key = $"adapters/{adapterId}", + ContentType = "application/zip", + Metadata = new Dictionary + { + ["EntryAssembly"] = Project + ".dll", + ["Hash"] = adapterId.Replace('.', '-') + "-" + Guid.NewGuid().ToString("N")[..8], + } + }); + } + + [ClassCleanup] + public static async Task ClassCleanup() + { + if (host != null) await host.StopAsync(); + host?.Dispose(); + try { Directory.Delete(workDirectory, true); } catch { } + } + + /// The command lines of this machine's running adapter processes, as any process can read them. + static List AdapterCommandLines() + { + using var ps = Process.Start(new ProcessStartInfo("ps", "-axo args") { RedirectStandardOutput = true })!; + var lines = ps.StandardOutput.ReadToEnd().Split('\n'); + ps.WaitForExit(); + return lines.Where(l => l.Contains(Project + ".dll") && l.Contains(workDirectory)).ToList(); + } + + static async Task<(string Password, List CommandLines)> RunAsync(string adapterId) + { + var service = host.Services.GetRequiredService(); + await service.StartAsync(adapterId, "corr", new Dictionary { ["Password"] = Secret }); + try + { + var password = await service.InvokeAsync("TestStartupValue", "Password"); + return (password, AdapterCommandLines()); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + + [TestMethod] + public async Task An_adapter_on_the_newer_SDK_gets_its_values_on_stdin_and_none_on_its_command_line() + { + if (OperatingSystem.IsWindows()) Assert.Inconclusive("reads command lines with ps"); + + var (password, commandLines) = await RunAsync("test.stdin.new"); + + Assert.AreEqual(Secret, password, "the value reached the adapter"); + var line = commandLines.Single(); + StringAssert.Contains(line, "--values-on-stdin"); + Assert.AreEqual(3, line.Split(' ', StringSplitOptions.RemoveEmptyEntries).Length, + $"nothing but dotnet, the assembly and the flag on the command line: {line}"); + } + + [TestMethod] + public async Task An_adapter_from_before_still_gets_its_values_as_arguments() + { + if (OperatingSystem.IsWindows()) Assert.Inconclusive("reads command lines with ps"); + + var (password, commandLines) = await RunAsync("test.stdin.old"); + + Assert.AreEqual(Secret, password); + var line = commandLines.Single(); + Assert.IsFalse(line.Contains("--values-on-stdin"), line); + Assert.AreEqual(5, line.Split(' ', StringSplitOptions.RemoveEmptyEntries).Length, + $"dotnet, the assembly and the three base64 values, as before: {line}"); + } + } +} diff --git a/SW.Serverless.sln b/SW.Serverless.sln index cd717b4..8f2b7f4 100644 --- a/SW.Serverless.sln +++ b/SW.Serverless.sln @@ -55,6 +55,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.Compat.Reside EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.CompatibilityTests", "SW.Serverless.CompatibilityTests\SW.Serverless.CompatibilityTests.csproj", "{8532437C-5983-48D0-AD79-666B5EA07617}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.Compat.ManifestV1002", "SW.Serverless.Compat.ManifestV1002\SW.Serverless.Compat.ManifestV1002.csproj", "{0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.UnitTests.GrpcClassicAdapter", "SW.Serverless.UnitTests.GrpcClassicAdapter\SW.Serverless.UnitTests.GrpcClassicAdapter.csproj", "{D77684BD-D24F-45AE-9B49-30278D6CCA3C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.Tooling", "SW.Serverless.Tooling\SW.Serverless.Tooling.csproj", "{C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.UnitTests.BitweenHandler", "SW.Serverless.UnitTests.BitweenHandler\SW.Serverless.UnitTests.BitweenHandler.csproj", "{DCF81A88-37CA-4746-A4D8-EADEF00422FD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.UnitTests.BitweenReceiver", "SW.Serverless.UnitTests.BitweenReceiver\SW.Serverless.UnitTests.BitweenReceiver.csproj", "{4D824901-1889-4BB2-9F72-2D33E5CFA7B3}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -341,6 +351,66 @@ Global {8532437C-5983-48D0-AD79-666B5EA07617}.Release|x64.Build.0 = Release|Any CPU {8532437C-5983-48D0-AD79-666B5EA07617}.Release|x86.ActiveCfg = Release|Any CPU {8532437C-5983-48D0-AD79-666B5EA07617}.Release|x86.Build.0 = Release|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Debug|x64.ActiveCfg = Debug|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Debug|x64.Build.0 = Debug|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Debug|x86.ActiveCfg = Debug|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Debug|x86.Build.0 = Debug|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Release|Any CPU.Build.0 = Release|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Release|x64.ActiveCfg = Release|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Release|x64.Build.0 = Release|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Release|x86.ActiveCfg = Release|Any CPU + {0C48B86B-F49C-43AB-A7EF-95D7C0368DE8}.Release|x86.Build.0 = Release|Any CPU + {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Debug|x64.ActiveCfg = Debug|Any CPU + {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Debug|x64.Build.0 = Debug|Any CPU + {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Debug|x86.ActiveCfg = Debug|Any CPU + {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Debug|x86.Build.0 = Debug|Any CPU + {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Release|Any CPU.Build.0 = Release|Any CPU + {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Release|x64.ActiveCfg = Release|Any CPU + {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Release|x64.Build.0 = Release|Any CPU + {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Release|x86.ActiveCfg = Release|Any CPU + {D77684BD-D24F-45AE-9B49-30278D6CCA3C}.Release|x86.Build.0 = Release|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Debug|x64.ActiveCfg = Debug|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Debug|x64.Build.0 = Debug|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Debug|x86.ActiveCfg = Debug|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Debug|x86.Build.0 = Debug|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|Any CPU.Build.0 = Release|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|x64.ActiveCfg = Release|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|x64.Build.0 = Release|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|x86.ActiveCfg = Release|Any CPU + {C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}.Release|x86.Build.0 = Release|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Debug|x64.ActiveCfg = Debug|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Debug|x64.Build.0 = Debug|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Debug|x86.ActiveCfg = Debug|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Debug|x86.Build.0 = Debug|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Release|Any CPU.Build.0 = Release|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Release|x64.ActiveCfg = Release|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Release|x64.Build.0 = Release|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Release|x86.ActiveCfg = Release|Any CPU + {DCF81A88-37CA-4746-A4D8-EADEF00422FD}.Release|x86.Build.0 = Release|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Debug|x64.ActiveCfg = Debug|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Debug|x64.Build.0 = Debug|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Debug|x86.ActiveCfg = Debug|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Debug|x86.Build.0 = Debug|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Release|Any CPU.Build.0 = Release|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Release|x64.ActiveCfg = Release|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Release|x64.Build.0 = Release|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Release|x86.ActiveCfg = Release|Any CPU + {4D824901-1889-4BB2-9F72-2D33E5CFA7B3}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/SW.Serverless/Extensions/IServiceCollectionExtensions.cs b/SW.Serverless/Extensions/IServiceCollectionExtensions.cs index 7924dd7..9524c56 100644 --- a/SW.Serverless/Extensions/IServiceCollectionExtensions.cs +++ b/SW.Serverless/Extensions/IServiceCollectionExtensions.cs @@ -17,11 +17,26 @@ public static IServiceCollection AddServerless(this IServiceCollection services, services.AddSingleton(serverlessOptions); services.AddTransient(); services.AddMemoryCache(); + services.TryAddSingleton(new Runtimes.AdapterRuntimeOptions()); + services.TryAddSingleton(); services.TryAddSingleton(); return services; } + /// + /// Where the host finds the interpreters adapters run on — python3, node and dotnet on the + /// PATH unless set. Call before AddServerless or AddResidentAdapters to take effect. + /// + public static IServiceCollection AddAdapterRuntimes(this IServiceCollection services, + Action configure) + { + var options = new Runtimes.AdapterRuntimeOptions(); + configure?.Invoke(options); + services.AddSingleton(options); + return services; + } + /// /// Adds the resident adapter runtime: a Kestrel endpoint on a Unix domain socket or named /// pipe that adapters dial, plus the supervisor that owns their processes. @@ -43,6 +58,8 @@ public static IServiceCollection AddResidentAdapters(this IServiceCollect // process, so a cursor saved on one node is invisible to the next one to run the // adapter. TryAdd, so a host that registered its own keeps it. services.TryAddSingleton(); + services.TryAddSingleton(new Runtimes.AdapterRuntimeOptions()); + services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.AddSingleton(); diff --git a/SW.Serverless/Resident/AdapterCommand.cs b/SW.Serverless/Resident/AdapterCommand.cs index b180f0c..c78fc1b 100644 --- a/SW.Serverless/Resident/AdapterCommand.cs +++ b/SW.Serverless/Resident/AdapterCommand.cs @@ -24,6 +24,12 @@ public class AdapterCommand /// public string ParameterSchema { get; set; } + /// The argument's JSON Schema, from an SDK in any language. Empty when not given. + public string InputSchema { get; set; } + + /// The result's JSON Schema, from an SDK in any language. Empty when not given. + public string OutputSchema { get; set; } + /// False for a command returning Task rather than Task<T>. public bool ReturnsValue { get; set; } @@ -35,4 +41,21 @@ public class AdapterCommand public override string ToString() => $"{Name}({ParameterType}){(ReturnsValue ? "" : " -> void")}"; } + + /// One setting an adapter says it reads, in any language. + public class AdapterSetting + { + public string Name { get; set; } + public string Description { get; set; } + public bool Required { get; set; } + + /// Masked wherever it's shown, never logged. + public bool Secret { get; set; } + + /// Empty when the setting has no default. + public string DefaultValue { get; set; } + + /// text, multiline, number, boolean, select or json. + public string Type { get; set; } + } } diff --git a/SW.Serverless/Resident/AdapterHostService.cs b/SW.Serverless/Resident/AdapterHostService.cs index fae5a83..3f91d51 100644 --- a/SW.Serverless/Resident/AdapterHostService.cs +++ b/SW.Serverless/Resident/AdapterHostService.cs @@ -63,12 +63,25 @@ public override async Task Attach(IAsyncStreamReader requestStream ParameterType = c.ParameterType, ParameterSchema = c.ParameterSchema, ReturnsValue = c.ReturnsValue, - Description = c.Description + Description = c.Description, + InputSchema = c.InputSchema, + OutputSchema = c.OutputSchema }) .OrderBy(c => c.Name) .ToArray(); instance.SdkVersion = first.Hello.SdkVersion; + // Empty from an SDK that predates them, which is fine: they describe, they don't decide. + instance.SdkLanguage = first.Hello.SdkLanguage; + instance.Settings = first.Hello.Settings + .Select(x => new AdapterSetting + { + Name = x.Name, Description = x.Description, Required = x.Required, Secret = x.Secret, + DefaultValue = x.DefaultValue, Type = x.Type + }) + .ToArray(); + instance.Kinds = first.Hello.Kinds.ToArray(); + instance.Contracts = first.Hello.Contracts.ToDictionary(kv => kv.Key, kv => kv.Value); instance.ProtocolVersion = first.Hello.ProtocolVersion; logger.LogInformation( diff --git a/SW.Serverless/Resident/AdapterProcessLauncher.cs b/SW.Serverless/Resident/AdapterProcessLauncher.cs index 1009e28..09b1d88 100644 --- a/SW.Serverless/Resident/AdapterProcessLauncher.cs +++ b/SW.Serverless/Resident/AdapterProcessLauncher.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.Logging; +using SW.Serverless.Runtimes; using SW.Serverless.Sdk.Resident; using System; using System.Diagnostics; @@ -15,11 +16,13 @@ internal class AdapterProcessLauncher { readonly ResidentOptions options; readonly ILogger logger; + readonly AdapterRuntimes runtimes; - public AdapterProcessLauncher(ResidentOptions options, ILogger logger) + public AdapterProcessLauncher(ResidentOptions options, ILogger logger, AdapterRuntimes runtimes) { this.options = options; this.logger = logger; + this.runtimes = runtimes; } public Process Launch(AdapterSpec spec, ResidentAdapterInstance instance) @@ -31,28 +34,42 @@ public Process Launch(AdapterSpec spec, ResidentAdapterInstance instance) if (!File.Exists(assembly)) throw new FileNotFoundException($"Adapter entry assembly not found.", assembly); - var executable = spec.Executable ?? "dotnet"; - var arguments = executable == "dotnet" ? $"\"{assembly}\"" : ""; - - var psi = new ProcessStartInfo(executable) - { - Arguments = arguments, - WorkingDirectory = Path.GetDirectoryName(assembly), - UseShellExecute = false, - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true - }; - - // Workstation GC: server GC allocates a heap and a GC thread per core, which is the - // wrong trade for an adapter that mostly waits on a socket. - if (options.UseWorkstationGc) psi.Environment["DOTNET_gcServer"] = "0"; - var hard = spec.HardMemoryLimitBytes > 0 ? spec.HardMemoryLimitBytes : options.HardMemoryLimitBytes; - if (hard > 0) + ProcessStartInfo psi; + if (!string.IsNullOrWhiteSpace(spec.Executable)) + { + // An executable set by the caller or the package's metadata, as before runtimes: + // dotnet with the assembly, anything else started bare. + var arguments = spec.Executable == "dotnet" ? $"\"{assembly}\"" : ""; + psi = new ProcessStartInfo(spec.Executable) + { + Arguments = arguments, + WorkingDirectory = Path.GetDirectoryName(assembly), + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + // Workstation GC: server GC allocates a heap and a GC thread per core, which is the + // wrong trade for an adapter that mostly waits on a socket. + if (options.UseWorkstationGc) psi.Environment["DOTNET_gcServer"] = "0"; + if (hard > 0) + { + psi.Environment["DOTNET_GCHeapHardLimit"] = hard.ToString("X"); + psi.Environment["DOTNET_GCConserveMemory"] = "5"; + } + } + else { - psi.Environment["DOTNET_GCHeapHardLimit"] = hard.ToString("X"); - psi.Environment["DOTNET_GCConserveMemory"] = "5"; + var runtime = runtimes.Find(spec.Runtime) + ?? throw new NotSupportedException( + $"Adapter '{spec.AdapterId}' needs the '{spec.Runtime}' runtime, which this host does not have."); + psi = runtime.StartInfo(assembly, new RuntimeLaunch + { + HardMemoryLimitBytes = hard, + UseWorkstationGc = options.UseWorkstationGc, + }); } var process = new Process { StartInfo = psi, EnableRaisingEvents = true }; diff --git a/SW.Serverless/Resident/AdapterSpec.cs b/SW.Serverless/Resident/AdapterSpec.cs index 8c1e5ca..1626615 100644 --- a/SW.Serverless/Resident/AdapterSpec.cs +++ b/SW.Serverless/Resident/AdapterSpec.cs @@ -19,9 +19,19 @@ public class AdapterSpec /// public string EntryAssemblyPath { get; set; } - /// Executable to launch. Defaults to `dotnet`; set for self-contained or non-.NET adapters. + /// + /// Executable to launch, overriding the runtime. Kept for callers that set it before + /// runtimes existed; an adapter installed from storage gets its launcher from + /// instead. + /// public string Executable { get; set; } + /// + /// The runtime it runs on — dotnet, exec, python, node — as its manifest names it. Filled + /// in when the adapter is resolved; empty means dotnet. + /// + public string Runtime { get; set; } + /// /// Which warm pool a POOLED rental belongs to. Left null the host derives one from the /// adapter id and a hash of , so that two configurations of the diff --git a/SW.Serverless/Resident/IResidentAdapterLocator.cs b/SW.Serverless/Resident/IResidentAdapterLocator.cs index c0de9be..4d9bb0a 100644 --- a/SW.Serverless/Resident/IResidentAdapterLocator.cs +++ b/SW.Serverless/Resident/IResidentAdapterLocator.cs @@ -17,6 +17,10 @@ public class ResolvedAdapter { public string EntryAssemblyPath { get; set; } public string Executable { get; set; } + + /// The runtime it runs on, from its manifest; empty means dotnet. + public string Runtime { get; set; } + public System.Collections.Generic.IDictionary AdapterValues { get; set; } /// The extraction directory, when installed from storage. Held while the adapter runs. @@ -36,6 +40,7 @@ public async Task ResolveAsync(AdapterSpec spec, CancellationTo { EntryAssemblyPath = spec.EntryAssemblyPath, Executable = spec.Executable, + Runtime = spec.Runtime, AdapterValues = spec.AdapterValues }; @@ -53,6 +58,7 @@ public async Task ResolveAsync(AdapterSpec spec, CancellationTo EntryAssemblyPath = installed.LocalPath, Directory = installed.Directory, Executable = spec.Executable ?? (values.TryGetValue("Executable", out var exe) ? exe : null), + Runtime = installed.Runtime, AdapterValues = values }; } diff --git a/SW.Serverless/Resident/InstanceHealth.cs b/SW.Serverless/Resident/InstanceHealth.cs index 5900d51..7d00a19 100644 --- a/SW.Serverless/Resident/InstanceHealth.cs +++ b/SW.Serverless/Resident/InstanceHealth.cs @@ -55,6 +55,10 @@ public class InstanceHealth public IReadOnlyCollection CommandDetails { get; set; } = Array.Empty(); public string SdkVersion { get; set; } + public string SdkLanguage { get; set; } + public IReadOnlyCollection Settings { get; set; } + public IReadOnlyCollection Kinds { get; set; } + public IReadOnlyDictionary Contracts { get; set; } public int ProtocolVersion { get; set; } public IReadOnlyDictionary StartupValues { get; set; } diff --git a/SW.Serverless/Resident/ResidentAdapterHost.cs b/SW.Serverless/Resident/ResidentAdapterHost.cs index 8d543a0..04740fb 100644 --- a/SW.Serverless/Resident/ResidentAdapterHost.cs +++ b/SW.Serverless/Resident/ResidentAdapterHost.cs @@ -44,7 +44,8 @@ public class ResidentAdapterHost : IResidentAdapterHost, IHostedService, IAsyncD readonly IResidentAdapterLocator locator; public ResidentAdapterHost(ResidentOptions options, IAdapterEventSink sink, - IAdapterStateStore stateStore, IResidentAdapterLocator locator, ILoggerFactory loggerFactory) + IAdapterStateStore stateStore, IResidentAdapterLocator locator, ILoggerFactory loggerFactory, + Runtimes.AdapterRuntimes runtimes = null) { this.options = options; this.sink = sink; @@ -52,7 +53,7 @@ public ResidentAdapterHost(ResidentOptions options, IAdapterEventSink sink, this.locator = locator; this.loggerFactory = loggerFactory; logger = loggerFactory.CreateLogger(); - launcher = new AdapterProcessLauncher(options, logger); + launcher = new AdapterProcessLauncher(options, logger, runtimes ?? new Runtimes.AdapterRuntimes(null)); } // ------------------------------------------------------------------ transport @@ -197,6 +198,7 @@ async Task SpawnAsync(Supervised supervised, CancellationToken cancellationToken var resolved = await locator.ResolveAsync(spec, cancellationToken); spec.EntryAssemblyPath = resolved.EntryAssemblyPath; spec.Executable = resolved.Executable ?? spec.Executable; + spec.Runtime = resolved.Runtime ?? spec.Runtime; if (resolved.AdapterValues != null) spec.AdapterValues = resolved.AdapterValues; var instance = new ResidentAdapterInstance( @@ -609,6 +611,10 @@ static InstanceHealth Describe(Supervised supervised) Commands = instance.Commands, CommandDetails = instance.CommandDetails, SdkVersion = instance.SdkVersion, + SdkLanguage = instance.SdkLanguage, + Settings = instance.Settings, + Kinds = instance.Kinds, + Contracts = instance.Contracts, ProtocolVersion = instance.ProtocolVersion, // Names only. These are connection strings and passwords, and health is what gets // shown on screens and serialised into logs. diff --git a/SW.Serverless/Resident/ResidentAdapterInstance.cs b/SW.Serverless/Resident/ResidentAdapterInstance.cs index 8268b20..91c16af 100644 --- a/SW.Serverless/Resident/ResidentAdapterInstance.cs +++ b/SW.Serverless/Resident/ResidentAdapterInstance.cs @@ -111,6 +111,18 @@ internal ResidentAdapterInstance(string adapterId, string instanceKey, string to internal IDisposable DirectoryLease { get; set; } public string SdkVersion { get; internal set; } + + /// The SDK's language — dotnet, python, node, go — or empty from an SDK that predates it. + public string SdkLanguage { get; internal set; } + + /// The settings the adapter says it reads; empty from an SDK that predates them. + public IReadOnlyCollection Settings { get; internal set; } = Array.Empty(); + + /// The kinds it implements for a contract, e.g. handler or receiver for Bitween. + public IReadOnlyCollection Kinds { get; internal set; } = Array.Empty(); + + /// The contracts it implements and their versions, e.g. bitween → 1. + public IReadOnlyDictionary Contracts { get; internal set; } = new Dictionary(); public int ProtocolVersion { get; internal set; } public IReadOnlyDictionary AdapterValues { get; internal set; } diff --git a/SW.Serverless/Runtimes/AdapterRuntimes.cs b/SW.Serverless/Runtimes/AdapterRuntimes.cs new file mode 100644 index 0000000..bdbec37 --- /dev/null +++ b/SW.Serverless/Runtimes/AdapterRuntimes.cs @@ -0,0 +1,236 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using SW.Serverless.Contract.Catalog; + +namespace SW.Serverless.Runtimes +{ + /// Where the host finds the interpreters its adapters run on. + public class AdapterRuntimeOptions + { + /// The Python interpreter. "python3" on the PATH unless set. + public string PythonExecutable { get; set; } = "python3"; + + /// The Node.js executable. "node" on the PATH unless set. + public string NodeExecutable { get; set; } = "node"; + + /// The .NET host. "dotnet" on the PATH unless set. + public string DotnetExecutable { get; set; } = "dotnet"; + } + + /// Whether a runtime is present on this host, and which version. + public sealed record RuntimeStatus(bool Available, string Version, string Reason) + { + public static RuntimeStatus Present(string version) => new(true, version, null); + public static RuntimeStatus Missing(string reason) => new(false, null, reason); + } + + /// What a launch is asked to apply, beside the entry it starts. + public sealed class RuntimeLaunch + { + /// A memory ceiling the runtime enforces itself, where it can. 0 for none. + public long HardMemoryLimitBytes { get; set; } + + /// .NET only: workstation GC, the right trade for an adapter that mostly waits. + public bool UseWorkstationGc { get; set; } + + /// Arguments after the entry, such as --describe. + public System.Collections.Generic.IReadOnlyList Arguments { get; set; } = Array.Empty(); + } + + /// + /// One runtime adapters can be written for: how to tell it's here, and how to start an adapter + /// on it. Named in an adapter's manifest; never a path a package can choose. + /// + public interface IAdapterRuntime + { + /// As manifests name it: dotnet, exec, python, node. + string Name { get; } + + Task DetectAsync(); + + /// How to start on this runtime, with its stdio redirected. + ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch); + } + + /// + /// The runtimes this host knows, and which of them it actually has. Detection runs each + /// runtime's version check once and remembers the answer for the life of the process. + /// + public class AdapterRuntimes + { + readonly Dictionary runtimes; + readonly ConcurrentDictionary>> detected = new(StringComparer.OrdinalIgnoreCase); + + public AdapterRuntimes(AdapterRuntimeOptions options) + { + options ??= new AdapterRuntimeOptions(); + runtimes = new IAdapterRuntime[] + { + new DotnetRuntime(options.DotnetExecutable), + new ExecRuntime(), + new PythonRuntime(options.PythonExecutable), + new NodeRuntime(options.NodeExecutable), + } + .ToDictionary(r => r.Name, StringComparer.OrdinalIgnoreCase); + } + + public IReadOnlyCollection Known => runtimes.Keys; + + /// The runtime named, or null when this host has no launcher for it. + public IAdapterRuntime Find(string name) => + runtimes.TryGetValue(string.IsNullOrWhiteSpace(name) ? AdapterManifest.DotnetRuntime : name, out var runtime) + ? runtime + : null; + + public Task StatusAsync(string name) + { + var runtime = Find(name); + if (runtime == null) return Task.FromResult(RuntimeStatus.Missing($"this host has no launcher for '{name}'")); + return detected.GetOrAdd(runtime.Name, _ => new Lazy>(runtime.DetectAsync)).Value; + } + + /// This host's platform as manifests name platforms: linux-x64, osx-arm64, win-x64… + public static string CurrentPlatform + { + get + { + var os = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "win" + : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? "osx" + : "linux"; + var arch = RuntimeInformation.OSArchitecture switch + { + Architecture.X64 => "x64", + Architecture.Arm64 => "arm64", + Architecture.X86 => "x86", + Architecture.Arm => "arm", + var other => other.ToString().ToLowerInvariant(), + }; + return $"{os}-{arch}"; + } + } + + /// Runs with and reads the first version in what it prints. + internal static async Task VersionOf(string executable, string arguments) + { + try + { + using var process = Process.Start(new ProcessStartInfo(executable, arguments) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }); + if (process == null) return RuntimeStatus.Missing($"'{executable}' could not be started"); + var output = await process.StandardOutput.ReadToEndAsync() + await process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(15_000)) + { + try { process.Kill(true); } catch { } + return RuntimeStatus.Missing($"'{executable} {arguments}' did not answer"); + } + var version = Regex.Match(output, @"\d+\.\d+(\.\d+)?"); + return process.ExitCode == 0 && version.Success + ? RuntimeStatus.Present(version.Value) + : RuntimeStatus.Missing($"'{executable} {arguments}' answered: {output.Trim()}"); + } + catch (Exception ex) when (ex is System.ComponentModel.Win32Exception or FileNotFoundException) + { + return RuntimeStatus.Missing($"'{executable}' is not installed on this host"); + } + } + + static ProcessStartInfo Redirected(string executable, string workingDirectory) => new(executable) + { + WorkingDirectory = workingDirectory, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + + sealed class DotnetRuntime(string executable) : IAdapterRuntime + { + public string Name => AdapterManifest.DotnetRuntime; + + // --list-runtimes, not --version: production images carry the runtime without the SDK, + // and there --version fails. The first line names the oldest runtime installed. + public Task DetectAsync() => VersionOf(executable, "--list-runtimes"); + + public ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch) + { + var psi = Redirected(executable, Path.GetDirectoryName(entryPath)); + psi.ArgumentList.Add(entryPath); + foreach (var argument in launch.Arguments) psi.ArgumentList.Add(argument); + if (launch.UseWorkstationGc) psi.Environment["DOTNET_gcServer"] = "0"; + if (launch.HardMemoryLimitBytes > 0) + { + psi.Environment["DOTNET_GCHeapHardLimit"] = launch.HardMemoryLimitBytes.ToString("X"); + psi.Environment["DOTNET_GCConserveMemory"] = "5"; + } + return psi; + } + } + + /// A self-contained executable: the entry itself is started. + sealed class ExecRuntime : IAdapterRuntime + { + public string Name => AdapterManifest.ExecRuntime; + + // Nothing to install: the package carries everything it needs. + public Task DetectAsync() => Task.FromResult(RuntimeStatus.Present("native")); + + public ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch) + { + var psi = Redirected(entryPath, Path.GetDirectoryName(entryPath)); + foreach (var argument in launch.Arguments) psi.ArgumentList.Add(argument); + return psi; + } + } + + sealed class PythonRuntime(string executable) : IAdapterRuntime + { + public string Name => AdapterManifest.PythonRuntime; + + public Task DetectAsync() => VersionOf(executable, "--version"); + + public ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch) + { + var psi = Redirected(executable, Path.GetDirectoryName(entryPath)); + // Unbuffered, so what it writes reaches the host's diagnostics when it's written. + psi.ArgumentList.Add("-u"); + psi.ArgumentList.Add(entryPath); + foreach (var argument in launch.Arguments) psi.ArgumentList.Add(argument); + psi.Environment["PYTHONUNBUFFERED"] = "1"; + psi.Environment["PYTHONDONTWRITEBYTECODE"] = "1"; + // Python has no heap ceiling of its own; the watchdog and, where the container + // allows them, cgroups are what hold it. + return psi; + } + } + + sealed class NodeRuntime(string executable) : IAdapterRuntime + { + public string Name => AdapterManifest.NodeRuntime; + + public Task DetectAsync() => VersionOf(executable, "--version"); + + public ProcessStartInfo StartInfo(string entryPath, RuntimeLaunch launch) + { + var psi = Redirected(executable, Path.GetDirectoryName(entryPath)); + if (launch.HardMemoryLimitBytes > 0) + // V8's own heap ceiling, in megabytes: an allocation past it fails inside the + // adapter, as DOTNET_GCHeapHardLimit does for .NET. + psi.ArgumentList.Add($"--max-old-space-size={Math.Max(16, launch.HardMemoryLimitBytes / (1024 * 1024))}"); + psi.ArgumentList.Add(entryPath); + foreach (var argument in launch.Arguments) psi.ArgumentList.Add(argument); + return psi; + } + } + } +} diff --git a/SW.Serverless/Runtimes/RuntimeVersionRange.cs b/SW.Serverless/Runtimes/RuntimeVersionRange.cs new file mode 100644 index 0000000..dcb21e0 --- /dev/null +++ b/SW.Serverless/Runtimes/RuntimeVersionRange.cs @@ -0,0 +1,58 @@ +using System; +using System.Linq; +using System.Text.RegularExpressions; + +namespace SW.Serverless.Runtimes +{ + /// + /// A manifest's runtimeVersion: comparisons joined by commas or spaces — ">=3.12", + /// ">=22, <24", "3.12" for exactly that minor line. Empty means any version. + /// + public static class RuntimeVersionRange + { + static readonly Regex Part = new(@"^(>=|<=|>|<|=|==)?\s*v?(\d+(?:\.\d+){0,2})$"); + + /// Whether is in ; a range that can't be read is not satisfied. + public static bool Satisfies(string range, string version) + { + if (string.IsNullOrWhiteSpace(range)) return true; + if (!TryVersion(version, out var actual)) return false; + + foreach (var raw in Regex.Split(range.Trim(), @"\s*,\s*|\s+(?=[<>=])")) + { + var match = Part.Match(raw.Trim()); + if (!match.Success || !TryVersion(match.Groups[2].Value, out var bound)) return false; + var length = match.Groups[2].Value.Split('.').Length; + var ok = match.Groups[1].Value switch + { + ">=" => actual >= bound, + ">" => actual > bound, + "<=" => actual <= bound, + "<" => actual < bound, + // A bare or "=" version matches every release that starts with it: 3.12 is any 3.12.x. + _ => Prefix(actual, length) == Prefix(bound, length), + }; + if (!ok) return false; + } + return true; + } + + public static bool IsReadable(string range) => + string.IsNullOrWhiteSpace(range) || + Regex.Split(range.Trim(), @"\s*,\s*|\s+(?=[<>=])").All(p => Part.IsMatch(p.Trim())); + + static bool TryVersion(string text, out Version version) + { + var parts = (text ?? "").Split('.'); + while (parts.Length < 3) parts = parts.Append("0").ToArray(); + return Version.TryParse(string.Join('.', parts.Take(3)), out version); + } + + static Version Prefix(Version v, int length) => length switch + { + 1 => new Version(v.Major, 0, 0), + 2 => new Version(v.Major, v.Minor, 0), + _ => new Version(v.Major, v.Minor, Math.Max(0, v.Build)), + }; + } +} diff --git a/SW.Serverless/Services/AdapterInstaller.cs b/SW.Serverless/Services/AdapterInstaller.cs index 0de886a..b8d19f0 100644 --- a/SW.Serverless/Services/AdapterInstaller.cs +++ b/SW.Serverless/Services/AdapterInstaller.cs @@ -1,3 +1,4 @@ +using SW.Serverless.Runtimes; using Microsoft.Extensions.Caching.Memory; using SW.PrimitiveTypes; using SW.Serverless.Contract.Catalog; @@ -25,13 +26,15 @@ public class AdapterInstaller readonly ServerlessOptions options; readonly IMemoryCache memoryCache; readonly ICloudFilesService cloudFilesService; + readonly AdapterRuntimes runtimes; public AdapterInstaller(ServerlessOptions options, IMemoryCache memoryCache, - ICloudFilesService cloudFilesService) + ICloudFilesService cloudFilesService, AdapterRuntimes runtimes = null) { this.options = options; this.memoryCache = memoryCache; this.cloudFilesService = cloudFilesService; + this.runtimes = runtimes ?? new AdapterRuntimes(null); } /// @@ -61,7 +64,7 @@ public async Task InstallAsync(string adapterId) PruneSupersededVersions(adapterId, directory); - ApplyManifest(metadata); + await ApplyManifestAsync(metadata); return metadata; } @@ -71,7 +74,7 @@ public async Task InstallAsync(string adapterId) /// the package, while metadata can be changed without touching it. A package without a /// manifest — every package published before them — is left exactly as its metadata says. /// - void ApplyManifest(InstalledAdapter installed) + async Task ApplyManifestAsync(InstalledAdapter installed) { if (installed.Manifest != null || string.IsNullOrEmpty(installed.Directory)) return; @@ -89,25 +92,54 @@ void ApplyManifest(InstalledAdapter installed) $"Adapter '{installed.AdapterId}' has an {AdapterManifest.FileName} that cannot be read: {ex.Message}", ex); } - // A runtime this host has no launcher for is refused now, with its name, rather than - // started with dotnet and failing in a way that says nothing about why. - if (!string.IsNullOrWhiteSpace(manifest.Runtime) && - !string.Equals(manifest.Runtime, AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase)) + // Everything that would stop it running is refused now, saying why, rather than found + // out by a process that fails to start, or starts on the wrong runtime. + var runtimeName = string.IsNullOrWhiteSpace(manifest.Runtime) ? AdapterManifest.DotnetRuntime : manifest.Runtime; + if (runtimes.Find(runtimeName) == null) throw new NotSupportedException( - $"Adapter '{installed.AdapterId}' needs the '{manifest.Runtime}' runtime, which this host does not have."); + $"Adapter '{installed.AdapterId}' needs the '{runtimeName}' runtime, which this host does not have."); + + // .NET adapters have always been started without asking whether dotnet is there, and + // still are: only a version the manifest asks for is checked. Every other runtime is + // checked before anything is started. + var isDotnet = string.Equals(runtimeName, AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase); + if (!isDotnet || !string.IsNullOrWhiteSpace(manifest.RuntimeVersion)) + { + var status = await runtimes.StatusAsync(runtimeName); + if (!status.Available) + throw new NotSupportedException( + $"Adapter '{installed.AdapterId}' needs the '{runtimeName}' runtime, which this host does not have: {status.Reason}."); + if (!RuntimeVersionRange.Satisfies(manifest.RuntimeVersion, status.Version)) + throw new NotSupportedException( + $"Adapter '{installed.AdapterId}' needs {runtimeName} {manifest.RuntimeVersion}; this host has {status.Version}."); + } + + var platform = AdapterRuntimes.CurrentPlatform; + if (manifest.Platforms is { Count: > 0 } platforms && + !platforms.Contains(platform, StringComparer.OrdinalIgnoreCase)) + throw new NotSupportedException( + $"Adapter '{installed.AdapterId}' is built for {string.Join(", ", platforms)}; this host is {platform}."); if (!HostInfo.Satisfies(manifest.Compatibility?.MinHostVersion)) throw new InvalidOperationException( $"Adapter '{installed.AdapterId}' needs SW.Serverless {manifest.Compatibility.MinHostVersion} or later; this host is {HostInfo.Version}."); - if (!string.IsNullOrWhiteSpace(manifest.Entry)) + var entry = manifest.EntryFor(platform); + if (!string.IsNullOrWhiteSpace(entry)) { - installed.LocalPath = ContainedPath(installed.Directory, manifest.Entry.Replace("\\", "/")) + installed.LocalPath = ContainedPath(installed.Directory, entry.Replace("\\", "/")) ?? throw new InvalidOperationException( - $"Adapter '{installed.AdapterId}' declares an entry outside its package: '{manifest.Entry}'."); - installed.EntryAssembly = manifest.Entry; + $"Adapter '{installed.AdapterId}' declares an entry outside its package: '{entry}'."); + installed.EntryAssembly = entry; } + // A zip doesn't reliably carry the execute bit, and an exec adapter's entry is the + // program itself. + if (string.Equals(runtimeName, AdapterManifest.ExecRuntime, StringComparison.OrdinalIgnoreCase) && + !OperatingSystem.IsWindows() && File.Exists(installed.LocalPath)) + File.SetUnixFileMode(installed.LocalPath, File.GetUnixFileMode(installed.LocalPath) | + UnixFileMode.UserExecute | UnixFileMode.GroupExecute); + installed.Manifest = manifest; } @@ -145,11 +177,18 @@ async Task ExtractAsync(string adapterId, string remoteKey, string directory) FileAccess.Read, FileShare.Read)) using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read)) { + var sourcePrefix = SourcePrefixOf(archive); long total = 0; foreach (var entry in archive.Entries) { if (string.IsNullOrEmpty(entry.Name)) continue; + // The source a package carries is for reading and rebuilding, never for + // running: left in storage rather than unpacked onto every host. + if (sourcePrefix != null && + entry.FullName.Replace("\\", "/").StartsWith(sourcePrefix, StringComparison.Ordinal)) + continue; + // An entry's name is the package author's to choose, and "../../app/x.dll" // would otherwise be written over the host's own files before any adapter // code ran. Every entry must land inside the extraction directory. @@ -190,6 +229,30 @@ async Task ExtractAsync(string adapterId, string remoteKey, string directory) } } + /// + /// The folder a package's manifest says holds its source, as a "folder/" prefix, or null + /// when the manifest declares none. Only a declared source folder is skipped: a package + /// from before manifests had source may well have a folder called "source" it needs to run. + /// + static string SourcePrefixOf(ZipArchive archive) + { + var manifestEntry = archive.GetEntry(AdapterManifest.FileName); + if (manifestEntry == null) return null; + try + { + using var reader = new StreamReader(manifestEntry.Open()); + var source = AdapterManifest.Parse(reader.ReadToEnd()).Source; + if (source == null || !AdapterManifest.IsPackagePath(source.Path ?? "")) return null; + return source.Path.Replace("\\", "/").TrimEnd('/') + "/"; + } + catch (System.Text.Json.JsonException) + { + // A manifest that can't be read is reported, with its reason, once the package is + // unpacked; here it only means there's no source folder to leave out. + return null; + } + } + /// /// under , or null when it would resolve /// outside it — an absolute path, or one climbing out with "..". @@ -264,14 +327,28 @@ async Task RemoteKeyOf(string adapterRef) { var root = options.AdapterRemotePath; var (adapterId, version) = AdapterCatalogPaths.Split(adapterRef); - if (version == null) return AdapterCatalogPaths.Current(root, adapterRef); + if (version == null) + { + var current = AdapterCatalogPaths.Current(root, adapterRef); + if (await ExistsAsync(current)) return current; + + // An adapter in a runtime older hosts can't run has no adapters/{id} — it is kept + // out of their sight on purpose — so its current version is the catalog's. + var entry = await new AdapterCatalogStore(cloudFilesService, root).GetAsync(adapterRef); + if (entry?.Current is { } currentVersion) + return AdapterCatalogPaths.Version(root, adapterRef, currentVersion); + return current; + } var key = AdapterCatalogPaths.Version(root, adapterId, version); - if ((await cloudFilesService.ListAsync(key)).Any(f => string.Equals(f.Key, key, StringComparison.Ordinal))) + if (await ExistsAsync(key)) return key; return AdapterCatalogPaths.LegacyVersion(root, adapterId, version); } + async Task ExistsAsync(string key) => + (await cloudFilesService.ListAsync(key)).Any(f => string.Equals(f.Key, key, StringComparison.Ordinal)); + public async Task GetMetadataAsync(string adapterId) { if (memoryCache.TryGetValue($"{NamingPrefix}.{adapterId}", out InstalledAdapter cached)) @@ -373,6 +450,9 @@ public class InstalledAdapter /// The package's own description, once installed. Null for a package without one. public AdapterManifest Manifest { get; set; } + /// The runtime it runs on: the manifest's, or dotnet for a package without one. + public string Runtime => string.IsNullOrWhiteSpace(Manifest?.Runtime) ? AdapterManifest.DotnetRuntime : Manifest.Runtime; + public IDictionary AdapterValues { get; set; } = new Dictionary(); } } diff --git a/SW.Serverless/Services/HostInfo.cs b/SW.Serverless/Services/HostInfo.cs index 5a8412a..9f9da0b 100644 --- a/SW.Serverless/Services/HostInfo.cs +++ b/SW.Serverless/Services/HostInfo.cs @@ -12,7 +12,7 @@ public static class HostInfo /// assembly cannot say which release it is. Raise it with every release that adds something /// an adapter could depend on. /// - public const string Baseline = "10.0.2"; + public const string Baseline = "10.1.0"; public static Version Version { get; } = Resolve(); diff --git a/SW.Serverless/Services/ServerlessService.cs b/SW.Serverless/Services/ServerlessService.cs index f4d115f..c0b2625 100644 --- a/SW.Serverless/Services/ServerlessService.cs +++ b/SW.Serverless/Services/ServerlessService.cs @@ -9,6 +9,7 @@ using System.Diagnostics; using System.IO; using System.IO.Compression; +using System.Linq; using System.Reflection; using System.Text; using System.Threading; @@ -34,7 +35,13 @@ public class ServerlessService : IServerlessService, IDisposable private ILogger adapterLogger; private readonly ICloudFilesService cloudFilesService; private readonly AdapterInstaller installer; + private readonly IServiceProvider serviceProvider; private IDisposable directoryLease; + + // Set when the adapter speaks the gRPC protocol rather than the classic text one: every + // non-.NET adapter, and a .NET one whose manifest opts in. The session then runs on a + // resident instance of its own, started here and stopped on dispose. + private GrpcSession grpc; /// /// Additive overload for hosts that never install adapters from cloud storage — the /// StartAsync(adapterId, correlationId, adapterPath, ...) path needs no ICloudFilesService. @@ -53,7 +60,9 @@ public ServerlessService(ServerlessOptions serverlessOptions, IMemoryCache memor this.memoryCache = memoryCache; this.loggerFactory = loggerFactory; this.cloudFilesService = cloudFilesService; - installer = new AdapterInstaller(serverlessOptions, memoryCache, cloudFilesService); + this.serviceProvider = serviceProvider; + installer = new AdapterInstaller(serverlessOptions, memoryCache, cloudFilesService, + serviceProvider?.GetService()); logger = loggerFactory.CreateLogger(); @@ -73,6 +82,16 @@ async public Task StartAsync(string adapterId, string correlationId, IDictionary } var adapterMetadata = await Install(adapterId); + + if (adapterMetadata.UsesGrpc) + { + if (processStarted || grpc != null) + throw new Exception("Already started."); + grpc = await GrpcSession.StartAsync(serviceProvider, adapterId, correlationId, startupValues, + adapterMetadata.AdapterValues); + return; + } + // Held while this process runs, so publishing a newer version can't prune its files. directoryLease = AdapterDirectoryLeases.Hold(adapterMetadata.Directory); @@ -116,11 +135,15 @@ Task StartAsync(string adapterId, AdapterMetadata adapterMetadata, string correl var serverlessOptionsBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(serverlessOptions))); var adapterValuesBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(adapterMetadata.AdapterValues))); + // On stdin for an adapter whose SDK reads them there, as arguments for every other. + var valuesOnStdin = adapterMetadata.ValuesOnStdin; process = new Process { StartInfo = new ProcessStartInfo("dotnet") { - Arguments = $"\"{adapterMetadata.LocalPath}\" {serverlessOptionsBase64} {startupValuesBase64} {adapterValuesBase64}", + Arguments = valuesOnStdin + ? $"\"{adapterMetadata.LocalPath}\" {Constants.ValuesOnStdinFlag}" + : $"\"{adapterMetadata.LocalPath}\" {serverlessOptionsBase64} {startupValuesBase64} {adapterValuesBase64}", WorkingDirectory = Path.GetDirectoryName(adapterMetadata.LocalPath), UseShellExecute = false, @@ -145,16 +168,30 @@ Task StartAsync(string adapterId, AdapterMetadata adapterMetadata, string correl process.BeginOutputReadLine(); process.BeginErrorReadLine(); + if (valuesOnStdin) + { + process.StandardInput.WriteLine(serverlessOptionsBase64); + process.StandardInput.WriteLine(startupValuesBase64); + process.StandardInput.WriteLine(adapterValuesBase64); + process.StandardInput.Flush(); + } + return Task.CompletedTask; } public Task> GetExpectedStartupValues() { + if (grpc != null) return Task.FromResult(grpc.ExpectedStartupValues()); return InvokeAsync>(Constants.ExpectedCommand, null); } async public Task InvokeAsync(string command, object input, int commandTimeout = 0) { + if (grpc != null) + { + await grpc.InvokeAsync(command, input, commandTimeout == 0 ? serverlessOptions.CommandTimeout : commandTimeout); + return; + } await InvokeAsync(command, input, commandTimeout); } @@ -167,6 +204,9 @@ async public Task InvokeAsync(string command, object input, in throw new ArgumentException("Invalid name.", nameof(command)); } + if (grpc != null) + return await grpc.InvokeAsync(command, input, commandTimeout); + if (!processStarted || process.HasExited || timedOut) throw new Exception("Process not started or terminated."); @@ -318,12 +358,45 @@ async Task Install(string adapterId) EntryAssembly = installed.EntryAssembly, LocalPath = installed.LocalPath, Directory = installed.Directory, - AdapterValues = installed.AdapterValues + AdapterValues = installed.AdapterValues, + UsesGrpc = UsesGrpc(installed), + ValuesOnStdin = ReadsValuesOnStdin(installed) }; } + /// + /// Whether a classic call to this adapter goes over gRPC: always for a runtime other than + /// .NET, whose SDKs speak only that, and for a .NET adapter whose manifest opts in with a + /// protocol of 2 or more. Every other .NET adapter keeps the text protocol it was built for. + /// + /// + /// Whether the adapter's SDK reads its values from stdin: known only from the SDK version + /// its manifest records. A package without one is from before, and gets them as arguments. + /// + internal static bool ReadsValuesOnStdin(InstalledAdapter installed) => + Version.TryParse((installed.Manifest?.SdkVersion ?? "").Split('-', '+')[0], out var sdk) && + sdk >= Version.Parse(Constants.ValuesOnStdinSince); + + internal static bool UsesGrpc(InstalledAdapter installed) => + !string.Equals(installed.Runtime, Contract.Catalog.AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase) || + installed.Manifest?.Protocol is { Min: >= 2 }; + public void Dispose() { + if (grpc != null) + { + try + { + grpc.StopAsync().GetAwaiter().GetResult(); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Service did not dispose properly."); + } + grpc = null; + return; + } + try { if (processStarted) @@ -363,6 +436,75 @@ private class AdapterMetadata public string LocalPath { get; set; } public string Directory { get; set; } public IDictionary AdapterValues { get; set; } = new Dictionary(); + public bool UsesGrpc { get; set; } + public bool ValuesOnStdin { get; set; } + } + + /// + /// A classic session over gRPC: a resident instance under a key of its own, started for + /// the session and stopped when it ends, called one command at a time as the text protocol + /// is. The resident host does the launching, handshake, timeouts and logs. + /// + private sealed class GrpcSession + { + readonly Resident.IResidentAdapterHost host; + readonly string adapterId; + readonly string instanceKey; + readonly Resident.ResidentAdapterInstance instance; + + GrpcSession(Resident.IResidentAdapterHost host, string adapterId, string instanceKey, + Resident.ResidentAdapterInstance instance) + { + this.host = host; + this.adapterId = adapterId; + this.instanceKey = instanceKey; + this.instance = instance; + } + + public static async Task StartAsync(IServiceProvider services, string adapterId, + string correlationId, IDictionary startupValues, IDictionary adapterValues) + { + var host = services?.GetService() + ?? throw new InvalidOperationException( + $"Adapter '{adapterId}' speaks the gRPC protocol, which runs on the resident adapter host. " + + "Register it with AddResidentAdapters."); + + var values = startupValues == null + ? new Dictionary() + : new Dictionary(startupValues); + values[Constants.CorrelationIdName] = correlationId; + + var instanceKey = $"classic-{Guid.NewGuid():N}"; + var instance = await host.StartExclusiveAsync(new Resident.AdapterSpec + { + AdapterId = adapterId, + InstanceKey = instanceKey, + StartupValues = values, + AdapterValues = adapterValues ?? new Dictionary(), + }); + return new GrpcSession(host, adapterId, instanceKey, instance); + } + + public Task InvokeAsync(string command, object input, int timeoutSeconds) => + instance.InvokeAsync(command, input, timeoutSeconds); + + public Task InvokeAsync(string command, object input, int timeoutSeconds) => + instance.InvokeAsync(command, input, timeoutSeconds); + + /// The settings it declared in its handshake, as the classic protocol has always described them. + public IDictionary ExpectedStartupValues() => + instance.Settings.ToDictionary( + s => s.Name, + s => new StartupValue + { + Optional = !s.Required, + Default = string.IsNullOrEmpty(s.DefaultValue) ? null : s.DefaultValue, + Type = s.Type, + Private = s.Secret, + Description = s.Description, + }); + + public Task StopAsync() => host.StopAsync(adapterId, instanceKey, drain: false); } } }