Multi-language adapters, the serverless CLI and source in packages (10.1)

.github/workflows/nuget-publish.yml

@@ -17,14 +17,16 @@ jobs:
       security-events: read   # critical-vuln gate job
     uses: simplify9/.github/.github/workflows/reusable-nuget-publish.yml@0f33f79176152bc11acf0ff08ed970ae8b429b3d # main
     with:
-      # Contract FIRST: both of the others depend on it, and publishing them without it
-      # produces packages whose dependency cannot resolve — which is what 8.1.13 shipped as.
+      # Contract FIRST: the others depend on it, and publishing them without it produces
+      # packages whose dependency cannot resolve — which is what 8.1.13 shipped as. Tooling
+      # last: it depends on the host and the contract.
       nuget-projects: >-
         SW.Serverless.Contract/SW.Serverless.Contract.csproj
         SW.Serverless/SW.Serverless.csproj
         SW.Serverless.Sdk/SW.Serverless.Sdk.csproj
+        SW.Serverless.Tooling/SW.Serverless.Tooling.csproj
       major-version: '10'
-      minor-version: '0'
+      minor-version: '1'
       dotnet-version: '10.0.x'
       run-tests: true
       test-projects: '**/*UnitTests/*.csproj'

.gitignore

@@ -337,3 +337,4 @@ ASALocalRun/
 
 binaries
 out
+.test-work/

SW.Serverless.Compat.ManifestV1002/Program.cs

@@ -0,0 +1,12 @@
+using System;
+using System.IO;
+using SW.Serverless.Contract.Catalog;
+
+// Reads the manifest at args[0] with the 10.0.2 parser and writes it back: what a released tool
+// does to a manifest a newer one wrote. Prints the problems 10.0.2 finds, then the JSON.
+var manifest = AdapterManifest.Parse(File.ReadAllText(args[0]));
+foreach (var problem in manifest.Validate())
+    Console.WriteLine("PROBLEM:" + problem);
+Console.WriteLine("ENTRY:" + manifest.Entry);
+Console.WriteLine("RUNTIME:" + manifest.Runtime);
+Console.WriteLine("JSON:" + manifest.ToJson().ReplaceLineEndings(" "));

SW.Serverless.Compat.ManifestV1002/SW.Serverless.Compat.ManifestV1002.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <!-- The manifest reader as published in SimplyWorks.Serverless.Contract 10.0.2, the first to
+         read manifests and what released Bitween reads them with. Never upgrade this reference:
+         it is the old side of the compatibility tests. -->
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net10.0</TargetFramework>
+        <IsPackable>false</IsPackable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="SimplyWorks.Serverless.Contract" Version="10.0.2" />
+    </ItemGroup>
+
+</Project>

SW.Serverless.CompatibilityTests/Compat.cs

@@ -8,7 +8,7 @@
 using SW.CloudFiles.Extensions;
 using SW.PrimitiveTypes;
 using SW.Serverless.Installer;
-using SW.Serverless.Installer.Shared;
+using SW.Serverless.Tooling;
 
 namespace SW.Serverless.CompatibilityTests;
 
@@ -146,6 +146,28 @@ public record OldHostRun(int ExitCode, List<string> Lines)
     }
 
     /// <summary>Runs the host built on the PUBLISHED SimplyWorks.Serverless 10.0.0 as its own process.</summary>
+    /// <summary>Reads a manifest file with the parser published in SimplyWorks.Serverless.Contract 10.0.2.</summary>
+    public static async Task<OldHostRun> ReadWithManifestV1002Async(string manifestPath)
+    {
+        var entry = System.IO.Path.Combine(BuildOutput("SW.Serverless.Compat.ManifestV1002"), "SW.Serverless.Compat.ManifestV1002.dll");
+        var start = new ProcessStartInfo("dotnet") { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false };
+        start.ArgumentList.Add(entry);
+        start.ArgumentList.Add(manifestPath);
+
+        using var process = Process.Start(start)!;
+        var stdout = process.StandardOutput.ReadToEndAsync();
+        var stderr = process.StandardError.ReadToEndAsync();
+        if (!process.WaitForExit(60_000))
+        {
+            process.Kill(true);
+            Assert.Fail("the 10.0.2 manifest reader did not finish within a minute");
+        }
+
+        var lines = (await stdout).Split('\n', StringSplitOptions.RemoveEmptyEntries).Select(l => l.TrimEnd('\r')).ToList();
+        lines.AddRange((await stderr).Split('\n', StringSplitOptions.RemoveEmptyEntries).Select(l => "stderr: " + l.TrimEnd('\r')));
+        return new OldHostRun(process.ExitCode, lines);
+    }
+
     public static async Task<OldHostRun> RunOldHostAsync(Bucket bucket, string adapter, params string[] args)
     {
         var entry = System.IO.Path.Combine(BuildOutput("SW.Serverless.Compat.OldHost"), "SW.Serverless.Compat.OldHost.dll");

SW.Serverless.CompatibilityTests/ListingAndManifestTests.cs

@@ -3,7 +3,7 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using SW.PrimitiveTypes;
 using SW.Serverless.Contract.Catalog;
-using SW.Serverless.Installer.Shared;
+using SW.Serverless.Tooling;
 
 namespace SW.Serverless.CompatibilityTests;
 

SW.Serverless.CompatibilityTests/MultiLanguageManifestTests.cs

@@ -0,0 +1,106 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using SW.Serverless.Contract.Catalog;
+
+namespace SW.Serverless.CompatibilityTests;
+
+/// <summary>
+/// The manifest fields multi-language adapters add — runtime version, platforms and their entries,
+/// contracts, the source the package carries — written by the current tools and read by the
+/// manifest parser released Bitween uses (SimplyWorks.Serverless.Contract 10.0.2). That parser
+/// must keep every one of them, find nothing wrong, and still see the fields it knows as before.
+/// </summary>
+[TestClass]
+public class MultiLanguageManifestTests
+{
+    static AdapterManifest Written() => new()
+    {
+        Id = "acme.orders",
+        Version = "2.1.0",
+        DisplayName = "Acme orders",
+        Kinds = { "handler" },
+        Runtime = AdapterManifest.PythonRuntime,
+        RuntimeVersion = ">=3.12",
+        Language = "python",
+        Entry = "main.py",
+        Lifecycle = AdapterManifest.ClassicLifecycle,
+        Platforms = new() { "linux-x64", "linux-arm64" },
+        Entries = new() { ["linux-x64"] = "x64/main.py", ["linux-arm64"] = "arm64/main.py" },
+        Contracts = new() { ["bitween"] = 1 },
+        Source = new AdapterSource
+        {
+            Files = new() { ["main.py"] = new string('a', 64), ["requirements.lock"] = new string('b', 64) },
+            BuildCommand = "serverless build",
+            Lockfiles = new() { "requirements.lock" },
+        },
+    };
+
+    [TestMethod]
+    public async Task The_released_manifest_parser_keeps_every_new_field_and_finds_nothing_wrong()
+    {
+        var written = Written();
+        Assert.AreEqual(0, written.Validate().Count, string.Join("; ", written.Validate()));
+        var path = Path.Combine(Path.GetTempPath(), $"manifest-{Guid.NewGuid():N}.json");
+        await File.WriteAllTextAsync(path, written.ToJson());
+        try
+        {
+            var read = await Compat.ReadWithManifestV1002Async(path);
+
+            Assert.AreEqual(0, read.ExitCode, read.ToString());
+            Assert.IsNull(read.Value("PROBLEM"), read.ToString());
+            Assert.AreEqual("main.py", read.Value("ENTRY"), "the default entry is what released readers use");
+            Assert.AreEqual("python", read.Value("RUNTIME"), "a released host sees the runtime, and refuses one it can't run");
+
+            var before = JsonNode.Parse(written.ToJson())!;
+            var after = JsonNode.Parse(read.Value("JSON")!)!;
+            foreach (var field in new[] { "runtimeVersion", "platforms", "entries", "contracts", "source" })
+                Assert.IsTrue(JsonNode.DeepEquals(before[field], after[field]), $"{field} changed in a round trip through 10.0.2:\n{after[field]}");
+        }
+        finally
+        {
+            File.Delete(path);
+        }
+    }
+
+    [TestMethod]
+    public void The_new_fields_round_trip_through_the_current_parser()
+    {
+        var again = AdapterManifest.Parse(Written().ToJson());
+
+        Assert.AreEqual(">=3.12", again.RuntimeVersion);
+        CollectionAssert.AreEqual(new[] { "linux-x64", "linux-arm64" }, again.Platforms);
+        Assert.AreEqual("arm64/main.py", again.EntryFor("linux-arm64"));
+        Assert.AreEqual("main.py", again.EntryFor("osx-arm64"), "a platform without its own entry uses the default");
+        Assert.AreEqual(1, again.Contracts!["bitween"]);
+        Assert.AreEqual(AdapterSource.DefaultPath, again.Source!.Path);
+        Assert.AreEqual(2, again.Source.Files.Count);
+        Assert.IsNull(again.Extensions, "nothing unknown to the current parser");
+    }
+
+    [TestMethod]
+    public void A_manifest_without_the_new_fields_is_read_as_a_dotnet_adapter_as_before()
+    {
+        var old = AdapterManifest.Parse("""{ "id": "legacy", "entry": "Legacy.dll" }""");
+
+        Assert.AreEqual(AdapterManifest.DotnetRuntime, old.Runtime);
+        Assert.AreEqual("Legacy.dll", old.EntryFor("linux-x64"));
+        Assert.IsNull(old.Platforms);
+        Assert.IsNull(old.Source);
+        Assert.AreEqual(0, old.Validate().Count);
+    }
+
+    [DataTestMethod]
+    [DataRow("""{ "runtime": "../bin/sh" }""", "runtime")]
+    [DataRow("""{ "platforms": ["Linux x64"] }""", "platform")]
+    [DataRow("""{ "platforms": ["linux-x64"], "entries": { "osx-arm64": "a" } }""", "doesn't list")]
+    [DataRow("""{ "platforms": ["linux-x64"], "entries": { "linux-x64": "../../etc/passwd" } }""", "inside the package")]
+    [DataRow("""{ "contracts": { "bitween": 0 } }""", "contract")]
+    [DataRow("""{ "source": { "files": { "main.py": "not-a-hash" } } }""", "SHA-256")]
+    [DataRow("""{ "source": { "path": "/abs", "files": {} } }""", "source.path")]
+    public void A_wrong_new_field_is_named_by_validation(string json, string expected)
+    {
+        var problems = AdapterManifest.Parse(json).Validate();
+        Assert.IsTrue(problems.Any(p => p.Contains(expected)), $"expected a problem about {expected}: {string.Join("; ", problems)}");
+    }
+}

SW.Serverless.CompatibilityTests/NewHostTests.cs

@@ -141,6 +141,14 @@ public async Task A_resident_adapter_on_sdk_10_0_0_runs_on_the_current_host()
             Assert.AreEqual(InstanceState.Ready, instance.State);
             Assert.AreEqual("v10:ping", await instance.InvokeAsync<string>("Echo", "ping"));
 
+            // The 10.0.0 SDK knows nothing of the language-neutral Hello fields: they're simply
+            // empty, and nothing depends on them being there.
+            var described = adapters.Describe().Single(h => h.InstanceKey == "compat");
+            Assert.AreEqual("", described.SdkLanguage);
+            Assert.AreEqual(0, described.Settings.Count);
+            Assert.AreEqual(0, described.Kinds.Count);
+            Assert.AreEqual(0, described.Contracts.Count);
+
             await adapters.StopAsync("sdk10.resident", "compat", drain: false);
         }
         finally

SW.Serverless.CompatibilityTests/PackageLayoutTests.cs

@@ -0,0 +1,160 @@
+using System.IO.Compression;
+using System.Security.Cryptography;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using SW.PrimitiveTypes;
+using SW.Serverless.Contract.Catalog;
+using SW.Serverless.Tooling;
+
+namespace SW.Serverless.CompatibilityTests;
+
+/// <summary>
+/// The package layout multi-language adapters bring, against the hosts already deployed:
+/// a package that carries its source runs on old and new hosts alike, and only new hosts leave
+/// the source unpacked; an adapter in another runtime is kept out of old hosts' sight entirely —
+/// they only ever run adapters/{id}, and always with dotnet — while new hosts find it in the catalog.
+/// </summary>
+[TestClass]
+public class PackageLayoutTests
+{
+    const string Sample = "SW.Serverless.Samples.Classic";
+
+    /// <summary>The sample's build output as a zip, with an adapter.json and any extra entries.</summary>
+    static string Package(AdapterManifest manifest, IDictionary<string, string>? extra = null)
+    {
+        var output = Compat.BuildOutput(Sample);
+        var zip = Path.Combine(Path.GetTempPath(), "swsl-compat", $"layout-{Guid.NewGuid():N}.zip");
+        Directory.CreateDirectory(Path.GetDirectoryName(zip)!);
+        using (var archive = ZipFile.Open(zip, ZipArchiveMode.Create))
+        {
+            foreach (var file in Directory.EnumerateFiles(output, "*", SearchOption.AllDirectories))
+                archive.CreateEntryFromFile(file, Path.GetRelativePath(output, file).Replace('\\', '/'), CompressionLevel.Fastest);
+            using (var writer = new StreamWriter(archive.CreateEntry(AdapterManifest.FileName).Open()))
+                writer.Write(manifest.ToJson());
+            foreach (var (name, content) in extra ?? new Dictionary<string, string>())
+                using (var writer = new StreamWriter(archive.CreateEntry(name).Open()))
+                    writer.Write(content);
+        }
+        return zip;
+    }
+
+    static string Sha256(string text) => Convert.ToHexString(SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(text))).ToLowerInvariant();
+
+    static AdapterManifest DotnetWithSource(string id) => new()
+    {
+        Id = id,
+        Entry = Sample + ".dll",
+        Source = new AdapterSource { Files = { ["Program.cs"] = Sha256("// the source"), ["Sample.csproj"] = Sha256("<Project />") } },
+    };
+
+    static AdapterManifest Python(string id) => new()
+    {
+        Id = id,
+        Runtime = AdapterManifest.PythonRuntime,
+        Language = "python",
+        Entry = "main.py",
+        Kinds = { "handler" },
+    };
+
+    static AdapterRepository Repository(Bucket bucket) => new(bucket.Files, _ => { });
+
+    [TestMethod]
+    public async Task A_package_carrying_its_source_runs_on_old_and_new_hosts_and_new_ones_leave_the_source_packed()
+    {
+        using var bucket = new Bucket();
+        const string id = "layout.withsource";
+        var zip = Package(DotnetWithSource(id), new Dictionary<string, string>
+        {
+            ["source/Program.cs"] = "// the source",
+            ["source/Sample.csproj"] = "<Project />",
+        });
+        await Repository(bucket).PublishVersionAsync(id, "1.0.0", zip,
+            new PackageInfo { EntryAssembly = Sample + ".dll", Manifest = DotnetWithSource(id) }, promote: true, "compat");
+
+        var old = await Compat.RunOldHostAsync(bucket, id, "--command", "Echo", "--input", "carried");
+        Assert.AreEqual(0, old.ExitCode, old.ToString());
+        Assert.AreEqual("carried", old.Value("RESULT"), "a host on the released 10.0.0 runs it as before");
+
+        using var host = Compat.NewHost(bucket);
+        using (var service = (ServerlessService)host.GetRequiredService<IServerlessService>())
+        {
+            await service.StartAsync(id, "correlation");
+            Assert.AreEqual("carried", await service.InvokeAsync<string>("Echo", "carried"));
+        }
+
+        var installed = await host.GetRequiredService<AdapterInstaller>().InstallAsync(id);
+        Assert.IsTrue(File.Exists(installed.LocalPath));
+        Assert.IsFalse(Directory.Exists(Path.Combine(installed.Directory, "source")), "the source was unpacked onto the host");
+    }
+
+    [TestMethod]
+    public async Task A_folder_called_source_in_a_package_without_declared_source_is_unpacked_as_before()
+    {
+        using var bucket = new Bucket();
+        const string id = "layout.ownsource";
+        var manifest = new AdapterManifest { Id = id, Entry = Sample + ".dll" };
+        await Repository(bucket).PublishVersionAsync(id, "1.0.0",
+            Package(manifest, new Dictionary<string, string> { ["source/needed-at-runtime.txt"] = "keep me" }),
+            new PackageInfo { EntryAssembly = Sample + ".dll", Manifest = manifest }, promote: true, "compat");
+
+        using var host = Compat.NewHost(bucket);
+        var installed = await host.GetRequiredService<AdapterInstaller>().InstallAsync(id);
+
+        Assert.AreEqual("keep me", await File.ReadAllTextAsync(Path.Combine(installed.Directory, "source", "needed-at-runtime.txt")));
+    }
+
+    [TestMethod]
+    public async Task An_adapter_in_another_runtime_is_out_of_old_hosts_sight_and_new_hosts_find_its_current_version()
+    {
+        using var bucket = new Bucket();
+        const string id = "layout.python";
+        var repository = Repository(bucket);
+        var package = new PackageInfo { EntryAssembly = "main.py", Manifest = Python(id) };
+        await repository.PublishVersionAsync(id, "1.0.0", Package(Python(id)), package, promote: true, "compat");
+        await repository.PublishVersionAsync(id, "1.1.0", Package(Python(id)), package, promote: false, "compat");
+
+        Assert.AreEqual(0, (await bucket.KeysAsync("adapters/")).Count, "nothing under adapters/, where old hosts and old Bitween look");
+        Assert.AreEqual("1.0.0", (await repository.LoadEntryAsync(id)).Current);
+
+        var old = await Compat.RunOldHostAsync(bucket, id, "--command", "Echo", "--input", "x");
+        Assert.AreNotEqual(0, old.ExitCode, "an old host must not find it, let alone start it with dotnet");
+
+        using (var host = Compat.NewHost(bucket))
+            Assert.AreEqual($"adapters-versions/{id}/1.0.0",
+                (await host.GetRequiredService<AdapterInstaller>().GetMetadataAsync(id)).RemoteKey);
+
+        await repository.PromoteAsync(id, "1.1.0", Path.Combine(bucket.Path, "..", "work-" + Guid.NewGuid().ToString("N")));
+
+        Assert.AreEqual(0, (await bucket.KeysAsync("adapters/")).Count, "promote doesn't put it there either");
+        Assert.AreEqual("1.1.0", (await repository.LoadEntryAsync(id)).Current);
+        using (var host = Compat.NewHost(bucket))
+            Assert.AreEqual($"adapters-versions/{id}/1.1.0",
+                (await host.GetRequiredService<AdapterInstaller>().GetMetadataAsync(id)).RemoteKey);
+    }
+
+    [TestMethod]
+    public async Task An_id_published_as_dotnet_cannot_move_to_another_runtime()
+    {
+        using var bucket = new Bucket();
+        const string id = "layout.switch";
+        var repository = Repository(bucket);
+        await repository.PublishVersionAsync(id, "1.0.0", Package(new AdapterManifest { Id = id, Entry = Sample + ".dll" }),
+            new PackageInfo { EntryAssembly = Sample + ".dll" }, promote: true, "compat");
+
+        var refused = await Assert.ThrowsExceptionAsync<SWException>(() => repository.PublishVersionAsync(id, "2.0.0",
+            Package(Python(id)), new PackageInfo { EntryAssembly = "main.py", Manifest = Python(id) }, promote: true, "compat"));
+        StringAssert.Contains(refused.Message, "new id");
+        Assert.AreEqual("1.0.0", (await repository.LoadEntryAsync(id)).Current);
+    }
+
+    [TestMethod]
+    public async Task An_adapter_in_another_runtime_must_be_published_with_a_version()
+    {
+        using var bucket = new Bucket();
+        var refused = await Assert.ThrowsExceptionAsync<SWException>(() => Repository(bucket).PublishUnversionedAsync("layout.unversioned",
+            Package(Python("layout.unversioned")), new PackageInfo { EntryAssembly = "main.py", Manifest = Python("layout.unversioned") }));
+
+        StringAssert.Contains(refused.Message, "with a version");
+        Assert.AreEqual(0, (await bucket.KeysAsync()).Count);
+    }
+}

SW.Serverless.CompatibilityTests/SW.Serverless.CompatibilityTests.csproj

@@ -30,6 +30,8 @@
         <!-- Build order only: these run as separate processes, or are packaged and installed. -->
         <ProjectReference Include="..\SW.Serverless.Compat.OldHost\SW.Serverless.Compat.OldHost.csproj"
                           ReferenceOutputAssembly="false" />
+        <ProjectReference Include="..\SW.Serverless.Compat.ManifestV1002\SW.Serverless.Compat.ManifestV1002.csproj"
+                          ReferenceOutputAssembly="false" />
         <ProjectReference Include="..\SW.Serverless.Compat.ClassicV10\SW.Serverless.Compat.ClassicV10.csproj"
                           ReferenceOutputAssembly="false" />
         <ProjectReference Include="..\SW.Serverless.Compat.ResidentV10\SW.Serverless.Compat.ResidentV10.csproj"