[wasm] Generate thunks for unmanaged calli signatures and allow thiscall

src/coreclr/tools/Common/JitInterface/WasmLowering.cs

@@ -680,7 +680,7 @@ public enum LoweringFlags
 
         public static WasmSignature GetSignature(MethodSignature signature, LoweringFlags flags)
         {
-            if (!flags.HasFlag(LoweringFlags.IsUnmanagedCallersOnly) && signature.Flags.HasFlag(MethodSignatureFlags.UnmanagedCallingConvention))
+            if ((signature.Flags & MethodSignatureFlags.UnmanagedCallingConventionMask) is not 0 and not MethodSignatureFlags.CallingConventionVarargs)
             {
                 flags |= LoweringFlags.IsUnmanagedCallersOnly;
             }

src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/WasmArgumentLayoutTests.cs

@@ -270,6 +270,22 @@ private static MethodDesc GetTaskReturningCoreLibMethod(ReadyToRunCompilerContex
         throw new InvalidOperationException($"No single-argument {type.Name.ToString()}.{methodName} overload in System.Private.CoreLib");
     }
 
+    [Theory]
+    [InlineData(MethodSignatureFlags.None, "iip")]
+    [InlineData(MethodSignatureFlags.CallingConventionVarargs, "iip")]
+    [InlineData(MethodSignatureFlags.UnmanagedCallingConventionCdecl, "ii")]
+    [InlineData(MethodSignatureFlags.UnmanagedCallingConventionStdCall, "ii")]
+    [InlineData(MethodSignatureFlags.UnmanagedCallingConventionThisCall, "ii")]
+    [InlineData(MethodSignatureFlags.UnmanagedCallingConvention, "ii")]
+    public void SignatureCallingConventionSelectsLowering(MethodSignatureFlags callingConvention, string expectedSignature)
+    {
+        ReadyToRunCompilerContext context = CreateWasmContext();
+        TypeDesc int32Type = context.GetWellKnownType(WellKnownType.Int32);
+        MethodSignature signature = new MethodSignature(callingConvention | MethodSignatureFlags.Static, 0, int32Type, [int32Type]);
+
+        Assert.Equal(expectedSignature, WasmLowering.GetSignature(signature, WasmLowering.LoweringFlags.None).SignatureString);
+    }
+
     [Theory]
     [InlineData(MethodSignatureFlags.None)]
     [InlineData(MethodSignatureFlags.CallingConventionVarargs)]
@@ -887,28 +903,6 @@ public static class Exports
         AssertPortableCallHelpersGeneratorRejects(source, $"has unsupported signature token '{expectedToken}'");
     }
 
-    [Fact]
-    public void PortableCallHelpersGeneratorRejectsHiddenReturnBufferCallbacks()
-    {
-        string source = """
-            using System.Runtime.InteropServices;
-
-            public struct Pair
-            {
-                public int First;
-                public int Second;
-            }
-
-            public static class Exports
-            {
-                [UnmanagedCallersOnly(EntryPoint = "callback")]
-                public static Pair Handle(int value) => default;
-            }
-            """;
-
-        AssertPortableCallHelpersGeneratorRejects(source, "uses a hidden return buffer");
-    }
-
     private void AssertPortableCallHelpersGeneratorRejects(string source, string expectedError)
     {
         string workingDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
@@ -1028,6 +1022,153 @@ public static class Exports
         }
     }
 
+    /// <summary>
+    /// The interpreter looks up a thunk for each unmanaged <c>calli</c> by signature, so the generator
+    /// has to emit one for every function pointer signature an app calls through, whatever the
+    /// calling convention spelling.
+    /// </summary>
+    [Fact]
+    public void PortableCallHelpersGeneratorEmitsThunksForUnmanagedCalliSites()
+    {
+        string source = """
+            public struct FloatPair
+            {
+                public float X;
+                public float Y;
+            }
+
+            public static unsafe class Calls
+            {
+                public static FloatPair MemberFunction(delegate* unmanaged[Cdecl, MemberFunction]<void*, long, FloatPair> fn)
+                    => fn(null, 1);
+
+                public static int ThisCall(delegate* unmanaged[Thiscall]<void*, double, double, double, int> fn)
+                    => fn(null, 1, 2, 3);
+
+                public static double Managed(delegate*<int, int, int, int, int, int, double> fn)
+                    => fn(1, 2, 3, 4, 5, 6);
+
+                public static T Generic<T>(delegate* unmanaged<T> fn)
+                    => fn();
+            }
+            """;
+
+        string workingDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+        Directory.CreateDirectory(workingDirectory);
+
+        try
+        {
+            string inputAssembly = CompileCallbackAssembly(source, Path.Combine(workingDirectory, "Calls.dll"), allowUnsafe: true);
+            string outputDirectory = Path.Combine(workingDirectory, "generated");
+
+            var options = new PortableCallHelpersGeneratorOptions
+            {
+                OutputDirectory = outputDirectory,
+                TargetOS = "browser",
+                PInvokeModules = new[] { "libSystem.Native" },
+            };
+
+            var log = new StringWriter();
+            int exitCode = PortableCallHelpersGenerator.Run(
+                CreateWasmContext(inputAssembly), options, new Logger(log, isVerbose: true));
+
+            Assert.True(exitCode == 0, log.ToString());
+
+            string thunks = File.ReadAllText(Path.Combine(outputDirectory, "callhelpers-interp-to-managed.cpp"));
+            Assert.Contains("\"MS8il\"", thunks);
+            Assert.Contains("\"Miiddd\"", thunks);
+            Assert.DoesNotContain("\"Mdiiiiii\"", thunks);
+        }
+        finally
+        {
+            try
+            {
+                Directory.Delete(workingDirectory, recursive: true);
+            }
+            catch (Exception e) when (e is IOException or UnauthorizedAccessException)
+            {
+            }
+        }
+    }
+
+    /// <summary>
+    /// Native code calls a reverse thunk with the wasm C ABI, which returns a struct it cannot pass as a
+    /// single scalar through a hidden leading pointer instead of a return value. The thunk has to take
+    /// that pointer and hand it to the interpreter as the return buffer, or the indirect call from
+    /// native code traps with a signature mismatch.
+    /// </summary>
+    [Fact]
+    public void PortableCallHelpersGeneratorReturnsStructsThroughHiddenPointerInReverseThunks()
+    {
+        string source = """
+            using System.Runtime.InteropServices;
+
+            public struct FloatPair
+            {
+                public float X;
+                public float Y;
+            }
+
+            public struct Width
+            {
+                public float Value;
+            }
+
+            public static class Callbacks
+            {
+                [UnmanagedCallersOnly]
+                public static FloatPair GetPair(nint self, int unused) => default;
+
+                [UnmanagedCallersOnly]
+                public static Width GetWidth(nint self) => default;
+
+                [UnmanagedCallersOnly(EntryPoint = "export_pair")]
+                public static FloatPair ExportPair(int value) => default;
+            }
+            """;
+
+        string workingDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+        Directory.CreateDirectory(workingDirectory);
+
+        try
+        {
+            string inputAssembly = CompileCallbackAssembly(source, Path.Combine(workingDirectory, "Callbacks.dll"));
+            string outputDirectory = Path.Combine(workingDirectory, "generated");
+
+            var options = new PortableCallHelpersGeneratorOptions
+            {
+                OutputDirectory = outputDirectory,
+                TargetOS = "browser",
+                PInvokeModules = new[] { "libSystem.Native" },
+            };
+
+            var log = new StringWriter();
+            int exitCode = PortableCallHelpersGenerator.Run(
+                CreateWasmContext(inputAssembly), options, new Logger(log, isVerbose: false));
+
+            Assert.True(exitCode == 0, log.ToString());
+
+            string reverse = File.ReadAllText(Path.Combine(outputDirectory, "callhelpers-reverse.cpp"));
+            Assert.Contains("static void Call_Callbacks__Callbacks_GetPair_I32_I32_RetS8(void * sret, void * arg0, int32_t arg1)", reverse);
+            Assert.Contains("static float Call_Callbacks__Callbacks_GetWidth_I32_RetF32(void * arg0)", reverse);
+            Assert.Contains("extern \"C\" void export_pair(void * sret, int32_t arg0)", reverse);
+            Assert.Contains("Call_Callbacks__Callbacks_ExportPair_I32_RetS8(sret, arg0);", reverse);
+            Assert.Contains("(int8_t*)sret, (PCODE)&Call_Callbacks__Callbacks_GetPair_I32_I32_RetS8", reverse);
+            Assert.Contains("((void(*)(void *, int32_t))r2r)(sret, arg0);", reverse);
+            Assert.DoesNotContain("RetS8(void * arg", reverse);
+        }
+        finally
+        {
+            try
+            {
+                Directory.Delete(workingDirectory, recursive: true);
+            }
+            catch (Exception e) when (e is IOException or UnauthorizedAccessException)
+            {
+            }
+        }
+    }
+
     /// <summary>
     /// The generated wrapper only needs to reach the R2R native entrypoint for callbacks exported by
     /// name: native code calls the extern "C" export (and thus the wrapper) directly. Every other
@@ -1101,13 +1242,13 @@ public static class Exports
     /// Builds an input assembly for the generator to scan. It references the same CoreLib the context
     /// reads, so the attributes it applies are the ones the type system will resolve.
     /// </summary>
-    private static string CompileCallbackAssembly(string source, string outputPath)
+    private static string CompileCallbackAssembly(string source, string outputPath, bool allowUnsafe = false)
     {
         CSharpCompilation compilation = CSharpCompilation.Create(
             Path.GetFileNameWithoutExtension(outputPath),
             new[] { CSharpSyntaxTree.ParseText(source) },
             new[] { MetadataReference.CreateFromFile(TestPaths.SystemPrivateCoreLibPath) },
-            new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary));
+            new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, allowUnsafe: allowUnsafe));
 
         EmitResult result = compilation.Emit(outputPath);
         Assert.True(result.Success,

src/coreclr/tools/aot/ILCompiler.ReadyToRun/PortableCallHelpers/PInvokeCollector.cs

@@ -6,6 +6,7 @@
 using System.Reflection.Metadata;
 using System.Reflection.Metadata.Ecma335;
 
+using Internal.IL;
 using Internal.JitInterface;
 using Internal.TypeSystem;
 using Internal.TypeSystem.Ecma;
@@ -138,6 +139,7 @@ public void CollectPInvokes(List<PInvokeInfo> pinvokes, List<PInvokeCallback> ca
                     CollectPInvokesForMethod(method);
                     if (IsMethodCallback(method))
                         callbacks.Add(new PInvokeCallback(method));
+                    CollectUnmanagedCalliSignatures(signatures, method);
                 }
                 catch (Exception ex) when (ex is not LogAsErrorException)
                 {
@@ -178,6 +180,80 @@ void CollectPInvokesForMethod(EcmaMethod method)
             }
         }
 
+        /// <summary>
+        /// Adds the signature of every <c>calli</c> through an unmanaged function pointer in the
+        /// method's body. The interpreter calls those through the same thunks as P/Invokes, and a
+        /// function pointer's signature appears nowhere else the scan looks.
+        /// </summary>
+        private void CollectUnmanagedCalliSignatures(Dictionary<string, MethodDesc> signatures, EcmaMethod method)
+        {
+            if (method.IsPInvoke || method.IsAbstract)
+                return;
+
+            MethodIL methodIL = EcmaMethodIL.Create(method);
+            if (methodIL is null)
+                return;
+
+            ILReader reader = new ILReader(methodIL.GetILBytes());
+            while (reader.HasNext)
+            {
+                ILOpcode opcode = reader.ReadILOpcode();
+                if (opcode != ILOpcode.calli)
+                {
+                    reader.Skip(opcode);
+                    continue;
+                }
+
+                if (methodIL.GetObject(reader.ReadILToken(), NotFoundBehavior.ReturnNull) is not MethodSignature calliSignature
+                    || (calliSignature.Flags & MethodSignatureFlags.UnmanagedCallingConventionMask) is 0 or MethodSignatureFlags.CallingConventionVarargs)
+                {
+                    continue;
+                }
+
+                if (IsUnsupportedOnPlatform(method))
+                    return;
+
+                // A signature over the enclosing generic parameters has no single native shape.
+                if (ContainsSignatureVariables(calliSignature))
+                {
+                    log.Verbose($"Skipping generic unmanaged calli in '{method.OwningType}.{method.Name.ToString()}'");
+                    continue;
+                }
+
+                string signature;
+                try
+                {
+                    signature = WasmLowering.GetSignature(calliSignature, WasmLowering.LoweringFlags.None).SignatureString;
+                    foreach (string token in InteropSignature.ParseSignatureTokens(signature))
+                        InteropSignature.TokenToNativeType(token);
+                }
+                catch (Exception ex) when (ex is InvalidSignatureCharException or LogAsErrorException or TypeSystemException)
+                {
+                    // A P/Invoke with such a signature is a build error, but a calli is only a call
+                    // site that may never run. Leave it to fail at run time rather than fail the build.
+                    log.Verbose($"Skipping unmanaged calli in '{method.OwningType}.{method.Name.ToString()}': {ex.Message}");
+                    continue;
+                }
+
+                if (signatures.TryAdd(signature, method))
+                    log.Verbose($"Adding calli signature {signature} for method '{method.OwningType}.{method.Name.ToString()}'");
+            }
+
+            static bool ContainsSignatureVariables(MethodSignature signature)
+            {
+                if (signature.ReturnType.ContainsSignatureVariables())
+                    return true;
+
+                for (int i = 0; i < signature.Length; i++)
+                {
+                    if (signature[i].ContainsSignatureVariables())
+                        return true;
+                }
+
+                return false;
+            }
+        }
+
         private void AddSignature(Dictionary<string, MethodDesc> signatures, MethodDesc method, WasmLowering.LoweringFlags flags, string kind)
         {
             string signature = InteropSignature.GetMethodSignature(method, flags);