diff --git a/.gitignore b/.gitignore index 43b4bb330b0..2dc12300ca2 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,9 @@ dlldata.c # Benchmark Results BenchmarkDotNet.Artifacts/ +# BenchmarkDotNet generated builds, written beside the benchmark project +.bdn/ + # .NET Core project.lock.json project.fragment.lock.json diff --git a/docs/benchmarkdotnet.md b/docs/benchmarkdotnet.md index 657c56bb8b9..7f58b03c474 100644 --- a/docs/benchmarkdotnet.md +++ b/docs/benchmarkdotnet.md @@ -294,7 +294,7 @@ M00_L00: The `--runtimes` or just `-r` allows you to run the benchmarks for **multiple Runtimes**. -Available options are: Mono, wasmnet70, CoreRT, netcoreapp3.1, net6.0, net7.0, net8.0, and net9.0. +Available options include: net8.0, net9.0, net10.0, net11.0, and the versioned forms of the other runtimes: mono11.0, nativeaot11.0, r2r11.0, monowasm11.0, monowasmaot11.0, corewasm11.0. Example: run the benchmarks for .NET 7.0 and 8.0: diff --git a/docs/benchmarking-workflow-dotnet-runtime.md b/docs/benchmarking-workflow-dotnet-runtime.md index 8980660135b..5a2fc0e821c 100644 --- a/docs/benchmarking-workflow-dotnet-runtime.md +++ b/docs/benchmarking-workflow-dotnet-runtime.md @@ -206,11 +206,11 @@ validated individually. #### Run the benchmarks with AOT -Essentially, add `--aotcompilermode wasm` to the `--bdn-arguments=".."`: +Essentially, add `--wasm-runtime-flavor MonoAOT` alongside `--wasm`: ```cmd -/path/to/dotnet/performance$ python3 ./scripts/benchmarks_ci.py --csproj src/benchmarks/micro/MicroBenchmarks.csproj -f net11.0 --dotnet-path artifacts/bin/dotnet-latest --wasm --run-isolated --bdn-artifacts artifacts/BenchmarkDotNet.Artifacts - --bdn-arguments="--category-exclusion-filter NoInterpreter NoWASM NoMono --aotcompilermode wasm --logBuildOutput --buildTimeout 3600 --filter " +/path/to/dotnet/performance$ python3 ./scripts/benchmarks_ci.py --csproj src/benchmarks/micro/MicroBenchmarks.csproj -f net11.0 --dotnet-path artifacts/bin/dotnet-latest --wasm --wasm-runtime-flavor MonoAOT --run-isolated --bdn-artifacts artifacts/BenchmarkDotNet.Artifacts + --bdn-arguments="--category-exclusion-filter NoInterpreter NoWASM NoMono --logBuildOutput --buildTimeout 3600 --filter " ``` #### Run CoreCLR WASM ReadyToRun with a nightly or VMR SDK diff --git a/docs/microbenchmark-design-guidelines.md b/docs/microbenchmark-design-guidelines.md index 3a85347702c..495ad374caa 100644 --- a/docs/microbenchmark-design-guidelines.md +++ b/docs/microbenchmark-design-guidelines.md @@ -411,9 +411,12 @@ However, **`[Params]` are applied to all the benchmarks in given class**. So if `[ArgumentsSource]` is xUnit's `[MemberData]` counterpart. +How the source maps onto the benchmark's parameters is read from its declared element type. + +For a **single argument** the source yields the parameter's own type: + ```cs -// for single argument it's an IEnumerable of objects (object), not IEnumerable of arrays of objects (object[]) -public IEnumerable Cultures() +public IEnumerable Cultures() { yield return new CultureInfo("fr"); yield return new CultureInfo("da"); @@ -427,6 +430,21 @@ public DateTime Parse(CultureInfo cultureInfo) => DateTime.Parse("10/10/2010 12:00:00 AM", cultureInfo); ``` +For **several arguments** it yields one element per case holding all of them, a `ValueTuple` naming each parameter's type: + +```cs +public IEnumerable<(string input, NumberStyles styles)> Inputs() +{ + yield return ("1234", NumberStyles.Integer); + yield return ("1,234", NumberStyles.AllowThousands); + yield return ("0x4D2", NumberStyles.HexNumber); +} + +[Benchmark] +[ArgumentsSource(nameof(Inputs))] +public int Parse(string input, NumberStyles styles) => int.Parse(input, styles); +``` + **Note:** If you need to use the argument in the setup method, then instead of using `ArgumentsSource` you should use `ParamsSource`. **Note:** `[ArgumentsSource]` methods are only called once per execution of the benchmark, and the cached results are used per invocation. @@ -445,7 +463,7 @@ public void Bitmap_FromStream(ImageTestData format) } } -public IEnumerable ImageFormats() => return new [] +public IEnumerable ImageFormats() => new [] { new ImageTestData(ImageFormat.Bmp), new ImageTestData(ImageFormat.Jpeg), diff --git a/eng/Versions.props b/eng/Versions.props index 3a80c2016b5..d288a68675e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,7 +11,7 @@ 11.0.0-rtm.26473.104 11.0.0-rtm.26473.104 11.0.0-rtm.26473.104 - 0.16.0-nightly.20260801.595 + 0.16.0-nightly.20260923.641 11.0.0-preview.5.26261.101 11.0.0-prerelease.26473.1 diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 7e04b64a6ae..3f047c6f69b 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -399,13 +399,12 @@ def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type: os.environ['RestoreAdditionalProjectSources'] = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmInterpreter, commit), "wasm_bundle") elif specific_run_type == RunType.WasmAOT: - benchmark_ci_args += ['--wasm', '--dotnet-path', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "dotnet")] + benchmark_ci_args += ['--wasm', '--wasm-runtime-flavor', 'MonoAOT', '--dotnet-path', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "dotnet")] bdn_args_unescaped += [ '--anyCategories', 'Libraries', 'Runtime', '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', '--cli', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "dotnet", "dotnet"), '--wasmEngine', parsed_args.wasm_engine_path, - '--aotcompilermode', 'wasm', '--logBuildOutput', '--generateBinLog' ] diff --git a/scripts/micro_benchmarks.py b/scripts/micro_benchmarks.py index b98bf4012c1..abfec26c3bc 100755 --- a/scripts/micro_benchmarks.py +++ b/scripts/micro_benchmarks.py @@ -145,8 +145,8 @@ def __get_bdn_arguments(user_input: str) -> list[str]: dest='wasm_runtime_flavor', required=False, default='Mono', - choices=['Mono', 'CoreCLR'], - help='Runtime flavor for WASM benchmarks: Mono (default) or CoreCLR' + choices=['Mono', 'MonoAOT', 'CoreCLR'], + help='Runtime flavor for WASM benchmarks: Mono interpreter (default), MonoAOT, or CoreCLR' ) parser.add_argument( @@ -298,22 +298,22 @@ def __get_benchmarkdotnet_arguments(framework: str, args: Any) -> list[str]: if framework.startswith("nativeaot"): run_args += ['--runtimes', framework] if args.wasm: - if framework == "net6.0": - run_args += ['--runtimes', 'wasm'] - elif framework == "net7.0": - run_args += ['--runtimes', 'wasmnet70'] - elif framework == "net8.0": - run_args += ['--runtimes', 'wasmnet80'] - elif framework == "net9.0": - run_args += ['--runtimes', 'wasmnet90'] - elif framework == "net10.0": - run_args += ['--runtimes', 'wasmnet10_0'] - elif framework == "net11.0": - run_args += ['--runtimes', 'wasmnet11_0'] - else: + wasm_moniker_prefix = { + 'Mono': 'monowasm', + 'MonoAOT': 'monowasmaot', + 'CoreCLR': 'corewasm', + }[args.wasm_runtime_flavor] + wasm_version = { + 'net6.0': '6.0', + 'net7.0': '7.0', + 'net8.0': '8.0', + 'net9.0': '9.0', + 'net10.0': '10.0', + 'net11.0': '11.0', + }.get(framework) + if wasm_version is None: raise ArgumentTypeError('Framework {} is not supported for wasm'.format(framework)) - if args.wasm_runtime_flavor != 'Mono': - run_args += ['--wasmRuntimeFlavor', args.wasm_runtime_flavor] + run_args += ['--runtimes', wasm_moniker_prefix + wasm_version] # Increase default 2 min build timeout to accommodate slow (or even very slow) hardware if not args.bdn_arguments or '--buildTimeout' not in args.bdn_arguments: diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index f3b8393240f..fee5505f441 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -554,7 +554,7 @@ def get_bdn_arguments( bdn_arguments += [ "--runtimes", "monoaotllvm", "--aotcompilerpath", "$HELIX_CORRELATION_PAYLOAD/monoaot/mono-aot-cross", - "--customruntimepack", "$HELIX_CORRELATION_PAYLOAD/monoaot/pack", + "--customruntimepack", "$HELIX_CORRELATION_PAYLOAD/monoaot/pack", "--aotcompilermode", "llvm", ] else: @@ -583,10 +583,7 @@ def get_bdn_arguments( bdn_arguments += ["\\\"--wasmArgs=--experimental-wasm-exnref\\\""] if is_aot: - bdn_arguments += [ - "--aotcompilermode", "wasm", - "--buildTimeout", "3600" - ] + bdn_arguments += ["--buildTimeout", "3600"] if runtime_type == "wasm_coreclr": category_exclusions += ["NoWASM", "NoWasmCoreCLR", "NoMono"] @@ -612,13 +609,13 @@ def get_bdn_arguments( ] if os_group == "windows": bdn_arguments += [ - "--runtimes", "r2r11_0", + "--runtimes", "r2r11.0", "--customruntimepack", "%HELIX_CORRELATION_PAYLOAD%\\r2r_interpreter\\runtimepack", "--aotcompilerpath", "%HELIX_CORRELATION_PAYLOAD%\\r2r_interpreter\\crossgen2", ] else: bdn_arguments += [ - "--runtimes", "r2r11_0", + "--runtimes", "r2r11.0", "--customruntimepack", "$HELIX_CORRELATION_PAYLOAD/r2r_interpreter/runtimepack", "--aotcompilerpath", "$HELIX_CORRELATION_PAYLOAD/r2r_interpreter/crossgen2", ] @@ -753,6 +750,7 @@ def get_work_item_command( bdn_artifacts_dir: str, wasm_coreclr: bool = False, wasm_ready_to_run: bool = False, + wasm_aot: bool = False, only_sanity_check: bool = False, wasm_workload_source: Optional[str] = None): if os_group == "windows": @@ -800,6 +798,8 @@ def get_work_item_command( "--wasm-workload-source", wasm_workload_source, ] + elif wasm_aot: + work_item_command += ["--wasm-runtime-flavor", "MonoAOT"] work_item_command += ["--bdn-artifacts", bdn_artifacts_dir] @@ -1496,6 +1496,7 @@ def get_work_item_command_for_artifact_dir(artifact_dir: str): artifact_dir, wasm_coreclr, wasm_coreclr and args.r2r_run_type == "r2r", + wasm_aot, args.only_sanity_check, helix_wasm_workload_source) diff --git a/src/benchmarks/micro/Program.cs b/src/benchmarks/micro/Program.cs index bca89678baa..d68add6797f 100644 --- a/src/benchmarks/micro/Program.cs +++ b/src/benchmarks/micro/Program.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.Runtime.Versioning; using System.Threading.Tasks; using BenchmarkDotNet.Running; using System.IO; @@ -15,6 +16,31 @@ namespace MicroBenchmarks { class Program { + // The MonoAotLLVM toolchain cannot build without these. Failing here reports an actionable message + // through the ArgumentException handler below, instead of passing an empty path into the generated + // project and failing deep inside MSBuild many minutes later. + private static string RequireMonoAotLLVMValue(List values, string parameter) + => values.Count > 0 + ? values[0] + : throw new ArgumentException($"{parameter} must be specified when running with --runtimes monoaotllvm"); + + // The TFM this assembly was compiled for. Environment.Version reports the runtime that actually + // loaded, which can roll forward to a newer major than --customruntimepack points at, producing an + // opaque restore/publish failure in the generated project. + private static string GetTargetFrameworkMoniker() + { + if (AppContext.TargetFrameworkName is string frameworkName) + { + var parsed = new FrameworkName(frameworkName); + if (parsed.Identifier == ".NETCoreApp") + { + return $"net{parsed.Version.Major}.{parsed.Version.Minor}"; + } + } + + return $"net{Environment.Version.Major}.0"; + } + static async Task Main(string[] args) { var argsList = new List(args); @@ -24,6 +50,7 @@ static async Task Main(string[] args) List categoryExclusionFilterValue; List msBuildArguments; bool getDiffableDisasm; + MonoAotLLVMToolChain monoAotToolchain = null; // Parse and remove any additional parameters that we need that aren't part of BDN try @@ -35,6 +62,58 @@ static async Task Main(string[] args) argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--msbuild-arguments", out msBuildArguments); CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--disasm-diff", out getDiffableDisasm); + // Extract monoaotllvm args not recognized by BDN and build the toolchain. + // BDN spells this option "-r, --runtimes" and matches monikers case-insensitively, so accept both forms here. + string runtimesArg = argsList.Contains("--runtimes") ? "--runtimes" : "-r"; + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, runtimesArg, out var runtimesValues); + int monoAotLLVMIndex = runtimesValues.FindIndex(runtime => string.Equals(runtime, "monoaotllvm", StringComparison.OrdinalIgnoreCase)); + if (monoAotLLVMIndex >= 0) + { + runtimesValues.RemoveAt(monoAotLLVMIndex); + if (runtimesValues.Count > 0) + { + argsList.Add(runtimesArg); + argsList.AddRange(runtimesValues); + } + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--cli", out var cliPathValue); + if (cliPathValue.Count > 0) + { + argsList.Add("--cli"); + argsList.AddRange(cliPathValue); + } + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--packages", out var packagesPathValue); + if (packagesPathValue.Count > 0) + { + argsList.Add("--packages"); + argsList.AddRange(packagesPathValue); + } + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--aotcompilerpath", out var aotCompilerPathValue); + string aotCompilerPath = RequireMonoAotLLVMValue(aotCompilerPathValue, "--aotcompilerpath"); + argsList.Add("--aotcompilerpath"); + argsList.AddRange(aotCompilerPathValue); + + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--customruntimepack", out var customRuntimePackValue); + string customRuntimePack = RequireMonoAotLLVMValue(customRuntimePackValue, "--customruntimepack"); + argsList.Add("--customruntimepack"); + argsList.AddRange(customRuntimePackValue); + + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--aotcompilermode", out var aotCompilerModeValue); + monoAotToolchain = new MonoAotLLVMToolChain(new MonoAotLLVMRuntime(Environment.Version), new() + { + CliPath = cliPathValue.Count > 0 ? new(cliPathValue[0]) : null, + PackagesPath = packagesPathValue.Count > 0 ? new(packagesPathValue[0]) : null, + TargetFrameworkMoniker = GetTargetFrameworkMoniker(), + CustomRuntimePack = customRuntimePack, + AotCompilerPath = aotCompilerPath, + AotCompilerMode = aotCompilerModeValue.Count > 0 ? Enum.Parse(aotCompilerModeValue[0], ignoreCase: true) : MonoAotCompilerMode.mini + }); + } + else if (runtimesValues.Count > 0) + { + argsList.Add(runtimesArg); + argsList.AddRange(runtimesValues); + } + CommandLineOptions.ValidatePartitionParameters(partitionCount, partitionIndex); } catch (ArgumentException e) @@ -43,11 +122,6 @@ static async Task Main(string[] args) return 1; } - // Use RunAsync (not Run) so BDN does not install its single-threaded - // BenchmarkDotNetSynchronizationContext on the entrypoint thread. The sync - // entrypoint installs that context before benchmark discovery, which - // deadlocks any sync-over-async work performed by [ParamsSource]/[ArgumentsSource] - // callbacks (e.g. SslStreamTests.GetTls13Support). var summaries = await BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) .RunAsync(argsList.ToArray(), @@ -59,7 +133,8 @@ static async Task Main(string[] args) exclusionFilterValue: exclusionFilterValue, categoryExclusionFilterValue: categoryExclusionFilterValue, getDiffableDisasm: getDiffableDisasm, - msBuildArguments: msBuildArguments)) + msBuildArguments: msBuildArguments, + toolchain: monoAotToolchain)) .ConfigureAwait(false); return summaries.ToExitCode(); diff --git a/src/benchmarks/micro/libraries/System.Buffers/Base64Tests.cs b/src/benchmarks/micro/libraries/System.Buffers/Base64Tests.cs index 268cfd0581a..1d64cfa13b0 100644 --- a/src/benchmarks/micro/libraries/System.Buffers/Base64Tests.cs +++ b/src/benchmarks/micro/libraries/System.Buffers/Base64Tests.cs @@ -99,7 +99,6 @@ public void SetupBase64IsValid() // To make the results stable the Iteration needs to last at least 100ms, this is why we are using bigger value for NumberOfBytes // Due to limitation of BDN, where Params have no Target and are applied to entire class the benchmarks live in a separate class. [BenchmarkCategory(Categories.Libraries, Categories.NoMono)] - [WarmupCount(30)] // make sure it's promoted to Tier 1 public class Base64EncodeDecodeInPlaceTests { [Params(1000 * 1000 * 200)] // allows for stable iteration around 200ms diff --git a/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs b/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs index 9adc9959fc7..c77a883706e 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs @@ -2,29 +2,18 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System.Linq; using System.Threading; using System.Threading.Tasks; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Attributes.Filters; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Filters; using MicroBenchmarks; namespace System.Collections.Concurrent { - internal class AddRemoveFromDifferentThreadsDisabledConfig : ManualConfig - { - public AddRemoveFromDifferentThreadsDisabledConfig() => AddFilter(new SimpleFilter(_ => false)); - } - [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections, Categories.NoWASM)] [GenericTypeArguments(typeof(int))] // value type [GenericTypeArguments(typeof(string))] // reference type - [MinWarmupCount(6, forceAutoWarmup: true)] - [MaxWarmupCount(10, forceAutoWarmup: true)] [AotFilter("It hangs. https://github.com/dotnet/runtime/issues/66987")] - [Config(typeof(AddRemoveFromDifferentThreadsDisabledConfig))] public class AddRemoveFromDifferentThreads { const int NumThreads = 2; diff --git a/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromSameThreads.cs b/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromSameThreads.cs index c7659a5fc52..3927123c0e7 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromSameThreads.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromSameThreads.cs @@ -6,23 +6,13 @@ using System.Threading; using System.Threading.Tasks; using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Filters; using MicroBenchmarks; namespace System.Collections.Concurrent { - internal class AddRemoveFromSameThreadsDisabledConfig : ManualConfig - { - public AddRemoveFromSameThreadsDisabledConfig() => AddFilter(new SimpleFilter(_ => false)); - } - [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections, Categories.NoWASM)] [GenericTypeArguments(typeof(int))] // value type [GenericTypeArguments(typeof(string))] // reference type - [MinWarmupCount(6, forceAutoWarmup: true)] - [MaxWarmupCount(10, forceAutoWarmup: true)] - [Config(typeof(AddRemoveFromSameThreadsDisabledConfig))] public class AddRemoveFromSameThreads { const int NumThreads = 2; diff --git a/src/benchmarks/micro/libraries/System.Collections/Sort.cs b/src/benchmarks/micro/libraries/System.Collections/Sort.cs index c57709e44d0..94b3c0e6ab7 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Sort.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Sort.cs @@ -17,7 +17,6 @@ namespace System.Collections [GenericTypeArguments(typeof(IntClass))] // custom reference type, sort in managed code [GenericTypeArguments(typeof(BigStruct))] // custom value type, sort in managed code [InvocationCount(InvocationsPerIteration)] - [MinWarmupCount(6, forceAutoWarmup: true)] // when InvocationCount is set, BDN does not run Pilot Stage, so to get the code promoted to Tier 1 before Actual Workload, we enforce more Warmups public class Sort where T : IComparable { private static readonly ComparableComparerClass _comparableComparerClass = new ComparableComparerClass(); diff --git a/src/benchmarks/micro/libraries/System.Net.Http/Perf.SocketsHttpHandler.cs b/src/benchmarks/micro/libraries/System.Net.Http/Perf.SocketsHttpHandler.cs index 5be3b82ec34..af4644c1c7a 100644 --- a/src/benchmarks/micro/libraries/System.Net.Http/Perf.SocketsHttpHandler.cs +++ b/src/benchmarks/micro/libraries/System.Net.Http/Perf.SocketsHttpHandler.cs @@ -147,12 +147,12 @@ public async Task Get_EnumerateHeaders_Unvalidated() #endif [GlobalCleanup] - public void Cleanup() + public async Task Cleanup() { _invoker.Dispose(); _handler.Dispose(); _listener.Dispose(); - _serverTask.GetAwaiter().GetResult(); + await _serverTask; _serverCert.Dispose(); } } diff --git a/src/benchmarks/micro/libraries/System.Net.Http/Perf.WinHttpHandler.cs b/src/benchmarks/micro/libraries/System.Net.Http/Perf.WinHttpHandler.cs index f07e4f09ec4..eb70d587e05 100644 --- a/src/benchmarks/micro/libraries/System.Net.Http/Perf.WinHttpHandler.cs +++ b/src/benchmarks/micro/libraries/System.Net.Http/Perf.WinHttpHandler.cs @@ -114,12 +114,12 @@ public async Task Get() } [GlobalCleanup] - public void Cleanup() + public async Task Cleanup() { _invoker.Dispose(); _handler.Dispose(); _listener.Dispose(); - _serverTask.GetAwaiter().GetResult(); + await _serverTask; _serverCert.Dispose(); } } diff --git a/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs b/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs index f2c33c61242..4b534069365 100644 --- a/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs +++ b/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs @@ -6,22 +6,20 @@ using System.Security.Authentication; using System.Threading.Tasks; using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Extensions; using MicroBenchmarks; namespace System.Net.Security.Tests { public partial class SslStreamTests { - private static Lazy s_supportsTls13 = new Lazy(GetTls13Support); - public static bool SupportsTls13 => s_supportsTls13.Value; + private static readonly Lazy> s_supportsTls13 = new(GetTls13SupportAsync); - public static IEnumerable TlsProtocols() + public static async IAsyncEnumerable TlsProtocols() { - yield return new object[] { SslProtocols.Tls12 }; - if (SupportsTls13) + yield return SslProtocols.Tls12; + if (await s_supportsTls13.Value) { - yield return new object[] { SslProtocols.Tls13 }; + yield return SslProtocols.Tls13; } } @@ -50,11 +48,11 @@ public static IEnumerable TlsProtocols() [ArgumentsSource(nameof(TlsProtocols))] public Task HandshakeRSA4096CertAsync(SslProtocols protocol) => SslStreamTests.HandshakeAsync(SslStreamTests._rsa4096Cert, protocol); - private static bool GetTls13Support() + private static async Task GetTls13SupportAsync() { try { - SslStreamTests.HandshakeAsync(SslStreamTests._rsa2048Cert, SslProtocols.Tls13).GetAwaiter().GetResult(); + await SslStreamTests.HandshakeAsync(SslStreamTests._rsa2048Cert, SslProtocols.Tls13); return true; } catch diff --git a/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.cs b/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.cs index 71adad43b9a..55d01758d33 100644 --- a/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.cs +++ b/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.cs @@ -36,7 +36,7 @@ public partial class SslStreamTests private PipeStream _clientPipe, _serverPipe; // used for handshake tests [GlobalSetup] - public void Setup() + public async Task Setup() { string pipeName = "SetupTlsHandshakePipe"; using (var listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)) @@ -51,7 +51,7 @@ public void Setup() _sslClient = new SslStream(new NetworkStream(client, ownsSocket: true), leaveInnerStreamOpen: false, delegate { return true; }); _sslServer = new SslStream(new NetworkStream(server, ownsSocket: true), leaveInnerStreamOpen: false, delegate { return true; }); - Task.WaitAll( + await Task.WhenAll( _sslClient.AuthenticateAsClientAsync("localhost", null, SslProtocols.None, checkCertificateRevocation: false), _sslServer.AuthenticateAsServerAsync(_cert, clientCertificateRequired: false, SslProtocols.None, checkCertificateRevocation: false)); @@ -80,7 +80,7 @@ public void Setup() // Create PIPE Pair. var pipeServer = new NamedPipeServerStream(pipeName, PipeDirection.InOut, 1, PipeTransmissionMode.Byte, PipeOptions.Asynchronous | PipeOptions.WriteThrough); var pipeClient = new NamedPipeClientStream(".", pipeName, PipeDirection.InOut, PipeOptions.Asynchronous | PipeOptions.WriteThrough); - Task.WaitAll(pipeServer.WaitForConnectionAsync(), pipeClient.ConnectAsync()); + await Task.WhenAll(pipeServer.WaitForConnectionAsync(), pipeClient.ConnectAsync()); _serverPipe = pipeServer; _clientPipe = pipeClient; } diff --git a/src/benchmarks/micro/libraries/System.Runtime/Perf.Uri.cs b/src/benchmarks/micro/libraries/System.Runtime/Perf.Uri.cs index 32e60716ba9..b4424a34dae 100644 --- a/src/benchmarks/micro/libraries/System.Runtime/Perf.Uri.cs +++ b/src/benchmarks/micro/libraries/System.Runtime/Perf.Uri.cs @@ -12,21 +12,21 @@ namespace System.Tests [BenchmarkCategory(Categories.Libraries)] public class Perf_Uri { - public static IEnumerable Ctor_MemberData() + public static IEnumerable Ctor_MemberData() { - yield return new object[] { "http://dot.net" }; - yield return new object[] { "https://contoso.com" }; - yield return new object[] { "https://CONTOSO.com" }; - yield return new object[] { "https://a.much.longer.domain.name" }; - yield return new object[] { "http://h\u00F6st.with.\u00FCnicode" }; - yield return new object[] { "http://xn--hst-sna.with.xn--nicode-2ya" }; + yield return "http://dot.net"; + yield return "https://contoso.com"; + yield return "https://CONTOSO.com"; + yield return "https://a.much.longer.domain.name"; + yield return "http://h\u00F6st.with.\u00FCnicode"; + yield return "http://xn--hst-sna.with.xn--nicode-2ya"; } - public static IEnumerable CtorIdnHostPathAndQuery_MemberData() + public static IEnumerable CtorIdnHostPathAndQuery_MemberData() { - foreach (object[] schemeAndAuthority in Ctor_MemberData()) + foreach (string schemeAndAuthority in Ctor_MemberData()) { - yield return new object[] { $"{schemeAndAuthority[0]}/path/with?key=value#fragment" }; + yield return $"{schemeAndAuthority}/path/with?key=value#fragment"; } string[] paths = new[] @@ -41,16 +41,16 @@ public static IEnumerable CtorIdnHostPathAndQuery_MemberData() foreach (string path in paths) { - yield return new object[] { $"http://host{path}" }; + yield return $"http://host{path}"; } } - public static IEnumerable EscapeDataString_MemberData() + public static IEnumerable EscapeDataString_MemberData() { - yield return new object[] { new string('a', 1000) }; // Nothing to escape - yield return new object[] { new string('{', 1000) }; // ASCII that needs escaping - yield return new object[] { new string('\u00FC', 1000) }; // Unicode - yield return new object[] { string.Concat(Enumerable.Repeat("a{\u00FC", 333)) }; + yield return new string('a', 1000); // Nothing to escape + yield return new string('{', 1000); // ASCII that needs escaping + yield return new string('\u00FC', 1000); // Unicode + yield return string.Concat(Enumerable.Repeat("a{\u00FC", 333)); } private static readonly Uri _uri = new Uri("http://contoso.com/path/with?key=value#fragment"); diff --git a/src/benchmarks/micro/libraries/System.Threading.Tasks.Extensions/Perf.ValueTask.cs b/src/benchmarks/micro/libraries/System.Threading.Tasks.Extensions/Perf.ValueTask.cs index b4b04834303..c42f6d5e189 100644 --- a/src/benchmarks/micro/libraries/System.Threading.Tasks.Extensions/Perf.ValueTask.cs +++ b/src/benchmarks/micro/libraries/System.Threading.Tasks.Extensions/Perf.ValueTask.cs @@ -11,8 +11,6 @@ namespace System.Threading.Tasks { [BenchmarkCategory(Categories.Libraries)] - [MinWarmupCount(2, forceAutoWarmup: true)] // these benchmarks require more warmups than in our default config - [MaxWarmupCount(10, forceAutoWarmup: true)] public class ValueTaskPerfTest { private Task _completedTask = Task.FromResult(42); diff --git a/src/benchmarks/micro/runtime/Span/Sorting.cs b/src/benchmarks/micro/runtime/Span/Sorting.cs index 969f35c083a..6c5e0a95f89 100644 --- a/src/benchmarks/micro/runtime/Span/Sorting.cs +++ b/src/benchmarks/micro/runtime/Span/Sorting.cs @@ -13,7 +13,6 @@ namespace Span { [BenchmarkCategory(Categories.Runtime)] [InvocationCount(InvocationsPerIteration)] - [MinWarmupCount(6, forceAutoWarmup: true)] // when InvocationCount is set, BDN does not run Pilot Stage, so to get the code promoted to Tier 1 before Actual Workload, we enforce more Warmups public class Sorting { private const int InvocationsPerIteration = 10_000; diff --git a/src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj b/src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj index 0b437d5164f..b7f01245234 100644 --- a/src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj +++ b/src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/harness/BenchmarkDotNet.Extensions/AotFilterAttribute.cs b/src/harness/BenchmarkDotNet.Extensions/AotFilterAttribute.cs new file mode 100644 index 00000000000..bf89591a716 --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/AotFilterAttribute.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Filters; + +namespace BenchmarkDotNet.Attributes.Filters; + +public class AotFilterAttribute : FilterConfigBaseAttribute +{ + public AotFilterAttribute(string? reason = null) + : base(new SimpleFilter(benchmark => benchmark.GetRuntime() is not (NativeAotRuntime or MonoAotLLVMRuntime))) + { + } +} diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotCompilerMode.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotCompilerMode.cs new file mode 100644 index 00000000000..fa9433b9a9a --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotCompilerMode.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace BenchmarkDotNet.Extensions; + +public enum MonoAotCompilerMode +{ + mini, + llvm +} diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMBuilder.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMBuilder.cs new file mode 100644 index 00000000000..612e71c8cbc --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMBuilder.cs @@ -0,0 +1,129 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.IO; +using System.Runtime.InteropServices; +using System.Xml.Linq; +using BenchmarkDotNet.Code; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.CsProj; + +namespace BenchmarkDotNet.Extensions; + +public class MonoAotLLVMBuilder : CsProjBuilder +{ + private readonly MonoAotLLVMSettings _settings; + + public MonoAotLLVMBuilder(MonoAotLLVMSettings settings) : base(settings) + { + _settings = settings; + BenchmarkRunCallType = CodeGenBenchmarkRunCallType.Direct; + } + + // The AOT compilation runs after Publish, so the app has to be published rather than built. + protected override bool PublishesOutput => true; + + // The app is published against the custom runtime pack, which only provides Microsoft.NETCore.App, so the + // frameworks another SDK would reference could not be resolved. + protected override string GetSdkName(XElement benchmarkProject) => "Microsoft.NET.Sdk"; + + protected override void AddEarlyProperties(XElement project, BuildPartition buildPartition, ArtifactsPaths artifactsPaths, FileInfo projectFile) + { + project.Add(new XElement("PropertyGroup", + new XElement("OriginalCSProjPath", projectFile.FullName), + new XElement("MonoPropsPath", "$([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Mono.props'))"), + new XElement("MonoTargetsPath", "$([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Mono.targets'))"))); + + project.Add(new XElement("Import", + new XAttribute("Project", "$(MonoPropsPath)"), + new XAttribute("Condition", "Exists($(MonoPropsPath))"))); + + base.AddEarlyProperties(project, buildPartition, artifactsPaths, projectFile); + } + + protected override void AddProjectContent(XElement project, BuildPartition buildPartition, ArtifactsPaths artifactsPaths, FileInfo projectFile) + { + base.AddProjectContent(project, buildPartition, artifactsPaths, projectFile); + + project.Add(new XElement("PropertyGroup", + new XElement("MicrosoftNetCoreAppRuntimePackDir", _settings.CustomRuntimePack), + new XElement("RuntimeIdentifier", RuntimeInformation.RuntimeIdentifier), + new XElement("EnableTargetingPackDownload", "false"), + new XElement("PublishTrimmed", "false"), + new XElement("ErrorOnDuplicatePublishOutputFiles", "false"))); + + // Self-contained is what makes the publish use the custom runtime pack. BenchmarkDotNet's reference-gathering + // build disables the app host, which self-contained requires, so it is left out of that pass. It is set here + // rather than passed as --self-contained, which would also restore the benchmark project for this RID. + project.Add(new XElement("PropertyGroup", + new XAttribute("Condition", "'$(BenchmarkDotNetGatherReferences)' != 'true'"), + new XElement("SelfContained", "true"))); + + project.Add(new XElement("ItemGroup", + new XElement("PackageReference", + new XAttribute("Include", "Microsoft.NET.Runtime.MonoAOTCompiler.Task"), + new XAttribute("Version", "6.0.0-*"), + new XAttribute("GeneratePathProperty", "true")))); + + project.Add(new XComment(" Redirect 'dotnet publish' to in-tree runtime pack ")); + project.Add(new XElement("Target", + new XAttribute("Name", "TrickRuntimePackLocation"), + new XAttribute("AfterTargets", "ProcessFrameworkReferences"), + new XElement("ItemGroup", + new XElement("RuntimePack", + new XElement("PackageDirectory", "$(MicrosoftNetCoreAppRuntimePackDir)"))), + new XElement("Message", + new XAttribute("Text", "Packaged ID: %(RuntimePack.PackageDirectory)"), + new XAttribute("Importance", "high")))); + + project.Add(new XElement("UsingTask", + new XAttribute("TaskName", "MonoAOTCompiler"), + new XAttribute("AssemblyFile", "$(PkgMicrosoft_NET_Runtime_MonoAOTCompiler_Task)/MonoAOTCompiler.dll"))); + + project.Add(new XElement("Target", + new XAttribute("Name", "AotApp"), + new XAttribute("AfterTargets", "Publish"), + new XElement("PropertyGroup", + new XElement("PublishDirFullPath", "$([System.IO.Path]::GetFullPath($(PublishDir)))"), + new XElement("SharedLibraryType", new XAttribute("Condition", "$([MSBuild]::IsOSPlatform('OSX'))"), "Dylib"), + new XElement("SharedLibraryType", new XAttribute("Condition", "$([MSBuild]::IsOSPlatform('Windows'))"), "Dll"), + new XElement("SharedLibraryType", new XAttribute("Condition", "'$(SharedLibraryType)' == ''"), "So")), + new XElement("ItemGroup", + new XElement("AotInputAssemblies", + new XAttribute("Include", "$(PublishDirFullPath)\\*.dll"), + new XElement("AotArguments", "mcpu=native"))), + new XElement("MonoAOTCompiler", + new XAttribute("CompilerBinaryPath", _settings.AotCompilerPath), + new XAttribute("Mode", "Normal"), + new XAttribute("OutputType", "Library"), + new XAttribute("LibraryFormat", "$(SharedLibraryType)"), + new XAttribute("Assemblies", "@(AotInputAssemblies)"), + new XAttribute("UseLLVM", _settings.AotCompilerMode == MonoAotCompilerMode.llvm ? "true" : "false"), + new XAttribute("LLVMPath", "$(MicrosoftNetCoreAppRuntimePackDir)\\runtimes\\$(RuntimeIdentifier)\\native"), + new XAttribute("OutputDir", "$(PublishDir)"), + new XAttribute("UseAotDataFile", "false"), + new XAttribute("IntermediateOutputPath", "$(IntermediateOutputPath)"), + new XElement("Output", + new XAttribute("TaskParameter", "CompiledAssemblies"), + new XAttribute("ItemName", "BundleAssemblies"))), + new XElement("Message", + new XAttribute("Text", "CompiledAssemblies: $(BundleAssemblies)"), + new XAttribute("Importance", "high")))); + } + + protected override void AddLateProperties(XElement project, BuildPartition buildPartition, ArtifactsPaths artifactsPaths, FileInfo projectFile) + { + base.AddLateProperties(project, buildPartition, artifactsPaths, projectFile); + + // After the late properties so the Mono targets can override anything the generated project defines. + project.Add(new XElement("Import", + new XAttribute("Project", "$(MonoTargetsPath)"), + new XAttribute("Condition", "Exists($(MonoTargetsPath))"))); + } + + // A self-contained publish produces an app host, so the executable is not the managed dll the base returns. + protected override string GetExecutableExtension() + => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".exe" : string.Empty; +} diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMRuntime.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMRuntime.cs new file mode 100644 index 00000000000..10d316c171c --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMRuntime.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; + +namespace BenchmarkDotNet.Extensions; + +public sealed class MonoAotLLVMRuntime(Version version) : Runtime +{ + public override string Name => "MonoAOTLLVM"; + + public override Version Version { get; } = version; + + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) + => throw new NotSupportedException("MonoAOTLLVM benchmarks must set the toolchain explicitly."); +} diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMSettings.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMSettings.cs new file mode 100644 index 00000000000..ae1ba8518b4 --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMSettings.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Extensions; + +public sealed record MonoAotLLVMSettings : DotNetCliSettings +{ + public required string CustomRuntimePack { get; init; } + public required string AotCompilerPath { get; init; } + public required MonoAotCompilerMode AotCompilerMode { get; init; } +} diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMToolChain.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMToolChain.cs new file mode 100644 index 00000000000..7a1928f5a87 --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMToolChain.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using BenchmarkDotNet.Toolchains; + +namespace BenchmarkDotNet.Extensions; + +public class MonoAotLLVMToolChain(MonoAotLLVMRuntime runtime, MonoAotLLVMSettings settings) + : Toolchain("MonoAotLLVM", runtime, new MonoAotLLVMBuilder(settings), Toolchains.Executor.Instance) +{ +} diff --git a/src/harness/BenchmarkDotNet.Extensions/Polyfills/AsyncEnumerable.cs b/src/harness/BenchmarkDotNet.Extensions/Polyfills/AsyncEnumerable.cs new file mode 100644 index 00000000000..7f6eb880933 --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/Polyfills/AsyncEnumerable.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#if !NET10_0_OR_GREATER +using System.Collections.Generic; + +namespace System.Linq; + +// System.Linq.AsyncEnumerable ships in the shared framework from .NET 10. This project targets net8.0 (the lowest TFM the perf pipelines exercise), +// so the member it needs is polyfilled here rather than taking the System.Linq.AsyncEnumerable package. This mirrors BenchmarkDotNet's own polyfill. +internal static class AsyncEnumerable +{ + public static IAsyncEnumerable ToAsyncEnumerable(this IEnumerable source) + { + ArgumentNullException.ThrowIfNull(source); + + return FromIterator(source); + + static async IAsyncEnumerable FromIterator(IEnumerable source) + { + foreach (TSource element in source) + { + yield return element; + } + } + } +} +#endif diff --git a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs index 2630289ae2c..01018485164 100644 --- a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs +++ b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs @@ -1,11 +1,13 @@ using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Exporters.Json; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Toolchains; using Perfolizer.Horology; using Reporting; using System; @@ -29,19 +31,24 @@ public static IConfig Create( List? categoryExclusionFilterValue = null, Job? job = null, bool getDiffableDisasm = false, - IReadOnlyList? msBuildArguments = null) + IReadOnlyList? msBuildArguments = null, + IToolchain? toolchain = null) { if (job is null) { - #pragma warning disable CS0618 // WithEvaluateOverhead is obsolete but needed for WASM accuracy job = Job.Default - .WithWarmupCount(1) // 1 warmup is enough for our purpose + .WithJitTieringMode(JitTieringMode.Force) // Force benchmarks to be promoted to tier 1, even for longer-running benchmarks that the default mode might skip (e.g. large InvocationsPerIteration) + .WithWarmupCount(0) // Jit stage already warms the benchmarks .WithIterationTime(TimeInterval.FromMilliseconds(250)) // the default is 0.5s per iteration, which is slighlty too much for us .WithMinIterationCount(15) .WithMaxIterationCount(20) // we don't want to run more that 20 iterations .WithEvaluateOverhead(Environment.GetEnvironmentVariable("PERFLAB_EVALUATE_OVERHEAD") == "1") // WASM has significant method-call overhead (1-10ns); subtract it when enabled .DontEnforcePowerPlan(); // make sure BDN does not try to enforce High Performance power plan on Windows - #pragma warning restore CS0618 + } + + if (toolchain is not null) + { + job = job.WithToolchain(toolchain); } if (msBuildArguments is not null && msBuildArguments.Count > 0)