From 7a0e02e89cb898d4eebb84d64ee9b4b865e89784 Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Thu, 13 Aug 2026 23:00:43 -0400 Subject: [PATCH 1/5] Add MonoAotLLVMToolchain. Update recommended config. Reenable concurrent benchmarks. --- scripts/benchmarks_local.py | 3 +- scripts/micro_benchmarks.py | 34 +++---- scripts/run_performance_job.py | 17 ++-- src/benchmarks/micro/Program.cs | 67 ++++++++++++- .../libraries/System.Buffers/Base64Tests.cs | 1 - .../AddRemoveFromDifferentThreads.cs | 11 --- .../Concurrent/AddRemoveFromSameThreads.cs | 10 -- .../libraries/System.Collections/Sort.cs | 1 - .../Perf.ValueTask.cs | 2 - src/benchmarks/micro/runtime/Span/Sorting.cs | 1 - .../AotFilterAttribute.cs | 17 ++++ .../BenchmarkDotNet.Extensions.csproj | 3 + .../MonoAotLLVM/MonoAotCompilerMode.cs | 11 +++ .../MonoAotLLVM/MonoAotLLVMCsProj.txt | 96 +++++++++++++++++++ .../MonoAotLLVM/MonoAotLLVMGenerator.cs | 77 +++++++++++++++ .../MonoAotLLVM/MonoAotLLVMRuntime.cs | 20 ++++ .../MonoAotLLVM/MonoAotLLVMSettings.cs | 14 +++ .../MonoAotLLVM/MonoAotLLVMToolChain.cs | 16 ++++ .../RecommendedConfig.cs | 15 ++- 19 files changed, 357 insertions(+), 59 deletions(-) create mode 100644 src/harness/BenchmarkDotNet.Extensions/AotFilterAttribute.cs create mode 100644 src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotCompilerMode.cs create mode 100644 src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMCsProj.txt create mode 100644 src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMGenerator.cs create mode 100644 src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMRuntime.cs create mode 100644 src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMSettings.cs create mode 100644 src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMToolChain.cs diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 7e04b64a6ae..3f047c6f69b 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -399,13 +399,12 @@ def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type: os.environ['RestoreAdditionalProjectSources'] = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmInterpreter, commit), "wasm_bundle") elif specific_run_type == RunType.WasmAOT: - benchmark_ci_args += ['--wasm', '--dotnet-path', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "dotnet")] + benchmark_ci_args += ['--wasm', '--wasm-runtime-flavor', 'MonoAOT', '--dotnet-path', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "dotnet")] bdn_args_unescaped += [ '--anyCategories', 'Libraries', 'Runtime', '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', '--cli', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "dotnet", "dotnet"), '--wasmEngine', parsed_args.wasm_engine_path, - '--aotcompilermode', 'wasm', '--logBuildOutput', '--generateBinLog' ] diff --git a/scripts/micro_benchmarks.py b/scripts/micro_benchmarks.py index f6ccb81214f..6f434321f08 100755 --- a/scripts/micro_benchmarks.py +++ b/scripts/micro_benchmarks.py @@ -145,8 +145,8 @@ def __get_bdn_arguments(user_input: str) -> list[str]: dest='wasm_runtime_flavor', required=False, default='Mono', - choices=['Mono', 'CoreCLR'], - help='Runtime flavor for WASM benchmarks: Mono (default) or CoreCLR' + choices=['Mono', 'MonoAOT', 'CoreCLR'], + help='Runtime flavor for WASM benchmarks: Mono interpreter (default), MonoAOT, or CoreCLR' ) parser.add_argument( @@ -268,22 +268,22 @@ def __get_benchmarkdotnet_arguments(framework: str, args: Any) -> list[str]: if framework.startswith("nativeaot"): run_args += ['--runtimes', framework] if args.wasm: - if framework == "net6.0": - run_args += ['--runtimes', 'wasm'] - elif framework == "net7.0": - run_args += ['--runtimes', 'wasmnet70'] - elif framework == "net8.0": - run_args += ['--runtimes', 'wasmnet80'] - elif framework == "net9.0": - run_args += ['--runtimes', 'wasmnet90'] - elif framework == "net10.0": - run_args += ['--runtimes', 'wasmnet10_0'] - elif framework == "net11.0": - run_args += ['--runtimes', 'wasmnet11_0'] - else: + wasm_moniker_prefix = { + 'Mono': 'monowasm', + 'MonoAOT': 'monowasmaot', + 'CoreCLR': 'corewasm', + }[args.wasm_runtime_flavor] + wasm_version = { + 'net6.0': '6.0', + 'net7.0': '7.0', + 'net8.0': '8.0', + 'net9.0': '9.0', + 'net10.0': '10.0', + 'net11.0': '11.0', + }.get(framework) + if wasm_version is None: raise ArgumentTypeError('Framework {} is not supported for wasm'.format(framework)) - if args.wasm_runtime_flavor != 'Mono': - run_args += ['--wasmRuntimeFlavor', args.wasm_runtime_flavor] + run_args += ['--runtimes', wasm_moniker_prefix + wasm_version] # Increase default 2 min build timeout to accommodate slow (or even very slow) hardware if not args.bdn_arguments or '--buildTimeout' not in args.bdn_arguments: diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index 2f3aefc0161..d19b01ea901 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -501,7 +501,7 @@ def get_bdn_arguments( bdn_arguments += [ "--runtimes", "monoaotllvm", "--aotcompilerpath", "$HELIX_CORRELATION_PAYLOAD/monoaot/mono-aot-cross", - "--customruntimepack", "$HELIX_CORRELATION_PAYLOAD/monoaot/pack", + "--customruntimepack", "$HELIX_CORRELATION_PAYLOAD/monoaot/pack", "--aotcompilermode", "llvm", ] else: @@ -530,10 +530,7 @@ def get_bdn_arguments( bdn_arguments += ["\\\"--wasmArgs=--experimental-wasm-exnref\\\""] if is_aot: - bdn_arguments += [ - "--aotcompilermode", "wasm", - "--buildTimeout", "3600" - ] + bdn_arguments += ["--buildTimeout", "3600"] if runtime_type == "wasm_coreclr": category_exclusions += ["NoWASM", "NoWasmCoreCLR", "NoMono"] @@ -555,13 +552,13 @@ def get_bdn_arguments( if runtime_type == "coreclr_r2r_interpreter": if os_group == "windows": bdn_arguments += [ - "--runtimes", "r2r11_0", + "--runtimes", "r2r11.0", "--customruntimepack", "%HELIX_CORRELATION_PAYLOAD%\\r2r_interpreter\\runtimepack", "--aotcompilerpath", "%HELIX_CORRELATION_PAYLOAD%\\r2r_interpreter\\crossgen2", ] else: bdn_arguments += [ - "--runtimes", "r2r11_0", + "--runtimes", "r2r11.0", "--customruntimepack", "$HELIX_CORRELATION_PAYLOAD/r2r_interpreter/runtimepack", "--aotcompilerpath", "$HELIX_CORRELATION_PAYLOAD/r2r_interpreter/crossgen2", ] @@ -684,7 +681,7 @@ def get_run_configurations( return configurations -def get_work_item_command(os_group: str, target_csproj: str, architecture: str, perf_lab_framework: str, internal: bool, wasm: bool, bdn_artifacts_dir: str, wasm_coreclr: bool = False, only_sanity_check: bool = False): +def get_work_item_command(os_group: str, target_csproj: str, architecture: str, perf_lab_framework: str, internal: bool, wasm: bool, bdn_artifacts_dir: str, wasm_coreclr: bool = False, wasm_aot: bool = False, only_sanity_check: bool = False): if os_group == "windows": work_item_command = [ "python", @@ -714,6 +711,8 @@ def get_work_item_command(os_group: str, target_csproj: str, architecture: str, work_item_command += ["--run-isolated", "--wasm", "--dotnet-path", "$HELIX_CORRELATION_PAYLOAD/dotnet/"] if wasm_coreclr: work_item_command += ["--wasm-runtime-flavor", "CoreCLR"] + elif wasm_aot: + work_item_command += ["--wasm-runtime-flavor", "MonoAOT"] work_item_command += ["--bdn-artifacts", bdn_artifacts_dir] @@ -1360,7 +1359,7 @@ def get_bdn_args_for_coreroot_dir(coreroot_dir: Optional[str]): def get_work_item_command_for_artifact_dir(artifact_dir: str): assert args.target_csproj is not None - return get_work_item_command(args.os_group, args.target_csproj, args.architecture, perf_lab_framework, args.internal, wasm, artifact_dir, wasm_coreclr, args.only_sanity_check) + return get_work_item_command(args.os_group, args.target_csproj, args.architecture, perf_lab_framework, args.internal, wasm, artifact_dir, wasm_coreclr, wasm_aot, args.only_sanity_check) work_item_command = get_work_item_command_for_artifact_dir(bdn_artifacts_directory) baseline_work_item_command = get_work_item_command_for_artifact_dir(bdn_baseline_artifacts_dir) diff --git a/src/benchmarks/micro/Program.cs b/src/benchmarks/micro/Program.cs index 8d2a590cae9..aa974b7b031 100644 --- a/src/benchmarks/micro/Program.cs +++ b/src/benchmarks/micro/Program.cs @@ -23,6 +23,7 @@ static async Task Main(string[] args) List exclusionFilterValue; List categoryExclusionFilterValue; bool getDiffableDisasm; + MonoAotLLVMToolChain monoAotToolchain = null; // Parse and remove any additional parameters that we need that aren't part of BDN try @@ -33,6 +34,69 @@ static async Task Main(string[] args) argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--category-exclusion-filter", out categoryExclusionFilterValue); CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--disasm-diff", out getDiffableDisasm); + // Extract monoaotllvm args not recognized by BDN and build the toolchain. + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--runtimes", out var runtimesValues); + if (runtimesValues.Remove("monoaotllvm")) + { + if (runtimesValues.Count > 0) + { + argsList.Add("--runtimes"); + argsList.AddRange(runtimesValues); + } + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--cli", out var cliPathValue); + if (cliPathValue.Count > 0) + { + argsList.Add("--cli"); + argsList.AddRange(cliPathValue); + } + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--packages", out var packagesPathValue); + if (packagesPathValue.Count > 0) + { + argsList.Add("--packages"); + argsList.AddRange(packagesPathValue); + } + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--aotcompilerpath", out var aotCompilerPathValue); + string aotCompilerPath; + if (aotCompilerPathValue.Count > 0) + { + aotCompilerPath = aotCompilerPathValue[0]; + argsList.Add("--aotcompilerpath"); + argsList.AddRange(aotCompilerPathValue); + } + else + { + aotCompilerPath = ""; + } + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--customruntimepack", out var customRuntimePackValue); + string customRuntimePack; + if (customRuntimePackValue.Count > 0) + { + customRuntimePack = customRuntimePackValue[0]; + argsList.Add("--customruntimepack"); + argsList.AddRange(customRuntimePackValue); + + } + else + { + customRuntimePack = ""; + } + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--aotcompilermode", out var aotCompilerModeValue); + monoAotToolchain = new MonoAotLLVMToolChain(new MonoAotLLVMRuntime(Environment.Version), new() + { + CliPath = cliPathValue.Count > 0 ? new(cliPathValue[0]) : null, + PackagesPath = packagesPathValue.Count > 0 ? new(packagesPathValue[0]) : null, + TargetFrameworkMoniker = $"net{Environment.Version.Major}.0", + CustomRuntimePack = customRuntimePack, + AotCompilerPath = aotCompilerPath, + AotCompilerMode = aotCompilerModeValue.Count > 0 ? Enum.Parse(aotCompilerModeValue[0], ignoreCase: true) : 0 + }); + } + else + { + argsList.Add("--runtimes"); + argsList.AddRange(runtimesValues); + } + CommandLineOptions.ValidatePartitionParameters(partitionCount, partitionIndex); } catch (ArgumentException e) @@ -56,7 +120,8 @@ static async Task Main(string[] args) partitionIndex: partitionIndex, exclusionFilterValue: exclusionFilterValue, categoryExclusionFilterValue: categoryExclusionFilterValue, - getDiffableDisasm: getDiffableDisasm) + getDiffableDisasm: getDiffableDisasm, + toolchain: monoAotToolchain) .AddValidator(new NoWasmValidator(Categories.NoWASM))) .ConfigureAwait(false); diff --git a/src/benchmarks/micro/libraries/System.Buffers/Base64Tests.cs b/src/benchmarks/micro/libraries/System.Buffers/Base64Tests.cs index 268cfd0581a..1d64cfa13b0 100644 --- a/src/benchmarks/micro/libraries/System.Buffers/Base64Tests.cs +++ b/src/benchmarks/micro/libraries/System.Buffers/Base64Tests.cs @@ -99,7 +99,6 @@ public void SetupBase64IsValid() // To make the results stable the Iteration needs to last at least 100ms, this is why we are using bigger value for NumberOfBytes // Due to limitation of BDN, where Params have no Target and are applied to entire class the benchmarks live in a separate class. [BenchmarkCategory(Categories.Libraries, Categories.NoMono)] - [WarmupCount(30)] // make sure it's promoted to Tier 1 public class Base64EncodeDecodeInPlaceTests { [Params(1000 * 1000 * 200)] // allows for stable iteration around 200ms diff --git a/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs b/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs index 9adc9959fc7..c77a883706e 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs @@ -2,29 +2,18 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System.Linq; using System.Threading; using System.Threading.Tasks; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Attributes.Filters; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Filters; using MicroBenchmarks; namespace System.Collections.Concurrent { - internal class AddRemoveFromDifferentThreadsDisabledConfig : ManualConfig - { - public AddRemoveFromDifferentThreadsDisabledConfig() => AddFilter(new SimpleFilter(_ => false)); - } - [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections, Categories.NoWASM)] [GenericTypeArguments(typeof(int))] // value type [GenericTypeArguments(typeof(string))] // reference type - [MinWarmupCount(6, forceAutoWarmup: true)] - [MaxWarmupCount(10, forceAutoWarmup: true)] [AotFilter("It hangs. https://github.com/dotnet/runtime/issues/66987")] - [Config(typeof(AddRemoveFromDifferentThreadsDisabledConfig))] public class AddRemoveFromDifferentThreads { const int NumThreads = 2; diff --git a/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromSameThreads.cs b/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromSameThreads.cs index c7659a5fc52..3927123c0e7 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromSameThreads.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromSameThreads.cs @@ -6,23 +6,13 @@ using System.Threading; using System.Threading.Tasks; using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Filters; using MicroBenchmarks; namespace System.Collections.Concurrent { - internal class AddRemoveFromSameThreadsDisabledConfig : ManualConfig - { - public AddRemoveFromSameThreadsDisabledConfig() => AddFilter(new SimpleFilter(_ => false)); - } - [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections, Categories.NoWASM)] [GenericTypeArguments(typeof(int))] // value type [GenericTypeArguments(typeof(string))] // reference type - [MinWarmupCount(6, forceAutoWarmup: true)] - [MaxWarmupCount(10, forceAutoWarmup: true)] - [Config(typeof(AddRemoveFromSameThreadsDisabledConfig))] public class AddRemoveFromSameThreads { const int NumThreads = 2; diff --git a/src/benchmarks/micro/libraries/System.Collections/Sort.cs b/src/benchmarks/micro/libraries/System.Collections/Sort.cs index c57709e44d0..94b3c0e6ab7 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Sort.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Sort.cs @@ -17,7 +17,6 @@ namespace System.Collections [GenericTypeArguments(typeof(IntClass))] // custom reference type, sort in managed code [GenericTypeArguments(typeof(BigStruct))] // custom value type, sort in managed code [InvocationCount(InvocationsPerIteration)] - [MinWarmupCount(6, forceAutoWarmup: true)] // when InvocationCount is set, BDN does not run Pilot Stage, so to get the code promoted to Tier 1 before Actual Workload, we enforce more Warmups public class Sort where T : IComparable { private static readonly ComparableComparerClass _comparableComparerClass = new ComparableComparerClass(); diff --git a/src/benchmarks/micro/libraries/System.Threading.Tasks.Extensions/Perf.ValueTask.cs b/src/benchmarks/micro/libraries/System.Threading.Tasks.Extensions/Perf.ValueTask.cs index c02f2653bd7..0f705875b2d 100644 --- a/src/benchmarks/micro/libraries/System.Threading.Tasks.Extensions/Perf.ValueTask.cs +++ b/src/benchmarks/micro/libraries/System.Threading.Tasks.Extensions/Perf.ValueTask.cs @@ -11,8 +11,6 @@ namespace System.Threading.Tasks { [BenchmarkCategory(Categories.Libraries, Categories.NoWASM)] - [MinWarmupCount(2, forceAutoWarmup: true)] // these benchmarks require more warmups than in our default config - [MaxWarmupCount(10, forceAutoWarmup: true)] public class ValueTaskPerfTest { private Task _completedTask = Task.FromResult(42); diff --git a/src/benchmarks/micro/runtime/Span/Sorting.cs b/src/benchmarks/micro/runtime/Span/Sorting.cs index 969f35c083a..6c5e0a95f89 100644 --- a/src/benchmarks/micro/runtime/Span/Sorting.cs +++ b/src/benchmarks/micro/runtime/Span/Sorting.cs @@ -13,7 +13,6 @@ namespace Span { [BenchmarkCategory(Categories.Runtime)] [InvocationCount(InvocationsPerIteration)] - [MinWarmupCount(6, forceAutoWarmup: true)] // when InvocationCount is set, BDN does not run Pilot Stage, so to get the code promoted to Tier 1 before Actual Workload, we enforce more Warmups public class Sorting { private const int InvocationsPerIteration = 10_000; diff --git a/src/harness/BenchmarkDotNet.Extensions/AotFilterAttribute.cs b/src/harness/BenchmarkDotNet.Extensions/AotFilterAttribute.cs new file mode 100644 index 00000000000..bf89591a716 --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/AotFilterAttribute.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Filters; + +namespace BenchmarkDotNet.Attributes.Filters; + +public class AotFilterAttribute : FilterConfigBaseAttribute +{ + public AotFilterAttribute(string? reason = null) + : base(new SimpleFilter(benchmark => benchmark.GetRuntime() is not (NativeAotRuntime or MonoAotLLVMRuntime))) + { + } +} diff --git a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj index a54dc819171..21ae4c8f73d 100644 --- a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj +++ b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj @@ -23,4 +23,7 @@ + + + diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotCompilerMode.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotCompilerMode.cs new file mode 100644 index 00000000000..fa9433b9a9a --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotCompilerMode.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace BenchmarkDotNet.Extensions; + +public enum MonoAotCompilerMode +{ + mini, + llvm +} diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMCsProj.txt b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMCsProj.txt new file mode 100644 index 00000000000..dd6fe93fded --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMCsProj.txt @@ -0,0 +1,96 @@ + + + $CSPROJPATH$ + $([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Mono.props')) + $([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Mono.targets')) + + + + + + Exe + False + false + false + $TFM$ + $RUNTIMEPACK$ + $RUNTIMEIDENTIFIER$ + false + false + $PROGRAMNAME$ + false + false + true + BenchmarkDotNet.Autogenerated.UniqueProgramName + true + + + + + + + + + + + + + + + + $COPIEDSETTINGS$ + + + + + latest + + + + + + + $(MicrosoftNetCoreAppRuntimePackDir) + + + + + + + + + + + $([System.IO.Path]::GetFullPath($(PublishDir))) + Dylib + Dll + So + + + + + mcpu=native + + + + + + + + + + + + + + diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMGenerator.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMGenerator.cs new file mode 100644 index 00000000000..926696fb416 --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMGenerator.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.IO; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; +using BenchmarkDotNet.Code; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.CsProj; + +namespace BenchmarkDotNet.Extensions; + +public class MonoAotLLVMGenerator : CsProjGenerator +{ + private readonly MonoAotLLVMSettings _settings; + + public MonoAotLLVMGenerator(MonoAotLLVMSettings settings) + : base(settings) + { + _settings = settings; + BenchmarkRunCallType = CodeGenBenchmarkRunCallType.Direct; + } + + protected override async ValueTask GenerateProjectAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger, CancellationToken cancellationToken) + { + BenchmarkCase benchmark = buildPartition.RepresentativeBenchmarkCase; + var projectFile = GetProjectFilePath(benchmark.Descriptor.Type, logger); + + var xmlDoc = new XmlDocument(); + xmlDoc.Load(projectFile.FullName); + var (customProperties, _) = GetSettingsThatNeedToBeCopied(xmlDoc, projectFile); + + string template = await LoadTemplateAsync(cancellationToken).ConfigureAwait(false); + string content = new StringBuilder(template) + .Replace("$CODEFILENAME$", Path.GetFileName(artifactsPaths.ProgramCodePath)) + .Replace("$CSPROJPATH$", projectFile.FullName) + .Replace("$TFM$", Settings.TargetFrameworkMoniker) + .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) + .Replace("$COPIEDSETTINGS$", customProperties) + .Replace("$RUNTIMEPACK$", _settings.CustomRuntimePack) + .Replace("$COMPILERBINARYPATH$", _settings.AotCompilerPath) + .Replace("$RUNTIMEIDENTIFIER$", RuntimeInformation.RuntimeIdentifier) + .Replace("$USELLVM$", _settings.AotCompilerMode == MonoAotCompilerMode.llvm ? "true" : "false") + .ToString(); + + await File.WriteAllTextAsync(artifactsPaths.ProjectFilePath, content, cancellationToken).ConfigureAwait(false); + + await GatherReferencesAsync(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false); + } + + private static async Task LoadTemplateAsync(CancellationToken cancellationToken) + { + var assembly = typeof(MonoAotLLVMGenerator).Assembly; + string resourceName = System.Array.Find(assembly.GetManifestResourceNames(), n => n.EndsWith("MonoAOTLLVMCsProj.txt")) + ?? throw new FileNotFoundException("Embedded resource MonoAOTLLVMCsProj.txt not found"); + using var stream = assembly.GetManifestResourceStream(resourceName)!; + using var reader = new StreamReader(stream); + return await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false); + } + + protected override string GetPublishDirectoryPath(string buildArtifactsDirectoryPath, string configuration) + => Path.Combine(GetBinariesDirectoryPath(buildArtifactsDirectoryPath, configuration), "publish"); + + protected override string GetExecutablePath(string binariesDirectoryPath, string programName) + => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? Path.Combine(binariesDirectoryPath, "publish", $"{programName}.exe") + : Path.Combine(binariesDirectoryPath, "publish", programName); + + protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) + => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, Settings.TargetFrameworkMoniker, RuntimeInformation.RuntimeIdentifier); +} diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMRuntime.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMRuntime.cs new file mode 100644 index 00000000000..10d316c171c --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMRuntime.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; + +namespace BenchmarkDotNet.Extensions; + +public sealed class MonoAotLLVMRuntime(Version version) : Runtime +{ + public override string Name => "MonoAOTLLVM"; + + public override Version Version { get; } = version; + + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) + => throw new NotSupportedException("MonoAOTLLVM benchmarks must set the toolchain explicitly."); +} diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMSettings.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMSettings.cs new file mode 100644 index 00000000000..ae1ba8518b4 --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMSettings.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Extensions; + +public sealed record MonoAotLLVMSettings : DotNetCliSettings +{ + public required string CustomRuntimePack { get; init; } + public required string AotCompilerPath { get; init; } + public required MonoAotCompilerMode AotCompilerMode { get; init; } +} diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMToolChain.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMToolChain.cs new file mode 100644 index 00000000000..70c174ec4ec --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMToolChain.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Extensions; + +public class MonoAotLLVMToolChain(MonoAotLLVMRuntime runtime, MonoAotLLVMSettings settings) + : Toolchain("MonoAotLLVM", runtime, + new MonoAotLLVMGenerator(settings), + new DotNetCliBuilder(settings), + new Executor()) +{ +} diff --git a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs index c8a469ba68e..a868cd88f36 100644 --- a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs +++ b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs @@ -1,11 +1,13 @@ using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Exporters.Json; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Toolchains; using Perfolizer.Horology; using Reporting; using System; @@ -28,19 +30,24 @@ public static IConfig Create( List? exclusionFilterValue = null, List? categoryExclusionFilterValue = null, Job? job = null, - bool getDiffableDisasm = false) + bool getDiffableDisasm = false, + IToolchain? toolchain = null) { if (job is null) { - #pragma warning disable CS0618 // WithEvaluateOverhead is obsolete but needed for WASM accuracy job = Job.Default - .WithWarmupCount(1) // 1 warmup is enough for our purpose + .WithJitTieringMode(JitTieringMode.Force) // Force benchmarks to be promoted to tier 1, even for longer-running benchmarks that the default mode might skip (e.g. large InvocationsPerIteration) + .WithWarmupCount(0) // Jit stage already warms the benchmarks .WithIterationTime(TimeInterval.FromMilliseconds(250)) // the default is 0.5s per iteration, which is slighlty too much for us .WithMinIterationCount(15) .WithMaxIterationCount(20) // we don't want to run more that 20 iterations .WithEvaluateOverhead(Environment.GetEnvironmentVariable("PERFLAB_EVALUATE_OVERHEAD") == "1") // WASM has significant method-call overhead (1-10ns); subtract it when enabled .DontEnforcePowerPlan(); // make sure BDN does not try to enforce High Performance power plan on Windows - #pragma warning restore CS0618 + } + + if (toolchain is not null) + { + job = job.WithToolchain(toolchain); } var config = ManualConfig.CreateEmpty() From 39dce0a21ec9770c3149dd7d7a26a158eaf9e544 Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Tue, 8 Sep 2026 04:03:55 -0400 Subject: [PATCH 2/5] Update BDN nightly version and async setup/cleanup/ArgumentsSource. --- eng/Versions.props | 2 +- src/benchmarks/micro/Program.cs | 5 ---- .../Perf.SocketsHttpHandler.cs | 4 +-- .../System.Net.Http/Perf.WinHttpHandler.cs | 4 +-- .../SslStreamTests.Protocols.cs | 12 ++++---- .../System.Net.Security/SslStreamTests.cs | 6 ++-- .../Roslyn/CompilerBenchmarks.csproj | 2 +- .../Polyfills/AsyncEnumerable.cs | 29 +++++++++++++++++++ 8 files changed, 43 insertions(+), 21 deletions(-) create mode 100644 src/harness/BenchmarkDotNet.Extensions/Polyfills/AsyncEnumerable.cs diff --git a/eng/Versions.props b/eng/Versions.props index 9f590a4d795..17618581057 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,7 +11,7 @@ 11.0.0-preview.5.26261.101 11.0.0-preview.5.26261.101 11.0.0-preview.5.26261.101 - 0.16.0-nightly.20260801.595 + 0.16.0-nightly.20260908.616 11.0.0-preview.5.26261.101 11.0.0-prerelease.26330.1 diff --git a/src/benchmarks/micro/Program.cs b/src/benchmarks/micro/Program.cs index aa974b7b031..29d75caac41 100644 --- a/src/benchmarks/micro/Program.cs +++ b/src/benchmarks/micro/Program.cs @@ -105,11 +105,6 @@ static async Task Main(string[] args) return 1; } - // Use RunAsync (not Run) so BDN does not install its single-threaded - // BenchmarkDotNetSynchronizationContext on the entrypoint thread. The sync - // entrypoint installs that context before benchmark discovery, which - // deadlocks any sync-over-async work performed by [ParamsSource]/[ArgumentsSource] - // callbacks (e.g. SslStreamTests.GetTls13Support). var summaries = await BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) .RunAsync(argsList.ToArray(), diff --git a/src/benchmarks/micro/libraries/System.Net.Http/Perf.SocketsHttpHandler.cs b/src/benchmarks/micro/libraries/System.Net.Http/Perf.SocketsHttpHandler.cs index 5be3b82ec34..af4644c1c7a 100644 --- a/src/benchmarks/micro/libraries/System.Net.Http/Perf.SocketsHttpHandler.cs +++ b/src/benchmarks/micro/libraries/System.Net.Http/Perf.SocketsHttpHandler.cs @@ -147,12 +147,12 @@ public async Task Get_EnumerateHeaders_Unvalidated() #endif [GlobalCleanup] - public void Cleanup() + public async Task Cleanup() { _invoker.Dispose(); _handler.Dispose(); _listener.Dispose(); - _serverTask.GetAwaiter().GetResult(); + await _serverTask; _serverCert.Dispose(); } } diff --git a/src/benchmarks/micro/libraries/System.Net.Http/Perf.WinHttpHandler.cs b/src/benchmarks/micro/libraries/System.Net.Http/Perf.WinHttpHandler.cs index f07e4f09ec4..eb70d587e05 100644 --- a/src/benchmarks/micro/libraries/System.Net.Http/Perf.WinHttpHandler.cs +++ b/src/benchmarks/micro/libraries/System.Net.Http/Perf.WinHttpHandler.cs @@ -114,12 +114,12 @@ public async Task Get() } [GlobalCleanup] - public void Cleanup() + public async Task Cleanup() { _invoker.Dispose(); _handler.Dispose(); _listener.Dispose(); - _serverTask.GetAwaiter().GetResult(); + await _serverTask; _serverCert.Dispose(); } } diff --git a/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs b/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs index f2c33c61242..6587570d0a8 100644 --- a/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs +++ b/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs @@ -6,20 +6,18 @@ using System.Security.Authentication; using System.Threading.Tasks; using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Extensions; using MicroBenchmarks; namespace System.Net.Security.Tests { public partial class SslStreamTests { - private static Lazy s_supportsTls13 = new Lazy(GetTls13Support); - public static bool SupportsTls13 => s_supportsTls13.Value; + private static readonly Task s_supportsTls13 = GetTls13SupportAsync(); - public static IEnumerable TlsProtocols() + public static async IAsyncEnumerable TlsProtocols() { yield return new object[] { SslProtocols.Tls12 }; - if (SupportsTls13) + if (await s_supportsTls13) { yield return new object[] { SslProtocols.Tls13 }; } @@ -50,11 +48,11 @@ public static IEnumerable TlsProtocols() [ArgumentsSource(nameof(TlsProtocols))] public Task HandshakeRSA4096CertAsync(SslProtocols protocol) => SslStreamTests.HandshakeAsync(SslStreamTests._rsa4096Cert, protocol); - private static bool GetTls13Support() + private static async Task GetTls13SupportAsync() { try { - SslStreamTests.HandshakeAsync(SslStreamTests._rsa2048Cert, SslProtocols.Tls13).GetAwaiter().GetResult(); + await SslStreamTests.HandshakeAsync(SslStreamTests._rsa2048Cert, SslProtocols.Tls13); return true; } catch diff --git a/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.cs b/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.cs index 71adad43b9a..55d01758d33 100644 --- a/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.cs +++ b/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.cs @@ -36,7 +36,7 @@ public partial class SslStreamTests private PipeStream _clientPipe, _serverPipe; // used for handshake tests [GlobalSetup] - public void Setup() + public async Task Setup() { string pipeName = "SetupTlsHandshakePipe"; using (var listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)) @@ -51,7 +51,7 @@ public void Setup() _sslClient = new SslStream(new NetworkStream(client, ownsSocket: true), leaveInnerStreamOpen: false, delegate { return true; }); _sslServer = new SslStream(new NetworkStream(server, ownsSocket: true), leaveInnerStreamOpen: false, delegate { return true; }); - Task.WaitAll( + await Task.WhenAll( _sslClient.AuthenticateAsClientAsync("localhost", null, SslProtocols.None, checkCertificateRevocation: false), _sslServer.AuthenticateAsServerAsync(_cert, clientCertificateRequired: false, SslProtocols.None, checkCertificateRevocation: false)); @@ -80,7 +80,7 @@ public void Setup() // Create PIPE Pair. var pipeServer = new NamedPipeServerStream(pipeName, PipeDirection.InOut, 1, PipeTransmissionMode.Byte, PipeOptions.Asynchronous | PipeOptions.WriteThrough); var pipeClient = new NamedPipeClientStream(".", pipeName, PipeDirection.InOut, PipeOptions.Asynchronous | PipeOptions.WriteThrough); - Task.WaitAll(pipeServer.WaitForConnectionAsync(), pipeClient.ConnectAsync()); + await Task.WhenAll(pipeServer.WaitForConnectionAsync(), pipeClient.ConnectAsync()); _serverPipe = pipeServer; _clientPipe = pipeClient; } diff --git a/src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj b/src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj index 0b437d5164f..b7f01245234 100644 --- a/src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj +++ b/src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/harness/BenchmarkDotNet.Extensions/Polyfills/AsyncEnumerable.cs b/src/harness/BenchmarkDotNet.Extensions/Polyfills/AsyncEnumerable.cs new file mode 100644 index 00000000000..7f6eb880933 --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/Polyfills/AsyncEnumerable.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#if !NET10_0_OR_GREATER +using System.Collections.Generic; + +namespace System.Linq; + +// System.Linq.AsyncEnumerable ships in the shared framework from .NET 10. This project targets net8.0 (the lowest TFM the perf pipelines exercise), +// so the member it needs is polyfilled here rather than taking the System.Linq.AsyncEnumerable package. This mirrors BenchmarkDotNet's own polyfill. +internal static class AsyncEnumerable +{ + public static IAsyncEnumerable ToAsyncEnumerable(this IEnumerable source) + { + ArgumentNullException.ThrowIfNull(source); + + return FromIterator(source); + + static async IAsyncEnumerable FromIterator(IEnumerable source) + { + foreach (TSource element in source) + { + yield return element; + } + } + } +} +#endif From f19464d79d76798470a176d3e4a71b4b2b2de1ab Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Tue, 8 Sep 2026 04:33:13 -0400 Subject: [PATCH 3/5] Fixes and docs update. --- docs/benchmarkdotnet.md | 2 +- docs/benchmarking-workflow-dotnet-runtime.md | 6 +- src/benchmarks/micro/Program.cs | 73 +++++++++++-------- .../SslStreamTests.Protocols.cs | 4 +- .../MonoAotLLVM/MonoAotLLVMGenerator.cs | 5 +- 5 files changed, 53 insertions(+), 37 deletions(-) diff --git a/docs/benchmarkdotnet.md b/docs/benchmarkdotnet.md index be429f57ad7..b841f18bfe9 100644 --- a/docs/benchmarkdotnet.md +++ b/docs/benchmarkdotnet.md @@ -289,7 +289,7 @@ M00_L00: The `--runtimes` or just `-r` allows you to run the benchmarks for **multiple Runtimes**. -Available options are: Mono, wasmnet70, CoreRT, netcoreapp3.1, net6.0, net7.0, net8.0, and net9.0. +Available options include: net8.0, net9.0, net10.0, net11.0, and the versioned forms of the other runtimes: mono11.0, nativeaot11.0, r2r11.0, monowasm11.0, monowasmaot11.0, corewasm11.0. Example: run the benchmarks for .NET 7.0 and 8.0: diff --git a/docs/benchmarking-workflow-dotnet-runtime.md b/docs/benchmarking-workflow-dotnet-runtime.md index 44d1ecd0d75..bc6bde157dc 100644 --- a/docs/benchmarking-workflow-dotnet-runtime.md +++ b/docs/benchmarking-workflow-dotnet-runtime.md @@ -197,11 +197,11 @@ Make sure you have the v8 engine installed and in the PATH. Follow the installat #### Run the benchmarks with AOT -Essentially, add `--aotcompilermode wasm` to the `--bdn-arguments=".."`: +Essentially, add `--wasm-runtime-flavor MonoAOT` alongside `--wasm`: ```cmd -/path/to/dotnet/performance$ python3 ./scripts/benchmarks_ci.py --csproj src/benchmarks/micro/MicroBenchmarks.csproj -f net11.0 --dotnet-path artifacts/bin/dotnet-latest --wasm --run-isolated --bdn-artifacts artifacts/BenchmarkDotNet.Artifacts - --bdn-arguments="--category-exclusion-filter NoInterpreter NoWASM NoMono --aotcompilermode wasm --logBuildOutput --buildTimeout 3600 --filter " +/path/to/dotnet/performance$ python3 ./scripts/benchmarks_ci.py --csproj src/benchmarks/micro/MicroBenchmarks.csproj -f net11.0 --dotnet-path artifacts/bin/dotnet-latest --wasm --wasm-runtime-flavor MonoAOT --run-isolated --bdn-artifacts artifacts/BenchmarkDotNet.Artifacts + --bdn-arguments="--category-exclusion-filter NoInterpreter NoWASM NoMono --logBuildOutput --buildTimeout 3600 --filter " ``` #### Note about "file ... being used by another process" error diff --git a/src/benchmarks/micro/Program.cs b/src/benchmarks/micro/Program.cs index 29d75caac41..9d20a716c8d 100644 --- a/src/benchmarks/micro/Program.cs +++ b/src/benchmarks/micro/Program.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.Runtime.Versioning; using System.Threading.Tasks; using BenchmarkDotNet.Running; using System.IO; @@ -15,6 +16,31 @@ namespace MicroBenchmarks { class Program { + // The MonoAotLLVM toolchain cannot build without these. Failing here reports an actionable message + // through the ArgumentException handler below, instead of passing an empty path into the generated + // project and failing deep inside MSBuild many minutes later. + private static string RequireMonoAotLLVMValue(List values, string parameter) + => values.Count > 0 + ? values[0] + : throw new ArgumentException($"{parameter} must be specified when running with --runtimes monoaotllvm"); + + // The TFM this assembly was compiled for. Environment.Version reports the runtime that actually + // loaded, which can roll forward to a newer major than --customruntimepack points at, producing an + // opaque restore/publish failure in the generated project. + private static string GetTargetFrameworkMoniker() + { + if (AppContext.TargetFrameworkName is string frameworkName) + { + var parsed = new FrameworkName(frameworkName); + if (parsed.Identifier == ".NETCoreApp") + { + return $"net{parsed.Version.Major}.{parsed.Version.Minor}"; + } + } + + return $"net{Environment.Version.Major}.0"; + } + static async Task Main(string[] args) { var argsList = new List(args); @@ -35,12 +61,16 @@ static async Task Main(string[] args) CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--disasm-diff", out getDiffableDisasm); // Extract monoaotllvm args not recognized by BDN and build the toolchain. - argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--runtimes", out var runtimesValues); - if (runtimesValues.Remove("monoaotllvm")) + // BDN spells this option "-r, --runtimes" and matches monikers case-insensitively, so accept both forms here. + string runtimesArg = argsList.Contains("--runtimes") ? "--runtimes" : "-r"; + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, runtimesArg, out var runtimesValues); + int monoAotLLVMIndex = runtimesValues.FindIndex(runtime => string.Equals(runtime, "monoaotllvm", StringComparison.OrdinalIgnoreCase)); + if (monoAotLLVMIndex >= 0) { + runtimesValues.RemoveAt(monoAotLLVMIndex); if (runtimesValues.Count > 0) { - argsList.Add("--runtimes"); + argsList.Add(runtimesArg); argsList.AddRange(runtimesValues); } argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--cli", out var cliPathValue); @@ -56,44 +86,29 @@ static async Task Main(string[] args) argsList.AddRange(packagesPathValue); } argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--aotcompilerpath", out var aotCompilerPathValue); - string aotCompilerPath; - if (aotCompilerPathValue.Count > 0) - { - aotCompilerPath = aotCompilerPathValue[0]; - argsList.Add("--aotcompilerpath"); - argsList.AddRange(aotCompilerPathValue); - } - else - { - aotCompilerPath = ""; - } + string aotCompilerPath = RequireMonoAotLLVMValue(aotCompilerPathValue, "--aotcompilerpath"); + argsList.Add("--aotcompilerpath"); + argsList.AddRange(aotCompilerPathValue); + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--customruntimepack", out var customRuntimePackValue); - string customRuntimePack; - if (customRuntimePackValue.Count > 0) - { - customRuntimePack = customRuntimePackValue[0]; - argsList.Add("--customruntimepack"); - argsList.AddRange(customRuntimePackValue); + string customRuntimePack = RequireMonoAotLLVMValue(customRuntimePackValue, "--customruntimepack"); + argsList.Add("--customruntimepack"); + argsList.AddRange(customRuntimePackValue); - } - else - { - customRuntimePack = ""; - } argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--aotcompilermode", out var aotCompilerModeValue); monoAotToolchain = new MonoAotLLVMToolChain(new MonoAotLLVMRuntime(Environment.Version), new() { CliPath = cliPathValue.Count > 0 ? new(cliPathValue[0]) : null, PackagesPath = packagesPathValue.Count > 0 ? new(packagesPathValue[0]) : null, - TargetFrameworkMoniker = $"net{Environment.Version.Major}.0", + TargetFrameworkMoniker = GetTargetFrameworkMoniker(), CustomRuntimePack = customRuntimePack, AotCompilerPath = aotCompilerPath, - AotCompilerMode = aotCompilerModeValue.Count > 0 ? Enum.Parse(aotCompilerModeValue[0], ignoreCase: true) : 0 + AotCompilerMode = aotCompilerModeValue.Count > 0 ? Enum.Parse(aotCompilerModeValue[0], ignoreCase: true) : MonoAotCompilerMode.mini }); } - else + else if (runtimesValues.Count > 0) { - argsList.Add("--runtimes"); + argsList.Add(runtimesArg); argsList.AddRange(runtimesValues); } diff --git a/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs b/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs index 6587570d0a8..f3afea8d315 100644 --- a/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs +++ b/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs @@ -12,12 +12,12 @@ namespace System.Net.Security.Tests { public partial class SslStreamTests { - private static readonly Task s_supportsTls13 = GetTls13SupportAsync(); + private static readonly Lazy> s_supportsTls13 = new(GetTls13SupportAsync); public static async IAsyncEnumerable TlsProtocols() { yield return new object[] { SslProtocols.Tls12 }; - if (await s_supportsTls13) + if (await s_supportsTls13.Value) { yield return new object[] { SslProtocols.Tls13 }; } diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMGenerator.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMGenerator.cs index 926696fb416..39ccde5ad06 100644 --- a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMGenerator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMGenerator.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.IO; using System.Runtime.InteropServices; using System.Text; @@ -57,8 +58,8 @@ protected override async ValueTask GenerateProjectAsync(BuildPartition buildPart private static async Task LoadTemplateAsync(CancellationToken cancellationToken) { var assembly = typeof(MonoAotLLVMGenerator).Assembly; - string resourceName = System.Array.Find(assembly.GetManifestResourceNames(), n => n.EndsWith("MonoAOTLLVMCsProj.txt")) - ?? throw new FileNotFoundException("Embedded resource MonoAOTLLVMCsProj.txt not found"); + string resourceName = System.Array.Find(assembly.GetManifestResourceNames(), n => n.EndsWith("MonoAotLLVMCsProj.txt", StringComparison.OrdinalIgnoreCase)) + ?? throw new FileNotFoundException("Embedded resource MonoAotLLVMCsProj.txt not found"); using var stream = assembly.GetManifestResourceStream(resourceName)!; using var reader = new StreamReader(stream); return await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false); From 7447f93ce61a5cf220d06d48265b5179932d6f99 Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Thu, 17 Sep 2026 06:43:45 -0400 Subject: [PATCH 4/5] Update to latest nightly BDN. --- eng/Versions.props | 2 +- .../BenchmarkDotNet.Extensions.csproj | 3 - .../MonoAotLLVM/MonoAotLLVMBuilder.cs | 136 ++++++++++++++++++ .../MonoAotLLVM/MonoAotLLVMCsProj.txt | 96 ------------- .../MonoAotLLVM/MonoAotLLVMGenerator.cs | 78 ---------- .../MonoAotLLVM/MonoAotLLVMToolChain.cs | 6 +- 6 files changed, 138 insertions(+), 183 deletions(-) create mode 100644 src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMBuilder.cs delete mode 100644 src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMCsProj.txt delete mode 100644 src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMGenerator.cs diff --git a/eng/Versions.props b/eng/Versions.props index 7fb04aac0ee..cbf54a93ccf 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,7 +11,7 @@ 11.0.0-rc.2.26465.102 11.0.0-rc.2.26465.102 11.0.0-rc.2.26465.102 - 0.16.0-nightly.20260908.616 + 0.16.0-nightly.20260917.628 11.0.0-preview.5.26261.101 11.0.0-prerelease.26465.1 diff --git a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj index 21ae4c8f73d..a54dc819171 100644 --- a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj +++ b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj @@ -23,7 +23,4 @@ - - - diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMBuilder.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMBuilder.cs new file mode 100644 index 00000000000..ddf51c364d7 --- /dev/null +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMBuilder.cs @@ -0,0 +1,136 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.IO; +using System.Runtime.InteropServices; +using System.Xml.Linq; +using BenchmarkDotNet.Code; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.CsProj; + +namespace BenchmarkDotNet.Extensions; + +public class MonoAotLLVMBuilder : CsProjBuilder +{ + private readonly MonoAotLLVMSettings _settings; + + public MonoAotLLVMBuilder(MonoAotLLVMSettings settings) : base(settings) + { + _settings = settings; + BenchmarkRunCallType = CodeGenBenchmarkRunCallType.Direct; + } + + // The AOT compilation runs after Publish, so the app has to be published rather than built. + protected override bool PublishesOutput => true; + + // The app is published against the custom runtime pack, which only provides Microsoft.NETCore.App, so the + // frameworks another SDK would reference could not be resolved. + protected override string GetSdkName(XElement benchmarkProject) => "Microsoft.NET.Sdk"; + + protected override void AddEarlyProperties(XElement project, BuildPartition buildPartition, ArtifactsPaths artifactsPaths, FileInfo projectFile) + { + project.Add(new XElement("PropertyGroup", + new XElement("OriginalCSProjPath", projectFile.FullName), + new XElement("MonoPropsPath", "$([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Mono.props'))"), + new XElement("MonoTargetsPath", "$([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Mono.targets'))"))); + + project.Add(new XElement("Import", + new XAttribute("Project", "$(MonoPropsPath)"), + new XAttribute("Condition", "Exists($(MonoPropsPath))"))); + + base.AddEarlyProperties(project, buildPartition, artifactsPaths, projectFile); + } + + protected override void AddProjectContent(XElement project, BuildPartition buildPartition, ArtifactsPaths artifactsPaths, FileInfo projectFile) + { + base.AddProjectContent(project, buildPartition, artifactsPaths, projectFile); + + project.Add(new XElement("PropertyGroup", + new XElement("MicrosoftNetCoreAppRuntimePackDir", _settings.CustomRuntimePack), + new XElement("RuntimeIdentifier", RuntimeInformation.RuntimeIdentifier), + new XElement("EnableTargetingPackDownload", "false"), + new XElement("PublishTrimmed", "false"), + new XElement("ErrorOnDuplicatePublishOutputFiles", "false"))); + + // Self-contained is what makes the publish use the custom runtime pack. BenchmarkDotNet's reference-gathering + // build disables the app host, which self-contained requires, so it is left out of that pass. It is set here + // rather than passed as --self-contained, which would also restore the benchmark project for this RID. + project.Add(new XElement("PropertyGroup", + new XAttribute("Condition", "'$(BenchmarkDotNetGatherReferences)' != 'true'"), + new XElement("SelfContained", "true"))); + + project.Add(new XElement("ItemGroup", + new XElement("PackageReference", + new XAttribute("Include", "Microsoft.NET.Runtime.MonoAOTCompiler.Task"), + new XAttribute("Version", "6.0.0-*"), + new XAttribute("GeneratePathProperty", "true")))); + + project.Add(new XComment(" Redirect 'dotnet publish' to in-tree runtime pack ")); + project.Add(new XElement("Target", + new XAttribute("Name", "TrickRuntimePackLocation"), + new XAttribute("AfterTargets", "ProcessFrameworkReferences"), + new XElement("ItemGroup", + new XElement("RuntimePack", + new XElement("PackageDirectory", "$(MicrosoftNetCoreAppRuntimePackDir)"))), + new XElement("Message", + new XAttribute("Text", "Packaged ID: %(RuntimePack.PackageDirectory)"), + new XAttribute("Importance", "high")))); + + project.Add(new XElement("UsingTask", + new XAttribute("TaskName", "MonoAOTCompiler"), + new XAttribute("AssemblyFile", "$(PkgMicrosoft_NET_Runtime_MonoAOTCompiler_Task)/MonoAOTCompiler.dll"))); + + project.Add(new XElement("Target", + new XAttribute("Name", "AotApp"), + new XAttribute("AfterTargets", "Publish"), + new XElement("PropertyGroup", + new XElement("PublishDirFullPath", "$([System.IO.Path]::GetFullPath($(PublishDir)))"), + new XElement("SharedLibraryType", new XAttribute("Condition", "$([MSBuild]::IsOSPlatform('OSX'))"), "Dylib"), + new XElement("SharedLibraryType", new XAttribute("Condition", "$([MSBuild]::IsOSPlatform('Windows'))"), "Dll"), + new XElement("SharedLibraryType", new XAttribute("Condition", "'$(SharedLibraryType)' == ''"), "So")), + new XElement("ItemGroup", + new XElement("AotInputAssemblies", + new XAttribute("Include", "$(PublishDirFullPath)\\*.dll"), + new XElement("AotArguments", "mcpu=native"))), + new XElement("MonoAOTCompiler", + new XAttribute("CompilerBinaryPath", _settings.AotCompilerPath), + new XAttribute("Mode", "Normal"), + new XAttribute("OutputType", "Library"), + new XAttribute("LibraryFormat", "$(SharedLibraryType)"), + new XAttribute("Assemblies", "@(AotInputAssemblies)"), + new XAttribute("UseLLVM", _settings.AotCompilerMode == MonoAotCompilerMode.llvm ? "true" : "false"), + new XAttribute("LLVMPath", "$(MicrosoftNetCoreAppRuntimePackDir)\\runtimes\\$(RuntimeIdentifier)\\native"), + new XAttribute("OutputDir", "$(PublishDir)"), + new XAttribute("UseAotDataFile", "false"), + new XAttribute("IntermediateOutputPath", "$(IntermediateOutputPath)"), + new XElement("Output", + new XAttribute("TaskParameter", "CompiledAssemblies"), + new XAttribute("ItemName", "BundleAssemblies"))), + new XElement("Message", + new XAttribute("Text", "CompiledAssemblies: $(BundleAssemblies)"), + new XAttribute("Importance", "high")))); + } + + protected override void AddLateProperties(XElement project, BuildPartition buildPartition, ArtifactsPaths artifactsPaths, FileInfo projectFile) + { + base.AddLateProperties(project, buildPartition, artifactsPaths, projectFile); + + // After the late properties so the Mono targets can override anything the generated project defines. + project.Add(new XElement("Import", + new XAttribute("Project", "$(MonoTargetsPath)"), + new XAttribute("Condition", "Exists($(MonoTargetsPath))"))); + } + + protected override string GetPublishDirectoryPath(string buildArtifactsDirectoryPath, string configuration) + => Path.Combine(GetBinariesDirectoryPath(buildArtifactsDirectoryPath, configuration), "publish"); + + protected override string GetExecutablePath(string binariesDirectoryPath, string programName) + => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? Path.Combine(binariesDirectoryPath, "publish", $"{programName}.exe") + : Path.Combine(binariesDirectoryPath, "publish", programName); + + protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) + => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, Settings.TargetFrameworkMoniker, RuntimeInformation.RuntimeIdentifier); +} diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMCsProj.txt b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMCsProj.txt deleted file mode 100644 index dd6fe93fded..00000000000 --- a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMCsProj.txt +++ /dev/null @@ -1,96 +0,0 @@ - - - $CSPROJPATH$ - $([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Mono.props')) - $([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Mono.targets')) - - - - - - Exe - False - false - false - $TFM$ - $RUNTIMEPACK$ - $RUNTIMEIDENTIFIER$ - false - false - $PROGRAMNAME$ - false - false - true - BenchmarkDotNet.Autogenerated.UniqueProgramName - true - - - - - - - - - - - - - - - - $COPIEDSETTINGS$ - - - - - latest - - - - - - - $(MicrosoftNetCoreAppRuntimePackDir) - - - - - - - - - - - $([System.IO.Path]::GetFullPath($(PublishDir))) - Dylib - Dll - So - - - - - mcpu=native - - - - - - - - - - - - - - diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMGenerator.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMGenerator.cs deleted file mode 100644 index 39ccde5ad06..00000000000 --- a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMGenerator.cs +++ /dev/null @@ -1,78 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.IO; -using System.Runtime.InteropServices; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using System.Xml; -using BenchmarkDotNet.Code; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Toolchains.CsProj; - -namespace BenchmarkDotNet.Extensions; - -public class MonoAotLLVMGenerator : CsProjGenerator -{ - private readonly MonoAotLLVMSettings _settings; - - public MonoAotLLVMGenerator(MonoAotLLVMSettings settings) - : base(settings) - { - _settings = settings; - BenchmarkRunCallType = CodeGenBenchmarkRunCallType.Direct; - } - - protected override async ValueTask GenerateProjectAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger, CancellationToken cancellationToken) - { - BenchmarkCase benchmark = buildPartition.RepresentativeBenchmarkCase; - var projectFile = GetProjectFilePath(benchmark.Descriptor.Type, logger); - - var xmlDoc = new XmlDocument(); - xmlDoc.Load(projectFile.FullName); - var (customProperties, _) = GetSettingsThatNeedToBeCopied(xmlDoc, projectFile); - - string template = await LoadTemplateAsync(cancellationToken).ConfigureAwait(false); - string content = new StringBuilder(template) - .Replace("$CODEFILENAME$", Path.GetFileName(artifactsPaths.ProgramCodePath)) - .Replace("$CSPROJPATH$", projectFile.FullName) - .Replace("$TFM$", Settings.TargetFrameworkMoniker) - .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) - .Replace("$COPIEDSETTINGS$", customProperties) - .Replace("$RUNTIMEPACK$", _settings.CustomRuntimePack) - .Replace("$COMPILERBINARYPATH$", _settings.AotCompilerPath) - .Replace("$RUNTIMEIDENTIFIER$", RuntimeInformation.RuntimeIdentifier) - .Replace("$USELLVM$", _settings.AotCompilerMode == MonoAotCompilerMode.llvm ? "true" : "false") - .ToString(); - - await File.WriteAllTextAsync(artifactsPaths.ProjectFilePath, content, cancellationToken).ConfigureAwait(false); - - await GatherReferencesAsync(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false); - } - - private static async Task LoadTemplateAsync(CancellationToken cancellationToken) - { - var assembly = typeof(MonoAotLLVMGenerator).Assembly; - string resourceName = System.Array.Find(assembly.GetManifestResourceNames(), n => n.EndsWith("MonoAotLLVMCsProj.txt", StringComparison.OrdinalIgnoreCase)) - ?? throw new FileNotFoundException("Embedded resource MonoAotLLVMCsProj.txt not found"); - using var stream = assembly.GetManifestResourceStream(resourceName)!; - using var reader = new StreamReader(stream); - return await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false); - } - - protected override string GetPublishDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(GetBinariesDirectoryPath(buildArtifactsDirectoryPath, configuration), "publish"); - - protected override string GetExecutablePath(string binariesDirectoryPath, string programName) - => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? Path.Combine(binariesDirectoryPath, "publish", $"{programName}.exe") - : Path.Combine(binariesDirectoryPath, "publish", programName); - - protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, Settings.TargetFrameworkMoniker, RuntimeInformation.RuntimeIdentifier); -} diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMToolChain.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMToolChain.cs index 70c174ec4ec..7a1928f5a87 100644 --- a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMToolChain.cs +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMToolChain.cs @@ -3,14 +3,10 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Toolchains.DotNetCli; namespace BenchmarkDotNet.Extensions; public class MonoAotLLVMToolChain(MonoAotLLVMRuntime runtime, MonoAotLLVMSettings settings) - : Toolchain("MonoAotLLVM", runtime, - new MonoAotLLVMGenerator(settings), - new DotNetCliBuilder(settings), - new Executor()) + : Toolchain("MonoAotLLVM", runtime, new MonoAotLLVMBuilder(settings), Toolchains.Executor.Instance) { } From e60d3215ae62d67f3427178327b18f7faca2a16c Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Thu, 24 Sep 2026 08:02:32 -0400 Subject: [PATCH 5/5] Update to latest nightly BDN. --- .gitignore | 3 ++ docs/microbenchmark-design-guidelines.md | 24 ++++++++++++-- eng/Versions.props | 2 +- .../SslStreamTests.Protocols.cs | 6 ++-- .../libraries/System.Runtime/Perf.Uri.cs | 32 +++++++++---------- .../MonoAotLLVM/MonoAotLLVMBuilder.cs | 13 ++------ 6 files changed, 47 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 43b4bb330b0..2dc12300ca2 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,9 @@ dlldata.c # Benchmark Results BenchmarkDotNet.Artifacts/ +# BenchmarkDotNet generated builds, written beside the benchmark project +.bdn/ + # .NET Core project.lock.json project.fragment.lock.json diff --git a/docs/microbenchmark-design-guidelines.md b/docs/microbenchmark-design-guidelines.md index 3a85347702c..495ad374caa 100644 --- a/docs/microbenchmark-design-guidelines.md +++ b/docs/microbenchmark-design-guidelines.md @@ -411,9 +411,12 @@ However, **`[Params]` are applied to all the benchmarks in given class**. So if `[ArgumentsSource]` is xUnit's `[MemberData]` counterpart. +How the source maps onto the benchmark's parameters is read from its declared element type. + +For a **single argument** the source yields the parameter's own type: + ```cs -// for single argument it's an IEnumerable of objects (object), not IEnumerable of arrays of objects (object[]) -public IEnumerable Cultures() +public IEnumerable Cultures() { yield return new CultureInfo("fr"); yield return new CultureInfo("da"); @@ -427,6 +430,21 @@ public DateTime Parse(CultureInfo cultureInfo) => DateTime.Parse("10/10/2010 12:00:00 AM", cultureInfo); ``` +For **several arguments** it yields one element per case holding all of them, a `ValueTuple` naming each parameter's type: + +```cs +public IEnumerable<(string input, NumberStyles styles)> Inputs() +{ + yield return ("1234", NumberStyles.Integer); + yield return ("1,234", NumberStyles.AllowThousands); + yield return ("0x4D2", NumberStyles.HexNumber); +} + +[Benchmark] +[ArgumentsSource(nameof(Inputs))] +public int Parse(string input, NumberStyles styles) => int.Parse(input, styles); +``` + **Note:** If you need to use the argument in the setup method, then instead of using `ArgumentsSource` you should use `ParamsSource`. **Note:** `[ArgumentsSource]` methods are only called once per execution of the benchmark, and the cached results are used per invocation. @@ -445,7 +463,7 @@ public void Bitmap_FromStream(ImageTestData format) } } -public IEnumerable ImageFormats() => return new [] +public IEnumerable ImageFormats() => new [] { new ImageTestData(ImageFormat.Bmp), new ImageTestData(ImageFormat.Jpeg), diff --git a/eng/Versions.props b/eng/Versions.props index cbf54a93ccf..1c276575e60 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,7 +11,7 @@ 11.0.0-rc.2.26465.102 11.0.0-rc.2.26465.102 11.0.0-rc.2.26465.102 - 0.16.0-nightly.20260917.628 + 0.16.0-nightly.20260923.641 11.0.0-preview.5.26261.101 11.0.0-prerelease.26465.1 diff --git a/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs b/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs index f3afea8d315..4b534069365 100644 --- a/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs +++ b/src/benchmarks/micro/libraries/System.Net.Security/SslStreamTests.Protocols.cs @@ -14,12 +14,12 @@ public partial class SslStreamTests { private static readonly Lazy> s_supportsTls13 = new(GetTls13SupportAsync); - public static async IAsyncEnumerable TlsProtocols() + public static async IAsyncEnumerable TlsProtocols() { - yield return new object[] { SslProtocols.Tls12 }; + yield return SslProtocols.Tls12; if (await s_supportsTls13.Value) { - yield return new object[] { SslProtocols.Tls13 }; + yield return SslProtocols.Tls13; } } diff --git a/src/benchmarks/micro/libraries/System.Runtime/Perf.Uri.cs b/src/benchmarks/micro/libraries/System.Runtime/Perf.Uri.cs index 32e60716ba9..b4424a34dae 100644 --- a/src/benchmarks/micro/libraries/System.Runtime/Perf.Uri.cs +++ b/src/benchmarks/micro/libraries/System.Runtime/Perf.Uri.cs @@ -12,21 +12,21 @@ namespace System.Tests [BenchmarkCategory(Categories.Libraries)] public class Perf_Uri { - public static IEnumerable Ctor_MemberData() + public static IEnumerable Ctor_MemberData() { - yield return new object[] { "http://dot.net" }; - yield return new object[] { "https://contoso.com" }; - yield return new object[] { "https://CONTOSO.com" }; - yield return new object[] { "https://a.much.longer.domain.name" }; - yield return new object[] { "http://h\u00F6st.with.\u00FCnicode" }; - yield return new object[] { "http://xn--hst-sna.with.xn--nicode-2ya" }; + yield return "http://dot.net"; + yield return "https://contoso.com"; + yield return "https://CONTOSO.com"; + yield return "https://a.much.longer.domain.name"; + yield return "http://h\u00F6st.with.\u00FCnicode"; + yield return "http://xn--hst-sna.with.xn--nicode-2ya"; } - public static IEnumerable CtorIdnHostPathAndQuery_MemberData() + public static IEnumerable CtorIdnHostPathAndQuery_MemberData() { - foreach (object[] schemeAndAuthority in Ctor_MemberData()) + foreach (string schemeAndAuthority in Ctor_MemberData()) { - yield return new object[] { $"{schemeAndAuthority[0]}/path/with?key=value#fragment" }; + yield return $"{schemeAndAuthority}/path/with?key=value#fragment"; } string[] paths = new[] @@ -41,16 +41,16 @@ public static IEnumerable CtorIdnHostPathAndQuery_MemberData() foreach (string path in paths) { - yield return new object[] { $"http://host{path}" }; + yield return $"http://host{path}"; } } - public static IEnumerable EscapeDataString_MemberData() + public static IEnumerable EscapeDataString_MemberData() { - yield return new object[] { new string('a', 1000) }; // Nothing to escape - yield return new object[] { new string('{', 1000) }; // ASCII that needs escaping - yield return new object[] { new string('\u00FC', 1000) }; // Unicode - yield return new object[] { string.Concat(Enumerable.Repeat("a{\u00FC", 333)) }; + yield return new string('a', 1000); // Nothing to escape + yield return new string('{', 1000); // ASCII that needs escaping + yield return new string('\u00FC', 1000); // Unicode + yield return string.Concat(Enumerable.Repeat("a{\u00FC", 333)); } private static readonly Uri _uri = new Uri("http://contoso.com/path/with?key=value#fragment"); diff --git a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMBuilder.cs b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMBuilder.cs index ddf51c364d7..612e71c8cbc 100644 --- a/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMBuilder.cs +++ b/src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotLLVMBuilder.cs @@ -123,14 +123,7 @@ protected override void AddLateProperties(XElement project, BuildPartition build new XAttribute("Condition", "Exists($(MonoTargetsPath))"))); } - protected override string GetPublishDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(GetBinariesDirectoryPath(buildArtifactsDirectoryPath, configuration), "publish"); - - protected override string GetExecutablePath(string binariesDirectoryPath, string programName) - => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? Path.Combine(binariesDirectoryPath, "publish", $"{programName}.exe") - : Path.Combine(binariesDirectoryPath, "publish", programName); - - protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, Settings.TargetFrameworkMoniker, RuntimeInformation.RuntimeIdentifier); + // A self-contained publish produces an app host, so the executable is not the managed dll the base returns. + protected override string GetExecutableExtension() + => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".exe" : string.Empty; }