From 9b5ea97dd0a381dd43f01406a6b95b5f392d6e63 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 4 Nov 2026 16:22:24 -0800 Subject: [PATCH 1/5] Keep reference gathering framework-dependent Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Toolchains/CsProj/CsProjBuilder.cs | 4 +- .../CsProjBuilderTests.cs | 106 +++++++++++++++++- 2 files changed, 108 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs index 2be677a661..ec643211f4 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs @@ -196,7 +196,7 @@ protected static async ValueTask SaveXmlAsync(XDocument document, string path, C /// that a path holding an XML character cannot corrupt the file, and it is complete when written: the /// reference-gathering pass adds a file beside it rather than editing it. /// - protected XDocument CreateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) + protected internal XDocument CreateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) { var projectFile = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger); @@ -319,6 +319,8 @@ protected void AddGatheredReferences(XElement project) project.Add(new XElement("PropertyGroup", new XAttribute("Condition", GatheringCondition), new XElement("SkipCompilerExecution", "true"), + // A self-contained executable requires an apphost, which this pass never produces. + new XElement("SelfContained", "false"), // Both of these copy a build product that no longer gets produced. new XElement("UseAppHost", "false"), new XElement("SkipCopyBuildProduct", "true"))); diff --git a/tests/BenchmarkDotNet.Tests/CsProjBuilderTests.cs b/tests/BenchmarkDotNet.Tests/CsProjBuilderTests.cs index 4ed7b0ee3b..19e5264079 100644 --- a/tests/BenchmarkDotNet.Tests/CsProjBuilderTests.cs +++ b/tests/BenchmarkDotNet.Tests/CsProjBuilderTests.cs @@ -2,20 +2,27 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Parameters; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Mocks; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.Mono; +using BenchmarkDotNet.Toolchains.NativeAot; using BenchmarkDotNet.Toolchains.NetCoreApp; +using BenchmarkDotNet.Toolchains.R2R; using JetBrains.Annotations; using System.Reflection; using System.Xml.Linq; namespace BenchmarkDotNet.Tests { - public class CsProjBuilderTests + public class CsProjBuilderTests(ITestOutputHelper output) { + private const string GatheringCondition = "'$(BenchmarkDotNetGatherReferences)' == 'true'"; + private const string NotGatheringCondition = "'$(BenchmarkDotNetGatherReferences)' != 'true'"; private FileInfo TestAssemblyFileInfo = new FileInfo(typeof(CsProjBuilderTests).Assembly.Location); private const string runtimeHostConfigurationOptionChunk = """ @@ -279,6 +286,103 @@ public void FreshPackagesFolderIsRestoredInsideTheBuildDirectory() Assert.Equal(explicitPackages.FullName, Resolve(new NetCoreAppSettings { UseFreshPackages = true, PackagesPath = explicitPackages })); } + [Theory] + [InlineData("net10.0", "CoreCLR")] + [InlineData("net11.0", "CoreCLR")] + [InlineData("net12.0", "CoreCLR")] + [InlineData("net10.0", "R2R")] + [InlineData("net11.0", "R2R")] + [InlineData("net12.0", "R2R")] + [InlineData("net10.0", "Mono")] + [InlineData("net11.0", "Mono")] + [InlineData("net12.0", "Mono")] + [InlineData("net10.0", "NativeAOT")] + [InlineData("net11.0", "NativeAOT")] + [InlineData("net12.0", "NativeAOT")] + public void ReferenceGatheringDoesNotChangeTheFinalDeploymentMode(string targetFrameworkMoniker, string toolchain) + { + var runtimePack = new DirectoryInfo(Path.Combine(Path.GetTempPath(), "runtime & 'pack'")); + var crossgen2Pack = new FileInfo(Path.Combine(Path.GetTempPath(), "crossgen2 & 'pack'")); + CsProjBuilder builder = toolchain switch + { + "CoreCLR" => new CsProjBuilder(new NetCoreAppSettings { TargetFrameworkMoniker = targetFrameworkMoniker }), + "R2R" => new R2RBuilder(new R2RSettings + { + TargetFrameworkMoniker = targetFrameworkMoniker, + CustomRuntimePack = runtimePack, + Crossgen2Pack = crossgen2Pack + }), + "Mono" => new CsProjMonoBuilder(new MonoCoreSettings { TargetFrameworkMoniker = targetFrameworkMoniker }), + "NativeAOT" => new CsProjNativeAotBuilder(new NativeAotSettings { TargetFrameworkMoniker = targetFrameworkMoniker }), + _ => throw new ArgumentOutOfRangeException(nameof(toolchain)) + }; + var config = ManualConfig.CreateEmpty().CreateImmutableConfig(); + var benchmarkCase = BenchmarkCase.Create(new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo), + Job.Default, ParameterInstances.Empty, config); + var partition = new BuildPartition([new BenchmarkBuildInfo(benchmarkCase, config, 0, new([]))], BenchmarkRunnerClean.DefaultResolver); + var project = XDocument.Parse(builder.CreateProject(partition, ArtifactsPaths.Empty, NullLogger.Instance).ToString()).Root!; + output.WriteLine(project.ToString()); + + var finalProperties = GetDeploymentProperties(project, gathering: false); + if (toolchain is "R2R" or "Mono") + { + Assert.Equal("true", finalProperties["SelfContained"]); + Assert.Equal(RuntimeInformation.GetPortableRuntimeIdentifier(), finalProperties["RuntimeIdentifier"]); + } + else + Assert.False(finalProperties.ContainsKey("SelfContained")); + + foreach (string property in new[] { "UseAppHost", "SkipCompilerExecution", "SkipCopyBuildProduct" }) + Assert.False(finalProperties.ContainsKey(property)); + + if (toolchain == "R2R") + { + Assert.Equal("true", finalProperties["PublishReadyToRun"]); + Assert.Equal("true", finalProperties["PublishReadyToRunComposite"]); + var target = Assert.Single(project.Elements("Target"), target => (string?)target.Attribute("Name") == "TrickRuntimePackLocation"); + Assert.Equal("ProcessFrameworkReferences", (string?)target.Attribute("AfterTargets")); + Assert.Null(target.Attribute("Condition")); + Assert.Equal(runtimePack.FullName, target.Element("ItemGroup")!.Element("RuntimePack")!.Element("PackageDirectory")!.Value); + Assert.Equal(crossgen2Pack.FullName, target.Element("ItemGroup")!.Element("Crossgen2Pack")!.Element("PackageDirectory")!.Value); + } + else if (toolchain == "Mono") + Assert.Equal("true", finalProperties["UseMonoRuntime"]); + else if (toolchain == "NativeAOT") + Assert.Equal("true", finalProperties["PublishAot"]); + + var referenceTarget = Assert.Single(project.Elements("Target"), target => (string?)target.Attribute("Name") == "BdnWriteReferences"); + Assert.Equal(GatheringCondition, (string?)referenceTarget.Attribute("Condition")); + var referenceImport = Assert.Single(project.Elements("Import")); + Assert.Equal("$(MSBuildThisFileDirectory)ref.props", (string?)referenceImport.Attribute("Project")); + Assert.Equal($"{NotGatheringCondition} And Exists('$(MSBuildThisFileDirectory)ref.props')", (string?)referenceImport.Attribute("Condition")); + + var gatherProperties = GetDeploymentProperties(project, gathering: true); + Assert.Equal("false", gatherProperties["UseAppHost"]); + Assert.Equal("true", gatherProperties["SkipCompilerExecution"]); + Assert.Equal("true", gatherProperties["SkipCopyBuildProduct"]); + Assert.True(gatherProperties.TryGetValue("SelfContained", out string? selfContained)); + Assert.Equal("false", selfContained); + Assert.Equal("false", Assert.Single(project.Elements("PropertyGroup") + .Where(group => (string?)group.Attribute("Condition") == GatheringCondition) + .Elements("SelfContained")).Value); + } + + // Evaluate the generated deployment declarations in document order, not SDK defaults or imported targets. + private static Dictionary GetDeploymentProperties(XElement project, bool gathering) + { + var properties = new Dictionary(); + foreach (var group in project.Elements("PropertyGroup")) + { + string? condition = (string?)group.Attribute("Condition"); + if (condition is not null && condition != (gathering ? GatheringCondition : NotGatheringCondition)) + continue; + + foreach (var property in group.Elements().Where(property => property.Attribute("Condition") is null)) + properties[property.Name.LocalName] = property.Value; + } + return properties; + } + [Fact] public void FindProjectFileIgnoresDotAndHiddenDirectories() { From b9fb9afd2f9b2f94885659302978e2c6fbfd6842 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 5 Nov 2026 11:36:13 -0800 Subject: [PATCH 2/5] Exercise R2R reference gathering in integration tests Run the full gather/publish path and verify usable gathered references before checking composite output and execution. Keep this SDK-only case enabled for draft PRs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../R2RTests.cs | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs b/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs index ae397794cf..f5a07de418 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs @@ -2,8 +2,11 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.R2R; +using System.Reflection; +using System.Xml.Linq; namespace BenchmarkDotNet.IntegrationTests { @@ -11,6 +14,65 @@ public class R2RTests : BenchmarkTestExecutor { public R2RTests(ITestOutputHelper output) : base(output) { } + [FactEnvSpecific("R2R requires .NET Core runtime", EnvRequirement.DotNetCoreOnly)] + public void R2RToolchainGathersReferencesBeforePublishing() + { + var logger = new AccumulationLogger(); + var config = ManualConfig.CreateEmpty() + .AddLogger(logger) + .AddJob(Job.Dry + .WithToolchain(CsProjR2RToolchain.R2R10_0) + // Explicitly selecting Release bypasses the integration-test no-dependencies shortcut. + .WithCustomBuildConfiguration(InfrastructureMode.ReleaseConfigurationName)) + .WithBuildTimeout(TimeSpan.FromSeconds(540)) + .WithOption(ConfigOptions.KeepBenchmarkFiles, true) + .WithOption(ConfigOptions.LogBuildOutput, true); + var summary = CanExecute(config, fullValidation: false); + string log = logger.GetLog(); + Assert.True(summary.Reports.Length == 1, log); + var report = summary.Reports[0]; + var paths = report.BuildResult.ArtifactsPaths; + Output.WriteLine($"Reference-gathering artifacts: {paths.BuildArtifactsDirectoryPath}"); + + string referencesList = Path.Combine(paths.BuildArtifactsDirectoryPath, "ref.txt"); + string referencesProps = Path.Combine(paths.BuildArtifactsDirectoryPath, "ref.props"); + Assert.True(File.Exists(referencesList) && new FileInfo(referencesList).Length > 0, + $"Reference gathering did not produce a nonempty ref.txt.{Environment.NewLine}{log}"); + Assert.True(File.Exists(referencesProps), + $"Reference gathering did not complete successfully (ref.props missing).{Environment.NewLine}{log}"); + + var gatheredPaths = File.ReadAllLines(referencesList).Select(line => line.Split('|')[0]).ToArray(); + var references = XDocument.Load(referencesProps).Descendants("Reference").ToArray(); + foreach (string assemblyName in new[] { typeof(Job).Assembly.GetName().Name!, typeof(R2RBenchmark).Assembly.GetName().Name! }) + { + var matchingReferences = references.Where(reference => (string?)reference.Attribute("Include") == assemblyName).ToArray(); + Assert.True(matchingReferences.Length == 1, $"Expected gathered reference {assemblyName}.{Environment.NewLine}{log}"); + string? hintPath = (string?)matchingReferences[0].Element("HintPath"); + Assert.True(hintPath is not null && File.Exists(hintPath), $"Gathered reference {assemblyName} is missing.{Environment.NewLine}{log}"); + Assert.True(AssemblyName.GetAssemblyName(hintPath!).Name == assemblyName, log); + Assert.True(gatheredPaths.Any(path => string.Equals(path, hintPath, StringComparison.OrdinalIgnoreCase)), log); + } + + Assert.True(log.Contains("BenchmarkDotNetGatherReferences=true"), log); + Assert.False(log.Contains("--no-dependencies"), log); + Assert.False(log.Contains("Failed to gather assembly references"), log); + Assert.True(report.BuildResult.IsBuildSuccess, log); + Assert.True(File.Exists(paths.ExecutablePath), log); + string compositeAssembly = Path.ChangeExtension(paths.ExecutablePath, ".r2r.dll"); + Assert.True(File.Exists(compositeAssembly) && new FileInfo(compositeAssembly).Length > 0, log); + Assert.True(report.ExecuteResults.Count > 0 && report.ExecuteResults.All(result => result.IsSuccess), log); + Assert.True(report.Success, log); + + var project = XDocument.Load(paths.ProjectFilePath); + var r2rProperties = Assert.Single(project.Root!.Elements("PropertyGroup"), + group => group.Element("PublishReadyToRunComposite") is not null); + Assert.Equal("true", (string?)r2rProperties.Element("SelfContained")); + Assert.Equal("true", (string?)r2rProperties.Element("PublishReadyToRun")); + Assert.Equal("true", (string?)r2rProperties.Element("PublishReadyToRunComposite")); + Assert.Equal(BenchmarkDotNet.Portability.RuntimeInformation.GetPortableRuntimeIdentifier(), + (string?)r2rProperties.Element("RuntimeIdentifier")); + } + [FactEnvSpecific("R2R requires .NET Core runtime", EnvRequirement.DotNetCoreOnly, EnvRequirement.NonGitHubDraftPR)] public void R2RToolchainCanExecuteBenchmarks() { From c4b6bb8bc6ad88377ac42c02d0e051ced819e294 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 5 Nov 2026 12:45:42 -0800 Subject: [PATCH 3/5] Use integration coverage and fix Unix composite assertion Remove the generated-property theory and its test-only access change. Preserve the full R2R regression and locate its composite output by program name, not the extensionless Unix apphost path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Toolchains/CsProj/CsProjBuilder.cs | 2 +- .../R2RTests.cs | 5 +- .../CsProjBuilderTests.cs | 106 +----------------- 3 files changed, 5 insertions(+), 108 deletions(-) diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs index ec643211f4..46dce5ca63 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs @@ -196,7 +196,7 @@ protected static async ValueTask SaveXmlAsync(XDocument document, string path, C /// that a path holding an XML character cannot corrupt the file, and it is complete when written: the /// reference-gathering pass adds a file beside it rather than editing it. /// - protected internal XDocument CreateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) + protected XDocument CreateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) { var projectFile = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger); diff --git a/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs b/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs index f5a07de418..69f049b762 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs @@ -58,8 +58,9 @@ public void R2RToolchainGathersReferencesBeforePublishing() Assert.False(log.Contains("Failed to gather assembly references"), log); Assert.True(report.BuildResult.IsBuildSuccess, log); Assert.True(File.Exists(paths.ExecutablePath), log); - string compositeAssembly = Path.ChangeExtension(paths.ExecutablePath, ".r2r.dll"); - Assert.True(File.Exists(compositeAssembly) && new FileInfo(compositeAssembly).Length > 0, log); + string compositeAssembly = Path.Combine(paths.BinariesDirectoryPath, $"{paths.ProgramName}.r2r.dll"); + Assert.True(File.Exists(compositeAssembly) && new FileInfo(compositeAssembly).Length > 0, + $"Expected composite assembly: {compositeAssembly}{Environment.NewLine}{log}"); Assert.True(report.ExecuteResults.Count > 0 && report.ExecuteResults.All(result => result.IsSuccess), log); Assert.True(report.Success, log); diff --git a/tests/BenchmarkDotNet.Tests/CsProjBuilderTests.cs b/tests/BenchmarkDotNet.Tests/CsProjBuilderTests.cs index 19e5264079..4ed7b0ee3b 100644 --- a/tests/BenchmarkDotNet.Tests/CsProjBuilderTests.cs +++ b/tests/BenchmarkDotNet.Tests/CsProjBuilderTests.cs @@ -2,27 +2,20 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Parameters; -using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Mocks; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.CsProj; -using BenchmarkDotNet.Toolchains.Mono; -using BenchmarkDotNet.Toolchains.NativeAot; using BenchmarkDotNet.Toolchains.NetCoreApp; -using BenchmarkDotNet.Toolchains.R2R; using JetBrains.Annotations; using System.Reflection; using System.Xml.Linq; namespace BenchmarkDotNet.Tests { - public class CsProjBuilderTests(ITestOutputHelper output) + public class CsProjBuilderTests { - private const string GatheringCondition = "'$(BenchmarkDotNetGatherReferences)' == 'true'"; - private const string NotGatheringCondition = "'$(BenchmarkDotNetGatherReferences)' != 'true'"; private FileInfo TestAssemblyFileInfo = new FileInfo(typeof(CsProjBuilderTests).Assembly.Location); private const string runtimeHostConfigurationOptionChunk = """ @@ -286,103 +279,6 @@ public void FreshPackagesFolderIsRestoredInsideTheBuildDirectory() Assert.Equal(explicitPackages.FullName, Resolve(new NetCoreAppSettings { UseFreshPackages = true, PackagesPath = explicitPackages })); } - [Theory] - [InlineData("net10.0", "CoreCLR")] - [InlineData("net11.0", "CoreCLR")] - [InlineData("net12.0", "CoreCLR")] - [InlineData("net10.0", "R2R")] - [InlineData("net11.0", "R2R")] - [InlineData("net12.0", "R2R")] - [InlineData("net10.0", "Mono")] - [InlineData("net11.0", "Mono")] - [InlineData("net12.0", "Mono")] - [InlineData("net10.0", "NativeAOT")] - [InlineData("net11.0", "NativeAOT")] - [InlineData("net12.0", "NativeAOT")] - public void ReferenceGatheringDoesNotChangeTheFinalDeploymentMode(string targetFrameworkMoniker, string toolchain) - { - var runtimePack = new DirectoryInfo(Path.Combine(Path.GetTempPath(), "runtime & 'pack'")); - var crossgen2Pack = new FileInfo(Path.Combine(Path.GetTempPath(), "crossgen2 & 'pack'")); - CsProjBuilder builder = toolchain switch - { - "CoreCLR" => new CsProjBuilder(new NetCoreAppSettings { TargetFrameworkMoniker = targetFrameworkMoniker }), - "R2R" => new R2RBuilder(new R2RSettings - { - TargetFrameworkMoniker = targetFrameworkMoniker, - CustomRuntimePack = runtimePack, - Crossgen2Pack = crossgen2Pack - }), - "Mono" => new CsProjMonoBuilder(new MonoCoreSettings { TargetFrameworkMoniker = targetFrameworkMoniker }), - "NativeAOT" => new CsProjNativeAotBuilder(new NativeAotSettings { TargetFrameworkMoniker = targetFrameworkMoniker }), - _ => throw new ArgumentOutOfRangeException(nameof(toolchain)) - }; - var config = ManualConfig.CreateEmpty().CreateImmutableConfig(); - var benchmarkCase = BenchmarkCase.Create(new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo), - Job.Default, ParameterInstances.Empty, config); - var partition = new BuildPartition([new BenchmarkBuildInfo(benchmarkCase, config, 0, new([]))], BenchmarkRunnerClean.DefaultResolver); - var project = XDocument.Parse(builder.CreateProject(partition, ArtifactsPaths.Empty, NullLogger.Instance).ToString()).Root!; - output.WriteLine(project.ToString()); - - var finalProperties = GetDeploymentProperties(project, gathering: false); - if (toolchain is "R2R" or "Mono") - { - Assert.Equal("true", finalProperties["SelfContained"]); - Assert.Equal(RuntimeInformation.GetPortableRuntimeIdentifier(), finalProperties["RuntimeIdentifier"]); - } - else - Assert.False(finalProperties.ContainsKey("SelfContained")); - - foreach (string property in new[] { "UseAppHost", "SkipCompilerExecution", "SkipCopyBuildProduct" }) - Assert.False(finalProperties.ContainsKey(property)); - - if (toolchain == "R2R") - { - Assert.Equal("true", finalProperties["PublishReadyToRun"]); - Assert.Equal("true", finalProperties["PublishReadyToRunComposite"]); - var target = Assert.Single(project.Elements("Target"), target => (string?)target.Attribute("Name") == "TrickRuntimePackLocation"); - Assert.Equal("ProcessFrameworkReferences", (string?)target.Attribute("AfterTargets")); - Assert.Null(target.Attribute("Condition")); - Assert.Equal(runtimePack.FullName, target.Element("ItemGroup")!.Element("RuntimePack")!.Element("PackageDirectory")!.Value); - Assert.Equal(crossgen2Pack.FullName, target.Element("ItemGroup")!.Element("Crossgen2Pack")!.Element("PackageDirectory")!.Value); - } - else if (toolchain == "Mono") - Assert.Equal("true", finalProperties["UseMonoRuntime"]); - else if (toolchain == "NativeAOT") - Assert.Equal("true", finalProperties["PublishAot"]); - - var referenceTarget = Assert.Single(project.Elements("Target"), target => (string?)target.Attribute("Name") == "BdnWriteReferences"); - Assert.Equal(GatheringCondition, (string?)referenceTarget.Attribute("Condition")); - var referenceImport = Assert.Single(project.Elements("Import")); - Assert.Equal("$(MSBuildThisFileDirectory)ref.props", (string?)referenceImport.Attribute("Project")); - Assert.Equal($"{NotGatheringCondition} And Exists('$(MSBuildThisFileDirectory)ref.props')", (string?)referenceImport.Attribute("Condition")); - - var gatherProperties = GetDeploymentProperties(project, gathering: true); - Assert.Equal("false", gatherProperties["UseAppHost"]); - Assert.Equal("true", gatherProperties["SkipCompilerExecution"]); - Assert.Equal("true", gatherProperties["SkipCopyBuildProduct"]); - Assert.True(gatherProperties.TryGetValue("SelfContained", out string? selfContained)); - Assert.Equal("false", selfContained); - Assert.Equal("false", Assert.Single(project.Elements("PropertyGroup") - .Where(group => (string?)group.Attribute("Condition") == GatheringCondition) - .Elements("SelfContained")).Value); - } - - // Evaluate the generated deployment declarations in document order, not SDK defaults or imported targets. - private static Dictionary GetDeploymentProperties(XElement project, bool gathering) - { - var properties = new Dictionary(); - foreach (var group in project.Elements("PropertyGroup")) - { - string? condition = (string?)group.Attribute("Condition"); - if (condition is not null && condition != (gathering ? GatheringCondition : NotGatheringCondition)) - continue; - - foreach (var property in group.Elements().Where(property => property.Attribute("Condition") is null)) - properties[property.Name.LocalName] = property.Value; - } - return properties; - } - [Fact] public void FindProjectFileIgnoresDotAndHiddenDirectories() { From 1aac6ebbca0160b875dfec6f3a31d24dd8c40b1b Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 5 Nov 2026 21:58:01 -0800 Subject: [PATCH 4/5] Use full integration builds for publishing toolchains Keep the no-dependencies shortcut for non-publishing CLI builders. Reuse the existing publish decision internally, cover toolchain selection, and let the R2R regression use the shared path without an explicit Release override. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/BenchmarkDotNet/Running/BuildPartition.cs | 3 +- .../Toolchains/DotNetCli/DotNetCliBuilder.cs | 2 + .../IntegrationTestSetupTests.cs | 54 +++++++++++++++++++ .../R2RTests.cs | 4 +- 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BuildPartition.cs b/src/BenchmarkDotNet/Running/BuildPartition.cs index f7397e7372..c1ce4f905c 100644 --- a/src/BenchmarkDotNet/Running/BuildPartition.cs +++ b/src/BenchmarkDotNet/Running/BuildPartition.cs @@ -131,7 +131,8 @@ internal bool ForcedNoDependenciesForIntegrationTests if (!XUnitHelper.IsIntegrationTest.Value || !RuntimeInformation.IsNetCore) return false; - if (RepresentativeBenchmarkCase.GetToolchain().Builder is not DotNetCliBuilder) + // Publishing toolchains need the full reference-gathering and publish path. + if (RepresentativeBenchmarkCase.GetToolchain().Builder is not DotNetCliBuilder { IsPublishing: false }) return false; return !RepresentativeBenchmarkCase.Job.HasDynamicBuildCharacteristic(); diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs index bbc40c3bde..6fce1b7add 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs @@ -24,6 +24,8 @@ public abstract class DotNetCliBuilder(DotNetCliSettings settings) : BuilderBase /// protected virtual bool PublishesOutput => false; + internal bool IsPublishing => PublishesOutput; + /// /// Extra arguments appended to every cli command. /// diff --git a/tests/BenchmarkDotNet.IntegrationTests/IntegrationTestSetupTests.cs b/tests/BenchmarkDotNet.IntegrationTests/IntegrationTestSetupTests.cs index 0bc9a6d365..1f0e3f0284 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/IntegrationTestSetupTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/IntegrationTestSetupTests.cs @@ -1,4 +1,14 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Parameters; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Mocks; +using BenchmarkDotNet.Toolchains.CoreRun; +using BenchmarkDotNet.Toolchains.InProcess.NoEmit; +using BenchmarkDotNet.Toolchains.Mono; namespace BenchmarkDotNet.IntegrationTests; @@ -6,4 +16,48 @@ public class IntegrationTestSetupTests { [Fact] public void IntegrationTestsAreDetected() => Assert.True(XUnitHelper.IsIntegrationTest.Value); + + [Theory] + [InlineData("net10.0", "default", true)] + [InlineData("net10.0", "configuration", false)] + [InlineData("net10.0", "arguments", false)] + [InlineData("r2r10.0", "default", false)] + [InlineData("r2r10.0", "configuration", false)] + [InlineData("r2r10.0", "arguments", false)] + [InlineData("mono10.0", "default", false)] + [InlineData("nativeaot10.0", "default", false)] + [InlineData("monowasm10.0", "default", false)] + [InlineData("monowasmaot10.0", "default", false)] + [InlineData("corewasm11.0", "default", false)] + [InlineData("CoreRun", "default", false)] + [InlineData("InProcess", "default", false)] + public void OnlyNonPublishingCliBuildsUseTheNoDependenciesShortcut(string toolchain, string buildCharacteristic, bool usesShortcut) + { + var job = toolchain switch + { + "mono10.0" => Job.Dry.WithToolchain(CsProjMonoCoreToolchain.Mono10_0), + "CoreRun" => Job.Dry.WithToolchain(CoreRunToolchain.From(new CoreRunSettings + { + // This case inspects selection only; no CoreRun executable is needed or executed. + SourceCoreRun = new FileInfo(typeof(IntegrationTestSetupTests).Assembly.Location), + CreateCopy = false, + TargetFrameworkMoniker = "net10.0" + })), + "InProcess" => Job.Dry.WithToolchain(InProcessNoEmitToolchain.Default), + _ => Job.Dry.WithRuntime(Runtime.Parse(toolchain)) + }; + job = buildCharacteristic switch + { + "default" => job, + "configuration" => job.WithCustomBuildConfiguration(InfrastructureMode.ReleaseConfigurationName), + "arguments" => job.WithArguments([]), + _ => throw new ArgumentOutOfRangeException(nameof(buildCharacteristic)) + }; + var config = ManualConfig.CreateEmpty().CreateImmutableConfig(); + var benchmark = BenchmarkCase.Create(new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo), + job, ParameterInstances.Empty, config); + var partition = new BuildPartition([new BenchmarkBuildInfo(benchmark, config, 0, new([]))], BenchmarkRunnerClean.DefaultResolver); + + Assert.Equal(usesShortcut && RuntimeInformation.IsNetCore, partition.ForcedNoDependenciesForIntegrationTests); + } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs b/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs index 69f049b762..ffb10cb913 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs @@ -21,9 +21,7 @@ public void R2RToolchainGathersReferencesBeforePublishing() var config = ManualConfig.CreateEmpty() .AddLogger(logger) .AddJob(Job.Dry - .WithToolchain(CsProjR2RToolchain.R2R10_0) - // Explicitly selecting Release bypasses the integration-test no-dependencies shortcut. - .WithCustomBuildConfiguration(InfrastructureMode.ReleaseConfigurationName)) + .WithToolchain(CsProjR2RToolchain.R2R10_0)) .WithBuildTimeout(TimeSpan.FromSeconds(540)) .WithOption(ConfigOptions.KeepBenchmarkFiles, true) .WithOption(ConfigOptions.LogBuildOutput, true); From c3d03fba217ea3000c43fbb9c35433d9f82935fe Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Fri, 6 Nov 2026 01:22:05 -0500 Subject: [PATCH 5/5] Fail integration tests on logged warnings A failed reference-gathering pass is only a warning, and the benchmark still builds and runs, so CanExecute's full validation now fails on any logged warning. Analyser conclusions are excluded since they depend on the machine. Tests that expect warnings opt out with allowWarnings, as the NativeAOT tests do for BenchmarkDotNet's own trim and AOT warnings. With publishing toolchains going through reference gathering, that check catches a Wasm failure: the static web assets pipeline converts the program's assembly, which the gathering pass never compiles. The pass only resolves references, so it disables static web assets. The other integration tests now cover gathering for publishing toolchains, which makes the dedicated R2R gathering test redundant. Co-Authored-By: Claude Opus 5.5 --- .../Toolchains/CsProj/CsProjBuilder.cs | 6 +- .../NativeAotTests.cs | 7 ++- .../R2RTests.cs | 61 ------------------- .../Shared/BenchmarkTestExecutor.cs | 29 +++++---- .../Shared/WarningCollectingLogger.cs | 58 ++++++++++++++++++ 5 files changed, 85 insertions(+), 76 deletions(-) create mode 100644 tests/BenchmarkDotNet.IntegrationTests/Shared/WarningCollectingLogger.cs diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs index 46dce5ca63..1143500364 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs @@ -321,9 +321,11 @@ protected void AddGatheredReferences(XElement project) new XElement("SkipCompilerExecution", "true"), // A self-contained executable requires an apphost, which this pass never produces. new XElement("SelfContained", "false"), - // Both of these copy a build product that no longer gets produced. + // All of these process a build product that no longer gets produced (static web assets convert + // and fingerprint the program's assembly, for one, which is how Wasm lays out its app). new XElement("UseAppHost", "false"), - new XElement("SkipCopyBuildProduct", "true"))); + new XElement("SkipCopyBuildProduct", "true"), + new XElement("StaticWebAssetsEnabled", "false"))); project.Add(new XElement("Target", new XAttribute("Name", "BdnWriteReferences"), diff --git a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs index 5ccb13a7c0..6feee56456 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs @@ -34,6 +34,9 @@ private ManualConfig GetConfig() .WithEnvironmentVariable(NativeAotBenchmark.EnvVarKey, IsAvx2Supported().ToString().ToLower())); } + // BenchmarkDotNet itself is not trim or AOT safe, so publishing reports IL2104 and IL3053 for its assembly. + private const bool AllowWarnings = true; + [FactEnvSpecific("It's impossible to reliably detect the version of NativeAOT if the process is not a .NET Core or NativeAOT process", EnvRequirement.DotNetCoreOnly, EnvRequirement.NonGitHubDraftPR)] public void LatestNativeAotVersionIsSupported() { @@ -42,7 +45,7 @@ public void LatestNativeAotVersionIsSupported() try { - CanExecute(GetConfig()); + CanExecute(GetConfig(), allowWarnings: AllowWarnings); } catch (MisconfiguredEnvironmentException e) { @@ -66,7 +69,7 @@ public void NativeAotSupportsInProcessDiagnosers() try { - CanExecute(config); + CanExecute(config, allowWarnings: AllowWarnings); } catch (MisconfiguredEnvironmentException e) { diff --git a/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs b/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs index ffb10cb913..ae397794cf 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs @@ -2,11 +2,8 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.R2R; -using System.Reflection; -using System.Xml.Linq; namespace BenchmarkDotNet.IntegrationTests { @@ -14,64 +11,6 @@ public class R2RTests : BenchmarkTestExecutor { public R2RTests(ITestOutputHelper output) : base(output) { } - [FactEnvSpecific("R2R requires .NET Core runtime", EnvRequirement.DotNetCoreOnly)] - public void R2RToolchainGathersReferencesBeforePublishing() - { - var logger = new AccumulationLogger(); - var config = ManualConfig.CreateEmpty() - .AddLogger(logger) - .AddJob(Job.Dry - .WithToolchain(CsProjR2RToolchain.R2R10_0)) - .WithBuildTimeout(TimeSpan.FromSeconds(540)) - .WithOption(ConfigOptions.KeepBenchmarkFiles, true) - .WithOption(ConfigOptions.LogBuildOutput, true); - var summary = CanExecute(config, fullValidation: false); - string log = logger.GetLog(); - Assert.True(summary.Reports.Length == 1, log); - var report = summary.Reports[0]; - var paths = report.BuildResult.ArtifactsPaths; - Output.WriteLine($"Reference-gathering artifacts: {paths.BuildArtifactsDirectoryPath}"); - - string referencesList = Path.Combine(paths.BuildArtifactsDirectoryPath, "ref.txt"); - string referencesProps = Path.Combine(paths.BuildArtifactsDirectoryPath, "ref.props"); - Assert.True(File.Exists(referencesList) && new FileInfo(referencesList).Length > 0, - $"Reference gathering did not produce a nonempty ref.txt.{Environment.NewLine}{log}"); - Assert.True(File.Exists(referencesProps), - $"Reference gathering did not complete successfully (ref.props missing).{Environment.NewLine}{log}"); - - var gatheredPaths = File.ReadAllLines(referencesList).Select(line => line.Split('|')[0]).ToArray(); - var references = XDocument.Load(referencesProps).Descendants("Reference").ToArray(); - foreach (string assemblyName in new[] { typeof(Job).Assembly.GetName().Name!, typeof(R2RBenchmark).Assembly.GetName().Name! }) - { - var matchingReferences = references.Where(reference => (string?)reference.Attribute("Include") == assemblyName).ToArray(); - Assert.True(matchingReferences.Length == 1, $"Expected gathered reference {assemblyName}.{Environment.NewLine}{log}"); - string? hintPath = (string?)matchingReferences[0].Element("HintPath"); - Assert.True(hintPath is not null && File.Exists(hintPath), $"Gathered reference {assemblyName} is missing.{Environment.NewLine}{log}"); - Assert.True(AssemblyName.GetAssemblyName(hintPath!).Name == assemblyName, log); - Assert.True(gatheredPaths.Any(path => string.Equals(path, hintPath, StringComparison.OrdinalIgnoreCase)), log); - } - - Assert.True(log.Contains("BenchmarkDotNetGatherReferences=true"), log); - Assert.False(log.Contains("--no-dependencies"), log); - Assert.False(log.Contains("Failed to gather assembly references"), log); - Assert.True(report.BuildResult.IsBuildSuccess, log); - Assert.True(File.Exists(paths.ExecutablePath), log); - string compositeAssembly = Path.Combine(paths.BinariesDirectoryPath, $"{paths.ProgramName}.r2r.dll"); - Assert.True(File.Exists(compositeAssembly) && new FileInfo(compositeAssembly).Length > 0, - $"Expected composite assembly: {compositeAssembly}{Environment.NewLine}{log}"); - Assert.True(report.ExecuteResults.Count > 0 && report.ExecuteResults.All(result => result.IsSuccess), log); - Assert.True(report.Success, log); - - var project = XDocument.Load(paths.ProjectFilePath); - var r2rProperties = Assert.Single(project.Root!.Elements("PropertyGroup"), - group => group.Element("PublishReadyToRunComposite") is not null); - Assert.Equal("true", (string?)r2rProperties.Element("SelfContained")); - Assert.Equal("true", (string?)r2rProperties.Element("PublishReadyToRun")); - Assert.Equal("true", (string?)r2rProperties.Element("PublishReadyToRunComposite")); - Assert.Equal(BenchmarkDotNet.Portability.RuntimeInformation.GetPortableRuntimeIdentifier(), - (string?)r2rProperties.Element("RuntimeIdentifier")); - } - [FactEnvSpecific("R2R requires .NET Core runtime", EnvRequirement.DotNetCoreOnly, EnvRequirement.NonGitHubDraftPR)] public void R2RToolchainCanExecuteBenchmarks() { diff --git a/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs b/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs index 58c72ad38f..7c8fb9bfe2 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs @@ -26,9 +26,10 @@ protected BenchmarkTestExecutor(ITestOutputHelper output) /// type that defines Benchmarks /// Optional custom config to be used instead of the default /// Optional: disable validation (default = true/enabled) + /// Optional: don't fail the validation on logged warnings (default = false) /// The summary from the benchmark run - public Summary CanExecute(IConfig? config = null, bool fullValidation = true) - => CanExecute(typeof(TBenchmark), config, fullValidation); + public Summary CanExecute(IConfig? config = null, bool fullValidation = true, bool allowWarnings = false) + => CanExecute(typeof(TBenchmark), config, fullValidation, allowWarnings); /// /// Runs Benchmarks with the most simple config (SingleRunFastConfig) @@ -38,31 +39,34 @@ public Summary CanExecute(IConfig? config = null, bool fullValidatio /// type that defines Benchmarks /// Optional custom config to be used instead of the default /// Optional: disable validation (default = true/enabled) + /// Optional: don't fail the validation on logged warnings (default = false) /// The summary from the benchmark run - public Summary CanExecute(Type type, IConfig? config = null, bool fullValidation = true) + public Summary CanExecute(Type type, IConfig? config = null, bool fullValidation = true, bool allowWarnings = false) { // Make sure we ALWAYS combine the Config (default or passed in) with any Config applied to the Type/Class - var summary = BenchmarkRunner.Run(type, GetMinimalConfig(config)); + var warnings = new WarningCollectingLogger(); + var summary = BenchmarkRunner.Run(type, GetMinimalConfig(config).AddLogger(warnings)); if (fullValidation) { - ValidateSummary(summary); + ValidateSummary(summary, warnings, allowWarnings); } return summary; } - public ValueTask CanExecuteAsync(IConfig? config = null, bool fullValidation = true) - => CanExecuteAsync(typeof(TBenchmark), config, fullValidation); + public ValueTask CanExecuteAsync(IConfig? config = null, bool fullValidation = true, bool allowWarnings = false) + => CanExecuteAsync(typeof(TBenchmark), config, fullValidation, allowWarnings); - public async ValueTask CanExecuteAsync(Type type, IConfig? config = null, bool fullValidation = true) + public async ValueTask CanExecuteAsync(Type type, IConfig? config = null, bool fullValidation = true, bool allowWarnings = false) { // Make sure we ALWAYS combine the Config (default or passed in) with any Config applied to the Type/Class - var summary = await BenchmarkRunner.RunAsync(type, GetMinimalConfig(config)); + var warnings = new WarningCollectingLogger(); + var summary = await BenchmarkRunner.RunAsync(type, GetMinimalConfig(config).AddLogger(warnings)); if (fullValidation) { - ValidateSummary(summary); + ValidateSummary(summary, warnings, allowWarnings); } return summary; @@ -85,7 +89,7 @@ private IConfig GetMinimalConfig(IConfig? config) return config!; } - private static void ValidateSummary(Summary summary) + private static void ValidateSummary(Summary summary, WarningCollectingLogger warnings, bool allowWarnings) { Assert.False(summary.HasCriticalValidationErrors, "The \"Summary\" should have NOT \"HasCriticalValidationErrors\""); @@ -103,6 +107,9 @@ private static void ValidateSummary(Summary summary) Assert.True(summary.Reports.All(r => r.ExecuteResults.All(er => er.IsSuccess)), "All reports should have succeeded to execute"); + + Assert.True(allowWarnings || warnings.Warnings.Count == 0, + "The run should not log any warnings:" + Environment.NewLine + string.Join(Environment.NewLine, warnings.Warnings)); } protected IConfig CreateSimpleConfig(OutputLogger? logger = null, Job? job = null) diff --git a/tests/BenchmarkDotNet.IntegrationTests/Shared/WarningCollectingLogger.cs b/tests/BenchmarkDotNet.IntegrationTests/Shared/WarningCollectingLogger.cs new file mode 100644 index 0000000000..bf93c769ae --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/Shared/WarningCollectingLogger.cs @@ -0,0 +1,58 @@ +using BenchmarkDotNet.Loggers; + +namespace BenchmarkDotNet.IntegrationTests; + +/// +/// Collects the warnings a benchmark run logs, so that a test can fail on them. +/// +/// +/// Analyser conclusions are skipped: they describe the measurements and the machine (a tiny Dry run, an antivirus, +/// a virtual machine) rather than a problem with the run itself, and the summary keeps them anyway. +/// +internal sealed class WarningCollectingLogger : ILogger +{ + // The header ConclusionHelper prints before its warning conclusions, which last until something else is written. + private const string ConclusionsHeader = "// * Warnings *"; + + private readonly List warnings = []; + private string currentLine = ""; + private bool inConclusions; + + public string Id => nameof(WarningCollectingLogger); + public int Priority => 0; + + public IReadOnlyList Warnings => warnings; + + public void Write(LogKind logKind, string text) + { + Track(logKind, text); + if (logKind == LogKind.Warning) + currentLine += text; + } + + public void WriteLine() => EndLine(); + + public void WriteLine(LogKind logKind, string text) + { + Write(logKind, text); + EndLine(); + } + + public void Flush() { } + + private void Track(LogKind logKind, string text) + { + if (logKind != LogKind.Warning) + inConclusions = false; + else if (text == ConclusionsHeader) + inConclusions = true; + } + + private void EndLine() + { + // ConclusionHelper writes an empty warning line before its header. + if (!inConclusions && currentLine.Length > 0) + warnings.Add(currentLine); + currentLine = ""; + } +}