diff --git a/src/BenchmarkDotNet/Running/BuildPartition.cs b/src/BenchmarkDotNet/Running/BuildPartition.cs index f7397e7372..c1ce4f905c 100644 --- a/src/BenchmarkDotNet/Running/BuildPartition.cs +++ b/src/BenchmarkDotNet/Running/BuildPartition.cs @@ -131,7 +131,8 @@ internal bool ForcedNoDependenciesForIntegrationTests if (!XUnitHelper.IsIntegrationTest.Value || !RuntimeInformation.IsNetCore) return false; - if (RepresentativeBenchmarkCase.GetToolchain().Builder is not DotNetCliBuilder) + // Publishing toolchains need the full reference-gathering and publish path. + if (RepresentativeBenchmarkCase.GetToolchain().Builder is not DotNetCliBuilder { IsPublishing: false }) return false; return !RepresentativeBenchmarkCase.Job.HasDynamicBuildCharacteristic(); diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs index 2be677a661..1143500364 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs @@ -319,9 +319,13 @@ protected void AddGatheredReferences(XElement project) project.Add(new XElement("PropertyGroup", new XAttribute("Condition", GatheringCondition), new XElement("SkipCompilerExecution", "true"), - // Both of these copy a build product that no longer gets produced. + // A self-contained executable requires an apphost, which this pass never produces. + new XElement("SelfContained", "false"), + // All of these process a build product that no longer gets produced (static web assets convert + // and fingerprint the program's assembly, for one, which is how Wasm lays out its app). new XElement("UseAppHost", "false"), - new XElement("SkipCopyBuildProduct", "true"))); + new XElement("SkipCopyBuildProduct", "true"), + new XElement("StaticWebAssetsEnabled", "false"))); project.Add(new XElement("Target", new XAttribute("Name", "BdnWriteReferences"), diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs index bbc40c3bde..6fce1b7add 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs @@ -24,6 +24,8 @@ public abstract class DotNetCliBuilder(DotNetCliSettings settings) : BuilderBase /// protected virtual bool PublishesOutput => false; + internal bool IsPublishing => PublishesOutput; + /// /// Extra arguments appended to every cli command. /// diff --git a/tests/BenchmarkDotNet.IntegrationTests/IntegrationTestSetupTests.cs b/tests/BenchmarkDotNet.IntegrationTests/IntegrationTestSetupTests.cs index 0bc9a6d365..1f0e3f0284 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/IntegrationTestSetupTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/IntegrationTestSetupTests.cs @@ -1,4 +1,14 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Parameters; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Mocks; +using BenchmarkDotNet.Toolchains.CoreRun; +using BenchmarkDotNet.Toolchains.InProcess.NoEmit; +using BenchmarkDotNet.Toolchains.Mono; namespace BenchmarkDotNet.IntegrationTests; @@ -6,4 +16,48 @@ public class IntegrationTestSetupTests { [Fact] public void IntegrationTestsAreDetected() => Assert.True(XUnitHelper.IsIntegrationTest.Value); + + [Theory] + [InlineData("net10.0", "default", true)] + [InlineData("net10.0", "configuration", false)] + [InlineData("net10.0", "arguments", false)] + [InlineData("r2r10.0", "default", false)] + [InlineData("r2r10.0", "configuration", false)] + [InlineData("r2r10.0", "arguments", false)] + [InlineData("mono10.0", "default", false)] + [InlineData("nativeaot10.0", "default", false)] + [InlineData("monowasm10.0", "default", false)] + [InlineData("monowasmaot10.0", "default", false)] + [InlineData("corewasm11.0", "default", false)] + [InlineData("CoreRun", "default", false)] + [InlineData("InProcess", "default", false)] + public void OnlyNonPublishingCliBuildsUseTheNoDependenciesShortcut(string toolchain, string buildCharacteristic, bool usesShortcut) + { + var job = toolchain switch + { + "mono10.0" => Job.Dry.WithToolchain(CsProjMonoCoreToolchain.Mono10_0), + "CoreRun" => Job.Dry.WithToolchain(CoreRunToolchain.From(new CoreRunSettings + { + // This case inspects selection only; no CoreRun executable is needed or executed. + SourceCoreRun = new FileInfo(typeof(IntegrationTestSetupTests).Assembly.Location), + CreateCopy = false, + TargetFrameworkMoniker = "net10.0" + })), + "InProcess" => Job.Dry.WithToolchain(InProcessNoEmitToolchain.Default), + _ => Job.Dry.WithRuntime(Runtime.Parse(toolchain)) + }; + job = buildCharacteristic switch + { + "default" => job, + "configuration" => job.WithCustomBuildConfiguration(InfrastructureMode.ReleaseConfigurationName), + "arguments" => job.WithArguments([]), + _ => throw new ArgumentOutOfRangeException(nameof(buildCharacteristic)) + }; + var config = ManualConfig.CreateEmpty().CreateImmutableConfig(); + var benchmark = BenchmarkCase.Create(new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo), + job, ParameterInstances.Empty, config); + var partition = new BuildPartition([new BenchmarkBuildInfo(benchmark, config, 0, new([]))], BenchmarkRunnerClean.DefaultResolver); + + Assert.Equal(usesShortcut && RuntimeInformation.IsNetCore, partition.ForcedNoDependenciesForIntegrationTests); + } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs index 5ccb13a7c0..6feee56456 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs @@ -34,6 +34,9 @@ private ManualConfig GetConfig() .WithEnvironmentVariable(NativeAotBenchmark.EnvVarKey, IsAvx2Supported().ToString().ToLower())); } + // BenchmarkDotNet itself is not trim or AOT safe, so publishing reports IL2104 and IL3053 for its assembly. + private const bool AllowWarnings = true; + [FactEnvSpecific("It's impossible to reliably detect the version of NativeAOT if the process is not a .NET Core or NativeAOT process", EnvRequirement.DotNetCoreOnly, EnvRequirement.NonGitHubDraftPR)] public void LatestNativeAotVersionIsSupported() { @@ -42,7 +45,7 @@ public void LatestNativeAotVersionIsSupported() try { - CanExecute(GetConfig()); + CanExecute(GetConfig(), allowWarnings: AllowWarnings); } catch (MisconfiguredEnvironmentException e) { @@ -66,7 +69,7 @@ public void NativeAotSupportsInProcessDiagnosers() try { - CanExecute(config); + CanExecute(config, allowWarnings: AllowWarnings); } catch (MisconfiguredEnvironmentException e) { diff --git a/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs b/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs index 58c72ad38f..7c8fb9bfe2 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs @@ -26,9 +26,10 @@ protected BenchmarkTestExecutor(ITestOutputHelper output) /// type that defines Benchmarks /// Optional custom config to be used instead of the default /// Optional: disable validation (default = true/enabled) + /// Optional: don't fail the validation on logged warnings (default = false) /// The summary from the benchmark run - public Summary CanExecute(IConfig? config = null, bool fullValidation = true) - => CanExecute(typeof(TBenchmark), config, fullValidation); + public Summary CanExecute(IConfig? config = null, bool fullValidation = true, bool allowWarnings = false) + => CanExecute(typeof(TBenchmark), config, fullValidation, allowWarnings); /// /// Runs Benchmarks with the most simple config (SingleRunFastConfig) @@ -38,31 +39,34 @@ public Summary CanExecute(IConfig? config = null, bool fullValidatio /// type that defines Benchmarks /// Optional custom config to be used instead of the default /// Optional: disable validation (default = true/enabled) + /// Optional: don't fail the validation on logged warnings (default = false) /// The summary from the benchmark run - public Summary CanExecute(Type type, IConfig? config = null, bool fullValidation = true) + public Summary CanExecute(Type type, IConfig? config = null, bool fullValidation = true, bool allowWarnings = false) { // Make sure we ALWAYS combine the Config (default or passed in) with any Config applied to the Type/Class - var summary = BenchmarkRunner.Run(type, GetMinimalConfig(config)); + var warnings = new WarningCollectingLogger(); + var summary = BenchmarkRunner.Run(type, GetMinimalConfig(config).AddLogger(warnings)); if (fullValidation) { - ValidateSummary(summary); + ValidateSummary(summary, warnings, allowWarnings); } return summary; } - public ValueTask CanExecuteAsync(IConfig? config = null, bool fullValidation = true) - => CanExecuteAsync(typeof(TBenchmark), config, fullValidation); + public ValueTask CanExecuteAsync(IConfig? config = null, bool fullValidation = true, bool allowWarnings = false) + => CanExecuteAsync(typeof(TBenchmark), config, fullValidation, allowWarnings); - public async ValueTask CanExecuteAsync(Type type, IConfig? config = null, bool fullValidation = true) + public async ValueTask CanExecuteAsync(Type type, IConfig? config = null, bool fullValidation = true, bool allowWarnings = false) { // Make sure we ALWAYS combine the Config (default or passed in) with any Config applied to the Type/Class - var summary = await BenchmarkRunner.RunAsync(type, GetMinimalConfig(config)); + var warnings = new WarningCollectingLogger(); + var summary = await BenchmarkRunner.RunAsync(type, GetMinimalConfig(config).AddLogger(warnings)); if (fullValidation) { - ValidateSummary(summary); + ValidateSummary(summary, warnings, allowWarnings); } return summary; @@ -85,7 +89,7 @@ private IConfig GetMinimalConfig(IConfig? config) return config!; } - private static void ValidateSummary(Summary summary) + private static void ValidateSummary(Summary summary, WarningCollectingLogger warnings, bool allowWarnings) { Assert.False(summary.HasCriticalValidationErrors, "The \"Summary\" should have NOT \"HasCriticalValidationErrors\""); @@ -103,6 +107,9 @@ private static void ValidateSummary(Summary summary) Assert.True(summary.Reports.All(r => r.ExecuteResults.All(er => er.IsSuccess)), "All reports should have succeeded to execute"); + + Assert.True(allowWarnings || warnings.Warnings.Count == 0, + "The run should not log any warnings:" + Environment.NewLine + string.Join(Environment.NewLine, warnings.Warnings)); } protected IConfig CreateSimpleConfig(OutputLogger? logger = null, Job? job = null) diff --git a/tests/BenchmarkDotNet.IntegrationTests/Shared/WarningCollectingLogger.cs b/tests/BenchmarkDotNet.IntegrationTests/Shared/WarningCollectingLogger.cs new file mode 100644 index 0000000000..bf93c769ae --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/Shared/WarningCollectingLogger.cs @@ -0,0 +1,58 @@ +using BenchmarkDotNet.Loggers; + +namespace BenchmarkDotNet.IntegrationTests; + +/// +/// Collects the warnings a benchmark run logs, so that a test can fail on them. +/// +/// +/// Analyser conclusions are skipped: they describe the measurements and the machine (a tiny Dry run, an antivirus, +/// a virtual machine) rather than a problem with the run itself, and the summary keeps them anyway. +/// +internal sealed class WarningCollectingLogger : ILogger +{ + // The header ConclusionHelper prints before its warning conclusions, which last until something else is written. + private const string ConclusionsHeader = "// * Warnings *"; + + private readonly List warnings = []; + private string currentLine = ""; + private bool inConclusions; + + public string Id => nameof(WarningCollectingLogger); + public int Priority => 0; + + public IReadOnlyList Warnings => warnings; + + public void Write(LogKind logKind, string text) + { + Track(logKind, text); + if (logKind == LogKind.Warning) + currentLine += text; + } + + public void WriteLine() => EndLine(); + + public void WriteLine(LogKind logKind, string text) + { + Write(logKind, text); + EndLine(); + } + + public void Flush() { } + + private void Track(LogKind logKind, string text) + { + if (logKind != LogKind.Warning) + inConclusions = false; + else if (text == ConclusionsHeader) + inConclusions = true; + } + + private void EndLine() + { + // ConclusionHelper writes an empty warning line before its header. + if (!inConclusions && currentLine.Length > 0) + warnings.Add(currentLine); + currentLine = ""; + } +}