Fail integration tests on logged warnings

src/BenchmarkDotNet/Toolchains/CsProj/CsProjBuilder.cs

@@ -321,9 +321,11 @@ protected void AddGatheredReferences(XElement project)
                 new XElement("SkipCompilerExecution", "true"),
                 // A self-contained executable requires an apphost, which this pass never produces.
                 new XElement("SelfContained", "false"),
-                // Both of these copy a build product that no longer gets produced.
+                // All of these process a build product that no longer gets produced (static web assets convert
+                // and fingerprint the program's assembly, for one, which is how Wasm lays out its app).
                 new XElement("UseAppHost", "false"),
-                new XElement("SkipCopyBuildProduct", "true")));
+                new XElement("SkipCopyBuildProduct", "true"),
+                new XElement("StaticWebAssetsEnabled", "false")));
 
             project.Add(new XElement("Target",
                 new XAttribute("Name", "BdnWriteReferences"),

tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs

@@ -34,6 +34,9 @@ private ManualConfig GetConfig()
                     .WithEnvironmentVariable(NativeAotBenchmark.EnvVarKey, IsAvx2Supported().ToString().ToLower()));
         }
 
+        // BenchmarkDotNet itself is not trim or AOT safe, so publishing reports IL2104 and IL3053 for its assembly.
+        private const bool AllowWarnings = true;
+
         [FactEnvSpecific("It's impossible to reliably detect the version of NativeAOT if the process is not a .NET Core or NativeAOT process", EnvRequirement.DotNetCoreOnly, EnvRequirement.NonGitHubDraftPR)]
         public void LatestNativeAotVersionIsSupported()
         {
@@ -42,7 +45,7 @@ public void LatestNativeAotVersionIsSupported()
 
             try
             {
-                CanExecute<NativeAotBenchmark>(GetConfig());
+                CanExecute<NativeAotBenchmark>(GetConfig(), allowWarnings: AllowWarnings);
             }
             catch (MisconfiguredEnvironmentException e)
             {
@@ -66,7 +69,7 @@ public void NativeAotSupportsInProcessDiagnosers()
 
                 try
                 {
-                    CanExecute<NativeAotBenchmark>(config);
+                    CanExecute<NativeAotBenchmark>(config, allowWarnings: AllowWarnings);
                 }
                 catch (MisconfiguredEnvironmentException e)
                 {

tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs

@@ -2,76 +2,15 @@
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.IntegrationTests.Xunit;
 using BenchmarkDotNet.Jobs;
-using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Tests.XUnit;
 using BenchmarkDotNet.Toolchains.R2R;
-using System.Reflection;
-using System.Xml.Linq;
 
 namespace BenchmarkDotNet.IntegrationTests
 {
     public class R2RTests : BenchmarkTestExecutor
     {
         public R2RTests(ITestOutputHelper output) : base(output) { }
 
-        [FactEnvSpecific("R2R requires .NET Core runtime", EnvRequirement.DotNetCoreOnly)]
-        public void R2RToolchainGathersReferencesBeforePublishing()
-        {
-            var logger = new AccumulationLogger();
-            var config = ManualConfig.CreateEmpty()
-                .AddLogger(logger)
-                .AddJob(Job.Dry
-                    .WithToolchain(CsProjR2RToolchain.R2R10_0))
-                .WithBuildTimeout(TimeSpan.FromSeconds(540))
-                .WithOption(ConfigOptions.KeepBenchmarkFiles, true)
-                .WithOption(ConfigOptions.LogBuildOutput, true);
-            var summary = CanExecute<R2RBenchmark>(config, fullValidation: false);
-            string log = logger.GetLog();
-            Assert.True(summary.Reports.Length == 1, log);
-            var report = summary.Reports[0];
-            var paths = report.BuildResult.ArtifactsPaths;
-            Output.WriteLine($"Reference-gathering artifacts: {paths.BuildArtifactsDirectoryPath}");
-
-            string referencesList = Path.Combine(paths.BuildArtifactsDirectoryPath, "ref.txt");
-            string referencesProps = Path.Combine(paths.BuildArtifactsDirectoryPath, "ref.props");
-            Assert.True(File.Exists(referencesList) && new FileInfo(referencesList).Length > 0,
-                $"Reference gathering did not produce a nonempty ref.txt.{Environment.NewLine}{log}");
-            Assert.True(File.Exists(referencesProps),
-                $"Reference gathering did not complete successfully (ref.props missing).{Environment.NewLine}{log}");
-
-            var gatheredPaths = File.ReadAllLines(referencesList).Select(line => line.Split('|')[0]).ToArray();
-            var references = XDocument.Load(referencesProps).Descendants("Reference").ToArray();
-            foreach (string assemblyName in new[] { typeof(Job).Assembly.GetName().Name!, typeof(R2RBenchmark).Assembly.GetName().Name! })
-            {
-                var matchingReferences = references.Where(reference => (string?)reference.Attribute("Include") == assemblyName).ToArray();
-                Assert.True(matchingReferences.Length == 1, $"Expected gathered reference {assemblyName}.{Environment.NewLine}{log}");
-                string? hintPath = (string?)matchingReferences[0].Element("HintPath");
-                Assert.True(hintPath is not null && File.Exists(hintPath), $"Gathered reference {assemblyName} is missing.{Environment.NewLine}{log}");
-                Assert.True(AssemblyName.GetAssemblyName(hintPath!).Name == assemblyName, log);
-                Assert.True(gatheredPaths.Any(path => string.Equals(path, hintPath, StringComparison.OrdinalIgnoreCase)), log);
-            }
-
-            Assert.True(log.Contains("BenchmarkDotNetGatherReferences=true"), log);
-            Assert.False(log.Contains("--no-dependencies"), log);
-            Assert.False(log.Contains("Failed to gather assembly references"), log);
-            Assert.True(report.BuildResult.IsBuildSuccess, log);
-            Assert.True(File.Exists(paths.ExecutablePath), log);
-            string compositeAssembly = Path.Combine(paths.BinariesDirectoryPath, $"{paths.ProgramName}.r2r.dll");
-            Assert.True(File.Exists(compositeAssembly) && new FileInfo(compositeAssembly).Length > 0,
-                $"Expected composite assembly: {compositeAssembly}{Environment.NewLine}{log}");
-            Assert.True(report.ExecuteResults.Count > 0 && report.ExecuteResults.All(result => result.IsSuccess), log);
-            Assert.True(report.Success, log);
-
-            var project = XDocument.Load(paths.ProjectFilePath);
-            var r2rProperties = Assert.Single(project.Root!.Elements("PropertyGroup"),
-                group => group.Element("PublishReadyToRunComposite") is not null);
-            Assert.Equal("true", (string?)r2rProperties.Element("SelfContained"));
-            Assert.Equal("true", (string?)r2rProperties.Element("PublishReadyToRun"));
-            Assert.Equal("true", (string?)r2rProperties.Element("PublishReadyToRunComposite"));
-            Assert.Equal(BenchmarkDotNet.Portability.RuntimeInformation.GetPortableRuntimeIdentifier(),
-                (string?)r2rProperties.Element("RuntimeIdentifier"));
-        }
-
         [FactEnvSpecific("R2R requires .NET Core runtime", EnvRequirement.DotNetCoreOnly, EnvRequirement.NonGitHubDraftPR)]
         public void R2RToolchainCanExecuteBenchmarks()
         {

tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs

@@ -26,9 +26,10 @@ protected BenchmarkTestExecutor(ITestOutputHelper output)
         /// <typeparam name="TBenchmark">type that defines Benchmarks</typeparam>
         /// <param name="config">Optional custom config to be used instead of the default</param>
         /// <param name="fullValidation">Optional: disable validation (default = true/enabled)</param>
+        /// <param name="allowWarnings">Optional: don't fail the validation on logged warnings (default = false)</param>
         /// <returns>The summary from the benchmark run</returns>
-        public Summary CanExecute<TBenchmark>(IConfig? config = null, bool fullValidation = true)
-            => CanExecute(typeof(TBenchmark), config, fullValidation);
+        public Summary CanExecute<TBenchmark>(IConfig? config = null, bool fullValidation = true, bool allowWarnings = false)
+            => CanExecute(typeof(TBenchmark), config, fullValidation, allowWarnings);
 
         /// <summary>
         /// Runs Benchmarks with the most simple config (SingleRunFastConfig)
@@ -38,31 +39,34 @@ public Summary CanExecute<TBenchmark>(IConfig? config = null, bool fullValidatio
         /// <param name="type">type that defines Benchmarks</param>
         /// <param name="config">Optional custom config to be used instead of the default</param>
         /// <param name="fullValidation">Optional: disable validation (default = true/enabled)</param>
+        /// <param name="allowWarnings">Optional: don't fail the validation on logged warnings (default = false)</param>
         /// <returns>The summary from the benchmark run</returns>
-        public Summary CanExecute(Type type, IConfig? config = null, bool fullValidation = true)
+        public Summary CanExecute(Type type, IConfig? config = null, bool fullValidation = true, bool allowWarnings = false)
         {
             // Make sure we ALWAYS combine the Config (default or passed in) with any Config applied to the Type/Class
-            var summary = BenchmarkRunner.Run(type, GetMinimalConfig(config));
+            var warnings = new WarningCollectingLogger();
+            var summary = BenchmarkRunner.Run(type, GetMinimalConfig(config).AddLogger(warnings));
 
             if (fullValidation)
             {
-                ValidateSummary(summary);
+                ValidateSummary(summary, warnings, allowWarnings);
             }
 
             return summary;
         }
 
-        public ValueTask<Summary> CanExecuteAsync<TBenchmark>(IConfig? config = null, bool fullValidation = true)
-            => CanExecuteAsync(typeof(TBenchmark), config, fullValidation);
+        public ValueTask<Summary> CanExecuteAsync<TBenchmark>(IConfig? config = null, bool fullValidation = true, bool allowWarnings = false)
+            => CanExecuteAsync(typeof(TBenchmark), config, fullValidation, allowWarnings);
 
-        public async ValueTask<Summary> CanExecuteAsync(Type type, IConfig? config = null, bool fullValidation = true)
+        public async ValueTask<Summary> CanExecuteAsync(Type type, IConfig? config = null, bool fullValidation = true, bool allowWarnings = false)
         {
             // Make sure we ALWAYS combine the Config (default or passed in) with any Config applied to the Type/Class
-            var summary = await BenchmarkRunner.RunAsync(type, GetMinimalConfig(config));
+            var warnings = new WarningCollectingLogger();
+            var summary = await BenchmarkRunner.RunAsync(type, GetMinimalConfig(config).AddLogger(warnings));
 
             if (fullValidation)
             {
-                ValidateSummary(summary);
+                ValidateSummary(summary, warnings, allowWarnings);
             }
 
             return summary;
@@ -85,7 +89,7 @@ private IConfig GetMinimalConfig(IConfig? config)
             return config!;
         }
 
-        private static void ValidateSummary(Summary summary)
+        private static void ValidateSummary(Summary summary, WarningCollectingLogger warnings, bool allowWarnings)
         {
             Assert.False(summary.HasCriticalValidationErrors, "The \"Summary\" should have NOT \"HasCriticalValidationErrors\"");
 
@@ -103,6 +107,9 @@ private static void ValidateSummary(Summary summary)
 
             Assert.True(summary.Reports.All(r => r.ExecuteResults.All(er => er.IsSuccess)),
                 "All reports should have succeeded to execute");
+
+            Assert.True(allowWarnings || warnings.Warnings.Count == 0,
+                "The run should not log any warnings:" + Environment.NewLine + string.Join(Environment.NewLine, warnings.Warnings));
         }
 
         protected IConfig CreateSimpleConfig(OutputLogger? logger = null, Job? job = null)

tests/BenchmarkDotNet.IntegrationTests/Shared/WarningCollectingLogger.cs

@@ -0,0 +1,58 @@
+using BenchmarkDotNet.Loggers;
+
+namespace BenchmarkDotNet.IntegrationTests;
+
+/// <summary>
+/// Collects the warnings a benchmark run logs, so that a test can fail on them.
+/// </summary>
+/// <remarks>
+/// Analyser conclusions are skipped: they describe the measurements and the machine (a tiny Dry run, an antivirus,
+/// a virtual machine) rather than a problem with the run itself, and the summary keeps them anyway.
+/// </remarks>
+internal sealed class WarningCollectingLogger : ILogger
+{
+    // The header ConclusionHelper prints before its warning conclusions, which last until something else is written.
+    private const string ConclusionsHeader = "// * Warnings *";
+
+    private readonly List<string> warnings = [];
+    private string currentLine = "";
+    private bool inConclusions;
+
+    public string Id => nameof(WarningCollectingLogger);
+    public int Priority => 0;
+
+    public IReadOnlyList<string> Warnings => warnings;
+
+    public void Write(LogKind logKind, string text)
+    {
+        Track(logKind, text);
+        if (logKind == LogKind.Warning)
+            currentLine += text;
+    }
+
+    public void WriteLine() => EndLine();
+
+    public void WriteLine(LogKind logKind, string text)
+    {
+        Write(logKind, text);
+        EndLine();
+    }
+
+    public void Flush() { }
+
+    private void Track(LogKind logKind, string text)
+    {
+        if (logKind != LogKind.Warning)
+            inConclusions = false;
+        else if (text == ConclusionsHeader)
+            inConclusions = true;
+    }
+
+    private void EndLine()
+    {
+        // ConclusionHelper writes an empty warning line before its header.
+        if (!inConclusions && currentLine.Length > 0)
+            warnings.Add(currentLine);
+        currentLine = "";
+    }
+}