C#: Use feed management in the remaining restore flows.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -80,11 +80,6 @@ private string GetRestoreArgs(RestoreSettings restoreSettings)
                 args += $" /p:TargetFrameworkRootPath=\"{path}\" /p:NetCoreTargetingPackRoot=\"{path}\" /p:AllowMissingPrunePackageData=true";
             }
 
-            if (restoreSettings.PathToNugetConfig != null)
-            {
-                args += $" --configfile \"{restoreSettings.PathToNugetConfig}\"";
-            }
-
             if (restoreSettings.ForceReevaluation)
             {
                 args += " --force";

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs

@@ -24,22 +24,97 @@ internal sealed partial class FeedManager : IDisposable
         private readonly FileProvider fileProvider;
         private readonly DependabotProxy? dependabotProxy;
         private readonly DependencyDirectory emptyPackageDirectory;
+        private readonly ImmutableHashSet<string> privateRegistryFeeds;
 
-        public ImmutableHashSet<string> PrivateRegistryFeeds { get; }
+        /// <summary>
+        /// Gets whether there are private package registries configured for C#.
+        /// </summary>
         public bool HasPrivateRegistryFeeds { get; }
+
+        /// <summary>
+        /// Gets whether the reachability of the NuGet feeds should be checked before using them for restore.
+        /// </summary>
         public bool CheckNugetFeedResponsiveness { get; } = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
 
+        private readonly Lazy<ImmutableHashSet<string>> lazyExplicitFeeds;
+
+        /// <summary>
+        /// Gets the list of NuGet feeds that are explicitly configured
+        /// - NuGet configuration files.
+        /// - Private package registries that are configured for C#.
+        /// </summary>
+        public ImmutableHashSet<string> ExplicitFeeds => lazyExplicitFeeds.Value;
+
+        private readonly Lazy<ImmutableHashSet<string>> lazyAllFeeds;
+
+        /// <summary>
+        /// Gets the list of all NuGet feeds that are configured in the environment. That is
+        /// - Explicit feeds
+        /// - Inherited feeds from the machine and environment (if not explicitly disabled by a
+        /// root directory NuGet configuration).
+        /// </summary>
+        public ImmutableHashSet<string> AllFeeds => lazyAllFeeds.Value;
+
+        /// <summary>
+        /// Gets the list of inherited NuGet feeds that are configured in the environment.
+        /// </summary>
+        public ImmutableHashSet<string> InheritedFeeds => AllFeeds.Except(ExplicitFeeds).ToImmutableHashSet();
+
+        private readonly Lazy<(bool, ImmutableHashSet<string>)> lazyReachableExplicitFeeds;
+
+        /// <summary>
+        /// Gets whether there was a timeout when checking the reachability of the explicitly configured NuGet feeds.
+        /// </summary>
+        public bool ExplicitFeedTimeout => lazyReachableExplicitFeeds.Value.Item1;
+
+        /// <summary>
+        /// Gets the list of reachable NuGet feeds that are explicitly configured.
+        /// </summary>
+        public ImmutableHashSet<string> ReachableExplicitFeeds => lazyReachableExplicitFeeds.Value.Item2;
+
+        private readonly Lazy<ImmutableHashSet<string>> lazyReachableFeeds;
+        /// <summary>
+        /// Gets the list of reachable NuGet feeds that are configured in the environment.
+        /// </summary>
+        public ImmutableHashSet<string> ReachableFeeds => lazyReachableFeeds.Value;
+
+        private readonly Lazy<ImmutableHashSet<string>> lazyReachableFallbackFeeds;
+        /// <summary>
+        /// Gets the list of reachable NuGet feeds that are configured as fallback feeds.
+        /// </summary>
+        public ImmutableHashSet<string> ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value;
+
         public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotProxy, FileProvider fileProvider)
         {
             this.logger = logger;
             this.dotnet = dotnet;
             this.dependabotProxy = dependabotProxy;
             this.fileProvider = fileProvider;
-            PrivateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
-            HasPrivateRegistryFeeds = PrivateRegistryFeeds.Count > 0;
+            privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
+            HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0;
             emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);
+
+            lazyExplicitFeeds = new Lazy<ImmutableHashSet<string>>(GetExplicitFeeds);
+            lazyAllFeeds = new Lazy<ImmutableHashSet<string>>(GetAllFeeds);
+            lazyReachableExplicitFeeds = new Lazy<(bool, ImmutableHashSet<string>)>(() =>
+            {
+                var timeout = CheckSpecifiedFeeds(ExplicitFeeds, out var reachableFeeds);
+                return (timeout, reachableFeeds);
+            });
+            lazyReachableFeeds = new Lazy<ImmutableHashSet<string>>(() =>
+            {
+                // Inherited feeds should only be used, if they are indeed reachable (as they may be environment specific).
+                CheckSpecifiedFeeds(InheritedFeeds, out var reachableInheritedFeeds);
+                return ReachableExplicitFeeds.Union(reachableInheritedFeeds).ToImmutableHashSet();
+            });
+            lazyReachableFallbackFeeds = new Lazy<ImmutableHashSet<string>>(() =>
+            {
+                var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
+                return reachableFallbackFeeds.ToImmutableHashSet();
+            });
         }
 
+
         private string? GetDirectoryName(string path)
         {
             try
@@ -88,6 +163,13 @@ private IEnumerable<string> GetFeedsFromFolder(string folderPath) =>
         private IEnumerable<string> GetFeedsFromNugetConfig(string nugetConfigPath) =>
             GetFeeds(() => dotnet.GetNugetFeeds(nugetConfigPath));
 
+        /// <summary>
+        /// Constructs the NuGet sources argument for the restore command based on the given feeds.
+        /// If there are no feeds, a dummy source argument is added to override any default feeds that `restore` would use.
+        /// </summary>
+        /// <param name="feeds">The list of feeds to use for the restore command.</param>
+        /// <param name="sourceArgumentPrefix">The prefix to use for each source argument (e.g., "-s").</param>
+        /// <returns>The constructed NuGet sources argument for the restore command.</returns>
         public string FeedsToRestoreArgument(IEnumerable<string> feeds, string sourceArgumentPrefix)
         {
             // If there are no feeds, we want to override any default feeds that `restore` would use by passing a dummy source argument.
@@ -107,30 +189,58 @@ public string FeedsToRestoreArgument(IEnumerable<string> feeds, string sourceArg
             return feedArgs.ToString();
         }
 
+        private IEnumerable<string> FeedsToUseAux(HashSet<string> feedsToConsider)
+        {
+            if (HasPrivateRegistryFeeds)
+            {
+                feedsToConsider.UnionWith(privateRegistryFeeds);
+            }
+
+            var feedsToUse = CheckNugetFeedResponsiveness
+                ? feedsToConsider.Where(ReachableFeeds.Contains)
+                : feedsToConsider;
+
+            return feedsToUse;
+        }
+
         /// <summary>
         /// Constructs the list of NuGet sources to use for this restore.
         /// (1) Use the feeds we get from `dotnet nuget list source`
         /// (2) Use private registries, if they are configured
         /// </summary>
         /// <param name="path">Path to project/solution/packages.config</param>
-        /// <param name="reachableFeeds">The set of reachable NuGet feeds.</param>
         /// <returns>The list of NuGet feeds to use for this restore.</returns>
-        public IEnumerable<string> FeedsToUse(string path, HashSet<string> reachableFeeds)
+        public IEnumerable<string> FeedsToUse(string path)
         {
             // Find the path specific feeds.
             var folder = GetDirectoryName(path);
             var feedsToConsider = folder is not null ? GetFeedsFromFolder(folder).ToHashSet() : new HashSet<string>();
 
-            if (HasPrivateRegistryFeeds)
-            {
-                feedsToConsider.UnionWith(PrivateRegistryFeeds);
-            }
+            return FeedsToUseAux(feedsToConsider);
+        }
 
-            var feedsToUse = CheckNugetFeedResponsiveness
-                ? feedsToConsider.Where(reachableFeeds.Contains)
-                : feedsToConsider;
+        /// <summary>
+        /// Constructs the list of NuGet sources to use for this restore.
+        /// (1) Use the feeds we get from `dotnet nuget list source --configfile`
+        /// (2) Use private registries, if they are configured
+        /// </summary>
+        /// <param name="config">Path to the NuGet configuration file.</param>
+        /// <returns>The list of NuGet feeds to use for this restore.</returns>
+        public IEnumerable<string> FeedsToUseFromConfig(string config)
+        {
+            var feedsToConsider = GetFeedsFromNugetConfig(config).ToHashSet();
 
-            return feedsToUse;
+            return FeedsToUseAux(feedsToConsider);
+        }
+
+        /// <summary>
+        /// Constructs the NuGet sources argument for the `dotnet restore` command based on the given feeds.
+        /// </summary>
+        /// <param name="feeds">The list of NuGet feeds to use for the restore command.</param>
+        /// <returns>A string representing the NuGet sources argument for the `dotnet restore` command.</returns>
+        public string FeedsToDotnetRestoreArgument(IEnumerable<string> feeds)
+        {
+            return FeedsToRestoreArgument(feeds, "-s");
         }
 
         /// <summary>
@@ -139,19 +249,18 @@ public IEnumerable<string> FeedsToUse(string path, HashSet<string> reachableFeed
         /// (2) Use private registries, if they are configured
         /// </summary>
         /// <param name="path">Path to project/solution</param>
-        /// <param name="reachableFeeds">The set of reachable NuGet feeds.</param>
-        /// <returns>A string representing the NuGet sources argument for the restore command.</returns>
-        public string? MakeDotnetRestoreSourcesArgument(string path, HashSet<string> reachableFeeds)
+        /// <returns>A string representing the NuGet sources argument for the `dotnet restore` command.</returns>
+        public string? MakeDotnetRestoreSourcesArgument(string path)
         {
             // Do not construct a set of explicit NuGet sources to use for restore.
             if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds)
             {
                 return null;
             }
 
-            var feedsToUse = FeedsToUse(path, reachableFeeds);
+            var feedsToUse = FeedsToUse(path);
 
-            return FeedsToRestoreArgument(feedsToUse, "-s");
+            return FeedsToDotnetRestoreArgument(feedsToUse);
         }
 
         private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback)
@@ -271,7 +380,7 @@ private HashSet<string> GetExcludedFeeds()
         /// True if there is a timeout when trying to reach the feeds (excluding any feeds that are configured
         /// to be excluded from the check) or false otherwise.
         /// </returns>
-        public bool CheckSpecifiedFeeds(HashSet<string> feeds, out HashSet<string> reachableFeeds)
+        private bool CheckSpecifiedFeeds(ImmutableHashSet<string> feeds, out ImmutableHashSet<string> reachableFeeds)
         {
             // Exclude any feeds from the feed check that are configured by the corresponding environment variable.
             // These feeds are always assumed to be reachable.
@@ -287,14 +396,19 @@ public bool CheckSpecifiedFeeds(HashSet<string> feeds, out HashSet<string> reach
                 return true;
             }).ToHashSet();
 
-            reachableFeeds = GetReachableNuGetFeeds(feedsToCheck, isFallback: false, out var isTimeout).ToHashSet();
+            var reachable = GetReachableNuGetFeeds(feedsToCheck, isFallback: false, out var isTimeout);
 
             // Always consider feeds excluded for the reachability check as reachable.
-            reachableFeeds.UnionWith(feeds.Where(feed => excludedFeeds.Contains(feed)));
+            reachableFeeds = reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet();
 
             return isTimeout;
         }
 
+        /// <summary>
+        /// Return true if the default NuGet feed is reachable, false otherwise.
+        /// If the reachability check is disabled, this method will always return true.
+        /// </summary>
+        /// <returns>True if the default NuGet feed is reachable, false otherwise.</returns>
         public bool IsDefaultFeedReachable()
         {
             if (CheckNugetFeedResponsiveness)
@@ -342,7 +456,7 @@ private List<string> GetReachableNuGetFeeds(HashSet<string> feedsToCheck, bool i
             return reachableFeeds;
         }
 
-        public List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNugetConfigs)
+        private List<string> GetReachableFallbackNugetFeeds()
         {
             var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
             if (fallbackFeeds.Count == 0)
@@ -353,19 +467,19 @@ public List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNug
                 var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
                 logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");
 
-                if (shouldAddNugetConfigFeeds && feedsFromNugetConfigs?.Count > 0)
+                if (shouldAddNugetConfigFeeds && ExplicitFeeds.Count > 0)
                 {
-                    // There are some feeds in `feedsFromNugetConfigs` that have already been checked for reachability, we could skip those.
-                    // But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again is safer.
-                    fallbackFeeds.UnionWith(feedsFromNugetConfigs);
-                    logger.LogInfo($"Using NuGet feeds from nuget.config files as fallback feeds: {string.Join(", ", feedsFromNugetConfigs.OrderBy(f => f))}");
+                    // Feeds in `ExplicitFeeds` may already been checked for reachability.
+                    // But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again.
+                    fallbackFeeds.UnionWith(ExplicitFeeds);
+                    logger.LogInfo($"Using NuGet feeds from nuget.config files as fallback feeds: {string.Join(", ", ExplicitFeeds.OrderBy(f => f))}");
                 }
             }
 
             return GetReachableNuGetFeeds(fallbackFeeds, isFallback: true, out var _);
         }
 
-        public (HashSet<string> explicitFeeds, HashSet<string> allFeeds) GetAllFeeds()
+        private ImmutableHashSet<string> GetExplicitFeeds()
         {
             var nugetConfigs = fileProvider.NugetConfigs;
 
@@ -387,14 +501,21 @@ public List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNug
             // in addition to the ones that are configured in `nuget.config` files.
             if (HasPrivateRegistryFeeds)
             {
-                logger.LogInfo($"Found {PrivateRegistryFeeds.Count} private registry feeds configured for C#: {string.Join(", ", PrivateRegistryFeeds.OrderBy(f => f))}");
-                explicitFeeds.UnionWith(PrivateRegistryFeeds);
+                logger.LogInfo($"Found {privateRegistryFeeds.Count} private registry feeds configured for C#: {string.Join(", ", privateRegistryFeeds.OrderBy(f => f))}");
+                explicitFeeds.UnionWith(privateRegistryFeeds);
             }
 
+            return explicitFeeds.ToImmutableHashSet();
+        }
+
+        private ImmutableHashSet<string> GetAllFeeds()
+        {
+            var nugetConfigs = fileProvider.NugetConfigs;
+
             HashSet<string> allFeeds = [];
 
             // Add all explicitFeeds to the set of all feeds.
-            allFeeds.UnionWith(explicitFeeds);
+            allFeeds.UnionWith(ExplicitFeeds);
 
             // Obtain the list of feeds from the root source directory.
             // If a NuGet file is present it will be respected, otherwise we will just get the machine/environment specific feeds.
@@ -414,7 +535,7 @@ public List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNug
 
             logger.LogInfo($"Found {allFeeds.Count} NuGet feeds (with inherited ones) in nuget.config files: {string.Join(", ", allFeeds.OrderBy(f => f))}");
 
-            return (explicitFeeds, allFeeds);
+            return allFeeds.ToImmutableHashSet();
         }
 
         [GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -17,7 +17,7 @@ public interface IDotNet
         IList<string> GetNugetFeedsFromFolder(string folderPath);
     }
 
-    public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? NugetSources = null, string? PathToNugetConfig = null, bool ForceReevaluation = false, bool TargetWindows = false);
+    public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? NugetSources = null, bool ForceReevaluation = false, bool TargetWindows = false);
 
     public partial record class RestoreResult(bool Success, IList<string> Output)
     {