Fixed CI issue related to needs upgrade checks

Author: jschick04

src/EventLogExpert.EventDbTool/DbToolCommand.cs

@@ -4,7 +4,6 @@
 using EventLogExpert.Eventing.Helpers;
 using EventLogExpert.Eventing.Providers;
 using EventLogExpert.Eventing.Readers;
-using System.Text.RegularExpressions;
 
 namespace EventLogExpert.EventDbTool;
 
@@ -14,21 +13,21 @@ public class DbToolCommand(ITraceLogger logger)
 
     protected ITraceLogger Logger => logger;
 
-    protected static List<string> GetLocalProviderNames(string? filter)
+    protected static List<string> GetLocalProviderNames(string? filter, ITraceLogger logger)
     {
         var providers = new List<string>(EventLogSession.GlobalSession.GetProviderNames().Distinct().OrderBy(name => name));
 
-        if (string.IsNullOrEmpty(filter)) { return providers; }
-
-        var regex = new Regex(filter, RegexOptions.IgnoreCase);
-        providers = providers.Where(p => regex.IsMatch(p)).ToList();
+        if (!RegexHelper.TryCreate(filter, logger, out var regex))
+        {
+            return [];
+        }
 
-        return providers;
+        return regex is null ? providers : providers.Where(p => regex.IsMatch(p)).ToList();
     }
 
     protected IEnumerable<ProviderDetails> LoadLocalProviders(string? filter, IReadOnlySet<string>? skipProviderNames = null)
     {
-        foreach (var providerName in GetLocalProviderNames(filter))
+        foreach (var providerName in GetLocalProviderNames(filter, Logger))
         {
             // Skip BEFORE resolving so we don't pay the cost of loading metadata for providers we
             // are about to discard (e.g. when --skip-providers-in-file lists most local providers).

src/EventLogExpert.EventDbTool/DiffDatabaseCommand.cs

@@ -81,14 +81,13 @@ private void DiffDatabase(string firstSource, string secondSource, string newDb)
 
         using var newDbContext = new EventProviderDbContext(newDb, false, Logger);
 
-        foreach (var details in ProviderSource.LoadProviders(secondSource, Logger))
-        {
-            if (firstProviderNames.Contains(details.ProviderName))
-            {
-                Logger.Info($"Skipping {details.ProviderName} because it is present in both sources.");
-                continue;
-            }
+        // Pass firstProviderNames as the skip set so providers present in the first source are
+        // never resolved from the second source's metadata path. This is especially important when
+        // the second source is .evtx+MTA, where each provider triggers an expensive load.
+        Logger.Info($"Skipping up to {firstProviderNames.Count} provider name(s) from the second source that also appear in the first source.");
 
+        foreach (var details in ProviderSource.LoadProviders(secondSource, Logger, filter: null, skipProviderNames: firstProviderNames))
+        {
             Logger.Info($"Copying {details.ProviderName} because it is present in second source but not first.");
 
             newDbContext.ProviderDetails.Add(new ProviderDetails

src/EventLogExpert.EventDbTool/MergeDatabaseCommand.cs

@@ -104,15 +104,28 @@ private void MergeDatabase(string source, string targetFile, bool overwriteProvi
             {
                 Logger.Info($"Removing these providers from the target database...");
 
-                // Single round-trip to load just the rows we need to remove. Since these names came
-                // from the same DB, exact (binary) matching here is correct.
-                var toRemove = targetContext.ProviderDetails
-                    .Where(p => targetMatchingNames.Contains(p.ProviderName))
-                    .ToList();
+                // Chunk the IN-clause to stay below SQLite's parameter limit (default 999). Without
+                // chunking, an --overwrite of a large overlap could throw at runtime.
+                const int maxInClauseParameters = 500;
+                var removed = 0;
+
+                for (var offset = 0; offset < targetMatchingNames.Count; offset += maxInClauseParameters)
+                {
+                    var chunk = targetMatchingNames
+                        .Skip(offset)
+                        .Take(maxInClauseParameters)
+                        .ToList();
+
+                    var toRemove = targetContext.ProviderDetails
+                        .Where(p => chunk.Contains(p.ProviderName))
+                        .ToList();
+
+                    targetContext.RemoveRange(toRemove);
+                    removed += toRemove.Count;
+                }
 
-                targetContext.RemoveRange(toRemove);
                 targetContext.SaveChanges();
-                Logger.Info($"Removal of {toRemove.Count} provider row(s) completed.");
+                Logger.Info($"Removal of {removed} provider row(s) completed.");
             }
             else
             {

src/EventLogExpert.EventDbTool/MtaProviderSource.cs

@@ -22,7 +22,13 @@ internal static class MtaProviderSource
     public static IReadOnlyList<string> DiscoverProviderNames(
         string evtxPath,
         ITraceLogger logger,
-        string? filter = null)
+        string? filter = null) =>
+        !RegexHelper.TryCreate(filter, logger, out var regex) ? [] : DiscoverProviderNamesCore(evtxPath, logger, regex);
+
+    private static IReadOnlyList<string> DiscoverProviderNamesCore(
+        string evtxPath,
+        ITraceLogger logger,
+        Regex? regex)
     {
         if (!File.Exists(evtxPath))
         {
@@ -42,10 +48,10 @@ public static IReadOnlyList<string> DiscoverProviderNames(
                 return [];
             }
 
+            // EventLogReader.TryGetEvents returns false (not an empty batch) when there are no more
+            // events, so a separate batch.Length == 0 check is unnecessary.
             while (reader.TryGetEvents(out var batch))
             {
-                if (batch.Length == 0) { break; }
-
                 foreach (var record in batch)
                 {
                     if (!string.IsNullOrEmpty(record.ProviderName))
@@ -61,14 +67,7 @@ public static IReadOnlyList<string> DiscoverProviderNames(
             return [];
         }
 
-        if (string.IsNullOrEmpty(filter))
-        {
-            return providerNames.ToList();
-        }
-
-        var regex = new Regex(filter, RegexOptions.IgnoreCase);
-
-        return providerNames.Where(n => regex.IsMatch(n)).ToList();
+        return regex is null ? providerNames.ToList() : providerNames.Where(n => regex.IsMatch(n)).ToList();
     }
 
     /// <summary>
@@ -122,7 +121,8 @@ public static IEnumerable<ProviderDetails> LoadProviders(
         string evtxPath,
         ITraceLogger logger,
         string? filter = null) =>
-        LoadProvidersCore(evtxPath, logger, string.IsNullOrEmpty(filter) ? null : new Regex(filter, RegexOptions.IgnoreCase), null, null);
+        !RegexHelper.TryCreate(filter, logger, out var regex) ? [] :
+            LoadProvidersCore(evtxPath, logger, regex, null, null);
 
     /// <summary>
     ///     Internal overload used by <see cref="ProviderSource" /> so name-based filtering and the de-dup
@@ -151,7 +151,7 @@ private static IEnumerable<ProviderDetails> LoadProvidersCore(
         IReadOnlySet<string>? skipProviderNames,
         HashSet<string>? seen)
     {
-        var providerNames = DiscoverProviderNames(evtxPath, logger);
+        var providerNames = DiscoverProviderNamesCore(evtxPath, logger, regex: null);
 
         if (providerNames.Count == 0) { yield break; }
 

src/EventLogExpert.EventDbTool/ProviderSource.cs

@@ -19,12 +19,24 @@ internal static class ProviderSource
     private const string DbExtension = ".db";
     private const string EvtxExtension = ".evtx";
 
+    /// <summary>
+    ///     Conservative cap on the number of parameters in a single <c>Where(... Contains)</c> SQL IN
+    ///     clause. SQLite's default limit is 999 parameters; we stay well under that so the same code
+    ///     works on older SQLite builds too. Larger requests are split into multiple round-trips.
+    /// </summary>
+    private const int MaxInClauseParameters = 500;
+
     /// <summary>
     ///     Returns the distinct provider names available from <paramref name="path" />, applying an optional
     ///     case-insensitive regex <paramref name="filter" />. Does not load full provider details.
     /// </summary>
     public static IReadOnlyList<string> LoadProviderNames(string path, ITraceLogger logger, string? filter = null)
     {
+        if (!RegexHelper.TryCreate(filter, logger, out var regex))
+        {
+            return [];
+        }
+
         var names = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
 
         foreach (var file in EnumerateSourceFiles(path))
@@ -35,7 +47,9 @@ public static IReadOnlyList<string> LoadProviderNames(string path, ITraceLogger
             }
         }
 
-        return ApplyFilter(names, filter).ToList();
+        if (regex is null) { return names.ToList(); }
+
+        return names.Where(n => regex.IsMatch(n)).ToList();
     }
 
     /// <summary>
@@ -49,19 +63,9 @@ public static IEnumerable<ProviderDetails> LoadProviders(
         string path,
         ITraceLogger logger,
         string? filter = null,
-        IReadOnlySet<string>? skipProviderNames = null)
-    {
-        Regex? regex = string.IsNullOrEmpty(filter) ? null : new Regex(filter, RegexOptions.IgnoreCase);
-        var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
-
-        foreach (var file in EnumerateSourceFiles(path))
-        {
-            foreach (var details in LoadDetailsFromFile(file, logger, regex, skipProviderNames, seen))
-            {
-                yield return details;
-            }
-        }
-    }
+        IReadOnlySet<string>? skipProviderNames = null) =>
+        !RegexHelper.TryCreate(filter, logger, out var regex) ? [] :
+            LoadProvidersIterator(path, logger, regex, skipProviderNames);
 
     /// <summary>Validates that <paramref name="path" /> exists and has a recognized form.</summary>
     public static bool TryValidate(string path, ITraceLogger logger)
@@ -98,15 +102,6 @@ internal static bool ShouldInclude(
         return seen.Add(providerName);
     }
 
-    private static IEnumerable<string> ApplyFilter(IEnumerable<string> names, string? filter)
-    {
-        if (string.IsNullOrEmpty(filter)) { return names; }
-
-        var regex = new Regex(filter, RegexOptions.IgnoreCase);
-
-        return names.Where(n => regex.IsMatch(n));
-    }
-
     /// <summary>
     ///     Expands <paramref name="path" /> into the ordered list of source files: a single .db or .evtx
     ///     when given a file; or all *.db files (sorted) followed by all *.evtx files (sorted) when given a
@@ -155,9 +150,21 @@ private static IEnumerable<ProviderDetails> LoadDetailsFromFile(
 
             if (namesToLoad.Count == 0) { return []; }
 
-            return ctx.ProviderDetails
-                .Where(p => namesToLoad.Contains(p.ProviderName))
-                .ToList();
+            // Chunk the IN-clause to stay below SQLite's parameter limit (default 999). This keeps
+            // the optimization in finding #4 working for source DBs with thousands of providers.
+            var loaded = new List<ProviderDetails>(namesToLoad.Count);
+
+            for (var offset = 0; offset < namesToLoad.Count; offset += MaxInClauseParameters)
+            {
+                var chunk = namesToLoad
+                    .Skip(offset)
+                    .Take(MaxInClauseParameters)
+                    .ToList();
+
+                loaded.AddRange(ctx.ProviderDetails.Where(p => chunk.Contains(p.ProviderName)));
+            }
+
+            return loaded;
         }
 
         if (string.Equals(ext, EvtxExtension, StringComparison.OrdinalIgnoreCase))
@@ -190,4 +197,21 @@ private static IEnumerable<string> LoadNamesFromFile(string file, ITraceLogger l
 
         return [];
     }
+
+    private static IEnumerable<ProviderDetails> LoadProvidersIterator(
+        string path,
+        ITraceLogger logger,
+        Regex? regex,
+        IReadOnlySet<string>? skipProviderNames)
+    {
+        var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+        foreach (var file in EnumerateSourceFiles(path))
+        {
+            foreach (var details in LoadDetailsFromFile(file, logger, regex, skipProviderNames, seen))
+            {
+                yield return details;
+            }
+        }
+    }
 }

src/EventLogExpert.EventDbTool/RegexHelper.cs

@@ -0,0 +1,46 @@
+// // Copyright (c) Microsoft Corporation.
+// // Licensed under the MIT License.
+
+using EventLogExpert.Eventing.Helpers;
+using System.Text.RegularExpressions;
+
+namespace EventLogExpert.EventDbTool;
+
+/// <summary>
+///     Centralized creation of <see cref="Regex" /> instances for user-supplied <c>--filter</c> patterns.
+///     Always sets a match timeout to bound worst-case execution against catastrophic backtracking, and
+///     converts <see cref="ArgumentException" /> from invalid patterns into a logged error rather than
+///     letting it terminate the process.
+/// </summary>
+internal static class RegexHelper
+{
+    /// <summary>Maximum time a single regex match is allowed to take before throwing.</summary>
+    private static readonly TimeSpan s_matchTimeout = TimeSpan.FromSeconds(1);
+
+    /// <summary>
+    ///     Attempts to compile <paramref name="pattern" /> into a case-insensitive <see cref="Regex" /> with
+    ///     a bounded match timeout. A null/empty pattern is treated as "no filter": <paramref name="regex" />
+    ///     is set to <see langword="null" /> and the method still returns <see langword="true" /> so callers
+    ///     can distinguish an absent filter from a malformed one.
+    /// </summary>
+    public static bool TryCreate(string? pattern, ITraceLogger logger, out Regex? regex)
+    {
+        if (string.IsNullOrEmpty(pattern))
+        {
+            regex = null;
+            return true;
+        }
+
+        try
+        {
+            regex = new Regex(pattern, RegexOptions.IgnoreCase, s_matchTimeout);
+            return true;
+        }
+        catch (ArgumentException ex)
+        {
+            logger.Error($"Invalid --filter regex '{pattern}': {ex.Message}");
+            regex = null;
+            return false;
+        }
+    }
+}