Fixed CI issue related to needs upgrade checks

Author: jschick04

src/EventLogExpert.EventDbTool/CreateDatabaseCommand.cs

@@ -6,6 +6,7 @@
 using EventLogExpert.Eventing.Providers;
 using Microsoft.Extensions.DependencyInjection;
 using System.CommandLine;
+using System.Text.RegularExpressions;
 
 namespace EventLogExpert.EventDbTool;
 
@@ -83,50 +84,59 @@ private void CreateDatabase(string path, string? source, string? filter, string?
             return;
         }
 
-        if (source is not null && !ProviderSource.TryValidate(source, Logger)) { return; }
+        if (!RegexHelper.TryCreate(filter, Logger, out _)) { return; }
 
-        HashSet<string> skipProviderNames = new(StringComparer.OrdinalIgnoreCase);
+        if (source is not null && !ProviderSource.TryValidate(source, Logger)) { return; }
 
-        if (!string.IsNullOrWhiteSpace(skipProvidersInFile))
+        try
         {
-            if (!ProviderSource.TryValidate(skipProvidersInFile, Logger)) { return; }
+            HashSet<string> skipProviderNames = new(StringComparer.OrdinalIgnoreCase);
 
-            foreach (var name in ProviderSource.LoadProviderNames(skipProvidersInFile, Logger))
+            if (!string.IsNullOrWhiteSpace(skipProvidersInFile))
             {
-                skipProviderNames.Add(name);
-            }
+                if (!ProviderSource.TryValidate(skipProvidersInFile, Logger)) { return; }
 
-            Logger.Info($"Found {skipProviderNames.Count} providers in {skipProvidersInFile}. These will not be included in the new database.");
-        }
+                foreach (var name in ProviderSource.LoadProviderNames(skipProvidersInFile, Logger))
+                {
+                    skipProviderNames.Add(name);
+                }
 
-        IEnumerable<ProviderDetails> providersToAdd = source is null
-            ? LoadLocalProviders(filter, skipProviderNames)
-            : ProviderSource.LoadProviders(source, Logger, filter, skipProviderNames);
+                Logger.Info($"Found {skipProviderNames.Count} providers in {skipProvidersInFile}. These will not be included in the new database.");
+            }
 
-        var providersNotSkipped = providersToAdd.ToList();
+            IEnumerable<ProviderDetails> providersToAdd = source is null
+                ? LoadLocalProviders(filter, skipProviderNames)
+                : ProviderSource.LoadProviders(source, Logger, filter, skipProviderNames);
 
-        if (providersNotSkipped.Count == 0)
-        {
-            Logger.Warn($"No providers to add to the new database.");
-            return;
-        }
+            var providersNotSkipped = providersToAdd.ToList();
+
+            if (providersNotSkipped.Count == 0)
+            {
+                Logger.Warn($"No providers to add to the new database.");
+                return;
+            }
 
-        using var dbContext = new EventProviderDbContext(path, false, Logger);
+            using var dbContext = new EventProviderDbContext(path, false, Logger);
 
-        LogProviderDetailHeader(providersNotSkipped.Select(p => p.ProviderName));
+            LogProviderDetailHeader(providersNotSkipped.Select(p => p.ProviderName));
 
-        foreach (var details in providersNotSkipped)
-        {
-            dbContext.ProviderDetails.Add(details);
-            LogProviderDetails(details);
-        }
+            foreach (var details in providersNotSkipped)
+            {
+                dbContext.ProviderDetails.Add(details);
+                LogProviderDetails(details);
+            }
 
-        Logger.Info($"");
-        Logger.Info($"Saving database. Please wait...");
+            Logger.Info($"");
+            Logger.Info($"Saving database. Please wait...");
 
-        dbContext.SaveChanges();
+            dbContext.SaveChanges();
 
-        Logger.Info($"Done!");
+            Logger.Info($"Done!");
+        }
+        catch (RegexMatchTimeoutException)
+        {
+            Logger.Error($"The --filter regex timed out. The pattern may cause catastrophic backtracking.");
+        }
     }
 
 }

src/EventLogExpert.EventDbTool/DbToolCommand.cs

@@ -4,7 +4,6 @@
 using EventLogExpert.Eventing.Helpers;
 using EventLogExpert.Eventing.Providers;
 using EventLogExpert.Eventing.Readers;
-using System.Text.RegularExpressions;
 
 namespace EventLogExpert.EventDbTool;
 
@@ -14,21 +13,21 @@ public class DbToolCommand(ITraceLogger logger)
 
     protected ITraceLogger Logger => logger;
 
-    protected static List<string> GetLocalProviderNames(string? filter)
+    protected static List<string> GetLocalProviderNames(string? filter, ITraceLogger logger)
     {
         var providers = new List<string>(EventLogSession.GlobalSession.GetProviderNames().Distinct().OrderBy(name => name));
 
-        if (string.IsNullOrEmpty(filter)) { return providers; }
-
-        var regex = new Regex(filter, RegexOptions.IgnoreCase);
-        providers = providers.Where(p => regex.IsMatch(p)).ToList();
+        if (!RegexHelper.TryCreate(filter, logger, out var regex))
+        {
+            return [];
+        }
 
-        return providers;
+        return regex is null ? providers : providers.Where(p => regex.IsMatch(p)).ToList();
     }
 
     protected IEnumerable<ProviderDetails> LoadLocalProviders(string? filter, IReadOnlySet<string>? skipProviderNames = null)
     {
-        foreach (var providerName in GetLocalProviderNames(filter))
+        foreach (var providerName in GetLocalProviderNames(filter, Logger))
         {
             // Skip BEFORE resolving so we don't pay the cost of loading metadata for providers we
             // are about to discard (e.g. when --skip-providers-in-file lists most local providers).

src/EventLogExpert.EventDbTool/DiffDatabaseCommand.cs

@@ -81,14 +81,13 @@ private void DiffDatabase(string firstSource, string secondSource, string newDb)
 
         using var newDbContext = new EventProviderDbContext(newDb, false, Logger);
 
-        foreach (var details in ProviderSource.LoadProviders(secondSource, Logger))
-        {
-            if (firstProviderNames.Contains(details.ProviderName))
-            {
-                Logger.Info($"Skipping {details.ProviderName} because it is present in both sources.");
-                continue;
-            }
+        // Pass firstProviderNames as the skip set so providers present in the first source are
+        // never resolved from the second source's metadata path. This is especially important when
+        // the second source is .evtx+MTA, where each provider triggers an expensive load.
+        Logger.Info($"Skipping up to {firstProviderNames.Count} provider name(s) from the second source that also appear in the first source.");
 
+        foreach (var details in ProviderSource.LoadProviders(secondSource, Logger, filter: null, skipProviderNames: firstProviderNames))
+        {
             Logger.Info($"Copying {details.ProviderName} because it is present in second source but not first.");
 
             newDbContext.ProviderDetails.Add(new ProviderDetails

src/EventLogExpert.EventDbTool/MergeDatabaseCommand.cs

@@ -104,15 +104,28 @@ private void MergeDatabase(string source, string targetFile, bool overwriteProvi
             {
                 Logger.Info($"Removing these providers from the target database...");
 
-                // Single round-trip to load just the rows we need to remove. Since these names came
-                // from the same DB, exact (binary) matching here is correct.
-                var toRemove = targetContext.ProviderDetails
-                    .Where(p => targetMatchingNames.Contains(p.ProviderName))
-                    .ToList();
+                // Chunk the IN-clause to stay below SQLite's parameter limit (default 999). Without
+                // chunking, an --overwrite of a large overlap could throw at runtime.
+                const int maxInClauseParameters = 500;
+                var removed = 0;
+
+                for (var offset = 0; offset < targetMatchingNames.Count; offset += maxInClauseParameters)
+                {
+                    var chunk = targetMatchingNames
+                        .Skip(offset)
+                        .Take(maxInClauseParameters)
+                        .ToList();
+
+                    var toRemove = targetContext.ProviderDetails
+                        .Where(p => chunk.Contains(p.ProviderName))
+                        .ToList();
+
+                    targetContext.RemoveRange(toRemove);
+                    removed += toRemove.Count;
+                }
 
-                targetContext.RemoveRange(toRemove);
                 targetContext.SaveChanges();
-                Logger.Info($"Removal of {toRemove.Count} provider row(s) completed.");
+                Logger.Info($"Removal of {removed} provider row(s) completed.");
             }
             else
             {

src/EventLogExpert.EventDbTool/MtaProviderSource.cs

@@ -22,7 +22,13 @@ internal static class MtaProviderSource
     public static IReadOnlyList<string> DiscoverProviderNames(
         string evtxPath,
         ITraceLogger logger,
-        string? filter = null)
+        string? filter = null) =>
+        !RegexHelper.TryCreate(filter, logger, out var regex) ? [] : DiscoverProviderNamesCore(evtxPath, logger, regex);
+
+    private static IReadOnlyList<string> DiscoverProviderNamesCore(
+        string evtxPath,
+        ITraceLogger logger,
+        Regex? regex)
     {
         if (!File.Exists(evtxPath))
         {
@@ -42,10 +48,11 @@ public static IReadOnlyList<string> DiscoverProviderNames(
                 return [];
             }
 
+            // TryGetEvents returns false both for normal end-of-results (ERROR_NO_MORE_ITEMS) and
+            // for read errors (corruption, access denied, etc.). Check LastErrorCode to surface
+            // non-terminal failures so users can distinguish "0 events" from "could not read the log".
             while (reader.TryGetEvents(out var batch))
             {
-                if (batch.Length == 0) { break; }
-
                 foreach (var record in batch)
                 {
                     if (!string.IsNullOrEmpty(record.ProviderName))
@@ -54,21 +61,21 @@ public static IReadOnlyList<string> DiscoverProviderNames(
                     }
                 }
             }
+
+            if (reader.LastErrorCode is not null)
+            {
+                logger.Warn(
+                    $"Reading {evtxPath} may be incomplete. " +
+                    $"EvtNext failed with Win32 error code {reader.LastErrorCode}.");
+            }
         }
         catch (Exception ex)
         {
             logger.Error($"Failed to read events from {evtxPath}: {ex.Message}");
             return [];
         }
 
-        if (string.IsNullOrEmpty(filter))
-        {
-            return providerNames.ToList();
-        }
-
-        var regex = new Regex(filter, RegexOptions.IgnoreCase);
-
-        return providerNames.Where(n => regex.IsMatch(n)).ToList();
+        return regex is null ? providerNames.ToList() : providerNames.Where(n => regex.IsMatch(n)).ToList();
     }
 
     /// <summary>
@@ -122,7 +129,8 @@ public static IEnumerable<ProviderDetails> LoadProviders(
         string evtxPath,
         ITraceLogger logger,
         string? filter = null) =>
-        LoadProvidersCore(evtxPath, logger, string.IsNullOrEmpty(filter) ? null : new Regex(filter, RegexOptions.IgnoreCase), null, null);
+        !RegexHelper.TryCreate(filter, logger, out var regex) ? [] :
+            LoadProvidersCore(evtxPath, logger, regex, null, null);
 
     /// <summary>
     ///     Internal overload used by <see cref="ProviderSource" /> so name-based filtering and the de-dup
@@ -151,7 +159,7 @@ private static IEnumerable<ProviderDetails> LoadProvidersCore(
         IReadOnlySet<string>? skipProviderNames,
         HashSet<string>? seen)
     {
-        var providerNames = DiscoverProviderNames(evtxPath, logger);
+        var providerNames = DiscoverProviderNamesCore(evtxPath, logger, regex: null);
 
         if (providerNames.Count == 0) { yield break; }
 

src/EventLogExpert.EventDbTool/ProviderSource.cs

@@ -19,12 +19,24 @@ internal static class ProviderSource
     private const string DbExtension = ".db";
     private const string EvtxExtension = ".evtx";
 
+    /// <summary>
+    ///     Conservative cap on the number of parameters in a single <c>Where(... Contains)</c> SQL IN
+    ///     clause. SQLite's default limit is 999 parameters; we stay well under that so the same code
+    ///     works on older SQLite builds too. Larger requests are split into multiple round-trips.
+    /// </summary>
+    private const int MaxInClauseParameters = 500;
+
     /// <summary>
     ///     Returns the distinct provider names available from <paramref name="path" />, applying an optional
     ///     case-insensitive regex <paramref name="filter" />. Does not load full provider details.
     /// </summary>
     public static IReadOnlyList<string> LoadProviderNames(string path, ITraceLogger logger, string? filter = null)
     {
+        if (!RegexHelper.TryCreate(filter, logger, out var regex))
+        {
+            return [];
+        }
+
         var names = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
 
         foreach (var file in EnumerateSourceFiles(path))
@@ -35,7 +47,9 @@ public static IReadOnlyList<string> LoadProviderNames(string path, ITraceLogger
             }
         }
 
-        return ApplyFilter(names, filter).ToList();
+        if (regex is null) { return names.ToList(); }
+
+        return names.Where(n => regex.IsMatch(n)).ToList();
     }
 
     /// <summary>
@@ -49,19 +63,9 @@ public static IEnumerable<ProviderDetails> LoadProviders(
         string path,
         ITraceLogger logger,
         string? filter = null,
-        IReadOnlySet<string>? skipProviderNames = null)
-    {
-        Regex? regex = string.IsNullOrEmpty(filter) ? null : new Regex(filter, RegexOptions.IgnoreCase);
-        var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
-
-        foreach (var file in EnumerateSourceFiles(path))
-        {
-            foreach (var details in LoadDetailsFromFile(file, logger, regex, skipProviderNames, seen))
-            {
-                yield return details;
-            }
-        }
-    }
+        IReadOnlySet<string>? skipProviderNames = null) =>
+        !RegexHelper.TryCreate(filter, logger, out var regex) ? [] :
+            LoadProvidersIterator(path, logger, regex, skipProviderNames);
 
     /// <summary>Validates that <paramref name="path" /> exists and has a recognized form.</summary>
     public static bool TryValidate(string path, ITraceLogger logger)
@@ -98,15 +102,6 @@ internal static bool ShouldInclude(
         return seen.Add(providerName);
     }
 
-    private static IEnumerable<string> ApplyFilter(IEnumerable<string> names, string? filter)
-    {
-        if (string.IsNullOrEmpty(filter)) { return names; }
-
-        var regex = new Regex(filter, RegexOptions.IgnoreCase);
-
-        return names.Where(n => regex.IsMatch(n));
-    }
-
     /// <summary>
     ///     Expands <paramref name="path" /> into the ordered list of source files: a single .db or .evtx
     ///     when given a file; or all *.db files (sorted) followed by all *.evtx files (sorted) when given a
@@ -155,14 +150,26 @@ private static IEnumerable<ProviderDetails> LoadDetailsFromFile(
 
             if (namesToLoad.Count == 0) { return []; }
 
-            return ctx.ProviderDetails
-                .Where(p => namesToLoad.Contains(p.ProviderName))
-                .ToList();
+            // Chunk the IN-clause to stay below SQLite's parameter limit (default 999). This keeps
+            // the optimization in finding #4 working for source DBs with thousands of providers.
+            var loaded = new List<ProviderDetails>(namesToLoad.Count);
+
+            for (var offset = 0; offset < namesToLoad.Count; offset += MaxInClauseParameters)
+            {
+                var chunk = namesToLoad
+                    .Skip(offset)
+                    .Take(MaxInClauseParameters)
+                    .ToList();
+
+                loaded.AddRange(ctx.ProviderDetails.Where(p => chunk.Contains(p.ProviderName)));
+            }
+
+            return loaded;
         }
 
         if (string.Equals(ext, EvtxExtension, StringComparison.OrdinalIgnoreCase))
         {
-            return MtaProviderSource.LoadProviders(file, logger, regex, skipProviderNames, seen).ToList();
+            return MtaProviderSource.LoadProviders(file, logger, regex, skipProviderNames, seen);
         }
 
         logger.Warn($"Skipping unsupported source file: {file}");
@@ -190,4 +197,21 @@ private static IEnumerable<string> LoadNamesFromFile(string file, ITraceLogger l
 
         return [];
     }
+
+    private static IEnumerable<ProviderDetails> LoadProvidersIterator(
+        string path,
+        ITraceLogger logger,
+        Regex? regex,
+        IReadOnlySet<string>? skipProviderNames)
+    {
+        var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+        foreach (var file in EnumerateSourceFiles(path))
+        {
+            foreach (var details in LoadDetailsFromFile(file, logger, regex, skipProviderNames, seen))
+            {
+                yield return details;
+            }
+        }
+    }
 }