Fixed CI issue related to needs upgrade checks

Author: jschick04

src/EventLogExpert.EventDbTool/CreateDatabaseCommand.cs

@@ -6,6 +6,7 @@
 using EventLogExpert.Eventing.Providers;
 using Microsoft.Extensions.DependencyInjection;
 using System.CommandLine;
+using System.Text.RegularExpressions;
 
 namespace EventLogExpert.EventDbTool;
 
@@ -83,50 +84,94 @@ private void CreateDatabase(string path, string? source, string? filter, string?
             return;
         }
 
-        if (source is not null && !ProviderSource.TryValidate(source, Logger)) { return; }
+        if (!RegexHelper.TryCreate(filter, Logger, out var regex)) { return; }
 
-        HashSet<string> skipProviderNames = new(StringComparer.OrdinalIgnoreCase);
+        if (source is not null && !ProviderSource.TryValidate(source, Logger)) { return; }
 
-        if (!string.IsNullOrWhiteSpace(skipProvidersInFile))
+        try
         {
-            if (!ProviderSource.TryValidate(skipProvidersInFile, Logger)) { return; }
+            HashSet<string> skipProviderNames = new(StringComparer.OrdinalIgnoreCase);
 
-            foreach (var name in ProviderSource.LoadProviderNames(skipProvidersInFile, Logger))
+            if (!string.IsNullOrWhiteSpace(skipProvidersInFile))
             {
-                skipProviderNames.Add(name);
+                if (!ProviderSource.TryValidate(skipProvidersInFile, Logger)) { return; }
+
+                foreach (var name in ProviderSource.LoadProviderNames(skipProvidersInFile, Logger))
+                {
+                    skipProviderNames.Add(name);
+                }
+
+                Logger.Info($"Found {skipProviderNames.Count} providers in {skipProvidersInFile}. These will not be included in the new database.");
             }
 
-            Logger.Info($"Found {skipProviderNames.Count} providers in {skipProvidersInFile}. These will not be included in the new database.");
-        }
+            // Load provider names first (cheap string-only query) for the empty check and header
+            // formatting. This avoids materializing all ProviderDetails (with large compressed
+            // payloads) just to compute the column widths.
+            IReadOnlyList<string> providerNames = source is null
+                ? GetLocalProviderNames(regex)
+                    .Where(n => !skipProviderNames.Contains(n)).ToList()
+                : ProviderSource.LoadProviderNames(source, Logger, regex)
+                    .Where(n => !skipProviderNames.Contains(n)).ToList();
 
-        IEnumerable<ProviderDetails> providersToAdd = source is null
-            ? LoadLocalProviders(filter, skipProviderNames)
-            : ProviderSource.LoadProviders(source, Logger, filter, skipProviderNames);
+            if (providerNames.Count == 0)
+            {
+                Logger.Warn($"No providers to add to the new database.");
+                return;
+            }
 
-        var providersNotSkipped = providersToAdd.ToList();
+            LogProviderDetailHeader(providerNames);
 
-        if (providersNotSkipped.Count == 0)
-        {
-            Logger.Warn($"No providers to add to the new database.");
-            return;
-        }
+            // Defer creating the DbContext (and therefore the .db file on disk) until we have
+            // at least one provider to persist. This prevents leaving an empty database behind
+            // when no provider details could be resolved (e.g., .evtx without LocaleMetaData).
+            EventProviderDbContext? dbContext = null;
+
+            try
+            {
+                // Stream details directly into the DbContext. Batch saves prevent the change tracker
+                // from accumulating all entities in memory at once.
+                const int batchSize = 100;
+                var count = 0;
 
-        using var dbContext = new EventProviderDbContext(path, false, Logger);
+                IEnumerable<ProviderDetails> providersToAdd = source is null
+                    ? LoadLocalProviders(regex, skipProviderNames)
+                    : ProviderSource.LoadProviders(source, Logger, regex, skipProviderNames);
 
-        LogProviderDetailHeader(providersNotSkipped.Select(p => p.ProviderName));
+                foreach (var details in providersToAdd)
+                {
+                    dbContext ??= new EventProviderDbContext(path, false, Logger);
+                    dbContext.ProviderDetails.Add(details);
+                    LogProviderDetails(details);
+                    count++;
 
-        foreach (var details in providersNotSkipped)
-        {
-            dbContext.ProviderDetails.Add(details);
-            LogProviderDetails(details);
-        }
+                    if (count % batchSize != 0) { continue; }
+                    dbContext.SaveChanges();
+                    dbContext.ChangeTracker.Clear();
+                }
 
-        Logger.Info($"");
-        Logger.Info($"Saving database. Please wait...");
+                if (dbContext is null)
+                {
+                    Logger.Warn($"No provider details could be resolved from the source. Database was not created.");
 
-        dbContext.SaveChanges();
+                    return;
+                }
 
-        Logger.Info($"Done!");
+                Logger.Info($"");
+                Logger.Info($"Saving database. Please wait...");
+
+                dbContext.SaveChanges();
+
+                Logger.Info($"Done!");
+            }
+            finally
+            {
+                dbContext?.Dispose();
+            }
+        }
+        catch (RegexMatchTimeoutException)
+        {
+            Logger.Error($"The --filter regex timed out. The pattern may cause catastrophic backtracking.");
+        }
     }
 
 }

src/EventLogExpert.EventDbTool/DbToolCommand.cs

@@ -14,21 +14,29 @@ public class DbToolCommand(ITraceLogger logger)
 
     protected ITraceLogger Logger => logger;
 
-    protected static List<string> GetLocalProviderNames(string? filter)
+    protected static List<string> GetLocalProviderNames(string? filter, ITraceLogger logger) =>
+        !RegexHelper.TryCreate(filter, logger, out var regex) ? [] : GetLocalProviderNames(regex);
+
+    protected static List<string> GetLocalProviderNames(Regex? regex)
     {
         var providers = new List<string>(EventLogSession.GlobalSession.GetProviderNames().Distinct().OrderBy(name => name));
 
-        if (string.IsNullOrEmpty(filter)) { return providers; }
+        return regex is null ? providers : providers.Where(p => regex.IsMatch(p)).ToList();
+    }
 
-        var regex = new Regex(filter, RegexOptions.IgnoreCase);
-        providers = providers.Where(p => regex.IsMatch(p)).ToList();
+    protected IEnumerable<ProviderDetails> LoadLocalProviders(string? filter, IReadOnlySet<string>? skipProviderNames = null)
+    {
+        if (!RegexHelper.TryCreate(filter, Logger, out var regex)) { yield break; }
 
-        return providers;
+        foreach (var details in LoadLocalProviders(regex, skipProviderNames))
+        {
+            yield return details;
+        }
     }
 
-    protected IEnumerable<ProviderDetails> LoadLocalProviders(string? filter, IReadOnlySet<string>? skipProviderNames = null)
+    protected IEnumerable<ProviderDetails> LoadLocalProviders(Regex? regex, IReadOnlySet<string>? skipProviderNames = null)
     {
-        foreach (var providerName in GetLocalProviderNames(filter))
+        foreach (var providerName in GetLocalProviderNames(regex))
         {
             // Skip BEFORE resolving so we don't pay the cost of loading metadata for providers we
             // are about to discard (e.g. when --skip-providers-in-file lists most local providers).

src/EventLogExpert.EventDbTool/DiffDatabaseCommand.cs

@@ -81,14 +81,13 @@ private void DiffDatabase(string firstSource, string secondSource, string newDb)
 
         using var newDbContext = new EventProviderDbContext(newDb, false, Logger);
 
-        foreach (var details in ProviderSource.LoadProviders(secondSource, Logger))
-        {
-            if (firstProviderNames.Contains(details.ProviderName))
-            {
-                Logger.Info($"Skipping {details.ProviderName} because it is present in both sources.");
-                continue;
-            }
+        // Pass firstProviderNames as the skip set so providers present in the first source are
+        // never resolved from the second source's metadata path. This is especially important when
+        // the second source is .evtx+MTA, where each provider triggers an expensive load.
+        Logger.Info($"Skipping up to {firstProviderNames.Count} provider name(s) from the second source that also appear in the first source.");
 
+        foreach (var details in ProviderSource.LoadProviders(secondSource, Logger, filter: null, skipProviderNames: firstProviderNames))
+        {
             Logger.Info($"Copying {details.ProviderName} because it is present in second source but not first.");
 
             newDbContext.ProviderDetails.Add(new ProviderDetails

src/EventLogExpert.EventDbTool/MergeDatabaseCommand.cs

@@ -104,15 +104,24 @@ private void MergeDatabase(string source, string targetFile, bool overwriteProvi
             {
                 Logger.Info($"Removing these providers from the target database...");
 
-                // Single round-trip to load just the rows we need to remove. Since these names came
-                // from the same DB, exact (binary) matching here is correct.
-                var toRemove = targetContext.ProviderDetails
-                    .Where(p => targetMatchingNames.Contains(p.ProviderName))
-                    .ToList();
-
-                targetContext.RemoveRange(toRemove);
-                targetContext.SaveChanges();
-                Logger.Info($"Removal of {toRemove.Count} provider row(s) completed.");
+                // Chunk the IN-clause to stay below SQLite's parameter limit (default 999). Without
+                // chunking, an --overwrite of a large overlap could throw at runtime.
+                // ExecuteDelete() issues a SQL DELETE directly, avoiding change-tracker overhead.
+                var removed = 0;
+
+                for (var offset = 0; offset < targetMatchingNames.Count; offset += ProviderSource.MaxInClauseParameters)
+                {
+                    var chunk = targetMatchingNames
+                        .Skip(offset)
+                        .Take(ProviderSource.MaxInClauseParameters)
+                        .ToList();
+
+                    removed += targetContext.ProviderDetails
+                        .Where(p => chunk.Contains(p.ProviderName))
+                        .ExecuteDelete();
+                }
+
+                Logger.Info($"Removal of {removed} provider row(s) completed.");
             }
             else
             {

src/EventLogExpert.EventDbTool/MtaProviderSource.cs

@@ -22,7 +22,13 @@ internal static class MtaProviderSource
     public static IReadOnlyList<string> DiscoverProviderNames(
         string evtxPath,
         ITraceLogger logger,
-        string? filter = null)
+        string? filter = null) =>
+        !RegexHelper.TryCreate(filter, logger, out var regex) ? [] : DiscoverProviderNamesCore(evtxPath, logger, regex);
+
+    private static IReadOnlyList<string> DiscoverProviderNamesCore(
+        string evtxPath,
+        ITraceLogger logger,
+        Regex? regex)
     {
         if (!File.Exists(evtxPath))
         {
@@ -42,10 +48,11 @@ public static IReadOnlyList<string> DiscoverProviderNames(
                 return [];
             }
 
+            // TryGetEvents returns false both for normal end-of-results (ERROR_NO_MORE_ITEMS) and
+            // for read errors (corruption, access denied, etc.). Check LastErrorCode to surface
+            // non-terminal failures so users can distinguish "0 events" from "could not read the log".
             while (reader.TryGetEvents(out var batch))
             {
-                if (batch.Length == 0) { break; }
-
                 foreach (var record in batch)
                 {
                     if (!string.IsNullOrEmpty(record.ProviderName))
@@ -54,21 +61,21 @@ public static IReadOnlyList<string> DiscoverProviderNames(
                     }
                 }
             }
+
+            if (reader.LastErrorCode is not null)
+            {
+                logger.Warn(
+                    $"Reading {evtxPath} may be incomplete. " +
+                    $"EvtNext failed with Win32 error code {reader.LastErrorCode}.");
+            }
         }
         catch (Exception ex)
         {
             logger.Error($"Failed to read events from {evtxPath}: {ex.Message}");
             return [];
         }
 
-        if (string.IsNullOrEmpty(filter))
-        {
-            return providerNames.ToList();
-        }
-
-        var regex = new Regex(filter, RegexOptions.IgnoreCase);
-
-        return providerNames.Where(n => regex.IsMatch(n)).ToList();
+        return regex is null ? providerNames.ToList() : providerNames.Where(n => regex.IsMatch(n)).ToList();
     }
 
     /// <summary>
@@ -122,7 +129,8 @@ public static IEnumerable<ProviderDetails> LoadProviders(
         string evtxPath,
         ITraceLogger logger,
         string? filter = null) =>
-        LoadProvidersCore(evtxPath, logger, string.IsNullOrEmpty(filter) ? null : new Regex(filter, RegexOptions.IgnoreCase), null, null);
+        !RegexHelper.TryCreate(filter, logger, out var regex) ? [] :
+            LoadProvidersCore(evtxPath, logger, regex, null, null);
 
     /// <summary>
     ///     Internal overload used by <see cref="ProviderSource" /> so name-based filtering and the de-dup
@@ -151,7 +159,7 @@ private static IEnumerable<ProviderDetails> LoadProvidersCore(
         IReadOnlySet<string>? skipProviderNames,
         HashSet<string>? seen)
     {
-        var providerNames = DiscoverProviderNames(evtxPath, logger);
+        var providerNames = DiscoverProviderNamesCore(evtxPath, logger, regex: null);
 
         if (providerNames.Count == 0) { yield break; }
 
@@ -165,7 +173,7 @@ private static IEnumerable<ProviderDetails> LoadProvidersCore(
         {
             if (regex is not null && !regex.IsMatch(providerName)) { continue; }
             if (skipProviderNames is not null && skipProviderNames.Contains(providerName)) { continue; }
-            if (seen is not null && !seen.Add(providerName)) { continue; }
+            if (seen is not null && seen.Contains(providerName)) { continue; }
 
             var details = new EventMessageProvider(providerName, null, mtaFiles, logger).LoadProviderDetails();
 
@@ -175,6 +183,10 @@ private static IEnumerable<ProviderDetails> LoadProvidersCore(
                 continue;
             }
 
+            // Mark as seen only after confirming the provider has data, so that an empty/missing
+            // provider from one .evtx does not block loading it from a later source file.
+            seen?.Add(providerName);
+
             yield return details;
         }
     }