Updated dbtool to support MTA files

src/EventLogExpert.EventDbTool/CreateDatabaseCommand.cs

@@ -1,11 +1,12 @@
-// // Copyright (c) Microsoft Corporation.
+// // Copyright (c) Microsoft Corporation.
 // // Licensed under the MIT License.
 
 using EventLogExpert.Eventing.EventProviderDatabase;
 using EventLogExpert.Eventing.Helpers;
 using EventLogExpert.Eventing.Providers;
 using Microsoft.Extensions.DependencyInjection;
 using System.CommandLine;
+using System.Text.RegularExpressions;
 
 namespace EventLogExpert.EventDbTool;
 
@@ -20,6 +21,14 @@ public static Command GetCommand()
             Description = "File to create. Must have a .db extension."
         };
 
+        Argument<string?> sourceArgument = new("source")
+        {
+            Description = "Optional provider source: a .db file, an exported .evtx file, or a folder containing " +
+                ".db and/or .evtx files (top-level only). When omitted, local providers on this machine are used. " +
+                "When supplied, ONLY the source is used (no fallback to local providers).",
+            Arity = ArgumentArity.ZeroOrOne
+        };
+
         Option<string> filterOption = new("--filter")
         {
             Description = "Only providers matching specified regex string will be added to the database."
@@ -28,7 +37,8 @@ public static Command GetCommand()
         Option<string> skipProvidersInFileOption = new("--skip-providers-in-file")
         {
             Description =
-                "Any providers found in the specified database file will not be included in the new database. " +
+                "Any providers found in the specified source (a .db file, an exported .evtx file, or a folder " +
+                "containing them, top-level only) will not be included in the new database. " +
                 "For example, when creating a database of event providers for Exchange Server, it may be useful " +
                 "to provide a database of all providers from a fresh OS install with no other products. That way, all the " +
                 "OS providers are skipped, and only providers added by Exchange or other installed products " +
@@ -41,6 +51,7 @@ public static Command GetCommand()
         };
 
         createDatabaseCommand.Arguments.Add(fileArgument);
+        createDatabaseCommand.Arguments.Add(sourceArgument);
         createDatabaseCommand.Options.Add(filterOption);
         createDatabaseCommand.Options.Add(skipProvidersInFileOption);
         createDatabaseCommand.Options.Add(verboseOption);
@@ -51,84 +62,116 @@ public static Command GetCommand()
             new CreateDatabaseCommand(sp.GetRequiredService<ITraceLogger>())
                 .CreateDatabase(
                     result.GetRequiredValue(fileArgument),
+                    result.GetValue(sourceArgument),
                     result.GetValue(filterOption),
                     result.GetValue(skipProvidersInFileOption));
         });
 
         return createDatabaseCommand;
     }
 
-    private void CreateDatabase(string path, string? filter, string? skipProvidersInFile)
+    private void CreateDatabase(string path, string? source, string? filter, string? skipProvidersInFile)
     {
         if (File.Exists(path))
         {
             Logger.Error($"Cannot create database because file already exists: {path}");
             return;
         }
 
-        if (Path.GetExtension(path) != ".db")
+        if (!string.Equals(Path.GetExtension(path), ".db", StringComparison.OrdinalIgnoreCase))
         {
             Logger.Error($"File extension must be .db.");
             return;
         }
 
-        HashSet<string> skipProviderNames = [];
+        if (!RegexHelper.TryCreate(filter, Logger, out var regex)) { return; }
+
+        if (source is not null && !ProviderSource.TryValidate(source, Logger)) { return; }
 
-        if (!string.IsNullOrWhiteSpace(skipProvidersInFile))
+        try
         {
-            if (!File.Exists(skipProvidersInFile))
+            HashSet<string> skipProviderNames = new(StringComparer.OrdinalIgnoreCase);
+
+            if (!string.IsNullOrWhiteSpace(skipProvidersInFile))
             {
-                Logger.Error($"File not found: {skipProvidersInFile}");
-                return;
+                if (!ProviderSource.TryValidate(skipProvidersInFile, Logger)) { return; }
+
+                foreach (var name in ProviderSource.LoadProviderNames(skipProvidersInFile, Logger))
+                {
+                    skipProviderNames.Add(name);
+                }
+
+                Logger.Info($"Found {skipProviderNames.Count} providers in {skipProvidersInFile}. These will not be included in the new database.");
             }
 
-            using var skipDbContext = new EventProviderDbContext(skipProvidersInFile, true, Logger);
+            // Load provider names first (cheap string-only query) for the empty check and header
+            // formatting. This avoids materializing all ProviderDetails (with large compressed
+            // payloads) just to compute the column widths.
+            IReadOnlyList<string> providerNames = source is null
+                ? GetLocalProviderNames(regex)
+                    .Where(n => !skipProviderNames.Contains(n)).ToList()
+                : ProviderSource.LoadProviderNames(source, Logger, regex)
+                    .Where(n => !skipProviderNames.Contains(n)).ToList();
 
-            foreach (var provider in skipDbContext.ProviderDetails)
+            if (providerNames.Count == 0)
             {
-                skipProviderNames.Add(provider.ProviderName);
+                Logger.Warn($"No providers to add to the new database.");
+                return;
             }
 
-            Logger.Info($"Found {skipProviderNames.Count} providers in file {skipProvidersInFile}. These will not be included in the new database.");
-        }
-
-        var providerNames = GetLocalProviderNames(filter);
+            LogProviderDetailHeader(providerNames);
 
-        if (!providerNames.Any())
-        {
-            Logger.Warn($"No providers found matching filter {filter}.");
-            return;
-        }
+            // Defer creating the DbContext (and therefore the .db file on disk) until we have
+            // at least one provider to persist. This prevents leaving an empty database behind
+            // when no provider details could be resolved (e.g., .evtx without LocaleMetaData).
+            EventProviderDbContext? dbContext = null;
 
-        var providerNamesNotSkipped = providerNames.Where(name => !skipProviderNames.Contains(name)).ToList();
+            try
+            {
+                // Stream details directly into the DbContext. Batch saves prevent the change tracker
+                // from accumulating all entities in memory at once.
+                const int batchSize = 100;
+                var count = 0;
 
-        var numberSkipped = providerNames.Count - providerNamesNotSkipped.Count;
+                IEnumerable<ProviderDetails> providersToAdd = source is null
+                    ? LoadLocalProviders(regex, skipProviderNames)
+                    : ProviderSource.LoadProviders(source, Logger, regex, skipProviderNames);
 
-        if (numberSkipped > 0)
-        {
-            Logger.Info($"{numberSkipped} providers were skipped due to being present in the specified database.");
-        }
+                foreach (var details in providersToAdd)
+                {
+                    dbContext ??= new EventProviderDbContext(path, false, Logger);
+                    dbContext.ProviderDetails.Add(details);
+                    LogProviderDetails(details);
+                    count++;
 
-        using var dbContext = new EventProviderDbContext(path, false, Logger);
+                    if (count % batchSize != 0) { continue; }
+                    dbContext.SaveChanges();
+                    dbContext.ChangeTracker.Clear();
+                }
 
-        LogProviderDetailHeader(providerNamesNotSkipped);
+                if (dbContext is null)
+                {
+                    Logger.Warn($"No provider details could be resolved from the source. Database was not created.");
 
-        foreach (var providerName in providerNamesNotSkipped)
-        {
-            var provider = new EventMessageProvider(providerName, Logger);
+                    return;
+                }
 
-            var details = provider.LoadProviderDetails();
+                Logger.Info($"");
+                Logger.Info($"Saving database. Please wait...");
 
-            dbContext.ProviderDetails.Add(details);
+                dbContext.SaveChanges();
 
-            LogProviderDetails(details);
+                Logger.Info($"Done!");
+            }
+            finally
+            {
+                dbContext?.Dispose();
+            }
+        }
+        catch (RegexMatchTimeoutException)
+        {
+            Logger.Error($"The --filter regex timed out. The pattern may cause catastrophic backtracking.");
         }
-
-        Logger.Info($"");
-        Logger.Info($"Saving database. Please wait...");
-
-        dbContext.SaveChanges();
-
-        Logger.Info($"Done!");
     }
+
 }

src/EventLogExpert.EventDbTool/DbToolCommand.cs

@@ -14,16 +14,36 @@ public class DbToolCommand(ITraceLogger logger)
 
     protected ITraceLogger Logger => logger;
 
-    protected static List<string> GetLocalProviderNames(string? filter)
+    protected static List<string> GetLocalProviderNames(string? filter, ITraceLogger logger) =>
+        !RegexHelper.TryCreate(filter, logger, out var regex) ? [] : GetLocalProviderNames(regex);
+
+    protected static List<string> GetLocalProviderNames(Regex? regex)
     {
         var providers = new List<string>(EventLogSession.GlobalSession.GetProviderNames().Distinct().OrderBy(name => name));
 
-        if (string.IsNullOrEmpty(filter)) { return providers; }
+        return regex is null ? providers : providers.Where(p => regex.IsMatch(p)).ToList();
+    }
+
+    protected IEnumerable<ProviderDetails> LoadLocalProviders(string? filter, IReadOnlySet<string>? skipProviderNames = null)
+    {
+        if (!RegexHelper.TryCreate(filter, Logger, out var regex)) { yield break; }
 
-        var regex = new Regex(filter, RegexOptions.IgnoreCase);
-        providers = providers.Where(p => regex.IsMatch(p)).ToList();
+        foreach (var details in LoadLocalProviders(regex, skipProviderNames))
+        {
+            yield return details;
+        }
+    }
+
+    protected IEnumerable<ProviderDetails> LoadLocalProviders(Regex? regex, IReadOnlySet<string>? skipProviderNames = null)
+    {
+        foreach (var providerName in GetLocalProviderNames(regex))
+        {
+            // Skip BEFORE resolving so we don't pay the cost of loading metadata for providers we
+            // are about to discard (e.g. when --skip-providers-in-file lists most local providers).
+            if (skipProviderNames is not null && skipProviderNames.Contains(providerName)) { continue; }
 
-        return providers;
+            yield return new EventMessageProvider(providerName, Logger).LoadProviderDetails();
+        }
     }
 
     protected void LogProviderDetailHeader(IEnumerable<string> providerNames)