diff --git a/src/EventLogExpert.EventDbTool/CreateDatabaseCommand.cs b/src/EventLogExpert.EventDbTool/CreateDatabaseCommand.cs index 24847a1d..c23781d2 100644 --- a/src/EventLogExpert.EventDbTool/CreateDatabaseCommand.cs +++ b/src/EventLogExpert.EventDbTool/CreateDatabaseCommand.cs @@ -1,4 +1,4 @@ -// // Copyright (c) Microsoft Corporation. +// // Copyright (c) Microsoft Corporation. // // Licensed under the MIT License. using EventLogExpert.Eventing.EventProviderDatabase; @@ -6,6 +6,7 @@ using EventLogExpert.Eventing.Providers; using Microsoft.Extensions.DependencyInjection; using System.CommandLine; +using System.Text.RegularExpressions; namespace EventLogExpert.EventDbTool; @@ -20,6 +21,14 @@ public static Command GetCommand() Description = "File to create. Must have a .db extension." }; + Argument sourceArgument = new("source") + { + Description = "Optional provider source: a .db file, an exported .evtx file, or a folder containing " + + ".db and/or .evtx files (top-level only). When omitted, local providers on this machine are used. " + + "When supplied, ONLY the source is used (no fallback to local providers).", + Arity = ArgumentArity.ZeroOrOne + }; + Option filterOption = new("--filter") { Description = "Only providers matching specified regex string will be added to the database." @@ -28,7 +37,8 @@ public static Command GetCommand() Option skipProvidersInFileOption = new("--skip-providers-in-file") { Description = - "Any providers found in the specified database file will not be included in the new database. " + + "Any providers found in the specified source (a .db file, an exported .evtx file, or a folder " + + "containing them, top-level only) will not be included in the new database. " + "For example, when creating a database of event providers for Exchange Server, it may be useful " + "to provide a database of all providers from a fresh OS install with no other products. That way, all the " + "OS providers are skipped, and only providers added by Exchange or other installed products " + @@ -41,6 +51,7 @@ public static Command GetCommand() }; createDatabaseCommand.Arguments.Add(fileArgument); + createDatabaseCommand.Arguments.Add(sourceArgument); createDatabaseCommand.Options.Add(filterOption); createDatabaseCommand.Options.Add(skipProvidersInFileOption); createDatabaseCommand.Options.Add(verboseOption); @@ -51,6 +62,7 @@ public static Command GetCommand() new CreateDatabaseCommand(sp.GetRequiredService()) .CreateDatabase( result.GetRequiredValue(fileArgument), + result.GetValue(sourceArgument), result.GetValue(filterOption), result.GetValue(skipProvidersInFileOption)); }); @@ -58,7 +70,7 @@ public static Command GetCommand() return createDatabaseCommand; } - private void CreateDatabase(string path, string? filter, string? skipProvidersInFile) + private void CreateDatabase(string path, string? source, string? filter, string? skipProvidersInFile) { if (File.Exists(path)) { @@ -66,69 +78,126 @@ private void CreateDatabase(string path, string? filter, string? skipProvidersIn return; } - if (Path.GetExtension(path) != ".db") + if (!string.Equals(Path.GetExtension(path), ".db", StringComparison.OrdinalIgnoreCase)) { Logger.Error($"File extension must be .db."); return; } - HashSet skipProviderNames = []; + if (!RegexHelper.TryCreate(filter, Logger, out var regex)) { return; } + + if (source is not null && !ProviderSource.TryValidate(source, Logger)) { return; } - if (!string.IsNullOrWhiteSpace(skipProvidersInFile)) + try { - if (!File.Exists(skipProvidersInFile)) + HashSet skipProviderNames = new(StringComparer.OrdinalIgnoreCase); + + if (!string.IsNullOrWhiteSpace(skipProvidersInFile)) { - Logger.Error($"File not found: {skipProvidersInFile}"); - return; - } + if (!ProviderSource.TryValidate(skipProvidersInFile, Logger)) { return; } - using var skipDbContext = new EventProviderDbContext(skipProvidersInFile, true, Logger); + foreach (var name in ProviderSource.LoadProviderNames(skipProvidersInFile, Logger)) + { + skipProviderNames.Add(name); + } - foreach (var provider in skipDbContext.ProviderDetails) - { - skipProviderNames.Add(provider.ProviderName); + Logger.Info($"Found {skipProviderNames.Count} providers in {skipProvidersInFile}. These will not be included in the new database."); } - Logger.Info($"Found {skipProviderNames.Count} providers in file {skipProvidersInFile}. These will not be included in the new database."); - } + // Stream details directly into the DbContext. For .evtx sources, scanning the file + // is expensive, so we deliberately do NOT pre-scan provider names just to size the + // log header — that would cause a second full pass over the .evtx. Instead we + // buffer the first batch of resolved details, derive the header column width from + // those names, then log the header + buffered rows and continue streaming. + const int batchSize = 100; + var count = 0; + var headerLogged = false; + var pendingForHeader = new List(batchSize); + + // Defer creating the DbContext (and therefore the .db file on disk) until we have + // at least one provider to persist. This prevents leaving an empty database behind + // when no provider details could be resolved (e.g., .evtx without LocaleMetaData). + EventProviderDbContext? dbContext = null; + + try + { + IEnumerable providersToAdd = source is null + ? LoadLocalProviders(regex, skipProviderNames) + : ProviderSource.LoadProviders(source, Logger, regex, skipProviderNames); - var providerNames = GetLocalProviderNames(filter); + foreach (var details in providersToAdd) + { + if (!headerLogged) + { + pendingForHeader.Add(details); - if (!providerNames.Any()) - { - Logger.Warn($"No providers found matching filter {filter}."); - return; - } + if (pendingForHeader.Count < batchSize) { continue; } - var providerNamesNotSkipped = providerNames.Where(name => !skipProviderNames.Contains(name)).ToList(); + FlushHeaderAndBuffer(ref dbContext, path, pendingForHeader, ref count); + headerLogged = true; + continue; + } - var numberSkipped = providerNames.Count - providerNamesNotSkipped.Count; + dbContext ??= new EventProviderDbContext(path, false, Logger); + dbContext.ProviderDetails.Add(details); + LogProviderDetails(details); + count++; - if (numberSkipped > 0) - { - Logger.Info($"{numberSkipped} providers were skipped due to being present in the specified database."); - } + if (count % batchSize != 0) { continue; } + dbContext.SaveChanges(); + dbContext.ChangeTracker.Clear(); + } + + // Flush any buffered providers when the stream ended before the buffer filled. + if (!headerLogged && pendingForHeader.Count > 0) + { + FlushHeaderAndBuffer(ref dbContext, path, pendingForHeader, ref count); + } - using var dbContext = new EventProviderDbContext(path, false, Logger); + if (dbContext is null) + { + Logger.Warn($"No provider details could be resolved from the source. Database was not created."); - LogProviderDetailHeader(providerNamesNotSkipped); + return; + } - foreach (var providerName in providerNamesNotSkipped) + Logger.Info($""); + Logger.Info($"Saving database. Please wait..."); + + dbContext.SaveChanges(); + + Logger.Info($"Done!"); + } + finally + { + dbContext?.Dispose(); + } + } + catch (RegexMatchTimeoutException) { - var provider = new EventMessageProvider(providerName, Logger); + Logger.Error($"The --filter regex timed out. The pattern may cause catastrophic backtracking."); + } + } - var details = provider.LoadProviderDetails(); + private void FlushHeaderAndBuffer( + ref EventProviderDbContext? dbContext, + string path, + List buffer, + ref int count) + { + LogProviderDetailHeader(buffer.Select(p => p.ProviderName)); - dbContext.ProviderDetails.Add(details); + dbContext ??= new EventProviderDbContext(path, false, Logger); + foreach (var details in buffer) + { + dbContext.ProviderDetails.Add(details); LogProviderDetails(details); + count++; } - Logger.Info($""); - Logger.Info($"Saving database. Please wait..."); - dbContext.SaveChanges(); - - Logger.Info($"Done!"); + dbContext.ChangeTracker.Clear(); + buffer.Clear(); } } diff --git a/src/EventLogExpert.EventDbTool/DbToolCommand.cs b/src/EventLogExpert.EventDbTool/DbToolCommand.cs index 872b7248..a74c4680 100644 --- a/src/EventLogExpert.EventDbTool/DbToolCommand.cs +++ b/src/EventLogExpert.EventDbTool/DbToolCommand.cs @@ -14,16 +14,36 @@ public class DbToolCommand(ITraceLogger logger) protected ITraceLogger Logger => logger; - protected static List GetLocalProviderNames(string? filter) + protected static List GetLocalProviderNames(string? filter, ITraceLogger logger) => + !RegexHelper.TryCreate(filter, logger, out var regex) ? [] : GetLocalProviderNames(regex); + + protected static List GetLocalProviderNames(Regex? regex) { var providers = new List(EventLogSession.GlobalSession.GetProviderNames().Distinct().OrderBy(name => name)); - if (string.IsNullOrEmpty(filter)) { return providers; } + return regex is null ? providers : providers.Where(p => regex.IsMatch(p)).ToList(); + } + + protected IEnumerable LoadLocalProviders(string? filter, IReadOnlySet? skipProviderNames = null) + { + if (!RegexHelper.TryCreate(filter, Logger, out var regex)) { yield break; } - var regex = new Regex(filter, RegexOptions.IgnoreCase); - providers = providers.Where(p => regex.IsMatch(p)).ToList(); + foreach (var details in LoadLocalProviders(regex, skipProviderNames)) + { + yield return details; + } + } + + protected IEnumerable LoadLocalProviders(Regex? regex, IReadOnlySet? skipProviderNames = null) + { + foreach (var providerName in GetLocalProviderNames(regex)) + { + // Skip BEFORE resolving so we don't pay the cost of loading metadata for providers we + // are about to discard (e.g. when --skip-providers-in-file lists most local providers). + if (skipProviderNames is not null && skipProviderNames.Contains(providerName)) { continue; } - return providers; + yield return new EventMessageProvider(providerName, Logger).LoadProviderDetails(); + } } protected void LogProviderDetailHeader(IEnumerable providerNames) diff --git a/src/EventLogExpert.EventDbTool/DiffDatabaseCommand.cs b/src/EventLogExpert.EventDbTool/DiffDatabaseCommand.cs index b4bab2cd..bb3a0450 100644 --- a/src/EventLogExpert.EventDbTool/DiffDatabaseCommand.cs +++ b/src/EventLogExpert.EventDbTool/DiffDatabaseCommand.cs @@ -15,22 +15,22 @@ public static Command GetCommand() { Command diffDatabaseCommand = new( "diff", - "Given two databases, produces a third database containing all providers " + - "from the second database which are not in the first database."); + "Given two provider sources (each may be a .db, an exported .evtx, or a folder containing them), " + + "produces a database containing all providers from the second source which are not in the first source."); - Argument dbOneArgument = new("first db") + Argument firstArgument = new("first source") { - Description = "The first database to compare." + Description = "The first source to compare: a .db, an exported .evtx, or a folder containing .db and/or .evtx files (top-level only)." }; - Argument dbTwoArgument = new("second db") + Argument secondArgument = new("second source") { - Description = "The second database to compare." + Description = "The second source to compare: a .db, an exported .evtx, or a folder containing .db and/or .evtx files (top-level only)." }; Argument newDbArgument = new("new db") { - Description = "The new database containing only the providers in the second db which are not in the first db. Must have a .db extension." + Description = "The new database containing only the providers in the second source which are not in the first source. Must have a .db extension." }; Option verboseOption = new("--verbose") @@ -38,8 +38,8 @@ public static Command GetCommand() Description = "Verbose logging. May be useful for troubleshooting." }; - diffDatabaseCommand.Arguments.Add(dbOneArgument); - diffDatabaseCommand.Arguments.Add(dbTwoArgument); + diffDatabaseCommand.Arguments.Add(firstArgument); + diffDatabaseCommand.Arguments.Add(secondArgument); diffDatabaseCommand.Arguments.Add(newDbArgument); diffDatabaseCommand.Options.Add(verboseOption); @@ -48,23 +48,18 @@ public static Command GetCommand() using var sp = Program.BuildServiceProvider(action.GetValue(verboseOption)); new DiffDatabaseCommand(sp.GetRequiredService()) .DiffDatabase( - action.GetRequiredValue(dbOneArgument), - action.GetRequiredValue(dbTwoArgument), + action.GetRequiredValue(firstArgument), + action.GetRequiredValue(secondArgument), action.GetRequiredValue(newDbArgument)); }); return diffDatabaseCommand; } - private void DiffDatabase(string dbOne, string dbTwo, string newDb) + private void DiffDatabase(string firstSource, string secondSource, string newDb) { - foreach (var path in new[] { dbOne, dbTwo }) - { - if (File.Exists(path)) { continue; } - - Logger.Error($"File not found: {path}"); - return; - } + if (!ProviderSource.TryValidate(firstSource, Logger)) { return; } + if (!ProviderSource.TryValidate(secondSource, Logger)) { return; } if (File.Exists(newDb)) { @@ -72,38 +67,41 @@ private void DiffDatabase(string dbOne, string dbTwo, string newDb) return; } - if (Path.GetExtension(newDb) != ".db") + if (!string.Equals(Path.GetExtension(newDb), ".db", StringComparison.OrdinalIgnoreCase)) { Logger.Error($"New db path must have a .db extension."); return; } - var dbOneProviderNames = new HashSet(); - - using (var dbOneContext = new EventProviderDbContext(dbOne, true, Logger)) - { - dbOneContext.ProviderDetails.Select(p => p.ProviderName).ToList() - .ForEach(name => dbOneProviderNames.Add(name)); - } + var firstProviderNames = new HashSet( + ProviderSource.LoadProviderNames(firstSource, Logger), + StringComparer.OrdinalIgnoreCase); var providersCopied = new List(); - using var dbTwoContext = new EventProviderDbContext(dbTwo, true, Logger); - using var newDbContext = new EventProviderDbContext(newDb, false, Logger); + // Pass firstProviderNames as the skip set so providers present in the first source are + // never resolved from the second source's metadata path. This is especially important when + // the second source is .evtx+MTA, where each provider triggers an expensive load. + Logger.Info($"Skipping up to {firstProviderNames.Count} provider name(s) from the second source that also appear in the first source."); + + // Defer creating the DbContext (and therefore the .db file on disk) until at least one + // provider is actually about to be persisted. This prevents leaving an empty database + // behind when the second source yields no new providers. + EventProviderDbContext? newDbContext = null; - foreach (var details in dbTwoContext.ProviderDetails) + try { - if (dbOneProviderNames.Contains(details.ProviderName)) - { - Logger.Info($"Skipping {details.ProviderName} because it is present in both databases."); - } - else + foreach (var details in ProviderSource.LoadProviders(secondSource, Logger, filter: null, skipProviderNames: firstProviderNames)) { - Logger.Info($"Copying {details.ProviderName} because it is present in second db but not first db."); + Logger.Info($"Copying {details.ProviderName} because it is present in second source but not first."); + + newDbContext ??= new EventProviderDbContext(newDb, false, Logger); + newDbContext.ProviderDetails.Add(new ProviderDetails { ProviderName = details.ProviderName, Events = details.Events, + Parameters = details.Parameters, Keywords = details.Keywords, Messages = details.Messages, Opcodes = details.Opcodes, @@ -112,19 +110,27 @@ private void DiffDatabase(string dbOne, string dbTwo, string newDb) providersCopied.Add(details); } - } - newDbContext.SaveChanges(); + if (newDbContext is null) + { + Logger.Warn($"No providers in the second source are missing from the first. Database was not created."); + return; + } - if (providersCopied.Count <= 0) { return; } - - Logger.Info($"Providers copied to new database:"); - Logger.Info($""); - LogProviderDetailHeader(providersCopied.Select(p => p.ProviderName)); + newDbContext.SaveChanges(); - foreach (var provider in providersCopied) + Logger.Info($"Providers copied to new database:"); + Logger.Info($""); + LogProviderDetailHeader(providersCopied.Select(p => p.ProviderName)); + + foreach (var provider in providersCopied) + { + LogProviderDetails(provider); + } + } + finally { - LogProviderDetails(provider); + newDbContext?.Dispose(); } } } diff --git a/src/EventLogExpert.EventDbTool/MergeDatabaseCommand.cs b/src/EventLogExpert.EventDbTool/MergeDatabaseCommand.cs index b4e411e4..cfede9b8 100644 --- a/src/EventLogExpert.EventDbTool/MergeDatabaseCommand.cs +++ b/src/EventLogExpert.EventDbTool/MergeDatabaseCommand.cs @@ -4,6 +4,7 @@ using EventLogExpert.Eventing.EventProviderDatabase; using EventLogExpert.Eventing.Helpers; using EventLogExpert.Eventing.Providers; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using System.CommandLine; @@ -17,9 +18,9 @@ public static Command GetCommand() "merge", "Copies providers from a source into a target database."); - Argument sourceDatabaseArgument = new("source db") + Argument sourceArgument = new("source") { - Description = "The source database from which to copy providers." + Description = "The provider source: a .db file, an exported .evtx file, or a folder containing .db and/or .evtx files (top-level only)." }; Argument targetDatabaseArgument = new("target db") @@ -38,7 +39,7 @@ public static Command GetCommand() Description = "Enable verbose logging. May be useful for troubleshooting." }; - mergeDatabaseCommand.Arguments.Add(sourceDatabaseArgument); + mergeDatabaseCommand.Arguments.Add(sourceArgument); mergeDatabaseCommand.Arguments.Add(targetDatabaseArgument); mergeDatabaseCommand.Options.Add(overwriteOption); mergeDatabaseCommand.Options.Add(verboseOption); @@ -48,7 +49,7 @@ public static Command GetCommand() using var sp = Program.BuildServiceProvider(action.GetValue(verboseOption)); new MergeDatabaseCommand(sp.GetRequiredService()) .MergeDatabase( - action.GetRequiredValue(sourceDatabaseArgument), + action.GetRequiredValue(sourceArgument), action.GetRequiredValue(targetDatabaseArgument), action.GetValue(overwriteOption)); }); @@ -56,52 +57,87 @@ public static Command GetCommand() return mergeDatabaseCommand; } - private void MergeDatabase(string sourceFile, string targetFile, bool overwriteProviders) + private void MergeDatabase(string source, string targetFile, bool overwriteProviders) { - foreach (var path in new[] { sourceFile, targetFile }) - { - if (File.Exists(path)) { continue; } + if (!ProviderSource.TryValidate(source, Logger)) { return; } - Logger.Error($"File not found: {path}"); + if (!File.Exists(targetFile)) + { + Logger.Error($"File not found: {targetFile}"); return; } - var sourceProviders = new List(); + // Load only the cheap projection of source provider names first. This avoids resolving + // (and for .evtx+MTA sources, expensively materializing) provider details that will be + // skipped because they already exist in the target. + var sourceNames = new HashSet(ProviderSource.LoadProviderNames(source, Logger), StringComparer.OrdinalIgnoreCase); - using (var sourceContext = new EventProviderDbContext(sourceFile, true, Logger)) + if (sourceNames.Count == 0) { - sourceProviders.AddRange(sourceContext.ProviderDetails.ToList()); + Logger.Warn($"No providers were discovered in the source."); + return; } using var targetContext = new EventProviderDbContext(targetFile, false, Logger); - var providersAlreadyInTarget = new Dictionary(); + // Query the overlap in the database by chunking sourceNames into IN-clause batches, + // rather than pulling every target ProviderName into memory. Same chunk size as the + // delete loop below to stay below SQLite's default parameter limit (999). + var sourceNamesList = sourceNames.ToList(); + var targetMatchingNames = new List(); - foreach (var sourceProviderDetails in sourceProviders) + for (var offset = 0; offset < sourceNamesList.Count; offset += ProviderSource.MaxInClauseParameters) { - var existingProviderInTarget = targetContext.ProviderDetails.FirstOrDefault(p => p.ProviderName == sourceProviderDetails.ProviderName); - - if (existingProviderInTarget != null) - { - providersAlreadyInTarget.Add(existingProviderInTarget.ProviderName, existingProviderInTarget); - } + var chunk = sourceNamesList + .Skip(offset) + .Take(ProviderSource.MaxInClauseParameters) + .ToList(); + + // Use NOCASE collation on the column side so the IN-clause matches case-insensitively, + // matching the case-insensitive semantics used elsewhere in the tool (sourceNames is + // an OrdinalIgnoreCase HashSet). Without this, a source name "Microsoft-Foo" would + // not match a target row stored as "microsoft-foo", causing duplicates on copy or + // missed deletes when --overwrite is used. Using NOCASE on the column prevents PK + // index use for this query (table scan), but for an admin one-shot merge the + // correctness trade-off is acceptable. + targetMatchingNames.AddRange( + targetContext.ProviderDetails + .AsNoTracking() + .Where(p => chunk.Contains(EF.Functions.Collate(p.ProviderName, "NOCASE"))) + .Select(p => p.ProviderName)); } - if (providersAlreadyInTarget.Count > 0) + // Track the source-side provider names whose case-insensitive equivalent exists in target. + // ProviderName is the primary key in the target DB, so case-sensitive uniqueness identifies + // a row; the case-insensitive HashSet drives the no-overwrite skip on the source side. + var providerNamesInTarget = new HashSet(targetMatchingNames, StringComparer.OrdinalIgnoreCase); + + if (targetMatchingNames.Count > 0) { - Logger.Info($"The target database contains {providersAlreadyInTarget.Count} providers that are in the source."); + Logger.Info($"The target database contains {targetMatchingNames.Count} provider row(s) matching {providerNamesInTarget.Count} provider name(s) in the source."); if (overwriteProviders) { Logger.Info($"Removing these providers from the target database..."); - foreach (var provider in providersAlreadyInTarget.Values) + // Chunk the IN-clause to stay below SQLite's parameter limit (default 999). Without + // chunking, an --overwrite of a large overlap could throw at runtime. + // ExecuteDelete() issues a SQL DELETE directly, avoiding change-tracker overhead. + var removed = 0; + + for (var offset = 0; offset < targetMatchingNames.Count; offset += ProviderSource.MaxInClauseParameters) { - targetContext.Remove(provider); + var chunk = targetMatchingNames + .Skip(offset) + .Take(ProviderSource.MaxInClauseParameters) + .ToList(); + + removed += targetContext.ProviderDetails + .Where(p => chunk.Contains(p.ProviderName)) + .ExecuteDelete(); } - targetContext.SaveChanges(); - Logger.Info($"Removal of {providersAlreadyInTarget.Count} completed."); + Logger.Info($"Removal of {removed} provider row(s) completed."); } else { @@ -111,39 +147,62 @@ private void MergeDatabase(string sourceFile, string targetFile, bool overwriteP Logger.Info($"Copying providers from the source..."); - var providersCopied = new List(); + // When not overwriting, pass the overlap as the skip set so providers that already exist + // in the target are never resolved from the source's metadata path. When overwriting, no + // skip set is passed so all source providers are loaded and re-inserted. + var skipForLoad = overwriteProviders ? null : providerNamesInTarget; + + // Compute the expected copy set up front from cheap string sets so we can size the log + // header BEFORE streaming. This avoids retaining the full ProviderDetails objects in a + // summary list — those objects can be sizable for .evtx+MTA sources, defeating the + // ChangeTracker.Clear batching below. + var expectedCopiedNames = skipForLoad is null + ? sourceNames.ToList() + : sourceNames.Where(n => !skipForLoad.Contains(n)).ToList(); + + Logger.Info($""); + LogProviderDetailHeader(expectedCopiedNames); + + // Stream details into the DbContext with periodic SaveChanges. The pending batch list is + // bounded by batchSize so memory stays flat regardless of source size; details are logged + // AFTER each successful SaveChanges so the printed rows reflect what actually persisted + // (a SaveChanges failure won't cause the log to overstate progress). + const int batchSize = 100; + var copiedCount = 0; + var pendingBatch = new List(batchSize); - foreach (var provider in sourceProviders) + foreach (var provider in ProviderSource.LoadProviders(source, Logger, filter: null, skipProviderNames: skipForLoad)) { - if (providersAlreadyInTarget.ContainsKey(provider.ProviderName) && !overwriteProviders) - { - Logger.Info($"Skipping provider: {provider.ProviderName}"); - continue; - } + // ProviderSource yields fresh, non-tracked entities — Add(provider) is safe and avoids + // the redundant per-property projection that the previous implementation used. + targetContext.ProviderDetails.Add(provider); + pendingBatch.Add(provider); - targetContext.ProviderDetails.Add(new ProviderDetails - { - ProviderName = provider.ProviderName, - Events = provider.Events, - Parameters = provider.Parameters, - Keywords = provider.Keywords, - Messages = provider.Messages, - Opcodes = provider.Opcodes, - Tasks = provider.Tasks - }); - - providersCopied.Add(provider); + if (pendingBatch.Count < batchSize) { continue; } + + FlushBatch(targetContext, pendingBatch, ref copiedCount); } - targetContext.SaveChanges(); + if (pendingBatch.Count > 0) + { + FlushBatch(targetContext, pendingBatch, ref copiedCount); + } - Logger.Info($"Providers copied:"); Logger.Info($""); - LogProviderDetailHeader(providersCopied.Select(p => p.ProviderName)); + Logger.Info($"Copied {copiedCount} provider(s)."); + } - foreach (var provider in providersCopied) + private void FlushBatch(EventProviderDbContext context, List batch, ref int copiedCount) + { + context.SaveChanges(); + + foreach (var details in batch) { - LogProviderDetails(provider); + LogProviderDetails(details); } + + copiedCount += batch.Count; + batch.Clear(); + context.ChangeTracker.Clear(); } } diff --git a/src/EventLogExpert.EventDbTool/MtaProviderSource.cs b/src/EventLogExpert.EventDbTool/MtaProviderSource.cs new file mode 100644 index 00000000..1abc07d1 --- /dev/null +++ b/src/EventLogExpert.EventDbTool/MtaProviderSource.cs @@ -0,0 +1,203 @@ +// // Copyright (c) Microsoft Corporation. +// // Licensed under the MIT License. + +using EventLogExpert.Eventing.Helpers; +using EventLogExpert.Eventing.Providers; +using EventLogExpert.Eventing.Readers; +using System.Text.RegularExpressions; + +namespace EventLogExpert.EventDbTool; + +/// +/// Loads from an exported .evtx log paired with its sibling +/// LocaleMetaData/*.MTA files. Provider names are discovered by reading the .evtx events, +/// and the MTA files are used as the only metadata source (no registry/DLL fallback). +/// +internal static class MtaProviderSource +{ + /// + /// Reads and returns the distinct provider names referenced by its + /// event records. Does NOT require sibling LocaleMetaData/*.MTA files. + /// + public static IReadOnlyList DiscoverProviderNames( + string evtxPath, + ITraceLogger logger, + string? filter = null) => + !RegexHelper.TryCreate(filter, logger, out var regex) ? [] : DiscoverProviderNamesCore(evtxPath, logger, regex); + + private static IReadOnlyList DiscoverProviderNamesCore( + string evtxPath, + ITraceLogger logger, + Regex? regex) + { + if (!File.Exists(evtxPath)) + { + logger.Error($"Evtx file not found: {evtxPath}"); + return []; + } + + var providerNames = new SortedSet(StringComparer.OrdinalIgnoreCase); + + try + { + using var reader = new EventLogReader(evtxPath, PathType.FilePath); + + if (!reader.IsValid) + { + logger.Error($"Failed to open {evtxPath} for reading. The file may be missing, corrupt, or inaccessible."); + return []; + } + + // TryGetEvents returns false both for normal end-of-results (ERROR_NO_MORE_ITEMS) and + // for read errors (corruption, access denied, etc.). Check LastErrorCode to surface + // non-terminal failures so users can distinguish "0 events" from "could not read the log". + while (reader.TryGetEvents(out var batch)) + { + foreach (var record in batch) + { + if (!string.IsNullOrEmpty(record.ProviderName)) + { + providerNames.Add(record.ProviderName); + } + } + } + + if (reader.LastErrorCode is not null) + { + logger.Warn( + $"Reading {evtxPath} may be incomplete. " + + $"EvtNext failed with Win32 error code {reader.LastErrorCode}."); + } + } + catch (Exception ex) + { + logger.Error($"Failed to read events from {evtxPath}: {ex.Message}"); + return []; + } + + return regex is null ? providerNames.ToList() : providerNames.Where(n => regex.IsMatch(n)).ToList(); + } + + /// + /// Returns the sibling LocaleMetaData/*.MTA files for , or an empty + /// array if none are present. Logs an error in the empty case to surface the misconfiguration + /// without throwing. + /// + public static IReadOnlyList FindMtaFiles(string evtxPath, ITraceLogger logger) + { + var logDir = Path.GetDirectoryName(Path.GetFullPath(evtxPath)); + + if (logDir is null) + { + logger.Error($"Could not determine directory for {evtxPath}."); + return []; + } + + var localeDir = Path.Combine(logDir, "LocaleMetaData"); + + if (!Directory.Exists(localeDir)) + { + logger.Error( + $"No LocaleMetaData folder found next to {evtxPath}. " + + $"MTA resolution requires the sibling LocaleMetaData folder produced when the log was exported."); + + return []; + } + + string[] mtaFiles; + + try + { + mtaFiles = Directory.GetFiles(localeDir, "*.MTA"); + } + catch (Exception ex) when (ex is UnauthorizedAccessException or IOException) + { + logger.Error($"Cannot read LocaleMetaData folder '{localeDir}': {ex.Message}"); + return []; + } + + Array.Sort(mtaFiles, StringComparer.Ordinal); + + if (mtaFiles.Length == 0) + { + logger.Error($"LocaleMetaData folder at {localeDir} contains no MTA files."); + return []; + } + + logger.Info($"Using {mtaFiles.Length} locale metadata file(s) from {localeDir}."); + + return mtaFiles; + } + + /// + /// Discovers provider names from and yields a + /// for each that can be resolved exclusively from the sibling + /// LocaleMetaData/*.MTA files. Providers that cannot be resolved from any MTA file are + /// skipped with a warning so callers never persist empty placeholder providers (which would + /// defeat the "no local fallback" guarantee when the resulting database is consumed later). + /// + public static IEnumerable LoadProviders( + string evtxPath, + ITraceLogger logger, + string? filter = null) => + !RegexHelper.TryCreate(filter, logger, out var regex) ? [] : + LoadProvidersCore(evtxPath, logger, regex, null, null); + + /// + /// Internal overload used by so name-based filtering and the de-dup + /// seen set are applied BEFORE the expensive MTA resolution per provider. + /// + internal static IEnumerable LoadProviders( + string evtxPath, + ITraceLogger logger, + Regex? regex, + IReadOnlySet? skipProviderNames, + HashSet? seen) => + LoadProvidersCore(evtxPath, logger, regex, skipProviderNames, seen); + + private static bool IsEmpty(ProviderDetails details) => + details.Events.Count == 0 && + details.Keywords.Count == 0 && + details.Messages.Count == 0 && + details.Opcodes.Count == 0 && + details.Tasks.Count == 0 && + !details.Parameters.Any(); + + private static IEnumerable LoadProvidersCore( + string evtxPath, + ITraceLogger logger, + Regex? regex, + IReadOnlySet? skipProviderNames, + HashSet? seen) + { + var providerNames = DiscoverProviderNamesCore(evtxPath, logger, regex); + + if (providerNames.Count == 0) { yield break; } + + var mtaFiles = FindMtaFiles(evtxPath, logger); + + // Without MTA files, EventMessageProvider would fall back to the local registry/DLLs, + // producing data attributable to this machine rather than the exported log. Refuse. + if (mtaFiles.Count == 0) { yield break; } + + foreach (var providerName in providerNames) + { + if (skipProviderNames is not null && skipProviderNames.Contains(providerName)) { continue; } + if (seen is not null && seen.Contains(providerName)) { continue; } + + var details = new EventMessageProvider(providerName, null, mtaFiles, logger).LoadProviderDetails(); + + if (IsEmpty(details)) + { + logger.Warn($"Skipping {providerName}: not found in any MTA file next to {evtxPath}."); + continue; + } + + // Mark as seen only after confirming the provider has data, so that an empty/missing + // provider from one .evtx does not block loading it from a later source file. + seen?.Add(providerName); + + yield return details; + } + } +} diff --git a/src/EventLogExpert.EventDbTool/Program.cs b/src/EventLogExpert.EventDbTool/Program.cs index ff667d43..776a5c4f 100644 --- a/src/EventLogExpert.EventDbTool/Program.cs +++ b/src/EventLogExpert.EventDbTool/Program.cs @@ -19,8 +19,7 @@ private static async Task Main(string[] args) { RootCommand rootCommand = new("Tool used to create and modify databases for use with EventLogExpert"); - rootCommand.Subcommands.Add(ShowLocalCommand.GetCommand()); - rootCommand.Subcommands.Add(ShowDatabaseCommand.GetCommand()); + rootCommand.Subcommands.Add(ShowCommand.GetCommand()); rootCommand.Subcommands.Add(CreateDatabaseCommand.GetCommand()); rootCommand.Subcommands.Add(MergeDatabaseCommand.GetCommand()); rootCommand.Subcommands.Add(DiffDatabaseCommand.GetCommand()); diff --git a/src/EventLogExpert.EventDbTool/ProviderSource.cs b/src/EventLogExpert.EventDbTool/ProviderSource.cs new file mode 100644 index 00000000..09402eb6 --- /dev/null +++ b/src/EventLogExpert.EventDbTool/ProviderSource.cs @@ -0,0 +1,277 @@ +// // Copyright (c) Microsoft Corporation. +// // Licensed under the MIT License. + +using EventLogExpert.Eventing.EventProviderDatabase; +using EventLogExpert.Eventing.Helpers; +using EventLogExpert.Eventing.Providers; +using Microsoft.EntityFrameworkCore; +using System.Data.Common; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace EventLogExpert.EventDbTool; + +/// +/// Loads from a path that may be a .db file, an exported .evtx file, or a +/// folder. When the path is a folder, all top-level *.db files are processed first (sorted), followed by +/// all top-level *.evtx files (sorted). Subdirectories are not searched. +/// +internal static class ProviderSource +{ + /// + /// Conservative cap on the number of parameters in a single Where(... Contains) SQL IN + /// clause. SQLite's default limit is 999 parameters; we stay well under that so the same code + /// works on older SQLite builds too. Larger requests are split into multiple round-trips. + /// + internal const int MaxInClauseParameters = 500; + + private const string DbExtension = ".db"; + private const string EvtxExtension = ".evtx"; + + /// + /// Returns the distinct provider names available from , applying an optional + /// case-insensitive regex . Does not load full provider details. + /// + public static IReadOnlyList LoadProviderNames(string path, ITraceLogger logger, string? filter = null) => + !RegexHelper.TryCreate(filter, logger, out var regex) ? [] : LoadProviderNames(path, logger, regex); + + /// + public static IReadOnlyList LoadProviderNames(string path, ITraceLogger logger, Regex? regex) + { + var names = new SortedSet(StringComparer.OrdinalIgnoreCase); + + foreach (var file in EnumerateSourceFiles(path, logger)) + { + foreach (var name in LoadNamesFromFile(file, logger)) + { + names.Add(name); + } + } + + return regex is null ? names.ToList() : names.Where(n => regex.IsMatch(n)).ToList(); + } + + /// + /// Loads from , applying an optional + /// case-insensitive regex to provider names. When the same provider name + /// appears in multiple source files, the first occurrence wins (.db files are processed before .evtx). + /// Provider names contained in are excluded BEFORE details are + /// resolved, so callers using the skip set never pay the cost of loading metadata for excluded providers. + /// + public static IEnumerable LoadProviders( + string path, + ITraceLogger logger, + string? filter = null, + IReadOnlySet? skipProviderNames = null) => + !RegexHelper.TryCreate(filter, logger, out var regex) ? [] : + LoadProvidersIterator(path, logger, regex, skipProviderNames); + + /// + public static IEnumerable LoadProviders( + string path, + ITraceLogger logger, + Regex? regex, + IReadOnlySet? skipProviderNames = null) => + LoadProvidersIterator(path, logger, regex, skipProviderNames); + + /// Validates that exists and has a recognized form. + public static bool TryValidate(string path, ITraceLogger logger) + { + if (Directory.Exists(path)) + { + // Probe directory accessibility up front so callers fail fast with a clear message + // rather than silently producing wrong output later when EnumerateSourceFiles falls + // back to an empty sequence on an UnauthorizedAccessException/IOException. + try + { + Directory.GetFiles(path); + return true; + } + catch (Exception ex) when (ex is UnauthorizedAccessException or IOException) + { + logger.Error($"Cannot read source folder '{path}': {ex.Message}"); + return false; + } + } + + if (!File.Exists(path)) + { + logger.Error($"Source not found: {path}"); + return false; + } + + var extension = Path.GetExtension(path); + + if (string.Equals(extension, DbExtension, StringComparison.OrdinalIgnoreCase) || + string.Equals(extension, EvtxExtension, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + logger.Error($"Unsupported source file extension '{extension}'. Expected .db, .evtx, or a folder containing them."); + return false; + } + + /// + /// Expands into the ordered list of source files: a single .db or .evtx + /// when given a file; or all *.db files (sorted) followed by all *.evtx files (sorted) when given a + /// folder. + /// + private static IEnumerable EnumerateSourceFiles(string path, ITraceLogger logger) + { + if (!Directory.Exists(path)) + { + yield return path; + yield break; + } + + // TryValidate already probed accessibility, but a transient IO/permission change between + // validation and enumeration is still possible. Catch and log so the tool warns instead + // of crashing; an empty result here means the caller sees no source files (handled + // elsewhere with a "no providers" warning). + string[] allFiles; + + try + { + allFiles = Directory.GetFiles(path); + } + catch (Exception ex) when (ex is UnauthorizedAccessException or IOException) + { + logger.Error($"Cannot read source folder '{path}': {ex.Message}"); + yield break; + } + + // Use case-insensitive extension comparison so that .DB / .EVTX (and any other case + // variants permitted by case-sensitive directories on Windows) are picked up the same + // way TryValidate accepts them. Files are bucketed (.db first, then .evtx) and sorted + // OrdinalIgnoreCase within each bucket so first-occurrence-wins ordering is stable. + var dbFiles = allFiles + .Where(f => string.Equals(System.IO.Path.GetExtension(f), DbExtension, StringComparison.OrdinalIgnoreCase)) + .ToArray(); + Array.Sort(dbFiles, StringComparer.OrdinalIgnoreCase); + + foreach (var f in dbFiles) { yield return f; } + + var evtxFiles = allFiles + .Where(f => string.Equals(System.IO.Path.GetExtension(f), EvtxExtension, StringComparison.OrdinalIgnoreCase)) + .ToArray(); + Array.Sort(evtxFiles, StringComparer.OrdinalIgnoreCase); + + foreach (var f in evtxFiles) { yield return f; } + } + + private static IEnumerable LoadDetailsFromFile( + string file, + ITraceLogger logger, + Regex? regex, + IReadOnlySet? skipProviderNames, + HashSet seen) + { + var ext = Path.GetExtension(file); + + if (string.Equals(ext, DbExtension, StringComparison.OrdinalIgnoreCase)) + { + try + { + using var context = new EventProviderDbContext(file, true, logger); + context.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + + // Filter by name without mutating `seen` so that a subsequent catch does not + // permanently mark these names as loaded when they were never successfully read. + var allNames = context.ProviderDetails.Select(p => p.ProviderName).ToList(); + var namesToLoad = allNames + .Where(name => + { + if (seen.Contains(name)) { return false; } + if (regex is not null && !regex.IsMatch(name)) { return false; } + + return skipProviderNames is null || !skipProviderNames.Contains(name); + }) + .OrderBy(n => n, StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (namesToLoad.Count == 0) { return []; } + + // Chunk the IN-clause to stay below SQLite's parameter limit (default 999). + var loaded = new List(namesToLoad.Count); + + for (var offset = 0; offset < namesToLoad.Count; offset += MaxInClauseParameters) + { + var chunk = namesToLoad + .Skip(offset) + .Take(MaxInClauseParameters) + .ToList(); + + loaded.AddRange(context.ProviderDetails + .Where(p => chunk.Contains(p.ProviderName)) + .OrderBy(p => p.ProviderName)); + } + + // Mark as seen only after a successful load so a catch for a corrupt file does + // not prevent the same provider names from being loaded from a later source file. + foreach (var name in namesToLoad) { seen.Add(name); } + + return loaded; + } + catch (Exception ex) when (ex is DbException or JsonException or InvalidDataException) + { + logger.Warn($"Skipping invalid database file '{file}': {ex.Message}"); + return []; + } + } + + if (string.Equals(ext, EvtxExtension, StringComparison.OrdinalIgnoreCase)) + { + return MtaProviderSource.LoadProviders(file, logger, regex, skipProviderNames, seen); + } + + logger.Warn($"Skipping unsupported source file: {file}"); + + return []; + } + + private static IEnumerable LoadNamesFromFile(string file, ITraceLogger logger) + { + var ext = Path.GetExtension(file); + + if (string.Equals(ext, DbExtension, StringComparison.OrdinalIgnoreCase)) + { + try + { + using var providerContext = new EventProviderDbContext(file, true, logger); + return providerContext.ProviderDetails.AsNoTracking().Select(p => p.ProviderName).ToList(); + } + catch (DbException ex) + { + logger.Warn($"Skipping invalid database file '{file}': {ex.Message}"); + return []; + } + } + + if (string.Equals(ext, EvtxExtension, StringComparison.OrdinalIgnoreCase)) + { + return MtaProviderSource.DiscoverProviderNames(file, logger); + } + + logger.Warn($"Skipping unsupported source file: {file}"); + + return []; + } + + private static IEnumerable LoadProvidersIterator( + string path, + ITraceLogger logger, + Regex? regex, + IReadOnlySet? skipProviderNames) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var file in EnumerateSourceFiles(path, logger)) + { + foreach (var details in LoadDetailsFromFile(file, logger, regex, skipProviderNames, seen)) + { + yield return details; + } + } + } +} diff --git a/src/EventLogExpert.EventDbTool/RegexHelper.cs b/src/EventLogExpert.EventDbTool/RegexHelper.cs new file mode 100644 index 00000000..3aac00ff --- /dev/null +++ b/src/EventLogExpert.EventDbTool/RegexHelper.cs @@ -0,0 +1,46 @@ +// // Copyright (c) Microsoft Corporation. +// // Licensed under the MIT License. + +using EventLogExpert.Eventing.Helpers; +using System.Text.RegularExpressions; + +namespace EventLogExpert.EventDbTool; + +/// +/// Centralized creation of instances for user-supplied --filter patterns. +/// Always sets a match timeout to bound worst-case execution against catastrophic backtracking, and +/// converts from invalid patterns into a logged error rather than +/// letting it terminate the process. +/// +internal static class RegexHelper +{ + /// Maximum time a single regex match is allowed to take before throwing. + private static readonly TimeSpan s_matchTimeout = TimeSpan.FromSeconds(1); + + /// + /// Attempts to compile into a case-insensitive with + /// a bounded match timeout. A null/empty pattern is treated as "no filter": + /// is set to and the method still returns so callers + /// can distinguish an absent filter from a malformed one. + /// + public static bool TryCreate(string? pattern, ITraceLogger logger, out Regex? regex) + { + if (string.IsNullOrEmpty(pattern)) + { + regex = null; + return true; + } + + try + { + regex = new Regex(pattern, RegexOptions.IgnoreCase, s_matchTimeout); + return true; + } + catch (ArgumentException ex) + { + logger.Error($"Invalid --filter regex '{pattern}': {ex.Message}"); + regex = null; + return false; + } + } +} diff --git a/src/EventLogExpert.EventDbTool/ShowCommand.cs b/src/EventLogExpert.EventDbTool/ShowCommand.cs new file mode 100644 index 00000000..e8d68698 --- /dev/null +++ b/src/EventLogExpert.EventDbTool/ShowCommand.cs @@ -0,0 +1,95 @@ +// // Copyright (c) Microsoft Corporation. +// // Licensed under the MIT License. + +using EventLogExpert.Eventing.Helpers; +using EventLogExpert.Eventing.Providers; +using Microsoft.Extensions.DependencyInjection; +using System.CommandLine; +using System.Text.RegularExpressions; + +namespace EventLogExpert.EventDbTool; + +public class ShowCommand(ITraceLogger logger) : DbToolCommand(logger) +{ + public static Command GetCommand() + { + Command showCommand = new( + name: "show", + description: "List event providers. When no source is supplied, lists providers on the local machine. " + + "When a source is supplied, it may be a .db file created with this tool, an exported .evtx file " + + "(resolved via its sibling LocaleMetaData/*.MTA files), or a folder containing either."); + + Argument sourceArgument = new("source") + { + Description = "Optional source: a .db file, an exported .evtx file, or a folder containing .db and/or " + + ".evtx files (top-level only). When omitted, local providers on this machine are listed.", + Arity = ArgumentArity.ZeroOrOne + }; + + Option filterOption = new("--filter") + { + Description = "Filter for provider names matching the specified regex string." + }; + + Option verboseOption = new("--verbose") + { + Description = "Verbose logging. May be useful for troubleshooting." + }; + + showCommand.Arguments.Add(sourceArgument); + showCommand.Options.Add(filterOption); + showCommand.Options.Add(verboseOption); + + showCommand.SetAction(action => + { + using var sp = Program.BuildServiceProvider(action.GetValue(verboseOption)); + new ShowCommand(sp.GetRequiredService()) + .ShowProviderInfo( + action.GetValue(sourceArgument), + action.GetValue(filterOption)); + }); + + return showCommand; + } + + private void ShowProviderInfo(string? source, string? filter) + { + if (!RegexHelper.TryCreate(filter, Logger, out var regex)) { return; } + + try + { + IReadOnlyList providerNames; + IEnumerable providers; + + if (source is null) + { + providerNames = GetLocalProviderNames(regex); + providers = LoadLocalProviders(regex); + } + else + { + if (!ProviderSource.TryValidate(source, Logger)) { return; } + + providerNames = ProviderSource.LoadProviderNames(source, Logger, regex); + providers = ProviderSource.LoadProviders(source, Logger, regex); + } + + if (providerNames.Count == 0) + { + Logger.Warn($"No providers found."); + return; + } + + LogProviderDetailHeader(providerNames); + + foreach (var details in providers) + { + LogProviderDetails(details); + } + } + catch (RegexMatchTimeoutException) + { + Logger.Error($"The --filter regex timed out. The pattern may cause catastrophic backtracking."); + } + } +} diff --git a/src/EventLogExpert.EventDbTool/ShowDatabaseCommand.cs b/src/EventLogExpert.EventDbTool/ShowDatabaseCommand.cs deleted file mode 100644 index 06d17006..00000000 --- a/src/EventLogExpert.EventDbTool/ShowDatabaseCommand.cs +++ /dev/null @@ -1,74 +0,0 @@ -// // Copyright (c) Microsoft Corporation. -// // Licensed under the MIT License. - -using EventLogExpert.Eventing.EventProviderDatabase; -using EventLogExpert.Eventing.Helpers; -using Microsoft.Extensions.DependencyInjection; -using System.CommandLine; -using System.Text.RegularExpressions; - -namespace EventLogExpert.EventDbTool; - -public class ShowDatabaseCommand(ITraceLogger logger) : DbToolCommand(logger) -{ - public static Command GetCommand() - { - Command showDatabaseCommand = new(name: "showdatabase", description: "List the event providers from a database created with this tool."); - - Argument fileArgument = new("file") - { - Description = "The database file to show." - }; - - Option filterOption = new("--filter") - { - Description = "Filter for provider names matching the specified regex string." - }; - - Option verboseOption = new("--verbose") - { - Description = "Verbose logging. May be useful for troubleshooting." - }; - - showDatabaseCommand.Arguments.Add(fileArgument); - showDatabaseCommand.Options.Add(filterOption); - showDatabaseCommand.Options.Add(verboseOption); - - showDatabaseCommand.SetAction(action => - { - using var sp = Program.BuildServiceProvider(action.GetValue(verboseOption)); - new ShowDatabaseCommand(sp.GetRequiredService()) - .ShowProviderInfo( - action.GetRequiredValue(fileArgument), - action.GetValue(filterOption)); - }); - - return showDatabaseCommand; - } - - private void ShowProviderInfo(string file, string? filter) - { - if (!File.Exists(file)) - { - Logger.Error($"File not found: {file}"); - return; - } - - using var dbContext = new EventProviderDbContext(file, readOnly: true, Logger); - - var providerNames = dbContext.ProviderDetails.Select(p => p.ProviderName).OrderBy(name => name).ToList(); - - if (!string.IsNullOrEmpty(filter)) - { - var regex = new Regex(filter); - providerNames = providerNames.Where(p => regex.IsMatch(p)).ToList(); - } - - LogProviderDetailHeader(providerNames); - - foreach (var name in providerNames) - { - LogProviderDetails(dbContext.ProviderDetails.First(p => p.ProviderName == name)); - } - } -} diff --git a/src/EventLogExpert.EventDbTool/ShowLocalCommand.cs b/src/EventLogExpert.EventDbTool/ShowLocalCommand.cs deleted file mode 100644 index 0cf239eb..00000000 --- a/src/EventLogExpert.EventDbTool/ShowLocalCommand.cs +++ /dev/null @@ -1,56 +0,0 @@ -// // Copyright (c) Microsoft Corporation. -// // Licensed under the MIT License. - -using EventLogExpert.Eventing.Helpers; -using EventLogExpert.Eventing.Providers; -using Microsoft.Extensions.DependencyInjection; -using System.CommandLine; - -namespace EventLogExpert.EventDbTool; - -public class ShowLocalCommand(ITraceLogger logger) : DbToolCommand(logger) -{ - public static Command GetCommand() - { - Command showProvidersCommand = new( - "showlocal", - "List the event providers on the local machine."); - - Option filterOption = new("--filter") - { - Description = "Filter for provider names matching the specified regex string." - }; - - Option verboseOption = new("--verbose") - { - Description = "Verbose logging. May be useful for troubleshooting." - }; - - showProvidersCommand.Options.Add(filterOption); - showProvidersCommand.Options.Add(verboseOption); - - showProvidersCommand.SetAction(action => - { - using var sp = Program.BuildServiceProvider(action.GetValue(verboseOption)); - new ShowLocalCommand(sp.GetRequiredService()) - .ShowProviderInfo(action.GetValue(filterOption)); - }); - - return showProvidersCommand; - } - - private void ShowProviderInfo(string? filter) - { - var providerNames = GetLocalProviderNames(filter); - - LogProviderDetailHeader(providerNames); - - foreach (var providerName in providerNames) - { - var provider = new EventMessageProvider(providerName, Logger); - var details = provider.LoadProviderDetails(); - - LogProviderDetails(details); - } - } -} diff --git a/src/EventLogExpert.Eventing.Tests/EventProviderDatabase/EventProviderDbContextTests.cs b/src/EventLogExpert.Eventing.Tests/EventProviderDatabase/EventProviderDbContextTests.cs index 1e32cad0..3c3e2551 100644 --- a/src/EventLogExpert.Eventing.Tests/EventProviderDatabase/EventProviderDbContextTests.cs +++ b/src/EventLogExpert.Eventing.Tests/EventProviderDatabase/EventProviderDbContextTests.cs @@ -157,6 +157,38 @@ public void IsUpgradeNeeded_WithNewDatabase_ShouldReturnFalseFalse() Assert.False(needsV3); } + [Fact] + public void IsUpgradeNeeded_WithV1Schema_ShouldFlagBothUpgrades() + { + // Arrange — V1 schema had no Parameters column; Messages stored as JSON TEXT. + var dbPath = CreateTempDatabasePath(); + SeedLegacySchema(dbPath, includeParameters: false, parametersType: null, messagesType: "TEXT"); + + // Act + using var context = new EventProviderDbContext(dbPath, false); + var (needsV2, needsV3) = context.IsUpgradeNeeded(); + + // Assert + Assert.True(needsV2); + Assert.True(needsV3); + } + + [Fact] + public void IsUpgradeNeeded_WithV2Schema_ShouldFlagBothUpgrades() + { + // Arrange — V2 schema kept TEXT payloads and added Parameters as TEXT. + var dbPath = CreateTempDatabasePath(); + SeedLegacySchema(dbPath, includeParameters: true, parametersType: "TEXT", messagesType: "TEXT"); + + // Act + using var context = new EventProviderDbContext(dbPath, false); + var (needsV2, needsV3) = context.IsUpgradeNeeded(); + + // Assert + Assert.True(needsV2); + Assert.True(needsV3); + } + [Fact] public void Name_ShouldNotIncludeFileExtension() { @@ -189,6 +221,97 @@ public void PerformUpgradeIfNeeded_WithNewDatabase_ShouldDoNothing() Assert.Equal(initialSize, finalSize); } + [Fact] + public void PerformUpgradeIfNeeded_WithV1Schema_ShouldUpgradeAndLeaveParametersEmpty() + { + // Arrange — V1 row has no Parameters column; payloads are JSON TEXT. + var dbPath = CreateTempDatabasePath(); + SeedLegacySchema(dbPath, includeParameters: false, parametersType: null, messagesType: "TEXT"); + InsertLegacyRow( + dbPath, + providerName: "V1Provider", + messagesJson: "[{\"ShortId\":1,\"LogLink\":null,\"RawId\":1,\"Tag\":null,\"Template\":null,\"Text\":\"hello\"}]", + parametersJson: null, + eventsJson: "[]", + keywordsJson: "{}", + opcodesJson: "{}", + tasksJson: "{}"); + + // Act + using (var context = new EventProviderDbContext(dbPath, false)) + { + context.PerformUpgradeIfNeeded(); + } + + // Assert — schema is now V3 and the existing row is preserved with empty Parameters. + using var verify = new EventProviderDbContext(dbPath, true); + var (needsV2After, needsV3After) = verify.IsUpgradeNeeded(); + Assert.False(needsV2After); + Assert.False(needsV3After); + + var row = verify.ProviderDetails.Single(p => p.ProviderName == "V1Provider"); + Assert.Single(row.Messages); + Assert.Equal("hello", row.Messages[0].Text); + Assert.Empty(row.Parameters); + } + + [Fact] + public void PerformUpgradeIfNeeded_WithV2Schema_ShouldPreserveExistingParametersJson() + { + // Arrange — V2 row stores Parameters as JSON TEXT. + var dbPath = CreateTempDatabasePath(); + SeedLegacySchema(dbPath, includeParameters: true, parametersType: "TEXT", messagesType: "TEXT"); + InsertLegacyRow( + dbPath, + providerName: "V2Provider", + messagesJson: "[]", + parametersJson: "[{\"ShortId\":2,\"LogLink\":null,\"RawId\":2,\"Tag\":null,\"Template\":null,\"Text\":\"param-text\"}]", + eventsJson: "[]", + keywordsJson: "{}", + opcodesJson: "{}", + tasksJson: "{}"); + + // Act + using (var context = new EventProviderDbContext(dbPath, false)) + { + context.PerformUpgradeIfNeeded(); + } + + // Assert — Parameters JSON survived the destructive recreate cycle. + using var verify = new EventProviderDbContext(dbPath, true); + var row = verify.ProviderDetails.Single(p => p.ProviderName == "V2Provider"); + Assert.Single(row.Parameters); + Assert.Equal("param-text", row.Parameters.First().Text); + } + + [Fact] + public void PerformUpgradeIfNeeded_WithV2SchemaAndNullParameters_ShouldYieldEmptyParameters() + { + // Arrange — V2 row with NULL Parameters column. + var dbPath = CreateTempDatabasePath(); + SeedLegacySchema(dbPath, includeParameters: true, parametersType: "TEXT", messagesType: "TEXT"); + InsertLegacyRow( + dbPath, + providerName: "V2NullParams", + messagesJson: "[]", + parametersJson: null, + eventsJson: "[]", + keywordsJson: "{}", + opcodesJson: "{}", + tasksJson: "{}"); + + // Act + using (var context = new EventProviderDbContext(dbPath, false)) + { + context.PerformUpgradeIfNeeded(); + } + + // Assert + using var verify = new EventProviderDbContext(dbPath, true); + var row = verify.ProviderDetails.Single(p => p.ProviderName == "V2NullParams"); + Assert.Empty(row.Parameters); + } + [Fact] public void ProviderDetails_Delete_ShouldRemoveRecord() { @@ -536,6 +659,91 @@ private static void DeleteDatabaseFile(string path) } } + private static void InsertLegacyRow( + string dbPath, + string providerName, + string messagesJson, + string? parametersJson, + string eventsJson, + string keywordsJson, + string opcodesJson, + string tasksJson) + { + using var connection = new SqliteConnection($"Data Source={dbPath}"); + connection.Open(); + + // Detect whether the legacy schema includes Parameters; insert the right column list. + bool hasParameters; + using (var pragma = connection.CreateCommand()) + { + pragma.CommandText = "PRAGMA table_info(\"ProviderDetails\")"; + using var pr = pragma.ExecuteReader(); + hasParameters = false; + while (pr.Read()) + { + if (string.Equals(pr["name"]?.ToString(), "Parameters", StringComparison.Ordinal)) + { + hasParameters = true; + break; + } + } + } + + using var cmd = connection.CreateCommand(); + if (hasParameters) + { + cmd.CommandText = "INSERT INTO \"ProviderDetails\" (\"ProviderName\", \"Messages\", \"Events\", \"Keywords\", \"Opcodes\", \"Tasks\", \"Parameters\") " + + "VALUES ($name, $messages, $events, $keywords, $opcodes, $tasks, $parameters)"; + cmd.Parameters.AddWithValue("$parameters", (object?)parametersJson ?? DBNull.Value); + } + else + { + cmd.CommandText = "INSERT INTO \"ProviderDetails\" (\"ProviderName\", \"Messages\", \"Events\", \"Keywords\", \"Opcodes\", \"Tasks\") " + + "VALUES ($name, $messages, $events, $keywords, $opcodes, $tasks)"; + } + + cmd.Parameters.AddWithValue("$name", providerName); + cmd.Parameters.AddWithValue("$messages", messagesJson); + cmd.Parameters.AddWithValue("$events", eventsJson); + cmd.Parameters.AddWithValue("$keywords", keywordsJson); + cmd.Parameters.AddWithValue("$opcodes", opcodesJson); + cmd.Parameters.AddWithValue("$tasks", tasksJson); + cmd.ExecuteNonQuery(); + } + + private static void SeedLegacySchema( + string dbPath, + bool includeParameters, + string? parametersType, + string messagesType) + { + // Build a legacy ProviderDetails table directly via raw SQLite, before any + // EventProviderDbContext touches the file. EnsureCreated() will then be a no-op + // because the table already exists, so the legacy schema reaches IsUpgradeNeeded + // and PerformUpgradeIfNeeded unmodified. + using var connection = new SqliteConnection($"Data Source={dbPath}"); + connection.Open(); + + var columns = new List + { + "\"ProviderName\" TEXT NOT NULL CONSTRAINT \"PK_ProviderDetails\" PRIMARY KEY", + $"\"Messages\" {messagesType} NOT NULL", + $"\"Events\" {messagesType} NOT NULL", + $"\"Keywords\" {messagesType} NOT NULL", + $"\"Opcodes\" {messagesType} NOT NULL", + $"\"Tasks\" {messagesType} NOT NULL" + }; + + if (includeParameters) + { + columns.Add($"\"Parameters\" {parametersType}"); + } + + using var cmd = connection.CreateCommand(); + cmd.CommandText = $"CREATE TABLE \"ProviderDetails\" ({string.Join(", ", columns)})"; + cmd.ExecuteNonQuery(); + } + private string CreateTempDatabasePath() { var path = Path.Combine(Path.GetTempPath(), $"test_db_{Guid.NewGuid()}.db"); diff --git a/src/EventLogExpert.Eventing.Tests/Readers/EventLogReaderTests.cs b/src/EventLogExpert.Eventing.Tests/Readers/EventLogReaderTests.cs index ab5b8db1..48fa8b18 100644 --- a/src/EventLogExpert.Eventing.Tests/Readers/EventLogReaderTests.cs +++ b/src/EventLogExpert.Eventing.Tests/Readers/EventLogReaderTests.cs @@ -163,6 +163,65 @@ public void Dispose_WhenCalledMultipleTimes_ShouldNotThrow() reader.Dispose(); } + [Fact] + public void EndOfResults_AfterExhaustion_ShouldKeepBookmarkAndNotSetLastErrorCode() + { + // Use a small bounded .evtx fixture (max 5 events exported via wevtutil) so the + // exhaustion loop completes quickly regardless of the host machine's Application log + // size. Reading the live Application log here previously made the test runtime + // unbounded and CI-flaky. + + // Arrange + using var fixture = new SmallEvtxFixture(); + using var reader = new EventLogReader(fixture.FilePath, PathType.FilePath); + + // Act — exhaust the reader so TryGetEvents returns false with ERROR_NO_MORE_ITEMS + while (reader.TryGetEvents(out _)) { } + + string? bookmarkAfterExhaustion = reader.LastBookmark; + + // One additional read past end-of-results + reader.TryGetEvents(out _); + string? bookmarkAfterExtraRead = reader.LastBookmark; + + // Assert — bookmark is stable past EOF, and normal end-of-results does NOT set LastErrorCode + Assert.Equal(bookmarkAfterExhaustion, bookmarkAfterExtraRead); + Assert.Null(reader.LastErrorCode); + } + + [Fact] + public void IsValid_WhenApplicationLog_ShouldBeTrue() + { + // Arrange & Act + using var reader = new EventLogReader(Constants.ApplicationLogName, PathType.LogName); + + // Assert + Assert.True(reader.IsValid); + } + + [Fact] + public void IsValid_WhenEmptyLogName_ShouldBeFalse() + { + // Arrange & Act + using var reader = new EventLogReader(string.Empty, PathType.LogName); + + // Assert + Assert.False(reader.IsValid); + } + + [Fact] + public void IsValid_WhenInvalidLogName_ShouldBeFalse() + { + // Arrange + var invalidLogName = "NonExistentLog_" + Guid.NewGuid(); + + // Act + using var reader = new EventLogReader(invalidLogName, PathType.LogName); + + // Assert + Assert.False(reader.IsValid); + } + [Fact] public void LastBookmark_AfterTryGetEvents_ShouldBeSet() { @@ -221,22 +280,27 @@ public void LastBookmark_WhenMultipleBatches_ShouldUpdateWithEachBatch() } [Fact] - public void LastBookmark_WhenNoEventsReturned_ShouldRemainUnchanged() + public void LastErrorCode_WhenInitialized_ShouldBeNull() { - // Arrange + // Arrange & Act using var reader = new EventLogReader(Constants.ApplicationLogName, PathType.LogName); - // Read all events - while (reader.TryGetEvents(out _)) { } + // Assert + Assert.Null(reader.LastErrorCode); + } - string? bookmarkBefore = reader.LastBookmark; + [Fact] + public void LastErrorCode_WhenInvalidHandle_ShouldBeSet() + { + // Arrange — opening a non-existent log produces an invalid handle + var invalidLogName = "NonExistentLog_" + Guid.NewGuid(); + using var reader = new EventLogReader(invalidLogName, PathType.LogName); - // Act - Try to read when no events left + // Act reader.TryGetEvents(out _); - string? bookmarkAfter = reader.LastBookmark; - // Assert - Assert.Equal(bookmarkBefore, bookmarkAfter); + // Assert — the failure is not a normal EOF, so LastErrorCode should be set + Assert.NotNull(reader.LastErrorCode); } [Fact] diff --git a/src/EventLogExpert.Eventing.Tests/Readers/SmallEvtxFixture.cs b/src/EventLogExpert.Eventing.Tests/Readers/SmallEvtxFixture.cs new file mode 100644 index 00000000..90d0e496 --- /dev/null +++ b/src/EventLogExpert.Eventing.Tests/Readers/SmallEvtxFixture.cs @@ -0,0 +1,73 @@ +// // Copyright (c) Microsoft Corporation. +// // Licensed under the MIT License. + +using System.Diagnostics; + +namespace EventLogExpert.Eventing.Tests.Readers; + +/// +/// Creates a small temporary .evtx file by exporting at most 5 events from the local +/// Application log. Used by tests that need to exercise end-of-results behavior without +/// scanning the entire (potentially millions of records) Application log. +/// +internal sealed class SmallEvtxFixture : IDisposable +{ + public SmallEvtxFixture() + { + FilePath = Path.Combine(Path.GetTempPath(), $"elx-tests-{Guid.NewGuid():N}.evtx"); + + // Use the absolute path to wevtutil.exe rather than relying on PATH so the fixture is + // robust on machines where %PATH% has been customized. + var wevtutilPath = Path.Combine(Environment.SystemDirectory, "wevtutil.exe"); + + // /q with an EventRecordID range bounds the export to at most 5 events. wevtutil epl + // does not support a /count switch, so XPath is the supported way to cap output. + var psi = new ProcessStartInfo + { + FileName = wevtutilPath, + ArgumentList = + { + "epl", + "Application", + FilePath, + "/q:*[System[EventRecordID>=1 and EventRecordID<=5]]" + }, + UseShellExecute = false, + RedirectStandardError = true, + RedirectStandardOutput = true, + CreateNoWindow = true + }; + + using var proc = Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start wevtutil.exe."); + + if (!proc.WaitForExit(TimeSpan.FromSeconds(30))) + { + try { proc.Kill(entireProcessTree: true); } catch { /* best effort */ } + + throw new InvalidOperationException("wevtutil.exe did not complete within 30 seconds."); + } + + if (proc.ExitCode != 0) + { + var err = proc.StandardError.ReadToEnd(); + + throw new InvalidOperationException( + $"wevtutil.exe exited with code {proc.ExitCode}. Stderr: {err}"); + } + } + + public string FilePath { get; } + + public void Dispose() + { + try + { + if (File.Exists(FilePath)) { File.Delete(FilePath); } + } + catch + { + // Best-effort cleanup: a temp file left behind should not fail the test. + } + } +} diff --git a/src/EventLogExpert.Eventing/EventProviderDatabase/EventProviderDbContext.cs b/src/EventLogExpert.Eventing/EventProviderDatabase/EventProviderDbContext.cs index dcd7cd7b..320ccd02 100644 --- a/src/EventLogExpert.Eventing/EventProviderDatabase/EventProviderDbContext.cs +++ b/src/EventLogExpert.Eventing/EventProviderDatabase/EventProviderDbContext.cs @@ -5,15 +5,15 @@ using EventLogExpert.Eventing.Models; using EventLogExpert.Eventing.Providers; using Microsoft.EntityFrameworkCore; +using System.Data; using System.Text.Json; namespace EventLogExpert.Eventing.EventProviderDatabase; public sealed class EventProviderDbContext : DbContext { - private readonly bool _readOnly; - private readonly ITraceLogger? _logger; + private readonly bool _readOnly; public EventProviderDbContext(string path, bool readOnly, ITraceLogger? logger = null) { @@ -30,73 +30,72 @@ public EventProviderDbContext(string path, bool readOnly, ITraceLogger? logger = public string Name { get; } - private string Path { get; } - public DbSet ProviderDetails { get; set; } - protected override void OnConfiguring(DbContextOptionsBuilder options) - => options.UseSqlite($"Data Source={Path};Mode={(_readOnly ? "ReadOnly" : "ReadWriteCreate")}"); - - protected override void OnModelCreating(ModelBuilder modelBuilder) - { - modelBuilder.Entity() - .HasKey(e => e.ProviderName); - - modelBuilder.Entity() - .Property(e => e.Messages) - .HasConversion>>(); - - modelBuilder.Entity() - .Property(e => e.Parameters) - .HasConversion>>(); - - modelBuilder.Entity() - .Property(e => e.Events) - .HasConversion>>(); - - modelBuilder.Entity() - .Property(e => e.Keywords) - .HasConversion>>(); - - modelBuilder.Entity() - .Property(e => e.Opcodes) - .HasConversion>>(); - - modelBuilder.Entity() - .Property(e => e.Tasks) - .HasConversion>>(); - } + private string Path { get; } public (bool needsV2Upgrade, bool needsV3Upgrade) IsUpgradeNeeded() { + // Use PRAGMA table_info instead of substring-matching sqlite_schema.sql. The previous + // approach matched the literal text `"Parameters" BLOB NOT NULL`, which is brittle across + // EF Core / Microsoft.Data.Sqlite versions (whitespace, quoting, constraint ordering, case) + // and would silently flip a fresh V3 database into the upgrade-needed path, causing an + // unnecessary drop/vacuum/recreate cycle. var connection = Database.GetDbConnection(); Database.OpenConnection(); - using var command = connection.CreateCommand(); - command.CommandText = "SELECT * FROM sqlite_schema"; - using var reader = command.ExecuteReader(); - var needsV2Upgrade = false; - var needsV3Upgrade = true; - while (reader.Read()) + + try { - var val = reader["sql"]?.ToString(); - if (val?.Contains("\"Messages\" TEXT NOT NULL") ?? false) - { - needsV2Upgrade = true; - } + using var command = connection.CreateCommand(); + command.CommandText = "PRAGMA table_info(\"ProviderDetails\")"; + using var reader = command.ExecuteReader(); + + string? messagesType = null; + string? parametersType = null; + var hasAnyColumn = false; - if (val?.Contains("\"Parameters\" BLOB NOT NULL") ?? false) + while (reader.Read()) { - needsV3Upgrade = false; + hasAnyColumn = true; + + var name = reader["name"]?.ToString(); + var type = reader["type"]?.ToString(); + + if (string.Equals(name, "Messages", StringComparison.Ordinal)) + { + messagesType = type; + } + else if (string.Equals(name, "Parameters", StringComparison.Ordinal)) + { + parametersType = type; + } } - } - reader.Close(); + reader.Close(); - var needsUpgrade = needsV2Upgrade || needsV3Upgrade; + // V2 schema stored payload columns as JSON-encoded TEXT. V3 stores them as compressed BLOB. + // V1 had no Parameters column at all; that case naturally falls into needsV3Upgrade=true + // because parametersType remains null below. + var messagesIsText = string.Equals(messagesType?.Trim(), "TEXT", StringComparison.OrdinalIgnoreCase); + var parametersIsBlob = string.Equals(parametersType?.Trim(), "BLOB", StringComparison.OrdinalIgnoreCase); - _logger?.Debug($"{nameof(EventProviderDbContext)}.{nameof(IsUpgradeNeeded)}() for database {Path}. needsV2Upgrade: {needsV2Upgrade} needsV3Upgrade: {needsV3Upgrade}"); + // Only flag upgrades when the ProviderDetails table actually exists. If it does not, the + // database is either freshly created (EnsureCreated already ran in the constructor) or + // unrelated to this tool — neither case warrants the destructive drop/vacuum/recreate path. + var needsV2Upgrade = hasAnyColumn && messagesIsText; + var needsV3Upgrade = hasAnyColumn && !parametersIsBlob; - return (needsV2Upgrade, needsV3Upgrade); + _logger?.Debug($"{nameof(EventProviderDbContext)}.{nameof(IsUpgradeNeeded)}() for database {Path}. needsV2Upgrade: {needsV2Upgrade} needsV3Upgrade: {needsV3Upgrade}"); + + return (needsV2Upgrade, needsV3Upgrade); + } + finally + { + // Always close the connection we opened explicitly — leaving it open keeps the SQLite + // file locked, blocking other operations (including FileInfo.Length reads on some + // platforms) and undermining EF's normal short-lived-connection lifecycle. + Database.CloseConnection(); + } } public void PerformUpgradeIfNeeded() @@ -114,55 +113,67 @@ public void PerformUpgradeIfNeeded() var connection = Database.GetDbConnection(); Database.OpenConnection(); - using var command = connection.CreateCommand(); var allProviderDetails = new List(); - command.CommandText = "SELECT * FROM \"ProviderDetails\""; - var detailsReader = command.ExecuteReader(); - - if (needsV2Upgrade) + try { - while (detailsReader.Read()) + using var command = connection.CreateCommand(); + + command.CommandText = "SELECT * FROM \"ProviderDetails\""; + + using (var detailsReader = command.ExecuteReader()) { - var p = new ProviderDetails + if (needsV2Upgrade) { - ProviderName = (string)detailsReader["ProviderName"], - Messages = JsonSerializer.Deserialize>((string)detailsReader["Messages"]) ?? new List(), - Parameters = new List(), - Events = JsonSerializer.Deserialize>((string)detailsReader["Events"]) ?? new List(), - Keywords = JsonSerializer.Deserialize>((string)detailsReader["Keywords"]) ?? new Dictionary(), - Opcodes = JsonSerializer.Deserialize>((string)detailsReader["Opcodes"]) ?? new Dictionary(), - Tasks = JsonSerializer.Deserialize>((string)detailsReader["Tasks"]) ?? new Dictionary() - }; - allProviderDetails.Add(p); + while (detailsReader.Read()) + { + var providerName = (string)detailsReader["ProviderName"]; + var p = new ProviderDetails + { + ProviderName = providerName, + Messages = JsonSerializer.Deserialize>((string)detailsReader["Messages"]) ?? new List(), + Parameters = TryReadParametersJson(detailsReader, providerName), + Events = JsonSerializer.Deserialize>((string)detailsReader["Events"]) ?? new List(), + Keywords = JsonSerializer.Deserialize>((string)detailsReader["Keywords"]) ?? new Dictionary(), + Opcodes = JsonSerializer.Deserialize>((string)detailsReader["Opcodes"]) ?? new Dictionary(), + Tasks = JsonSerializer.Deserialize>((string)detailsReader["Tasks"]) ?? new Dictionary() + }; + allProviderDetails.Add(p); + } + } + else + { + while (detailsReader.Read()) + { + var providerName = (string)detailsReader["ProviderName"]; + var p = new ProviderDetails + { + ProviderName = providerName, + Messages = CompressedJsonValueConverter>.ConvertFromCompressedJson((byte[])detailsReader["Messages"]) ?? new List(), + Parameters = TryReadParametersJson(detailsReader, providerName), + Events = CompressedJsonValueConverter>.ConvertFromCompressedJson((byte[])detailsReader["Events"]) ?? new List(), + Keywords = CompressedJsonValueConverter>.ConvertFromCompressedJson((byte[])detailsReader["Keywords"]) ?? new Dictionary(), + Opcodes = CompressedJsonValueConverter>.ConvertFromCompressedJson((byte[])detailsReader["Opcodes"]) ?? new Dictionary(), + Tasks = CompressedJsonValueConverter>.ConvertFromCompressedJson((byte[])detailsReader["Tasks"]) ?? new Dictionary() + }; + allProviderDetails.Add(p); + } + } } + + command.CommandText = "DROP TABLE \"ProviderDetails\""; + command.ExecuteNonQuery(); + command.CommandText = "VACUUM"; + command.ExecuteNonQuery(); } - else + finally { - while (detailsReader.Read()) - { - var p = new ProviderDetails - { - ProviderName = (string)detailsReader["ProviderName"], - Messages = CompressedJsonValueConverter>.ConvertFromCompressedJson((byte[])detailsReader["Messages"]) ?? new List(), - Parameters = new List(), - Events = CompressedJsonValueConverter>.ConvertFromCompressedJson((byte[])detailsReader["Events"]) ?? new List(), - Keywords = CompressedJsonValueConverter>.ConvertFromCompressedJson((byte[])detailsReader["Keywords"]) ?? new Dictionary(), - Opcodes = CompressedJsonValueConverter>.ConvertFromCompressedJson((byte[])detailsReader["Opcodes"]) ?? new Dictionary(), - Tasks = CompressedJsonValueConverter>.ConvertFromCompressedJson((byte[])detailsReader["Tasks"]) ?? new Dictionary() - }; - allProviderDetails.Add(p); - } + // SaveChanges() below will reopen the connection on demand; releasing it here keeps the + // explicit open/close balanced and prevents leaking the lock if SaveChanges throws. + Database.CloseConnection(); } - detailsReader.Close(); - - command.CommandText = "DROP TABLE \"ProviderDetails\""; - command.ExecuteNonQuery(); - command.CommandText = "VACUUM"; - command.ExecuteNonQuery(); - Database.EnsureCreated(); foreach (var p in allProviderDetails) @@ -176,4 +187,81 @@ public void PerformUpgradeIfNeeded() _logger?.Info($"EventProviderDbContext upgrade completed. Size: {size} Path: {Path}"); } + + protected override void OnConfiguring(DbContextOptionsBuilder options) + => options.UseSqlite($"Data Source={Path};Mode={(_readOnly ? "ReadOnly" : "ReadWriteCreate")}"); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity() + .HasKey(e => e.ProviderName); + + modelBuilder.Entity() + .Property(e => e.Messages) + .HasConversion>>(); + + modelBuilder.Entity() + .Property(e => e.Parameters) + .HasConversion>>(); + + modelBuilder.Entity() + .Property(e => e.Events) + .HasConversion>>(); + + modelBuilder.Entity() + .Property(e => e.Keywords) + .HasConversion>>(); + + modelBuilder.Entity() + .Property(e => e.Opcodes) + .HasConversion>>(); + + modelBuilder.Entity() + .Property(e => e.Tasks) + .HasConversion>>(); + } + + /// + /// Reads the optional Parameters column from the pre-upgrade row, if present, and + /// deserializes the JSON payload. V1 schemas have no Parameters column at all (returns empty + /// list); V2 stored it as JSON-encoded TEXT (preserved here). Any failure to parse logs a + /// warning so silent data loss is diagnosable instead of opaque. + /// + private List TryReadParametersJson(IDataReader reader, string providerName) + { + for (var i = 0; i < reader.FieldCount; i++) + { + if (!string.Equals(reader.GetName(i), "Parameters", StringComparison.Ordinal)) + { + continue; + } + + if (reader.IsDBNull(i)) + { + return []; + } + + var raw = reader.GetValue(i); + + if (raw is string s) + { + if (string.IsNullOrEmpty(s)) { return []; } + + try + { + return JsonSerializer.Deserialize>(s) ?? []; + } + catch (JsonException ex) + { + _logger?.Warn($"EventProviderDbContext upgrade: failed to deserialize Parameters JSON for provider '{providerName}' in {Path}: {ex.Message}. Parameters will be empty after upgrade."); + return []; + } + } + + _logger?.Warn($"EventProviderDbContext upgrade: Parameters column for provider '{providerName}' in {Path} is of unexpected type '{raw.GetType().Name}'. Parameters will be empty after upgrade."); + return []; + } + + return []; + } } diff --git a/src/EventLogExpert.Eventing/Readers/EventLogReader.cs b/src/EventLogExpert.Eventing/Readers/EventLogReader.cs index 8119b54f..c957ee9a 100644 --- a/src/EventLogExpert.Eventing/Readers/EventLogReader.cs +++ b/src/EventLogExpert.Eventing/Readers/EventLogReader.cs @@ -16,8 +16,23 @@ public sealed partial class EventLogReader(string path, PathType pathType, bool private int _disposed; + /// + /// when the underlying EvtQuery handle was opened successfully. + /// When , the path could not be opened (invalid log name, missing or + /// corrupt .evtx file, access denied, etc.) and will not return events. + /// + public bool IsValid => _handle is { IsInvalid: false }; + public string? LastBookmark { get; private set; } + /// + /// When returns due to a Win32 error other + /// than ERROR_NO_MORE_ITEMS, this property contains the Win32 error code. A value of + /// means either no error occurred or the last failure was a normal + /// end-of-results condition. + /// + public int? LastErrorCode { get; private set; } + public void Dispose() { // Use Interlocked.CompareExchange for atomic check-and-set. @@ -50,10 +65,14 @@ public bool TryGetEvents(out EventRecord[] events, int batchSize = 30) if (!success) { + var error = Marshal.GetLastWin32Error(); + LastErrorCode = error != Interop.ERROR_NO_MORE_ITEMS ? error : null; events = []; return false; } + LastErrorCode = null; + using (_eventLock.EnterScope()) { LastBookmark = CreateBookmark(new EvtHandle(buffer[count - 1], false));