Skip to content

Commit be9681d

Browse files
jschick04jschick04
committed
Fixed CI issue related to needs upgrade checks
1 parent 64dc8ce commit be9681d

File tree

11 files changed

+454
-187
lines changed

11 files changed

+454
-187
lines changed

‎src/EventLogExpert.EventDbTool/CreateDatabaseCommand.cs‎

Lines changed: 57 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using EventLogExpert.Eventing.Providers;
77
using Microsoft.Extensions.DependencyInjection;
88
using System.CommandLine;
9+
using System.Text.RegularExpressions;
910

1011
namespace EventLogExpert.EventDbTool;
1112

@@ -83,50 +84,78 @@ private void CreateDatabase(string path, string? source, string? filter, string?
8384
return;
8485
}
8586

86-
if (source is not null && !ProviderSource.TryValidate(source, Logger)) { return; }
87+
if (!RegexHelper.TryCreate(filter, Logger, out _)) { return; }
8788

88-
HashSet<string> skipProviderNames = new(StringComparer.OrdinalIgnoreCase);
89+
if (source is not null && !ProviderSource.TryValidate(source, Logger)) { return; }
8990

90-
if (!string.IsNullOrWhiteSpace(skipProvidersInFile))
91+
try
9192
{
92-
if (!ProviderSource.TryValidate(skipProvidersInFile, Logger)) { return; }
93+
HashSet<string> skipProviderNames = new(StringComparer.OrdinalIgnoreCase);
9394

94-
foreach (var name in ProviderSource.LoadProviderNames(skipProvidersInFile, Logger))
95+
if (!string.IsNullOrWhiteSpace(skipProvidersInFile))
9596
{
96-
skipProviderNames.Add(name);
97+
if (!ProviderSource.TryValidate(skipProvidersInFile, Logger)) { return; }
98+
99+
foreach (var name in ProviderSource.LoadProviderNames(skipProvidersInFile, Logger))
100+
{
101+
skipProviderNames.Add(name);
102+
}
103+
104+
Logger.Info($"Found {skipProviderNames.Count} providers in {skipProvidersInFile}. These will not be included in the new database.");
97105
}
98106

99-
Logger.Info($"Found {skipProviderNames.Count} providers in {skipProvidersInFile}. These will not be included in the new database.");
100-
}
107+
// Load provider names first (cheap string-only query) for the empty check and header
108+
// formatting. This avoids materializing all ProviderDetails (with large compressed
109+
// payloads) just to compute the column widths.
110+
IReadOnlyList<string> providerNames = source is null
111+
? GetLocalProviderNames(filter, Logger)
112+
.Where(n => !skipProviderNames.Contains(n)).ToList()
113+
: ProviderSource.LoadProviderNames(source, Logger, filter)
114+
.Where(n => !skipProviderNames.Contains(n)).ToList();
101115

102-
IEnumerable<ProviderDetails> providersToAdd = source is null
103-
? LoadLocalProviders(filter, skipProviderNames)
104-
: ProviderSource.LoadProviders(source, Logger, filter, skipProviderNames);
116+
if (providerNames.Count == 0)
117+
{
118+
Logger.Warn($"No providers to add to the new database.");
119+
return;
120+
}
105121

106-
var providersNotSkipped = providersToAdd.ToList();
122+
using var dbContext = new EventProviderDbContext(path, false, Logger);
107123

108-
if (providersNotSkipped.Count == 0)
109-
{
110-
Logger.Warn($"No providers to add to the new database.");
111-
return;
112-
}
124+
LogProviderDetailHeader(providerNames);
113125

114-
using var dbContext = new EventProviderDbContext(path, false, Logger);
126+
// Stream details directly into the DbContext. Batch saves prevent the change tracker
127+
// from accumulating all entities in memory at once.
128+
const int batchSize = 100;
129+
var count = 0;
115130

116-
LogProviderDetailHeader(providersNotSkipped.Select(p => p.ProviderName));
131+
IEnumerable<ProviderDetails> providersToAdd = source is null
132+
? LoadLocalProviders(filter, skipProviderNames)
133+
: ProviderSource.LoadProviders(source, Logger, filter, skipProviderNames);
117134

118-
foreach (var details in providersNotSkipped)
119-
{
120-
dbContext.ProviderDetails.Add(details);
121-
LogProviderDetails(details);
122-
}
135+
foreach (var details in providersToAdd)
136+
{
137+
dbContext.ProviderDetails.Add(details);
138+
LogProviderDetails(details);
139+
count++;
140+
141+
if (count % batchSize == 0)
142+
{
143+
dbContext.SaveChanges();
144+
dbContext.ChangeTracker.Clear();
145+
}
146+
}
123147

124-
Logger.Info($"");
125-
Logger.Info($"Saving database. Please wait...");
148+
Logger.Info($"");
149+
Logger.Info($"Saving database. Please wait...");
126150

127-
dbContext.SaveChanges();
151+
dbContext.SaveChanges();
128152

129-
Logger.Info($"Done!");
153+
Logger.Info($"Done!");
154+
}
155+
catch (RegexMatchTimeoutException)
156+
{
157+
Logger.Error($"The --filter regex timed out. The pattern may cause catastrophic backtracking.");
158+
}
130159
}
131160

132161
}

‎src/EventLogExpert.EventDbTool/DbToolCommand.cs‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
using EventLogExpert.Eventing.Helpers;
55
using EventLogExpert.Eventing.Providers;
66
using EventLogExpert.Eventing.Readers;
7-
using System.Text.RegularExpressions;
87

98
namespace EventLogExpert.EventDbTool;
109

@@ -14,21 +13,21 @@ public class DbToolCommand(ITraceLogger logger)
1413

1514
protected ITraceLogger Logger => logger;
1615

17-
protected static List<string> GetLocalProviderNames(string? filter)
16+
protected static List<string> GetLocalProviderNames(string? filter, ITraceLogger logger)
1817
{
1918
var providers = new List<string>(EventLogSession.GlobalSession.GetProviderNames().Distinct().OrderBy(name => name));
2019

21-
if (string.IsNullOrEmpty(filter)) { return providers; }
22-
23-
var regex = new Regex(filter, RegexOptions.IgnoreCase);
24-
providers = providers.Where(p => regex.IsMatch(p)).ToList();
20+
if (!RegexHelper.TryCreate(filter, logger, out var regex))
21+
{
22+
return [];
23+
}
2524

26-
return providers;
25+
return regex is null ? providers : providers.Where(p => regex.IsMatch(p)).ToList();
2726
}
2827

2928
protected IEnumerable<ProviderDetails> LoadLocalProviders(string? filter, IReadOnlySet<string>? skipProviderNames = null)
3029
{
31-
foreach (var providerName in GetLocalProviderNames(filter))
30+
foreach (var providerName in GetLocalProviderNames(filter, Logger))
3231
{
3332
// Skip BEFORE resolving so we don't pay the cost of loading metadata for providers we
3433
// are about to discard (e.g. when --skip-providers-in-file lists most local providers).

‎src/EventLogExpert.EventDbTool/DiffDatabaseCommand.cs‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,14 +81,13 @@ private void DiffDatabase(string firstSource, string secondSource, string newDb)
8181

8282
using var newDbContext = new EventProviderDbContext(newDb, false, Logger);
8383

84-
foreach (var details in ProviderSource.LoadProviders(secondSource, Logger))
85-
{
86-
if (firstProviderNames.Contains(details.ProviderName))
87-
{
88-
Logger.Info($"Skipping {details.ProviderName} because it is present in both sources.");
89-
continue;
90-
}
84+
// Pass firstProviderNames as the skip set so providers present in the first source are
85+
// never resolved from the second source's metadata path. This is especially important when
86+
// the second source is .evtx+MTA, where each provider triggers an expensive load.
87+
Logger.Info($"Skipping up to {firstProviderNames.Count} provider name(s) from the second source that also appear in the first source.");
9188

89+
foreach (var details in ProviderSource.LoadProviders(secondSource, Logger, filter: null, skipProviderNames: firstProviderNames))
90+
{
9291
Logger.Info($"Copying {details.ProviderName} because it is present in second source but not first.");
9392

9493
newDbContext.ProviderDetails.Add(new ProviderDetails

‎src/EventLogExpert.EventDbTool/MergeDatabaseCommand.cs‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -104,15 +104,25 @@ private void MergeDatabase(string source, string targetFile, bool overwriteProvi
104104
{
105105
Logger.Info($"Removing these providers from the target database...");
106106

107-
// Single round-trip to load just the rows we need to remove. Since these names came
108-
// from the same DB, exact (binary) matching here is correct.
109-
var toRemove = targetContext.ProviderDetails
110-
.Where(p => targetMatchingNames.Contains(p.ProviderName))
111-
.ToList();
112-
113-
targetContext.RemoveRange(toRemove);
114-
targetContext.SaveChanges();
115-
Logger.Info($"Removal of {toRemove.Count} provider row(s) completed.");
107+
// Chunk the IN-clause to stay below SQLite's parameter limit (default 999). Without
108+
// chunking, an --overwrite of a large overlap could throw at runtime.
109+
// ExecuteDelete() issues a SQL DELETE directly, avoiding change-tracker overhead.
110+
const int maxInClauseParameters = 500;
111+
var removed = 0;
112+
113+
for (var offset = 0; offset < targetMatchingNames.Count; offset += maxInClauseParameters)
114+
{
115+
var chunk = targetMatchingNames
116+
.Skip(offset)
117+
.Take(maxInClauseParameters)
118+
.ToList();
119+
120+
removed += targetContext.ProviderDetails
121+
.Where(p => chunk.Contains(p.ProviderName))
122+
.ExecuteDelete();
123+
}
124+
125+
Logger.Info($"Removal of {removed} provider row(s) completed.");
116126
}
117127
else
118128
{

‎src/EventLogExpert.EventDbTool/MtaProviderSource.cs‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,13 @@ internal static class MtaProviderSource
2222
public static IReadOnlyList<string> DiscoverProviderNames(
2323
string evtxPath,
2424
ITraceLogger logger,
25-
string? filter = null)
25+
string? filter = null) =>
26+
!RegexHelper.TryCreate(filter, logger, out var regex) ? [] : DiscoverProviderNamesCore(evtxPath, logger, regex);
27+
28+
private static IReadOnlyList<string> DiscoverProviderNamesCore(
29+
string evtxPath,
30+
ITraceLogger logger,
31+
Regex? regex)
2632
{
2733
if (!File.Exists(evtxPath))
2834
{
@@ -42,10 +48,11 @@ public static IReadOnlyList<string> DiscoverProviderNames(
4248
return [];
4349
}
4450

51+
// TryGetEvents returns false both for normal end-of-results (ERROR_NO_MORE_ITEMS) and
52+
// for read errors (corruption, access denied, etc.). Check LastErrorCode to surface
53+
// non-terminal failures so users can distinguish "0 events" from "could not read the log".
4554
while (reader.TryGetEvents(out var batch))
4655
{
47-
if (batch.Length == 0) { break; }
48-
4956
foreach (var record in batch)
5057
{
5158
if (!string.IsNullOrEmpty(record.ProviderName))
@@ -54,21 +61,21 @@ public static IReadOnlyList<string> DiscoverProviderNames(
5461
}
5562
}
5663
}
64+
65+
if (reader.LastErrorCode is not null)
66+
{
67+
logger.Warn(
68+
$"Reading {evtxPath} may be incomplete. " +
69+
$"EvtNext failed with Win32 error code {reader.LastErrorCode}.");
70+
}
5771
}
5872
catch (Exception ex)
5973
{
6074
logger.Error($"Failed to read events from {evtxPath}: {ex.Message}");
6175
return [];
6276
}
6377

64-
if (string.IsNullOrEmpty(filter))
65-
{
66-
return providerNames.ToList();
67-
}
68-
69-
var regex = new Regex(filter, RegexOptions.IgnoreCase);
70-
71-
return providerNames.Where(n => regex.IsMatch(n)).ToList();
78+
return regex is null ? providerNames.ToList() : providerNames.Where(n => regex.IsMatch(n)).ToList();
7279
}
7380

7481
/// <summary>
@@ -122,7 +129,8 @@ public static IEnumerable<ProviderDetails> LoadProviders(
122129
string evtxPath,
123130
ITraceLogger logger,
124131
string? filter = null) =>
125-
LoadProvidersCore(evtxPath, logger, string.IsNullOrEmpty(filter) ? null : new Regex(filter, RegexOptions.IgnoreCase), null, null);
132+
!RegexHelper.TryCreate(filter, logger, out var regex) ? [] :
133+
LoadProvidersCore(evtxPath, logger, regex, null, null);
126134

127135
/// <summary>
128136
/// Internal overload used by <see cref="ProviderSource" /> so name-based filtering and the de-dup
@@ -151,7 +159,7 @@ private static IEnumerable<ProviderDetails> LoadProvidersCore(
151159
IReadOnlySet<string>? skipProviderNames,
152160
HashSet<string>? seen)
153161
{
154-
var providerNames = DiscoverProviderNames(evtxPath, logger);
162+
var providerNames = DiscoverProviderNamesCore(evtxPath, logger, regex: null);
155163

156164
if (providerNames.Count == 0) { yield break; }
157165

0 commit comments

Comments
 (0)