[main] GRD-122972: Add support for MongoDB shell regex

filter-plugin/logstash-filter-documentdb-aws-guardium/CHANGELOG.md

@@ -2,6 +2,9 @@
 
 Notable changes will be documented in this file.
 
+## [0.1.25]
+- GRD-122972: Add support for MongoDB shell regex
+
 ## [0.1.24]
 - GRD-126808: Add a support for MongoStyle audit logs
 

filter-plugin/logstash-filter-documentdb-aws-guardium/README.md

@@ -116,8 +116,9 @@ The Guardium universal connector is the Guardium entry point for native audit/pr
 
 ## 6. Limitations
 - Due to native limitations in Amazon DocumentDB, audit logs may truncate query details (1 KB limit), and profiler logs only capture slow queries (based on a configurable threshold, around 50 ms). As a result, full query visibility cannot be guaranteed.
-    - https://docs.aws.amazon.com/documentdb/latest/developerguide/event-auditing.html
-    - https://docs.aws.amazon.com/documentdb/latest/developerguide/profiling.html
+- DocumentDB writes regex arguments as `/pattern/flags` literals (MongoDB shell syntax), which are not valid JSON. The plugin automatically quotes these literals before parsing so that queries containing regular expressions are handled correctly.
+  - https://docs.aws.amazon.com/documentdb/latest/developerguide/event-auditing.html
+  - https://docs.aws.amazon.com/documentdb/latest/developerguide/profiling.html
 - DocumentDB Profiler logs capture any database operations that take longer than some period of time(e. g. 100 ms). If the threshold value is not configurable and set value is too high, then profiler logs may not get captured for every database operation.
 - The Following important fields couldn't be mapped with DocumentDB audit/profiler logs
   - Source program : Only available in case of "aggregate" query

filter-plugin/logstash-filter-documentdb-aws-guardium/VERSION

@@ -1 +1 @@
-0.1.24
+0.1.25

filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/DocumentdbGuardiumFilter.java

@@ -147,6 +147,12 @@ private void processAuditEvent(Event e, String messageString, FilterMatchListene
     } catch (OutOfMemoryError oom) {
       handleOutOfMemoryError(e, messageString, matchListener);
     } catch (JsonSyntaxException jse) {
+      // A bare BSON regex literal (e.g. /^foo/i) makes Gson fail. Sanitize and retry once.
+      String sanitized = StringUtils.sanitizeMongoBsonLiterals(messageString);
+      if (!sanitized.equals(messageString)) {
+        processAuditEvent(e, sanitized, matchListener);
+        return;
+      }
       handleJsonSyntaxError(e, messageString, jse, matchListener, true);
     } catch (Exception exception) {
       handleGenericError(e, messageString, exception, matchListener, true);
@@ -187,6 +193,12 @@ private void processProfilerEvent(
     } catch (OutOfMemoryError oom) {
       handleOutOfMemoryError(e, messageString, matchListener);
     } catch (JsonSyntaxException jse) {
+      // A bare BSON regex literal (e.g. /^foo/i) makes Gson fail. Sanitize and retry once.
+      String sanitized = StringUtils.sanitizeMongoBsonLiterals(messageString);
+      if (!sanitized.equals(messageString)) {
+        processProfilerEvent(e, sanitized, matchListener);
+        return;
+      }
       handleJsonSyntaxError(e, messageString, jse, matchListener, false);
     } catch (Exception exception) {
       handleGenericError(e, messageString, exception, matchListener, false);

filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/Parser.java

@@ -9,6 +9,7 @@
 import java.util.Iterator;
 import java.util.Map.Entry;
 
+import org.apache.commons.lang3.StringEscapeUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -242,7 +243,8 @@ public Construct parseAsConstructAuthCheck(final JsonObject data) {
       final Sentence sentence = parseSentenceAuthCheck(data);
       final Construct construct = new Construct();
       construct.sentences.add(sentence);
-      String fullSql = "\"atype\": "+data.get(Constants.FIELD_ATYPE).toString()+","+data.get(Constants.FIELD_PARAM);
+      String fullSql = StringEscapeUtils.unescapeJava(
+          "\"atype\": " + data.get(Constants.FIELD_ATYPE).toString() + "," + data.get(Constants.FIELD_PARAM));
       construct.setFullSql(fullSql);
       construct.setRedactedSensitiveDataSql(fullSql);
 
@@ -545,11 +547,15 @@ public Construct parseAsConstructDocumentDb(final JsonObject data) {
   	final Construct construct = new Construct();
   	construct.sentences.add(sentence);
   	if(data.has(Constants.FIELD_PARAM)) {
-  		construct.setFullSql("\"atype\": "+data.get(Constants.FIELD_ATYPE).toString()+","+data.get(Constants.FIELD_PARAM));
-  		construct.setRedactedSensitiveDataSql("\"atype\": "+data.get(Constants.FIELD_ATYPE).toString()+","+data.get(Constants.FIELD_PARAM));
+  		String fullSql = StringEscapeUtils.unescapeJava(
+  		    "\"atype\": " + data.get(Constants.FIELD_ATYPE).toString() + "," + data.get(Constants.FIELD_PARAM));
+  		construct.setFullSql(fullSql);
+  		construct.setRedactedSensitiveDataSql(fullSql);
   	}else if(data.has(Constants.FIELD_COMMAND)) {
-  		construct.setFullSql(data.get(Constants.FIELD_COMMAND).toString());
-  		construct.setRedactedSensitiveDataSql(data.get(Constants.FIELD_COMMAND).toString());
+  		String fullSql = StringEscapeUtils.unescapeJava(
+  		    data.get(Constants.FIELD_COMMAND).toString());
+  		construct.setFullSql(fullSql);
+  		construct.setRedactedSensitiveDataSql(fullSql);
   	}
   	return construct;
   } catch (final Exception e) {

filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/StringUtils.java

@@ -100,4 +100,86 @@ public static boolean containsAnyProfilerKey(String message) {
         }
         return false;
     }
+
+    /**
+     * Wraps bare BSON regex literals ({@code /pattern/flags}) in double quotes so the result
+     * is valid JSON. Slashes inside existing quoted strings are never touched.
+     */
+    public static String sanitizeMongoBsonLiterals(String json) {
+        if (json == null || json.isEmpty() || !json.contains("/")) {
+            return json;
+        }
+
+        StringBuilder out = new StringBuilder(json.length() + 16);
+        boolean inString = false;
+        boolean escaped  = false;
+
+        int i = 0;
+        while (i < json.length()) {
+            char c = json.charAt(i);
+
+            if (inString) {
+                out.append(c);
+                if (escaped)       { escaped = false; }
+                else if (c == '\\') { escaped = true;  }
+                else if (c == '"')  { inString = false; }
+                i++;
+                continue;
+            }
+
+            if (c == '"') {
+                inString = true;
+                out.append(c);
+                i++;
+                continue;
+            }
+
+            // Any unquoted '/' is a BSON regex literal — find its closing '/'
+            if (c == '/') {
+                int closingSlash = findRegexClosingSlash(json, i + 1);
+                int newline = json.indexOf('\n', i + 1);
+                if (closingSlash != -1 && (newline == -1 || closingSlash < newline)) {
+                    // Consume optional flags after the closing slash
+                    int flagsEnd = closingSlash + 1;
+                    while (flagsEnd < json.length()
+                            && Character.isLetterOrDigit(json.charAt(flagsEnd))) {
+                        flagsEnd++;
+                    }
+                    // Wrap in quotes, escaping '\' and '"' inside the pattern
+                    out.append('"');
+                    for (int j = i; j < flagsEnd; j++) {
+                        char p = json.charAt(j);
+                        if (p == '\\' || p == '"') out.append('\\');
+                        out.append(p);
+                    }
+                    out.append('"');
+                    i = flagsEnd;
+                    continue;
+                }
+            }
+
+            out.append(c);
+            i++;
+        }
+
+        return out.toString();
+    }
+
+    /**
+     * Returns the index of the closing {@code /} of a BSON regex literal, starting at
+     * {@code from} (one past the opening {@code /}). Backslash-escaped characters are
+     * skipped so an escaped {@code \/} inside the pattern is not treated as the delimiter.
+     * Returns {@code -1} if the end of the line is reached before a closing {@code /}.
+     */
+    private static int findRegexClosingSlash(String json, int from) {
+        int j = from;
+        while (j < json.length()) {
+            char c = json.charAt(j);
+            if (c == '\n') return -1;
+            if (c == '\\') { j += 2; continue; }
+            if (c == '/') return j;
+            j++;
+        }
+        return -1;
+    }
 }

filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/BsonRegexParserTest.java

@@ -0,0 +1,170 @@
+package com.ibm.guardium.documentdb;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import org.junit.jupiter.api.Test;
+import org.logstash.plugins.ContextImpl;
+
+import com.google.gson.Gson;
+import com.ibm.guardium.universalconnector.commons.GuardConstants;
+import com.ibm.guardium.universalconnector.commons.structures.Record;
+
+import co.elastic.logstash.api.Context;
+import co.elastic.logstash.api.Event;
+import co.elastic.logstash.api.FilterMatchListener;
+
+/** End-to-end tests for audit logs containing BSON regex literals. All message strings are synthetic. */
+public class BsonRegexParserTest {
+
+    private static final Context context = new ContextImpl(null, null);
+    private static final Gson gson = new Gson();
+
+    // ── helpers ───────────────────────────────────────────────────────────────
+
+    private static final DocumentdbGuardiumFilter filter =
+            new DocumentdbGuardiumFilter("test-id", null, context);
+
+    private static class CapturingMatchListener implements FilterMatchListener {
+        int count = 0;
+        @Override public void filterMatched(Event e) { count++; }
+    }
+
+    private Record runFilter(String message) {
+        Event e = new org.logstash.Event();
+        e.setField("message", message);
+        e.setField("serverHostnamePrefix", "test-cluster");
+        e.setField("event_id",
+                "0000000000000000000000000000000000000000000000000000000000000001");
+
+        CapturingMatchListener listener = new CapturingMatchListener();
+        Collection<Event> results = filter.filter(Collections.singletonList(e), listener);
+
+        assertEquals(1, results.size(), "filter should return exactly one event");
+        assertEquals(1, listener.count,  "event should be matched");
+        Object raw = e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME);
+        assertNotNull(raw, "GuardRecord field must be populated");
+        return gson.fromJson(raw.toString(), Record.class);
+    }
+
+    // ── test cases ────────────────────────────────────────────────────────────
+
+    /** find: bare regex as a field operator value. */
+    @Test
+    public void testFindWithNotRegex() {
+        String message =
+            "{\"atype\":\"authCheck\",\"ts\":1700000000000," +
+            "\"timestamp_utc\":\"2023-11-14 22:13:20.000\"," +
+            "\"remote_ip\":\"127.0.0.1:11111\"," +
+            "\"users\":[{\"user\":\"testuser\",\"db\":\"testdb\"}]," +
+            "\"param\":{\"command\":\"find\",\"ns\":\"testdb.col\"," +
+            "\"args\":{\"find\":\"col\"," +
+            "\"filter\":{\"a\":{\"b\":\"v\"}," +
+            "\"c\":{\"d\":/^abc/}," +
+            "\"e\":\"x\"}," +
+            "\"skip\":0,\"startTransaction\":false}," +
+            "\"result\":0}}";
+
+        Record record = runFilter(message);
+
+        assertNotNull(record.getData(),                "data must not be null");
+        assertNotNull(record.getData().getConstruct(), "construct must not be null");
+        assertNull(record.getException(),              "no exception expected");
+        assertEquals("find", record.getData().getConstruct().sentences.get(0).getVerb(),
+                "sentence verb must equal param.command");
+    }
+
+    /** find: array operator and regex as siblings on the same field. */
+    @Test
+    public void testFindWithNinAndNotRegex() {
+        String message =
+            "{\"atype\":\"authCheck\",\"ts\":1700000001000," +
+            "\"timestamp_utc\":\"2023-11-14 22:13:21.000\"," +
+            "\"remote_ip\":\"127.0.0.1:22222\"," +
+            "\"users\":[{\"user\":\"testuser\",\"db\":\"testdb\"}]," +
+            "\"param\":{\"command\":\"find\",\"ns\":\"testdb.col\"," +
+            "\"args\":{\"find\":\"col\"," +
+            "\"filter\":{\"a\":{\"b\":\"v\"}," +
+            "\"c\":{\"d\":[\"p\",\"q\"],\"e\":/^abc/}," +
+            "\"f\":\"x\"}," +
+            "\"skip\":0,\"startTransaction\":false}," +
+            "\"result\":0}}";
+
+        Record record = runFilter(message);
+
+        assertNotNull(record.getData());
+        assertNotNull(record.getData().getConstruct());
+        assertNull(record.getException(), "no exception expected");
+    }
+
+    /** find: nested array operators and regex combined in one filter. */
+    @Test
+    public void testFindWithAndNinNotRegex() {
+        String message =
+            "{\"atype\":\"authCheck\",\"ts\":1700000002000," +
+            "\"timestamp_utc\":\"2023-11-14 22:13:22.000\"," +
+            "\"remote_ip\":\"127.0.0.1:33333\"," +
+            "\"users\":[{\"user\":\"testuser\",\"db\":\"testdb\"}]," +
+            "\"param\":{\"command\":\"find\",\"ns\":\"testdb.col\"," +
+            "\"args\":{\"find\":\"col\"," +
+            "\"filter\":{\"a\":[{\"b\":{\"c\":[\"p\"]}},{\"d\":{\"e\":[\"q\"]}}]," +
+            "\"f\":{\"g\":[\"r\",\"s\",\"t\"],\"h\":/^abc/}," +
+            "\"i\":\"x\"}," +
+            "\"skip\":0,\"startTransaction\":false}," +
+            "\"result\":0}}";
+
+        Record record = runFilter(message);
+
+        assertNotNull(record.getData());
+        assertNotNull(record.getData().getConstruct());
+        assertNull(record.getException(), "no exception expected");
+    }
+
+    /** aggregate: regex with backslash escape sequence in a pipeline stage. */
+    @Test
+    public void testAggregateWithBackslashRegex() {
+        String message =
+            "{\"atype\":\"authCheck\",\"ts\":1700000003000," +
+            "\"timestamp_utc\":\"2023-11-14 22:13:23.000\"," +
+            "\"remote_ip\":\"127.0.0.1:44444\"," +
+            "\"users\":[{\"user\":\"testuser\",\"db\":\"testdb\"}]," +
+            "\"param\":{\"command\":\"aggregate\",\"ns\":\"testdb.col\"," +
+            "\"args\":{\"aggregate\":\"col\",\"allowDiskUse\":false," +
+            "\"cursor\":{\"batchSize\":256},\"explain\":false," +
+            "\"pipeline\":[" +
+            "{\"$match\":{\"a\":/.*\\wsome term.*/i,\"b\":{\"c\":[\"x\",\"y\"]}}}," +
+            "{\"$sort\":{\"a\":1}},{\"$skip\":0},{\"$limit\":100}]," +
+            "\"startTransaction\":false},\"result\":0}}";
+
+        Record record = runFilter(message);
+
+        assertNotNull(record.getData());
+        assertNotNull(record.getData().getConstruct());
+        assertNull(record.getException(), "no exception expected");
+    }
+
+    /** fullSql must preserve a backslash escape sequence, not double-escape it (unescapeJava regression). */
+    @Test
+    public void testFullSqlPreservesBackslashEscape() {
+        String message =
+            "{\"atype\":\"authCheck\",\"ts\":1700000004000," +
+            "\"timestamp_utc\":\"2023-11-14 22:13:24.000\"," +
+            "\"remote_ip\":\"127.0.0.1:55555\"," +
+            "\"users\":[{\"user\":\"testuser\",\"db\":\"testdb\"}]," +
+            "\"param\":{\"command\":\"aggregate\",\"ns\":\"testdb.col\"," +
+            "\"args\":{\"aggregate\":\"col\",\"allowDiskUse\":false," +
+            "\"cursor\":{\"batchSize\":256},\"explain\":false," +
+            "\"pipeline\":[" +
+            "{\"$match\":{\"a\":/.*\\wsome term.*/i,\"b\":{\"c\":[\"x\"]}}}]," +
+            "\"startTransaction\":false},\"result\":0}}";
+
+        Record record = runFilter(message);
+
+        String fullSql = record.getData().getConstruct().getFullSql();
+        assertNotNull(fullSql, "fullSql must not be null");
+        assertTrue(fullSql.contains("\\w"),    "fullSql must contain \\w");
+        assertFalse(fullSql.contains("\\\\w"), "fullSql must NOT contain \\\\w");
+    }
+}