diff --git a/packages/microsoft_exchange_online_message_trace/_dev/deploy/docker/files/config.yml b/packages/microsoft_exchange_online_message_trace/_dev/deploy/docker/files/config.yml index e595d7fac18..f36a5fa55b7 100644 --- a/packages/microsoft_exchange_online_message_trace/_dev/deploy/docker/files/config.yml +++ b/packages/microsoft_exchange_online_message_trace/_dev/deploy/docker/files/config.yml @@ -20,8 +20,9 @@ rules: # data: checks that the first page is requested with a broken skiptoken # (double-quoted dates — Microsoft Graph beta API defect), the cleaned # second page link is followed, the third page link (clean skiptoken with - # padding) is followed, and a new request (empty response) completes the - # time range. + # padding) is followed, the fourth page link (single-quote-escaped + # recipient address) is followed, and a new request (empty response) + # completes the time range. - path: /beta/admin/exchange/tracing/messageTraces methods: [GET] request_headers: @@ -78,6 +79,8 @@ rules: ] } # page 3: matched after the clean padded skiptoken is normalised (keys reordered). + # Returns a nextLink with a single-quote-escaped recipient address (Microsoft + # Graph beta API defect for addresses containing apostrophes). - path: /beta/admin/exchange/tracing/messageTraces methods: [GET] request_headers: @@ -93,7 +96,29 @@ rules: body: |- { "@odata.context": "https://graph.microsoft.com/beta/$metadata#admin/exchange/tracing/messageTraces", + "@odata.nextLink": "http://svc-graph-api:8080/beta/admin/exchange/tracing/messageTraces?$skiptoken=eyJTdGFydERhdGUiOiIyMDI1LTExLTI3VDE0OjE4OjM3LjYwNzAzOTBaIiwiRW5kRGF0ZSI6IjIwMjUtMTItMDJUMDU6MzQ6MTIuMzczMDAwMFoiLCJTdGFydGluZ1JlY2lwaWVudEFkZHJlc3MiOiInZGFuLm8nJ2Nvbm5vckBlbGFzdGljNTk1Lm9ubWljcm9zb2Z0LmNvbSciLCJSZXN1bHRTaXplIjoiNCJ9", "value": [ {"id":"48d3fcf5-2389-4390-926f-7f6e575b1fe6","messageId":"<123433a6-c7cb-473e-9c65-2ca41b93f2dd@az.westus3.microsoft.com>","status":"delivered","receivedDateTime":"2025-11-30T06:43:04.864Z","recipientAddress":"G@Elastic595.onmicrosoft.com","senderAddress":"MSSecurity-noreply@microsoft.com","subject":"Your weekly PIM digest for Elastic (ID: 60aa0e13-5d8c-4325-ad81-e22998b1649e)","size":327783,"fromIP":"2a02:cf47:648c:e1da:2155:5eb2:8ead:fb05","toIP":""} ] } + # page 4: matched after the single-quote-escaped skiptoken is cleaned + # (outer quotes stripped, doubled apostrophes unescaped, keys reordered). + - path: /beta/admin/exchange/tracing/messageTraces + methods: [GET] + request_headers: + Authorization: + - "Bearer test-access-token" + query_params: + $skiptoken: "eyJFbmREYXRlIjoiMjAyNS0xMi0wMlQwNTozNDoxMi4zNzMwMDAwWiIsIlJlc3VsdFNpemUiOiI0IiwiU3RhcnREYXRlIjoiMjAyNS0xMS0yN1QxNDoxODozNy42MDcwMzkwWiIsIlN0YXJ0aW5nUmVjaXBpZW50QWRkcmVzcyI6ImRhbi5vJ2Nvbm5vckBlbGFzdGljNTk1Lm9ubWljcm9zb2Z0LmNvbSJ9" + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: |- + { + "@odata.context": "https://graph.microsoft.com/beta/$metadata#admin/exchange/tracing/messageTraces", + "value": [ + {"id":"a1b2c3d4-e5f6-7890-abcd-ef1234567890","messageId":"<7a8b9c0d-e1f2-3456-7890-abcdef123456@az.northeurope.microsoft.com>","status":"delivered","receivedDateTime":"2025-11-29T14:22:31.109Z","recipientAddress":"dan.o'connor@elastic595.onmicrosoft.com","senderAddress":"MSSecurity-noreply@microsoft.com","subject":"Microsoft Entra ID Protection Weekly Digest","size":298412,"fromIP":"2a02:cf41:d675:a90d:c0ef:b23f:f053:947e","toIP":""} + ] + } diff --git a/packages/microsoft_exchange_online_message_trace/changelog.yml b/packages/microsoft_exchange_online_message_trace/changelog.yml index 192751f6abf..8926b80a4a9 100644 --- a/packages/microsoft_exchange_online_message_trace/changelog.yml +++ b/packages/microsoft_exchange_online_message_trace/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.2.2" + changes: + - description: Work around Microsoft Graph beta API defect where single-quote-escaped recipient addresses in the skiptoken cause pagination to fail with 400 errors. + type: bugfix + link: https://github.com/elastic/integrations/pull/19401 - version: "2.2.1" changes: - description: Use the ECS definition for email.attachments. diff --git a/packages/microsoft_exchange_online_message_trace/data_stream/log/_dev/test/system/test-cel-config.yml b/packages/microsoft_exchange_online_message_trace/data_stream/log/_dev/test/system/test-cel-config.yml index b3eaf99b8ab..d41fb6258a0 100644 --- a/packages/microsoft_exchange_online_message_trace/data_stream/log/_dev/test/system/test-cel-config.yml +++ b/packages/microsoft_exchange_online_message_trace/data_stream/log/_dev/test/system/test-cel-config.yml @@ -17,5 +17,5 @@ data_stream: preserve_original_event: true enable_request_tracer: true assert: - hit_count: 7 + hit_count: 8 wait_for_data_timeout: 30s diff --git a/packages/microsoft_exchange_online_message_trace/data_stream/log/agent/stream/cel.yml.hbs b/packages/microsoft_exchange_online_message_trace/data_stream/log/agent/stream/cel.yml.hbs index b1ec26f31ad..ca49722fb37 100644 --- a/packages/microsoft_exchange_online_message_trace/data_stream/log/agent/stream/cel.yml.hbs +++ b/packages/microsoft_exchange_online_message_trace/data_stream/log/agent/stream/cel.yml.hbs @@ -38,10 +38,11 @@ program: |- ( (state.?cursor.next_link.orValue(null) != null) ? // There is a next link to follow for the current range. - // Work around Microsoft Graph beta API defect where - // skiptoken values are double-JSON-encoded, causing - // 400 errors on the next page request. Fixing on read - // also repairs cursors poisoned before this fix. + // Work around Microsoft Graph beta API defects where + // skiptoken values are either double-JSON-encoded or + // wrapped in single quotes with doubled apostrophes, + // causing 400 errors on the next page request. Fixing + // on read also repairs cursors poisoned before this fix. { "url": state.cursor.next_link.parse_url().as(u, u.RawQuery.parse_query().as(q, @@ -58,6 +59,8 @@ program: |- { k: (v.has_prefix('"') && v.has_suffix('"')) ? v.decode_json().as(dv, is_error(dv) ? v : dv) + : (size(v) >= 2 && v.has_prefix("'") && v.has_suffix("'")) ? + try(string(v).substring(1, size(v)-1).replace_all("''", "'")).as(dv, is_error(dv) ? v : dv) : v } diff --git a/packages/microsoft_exchange_online_message_trace/data_stream/log/sample_event.json b/packages/microsoft_exchange_online_message_trace/data_stream/log/sample_event.json index d6787825e1e..d9ee2194a32 100644 --- a/packages/microsoft_exchange_online_message_trace/data_stream/log/sample_event.json +++ b/packages/microsoft_exchange_online_message_trace/data_stream/log/sample_event.json @@ -1,54 +1,56 @@ { - "@timestamp": "2022-10-21T17:25:36.969Z", + "@timestamp": "2025-11-29T14:22:31.109Z", "agent": { - "ephemeral_id": "11edfb81-b112-45ba-8f01-6e7483e450fa", - "id": "1c0788e9-492a-441e-acab-fc8c56281cf1", - "name": "elastic-agent-22259", + "ephemeral_id": "5348c840-193d-4e9d-8849-bb4d2c6b88f0", + "id": "ed6593d9-b251-42d5-9c26-b5222d3a9ce1", + "name": "elastic-agent-79838", "type": "filebeat", "version": "8.19.4" }, "data_stream": { "dataset": "microsoft_exchange_online_message_trace.log", - "namespace": "71098", + "namespace": "75301", "type": "logs" }, "destination": { - "domain": "contoso.com", - "registered_domain": "contoso.com", + "domain": "elastic595.onmicrosoft.com", + "registered_domain": "onmicrosoft.com", + "subdomain": "elastic595", "top_level_domain": "com", "user": { - "domain": "contoso.com", - "email": "linus@contoso.com", - "id": "linus@contoso.com", - "name": "linus" + "domain": "elastic595.onmicrosoft.com", + "email": "dan.o'connor@elastic595.onmicrosoft.com", + "id": "dan.o'connor@elastic595.onmicrosoft.com", + "name": "dan.o'connor" } }, "ecs": { "version": "8.11.0" }, "elastic_agent": { - "id": "1c0788e9-492a-441e-acab-fc8c56281cf1", + "id": "ed6593d9-b251-42d5-9c26-b5222d3a9ce1", "snapshot": false, "version": "8.19.4" }, "email": { "attachments": { "file": { - "size": 22761 + "size": 298412 } }, - "delivery_timestamp": "2022-10-21T17:25:36.969376Z", + "delivery_timestamp": "2025-11-29T14:22:31.109Z", + "direction": "external", "from": { "address": [ - "noreply@azure.microsoft.com" + "MSSecurity-noreply@microsoft.com" ] }, - "local_id": "a5e6dc0f-23df-4b20-d240-08dab38944a1", - "message_id": "", - "subject": "testmail 2", + "local_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "message_id": "<7a8b9c0d-e1f2-3456-7890-abcdef123456@az.northeurope.microsoft.com>", + "subject": "Microsoft Entra ID Protection Weekly Digest", "to": { "address": [ - "linus@contoso.com" + "dan.o'connor@elastic595.onmicrosoft.com" ] } }, @@ -58,63 +60,60 @@ "email" ], "dataset": "microsoft_exchange_online_message_trace.log", - "ingested": "2025-10-06T13:13:06Z", - "original": "{\"Organization\":\"contoso.com\",\"MessageId\":\"\\u003cGVAP278MB037586A65EF1FB2F844B0258DA2D9@GVAP278MB0375.CHEP278.PROD.OUTLOOK.COM\\u003e\",\"Received\":\"2022-10-21T17:25:36.969376Z\",\"SenderAddress\":\"noreply@azure.microsoft.com\",\"RecipientAddress\":\"linus@contoso.com\",\"Subject\":\"testmail 2\",\"Status\":\"Delivered\",\"ToIP\":null,\"FromIP\":\"40.107.23.54\",\"Size\":22761,\"MessageTraceId\":\"a5e6dc0f-23df-4b20-d240-08dab38944a1\",\"StartDate\":\"2022-10-21T09:40:10Z\",\"EndDate\":\"2022-10-22T09:40:10Z\",\"Index\":0}", + "ingested": "2026-06-05T13:14:47Z", + "original": "{\"fromIP\":\"2a02:cf41:d675:a90d:c0ef:b23f:f053:947e\",\"id\":\"a1b2c3d4-e5f6-7890-abcd-ef1234567890\",\"messageId\":\"\\u003c7a8b9c0d-e1f2-3456-7890-abcdef123456@az.northeurope.microsoft.com\\u003e\",\"receivedDateTime\":\"2025-11-29T14:22:31.109Z\",\"recipientAddress\":\"dan.o'connor@elastic595.onmicrosoft.com\",\"senderAddress\":\"MSSecurity-noreply@microsoft.com\",\"size\":298412,\"status\":\"delivered\",\"subject\":\"Microsoft Entra ID Protection Weekly Digest\",\"toIP\":\"\"}", "outcome": "success", "type": [ "info" ] }, "input": { - "type": "log" - }, - "log": { - "file": { - "path": "/tmp/service_logs/microsoft_exchange_online_message_trace_test.ndjson.log" - }, - "offset": 0 + "type": "cel" }, "microsoft": { "online_message_trace": { - "EndDate": "2022-10-22T09:40:10Z", - "FromIP": "40.107.23.54", - "Index": 0, - "MessageId": "", - "MessageTraceId": "a5e6dc0f-23df-4b20-d240-08dab38944a1", - "Organization": "contoso.com", - "Received": "2022-10-21T17:25:36.969376Z", - "RecipientAddress": "linus@contoso.com", - "SenderAddress": "noreply@azure.microsoft.com", - "Size": 22761, - "StartDate": "2022-10-21T09:40:10Z", - "Status": "Delivered", - "Subject": "testmail 2" + "FromIP": "2a02:cf41:d675:a90d:c0ef:b23f:f053:947e", + "MessageId": "<7a8b9c0d-e1f2-3456-7890-abcdef123456@az.northeurope.microsoft.com>", + "MessageTraceId": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "Received": "2025-11-29T14:22:31.109Z", + "RecipientAddress": "dan.o'connor@elastic595.onmicrosoft.com", + "SenderAddress": "MSSecurity-noreply@microsoft.com", + "Size": 298412, + "Status": "delivered", + "Subject": "Microsoft Entra ID Protection Weekly Digest" } }, "related": { "user": [ - "linus@contoso.com", - "noreply@azure.microsoft.com", - "linus", - "noreply" + "dan.o'connor@elastic595.onmicrosoft.com", + "MSSecurity-noreply@microsoft.com", + "dan.o'connor", + "MSSecurity-noreply" ] }, "source": { - "domain": "azure.microsoft.com", - "ip": "40.107.23.54", + "domain": "microsoft.com", + "geo": { + "continent_name": "Europe", + "country_iso_code": "NO", + "country_name": "Norway", + "location": { + "lat": 62, + "lon": 10 + } + }, + "ip": "2a02:cf41:d675:a90d:c0ef:b23f:f053:947e", "registered_domain": "microsoft.com", - "subdomain": "azure", "top_level_domain": "com", "user": { - "domain": "azure.microsoft.com", - "email": "noreply@azure.microsoft.com", - "id": "noreply@azure.microsoft.com", - "name": "noreply" + "domain": "microsoft.com", + "email": "MSSecurity-noreply@microsoft.com", + "id": "MSSecurity-noreply@microsoft.com", + "name": "MSSecurity-noreply" } }, "tags": [ "preserve_original_event", - "microsoft-defender-endpoint", "forwarded" ] } diff --git a/packages/microsoft_exchange_online_message_trace/docs/README.md b/packages/microsoft_exchange_online_message_trace/docs/README.md index 388f51720ac..24e8a992a9f 100644 --- a/packages/microsoft_exchange_online_message_trace/docs/README.md +++ b/packages/microsoft_exchange_online_message_trace/docs/README.md @@ -162,56 +162,58 @@ An example event for `log` looks as following: ```json { - "@timestamp": "2022-10-21T17:25:36.969Z", + "@timestamp": "2025-11-29T14:22:31.109Z", "agent": { - "ephemeral_id": "11edfb81-b112-45ba-8f01-6e7483e450fa", - "id": "1c0788e9-492a-441e-acab-fc8c56281cf1", - "name": "elastic-agent-22259", + "ephemeral_id": "5348c840-193d-4e9d-8849-bb4d2c6b88f0", + "id": "ed6593d9-b251-42d5-9c26-b5222d3a9ce1", + "name": "elastic-agent-79838", "type": "filebeat", "version": "8.19.4" }, "data_stream": { "dataset": "microsoft_exchange_online_message_trace.log", - "namespace": "71098", + "namespace": "75301", "type": "logs" }, "destination": { - "domain": "contoso.com", - "registered_domain": "contoso.com", + "domain": "elastic595.onmicrosoft.com", + "registered_domain": "onmicrosoft.com", + "subdomain": "elastic595", "top_level_domain": "com", "user": { - "domain": "contoso.com", - "email": "linus@contoso.com", - "id": "linus@contoso.com", - "name": "linus" + "domain": "elastic595.onmicrosoft.com", + "email": "dan.o'connor@elastic595.onmicrosoft.com", + "id": "dan.o'connor@elastic595.onmicrosoft.com", + "name": "dan.o'connor" } }, "ecs": { "version": "8.11.0" }, "elastic_agent": { - "id": "1c0788e9-492a-441e-acab-fc8c56281cf1", + "id": "ed6593d9-b251-42d5-9c26-b5222d3a9ce1", "snapshot": false, "version": "8.19.4" }, "email": { "attachments": { "file": { - "size": 22761 + "size": 298412 } }, - "delivery_timestamp": "2022-10-21T17:25:36.969376Z", + "delivery_timestamp": "2025-11-29T14:22:31.109Z", + "direction": "external", "from": { "address": [ - "noreply@azure.microsoft.com" + "MSSecurity-noreply@microsoft.com" ] }, - "local_id": "a5e6dc0f-23df-4b20-d240-08dab38944a1", - "message_id": "", - "subject": "testmail 2", + "local_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "message_id": "<7a8b9c0d-e1f2-3456-7890-abcdef123456@az.northeurope.microsoft.com>", + "subject": "Microsoft Entra ID Protection Weekly Digest", "to": { "address": [ - "linus@contoso.com" + "dan.o'connor@elastic595.onmicrosoft.com" ] } }, @@ -221,63 +223,60 @@ An example event for `log` looks as following: "email" ], "dataset": "microsoft_exchange_online_message_trace.log", - "ingested": "2025-10-06T13:13:06Z", - "original": "{\"Organization\":\"contoso.com\",\"MessageId\":\"\\u003cGVAP278MB037586A65EF1FB2F844B0258DA2D9@GVAP278MB0375.CHEP278.PROD.OUTLOOK.COM\\u003e\",\"Received\":\"2022-10-21T17:25:36.969376Z\",\"SenderAddress\":\"noreply@azure.microsoft.com\",\"RecipientAddress\":\"linus@contoso.com\",\"Subject\":\"testmail 2\",\"Status\":\"Delivered\",\"ToIP\":null,\"FromIP\":\"40.107.23.54\",\"Size\":22761,\"MessageTraceId\":\"a5e6dc0f-23df-4b20-d240-08dab38944a1\",\"StartDate\":\"2022-10-21T09:40:10Z\",\"EndDate\":\"2022-10-22T09:40:10Z\",\"Index\":0}", + "ingested": "2026-06-05T13:14:47Z", + "original": "{\"fromIP\":\"2a02:cf41:d675:a90d:c0ef:b23f:f053:947e\",\"id\":\"a1b2c3d4-e5f6-7890-abcd-ef1234567890\",\"messageId\":\"\\u003c7a8b9c0d-e1f2-3456-7890-abcdef123456@az.northeurope.microsoft.com\\u003e\",\"receivedDateTime\":\"2025-11-29T14:22:31.109Z\",\"recipientAddress\":\"dan.o'connor@elastic595.onmicrosoft.com\",\"senderAddress\":\"MSSecurity-noreply@microsoft.com\",\"size\":298412,\"status\":\"delivered\",\"subject\":\"Microsoft Entra ID Protection Weekly Digest\",\"toIP\":\"\"}", "outcome": "success", "type": [ "info" ] }, "input": { - "type": "log" - }, - "log": { - "file": { - "path": "/tmp/service_logs/microsoft_exchange_online_message_trace_test.ndjson.log" - }, - "offset": 0 + "type": "cel" }, "microsoft": { "online_message_trace": { - "EndDate": "2022-10-22T09:40:10Z", - "FromIP": "40.107.23.54", - "Index": 0, - "MessageId": "", - "MessageTraceId": "a5e6dc0f-23df-4b20-d240-08dab38944a1", - "Organization": "contoso.com", - "Received": "2022-10-21T17:25:36.969376Z", - "RecipientAddress": "linus@contoso.com", - "SenderAddress": "noreply@azure.microsoft.com", - "Size": 22761, - "StartDate": "2022-10-21T09:40:10Z", - "Status": "Delivered", - "Subject": "testmail 2" + "FromIP": "2a02:cf41:d675:a90d:c0ef:b23f:f053:947e", + "MessageId": "<7a8b9c0d-e1f2-3456-7890-abcdef123456@az.northeurope.microsoft.com>", + "MessageTraceId": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "Received": "2025-11-29T14:22:31.109Z", + "RecipientAddress": "dan.o'connor@elastic595.onmicrosoft.com", + "SenderAddress": "MSSecurity-noreply@microsoft.com", + "Size": 298412, + "Status": "delivered", + "Subject": "Microsoft Entra ID Protection Weekly Digest" } }, "related": { "user": [ - "linus@contoso.com", - "noreply@azure.microsoft.com", - "linus", - "noreply" + "dan.o'connor@elastic595.onmicrosoft.com", + "MSSecurity-noreply@microsoft.com", + "dan.o'connor", + "MSSecurity-noreply" ] }, "source": { - "domain": "azure.microsoft.com", - "ip": "40.107.23.54", + "domain": "microsoft.com", + "geo": { + "continent_name": "Europe", + "country_iso_code": "NO", + "country_name": "Norway", + "location": { + "lat": 62, + "lon": 10 + } + }, + "ip": "2a02:cf41:d675:a90d:c0ef:b23f:f053:947e", "registered_domain": "microsoft.com", - "subdomain": "azure", "top_level_domain": "com", "user": { - "domain": "azure.microsoft.com", - "email": "noreply@azure.microsoft.com", - "id": "noreply@azure.microsoft.com", - "name": "noreply" + "domain": "microsoft.com", + "email": "MSSecurity-noreply@microsoft.com", + "id": "MSSecurity-noreply@microsoft.com", + "name": "MSSecurity-noreply" } }, "tags": [ "preserve_original_event", - "microsoft-defender-endpoint", "forwarded" ] } diff --git a/packages/microsoft_exchange_online_message_trace/manifest.yml b/packages/microsoft_exchange_online_message_trace/manifest.yml index a505d0c85f6..83b98f71ff1 100644 --- a/packages/microsoft_exchange_online_message_trace/manifest.yml +++ b/packages/microsoft_exchange_online_message_trace/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.3.2" name: microsoft_exchange_online_message_trace title: "Microsoft Exchange Online Message Trace" -version: "2.2.1" +version: "2.2.2" description: "Microsoft Exchange Online Message Trace Integration" type: integration categories: