From 7761a35de9c06211aa71cea15a3bf1ec74f909cd Mon Sep 17 00:00:00 2001 From: StacieClark-Elastic Date: Mon, 20 Oct 2025 09:41:37 -0400 Subject: [PATCH 1/6] fix-parsing-error-due-to-duplicate-fields Fixes flattening error in Actions list when the list is encoded json string instead of json objects. Adds fields ActorInfoString, OperationCount, TokenObjectId, TokenTenantId. Added fields Messages and Folders as ExchangeMessages and ExchangeFolder for record type 50: ExchangeItemAggregated --- packages/o365/changelog.yml | 9 + .../pipeline/test-exchange-access-event.json | 222 +++++ ...t-exchange-access-event.json-expected.json | 787 ++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 58 +- .../o365/data_stream/audit/fields/fields.yml | 55 ++ packages/o365/docs/README.md | 14 + packages/o365/manifest.yml | 2 +- 7 files changed, 1143 insertions(+), 4 deletions(-) create mode 100644 packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json create mode 100644 packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json-expected.json diff --git a/packages/o365/changelog.yml b/packages/o365/changelog.yml index 3b07bb035c9..52ec00e3467 100644 --- a/packages/o365/changelog.yml +++ b/packages/o365/changelog.yml @@ -1,4 +1,13 @@ # newer versions go on top +- version: "2.31.1" + changes: + - description: >- + Fix flattening errors in Action List items due to duplicate QueryTime fields. + Added fields ActorInfoString, OperationCount, TokenObjectId, TokenTenantId. + Added fields Messages and Folders as ExchangeMessages and ExchangeFolders + for record type 50: `ExchangeItemAggregated`. + type: bugfix + link: https://github.com/elastic/integrations/pull/99999 - version: "2.31.0" changes: - description: Improve documentation. diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json new file mode 100644 index 00000000000..3dced223128 --- /dev/null +++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json @@ -0,0 +1,222 @@ +{ + "events": [ + { + "o365audit": { + "LogonUserSid": "S-1-5-21-1234567890-123456789-1234567890-12345678", + "AppAccessContext": { + "APIId": "abcdabcd-1234-12ab-1a2b-ad1234567890", + "ClientAppId": "abcdabcd-1234-12ab-1a2b-ad1234567890", + "IssuedAtTime": "2025-09-29T01:01:01", + "UniqueTokenId": "12345678-1234-1234-abcd-abcdef123456" + }, + "AppId": "abcdabcd-1234-12ab-1a2b-ad1234567890", + "ActorInfoString": "Client=REST;Client=RESTSystem;UserAgent=[NoUserAgent][AppId=abcdabcd-1234-12ab-1a2b-ad1234567890];", + "MailboxOwnerUPN": "user@example.com", + "MailboxOwnerSid": "S-1-5-21-1234567890-123456789-1234567890-12345678", + "LogonType": 0, + "ClientInfoString": "Client=REST;Client=RESTSystem;;", + "ResultStatus": "Succeeded", + "OrganizationName": "example.onmicrosoft.com", + "ExternalAccess": false, + "OperationProperties": [ + { + "Name": "AttachmentAccessType", + "Value": "Bind" + } + ], + "InternalLogonType": 0, + "MailboxGuid": "8b46a639-c47f-4634-b90c-2accecd337e3", + "UserKey": "abcdabcd-1234-12ab-1a2b-ad1234567890", + "TokenTenantId": "dcbadcba-1234-12ab-1a2b-ad1234567890", + "UserId": "user@example.com", + "UserType": 5, + "CreationTime": "2025-09-29T01:01:01", + "Version": 1, + "RecordType": 50, + "ClientIPAddress": "203.0.113.5", + "Operation": "AttachmentAccess", + "OrganizationId": "1234abcd-4321-dcba-43ab-1023456789ab", + "ClientAppId": "abcdabcd-1234-12ab-1a2b-ad1234567890", + "OriginatingServer": "imase12AA1234 (203.0.113.3)", + "Messages": [ + { + "Path": "Messages", + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDD", + "MessageItems": [ + { + "SizeInBytes": 2379, + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDDEEEEEE12345678901234567890123" + }, + { + "SizeInBytes": 7356, + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDDEEEEEE000000011111122222222222" + } + ] + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDlFFFF", + "MessageItems": [ + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDlFFFFEEEEEEaaaaaaaaaaaaaaaaaaaaaaa", + "SizeInBytes": 1156492 + } + ], + "Path": "Messages" + }, + { + "Path": "Messages", + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDvnQAA", + "MessageItems": [ + { + "SizeInBytes": 87052, + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDvnQAAEEEEEEAALE" + } + ] + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJk_eAAA", + "MessageItems": [ + { + "SizeInBytes": 267212, + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJk_eAAAEEEEEEAB_WWWWWWWWWWWWWWW-1234" + } + ], + "Path": "Messages" + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJ123456", + "MessageItems": [ + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJ1234AAEEEEEEADGGGGGGGGGGGGGGGGGGGGGG", + "SizeInBytes": 20477 + } + ], + "Path": "Messages" + } + ], + "Id": "88888888-4444-5555-6666-123456789012", + "Workload": "Exchange", + "OperationCount": 6 + } + }, + { + "o365audit": { + "StartTimeUtc": "2025-09-29T23:59:59", + "Actions": [ + "{\"$id\":\"1\",\"ActionId\":\"urn:EmailZapper:1234567890abcdef1234567890abcdef\",\"InvestigationId\":\"urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\",\"ActionApproval\":\"None\",\"ActionType\":\"EmailRemediation\",\"ActionStatus\":\"Skipped\",\"Entities\":[{\"$id\":\"2\",\"Recipient\":\"user@example.com\",\"Urls\":[\"https://exxample.com/fffffffff\",\"https://example.com/\",\"https://www.example.com/\",\"https://domain.com\",\"https://domain.com\",\"https://emailsg.example.com/wf/open?upn=1234\",\"https://apps.apple.com/us/app/example-com/id12343123444\",\"https://play.google.com/store/apps/details?id=com.example.example\",\"https://dl.example.com/boards/6667868113/groups/topics?dl_slug=resortinternet&dl_msgid=99999999-6666-5555-4444-1023456789012&dl_category=notifications_mailer-assign_person_to_pulse&dl_userid=12345678&dl_sessionid=12345678901234567890123456789012_0&dl_senderid=-4&dl_notificationappid=98776666&dl_notificationkindname=board_assigned_in_column&dl_notificationuuid=12345678901234567890123456789012_0\",\"https://dl.example.com/users/-4-automations?dl_slug=resortinternet&dl_msgid=99999999-6666-5555-4444-1023456789012&dl_category=notifications_mailer-assign_person_to_pulse&dl_userid=12345678&dl_sessionid=12345678901234567890123456789012_0&dl_senderid=-4&dl_notificationappid=12345678&dl_notificationkindname=board_assigned_in_column&dl_notificationuuid=12345678901234567890123456789012_0\"],\"Threats\":[\"ZapPhish\",\"HighConfPhish\"],\"Sender\":\"sales@example.com\",\"P1Sender\":\"1234565@example.com\",\"P1SenderDomain\":\"example.com\",\"SenderIP\":\"203.0.113.55\",\"P2Sender\":\"sales@example.com\",\"P2SenderDisplayName\":\"Postal_ProtocolAdminChecklnReportDocSubmissionRequestapEx12341234Serverange-reply\",\"P2SenderDomain\":\"example.com\",\"ReceivedDate\":\"2025-09-15T22:20:37\",\"NetworkMessageId\":\"33333333-eeee-4444-5555-999999999999\",\"InternetMessageId\":\"<1234@email.example.com>\",\"Subject\":\"Admin-Protocol-Tasks-Update on 9/15/2025\",\"AntispamDirection\":\"Inbound\",\"DeliveryAction\":\"Delivered\",\"Language\":\"en\",\"DeliveryLocation\":\"Quarantine\",\"OriginalDeliveryLocation\":\"Inbox\",\"AdditionalActionsAndResults\":[\"OriginalDelivery: [N/A]\"],\"AuthDetails\":[{\"Name\":\"SPF\",\"Value\":\"Pass\"},{\"Name\":\"DKIM\",\"Value\":\"Pass\"},{\"Name\":\"DMARC\",\"Value\":\"Pass\"},{\"Name\":\"Comp Auth\",\"Value\":\"pass\"}],\"SystemOverrides\":[],\"Type\":\"mailMessage\",\"Urn\":\"urn:MailEntity:aaaaaaaaaaaaaaaabbbbbbbbbbbbbbbb\",\"Source\":\"OATP\",\"FirstSeen\":\"2025-09-15T22:51:56\"}],\"RelatedAlertIds\":[\"33333333-2222-1111-7777-123456789012\"],\"StartTimeUtc\":\"2025-09-15T23:02:00\",\"EndTimeUtc\":\"2025-09-15T23:04:18Z\",\"LastUpdateTimeUtc\":\"2025-09-17T12:22:43.2513154Z\",\"TimestampUtc\":\"2025-09-15T23:04:18\",\"BulkName\":\"Malicious mail is zapped - urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\",\"ResourceIdentifiers\":[{\"$id\":\"3\",\"AadTenantId\":\"999999999-2222-1111-7777-123456789012\",\"Type\":\"AAD\"}],\"PendingType\":\"User\",\"Type\":\"InvestigationAction\",\"LogCreationTime\":\"2025-09-17T12:22:43.2513154Z\",\"MachineName\":\"AB12AB12AB123\",\"Description\":\"For malicious emails, you can move to junk, soft or hard delete from user's mailbox.\"}", + "{\"$id\":\"1\",\"ActionId\":\"urn:EmailZapper:4321567890abcdef1234567890abcdef\",\"InvestigationId\":\"urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\",\"ActionApproval\":\"None\",\"ActionType\":\"EmailRemediation\",\"ActionStatus\":\"Skipped\",\"Entities\":[{\"$id\":\"2\",\"NetworkMessageIds\":[\"88888888-eeee-4444-5555-999999999999\",\"33333333-eeee-4444-5555-999999999999\"],\"CountByThreatType\":{\"HighConfPhish\":1,\"Phish\":0,\"Malware\":0,\"Spam\":0},\"CountByProtectionStatus\":{\"Delivered\":1,\"Blocked\":1},\"CountByDeliveryLocation\":{\"Quarantine\":2},\"Query\":\"( (( (BodyFingerprintBin1:\\\"99999999999\\\") ) AND ( (SenderIp:\\\"203.0.113.55\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\",\"QueryTime\":\"2025-09-15T23:05:00Z\",\"MailCount\":2,\"IsVolumeAnamoly\":false,\"ClusterSourceIdentifier\":\"33333333-eeee-4444-5555-999999999999\",\"ClusterSourceType\":\"Similarity\",\"ClusterQueryStartTime\":\"2025-08-26T00:00:00Z\",\"ClusterQueryEndTime\":\"2025-09-15T23:05:00Z\",\"ClusterGroup\":\"BodyFingerprintBin1,SenderIp\",\"Type\":\"mailCluster\",\"ClusterBy\":\"BodyFingerprintBin1;SenderIp;ContentType\",\"ClusterByValue\":\"99999999999;203.0.113.5;1\",\"QueryStartTime\":\"8/26/2025 12:00:00 AM\",\"QueryTime\":\"9/15/2025 11:05:00 PM\",\"Urn\":\"urn:MailClusterEntity:98765432109876543210987654321098\",\"Source\":\"OATP\",\"FirstSeen\":\"2025-09-15T23:00:09\"}],\"RelatedAlertIds\":[\"33333333-2222-1111-7777-123456789012\"],\"StartTimeUtc\":\"2025-09-15T23:02:00\",\"EndTimeUtc\":\"2025-09-17T10:37:16\",\"LastUpdateTimeUtc\":\"2025-09-17T12:22:43.2525624Z\",\"TimestampUtc\":\"2025-09-17T10:37:16\",\"BulkName\":\"Malicious mail is zapped - urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\",\"ResourceIdentifiers\":[{\"$id\":\"3\",\"AadTenantId\":\"999999999-2222-1111-7777-123456789012\",\"Type\":\"AAD\"}],\"PendingType\":\"User\",\"Type\":\"InvestigationAction\",\"LogCreationTime\":\"2025-09-17T12:22:43.2525624Z\",\"MachineName\":\"AB12AB12AB123\",\"Description\":\"For malicious emails, you can move to junk, soft or hard delete from user's mailbox.\"}", + "{\"$id\":\"1\",\"ActionId\":\"urn:EmailZapper:6666567890abcdef1234567890abcdef\",\"InvestigationId\":\"urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\",\"ActionApproval\":\"None\",\"ActionType\":\"EmailRemediation\",\"ActionStatus\":\"Skipped\",\"Entities\":[{\"$id\":\"2\",\"NetworkMessageIds\":[\"88888888-eeee-4444-5555-999999999999\",\"33333333-eeee-4444-5555-999999999999\",\"44444444-eeee-4444-5555-999999999999\"],\"CountByThreatType\":{\"HighConfPhish\":2,\"Phish\":0,\"Malware\":0,\"Spam\":0},\"CountByProtectionStatus\":{\"Blocked\":2,\"Delivered\":1},\"CountByDeliveryLocation\":{\"Quarantine\":3},\"Query\":\"( (( (BodyFingerprintBin1:\\\"99999999999\\\") ) AND ( (P2SenderDomain:\\\"example.com\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\",\"QueryTime\":\"2025-09-15T23:05:00Z\",\"MailCount\":3,\"IsVolumeAnamoly\":false,\"ClusterSourceIdentifier\":\"33333333-eeee-4444-5555-999999999999\",\"ClusterSourceType\":\"Similarity\",\"ClusterQueryStartTime\":\"2025-08-26T00:00:00Z\",\"ClusterQueryEndTime\":\"2025-09-15T23:05:00Z\",\"ClusterGroup\":\"BodyFingerprintBin1,P2SenderDomain\",\"Type\":\"mailCluster\",\"ClusterBy\":\"BodyFingerprintBin1;P2SenderDomain;ContentType\",\"ClusterByValue\":\"99999999999;example.com;1\",\"QueryStartTime\":\"8/26/2025 12:00:00 AM\",\"QueryTime\":\"9/15/2025 11:05:00 PM\",\"Urn\":\"urn:MailClusterEntity:cccccccccccccccccccccccccccccccc\",\"Source\":\"OATP\",\"FirstSeen\":\"2025-09-15T23:00:09\"}],\"RelatedAlertIds\":[\"33333333-2222-1111-7777-123456789012\"],\"StartTimeUtc\":\"2025-09-15T23:02:00\",\"EndTimeUtc\":\"2025-09-17T10:37:16\",\"LastUpdateTimeUtc\":\"2025-09-17T12:22:43.2676112Z\",\"TimestampUtc\":\"2025-09-17T10:37:16\",\"BulkName\":\"Malicious mail is zapped - urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\",\"ResourceIdentifiers\":[{\"$id\":\"3\",\"AadTenantId\":\"999999999-2222-1111-7777-123456789012\",\"Type\":\"AAD\"}],\"PendingType\":\"User\",\"Type\":\"InvestigationAction\",\"LogCreationTime\":\"2025-09-17T12:22:43.2676112Z\",\"MachineName\":\"AB12AB12AB123\",\"Description\":\"For malicious emails, you can move to junk, soft or hard delete from user's mailbox.\"}" + ], + "InvestigationName": "Malicious mail is zapped - urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef", + "Operation": "AirInvestigationData", + "InvestigationType": "ZappedEmailInvestigation", + "UserId": "AirInvestigation", + "UserKey": "AirInvestigation", + "DeepLinkUrl": "https://security.microsoft.com/abc-investigation/urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890", + "Version": 1, + "InvestigationId": "urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef", + "EndTimeUtc": "2025-09-29T23:59:59", + "LastUpdateTimeUtc": "2025-09-29T23:59:59", + "Id": "44445555-2222-4444-8888-123456789012", + "Status": "Remediated", + "Data": "{\"Version\":\"3.0\",\"VendorName\":\"Microsoft\",\"ProviderName\":\"OATP\",\"AlertType\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"StartTimeUtc\":\"2025-09-29T23:59:59Z\",\"EndTimeUtc\":\"2025-09-29T23:59:59Z\",\"TimeGenerated\":\"2025-09-29T23:59:59.00Z\",\"ProcessingEndTime\":\"2025-09-29T23:59:59.0000000Z\",\"Status\":\"InProgress\",\"Severity\":\"Low\",\"ConfidenceLevel\":\"Unknown\",\"ConfidenceScore\":1,\"IsIncident\":false,\"ProviderAlertId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"SystemAlertId\":null,\"CorrelationKey\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Investigations\":[{\"$id\":\"1\",\"Id\":\"urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\",\"InvestigationStatus\":\"Running\"}],\"InvestigationIds\":[\"urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\"],\"Intent\":\"Probing\",\"ResourceIdentifiers\":[{\"$id\":\"2\",\"AadTenantId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Type\":\"AAD\"}],\"AzureResourceId\":null,\"WorkspaceId\":null,\"WorkspaceSubscriptionId\":null,\"WorkspaceResourceGroup\":null,\"AgentId\":null,\"AlertDisplayName\":\"Email reported by user as malware or phish\",\"Description\":\"This alert is triggered when any email message is reported as malware or phish by users -V1.0.0.3\",\"ExtendedLinks\":[{\"Href\":\"https://security.microsoft.com/viewalerts?id=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Category\":null,\"Label\":\"alert\",\"Type\":\"webLink\"}],\"Metadata\":{\"CustomApps\":null,\"GenericInfo\":null},\"Entities\":[{\"$id\":\"3\",\"Recipient\":\"user@example.com\",\"Urls\":[\"hxxp://test.local\",\"hxxp://test.local\",\"hxxp://test.local\"],\"Threats\":[\"HighConfPhish\"],\"Sender\":\"bounce@example.com\",\"P1Sender\":\"<>\",\"P1SenderDomain\":\"\",\"SenderIP\":\"81.2.69.144\",\"P2Sender\":\"bounce@example.com\",\"P2SenderDisplayName\":\"name\",\"P2SenderDomain\":\"example.com\",\"ReceivedDate\":\"2025-09-29T23:59:59\",\"NetworkMessageId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"InternetMessageId\":\"\",\"Subject\":\"subject\",\"AntispamDirection\":\"Inbound\",\"DeliveryAction\":\"Delivered\",\"ThreatDetectionMethods\":[\"MLModel\"],\"Language\":\"nb\",\"DeliveryLocation\":\"Inbox\",\"OriginalDeliveryLocation\":\"Inbox\",\"PhishConfidenceLevel\":\"High\",\"AdditionalActionsAndResults\":[\"OriginalDelivery: [N/A]\"],\"Connector\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\",\"AuthDetails\":[{\"Name\":\"SPF\",\"Value\":\"Pass\"},{\"Name\":\"DKIM\",\"Value\":\"None\"},{\"Name\":\"DMARC\",\"Value\":\"Fail\"},{\"Name\":\"Comp Auth\",\"Value\":\"fail\"}],\"SystemOverrides\":[],\"Type\":\"mailMessage\",\"Urn\":\"urn:MailEntity:dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"Source\":\"OATP\",\"FirstSeen\":\"2025-09-29T23:59:59\"},{\"$id\":\"4\",\"MailboxPrimaryAddress\":\"user@example.com\",\"Upn\":\"account@example.com\",\"AadId\":\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\",\"RiskLevel\":\"None\",\"Type\":\"mailbox\",\"Urn\":\"urn:UserEntity:abcdef1234567890abcdef1234567890\",\"Source\":\"OATP\",\"FirstSeen\":\"2025-09-29T23:59:59\"}],\"LogCreationTime\":\"2025-09-29T23:59:59.0000000Z\",\"MachineName\":\"ABCDEFGHIJK\",\"SourceTemplateType\":\"Activity_Single\",\"Category\":\"ThreatManagement\",\"SourceAlertType\":\"System\"}", + "CreationTime": "2025-09-29T03:11:01", + "RecordType": 64, + "RunningTime": 135051, + "OrganizationId": "999999999-2222-1111-7777-123456789012", + "ObjectId": "44445555-2222-4444-8888-123456789012", + "UserType": 4, + "Workload": "AirInvestigation" + } + }, + { + "o365audit": { + "LogonType": 0, + "ClientInfoString": "Client=WebServices;Apache-HttpAsyncClient/5.0[AppId=7777777-6666-aaaa-bbbb-123456789012];", + "UserId": "user@example.com", + "Id": "aaaaaaaa-bbbb-cccc-dddd-123456789012", + "UserType": 0, + "ClientIPAddress": "203.0.113.145", + "AppId": "7777777-6666-aaaa-bbbb-123456789012", + "InternalLogonType": 0, + "OriginatingServer": "AB8MB22NO1234 (203.0.113.8)", + "CreationTime": "2025-09-26T22:32:29", + "OrganizationId": "33333333-bbbb-cccc-dddd-123456789012", + "Folders": [ + { + "Path": "\\Sent Items", + "FolderItems": [ + { + "SizeInBytes": 14593, + "Id": "AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEpAAAJ", + "ImmutableId": "CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgO3wRAAAJ", + "InternetMessageId": "" + }, + { + "ImmutableId": "CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgO3gYAAAJ", + "InternetMessageId": "", + "SizeInBytes": 8526, + "Id": "AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEoAAAJ" + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEnAAAJ", + "ImmutableId": "CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAxaAAAJ", + "InternetMessageId": "", + "SizeInBytes": 99635 + }, + { + "ImmutableId": "CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAgkAAAJ", + "InternetMessageId": "", + "SizeInBytes": 6475, + "Id": "AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEmAAAJ" + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEElAAAJ", + "ImmutableId": "CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAgQAAAJ", + "InternetMessageId": "", + "SizeInBytes": 326463 + }, + { + "SizeInBytes": 1352491, + "Id": "AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEkAAAJ", + "ImmutableId": "CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgOv+cAAAJ", + "InternetMessageId": "" + } + ], + "Id": "LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLCCCCCCCCCCCCCCCCCCCCCCCC" + } + ], + "UserKey": "9876543210987656", + "RecordType": 50, + "AppAccessContext": { + "IssuedAtTime": "2025-09-26T22:27:27", + "UniqueTokenId": "ZZZZZZZZZZKKKKKKKKKKAA", + "AADSessionId": "dddddddd-aaaa-eeee-dddd-123456789012", + "APIId": "bbbbbbbb-aaaa-eeee-bbbb-123456789012", + "ClientAppId": "7777777-6666-aaaa-bbbb-123456789012" + }, + "MailboxGuid": "eeeeeeee-aaaa-1234-bbbb-123456789012", + "TokenTenantId": "33333333-bbbb-cccc-dddd-123456789012", + "Version": 1, + "ClientAppId": "7777777-6666-aaaa-bbbb-123456789012", + "TokenObjectId": "ffffffff-aaaa-1234-bbbb-123456789012", + "OperationCount": 6, + "ActorInfoString": "Client=WebServices;Apache-HttpAsyncClient/5.0[AppId=7777777-6666-aaaa-bbbb-123456789012];", + "OrganizationName": "example.onmicrosoft.com", + "LogonUserSid": "S-1-5-21-1234567890-1234567890-123456789012-88888888", + "Operation": "MailItemsAccessed", + "MailboxOwnerUPN": "user@example.com", + "Workload": "Exchange", + "OperationProperties": [ + { + "Name": "MailAccessType", + "Value": "Bind" + } + ], + "ExternalAccess": false, + "ResultStatus": "Succeeded", + "MailboxOwnerSid": "S-1-5-21-1234567890-1234567890-123456789012-88888888" + } + } + ] +} diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json-expected.json new file mode 100644 index 00000000000..af7832dcbf5 --- /dev/null +++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json-expected.json @@ -0,0 +1,787 @@ +{ + "expected": [ + { + "@timestamp": "2025-09-29T01:01:01.000Z", + "client": { + "address": "203.0.113.5", + "ip": "203.0.113.5" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "AttachmentAccess", + "category": [ + "web", + "email" + ], + "code": "ExchangeItemAggregated", + "id": "88888888-4444-5555-6666-123456789012", + "kind": "event", + "original": "{\"OrganizationName\":\"example.onmicrosoft.com\",\"ActorInfoString\":\"Client=REST;Client=RESTSystem;UserAgent=[NoUserAgent][AppId=abcdabcd-1234-12ab-1a2b-ad1234567890];\",\"UserKey\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"MailboxGuid\":\"8b46a639-c47f-4634-b90c-2accecd337e3\",\"Operation\":\"AttachmentAccess\",\"OrganizationId\":\"1234abcd-4321-dcba-43ab-1023456789ab\",\"ClientIPAddress\":\"203.0.113.5\",\"LogonUserSid\":\"S-1-5-21-1234567890-123456789-1234567890-12345678\",\"OriginatingServer\":\"imase12AA1234 (203.0.113.3)\",\"RecordType\":50,\"Version\":1,\"ClientInfoString\":\"Client=REST;Client=RESTSystem;;\",\"ClientAppId\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"MailboxOwnerUPN\":\"user@example.com\",\"OperationCount\":6,\"MailboxOwnerSid\":\"S-1-5-21-1234567890-123456789-1234567890-12345678\",\"Messages\":[{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":2379,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDDEEEEEE12345678901234567890123\"},{\"SizeInBytes\":7356,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDDEEEEEE000000011111122222222222\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDD\"},{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":1156492,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDlFFFFEEEEEEaaaaaaaaaaaaaaaaaaaaaaa\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDlFFFF\"},{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":87052,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDvnQAAEEEEEEAALE\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDvnQAA\"},{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":267212,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJk_eAAAEEEEEEAB_WWWWWWWWWWWWWWW-1234\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJk_eAAA\"},{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":20477,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJ1234AAEEEEEEADGGGGGGGGGGGGGGGGGGGGGG\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJ123456\"}],\"ResultStatus\":\"Succeeded\",\"ExternalAccess\":false,\"LogonType\":0,\"TokenTenantId\":\"dcbadcba-1234-12ab-1a2b-ad1234567890\",\"AppAccessContext\":{\"ClientAppId\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"UniqueTokenId\":\"12345678-1234-1234-abcd-abcdef123456\",\"APIId\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"IssuedAtTime\":\"2025-09-29T01:01:01\"},\"Workload\":\"Exchange\",\"InternalLogonType\":0,\"OperationProperties\":[{\"Value\":\"Bind\",\"Name\":\"AttachmentAccessType\"}],\"AppId\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"UserId\":\"user@example.com\",\"CreationTime\":\"2025-09-29T01:01:01\",\"Id\":\"88888888-4444-5555-6666-123456789012\",\"UserType\":5}", + "outcome": "success", + "provider": "Exchange", + "type": [ + "info", + "access" + ] + }, + "host": { + "id": "1234abcd-4321-dcba-43ab-1023456789ab", + "name": "example.com" + }, + "network": { + "type": "ipv4" + }, + "o365": { + "audit": { + "ActorInfoString": "Client=REST;Client=RESTSystem;UserAgent=[NoUserAgent][AppId=abcdabcd-1234-12ab-1a2b-ad1234567890];", + "AppAccessContext": { + "APIId": "abcdabcd-1234-12ab-1a2b-ad1234567890", + "ClientAppId": "abcdabcd-1234-12ab-1a2b-ad1234567890", + "IssuedAtTime": "2025-09-29T01:01:01", + "UniqueTokenId": "12345678-1234-1234-abcd-abcdef123456" + }, + "AppId": "abcdabcd-1234-12ab-1a2b-ad1234567890", + "ClientAppId": "abcdabcd-1234-12ab-1a2b-ad1234567890", + "ClientInfoString": "Client=REST;Client=RESTSystem;;", + "CreationTime": "2025-09-29T01:01:01", + "ExchangeMessages": [ + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDD", + "MessageItems": [ + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDDEEEEEE12345678901234567890123", + "SizeInBytes": 2379 + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDDEEEEEE000000011111122222222222", + "SizeInBytes": 7356 + } + ], + "Path": "Messages" + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDlFFFF", + "MessageItems": [ + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDlFFFFEEEEEEaaaaaaaaaaaaaaaaaaaaaaa", + "SizeInBytes": 1156492 + } + ], + "Path": "Messages" + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDvnQAA", + "MessageItems": [ + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDvnQAAEEEEEEAALE", + "SizeInBytes": 87052 + } + ], + "Path": "Messages" + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJk_eAAA", + "MessageItems": [ + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJk_eAAAEEEEEEAB_WWWWWWWWWWWWWWW-1234", + "SizeInBytes": 267212 + } + ], + "Path": "Messages" + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJ123456", + "MessageItems": [ + { + "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJ1234AAEEEEEEADGGGGGGGGGGGGGGGGGGGGGG", + "SizeInBytes": 20477 + } + ], + "Path": "Messages" + } + ], + "ExternalAccess": false, + "InternalLogonType": "0", + "LogonType": "0", + "LogonUserSid": "S-1-5-21-1234567890-123456789-1234567890-12345678", + "MailboxGuid": "8b46a639-c47f-4634-b90c-2accecd337e3", + "MailboxOwnerSid": "S-1-5-21-1234567890-123456789-1234567890-12345678", + "MailboxOwnerUPN": "user@example.com", + "OperationCount": "6", + "OperationProperties": [ + { + "Name": "AttachmentAccessType", + "Value": "Bind" + } + ], + "OrganizationName": "example.onmicrosoft.com", + "OriginatingServer": "imase12AA1234 (203.0.113.3)", + "RecordType": "50", + "ResultStatus": "Succeeded", + "TokenTenantId": "dcbadcba-1234-12ab-1a2b-ad1234567890", + "UserId": "user@example.com", + "UserKey": "abcdabcd-1234-12ab-1a2b-ad1234567890", + "UserType": "5", + "Version": "1" + } + }, + "organization": { + "id": "1234abcd-4321-dcba-43ab-1023456789ab" + }, + "related": { + "ip": [ + "203.0.113.5" + ], + "user": [ + "user" + ] + }, + "source": { + "as": { + "number": 64502, + "organization": { + "name": "Documentation ASN" + } + }, + "geo": { + "city_name": "Madrid", + "continent_name": "Europe", + "country_iso_code": "ES", + "country_name": "Spain", + "location": { + "lat": 40.41639, + "lon": -3.7025 + }, + "region_iso_code": "ES-M", + "region_name": "Madrid" + }, + "ip": "203.0.113.5" + }, + "tags": [ + "preserve_original_event" + ], + "token": { + "id": "12345678-1234-1234-abcd-abcdef123456" + }, + "user": { + "domain": "example.com", + "email": "user@example.com", + "id": "user@example.com", + "name": "user" + } + }, + { + "@timestamp": "2025-09-29T03:11:01.000Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "from": { + "address": [ + "bounce@example.com" + ] + }, + "local_id": [ + "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb" + ], + "message_id": [ + "" + ], + "sender": { + "address": [ + "<>" + ] + }, + "subject": [ + "subject" + ], + "to": { + "address": [ + "user@example.com" + ] + } + }, + "event": { + "action": "AirInvestigationData", + "category": [ + "web" + ], + "code": "AirInvestigation", + "id": "44445555-2222-4444-8888-123456789012", + "kind": "event", + "original": "{\"Status\":\"Remediated\",\"StartTimeUtc\":\"2025-09-29T23:59:59\",\"Actions\":[\"{\\\"$id\\\":\\\"1\\\",\\\"ActionId\\\":\\\"urn:EmailZapper:1234567890abcdef1234567890abcdef\\\",\\\"InvestigationId\\\":\\\"urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\\\",\\\"ActionApproval\\\":\\\"None\\\",\\\"ActionType\\\":\\\"EmailRemediation\\\",\\\"ActionStatus\\\":\\\"Skipped\\\",\\\"Entities\\\":[{\\\"$id\\\":\\\"2\\\",\\\"Recipient\\\":\\\"user@example.com\\\",\\\"Urls\\\":[\\\"https://exxample.com/fffffffff\\\",\\\"https://example.com/\\\",\\\"https://www.example.com/\\\",\\\"https://domain.com\\\",\\\"https://domain.com\\\",\\\"https://emailsg.example.com/wf/open?upn=1234\\\",\\\"https://apps.apple.com/us/app/example-com/id12343123444\\\",\\\"https://play.google.com/store/apps/details?id=com.example.example\\\",\\\"https://dl.example.com/boards/6667868113/groups/topics?dl_slug=resortinternet&dl_msgid=99999999-6666-5555-4444-1023456789012&dl_category=notifications_mailer-assign_person_to_pulse&dl_userid=12345678&dl_sessionid=12345678901234567890123456789012_0&dl_senderid=-4&dl_notificationappid=98776666&dl_notificationkindname=board_assigned_in_column&dl_notificationuuid=12345678901234567890123456789012_0\\\",\\\"https://dl.example.com/users/-4-automations?dl_slug=resortinternet&dl_msgid=99999999-6666-5555-4444-1023456789012&dl_category=notifications_mailer-assign_person_to_pulse&dl_userid=12345678&dl_sessionid=12345678901234567890123456789012_0&dl_senderid=-4&dl_notificationappid=12345678&dl_notificationkindname=board_assigned_in_column&dl_notificationuuid=12345678901234567890123456789012_0\\\"],\\\"Threats\\\":[\\\"ZapPhish\\\",\\\"HighConfPhish\\\"],\\\"Sender\\\":\\\"sales@example.com\\\",\\\"P1Sender\\\":\\\"1234565@example.com\\\",\\\"P1SenderDomain\\\":\\\"example.com\\\",\\\"SenderIP\\\":\\\"203.0.113.55\\\",\\\"P2Sender\\\":\\\"sales@example.com\\\",\\\"P2SenderDisplayName\\\":\\\"Postal_ProtocolAdminChecklnReportDocSubmissionRequestapEx12341234Serverange-reply\\\",\\\"P2SenderDomain\\\":\\\"example.com\\\",\\\"ReceivedDate\\\":\\\"2025-09-15T22:20:37\\\",\\\"NetworkMessageId\\\":\\\"33333333-eeee-4444-5555-999999999999\\\",\\\"InternetMessageId\\\":\\\"<1234@email.example.com>\\\",\\\"Subject\\\":\\\"Admin-Protocol-Tasks-Update on 9/15/2025\\\",\\\"AntispamDirection\\\":\\\"Inbound\\\",\\\"DeliveryAction\\\":\\\"Delivered\\\",\\\"Language\\\":\\\"en\\\",\\\"DeliveryLocation\\\":\\\"Quarantine\\\",\\\"OriginalDeliveryLocation\\\":\\\"Inbox\\\",\\\"AdditionalActionsAndResults\\\":[\\\"OriginalDelivery: [N/A]\\\"],\\\"AuthDetails\\\":[{\\\"Name\\\":\\\"SPF\\\",\\\"Value\\\":\\\"Pass\\\"},{\\\"Name\\\":\\\"DKIM\\\",\\\"Value\\\":\\\"Pass\\\"},{\\\"Name\\\":\\\"DMARC\\\",\\\"Value\\\":\\\"Pass\\\"},{\\\"Name\\\":\\\"Comp Auth\\\",\\\"Value\\\":\\\"pass\\\"}],\\\"SystemOverrides\\\":[],\\\"Type\\\":\\\"mailMessage\\\",\\\"Urn\\\":\\\"urn:MailEntity:aaaaaaaaaaaaaaaabbbbbbbbbbbbbbbb\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2025-09-15T22:51:56\\\"}],\\\"RelatedAlertIds\\\":[\\\"33333333-2222-1111-7777-123456789012\\\"],\\\"StartTimeUtc\\\":\\\"2025-09-15T23:02:00\\\",\\\"EndTimeUtc\\\":\\\"2025-09-15T23:04:18Z\\\",\\\"LastUpdateTimeUtc\\\":\\\"2025-09-17T12:22:43.2513154Z\\\",\\\"TimestampUtc\\\":\\\"2025-09-15T23:04:18\\\",\\\"BulkName\\\":\\\"Malicious mail is zapped - urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\\\",\\\"ResourceIdentifiers\\\":[{\\\"$id\\\":\\\"3\\\",\\\"AadTenantId\\\":\\\"999999999-2222-1111-7777-123456789012\\\",\\\"Type\\\":\\\"AAD\\\"}],\\\"PendingType\\\":\\\"User\\\",\\\"Type\\\":\\\"InvestigationAction\\\",\\\"LogCreationTime\\\":\\\"2025-09-17T12:22:43.2513154Z\\\",\\\"MachineName\\\":\\\"AB12AB12AB123\\\",\\\"Description\\\":\\\"For malicious emails, you can move to junk, soft or hard delete from user's mailbox.\\\"}\",\"{\\\"$id\\\":\\\"1\\\",\\\"ActionId\\\":\\\"urn:EmailZapper:4321567890abcdef1234567890abcdef\\\",\\\"InvestigationId\\\":\\\"urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\\\",\\\"ActionApproval\\\":\\\"None\\\",\\\"ActionType\\\":\\\"EmailRemediation\\\",\\\"ActionStatus\\\":\\\"Skipped\\\",\\\"Entities\\\":[{\\\"$id\\\":\\\"2\\\",\\\"NetworkMessageIds\\\":[\\\"88888888-eeee-4444-5555-999999999999\\\",\\\"33333333-eeee-4444-5555-999999999999\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":1,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Delivered\\\":1,\\\"Blocked\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":2},\\\"Query\\\":\\\"( (( (BodyFingerprintBin1:\\\\\\\"99999999999\\\\\\\") ) AND ( (SenderIp:\\\\\\\"203.0.113.55\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"2025-09-15T23:05:00Z\\\",\\\"MailCount\\\":2,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"33333333-eeee-4444-5555-999999999999\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2025-08-26T00:00:00Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2025-09-15T23:05:00Z\\\",\\\"ClusterGroup\\\":\\\"BodyFingerprintBin1,SenderIp\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"BodyFingerprintBin1;SenderIp;ContentType\\\",\\\"ClusterByValue\\\":\\\"99999999999;203.0.113.5;1\\\",\\\"QueryStartTime\\\":\\\"8/26/2025 12:00:00 AM\\\",\\\"QueryTime\\\":\\\"9/15/2025 11:05:00 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:98765432109876543210987654321098\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2025-09-15T23:00:09\\\"}],\\\"RelatedAlertIds\\\":[\\\"33333333-2222-1111-7777-123456789012\\\"],\\\"StartTimeUtc\\\":\\\"2025-09-15T23:02:00\\\",\\\"EndTimeUtc\\\":\\\"2025-09-17T10:37:16\\\",\\\"LastUpdateTimeUtc\\\":\\\"2025-09-17T12:22:43.2525624Z\\\",\\\"TimestampUtc\\\":\\\"2025-09-17T10:37:16\\\",\\\"BulkName\\\":\\\"Malicious mail is zapped - urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\\\",\\\"ResourceIdentifiers\\\":[{\\\"$id\\\":\\\"3\\\",\\\"AadTenantId\\\":\\\"999999999-2222-1111-7777-123456789012\\\",\\\"Type\\\":\\\"AAD\\\"}],\\\"PendingType\\\":\\\"User\\\",\\\"Type\\\":\\\"InvestigationAction\\\",\\\"LogCreationTime\\\":\\\"2025-09-17T12:22:43.2525624Z\\\",\\\"MachineName\\\":\\\"AB12AB12AB123\\\",\\\"Description\\\":\\\"For malicious emails, you can move to junk, soft or hard delete from user's mailbox.\\\"}\",\"{\\\"$id\\\":\\\"1\\\",\\\"ActionId\\\":\\\"urn:EmailZapper:6666567890abcdef1234567890abcdef\\\",\\\"InvestigationId\\\":\\\"urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\\\",\\\"ActionApproval\\\":\\\"None\\\",\\\"ActionType\\\":\\\"EmailRemediation\\\",\\\"ActionStatus\\\":\\\"Skipped\\\",\\\"Entities\\\":[{\\\"$id\\\":\\\"2\\\",\\\"NetworkMessageIds\\\":[\\\"88888888-eeee-4444-5555-999999999999\\\",\\\"33333333-eeee-4444-5555-999999999999\\\",\\\"44444444-eeee-4444-5555-999999999999\\\"],\\\"CountByThreatType\\\":{\\\"HighConfPhish\\\":2,\\\"Phish\\\":0,\\\"Malware\\\":0,\\\"Spam\\\":0},\\\"CountByProtectionStatus\\\":{\\\"Blocked\\\":2,\\\"Delivered\\\":1},\\\"CountByDeliveryLocation\\\":{\\\"Quarantine\\\":3},\\\"Query\\\":\\\"( (( (BodyFingerprintBin1:\\\\\\\"99999999999\\\\\\\") ) AND ( (P2SenderDomain:\\\\\\\"example.com\\\\\\\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))\\\",\\\"QueryTime\\\":\\\"2025-09-15T23:05:00Z\\\",\\\"MailCount\\\":3,\\\"IsVolumeAnamoly\\\":false,\\\"ClusterSourceIdentifier\\\":\\\"33333333-eeee-4444-5555-999999999999\\\",\\\"ClusterSourceType\\\":\\\"Similarity\\\",\\\"ClusterQueryStartTime\\\":\\\"2025-08-26T00:00:00Z\\\",\\\"ClusterQueryEndTime\\\":\\\"2025-09-15T23:05:00Z\\\",\\\"ClusterGroup\\\":\\\"BodyFingerprintBin1,P2SenderDomain\\\",\\\"Type\\\":\\\"mailCluster\\\",\\\"ClusterBy\\\":\\\"BodyFingerprintBin1;P2SenderDomain;ContentType\\\",\\\"ClusterByValue\\\":\\\"99999999999;example.com;1\\\",\\\"QueryStartTime\\\":\\\"8/26/2025 12:00:00 AM\\\",\\\"QueryTime\\\":\\\"9/15/2025 11:05:00 PM\\\",\\\"Urn\\\":\\\"urn:MailClusterEntity:cccccccccccccccccccccccccccccccc\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2025-09-15T23:00:09\\\"}],\\\"RelatedAlertIds\\\":[\\\"33333333-2222-1111-7777-123456789012\\\"],\\\"StartTimeUtc\\\":\\\"2025-09-15T23:02:00\\\",\\\"EndTimeUtc\\\":\\\"2025-09-17T10:37:16\\\",\\\"LastUpdateTimeUtc\\\":\\\"2025-09-17T12:22:43.2676112Z\\\",\\\"TimestampUtc\\\":\\\"2025-09-17T10:37:16\\\",\\\"BulkName\\\":\\\"Malicious mail is zapped - urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\\\",\\\"ResourceIdentifiers\\\":[{\\\"$id\\\":\\\"3\\\",\\\"AadTenantId\\\":\\\"999999999-2222-1111-7777-123456789012\\\",\\\"Type\\\":\\\"AAD\\\"}],\\\"PendingType\\\":\\\"User\\\",\\\"Type\\\":\\\"InvestigationAction\\\",\\\"LogCreationTime\\\":\\\"2025-09-17T12:22:43.2676112Z\\\",\\\"MachineName\\\":\\\"AB12AB12AB123\\\",\\\"Description\\\":\\\"For malicious emails, you can move to junk, soft or hard delete from user's mailbox.\\\"}\"],\"ObjectId\":\"44445555-2222-4444-8888-123456789012\",\"InvestigationType\":\"ZappedEmailInvestigation\",\"UserKey\":\"AirInvestigation\",\"Data\":\"{\\\"Version\\\":\\\"3.0\\\",\\\"VendorName\\\":\\\"Microsoft\\\",\\\"ProviderName\\\":\\\"OATP\\\",\\\"AlertType\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"StartTimeUtc\\\":\\\"2025-09-29T23:59:59Z\\\",\\\"EndTimeUtc\\\":\\\"2025-09-29T23:59:59Z\\\",\\\"TimeGenerated\\\":\\\"2025-09-29T23:59:59.00Z\\\",\\\"ProcessingEndTime\\\":\\\"2025-09-29T23:59:59.0000000Z\\\",\\\"Status\\\":\\\"InProgress\\\",\\\"Severity\\\":\\\"Low\\\",\\\"ConfidenceLevel\\\":\\\"Unknown\\\",\\\"ConfidenceScore\\\":1,\\\"IsIncident\\\":false,\\\"ProviderAlertId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"SystemAlertId\\\":null,\\\"CorrelationKey\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Investigations\\\":[{\\\"$id\\\":\\\"1\\\",\\\"Id\\\":\\\"urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\\\",\\\"InvestigationStatus\\\":\\\"Running\\\"}],\\\"InvestigationIds\\\":[\\\"urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\\\"],\\\"Intent\\\":\\\"Probing\\\",\\\"ResourceIdentifiers\\\":[{\\\"$id\\\":\\\"2\\\",\\\"AadTenantId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Type\\\":\\\"AAD\\\"}],\\\"AzureResourceId\\\":null,\\\"WorkspaceId\\\":null,\\\"WorkspaceSubscriptionId\\\":null,\\\"WorkspaceResourceGroup\\\":null,\\\"AgentId\\\":null,\\\"AlertDisplayName\\\":\\\"Email reported by user as malware or phish\\\",\\\"Description\\\":\\\"This alert is triggered when any email message is reported as malware or phish by users -V1.0.0.3\\\",\\\"ExtendedLinks\\\":[{\\\"Href\\\":\\\"https://security.microsoft.com/viewalerts?id=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Category\\\":null,\\\"Label\\\":\\\"alert\\\",\\\"Type\\\":\\\"webLink\\\"}],\\\"Metadata\\\":{\\\"CustomApps\\\":null,\\\"GenericInfo\\\":null},\\\"Entities\\\":[{\\\"$id\\\":\\\"3\\\",\\\"Recipient\\\":\\\"user@example.com\\\",\\\"Urls\\\":[\\\"hxxp://test.local\\\",\\\"hxxp://test.local\\\",\\\"hxxp://test.local\\\"],\\\"Threats\\\":[\\\"HighConfPhish\\\"],\\\"Sender\\\":\\\"bounce@example.com\\\",\\\"P1Sender\\\":\\\"<>\\\",\\\"P1SenderDomain\\\":\\\"\\\",\\\"SenderIP\\\":\\\"81.2.69.144\\\",\\\"P2Sender\\\":\\\"bounce@example.com\\\",\\\"P2SenderDisplayName\\\":\\\"name\\\",\\\"P2SenderDomain\\\":\\\"example.com\\\",\\\"ReceivedDate\\\":\\\"2025-09-29T23:59:59\\\",\\\"NetworkMessageId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"InternetMessageId\\\":\\\"\\\",\\\"Subject\\\":\\\"subject\\\",\\\"AntispamDirection\\\":\\\"Inbound\\\",\\\"DeliveryAction\\\":\\\"Delivered\\\",\\\"ThreatDetectionMethods\\\":[\\\"MLModel\\\"],\\\"Language\\\":\\\"nb\\\",\\\"DeliveryLocation\\\":\\\"Inbox\\\",\\\"OriginalDeliveryLocation\\\":\\\"Inbox\\\",\\\"PhishConfidenceLevel\\\":\\\"High\\\",\\\"AdditionalActionsAndResults\\\":[\\\"OriginalDelivery: [N/A]\\\"],\\\"Connector\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]\\\",\\\"AuthDetails\\\":[{\\\"Name\\\":\\\"SPF\\\",\\\"Value\\\":\\\"Pass\\\"},{\\\"Name\\\":\\\"DKIM\\\",\\\"Value\\\":\\\"None\\\"},{\\\"Name\\\":\\\"DMARC\\\",\\\"Value\\\":\\\"Fail\\\"},{\\\"Name\\\":\\\"Comp Auth\\\",\\\"Value\\\":\\\"fail\\\"}],\\\"SystemOverrides\\\":[],\\\"Type\\\":\\\"mailMessage\\\",\\\"Urn\\\":\\\"urn:MailEntity:dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2025-09-29T23:59:59\\\"},{\\\"$id\\\":\\\"4\\\",\\\"MailboxPrimaryAddress\\\":\\\"user@example.com\\\",\\\"Upn\\\":\\\"account@example.com\\\",\\\"AadId\\\":\\\"dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb\\\",\\\"RiskLevel\\\":\\\"None\\\",\\\"Type\\\":\\\"mailbox\\\",\\\"Urn\\\":\\\"urn:UserEntity:abcdef1234567890abcdef1234567890\\\",\\\"Source\\\":\\\"OATP\\\",\\\"FirstSeen\\\":\\\"2025-09-29T23:59:59\\\"}],\\\"LogCreationTime\\\":\\\"2025-09-29T23:59:59.0000000Z\\\",\\\"MachineName\\\":\\\"ABCDEFGHIJK\\\",\\\"SourceTemplateType\\\":\\\"Activity_Single\\\",\\\"Category\\\":\\\"ThreatManagement\\\",\\\"SourceAlertType\\\":\\\"System\\\"}\",\"DeepLinkUrl\":\"https://security.microsoft.com/abc-investigation/urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890\",\"Operation\":\"AirInvestigationData\",\"OrganizationId\":\"999999999-2222-1111-7777-123456789012\",\"EndTimeUtc\":\"2025-09-29T23:59:59\",\"InvestigationId\":\"urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\",\"Workload\":\"AirInvestigation\",\"RecordType\":64,\"Version\":1,\"UserId\":\"AirInvestigation\",\"CreationTime\":\"2025-09-29T03:11:01\",\"InvestigationName\":\"Malicious mail is zapped - urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef\",\"Id\":\"44445555-2222-4444-8888-123456789012\",\"RunningTime\":135051,\"UserType\":4,\"LastUpdateTimeUtc\":\"2025-09-29T23:59:59\"}", + "outcome": "success", + "provider": "AirInvestigation", + "type": [ + "info" + ] + }, + "host": { + "id": "999999999-2222-1111-7777-123456789012" + }, + "o365": { + "audit": { + "Actions": [ + { + "$id": "1", + "ActionApproval": "None", + "ActionId": "urn:EmailZapper:1234567890abcdef1234567890abcdef", + "ActionStatus": "Skipped", + "ActionType": "EmailRemediation", + "BulkName": "Malicious mail is zapped - urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef", + "Description": "For malicious emails, you can move to junk, soft or hard delete from user's mailbox.", + "EndTimeUtc": "2025-09-15T23:04:18Z", + "Entities": [ + { + "$id": "2", + "AdditionalActionsAndResults": [ + "OriginalDelivery: [N/A]" + ], + "AntispamDirection": "Inbound", + "AuthDetails": [ + { + "Name": "SPF", + "Value": "Pass" + }, + { + "Name": "DKIM", + "Value": "Pass" + }, + { + "Name": "DMARC", + "Value": "Pass" + }, + { + "Name": "Comp Auth", + "Value": "pass" + } + ], + "DeliveryAction": "Delivered", + "DeliveryLocation": "Quarantine", + "FirstSeen": "2025-09-15T22:51:56", + "InternetMessageId": "<1234@email.example.com>", + "Language": "en", + "NetworkMessageId": "33333333-eeee-4444-5555-999999999999", + "OriginalDeliveryLocation": "Inbox", + "P1Sender": "1234565@example.com", + "P1SenderDomain": "example.com", + "P2Sender": "sales@example.com", + "P2SenderDisplayName": "Postal_ProtocolAdminChecklnReportDocSubmissionRequestapEx12341234Serverange-reply", + "P2SenderDomain": "example.com", + "ReceivedDate": "2025-09-15T22:20:37", + "Recipient": "user@example.com", + "Sender": "sales@example.com", + "SenderIP": "203.0.113.55", + "Source": "OATP", + "Subject": "Admin-Protocol-Tasks-Update on 9/15/2025", + "Threats": [ + "ZapPhish", + "HighConfPhish" + ], + "Type": "mailMessage", + "Urls": [ + "https://exxample.com/fffffffff", + "https://example.com/", + "https://www.example.com/", + "https://domain.com", + "https://domain.com", + "https://emailsg.example.com/wf/open?upn=1234", + "https://apps.apple.com/us/app/example-com/id12343123444", + "https://play.google.com/store/apps/details?id=com.example.example", + "https://dl.example.com/boards/6667868113/groups/topics?dl_slug=resortinternet&dl_msgid=99999999-6666-5555-4444-1023456789012&dl_category=notifications_mailer-assign_person_to_pulse&dl_userid=12345678&dl_sessionid=12345678901234567890123456789012_0&dl_senderid=-4&dl_notificationappid=98776666&dl_notificationkindname=board_assigned_in_column&dl_notificationuuid=12345678901234567890123456789012_0", + "https://dl.example.com/users/-4-automations?dl_slug=resortinternet&dl_msgid=99999999-6666-5555-4444-1023456789012&dl_category=notifications_mailer-assign_person_to_pulse&dl_userid=12345678&dl_sessionid=12345678901234567890123456789012_0&dl_senderid=-4&dl_notificationappid=12345678&dl_notificationkindname=board_assigned_in_column&dl_notificationuuid=12345678901234567890123456789012_0" + ], + "Urn": "urn:MailEntity:aaaaaaaaaaaaaaaabbbbbbbbbbbbbbbb" + } + ], + "InvestigationId": "urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef", + "LastUpdateTimeUtc": "2025-09-17T12:22:43.2513154Z", + "LogCreationTime": "2025-09-17T12:22:43.2513154Z", + "MachineName": "AB12AB12AB123", + "PendingType": "User", + "RelatedAlertIds": [ + "33333333-2222-1111-7777-123456789012" + ], + "ResourceIdentifiers": [ + { + "$id": "3", + "AadTenantId": "999999999-2222-1111-7777-123456789012", + "Type": "AAD" + } + ], + "StartTimeUtc": "2025-09-15T23:02:00", + "TimestampUtc": "2025-09-15T23:04:18", + "Type": "InvestigationAction" + }, + { + "$id": "1", + "ActionApproval": "None", + "ActionId": "urn:EmailZapper:4321567890abcdef1234567890abcdef", + "ActionStatus": "Skipped", + "ActionType": "EmailRemediation", + "BulkName": "Malicious mail is zapped - urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef", + "Description": "For malicious emails, you can move to junk, soft or hard delete from user's mailbox.", + "EndTimeUtc": "2025-09-17T10:37:16", + "Entities": [ + { + "$id": "2", + "ClusterBy": "BodyFingerprintBin1;SenderIp;ContentType", + "ClusterByValue": "99999999999;203.0.113.5;1", + "ClusterGroup": "BodyFingerprintBin1,SenderIp", + "ClusterQueryEndTime": "2025-09-15T23:05:00Z", + "ClusterQueryStartTime": "2025-08-26T00:00:00Z", + "ClusterSourceIdentifier": "33333333-eeee-4444-5555-999999999999", + "ClusterSourceType": "Similarity", + "CountByDeliveryLocation": { + "Quarantine": 2 + }, + "CountByProtectionStatus": { + "Blocked": 1, + "Delivered": 1 + }, + "CountByThreatType": { + "HighConfPhish": 1, + "Malware": 0, + "Phish": 0, + "Spam": 0 + }, + "FirstSeen": "2025-09-15T23:00:09", + "IsVolumeAnamoly": false, + "MailCount": 2, + "NetworkMessageIds": [ + "88888888-eeee-4444-5555-999999999999", + "33333333-eeee-4444-5555-999999999999" + ], + "Query": "( (( (BodyFingerprintBin1:\"99999999999\") ) AND ( (SenderIp:\"203.0.113.55\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))", + "QueryStartTime": "8/26/2025 12:00:00 AM", + "QueryTime": "2025-09-15T23:05:00Z", + "Source": "OATP", + "Type": "mailCluster", + "Urn": "urn:MailClusterEntity:98765432109876543210987654321098" + } + ], + "InvestigationId": "urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef", + "LastUpdateTimeUtc": "2025-09-17T12:22:43.2525624Z", + "LogCreationTime": "2025-09-17T12:22:43.2525624Z", + "MachineName": "AB12AB12AB123", + "PendingType": "User", + "RelatedAlertIds": [ + "33333333-2222-1111-7777-123456789012" + ], + "ResourceIdentifiers": [ + { + "$id": "3", + "AadTenantId": "999999999-2222-1111-7777-123456789012", + "Type": "AAD" + } + ], + "StartTimeUtc": "2025-09-15T23:02:00", + "TimestampUtc": "2025-09-17T10:37:16", + "Type": "InvestigationAction" + }, + { + "$id": "1", + "ActionApproval": "None", + "ActionId": "urn:EmailZapper:6666567890abcdef1234567890abcdef", + "ActionStatus": "Skipped", + "ActionType": "EmailRemediation", + "BulkName": "Malicious mail is zapped - urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef", + "Description": "For malicious emails, you can move to junk, soft or hard delete from user's mailbox.", + "EndTimeUtc": "2025-09-17T10:37:16", + "Entities": [ + { + "$id": "2", + "ClusterBy": "BodyFingerprintBin1;P2SenderDomain;ContentType", + "ClusterByValue": "99999999999;example.com;1", + "ClusterGroup": "BodyFingerprintBin1,P2SenderDomain", + "ClusterQueryEndTime": "2025-09-15T23:05:00Z", + "ClusterQueryStartTime": "2025-08-26T00:00:00Z", + "ClusterSourceIdentifier": "33333333-eeee-4444-5555-999999999999", + "ClusterSourceType": "Similarity", + "CountByDeliveryLocation": { + "Quarantine": 3 + }, + "CountByProtectionStatus": { + "Blocked": 2, + "Delivered": 1 + }, + "CountByThreatType": { + "HighConfPhish": 2, + "Malware": 0, + "Phish": 0, + "Spam": 0 + }, + "FirstSeen": "2025-09-15T23:00:09", + "IsVolumeAnamoly": false, + "MailCount": 3, + "NetworkMessageIds": [ + "88888888-eeee-4444-5555-999999999999", + "33333333-eeee-4444-5555-999999999999", + "44444444-eeee-4444-5555-999999999999" + ], + "Query": "( (( (BodyFingerprintBin1:\"99999999999\") ) AND ( (P2SenderDomain:\"example.com\") ) AND ( (ContentType: 1) )) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:PhishEdu) AND NOT(XmiInfoTenantPolicyFinalVerdictSource:SecOps))", + "QueryStartTime": "8/26/2025 12:00:00 AM", + "QueryTime": "2025-09-15T23:05:00Z", + "Source": "OATP", + "Type": "mailCluster", + "Urn": "urn:MailClusterEntity:cccccccccccccccccccccccccccccccc" + } + ], + "InvestigationId": "urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef", + "LastUpdateTimeUtc": "2025-09-17T12:22:43.2676112Z", + "LogCreationTime": "2025-09-17T12:22:43.2676112Z", + "MachineName": "AB12AB12AB123", + "PendingType": "User", + "RelatedAlertIds": [ + "33333333-2222-1111-7777-123456789012" + ], + "ResourceIdentifiers": [ + { + "$id": "3", + "AadTenantId": "999999999-2222-1111-7777-123456789012", + "Type": "AAD" + } + ], + "StartTimeUtc": "2025-09-15T23:02:00", + "TimestampUtc": "2025-09-17T10:37:16", + "Type": "InvestigationAction" + } + ], + "CreationTime": "2025-09-29T03:11:01", + "Data": { + "flattened": { + "AlertDisplayName": "Email reported by user as malware or phish", + "AlertType": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb", + "Category": "ThreatManagement", + "ConfidenceLevel": "Unknown", + "ConfidenceScore": 1, + "CorrelationKey": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb", + "Description": "This alert is triggered when any email message is reported as malware or phish by users -V1.0.0.3", + "EndTimeUtc": "2025-09-29T23:59:59Z", + "Entities": [ + { + "$id": "3", + "AdditionalActionsAndResults": [ + "OriginalDelivery: [N/A]" + ], + "AntispamDirection": "Inbound", + "AuthDetails": [ + { + "Name": "SPF", + "Value": "Pass" + }, + { + "Name": "DKIM", + "Value": "None" + }, + { + "Name": "DMARC", + "Value": "Fail" + }, + { + "Name": "Comp Auth", + "Value": "fail" + } + ], + "Connector": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb: [Inbound from dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb]", + "DeliveryAction": "Delivered", + "DeliveryLocation": "Inbox", + "FirstSeen": "2025-09-29T23:59:59", + "InternetMessageId": "", + "Language": "nb", + "NetworkMessageId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb", + "OriginalDeliveryLocation": "Inbox", + "P1Sender": "<>", + "P2Sender": "bounce@example.com", + "P2SenderDisplayName": "name", + "P2SenderDomain": "example.com", + "PhishConfidenceLevel": "High", + "ReceivedDate": "2025-09-29T23:59:59", + "Recipient": "user@example.com", + "Sender": "bounce@example.com", + "SenderIP": "81.2.69.144", + "Source": "OATP", + "Subject": "subject", + "ThreatDetectionMethods": [ + "MLModel" + ], + "Threats": [ + "HighConfPhish" + ], + "Type": "mailMessage", + "Urls": [ + "hxxp://test.local", + "hxxp://test.local", + "hxxp://test.local" + ], + "Urn": "urn:MailEntity:dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb" + }, + { + "$id": "4", + "AadId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb", + "FirstSeen": "2025-09-29T23:59:59", + "MailboxPrimaryAddress": "user@example.com", + "RiskLevel": "None", + "Source": "OATP", + "Type": "mailbox", + "Upn": "account@example.com", + "Urn": "urn:UserEntity:abcdef1234567890abcdef1234567890" + } + ], + "ExtendedLinks": [ + { + "Href": "https://security.microsoft.com/viewalerts?id=dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb", + "Label": "alert", + "Type": "webLink" + } + ], + "Intent": "Probing", + "InvestigationIds": [ + "urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890" + ], + "Investigations": [ + { + "$id": "1", + "Id": "urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890", + "InvestigationStatus": "Running" + } + ], + "IsIncident": false, + "LogCreationTime": "2025-09-29T23:59:59.0000000Z", + "MachineName": "ABCDEFGHIJK", + "ProcessingEndTime": "2025-09-29T23:59:59.0000000Z", + "ProviderAlertId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb", + "ProviderName": "OATP", + "ResourceIdentifiers": [ + { + "$id": "2", + "AadTenantId": "dddddddd-cccc-eeee-aaaa-bbbbbbbbbbbb", + "Type": "AAD" + } + ], + "Severity": "Low", + "SourceAlertType": "System", + "SourceTemplateType": "Activity_Single", + "StartTimeUtc": "2025-09-29T23:59:59Z", + "Status": "InProgress", + "TimeGenerated": "2025-09-29T23:59:59.00Z", + "VendorName": "Microsoft", + "Version": "3.0" + } + }, + "DeepLinkUrl": "https://security.microsoft.com/abc-investigation/urn:SubmissionInvestigation:abcdef1234567890abcdef1234567890", + "EndTimeUtc": "2025-09-29T23:59:59.000Z", + "InvestigationId": "urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef", + "InvestigationName": "Malicious mail is zapped - urn:ZappedEmailInvestigation:7777567890abcdef1234567890abcdef", + "InvestigationType": "ZappedEmailInvestigation", + "LastUpdateTimeUtc": "2025-09-29T23:59:59.000Z", + "ObjectId": "44445555-2222-4444-8888-123456789012", + "OriginalDeliveryLocation": [ + "Inbox" + ], + "PhishConfidenceLevel": [ + "High" + ], + "RecordType": "64", + "RunningTime": "135051", + "StartTimeUtc": "2025-09-29T23:59:59.000Z", + "Status": "Remediated", + "ThreatDetectionMethods": [ + "MLModel" + ], + "UserId": "AirInvestigation", + "UserKey": "AirInvestigation", + "UserType": "4", + "Version": "1" + } + }, + "organization": { + "id": "999999999-2222-1111-7777-123456789012" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "account@example.com" + ] + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "email": [ + "user@example.com" + ], + "id": "AirInvestigation" + } + }, + { + "@timestamp": "2025-09-26T22:32:29.000Z", + "client": { + "address": "203.0.113.145", + "ip": "203.0.113.145" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "MailItemsAccessed", + "category": [ + "web", + "email" + ], + "code": "ExchangeItemAggregated", + "id": "aaaaaaaa-bbbb-cccc-dddd-123456789012", + "kind": "event", + "original": "{\"OrganizationName\":\"example.onmicrosoft.com\",\"ActorInfoString\":\"Client=WebServices;Apache-HttpAsyncClient/5.0[AppId=7777777-6666-aaaa-bbbb-123456789012];\",\"UserKey\":\"9876543210987656\",\"MailboxGuid\":\"eeeeeeee-aaaa-1234-bbbb-123456789012\",\"Operation\":\"MailItemsAccessed\",\"OrganizationId\":\"33333333-bbbb-cccc-dddd-123456789012\",\"ClientIPAddress\":\"203.0.113.145\",\"TokenObjectId\":\"ffffffff-aaaa-1234-bbbb-123456789012\",\"LogonUserSid\":\"S-1-5-21-1234567890-1234567890-123456789012-88888888\",\"OriginatingServer\":\"AB8MB22NO1234 (203.0.113.8)\",\"RecordType\":50,\"Version\":1,\"ClientInfoString\":\"Client=WebServices;Apache-HttpAsyncClient/5.0[AppId=7777777-6666-aaaa-bbbb-123456789012];\",\"ClientAppId\":\"7777777-6666-aaaa-bbbb-123456789012\",\"MailboxOwnerUPN\":\"user@example.com\",\"OperationCount\":6,\"Folders\":[{\"Path\":\"\\\\Sent Items\",\"Id\":\"LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLCCCCCCCCCCCCCCCCCCCCCCCC\",\"FolderItems\":[{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgO3wRAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":14593,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEpAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgO3gYAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":8526,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEoAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAxaAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":99635,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEnAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAgkAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":6475,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEmAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAgQAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":326463,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEElAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgOv+cAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":1352491,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEkAAAJ\"}]}],\"MailboxOwnerSid\":\"S-1-5-21-1234567890-1234567890-123456789012-88888888\",\"ResultStatus\":\"Succeeded\",\"ExternalAccess\":false,\"LogonType\":0,\"TokenTenantId\":\"33333333-bbbb-cccc-dddd-123456789012\",\"AppAccessContext\":{\"AADSessionId\":\"dddddddd-aaaa-eeee-dddd-123456789012\",\"ClientAppId\":\"7777777-6666-aaaa-bbbb-123456789012\",\"UniqueTokenId\":\"ZZZZZZZZZZKKKKKKKKKKAA\",\"APIId\":\"bbbbbbbb-aaaa-eeee-bbbb-123456789012\",\"IssuedAtTime\":\"2025-09-26T22:27:27\"},\"Workload\":\"Exchange\",\"InternalLogonType\":0,\"OperationProperties\":[{\"Value\":\"Bind\",\"Name\":\"MailAccessType\"}],\"AppId\":\"7777777-6666-aaaa-bbbb-123456789012\",\"UserId\":\"user@example.com\",\"CreationTime\":\"2025-09-26T22:32:29\",\"Id\":\"aaaaaaaa-bbbb-cccc-dddd-123456789012\",\"UserType\":0}", + "outcome": "success", + "provider": "Exchange", + "type": [ + "info", + "access" + ] + }, + "host": { + "id": "33333333-bbbb-cccc-dddd-123456789012", + "name": "example.com" + }, + "network": { + "type": "ipv4" + }, + "o365": { + "audit": { + "ActorInfoString": "Client=WebServices;Apache-HttpAsyncClient/5.0[AppId=7777777-6666-aaaa-bbbb-123456789012];", + "AppAccessContext": { + "AADSessionId": "dddddddd-aaaa-eeee-dddd-123456789012", + "APIId": "bbbbbbbb-aaaa-eeee-bbbb-123456789012", + "ClientAppId": "7777777-6666-aaaa-bbbb-123456789012", + "IssuedAtTime": "2025-09-26T22:27:27", + "UniqueTokenId": "ZZZZZZZZZZKKKKKKKKKKAA" + }, + "AppId": "7777777-6666-aaaa-bbbb-123456789012", + "ClientAppId": "7777777-6666-aaaa-bbbb-123456789012", + "ClientInfoString": "Client=WebServices;Apache-HttpAsyncClient/5.0[AppId=7777777-6666-aaaa-bbbb-123456789012];", + "CreationTime": "2025-09-26T22:32:29", + "ExchangeFolders": [ + { + "FolderItems": [ + { + "Id": "AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEpAAAJ", + "ImmutableId": "CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgO3wRAAAJ", + "InternetMessageId": "", + "SizeInBytes": 14593 + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEoAAAJ", + "ImmutableId": "CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgO3gYAAAJ", + "InternetMessageId": "", + "SizeInBytes": 8526 + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEnAAAJ", + "ImmutableId": "CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAxaAAAJ", + "InternetMessageId": "", + "SizeInBytes": 99635 + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEmAAAJ", + "ImmutableId": "CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAgkAAAJ", + "InternetMessageId": "", + "SizeInBytes": 6475 + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEElAAAJ", + "ImmutableId": "CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAgQAAAJ", + "InternetMessageId": "", + "SizeInBytes": 326463 + }, + { + "Id": "AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEkAAAJ", + "ImmutableId": "CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgOv+cAAAJ", + "InternetMessageId": "", + "SizeInBytes": 1352491 + } + ], + "Id": "LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLCCCCCCCCCCCCCCCCCCCCCCCC", + "Path": "\\Sent Items" + } + ], + "ExternalAccess": false, + "InternalLogonType": "0", + "LogonType": "0", + "LogonUserSid": "S-1-5-21-1234567890-1234567890-123456789012-88888888", + "MailboxGuid": "eeeeeeee-aaaa-1234-bbbb-123456789012", + "MailboxOwnerSid": "S-1-5-21-1234567890-1234567890-123456789012-88888888", + "MailboxOwnerUPN": "user@example.com", + "OperationCount": "6", + "OperationProperties": [ + { + "Name": "MailAccessType", + "Value": "Bind" + } + ], + "OrganizationName": "example.onmicrosoft.com", + "OriginatingServer": "AB8MB22NO1234 (203.0.113.8)", + "RecordType": "50", + "ResultStatus": "Succeeded", + "TokenObjectId": "ffffffff-aaaa-1234-bbbb-123456789012", + "TokenTenantId": "33333333-bbbb-cccc-dddd-123456789012", + "UserId": "user@example.com", + "UserKey": "9876543210987656", + "UserType": "0", + "Version": "1" + } + }, + "organization": { + "id": "33333333-bbbb-cccc-dddd-123456789012" + }, + "related": { + "ip": [ + "203.0.113.145" + ], + "user": [ + "user" + ] + }, + "session": { + "id": "dddddddd-aaaa-eeee-dddd-123456789012" + }, + "source": { + "as": { + "number": 64502, + "organization": { + "name": "Documentation ASN" + } + }, + "geo": { + "city_name": "Madrid", + "continent_name": "Europe", + "country_iso_code": "ES", + "country_name": "Spain", + "location": { + "lat": 40.41639, + "lon": -3.7025 + }, + "region_iso_code": "ES-M", + "region_name": "Madrid" + }, + "ip": "203.0.113.145" + }, + "tags": [ + "preserve_original_event" + ], + "token": { + "id": "ZZZZZZZZZZKKKKKKKKKKAA" + }, + "user": { + "domain": "example.com", + "email": "user@example.com", + "id": "user@example.com", + "name": "user" + } + } + ] +} diff --git a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 0ce7532a878..b23ae1418e2 100644 --- a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -132,11 +132,12 @@ processors: if (!(ctx.o365audit.Actions instanceof List)) { ctx.o365audit.Actions = [ctx.o365audit.Actions]; } + def regex = /,\"QueryTime\":\"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M\"|\"QueryTime\":\"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M\",/; for (def e: ctx.o365audit.Actions) { if (e instanceof Map) { actions.add(e); } else if (e instanceof String) { - ctx._tmp.action_strings.add(e); + ctx._tmp.action_strings.add(regex.matcher(e).replaceAll('')); } } if (actions.length == ctx.o365audit.Actions.length) { @@ -672,11 +673,11 @@ processors: target_field: file.extension ignore_missing: true if: ctx.event?.code != null && ["SharePointFileOperation", "SharePointSharingOperation"].contains(ctx.event.code) - - append: + - append: field: event.category value: file if: 'ctx.event?.action != null && ["FileAccessed", "FileDeleted", "FileDownloaded", "FileModified", "FileMoved", "FileRenamed", "FileRestored", "FileUploaded", "FolderCopied", "FolderCreated", "FolderDeleted", "FolderModified", "FolderMoved", "FolderRenamed", "FolderRestored"].contains(ctx.event?.action)' - - append: + - append: field: event.category value: configuration if: ctx.event?.action == "ComplianceSettingChanged" @@ -1398,6 +1399,26 @@ processors: } else { ctx.o365audit.YammerNetworkId = ctx.o365audit.YammerNetworkId.toString(); } + - script: + tag: convert_runningtime + description: Ensure that RunningTime is not rendered with e-notation or other numeric + if: ctx.o365audit?.RunningTime != null + source: |- + if (ctx.o365audit.RunningTime instanceof double) { + ctx.o365audit.RunningTime = ((long)ctx.o365audit.RunningTime).toString(); + } else { + ctx.o365audit.RunningTime = ctx.o365audit.RunningTime.toString(); + } + - script: + tag: convert_operationcount + description: Ensure that OperationCount is not rendered with e-notation or other numeric + if: ctx.o365audit?.OperationCount != null + source: |- + if (ctx.o365audit.OperationCount instanceof double) { + ctx.o365audit.OperationCount = ((long)ctx.o365audit.OperationCount).toString(); + } else { + ctx.o365audit.OperationCount = ctx.o365audit.OperationCount.toString(); + } - append: field: email.message_id value: "{{{o365audit.InternetMessageId}}}" @@ -1446,6 +1467,7 @@ processors: field: o365audit.EndTimeUtc target_field: o365audit.EndTimeUtc tag: date_EndTimeUtc + timezone: "UTC" formats: - ISO8601 if: ctx.o365audit?.EndTimeUtc != null @@ -1770,6 +1792,36 @@ processors: copy_from: o365audit.ApplicationDisplayName tag: set_application_name ignore_empty_value: true + + # ExchangeItemAggregated Schema + - append: + field: event.type + value: access + if: 'ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50"' + - append: + field: event.category + value: email + if: 'ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50"' + - rename: + field: o365audit.Messages + target_field: o365audit.ExchangeMessages + tag: rename_messages_exchange + if: 'ctx.o365audit?.Messages != null && ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50"' + - remove: + field: o365audit.Messages + tag: remove_messages_field + if: 'ctx.o365audit?.Messages != null' + description: 'remove o365audit.Messages if we have not explicitly renamed them based on record type' + - rename: + field: o365audit.Folders + target_field: o365audit.ExchangeFolders + tag: rename_folders_exchange + if: 'ctx.o365audit?.Folders != null && ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50"' + - remove: + field: o365audit.Folders + tag: remove_folders_field + if: 'ctx.o365audit?.Folders != null' + description: 'remove o365audit.Folders if we have not explicitly renamed them based on record type' - script: description: Handle _tmp.entities.ThreatDetectionMethods containing list of lists. lang: painless diff --git a/packages/o365/data_stream/audit/fields/fields.yml b/packages/o365/data_stream/audit/fields/fields.yml index 64081ffd3c1..d3d2a5e530a 100644 --- a/packages/o365/data_stream/audit/fields/fields.yml +++ b/packages/o365/data_stream/audit/fields/fields.yml @@ -16,6 +16,8 @@ type: keyword - name: ActorContextId type: keyword + - name: ActorInfoString + type: keyword - name: ActorIpAddress type: keyword - name: ActorUserId @@ -275,6 +277,53 @@ # not expressible here; object_type_mapping_type cannot be 'boolean'. object_type: keyword object_type_mapping_type: '*' + - name: ExchangeFolders + type: nested + description: List of folders + fields: + - name: Path + type: keyword + description: Path of the folder + - name: Id + type: keyword + description: Folder ID + - name: FolderItems + type: nested + description: Items in the folder + fields: + - name: SizeInBytes + type: long + description: Size of the item in bytes + - name: Id + type: keyword + description: Item ID + - name: ImmutableId + type: keyword + description: Immutable ID of the item + - name: InternetMessageId + type: keyword + description: Internet message ID + - name: ExchangeMessages + type: nested + description: List of messages + fields: + - name: Path + type: keyword + description: Path of the message + - name: Id + type: keyword + description: Message ID + - name: MessageItems + type: nested + description: Items in the message + fields: + - name: SizeInBytes + type: long + description: Size of the message item in bytes + - name: Id + type: keyword + description: Message item ID + - name: ExchangeMetaData type: group fields: @@ -415,6 +464,8 @@ type: keyword - name: Operation type: keyword + - name: OperationCount + type: keyword - name: OperationId type: keyword - name: OperationProperties @@ -604,6 +655,10 @@ type: keyword - name: ThreatDetectionMethods type: keyword + - name: TokenObjectId + type: keyword + - name: TokenTenantId + type: keyword - name: Timestamp type: keyword - name: UniqueSharingId diff --git a/packages/o365/docs/README.md b/packages/o365/docs/README.md index c2655f26d4f..d09117ab9e4 100644 --- a/packages/o365/docs/README.md +++ b/packages/o365/docs/README.md @@ -237,6 +237,7 @@ An example event for `audit` looks as following: | o365.audit.Actor.ID | | keyword | | o365.audit.Actor.Type | | keyword | | o365.audit.ActorContextId | | keyword | +| o365.audit.ActorInfoString | | keyword | | o365.audit.ActorIpAddress | | keyword | | o365.audit.ActorUserId | | keyword | | o365.audit.ActorYammerUserId | | keyword | @@ -356,6 +357,16 @@ An example event for `audit` looks as following: | o365.audit.EventDeepLink | | keyword | | o365.audit.EventSource | | keyword | | o365.audit.ExceptionInfo.\* | | object | +| o365.audit.ExchangeFolders.FolderItems.Id | Item ID | keyword | +| o365.audit.ExchangeFolders.FolderItems.ImmutableId | Immutable ID of the item | keyword | +| o365.audit.ExchangeFolders.FolderItems.InternetMessageId | Internet message ID | keyword | +| o365.audit.ExchangeFolders.FolderItems.SizeInBytes | Size of the item in bytes | long | +| o365.audit.ExchangeFolders.Id | Folder ID | keyword | +| o365.audit.ExchangeFolders.Path | Path of the folder | keyword | +| o365.audit.ExchangeMessages.Id | Message ID | keyword | +| o365.audit.ExchangeMessages.MessageItems.Id | Message item ID | keyword | +| o365.audit.ExchangeMessages.MessageItems.SizeInBytes | Size of the message item in bytes | long | +| o365.audit.ExchangeMessages.Path | Path of the message | keyword | | o365.audit.ExchangeMetaData.\* | | long | | o365.audit.ExchangeMetaData.CC | | keyword | | o365.audit.ExchangeMetaData.MessageID | | keyword | @@ -417,6 +428,7 @@ An example event for `audit` looks as following: | o365.audit.ObjectId | | keyword | | o365.audit.ObjectType | | keyword | | o365.audit.Operation | | keyword | +| o365.audit.OperationCount | | keyword | | o365.audit.OperationId | | keyword | | o365.audit.OperationProperties | | object | | o365.audit.OrganizationId | | keyword | @@ -501,6 +513,8 @@ An example event for `audit` looks as following: | o365.audit.TeamName | | keyword | | o365.audit.ThreatDetectionMethods | | keyword | | o365.audit.Timestamp | | keyword | +| o365.audit.TokenObjectId | | keyword | +| o365.audit.TokenTenantId | | keyword | | o365.audit.UniqueSharingId | | keyword | | o365.audit.UserAgent | | keyword | | o365.audit.UserId | | keyword | diff --git a/packages/o365/manifest.yml b/packages/o365/manifest.yml index 016ccf300ba..d2f4caa96cf 100644 --- a/packages/o365/manifest.yml +++ b/packages/o365/manifest.yml @@ -1,6 +1,6 @@ name: o365 title: Microsoft Office 365 -version: "2.31.0" +version: "2.31.1" description: Collect logs from Microsoft Office 365 with Elastic Agent. type: integration format_version: "3.2.3" From cc325a52e8885f6ec665b397c10ec3f2cce7ccb2 Mon Sep 17 00:00:00 2001 From: StacieClark-Elastic Date: Mon, 20 Oct 2025 14:40:50 -0400 Subject: [PATCH 2/6] fix-parsing-error-due-to-duplicate-fields added correct PR number --- packages/o365/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/o365/changelog.yml b/packages/o365/changelog.yml index 52ec00e3467..a9a8ee4233b 100644 --- a/packages/o365/changelog.yml +++ b/packages/o365/changelog.yml @@ -7,7 +7,7 @@ Added fields Messages and Folders as ExchangeMessages and ExchangeFolders for record type 50: `ExchangeItemAggregated`. type: bugfix - link: https://github.com/elastic/integrations/pull/99999 + link: https://github.com/elastic/integrations/pull/15699 - version: "2.31.0" changes: - description: Improve documentation. From 61da5f7e5a7fecf29d28f5f766deb3de46f8f605 Mon Sep 17 00:00:00 2001 From: StacieClark-Elastic Date: Tue, 21 Oct 2025 12:05:41 -0400 Subject: [PATCH 3/6] fix-parsing-error-due-to-duplicate-fields Added code to explcitly convert SizeInBytes field in ExchangeMessages and ExchangeFolders to long --- packages/o365/changelog.yml | 15 +++-- .../pipeline/test-exchange-access-event.json | 4 +- ...t-exchange-access-event.json-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 61 ++++++++++++++++--- .../o365/data_stream/audit/fields/fields.yml | 1 - 5 files changed, 67 insertions(+), 18 deletions(-) diff --git a/packages/o365/changelog.yml b/packages/o365/changelog.yml index a9a8ee4233b..3090a974f1e 100644 --- a/packages/o365/changelog.yml +++ b/packages/o365/changelog.yml @@ -2,10 +2,17 @@ - version: "2.31.1" changes: - description: >- - Fix flattening errors in Action List items due to duplicate QueryTime fields. - Added fields ActorInfoString, OperationCount, TokenObjectId, TokenTenantId. - Added fields Messages and Folders as ExchangeMessages and ExchangeFolders - for record type 50: `ExchangeItemAggregated`. + Fix flattening errors in `Action` List items due to duplicate `QueryTime` fields by removing duplicate field. + type: bugfix + link: https://github.com/elastic/integrations/pull/15699 + - description: >- + Fixes undefined errors by adding fields `ActorInfoString`, `OperationCount`, `TokenObjectId`, `TokenTenantId` + type: bugfix + link: https://github.com/elastic/integrations/pull/15699 + - description: >- + Fixes errors due to SizInBytes fields in `Messages` and `Folders` structures previously imported as long + and then being sent as floats. Moves the fields to explicitly defined fields `ExchangeMessages` and + `ExchangeFolders`and explicitly converts SizeInBytes to long or record type 50: `ExchangeItemAggregated`. type: bugfix link: https://github.com/elastic/integrations/pull/15699 - version: "2.31.0" diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json index 3dced223128..93bf900dbf8 100644 --- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json +++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json @@ -58,7 +58,7 @@ "MessageItems": [ { "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDlFFFFEEEEEEaaaaaaaaaaaaaaaaaaaaaaa", - "SizeInBytes": 1156492 + "SizeInBytes": 1156492.00 } ], "Path": "Messages" @@ -177,7 +177,7 @@ "SizeInBytes": 326463 }, { - "SizeInBytes": 1352491, + "SizeInBytes": 1352491.00, "Id": "AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEkAAAJ", "ImmutableId": "CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgOv+cAAAJ", "InternetMessageId": "" diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json-expected.json index af7832dcbf5..fc44ca597e1 100644 --- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json-expected.json +++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json-expected.json @@ -18,7 +18,7 @@ "code": "ExchangeItemAggregated", "id": "88888888-4444-5555-6666-123456789012", "kind": "event", - "original": "{\"OrganizationName\":\"example.onmicrosoft.com\",\"ActorInfoString\":\"Client=REST;Client=RESTSystem;UserAgent=[NoUserAgent][AppId=abcdabcd-1234-12ab-1a2b-ad1234567890];\",\"UserKey\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"MailboxGuid\":\"8b46a639-c47f-4634-b90c-2accecd337e3\",\"Operation\":\"AttachmentAccess\",\"OrganizationId\":\"1234abcd-4321-dcba-43ab-1023456789ab\",\"ClientIPAddress\":\"203.0.113.5\",\"LogonUserSid\":\"S-1-5-21-1234567890-123456789-1234567890-12345678\",\"OriginatingServer\":\"imase12AA1234 (203.0.113.3)\",\"RecordType\":50,\"Version\":1,\"ClientInfoString\":\"Client=REST;Client=RESTSystem;;\",\"ClientAppId\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"MailboxOwnerUPN\":\"user@example.com\",\"OperationCount\":6,\"MailboxOwnerSid\":\"S-1-5-21-1234567890-123456789-1234567890-12345678\",\"Messages\":[{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":2379,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDDEEEEEE12345678901234567890123\"},{\"SizeInBytes\":7356,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDDEEEEEE000000011111122222222222\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDD\"},{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":1156492,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDlFFFFEEEEEEaaaaaaaaaaaaaaaaaaaaaaa\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDlFFFF\"},{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":87052,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDvnQAAEEEEEEAALE\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDvnQAA\"},{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":267212,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJk_eAAAEEEEEEAB_WWWWWWWWWWWWWWW-1234\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJk_eAAA\"},{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":20477,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJ1234AAEEEEEEADGGGGGGGGGGGGGGGGGGGGGG\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJ123456\"}],\"ResultStatus\":\"Succeeded\",\"ExternalAccess\":false,\"LogonType\":0,\"TokenTenantId\":\"dcbadcba-1234-12ab-1a2b-ad1234567890\",\"AppAccessContext\":{\"ClientAppId\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"UniqueTokenId\":\"12345678-1234-1234-abcd-abcdef123456\",\"APIId\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"IssuedAtTime\":\"2025-09-29T01:01:01\"},\"Workload\":\"Exchange\",\"InternalLogonType\":0,\"OperationProperties\":[{\"Value\":\"Bind\",\"Name\":\"AttachmentAccessType\"}],\"AppId\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"UserId\":\"user@example.com\",\"CreationTime\":\"2025-09-29T01:01:01\",\"Id\":\"88888888-4444-5555-6666-123456789012\",\"UserType\":5}", + "original": "{\"OrganizationName\":\"example.onmicrosoft.com\",\"ActorInfoString\":\"Client=REST;Client=RESTSystem;UserAgent=[NoUserAgent][AppId=abcdabcd-1234-12ab-1a2b-ad1234567890];\",\"UserKey\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"MailboxGuid\":\"8b46a639-c47f-4634-b90c-2accecd337e3\",\"Operation\":\"AttachmentAccess\",\"OrganizationId\":\"1234abcd-4321-dcba-43ab-1023456789ab\",\"ClientIPAddress\":\"203.0.113.5\",\"LogonUserSid\":\"S-1-5-21-1234567890-123456789-1234567890-12345678\",\"OriginatingServer\":\"imase12AA1234 (203.0.113.3)\",\"RecordType\":50,\"Version\":1,\"ClientInfoString\":\"Client=REST;Client=RESTSystem;;\",\"ClientAppId\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"MailboxOwnerUPN\":\"user@example.com\",\"OperationCount\":6,\"MailboxOwnerSid\":\"S-1-5-21-1234567890-123456789-1234567890-12345678\",\"Messages\":[{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":2379,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDDEEEEEE12345678901234567890123\"},{\"SizeInBytes\":7356,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDDEEEEEE000000011111122222222222\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDD\"},{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":1156492.0,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDlFFFFEEEEEEaaaaaaaaaaaaaaaaaaaaaaa\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDlFFFF\"},{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":87052,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDvnQAAEEEEEEAALE\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDvnQAA\"},{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":267212,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJk_eAAAEEEEEEAB_WWWWWWWWWWWWWWW-1234\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJk_eAAA\"},{\"Path\":\"Messages\",\"MessageItems\":[{\"SizeInBytes\":20477,\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJ1234AAEEEEEEADGGGGGGGGGGGGGGGGGGGGGG\"}],\"Id\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCJJJJJJJJ123456\"}],\"ResultStatus\":\"Succeeded\",\"ExternalAccess\":false,\"LogonType\":0,\"TokenTenantId\":\"dcbadcba-1234-12ab-1a2b-ad1234567890\",\"AppAccessContext\":{\"ClientAppId\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"UniqueTokenId\":\"12345678-1234-1234-abcd-abcdef123456\",\"APIId\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"IssuedAtTime\":\"2025-09-29T01:01:01\"},\"Workload\":\"Exchange\",\"InternalLogonType\":0,\"OperationProperties\":[{\"Value\":\"Bind\",\"Name\":\"AttachmentAccessType\"}],\"AppId\":\"abcdabcd-1234-12ab-1a2b-ad1234567890\",\"UserId\":\"user@example.com\",\"CreationTime\":\"2025-09-29T01:01:01\",\"Id\":\"88888888-4444-5555-6666-123456789012\",\"UserType\":5}", "outcome": "success", "provider": "Exchange", "type": [ @@ -636,7 +636,7 @@ "code": "ExchangeItemAggregated", "id": "aaaaaaaa-bbbb-cccc-dddd-123456789012", "kind": "event", - "original": "{\"OrganizationName\":\"example.onmicrosoft.com\",\"ActorInfoString\":\"Client=WebServices;Apache-HttpAsyncClient/5.0[AppId=7777777-6666-aaaa-bbbb-123456789012];\",\"UserKey\":\"9876543210987656\",\"MailboxGuid\":\"eeeeeeee-aaaa-1234-bbbb-123456789012\",\"Operation\":\"MailItemsAccessed\",\"OrganizationId\":\"33333333-bbbb-cccc-dddd-123456789012\",\"ClientIPAddress\":\"203.0.113.145\",\"TokenObjectId\":\"ffffffff-aaaa-1234-bbbb-123456789012\",\"LogonUserSid\":\"S-1-5-21-1234567890-1234567890-123456789012-88888888\",\"OriginatingServer\":\"AB8MB22NO1234 (203.0.113.8)\",\"RecordType\":50,\"Version\":1,\"ClientInfoString\":\"Client=WebServices;Apache-HttpAsyncClient/5.0[AppId=7777777-6666-aaaa-bbbb-123456789012];\",\"ClientAppId\":\"7777777-6666-aaaa-bbbb-123456789012\",\"MailboxOwnerUPN\":\"user@example.com\",\"OperationCount\":6,\"Folders\":[{\"Path\":\"\\\\Sent Items\",\"Id\":\"LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLCCCCCCCCCCCCCCCCCCCCCCCC\",\"FolderItems\":[{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgO3wRAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":14593,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEpAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgO3gYAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":8526,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEoAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAxaAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":99635,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEnAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAgkAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":6475,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEmAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAgQAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":326463,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEElAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgOv+cAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":1352491,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEkAAAJ\"}]}],\"MailboxOwnerSid\":\"S-1-5-21-1234567890-1234567890-123456789012-88888888\",\"ResultStatus\":\"Succeeded\",\"ExternalAccess\":false,\"LogonType\":0,\"TokenTenantId\":\"33333333-bbbb-cccc-dddd-123456789012\",\"AppAccessContext\":{\"AADSessionId\":\"dddddddd-aaaa-eeee-dddd-123456789012\",\"ClientAppId\":\"7777777-6666-aaaa-bbbb-123456789012\",\"UniqueTokenId\":\"ZZZZZZZZZZKKKKKKKKKKAA\",\"APIId\":\"bbbbbbbb-aaaa-eeee-bbbb-123456789012\",\"IssuedAtTime\":\"2025-09-26T22:27:27\"},\"Workload\":\"Exchange\",\"InternalLogonType\":0,\"OperationProperties\":[{\"Value\":\"Bind\",\"Name\":\"MailAccessType\"}],\"AppId\":\"7777777-6666-aaaa-bbbb-123456789012\",\"UserId\":\"user@example.com\",\"CreationTime\":\"2025-09-26T22:32:29\",\"Id\":\"aaaaaaaa-bbbb-cccc-dddd-123456789012\",\"UserType\":0}", + "original": "{\"OrganizationName\":\"example.onmicrosoft.com\",\"ActorInfoString\":\"Client=WebServices;Apache-HttpAsyncClient/5.0[AppId=7777777-6666-aaaa-bbbb-123456789012];\",\"UserKey\":\"9876543210987656\",\"MailboxGuid\":\"eeeeeeee-aaaa-1234-bbbb-123456789012\",\"Operation\":\"MailItemsAccessed\",\"OrganizationId\":\"33333333-bbbb-cccc-dddd-123456789012\",\"ClientIPAddress\":\"203.0.113.145\",\"TokenObjectId\":\"ffffffff-aaaa-1234-bbbb-123456789012\",\"LogonUserSid\":\"S-1-5-21-1234567890-1234567890-123456789012-88888888\",\"OriginatingServer\":\"AB8MB22NO1234 (203.0.113.8)\",\"RecordType\":50,\"Version\":1,\"ClientInfoString\":\"Client=WebServices;Apache-HttpAsyncClient/5.0[AppId=7777777-6666-aaaa-bbbb-123456789012];\",\"ClientAppId\":\"7777777-6666-aaaa-bbbb-123456789012\",\"MailboxOwnerUPN\":\"user@example.com\",\"OperationCount\":6,\"Folders\":[{\"Path\":\"\\\\Sent Items\",\"Id\":\"LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLCCCCCCCCCCCCCCCCCCCCCCCC\",\"FolderItems\":[{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgO3wRAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":14593,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEpAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgO3gYAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":8526,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEoAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAxaAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":99635,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEnAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAgkAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":6475,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEmAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAAAAgQAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":326463,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEElAAAJ\"},{\"ImmutableId\":\"CCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEAAAgOv+cAAAJ\",\"InternetMessageId\":\"\",\"SizeInBytes\":1352491.0,\"Id\":\"AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEkAAAJ\"}]}],\"MailboxOwnerSid\":\"S-1-5-21-1234567890-1234567890-123456789012-88888888\",\"ResultStatus\":\"Succeeded\",\"ExternalAccess\":false,\"LogonType\":0,\"TokenTenantId\":\"33333333-bbbb-cccc-dddd-123456789012\",\"AppAccessContext\":{\"AADSessionId\":\"dddddddd-aaaa-eeee-dddd-123456789012\",\"ClientAppId\":\"7777777-6666-aaaa-bbbb-123456789012\",\"UniqueTokenId\":\"ZZZZZZZZZZKKKKKKKKKKAA\",\"APIId\":\"bbbbbbbb-aaaa-eeee-bbbb-123456789012\",\"IssuedAtTime\":\"2025-09-26T22:27:27\"},\"Workload\":\"Exchange\",\"InternalLogonType\":0,\"OperationProperties\":[{\"Value\":\"Bind\",\"Name\":\"MailAccessType\"}],\"AppId\":\"7777777-6666-aaaa-bbbb-123456789012\",\"UserId\":\"user@example.com\",\"CreationTime\":\"2025-09-26T22:32:29\",\"Id\":\"aaaaaaaa-bbbb-cccc-dddd-123456789012\",\"UserType\":0}", "outcome": "success", "provider": "Exchange", "type": [ diff --git a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index b23ae1418e2..1f887bf0a99 100644 --- a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -132,7 +132,11 @@ processors: if (!(ctx.o365audit.Actions instanceof List)) { ctx.o365audit.Actions = [ctx.o365audit.Actions]; } - def regex = /,\"QueryTime\":\"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M\"|\"QueryTime\":\"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M\",/; + /* + * Actions contains both a human readable `QueryTime` using AM/PM and an ISO8601 format `QueryTime` + * We remove the AM/PM containing `QueryTime` to avoid duplicate field errors on flattening. + */ + def regex = /,"QueryTime":"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M"|"QueryTime":"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M",/; for (def e: ctx.o365audit.Actions) { if (e instanceof Map) { actions.add(e); @@ -1414,7 +1418,7 @@ processors: description: Ensure that OperationCount is not rendered with e-notation or other numeric if: ctx.o365audit?.OperationCount != null source: |- - if (ctx.o365audit.OperationCount instanceof double) { + if (ctx.o365audit.OperationCount instanceof Number) { ctx.o365audit.OperationCount = ((long)ctx.o365audit.OperationCount).toString(); } else { ctx.o365audit.OperationCount = ctx.o365audit.OperationCount.toString(); @@ -1797,31 +1801,70 @@ processors: - append: field: event.type value: access - if: 'ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50"' + if: ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50" - append: field: event.category value: email - if: 'ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50"' + if: ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50" - rename: field: o365audit.Messages target_field: o365audit.ExchangeMessages tag: rename_messages_exchange - if: 'ctx.o365audit?.Messages != null && ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50"' + description: 'Move generic Messages field to the ExchangeMessages field defined by the ExchangeAggregatedMessage type' + if: ctx.o365audit?.Messages != null && ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50" + - script: + tag: convert_exchange_message_size_to_long + if: ctx.o365audit?.ExchangeMessages != null + lang: painless + source: | + for (def i = 0; i < ctx.o365audit.ExchangeMessages.length; i++) { + if (ctx.o365audit.ExchangeMessages[i].MessageItems != null) { + for (def j = 0; j < ctx.o365audit.ExchangeMessages[i].MessageItems.length; j++) { + def size = ctx.o365audit.ExchangeMessages[i].MessageItems[j].SizeInBytes; + if (size instanceof String) { + ctx.o365audit.ExchangeMessages[i].MessageItems[j].SizeInBytes = Long.parseLong(size); + } else { + ctx.o365audit.ExchangeMessages[i].MessageItems[j].SizeInBytes = (long)size; + } + } + } + } + - remove: field: o365audit.Messages tag: remove_messages_field - if: 'ctx.o365audit?.Messages != null' + if: ctx.o365audit?.Messages != null description: 'remove o365audit.Messages if we have not explicitly renamed them based on record type' + - rename: field: o365audit.Folders target_field: o365audit.ExchangeFolders tag: rename_folders_exchange - if: 'ctx.o365audit?.Folders != null && ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50"' + description: 'Move generic Folders field to the ExchangeFolders field defined by the ExchangeAggregatedFolder type' + if: ctx.o365audit?.Folders != null && ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50" + - script: + tag: convert_exchange_folder_size_to_long + if: ctx.o365audit?.ExchangeFolders != null + lang: painless + source: | + for (def i = 0; i < ctx.o365audit.ExchangeFolders.length; i++) { + if (ctx.o365audit.ExchangeFolders[i].FolderItems != null) { + for (def j = 0; j < ctx.o365audit.ExchangeFolders[i].FolderItems.length; j++) { + def size = ctx.o365audit.ExchangeFolders[i].FolderItems[j].SizeInBytes; + if (size instanceof String) { + ctx.o365audit.ExchangeFolders[i].FolderItems[j].SizeInBytes = Long.parseLong(size); + } else { + ctx.o365audit.ExchangeFolders[i].FolderItems[j].SizeInBytes = (long)size; + } + } + } + } + - remove: field: o365audit.Folders tag: remove_folders_field - if: 'ctx.o365audit?.Folders != null' - description: 'remove o365audit.Folders if we have not explicitly renamed them based on record type' + if: ctx.o365audit?.Folders != null + description: 'Remove o365audit.Folders if we have not explicitly renamed them based on record type' - script: description: Handle _tmp.entities.ThreatDetectionMethods containing list of lists. lang: painless diff --git a/packages/o365/data_stream/audit/fields/fields.yml b/packages/o365/data_stream/audit/fields/fields.yml index d3d2a5e530a..7e0d3c66888 100644 --- a/packages/o365/data_stream/audit/fields/fields.yml +++ b/packages/o365/data_stream/audit/fields/fields.yml @@ -323,7 +323,6 @@ - name: Id type: keyword description: Message item ID - - name: ExchangeMetaData type: group fields: From 7ec4b31b3b27d458bd81835aaff5aef17b04bf0c Mon Sep 17 00:00:00 2001 From: StacieClark-Elastic Date: Tue, 21 Oct 2025 12:21:29 -0400 Subject: [PATCH 4/6] fix-parsing-error-due-to-duplicate-fields changed Field names ExchangeMessages to ExchangeAggregatedMessages and ExchangeFolders to ExchangeAggregatedFolders --- packages/o365/changelog.yml | 4 +-- ...t-exchange-access-event.json-expected.json | 4 +-- .../elasticsearch/ingest_pipeline/default.yml | 36 +++++++++---------- .../o365/data_stream/audit/fields/fields.yml | 4 +-- packages/o365/docs/README.md | 20 +++++------ 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/packages/o365/changelog.yml b/packages/o365/changelog.yml index 3090a974f1e..bccf916d236 100644 --- a/packages/o365/changelog.yml +++ b/packages/o365/changelog.yml @@ -11,8 +11,8 @@ link: https://github.com/elastic/integrations/pull/15699 - description: >- Fixes errors due to SizInBytes fields in `Messages` and `Folders` structures previously imported as long - and then being sent as floats. Moves the fields to explicitly defined fields `ExchangeMessages` and - `ExchangeFolders`and explicitly converts SizeInBytes to long or record type 50: `ExchangeItemAggregated`. + and then being sent as floats. Moves the fields to explicitly defined fields `ExchangeAggregatedMessages` and + `ExchangeAggregatedFolders`and explicitly converts SizeInBytes to long for record type 50: `ExchangeItemAggregated`. type: bugfix link: https://github.com/elastic/integrations/pull/15699 - version: "2.31.0" diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json-expected.json index fc44ca597e1..23cad758533 100644 --- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json-expected.json +++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-exchange-access-event.json-expected.json @@ -46,7 +46,7 @@ "ClientAppId": "abcdabcd-1234-12ab-1a2b-ad1234567890", "ClientInfoString": "Client=REST;Client=RESTSystem;;", "CreationTime": "2025-09-29T01:01:01", - "ExchangeMessages": [ + "ExchangeAggregatedMessages": [ { "Id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBB_CCCCCCCCCCDDDDDDDDDDDDDD", "MessageItems": [ @@ -665,7 +665,7 @@ "ClientAppId": "7777777-6666-aaaa-bbbb-123456789012", "ClientInfoString": "Client=WebServices;Apache-HttpAsyncClient/5.0[AppId=7777777-6666-aaaa-bbbb-123456789012];", "CreationTime": "2025-09-26T22:32:29", - "ExchangeFolders": [ + "ExchangeAggregatedFolders": [ { "FolderItems": [ { diff --git a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 1f887bf0a99..816ee493817 100644 --- a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -1808,23 +1808,23 @@ processors: if: ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50" - rename: field: o365audit.Messages - target_field: o365audit.ExchangeMessages + target_field: o365audit.ExchangeAggregatedMessages tag: rename_messages_exchange - description: 'Move generic Messages field to the ExchangeMessages field defined by the ExchangeAggregatedMessage type' + description: 'Move generic Messages field to the ExchangeAggregatedMessages field type' if: ctx.o365audit?.Messages != null && ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50" - script: tag: convert_exchange_message_size_to_long - if: ctx.o365audit?.ExchangeMessages != null + if: ctx.o365audit?.ExchangeAggregatedMessages != null lang: painless source: | - for (def i = 0; i < ctx.o365audit.ExchangeMessages.length; i++) { - if (ctx.o365audit.ExchangeMessages[i].MessageItems != null) { - for (def j = 0; j < ctx.o365audit.ExchangeMessages[i].MessageItems.length; j++) { - def size = ctx.o365audit.ExchangeMessages[i].MessageItems[j].SizeInBytes; + for (def i = 0; i < ctx.o365audit.ExchangeAggregatedMessages.length; i++) { + if (ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems != null) { + for (def j = 0; j < ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems.length; j++) { + def size = ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes; if (size instanceof String) { - ctx.o365audit.ExchangeMessages[i].MessageItems[j].SizeInBytes = Long.parseLong(size); + ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes = Long.parseLong(size); } else { - ctx.o365audit.ExchangeMessages[i].MessageItems[j].SizeInBytes = (long)size; + ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes = (long)size; } } } @@ -1838,23 +1838,23 @@ processors: - rename: field: o365audit.Folders - target_field: o365audit.ExchangeFolders + target_field: o365audit.ExchangeAggregatedFolders tag: rename_folders_exchange - description: 'Move generic Folders field to the ExchangeFolders field defined by the ExchangeAggregatedFolder type' + description: 'Move generic Folders field to the O365 ExchangeAggregatedFolders field type' if: ctx.o365audit?.Folders != null && ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50" - script: tag: convert_exchange_folder_size_to_long - if: ctx.o365audit?.ExchangeFolders != null + if: ctx.o365audit?.ExchangeAggregatedFolders != null lang: painless source: | - for (def i = 0; i < ctx.o365audit.ExchangeFolders.length; i++) { - if (ctx.o365audit.ExchangeFolders[i].FolderItems != null) { - for (def j = 0; j < ctx.o365audit.ExchangeFolders[i].FolderItems.length; j++) { - def size = ctx.o365audit.ExchangeFolders[i].FolderItems[j].SizeInBytes; + for (def i = 0; i < ctx.o365audit.ExchangeAggregatedFolders.length; i++) { + if (ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems != null) { + for (def j = 0; j < ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems.length; j++) { + def size = ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes; if (size instanceof String) { - ctx.o365audit.ExchangeFolders[i].FolderItems[j].SizeInBytes = Long.parseLong(size); + ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes = Long.parseLong(size); } else { - ctx.o365audit.ExchangeFolders[i].FolderItems[j].SizeInBytes = (long)size; + ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes = (long)size; } } } diff --git a/packages/o365/data_stream/audit/fields/fields.yml b/packages/o365/data_stream/audit/fields/fields.yml index 7e0d3c66888..87a5db2b86d 100644 --- a/packages/o365/data_stream/audit/fields/fields.yml +++ b/packages/o365/data_stream/audit/fields/fields.yml @@ -277,7 +277,7 @@ # not expressible here; object_type_mapping_type cannot be 'boolean'. object_type: keyword object_type_mapping_type: '*' - - name: ExchangeFolders + - name: ExchangeAggregatedFolders type: nested description: List of folders fields: @@ -303,7 +303,7 @@ - name: InternetMessageId type: keyword description: Internet message ID - - name: ExchangeMessages + - name: ExchangeAggregatedMessages type: nested description: List of messages fields: diff --git a/packages/o365/docs/README.md b/packages/o365/docs/README.md index d09117ab9e4..1de68f998ce 100644 --- a/packages/o365/docs/README.md +++ b/packages/o365/docs/README.md @@ -357,16 +357,16 @@ An example event for `audit` looks as following: | o365.audit.EventDeepLink | | keyword | | o365.audit.EventSource | | keyword | | o365.audit.ExceptionInfo.\* | | object | -| o365.audit.ExchangeFolders.FolderItems.Id | Item ID | keyword | -| o365.audit.ExchangeFolders.FolderItems.ImmutableId | Immutable ID of the item | keyword | -| o365.audit.ExchangeFolders.FolderItems.InternetMessageId | Internet message ID | keyword | -| o365.audit.ExchangeFolders.FolderItems.SizeInBytes | Size of the item in bytes | long | -| o365.audit.ExchangeFolders.Id | Folder ID | keyword | -| o365.audit.ExchangeFolders.Path | Path of the folder | keyword | -| o365.audit.ExchangeMessages.Id | Message ID | keyword | -| o365.audit.ExchangeMessages.MessageItems.Id | Message item ID | keyword | -| o365.audit.ExchangeMessages.MessageItems.SizeInBytes | Size of the message item in bytes | long | -| o365.audit.ExchangeMessages.Path | Path of the message | keyword | +| o365.audit.ExchangeAggregatedFolders.FolderItems.Id | Item ID | keyword | +| o365.audit.ExchangeAggregatedFolders.FolderItems.ImmutableId | Immutable ID of the item | keyword | +| o365.audit.ExchangeAggregatedFolders.FolderItems.InternetMessageId | Internet message ID | keyword | +| o365.audit.ExchangeAggregatedFolders.FolderItems.SizeInBytes | Size of the item in bytes | long | +| o365.audit.ExchangeAggregatedFolders.Id | Folder ID | keyword | +| o365.audit.ExchangeAggregatedFolders.Path | Path of the folder | keyword | +| o365.audit.ExchangeAggregatedMessages.Id | Message ID | keyword | +| o365.audit.ExchangeAggregatedMessages.MessageItems.Id | Message item ID | keyword | +| o365.audit.ExchangeAggregatedMessages.MessageItems.SizeInBytes | Size of the message item in bytes | long | +| o365.audit.ExchangeAggregatedMessages.Path | Path of the message | keyword | | o365.audit.ExchangeMetaData.\* | | long | | o365.audit.ExchangeMetaData.CC | | keyword | | o365.audit.ExchangeMetaData.MessageID | | keyword | From 81cbb55e8f91c6df67a68027e2cc488bddd1dc74 Mon Sep 17 00:00:00 2001 From: StacieClark-Elastic Date: Wed, 22 Oct 2025 09:38:52 -0400 Subject: [PATCH 5/6] fix-parsing-error-due-to-duplicate-fields Don't remove Mesages or Folders field if they have not been renamed --- packages/o365/changelog.yml | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 68 ++++++++----------- 2 files changed, 31 insertions(+), 41 deletions(-) diff --git a/packages/o365/changelog.yml b/packages/o365/changelog.yml index bccf916d236..ee2f54e0e7e 100644 --- a/packages/o365/changelog.yml +++ b/packages/o365/changelog.yml @@ -3,14 +3,14 @@ changes: - description: >- Fix flattening errors in `Action` List items due to duplicate `QueryTime` fields by removing duplicate field. - type: bugfix + type: enhancement link: https://github.com/elastic/integrations/pull/15699 - description: >- Fixes undefined errors by adding fields `ActorInfoString`, `OperationCount`, `TokenObjectId`, `TokenTenantId` type: bugfix link: https://github.com/elastic/integrations/pull/15699 - description: >- - Fixes errors due to SizInBytes fields in `Messages` and `Folders` structures previously imported as long + Fixes errors due to SizeInBytes fields in `Messages` and `Folders` structures previously imported as long and then being sent as floats. Moves the fields to explicitly defined fields `ExchangeAggregatedMessages` and `ExchangeAggregatedFolders`and explicitly converts SizeInBytes to long for record type 50: `ExchangeItemAggregated`. type: bugfix diff --git a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 816ee493817..a1cf3e06447 100644 --- a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -132,16 +132,15 @@ processors: if (!(ctx.o365audit.Actions instanceof List)) { ctx.o365audit.Actions = [ctx.o365audit.Actions]; } - /* - * Actions contains both a human readable `QueryTime` using AM/PM and an ISO8601 format `QueryTime` - * We remove the AM/PM containing `QueryTime` to avoid duplicate field errors on flattening. - */ - def regex = /,"QueryTime":"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M"|"QueryTime":"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M",/; + + // Actions contains both a human readable `QueryTime` using AM/PM and an ISO8601 format `QueryTime` + // We remove the AM/PM containing `QueryTime` to avoid duplicate field errors on flattening. + def queryTimePattern = /,"QueryTime":"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M"|"QueryTime":"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M",/; for (def e: ctx.o365audit.Actions) { if (e instanceof Map) { actions.add(e); } else if (e instanceof String) { - ctx._tmp.action_strings.add(regex.matcher(e).replaceAll('')); + ctx._tmp.action_strings.add(queryTimePattern.matcher(e).replaceAll('')); } } if (actions.length == ctx.o365audit.Actions.length) { @@ -1801,70 +1800,61 @@ processors: - append: field: event.type value: access - if: ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50" + if: ctx.o365audit?.RecordType == "50" - append: field: event.category value: email - if: ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50" + if: ctx.o365audit?.RecordType == "50" - rename: field: o365audit.Messages target_field: o365audit.ExchangeAggregatedMessages tag: rename_messages_exchange - description: 'Move generic Messages field to the ExchangeAggregatedMessages field type' - if: ctx.o365audit?.Messages != null && ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50" + description: 'move generic Messages field to the ExchangeAggregatedMessages field type' + if: ctx.o365audit?.Messages != null && ctx.o365audit.RecordType == "50" - script: tag: convert_exchange_message_size_to_long if: ctx.o365audit?.ExchangeAggregatedMessages != null lang: painless source: | for (def i = 0; i < ctx.o365audit.ExchangeAggregatedMessages.length; i++) { - if (ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems != null) { - for (def j = 0; j < ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems.length; j++) { - def size = ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes; - if (size instanceof String) { - ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes = Long.parseLong(size); - } else { - ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes = (long)size; - } + if (ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems == null) { + continue; + } + for (def j = 0; j < ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems.length; j++) { + def size = ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes; + if (size instanceof String) { + ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes = Long.parseLong(size); + } else { + ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes = (long)size; } } } - - remove: - field: o365audit.Messages - tag: remove_messages_field - if: ctx.o365audit?.Messages != null - description: 'remove o365audit.Messages if we have not explicitly renamed them based on record type' - - rename: field: o365audit.Folders target_field: o365audit.ExchangeAggregatedFolders tag: rename_folders_exchange - description: 'Move generic Folders field to the O365 ExchangeAggregatedFolders field type' - if: ctx.o365audit?.Folders != null && ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50" + description: 'move generic Folders field to the O365 ExchangeAggregatedFolders field type' + if: ctx.o365audit?.Folders != null && ctx.o365audit.RecordType == "50" - script: tag: convert_exchange_folder_size_to_long if: ctx.o365audit?.ExchangeAggregatedFolders != null lang: painless source: | for (def i = 0; i < ctx.o365audit.ExchangeAggregatedFolders.length; i++) { - if (ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems != null) { - for (def j = 0; j < ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems.length; j++) { - def size = ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes; - if (size instanceof String) { - ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes = Long.parseLong(size); - } else { - ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes = (long)size; - } + if (ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems == null) { + continue; + } + for (def j = 0; j < ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems.length; j++) { + def size = ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes; + if (size instanceof String) { + ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes = Long.parseLong(size); + } else { + ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes = (long)size; } } } - - remove: - field: o365audit.Folders - tag: remove_folders_field - if: ctx.o365audit?.Folders != null - description: 'Remove o365audit.Folders if we have not explicitly renamed them based on record type' - script: description: Handle _tmp.entities.ThreatDetectionMethods containing list of lists. lang: painless From 862e0a50b51b037754fc87479f2bb91a89818d6b Mon Sep 17 00:00:00 2001 From: StacieClark-Elastic Date: Fri, 24 Oct 2025 15:53:49 -0400 Subject: [PATCH 6/6] fix-parsing-error-due-to-duplicate-fields incremented version to next major version --- packages/o365/changelog.yml | 2 +- packages/o365/manifest.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/o365/changelog.yml b/packages/o365/changelog.yml index ee2f54e0e7e..1a69f9e5fa8 100644 --- a/packages/o365/changelog.yml +++ b/packages/o365/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: "2.31.1" +- version: "2.32.0" changes: - description: >- Fix flattening errors in `Action` List items due to duplicate `QueryTime` fields by removing duplicate field. diff --git a/packages/o365/manifest.yml b/packages/o365/manifest.yml index d2f4caa96cf..d1539230bb9 100644 --- a/packages/o365/manifest.yml +++ b/packages/o365/manifest.yml @@ -1,6 +1,6 @@ name: o365 title: Microsoft Office 365 -version: "2.31.1" +version: "2.32.0" description: Collect logs from Microsoft Office 365 with Elastic Agent. type: integration format_version: "3.2.3"