Skip to content

Commit 0652bdb

Browse files
stefans-elasticlucian-ioan
stefans-elastic
and
lucian-ioan
authored
[azure][activitylogs] add json processor to responseBody and requestBody (#15690)
* add json processor to responseBody and requestBody * extra test case * update manifest and changelog * fix field name in processor * cover requestBody in tests * update expected results * Update packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml Co-authored-by: Lucian Ioan <[email protected]> * Update packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml Co-authored-by: Lucian Ioan <[email protected]> * Update packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml Co-authored-by: Lucian Ioan <[email protected]> * address PR comments * update test data with real field names --------- Co-authored-by: Lucian Ioan <[email protected]>
1 parent bb9b662 commit 0652bdb

File tree

6 files changed

+165
-4
lines changed

6 files changed

+165
-4
lines changed

‎packages/azure/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
- version: "1.29.0"
2+
changes:
3+
- description: Parse responseBody and requestBody json in activitylogs.
4+
type: enhancement
5+
link: https://github.com/elastic/integrations/pull/15690
16
- version: "1.28.7"
27
changes:
38
- description: Interim fix to support non-standard log events.

‎packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
1-
{"callerIpAddress":"81.2.69.144","category":"Action","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","durationMs":0,"identity":{"authorization":{"action":"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action","evidence":{"principalId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","principalType":"ServicePrincipal","role":"Azure EventGrid Service BuiltIn Role","roleAssignmentId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","roleAssignmentScope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53","roleDefinitionId":"8a4de8b5-095c-47d0-a96f-a75130c61d53"},"scope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey"},"claims":{"aio":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appidacr":"2","aud":"https://management.core.windows.net/","exp":"1571904826","http://schemas.microsoft.com/identity/claims/identityprovider":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","http://schemas.microsoft.com/identity/claims/objectidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.microsoft.com/identity/claims/tenantid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","iat":"1571875726","iss":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","nbf":"1571875726","uti":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ver":"1.0"}},"level":"Information","location":"global","operationName":"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION","resourceId":"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY","resultSignature":"Started.","resultType":"Start","time":"2019-10-24T00:13:46.3554259Z"}
1+
{"callerIpAddress":"81.2.69.144","category":"Action","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","durationMs":0,"identity":{"authorization":{"action":"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action","evidence":{"principalId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","principalType":"ServicePrincipal","role":"Azure EventGrid Service BuiltIn Role","roleAssignmentId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","roleAssignmentScope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53","roleDefinitionId":"8a4de8b5-095c-47d0-a96f-a75130c61d53"},"scope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey"},"claims":{"aio":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appidacr":"2","aud":"https://management.core.windows.net/","exp":"1571904826","http://schemas.microsoft.com/identity/claims/identityprovider":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","http://schemas.microsoft.com/identity/claims/objectidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.microsoft.com/identity/claims/tenantid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","iat":"1571875726","iss":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","nbf":"1571875726","uti":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ver":"1.0"}},"level":"Information","location":"global","operationName":"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION","resourceId":"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY","resultSignature":"Started.","resultType":"Start","time":"2019-10-24T00:13:46.3554259Z"}
2+
{"category":"ResourceHealth","correlationId":"1c867fe2-050c-4a74-bb1c-a83b15246fdd","level":"Information","operationName":"Microsoft.Resourcehealth/healthevent/Updated/action","properties":{"responseBody": "{\"sku\":{\"name\":\"Standard_LRS\",\"tier\":\"Standard\"},\"kind\":\"StorageV2\",\"id\":\"/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123\",\"location\":\"eastus\"}", "requestBody": "{\"id\":\"/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123\",\"kind\":\"StorageV2\",\"location\":\"eastus\",\"name\":\"mystorageacct123\",\"properties\":{\"creationTime\":\"2025-01-15T14:20:00.1234567Z\",\"primaryEndpoints\":{\"blob\":\"https://mystorageacct123.blob.core.windows.net/\",\"file\":\"https://mystorageacct123.file.core.windows.net/\"},\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\"},\"sku\":{\"name\":\"Standard_GRS\",\"tier\":\"Standard\"},\"type\":\"Microsoft.Storage/storageAccounts\"}", "eventCategory":"ResourceHealth","eventProperties":{"cause":"PlatformInitiated"}},"resourceId":"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration","resultType":"Updated","time":"2025-10-17T11:50:07.22Z"}
3+
{"category":"ResourceHealth","correlationId":"1c867fe2-050c-4a74-bb1c-a83b15246fdd","level":"Information","operationName":"Microsoft.Resourcehealth/healthevent/Updated/action","properties":{"responseBody": {"id":"\/subscriptions\/abc-123-your-sub-id\/resourceGroups\/my-resource-group\/providers\/Microsoft.Storage\/storageAccounts\/mystorageacct123","kind":"StorageV2","location":"eastus","sku":{"name":"Standard_LRS","tier":"Standard"}}, "requestBody": {"id":"\/subscriptions\/abc-123-your-sub-id\/resourceGroups\/my-resource-group\/providers\/Microsoft.Storage\/storageAccounts\/mystorageacct123","kind":"StorageV2","location":"eastus","name":"mystorageacct123","properties":{"creationTime":"2025-01-15T14:20:00.1234567Z","primaryEndpoints":{"blob":"https:\/\/mystorageacct123.blob.core.windows.net\/","file":"https:\/\/mystorageacct123.file.core.windows.net\/"},"provisioningState":"Succeeded","publicNetworkAccess":"Enabled"},"sku":{"name":"Standard_GRS","tier":"Standard"},"type":"Microsoft.Storage\/storageAccounts"}, "eventCategory":"ResourceHealth","eventProperties":{"cause":"PlatformInitiated"}},"resourceId":"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration","resultType":"Updated","time":"2025-10-17T11:50:07.22Z"}

‎packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json‎

Lines changed: 147 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,152 @@
110110
"tags": [
111111
"preserve_original_event"
112112
]
113+
},
114+
{
115+
"@timestamp": "2025-10-17T11:50:07.220Z",
116+
"azure": {
117+
"activitylogs": {
118+
"category": "ResourceHealth",
119+
"event_category": "ResourceHealth",
120+
"operation_name": "Microsoft.Resourcehealth/healthevent/Updated/action",
121+
"properties": {
122+
"eventProperties": {
123+
"cause": "PlatformInitiated"
124+
},
125+
"requestBody": {
126+
"id": "/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123",
127+
"kind": "StorageV2",
128+
"location": "eastus",
129+
"name": "mystorageacct123",
130+
"properties": {
131+
"creationTime": "2025-01-15T14:20:00.1234567Z",
132+
"primaryEndpoints": {
133+
"blob": "https://mystorageacct123.blob.core.windows.net/",
134+
"file": "https://mystorageacct123.file.core.windows.net/"
135+
},
136+
"provisioningState": "Succeeded",
137+
"publicNetworkAccess": "Enabled"
138+
},
139+
"sku": {
140+
"name": "Standard_GRS",
141+
"tier": "Standard"
142+
},
143+
"type": "Microsoft.Storage/storageAccounts"
144+
},
145+
"responseBody": {
146+
"id": "/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123",
147+
"kind": "StorageV2",
148+
"location": "eastus",
149+
"sku": {
150+
"name": "Standard_LRS",
151+
"tier": "Standard"
152+
}
153+
}
154+
},
155+
"result_type": "Updated"
156+
},
157+
"correlation_id": "1c867fe2-050c-4a74-bb1c-a83b15246fdd",
158+
"resource": {
159+
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration",
160+
"provider": "Microsoft.domainRegistration"
161+
},
162+
"subscription_id": "00000000-0000-0000-0000-000000000000"
163+
},
164+
"cloud": {
165+
"provider": "azure"
166+
},
167+
"ecs": {
168+
"version": "8.11.0"
169+
},
170+
"event": {
171+
"action": "Microsoft.Resourcehealth/healthevent/Updated/action",
172+
"kind": "event",
173+
"original": "{\"category\":\"ResourceHealth\",\"correlationId\":\"1c867fe2-050c-4a74-bb1c-a83b15246fdd\",\"level\":\"Information\",\"operationName\":\"Microsoft.Resourcehealth/healthevent/Updated/action\",\"properties\":{\"responseBody\": \"{\\\"sku\\\":{\\\"name\\\":\\\"Standard_LRS\\\",\\\"tier\\\":\\\"Standard\\\"},\\\"kind\\\":\\\"StorageV2\\\",\\\"id\\\":\\\"/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123\\\",\\\"location\\\":\\\"eastus\\\"}\", \"requestBody\": \"{\\\"id\\\":\\\"/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123\\\",\\\"kind\\\":\\\"StorageV2\\\",\\\"location\\\":\\\"eastus\\\",\\\"name\\\":\\\"mystorageacct123\\\",\\\"properties\\\":{\\\"creationTime\\\":\\\"2025-01-15T14:20:00.1234567Z\\\",\\\"primaryEndpoints\\\":{\\\"blob\\\":\\\"https://mystorageacct123.blob.core.windows.net/\\\",\\\"file\\\":\\\"https://mystorageacct123.file.core.windows.net/\\\"},\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"publicNetworkAccess\\\":\\\"Enabled\\\"},\\\"sku\\\":{\\\"name\\\":\\\"Standard_GRS\\\",\\\"tier\\\":\\\"Standard\\\"},\\\"type\\\":\\\"Microsoft.Storage/storageAccounts\\\"}\", \"eventCategory\":\"ResourceHealth\",\"eventProperties\":{\"cause\":\"PlatformInitiated\"}},\"resourceId\":\"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration\",\"resultType\":\"Updated\",\"time\":\"2025-10-17T11:50:07.22Z\"}"
174+
},
175+
"log": {
176+
"level": "Information"
177+
},
178+
"related": {
179+
"entity": [
180+
"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration"
181+
]
182+
},
183+
"tags": [
184+
"preserve_original_event"
185+
]
186+
},
187+
{
188+
"@timestamp": "2025-10-17T11:50:07.220Z",
189+
"azure": {
190+
"activitylogs": {
191+
"category": "ResourceHealth",
192+
"event_category": "ResourceHealth",
193+
"operation_name": "Microsoft.Resourcehealth/healthevent/Updated/action",
194+
"properties": {
195+
"eventProperties": {
196+
"cause": "PlatformInitiated"
197+
},
198+
"requestBody": {
199+
"id": "/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123",
200+
"kind": "StorageV2",
201+
"location": "eastus",
202+
"name": "mystorageacct123",
203+
"properties": {
204+
"creationTime": "2025-01-15T14:20:00.1234567Z",
205+
"primaryEndpoints": {
206+
"blob": "https://mystorageacct123.blob.core.windows.net/",
207+
"file": "https://mystorageacct123.file.core.windows.net/"
208+
},
209+
"provisioningState": "Succeeded",
210+
"publicNetworkAccess": "Enabled"
211+
},
212+
"sku": {
213+
"name": "Standard_GRS",
214+
"tier": "Standard"
215+
},
216+
"type": "Microsoft.Storage/storageAccounts"
217+
},
218+
"responseBody": {
219+
"id": "/subscriptions/abc-123-your-sub-id/resourceGroups/my-resource-group/providers/Microsoft.Storage/storageAccounts/mystorageacct123",
220+
"kind": "StorageV2",
221+
"location": "eastus",
222+
"sku": {
223+
"name": "Standard_LRS",
224+
"tier": "Standard"
225+
}
226+
}
227+
},
228+
"result_type": "Updated"
229+
},
230+
"correlation_id": "1c867fe2-050c-4a74-bb1c-a83b15246fdd",
231+
"resource": {
232+
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration",
233+
"provider": "Microsoft.domainRegistration"
234+
},
235+
"subscription_id": "00000000-0000-0000-0000-000000000000"
236+
},
237+
"cloud": {
238+
"provider": "azure"
239+
},
240+
"ecs": {
241+
"version": "8.11.0"
242+
},
243+
"event": {
244+
"action": "Microsoft.Resourcehealth/healthevent/Updated/action",
245+
"kind": "event",
246+
"original": "{\"category\":\"ResourceHealth\",\"correlationId\":\"1c867fe2-050c-4a74-bb1c-a83b15246fdd\",\"level\":\"Information\",\"operationName\":\"Microsoft.Resourcehealth/healthevent/Updated/action\",\"properties\":{\"responseBody\": {\"id\":\"\\/subscriptions\\/abc-123-your-sub-id\\/resourceGroups\\/my-resource-group\\/providers\\/Microsoft.Storage\\/storageAccounts\\/mystorageacct123\",\"kind\":\"StorageV2\",\"location\":\"eastus\",\"sku\":{\"name\":\"Standard_LRS\",\"tier\":\"Standard\"}}, \"requestBody\": {\"id\":\"\\/subscriptions\\/abc-123-your-sub-id\\/resourceGroups\\/my-resource-group\\/providers\\/Microsoft.Storage\\/storageAccounts\\/mystorageacct123\",\"kind\":\"StorageV2\",\"location\":\"eastus\",\"name\":\"mystorageacct123\",\"properties\":{\"creationTime\":\"2025-01-15T14:20:00.1234567Z\",\"primaryEndpoints\":{\"blob\":\"https:\\/\\/mystorageacct123.blob.core.windows.net\\/\",\"file\":\"https:\\/\\/mystorageacct123.file.core.windows.net\\/\"},\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\"},\"sku\":{\"name\":\"Standard_GRS\",\"tier\":\"Standard\"},\"type\":\"Microsoft.Storage\\/storageAccounts\"}, \"eventCategory\":\"ResourceHealth\",\"eventProperties\":{\"cause\":\"PlatformInitiated\"}},\"resourceId\":\"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration\",\"resultType\":\"Updated\",\"time\":\"2025-10-17T11:50:07.22Z\"}"
247+
},
248+
"log": {
249+
"level": "Information"
250+
},
251+
"related": {
252+
"entity": [
253+
"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration"
254+
]
255+
},
256+
"tags": [
257+
"preserve_original_event"
258+
]
113259
}
114260
]
115-
}
261+
}

‎packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,14 @@ processors:
9696
field: azure.activitylogs.properties
9797
if: "ctx.azure?.activitylogs?.properties instanceof String"
9898
ignore_failure: true
99+
- json:
100+
field: azure.activitylogs.properties.responseBody
101+
if: "ctx.azure?.activitylogs?.properties?.responseBody instanceof String"
102+
ignore_failure: true
103+
- json:
104+
field: azure.activitylogs.properties.requestBody
105+
if: "ctx.azure?.activitylogs?.properties?.requestBody instanceof String"
106+
ignore_failure: true
99107
- script:
100108
lang: painless
101109
source: >-

‎packages/azure/data_stream/application_gateway/_dev/test/pipeline/test-application-gateway-raw.log-expected.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -452,4 +452,4 @@
452452
}
453453
}
454454
]
455-
}
455+
}

‎packages/azure/manifest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: azure
22
title: Azure Logs
3-
version: "1.28.7"
3+
version: "1.29.0"
44
description: This Elastic integration collects logs from Azure
55
type: integration
66
icons:

0 commit comments

Comments
 (0)