Skip to content

Commit bb9b662

Browse files
moxarth-rathodmoxarth-rathod
authored
[Netskope] Add alerts_events_v2 data stream to fetch the data for alerts_v2 and events_v2 from a single queue (#15697)
netskope: add alerts_events_v2 data stream to fetch the data for alerts_v2 and events_v2 from a single queue
1 parent b434d5d commit bb9b662

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+2663
-4732
lines changed

‎packages/netskope/_dev/build/docs/README.md‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ Please make sure to use the given response formats.
4444

4545
Considering you already have an AWS S3 bucket setup, to configure it with Netskope, follow [these steps](https://docs.netskope.com/en/stream-logs-to-amazon-s3) to enable the log streaming.
4646

47+
**Note**: It is recommended to use the combined Alerts V2 and Events V2 data stream rather than configuring the individual Events V2 or Alerts V2 data stream. The alerts_events_v2 stream automatically directs logs to the appropriate individual data streams.
48+
4749
#### Collect data from Azure Blob Storage
4850

4951
1. If you already have an Azure storage container setup, configure it with Netskope via log streaming.
@@ -176,24 +178,12 @@ Default port: _9021_
176178

177179
{{event "alerts"}}
178180

179-
### Alerts V2
180-
181-
{{fields "alerts_v2"}}
182-
183-
{{event "alerts_v2"}}
184-
185181
### Events
186182

187183
{{fields "events"}}
188184

189185
{{event "events"}}
190186

191-
### Events V2
192-
193-
{{fields "events_v2"}}
194-
195-
{{event "events_v2"}}
196-
197187
### Transaction
198188

199189
{{fields "transaction"}}

‎packages/netskope/changelog.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,11 @@
11
# newer versions go on top
2+
- version: "3.0.0"
3+
changes:
4+
- description: >-
5+
Add alerts_events_v2 data stream to support fetching alert v2 and event v2 data from a single queue.
6+
Users using SQS input should consider disabling alerts_v2 and events_v2 to avoid conflicts, and use the combined data stream instead.
7+
type: breaking-change
8+
link: https://github.com/elastic/integrations/pull/15697
29
- version: "2.3.0"
310
changes:
411
- description: Add support for Transaction data stream.

‎packages/netskope/data_stream/alerts_events_v2/_dev/test/pipeline/test-combined.log‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{"_id": "5182808a2a99fc688d4a8057", "access_method": "API Connector", "account_id": "0u3700o60054", "account_name": "testing-iaas-policies", "acked": "true", "action": "block", "activity": "Delete", "alert": "yes", "alert_id": "9b302da498a1ed703495f527c1574b76", "alert_name": "[Web] Block Countries Managed", "alert_source": "DLP", "alert_type": "policy", "app": "Dropbox", "app_session_id": 130464223392976, "appcategory": "Technology", "appsuite": "Microsoft Live", "breach_date": 1700352376, "breach_id": "e8bcc837615516de9d338403caa57ac9", "breach_score": "70", "browser": "Chrome", "browser_session_id": 6013040120128864000, "cci": 31, "ccl": "excellent", "client_bytes": 3613916, "conn_duration": 47, "conn_endtime": 1700352376, "conn_starttime": 1700352376, "connection_id": 5047636402716176000, "custom_attr": {"usr_udf_employeeid": "A095301", "usr_display_name": "McKillip, William (A095301)", "usr_title": "Analyst - Business Solutions Analyst"}, "device": "Windows Device", "device_classification": "unknown", "dlp_file": "r5kessan.data", "dlp_incident_id": 5593408369243077000, "dlp_is_unique_count": "true", "dlp_parent_id": 6398957211952494000, "dlp_profile": "NSSF_Customer_Confidential_Keywords", "dlp_rule": "TennCare Member ID and Medical Lab Test", "dlp_rule_count": 1267, "dlp_rule_severity": "Critical", "dlp_unique_count": 386, "dns_profile": "dns profile for test_dns_profiles.json", "domain": "au-sonpo.my.test.com", "domain_ip": "81.2.69.192", "dst_country": "SE", "dst_geoip_src": 0, "dst_latitude": 58.4167, "dst_location": "Link\u00f6ping", "dst_longitude": 15.6167, "dst_region": "\u00d6sterg\u00f6tland County", "dst_timezone": "Europe/Stockholm", "dst_zipcode": "58183", "dsthost": "ftp.abcd.com", "dstip": "89.160.20.128", "dstport": 1143, "email_title": "TESTING MALWARE", "event_uuid": "b9874d0e-c68a-4917-90aa-4c3c2ed3d2de", "file_cls_encrypted": true, "file_exposure": "Private", "file_path": "/home/username/Documents/file.txt", "file_size": 0, "file_type": "JSON document", "from_user": "[email protected]", "hostname": "C02GH1DMMD6N", "iaas_remediated": "true", "iaas_remediated_by": "[email protected]", "iaas_remediated_on": 1565244616, "iaas_remediation_action": "Revoke security group ingress port 22", "instance": "Chrome-River", "instance_id": "trical forage", "instance_name": "test", "loc": "US", "local_sha1": "835b7286727edfbc20eae7e81405fe0a8c4bd302", "mal_id": "30520723a5b106e6d0aea46a87a35a5f", "mal_type": "PUA", "managed_app": "yes", "managementID": "5E2156872C1791458F39A3B0AC3303E5", "md5": "5a00bef704579c065e188ce8a11b7d53", "mime_type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "netskope_pop": "IN-DEL2", "nsdeviceuid": "BC5A83EE-5FF1-6F51-FDD1-84CAFBF60E9E", "numbytes": 10621990, "object": "EYR Corporate Calibration 2023 with Bottom 10 (exclude HR) - 20 Feb 2024.xlsx", "object_id": "e2771328-dcfe-4dd9-bd0d-7947f247057a", "object_type": "People & Blogs", "org": "testlogistics.com", "organization_unit": "netskope.local/Netskope/Active Users/US & International/Full Time", "os": "Mac OS X 14.3.1", "os_family": "Windows", "os_version": "iOS 17.2.1", "owner": "[email protected]", "page": "www.youtube.com", "parent_id": "/personal/bnelson_hudsoninsgroup_com/Documents/Desktop", "policy": "SSL-Do-Not-Decrypt-General", "pop_id": "0X0008", "record_type": "page", "referer": "https://www.mmafighting.com/", "region_id": "us-east-2", "region_name": "US East(Ohio)", "req_cnt": 2000, "request_id": 2780503013482479600, "resource_category": "Database", "resource_group": "Automation-021589709060", "resp_cnt": 2057, "sa_profile_name": "Infosec Profile", "sa_rule_name": "Google Workspace 3rd-party apps should have lower risk score than 'High'", "sa_rule_severity": "High", "sanctioned_instance": "Yes", "server_bytes": 1920409, "severity": "informational", "severity_level": "med", "sha256": "a52b02bf2f91163f17e3e6bb751a94d3f2411bb726c2c731681892e943ef5793", "shared_domains": "next15.com", "shared_with": "[email protected]", "sharedType": "public", "site": "movistar", "src_country": "SE", "src_geoip_src": 0, "src_latitude": 58.4167, "src_location": "Villagran", "src_longitude": 15.6167, "src_region": "\u00d6sterg\u00f6tland County", "src_timezone": "Europe/Stockholm", "src_zipcode": "58183", "srcip": "89.160.20.128", "subject": "2025 SF Materials - Molina AZ DSNP", "suppression_count": "234", "telemetry_app": "pndsn", "threat_type": "domain_category", "timestamp": 1708989085, "traffic_type": "CloudApp", "transaction_id": 7147084621365702000, "tss_mode": "inline", "two_factor_auth": "yes", "type": "anomaly", "ur_normalized": "[email protected]", "url": "ipcow.com/", "user": "[email protected]", "user_confidence_index": 100, "user_confidence_level": "high", "user_id": "[email protected]", "useragent": "Mozilla/5.0 (ZOOM.Mac 13.6 x86)", "usergroup": "//DynamicGroup//s_vish", "userip": "192.168.1.2", "userkey": "[email protected]", "watchlist_name": "C Suite3", "web_url": "https://netskopepmskope-my.sharepoint.com/personal/admin_netskopepmskope_onmicrosoft_com2/Documents/shubhushduasjdsa.txt"}
2+
{ "_id": "5182808a2a99fc688d4a8157", "access_method": "Client", "account_id": "533708960054", "account_name": "csa-rules-setup", "acked": "false", "action": "bypass", "activity": "Login Failed", "alert": "yes", "alert_id": "314eba43aa95c8ea4f7416732e2c1921", "alert_name": "[CASB] Alert on Upload and Download for Sensitive Keywords in Cloud Storage", "alert_source": "Malware", "alert_type": "anomaly", "app": "Google Drive", "app_session_id": 130464223392977, "appcategory": "Productivity", "appsuite": "Google Workspace", "breach_date": 1700352377, "breach_id": "e8bcc837615516de9d338403caa57ad0", "breach_score": "80", "browser": "Firefox", "browser_session_id": 6013040120128863784, "cci": 32, "ccl": "good", "client_bytes": 3613917, "conn_duration": 48, "conn_endtime": 1700352377, "conn_starttime": 1700352377, "connection_id": 5047636402716175951, "custom_attr": { "usr_udf_primarydomain": "MWI.INTERNAL", "usr_status": "Active", "usr_udf_businesssegmentlevel3": "Animal Health", "usr_udf_companyname": "MWI Veterinary Supply Company" }, "device": "Mac Device", "device_classification": "managed", "dlp_file": "file2.data", "dlp_incident_id": 5593408369243076226, "dlp_is_unique_count": "false", "dlp_parent_id": 6398957211952493729, "dlp_profile": "PII_Profile", "dlp_rule": "PCI Data", "dlp_rule_count": 1268, "dlp_rule_severity": "High", "dlp_unique_count": 387, "dns_profile": "dns profile 2", "domain": "us-sonpo.my.test.com", "domain_ip": "81.2.69.193", "dst_country": "US", "dst_geoip_src": 1, "dst_latitude": -33.8688, "dst_location": "Nishikata", "dst_longitude": 151.2093, "dst_region": "New South Wales", "dst_timezone": "America/New_York", "dst_zipcode": "2100", "dsthost": "ftp.efgh.com", "dstip": "81.2.69.193", "dstport": 1144, "email_title": "ALERT: Suspicious Activity", "event_uuid": "b9874d0e-c68a-4917-90aa-4c3c2ed3d2df", "file_cls_encrypted": false, "file_exposure": "Public", "file_path": "/home/username/Documents/file2.txt", "file_size": 1, "file_type": "PDF document", "from_user": "[email protected]", "hostname": "C02GH1DMMD6O", "iaas_remediated": "false", "iaas_remediated_by": "[email protected]", "iaas_remediated_on": 1565244617, "iaas_remediation_action": "Remove user access", "instance": "Salesforce", "instance_id": "clearview farm", "instance_name": "adminscope", "loc": "IN", "local_sha1": "835b7286727edfbc20eae7e81405fe0a8c4bd303", "mal_id": "30520723a5b106e6d0aea46a87a35a5g", "mal_type": "Malware", "managed_app": "no", "managementID": "5E2156872C1791458F39A3B0AC3303E6", "md5": "5a00bef704579c065e188ce8a11b7d54", "mime_type": "application/pdf", "netskope_pop": "US-NYC1", "nsdeviceuid": "BC5A83EE-5FF1-6F51-FDD1-84CAFBF60E9F", "numbytes": 10621991, "object": "Document2.pdf", "object_id": "e2771328-dcfe-4dd9-bd0d-7947f247057b", "object_type": "Document", "org": "example.com", "organization_unit": "netskope.local/Netskope/Active Users/EMEA/Full Time", "os": "Windows 10", "os_family": "Windows Server", "os_version": "iOS 16.5", "owner": "[email protected]", "page": "www.google.com", "parent_id": "/personal/ukrishnan_hudsoninsgroup_com/Documents/Surety%20Data%20Recon", "policy": "Block Social Media", "pop_id": "0X0009", "record_type": "alert", "referer": "https://www.example.com/", "region_id": "us-west-1", "region_name": "US West(California)", "req_cnt": 1478, "request_id": 2780503102603051008, "resource_category": "Storage", "resource_group": "ResourceGroup2", "resp_cnt": 2058, "sa_profile_name": "Compliance Profile", "sa_rule_name": "Block All External Sharing", "sa_rule_severity": "Medium", "sanctioned_instance": "No", "server_bytes": 1920410, "severity": "High", "severity_level": "high", "sha256": "a52b02bf2f91163f17e3e6bb751a94d3f2411bb726c2c731681892e943ef5794", "shared_domains": "example.com", "shared_with": "[email protected]", "sharedType": "private", "site": "site2", "src_country": "GE", "src_geoip_src": 1, "src_latitude": -33.8688, "src_location": "Jose Maria Morelos", "src_longitude": 2.3522, "src_region": "National Capital District (Port Moresby)", "src_timezone": "America/New_York", "src_zipcode": "2100", "srcip": "81.2.69.193", "subject": "2024 SF Materials - Molina CA DSNP", "suppression_count": "235", "telemetry_app": "app2", "threat_type": "malware", "timestamp": 1708989086, "traffic_type": "WebApp", "transaction_id": 7147084621365701240, "tss_mode": "offline", "two_factor_auth": "no", "type": "policy", "ur_normalized": "[email protected]", "url": "example.com/", "user": "[email protected]", "user_confidence_index": 99, "user_confidence_level": "medium", "user_id": "[email protected]", "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", "usergroup": "//DynamicGroup//s_john", "userip": "192.168.1.3", "userkey": "[email protected]", "watchlist_name": "C Suite2", "web_url": "https://netskopepmskope-my.sharepoint.com/personal/admin_netskopepmskope_onmicrosoft_com2/Documents/file2.txt" }
3+
{"_id": "5182808a2a99fc688d4a8457", "access_method": "Reverse Proxy", "action": "bypass", "app": "PrintDirect", "appcategory": "IaaS/PaaS", "cci": 71, "ccl": "medium", "client_bytes": 6610916, "client_packets": 300, "device": "Mac Device", "dns_profile": "adminB_dns_profile_block_policy.json", "domain": "metrogas.dummycompany.net", "domain_ip": "175.16.199.0", "dst_country": "IE", "dst_geoip_src": 0, "dst_latitude": 58.4167, "dst_location": "\u014ctemachi", "dst_longitude": 15.6167, "dst_region": "Mexico", "dst_zipcode": "L3R", "dsthost": "tcp.abcd.com", "dstip": "175.16.199.0", "dstport": 2022, "end_time": "2025-05-13T10:57:55+00:00", "hostname": "C02GH1DMND6N", "ip_protocol": "TCP", "network_session_id": "17713785972606802068", "numbytes": 22093509, "organization_unit": "netskope.local/Netskope/Active Users/US \\\\u0026 International/Full Time", "os": "Ventura", "os_version": "iOS 16.1.2", "policy": "Domain Controllers - MS Defender", "publisher_cn": "b412ed05f43e117f", "record_type": "network", "response_time": 127, "server_bytes": 67997, "server_packets": 1478, "session_duration": 5000, "site": "netxpro", "src_country": "SK", "src_geoip_src": 1, "src_latitude": 58.4167, "src_location": "Villagran", "src_longitude": 15.6167, "src_region": "Moscow Oblast", "src_zipcode": "14760", "srcip": "175.16.199.0", "srcport": 57018, "start_time": "2024-01-19T02:07:04+0000", "threat_type": "domain_category", "timestamp": 1708409385, "total_packets": 301760, "traffic_type": "CloudApp", "tunnel_id": "1992661906", "type": "breach", "ur_normalized": "[email protected]", "user": "[email protected]", "userip": "192.168.13.2", "userkey": "[email protected]"}
4+
{"_id": "5182808a2a99fc688d4a8157", "access_method": "Reverse Proxy", "action": "restrictAccess", "app": "Facebook", "app_session_id": 262512272630604, "appcategory": "Business Intelligence and Data Analytics", "browser": "Safari", "browser_session_id": 6697509106751238992, "cci": 71, "ccl": "unknown", "client_bytes": 3613917, "conn_duration": 22, "conn_endtime": 1700352377, "conn_starttime": 1700352376, "connection_id": 2544084735387872360, "device": "iPhone", "domain": "up1.dummycompany.com", "dst_country": "FR", "dst_geoip_src": 0, "dst_latitude": 15.6167, "dst_location": "Ebara", "dst_longitude": 15.6167, "dst_region": "Guangdong", "dst_timezone": "Europe/London", "dst_zipcode": "K0K", "dsthost": "ftp.abcd.com", "dstip": "175.16.199.0", "dstport": 6253, "hostname": "C02GH1DMMD6N", "netskope_pop": "IN-DEL2", "numbytes": 5354651, "org": "setupfordemo.com", "organization_unit": "netskope.local/Netskope/Active Users/US \\\\u0029 International/Full Time", "os": "Mac OS X 14.3.1", "os_family": "MacOS", "os_version": "Mac OSX 11.1.0", "page": "www.linkedin.com", "policy": "JetBrain No476", "record_type": "page", "req_cnt": 1721, "request_id": 2780503013482479616, "resp_cnt": 1807, "server_bytes": 3141, "severity": "medium", "site": "nequi", "src_country": "EE", "src_geoip_src": 2, "src_latitude": 58.4167, "src_location": "Conkal", "src_longitude": 15.6167, "src_region": "Astana", "src_timezone": "America/Bogota", "src_zipcode": "84189", "srcip": "175.16.199.0", "timestamp": 1708989385, "traffic_type": "Web", "transaction_id": 3363254757506630961, "type": "breach", "ur_normalized": "[email protected]", "url": "ipcow.com/", "user": "[email protected]", "useragent": "Mozilla/5.0 (ZOOM.Mac 13.6 x86)", "userip": "192.168.1.2", "userkey": "apatel97"}

0 commit comments

Comments
 (0)