diff --git a/packages/zeek/changelog.yml b/packages/zeek/changelog.yml index 1bf3ee52032..a5c591b3ef5 100644 --- a/packages/zeek/changelog.yml +++ b/packages/zeek/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.0.3" + changes: + - description: Generate processor tags and normalize error handler. + type: enhancement + link: https://github.com/elastic/integrations/pull/15721 - version: "3.0.2" changes: - description: Remove unused agent files. diff --git a/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml index 369b3b1fc87..bace0a8abe4 100644 --- a/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml @@ -2,45 +2,57 @@ description: Pipeline for normalizing Zeek capture_loss.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_capture_loss_62ef1b3a target_field: zeek.capture_loss field: _temp_ # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - date: + tag: date_zeek_capture_loss_ts_aa14509f field: zeek.capture_loss.ts formats: - UNIX - ISO8601 - set: + tag: set_event_kind_495d69f0 field: event.kind value: metric - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] - convert: + tag: convert_zeek_percent_lost_b0ba2ad8 field: zeek.percent_lost type: long ignore_missing: true - remove: + tag: remove_73e0896e field: - zeek.capture_loss.ts ignore_missing: true @@ -50,4 +62,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/capture_loss/manifest.yml b/packages/zeek/data_stream/capture_loss/manifest.yml index a19ed14e232..6e2843399e4 100644 --- a/packages/zeek/data_stream/capture_loss/manifest.yml +++ b/packages/zeek/data_stream/capture_loss/manifest.yml @@ -40,4 +40,3 @@ streams: template_path: log.yml.hbs title: Zeek capture_loss.log description: Collect Zeek capture_loss logs - \ No newline at end of file diff --git a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml index 834c2879035..946de9ffb4a 100644 --- a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml @@ -2,181 +2,226 @@ description: Pipeline for normalizing Zeek conn.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_connection_041442f1 field: _temp_ target_field: zeek.connection ignore_failure: true # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - dot_expander: + tag: dot_expander_id_orig_p_8fa63710 path: zeek.connection field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_19746738 path: zeek.connection field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_ee0b44c3 path: zeek.connection field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_61d8918b path: zeek.connection field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_connection_duration_to_temp_duration_201bfd9d field: zeek.connection.duration target_field: temp.duration ignore_missing: true - rename: + tag: rename_zeek_connection_id_orig_h_to_source_address_586eab3b field: zeek.connection.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_connection_id_orig_p_to_source_port_8a85f468 field: zeek.connection.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_connection_id_resp_h_to_destination_address_fd7a6257 field: zeek.connection.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_connection_id_resp_p_to_destination_port_26cf9664 field: zeek.connection.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_connection_proto_to_network_transport_749c7524 field: zeek.connection.proto target_field: network.transport ignore_missing: true - rename: + tag: rename_zeek_connection_service_to_network_protocol_9394a996 field: zeek.connection.service target_field: network.protocol ignore_missing: true - rename: + tag: rename_zeek_connection_uid_to_zeek_session_id_0aa6be04 field: zeek.connection.uid target_field: zeek.session_id ignore_missing: true - rename: + tag: rename_zeek_connection_orig_ip_bytes_to_source_bytes_0c856e1e field: zeek.connection.orig_ip_bytes target_field: source.bytes ignore_missing: true - rename: + tag: rename_zeek_connection_resp_ip_bytes_to_destination_bytes_424f8424 field: zeek.connection.resp_ip_bytes target_field: destination.bytes ignore_missing: true - rename: + tag: rename_zeek_connection_orig_pkts_to_source_packets_1ba73f9f field: zeek.connection.orig_pkts target_field: source.packets ignore_missing: true - rename: + tag: rename_zeek_connection_resp_pkts_to_destination_packets_78bf2a3b field: zeek.connection.resp_pkts target_field: destination.packets ignore_missing: true - rename: + tag: rename_zeek_connection_conn_state_to_zeek_connection_state_f67acac3 field: zeek.connection.conn_state target_field: zeek.connection.state ignore_missing: true - rename: + tag: rename_zeek_connection_orig_l2_addr_to_source_mac_67a0df2d field: zeek.connection.orig_l2_addr target_field: source.mac ignore_missing: true - rename: + tag: rename_zeek_connection_resp_l2_addr_to_destination_mac_c482941f field: zeek.connection.resp_l2_addr target_field: destination.mac ignore_missing: true - rename: + tag: rename_source_port_to_zeek_connection_icmp_type_1b384cfb field: source.port target_field: zeek.connection.icmp.type ignore_missing: true if: 'ctx.network?.transport == "icmp"' - rename: + tag: rename_destination_port_to_zeek_connection_icmp_code_c1cb67bd field: destination.port target_field: zeek.connection.icmp.code ignore_missing: true if: 'ctx.network?.transport == "icmp"' - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - community_id: + tag: community_id_f1f0d464 if: 'ctx.network?.transport != "icmp" && ctx.source?.port != 0 && ctx.destination?.port != 0' - community_id: + tag: community_id_2a7728bd icmp_type: zeek.connection.icmp.type icmp_code: zeek.connection.icmp.code if: 'ctx.network?.transport == "icmp"' - date: + tag: date_zeek_connection_ts_668d72d7 field: zeek.connection.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_connection_ts_07604b65 field: zeek.connection.ts - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - script: + tag: script_92d4fbc4 source: ctx.event.duration = Math.round(ctx.temp.duration * params.scale) params: scale: 1000000000 if: ctx.temp?.duration != null - append: + tag: append_tags_018d477a field: tags value: - local_orig if: ctx.zeek?.connection?.local_orig != null allow_duplicates: false - append: + tag: append_tags_4b61bc7e field: tags value: - local_resp if: ctx.zeek?.connection?.local_resp != null allow_duplicates: false - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: ctx.source?.ip != null allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: ctx.destination?.ip != null allow_duplicates: false - script: + tag: script_08005e97 source: ctx.network.packets = ctx.source.packets + ctx.destination.packets ignore_failure: true - script: + tag: script_06b63541 source: ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes ignore_failure: true - script: + tag: script_bd1f59ae source: |- if (ctx.zeek?.connection?.local_orig == null || ctx.zeek?.connection?.local_resp == null) { @@ -203,14 +248,17 @@ processors: return; } - geoip: + tag: geoip_destination_ip_to_destination_geo_942e2f6c field: destination.ip target_field: destination.geo if: ctx.destination?.ip != null - geoip: + tag: geoip_source_ip_to_source_geo_bace2435 field: source.ip target_field: source.geo if: ctx.source?.ip != null - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -219,6 +267,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -227,22 +276,27 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - script: + tag: script_0e29b9b2 params: S0: conn_str: "Connection attempt seen, no reply." @@ -320,6 +374,7 @@ processors: ctx.event.type = params[ctx.zeek.connection.state]["types"]; } - remove: + tag: remove_ee848475 field: - zeek.connection.id - zeek.connection.orig_bytes @@ -335,4 +390,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/connection/manifest.yml b/packages/zeek/data_stream/connection/manifest.yml index 2c2c56dfa86..4e0eb6e797f 100644 --- a/packages/zeek/data_stream/connection/manifest.yml +++ b/packages/zeek/data_stream/connection/manifest.yml @@ -39,3 +39,4 @@ streams: show_user: false description: > Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details. + diff --git a/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml index 3faddd529a5..e9055f07903 100644 --- a/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml @@ -2,114 +2,145 @@ description: Pipeline for normalizing Zeek dce_rpc.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_dce_rpc_fc55dca2 field: _temp_ target_field: zeek.dce_rpc # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_78fef4b8 field: network.protocol value: dce_rpc - dot_expander: + tag: dot_expander_id_orig_p_57b98fdc path: zeek.dce_rpc field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_ba641c44 path: zeek.dce_rpc field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_41c040dd path: zeek.dce_rpc field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_fe2d6745 path: zeek.dce_rpc field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_dce_rpc_id_orig_h_to_source_address_c3098dff field: zeek.dce_rpc.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_dce_rpc_id_orig_p_to_source_port_7303eb5c field: zeek.dce_rpc.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_dce_rpc_id_resp_h_to_destination_address_31f9febb field: zeek.dce_rpc.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_dce_rpc_id_resp_p_to_destination_port_a3197f68 field: zeek.dce_rpc.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_dce_rpc_uid_to_zeek_session_id_46986de0 field: zeek.dce_rpc.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - date: + tag: date_zeek_dce_rpc_ts_0544c56b field: zeek.dce_rpc.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_dce_rpc_ts_9a6ea701 field: zeek.dce_rpc.ts - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: ctx.source?.ip != null allow_duplicates: false - geoip: + tag: geoip_source_ip_to_source_geo_0494c6b1 field: source.ip target_field: source.geo - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -118,22 +149,27 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: ctx.destination?.ip != null allow_duplicates: false - geoip: + tag: geoip_destination_ip_to_destination_geo_394bcdb5 field: destination.ip target_field: destination.geo - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -142,20 +178,25 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - set: + tag: set_event_action_e9dfe97e field: event.action copy_from: zeek.dce_rpc.operation if: "ctx.zeek?.dce_rpc?.operation != null" - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_801cb4bc field: - zeek.dce_rpc.id ignore_missing: true @@ -165,4 +206,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml index 0e57ae51fcf..05315bfd4bd 100644 --- a/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml @@ -2,180 +2,227 @@ description: Pipeline for normalizing Zeek dhcp.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_dhcp_1a6c1843 field: _temp_ target_field: zeek.dhcp # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_network_transport_a6cfd568 field: network.transport value: udp - set: + tag: set_network_protocol_ea539779 field: network.protocol value: dhcp - rename: + tag: rename_zeek_dhcp_uids_to_zeek_session_id_fac1f710 field: zeek.dhcp.uids target_field: zeek.session_id ignore_missing: true - rename: + tag: rename_zeek_dhcp_assigned_addr_to_zeek_dhcp_address_assigned_453773ce field: zeek.dhcp.assigned_addr target_field: zeek.dhcp.address.assigned ignore_missing: true - rename: + tag: rename_zeek_dhcp_client_addr_to_zeek_dhcp_address_client_026c4398 field: zeek.dhcp.client_addr target_field: zeek.dhcp.address.client ignore_missing: true - rename: + tag: rename_zeek_dhcp_mac_to_zeek_dhcp_address_mac_287b8e3e field: zeek.dhcp.mac target_field: zeek.dhcp.address.mac ignore_missing: true - rename: + tag: rename_zeek_dhcp_requested_addr_to_zeek_dhcp_address_requested_afb9107c field: zeek.dhcp.requested_addr target_field: zeek.dhcp.address.requested ignore_missing: true - rename: + tag: rename_zeek_dhcp_server_addr_to_zeek_dhcp_address_server_fce0bc00 field: zeek.dhcp.server_addr target_field: zeek.dhcp.address.server ignore_missing: true - rename: + tag: rename_zeek_dhcp_host_name_to_zeek_dhcp_hostname_c81f3f49 field: zeek.dhcp.host_name target_field: zeek.dhcp.hostname ignore_missing: true - rename: + tag: rename_zeek_dhcp_client_message_to_zeek_dhcp_msg_client_b561f5cf field: zeek.dhcp.client_message target_field: zeek.dhcp.msg.client ignore_missing: true - rename: + tag: rename_zeek_dhcp_server_message_to_zeek_dhcp_msg_server_4520d757 field: zeek.dhcp.server_message target_field: zeek.dhcp.msg.server ignore_missing: true - rename: + tag: rename_zeek_dhcp_msg_types_to_zeek_dhcp_msg_types_9314049b field: zeek.dhcp.msg_types target_field: zeek.dhcp.msg.types ignore_missing: true - rename: + tag: rename_zeek_dhcp_msg_orig_to_zeek_dhcp_msg_origin_818e97d0 field: zeek.dhcp.msg_orig target_field: zeek.dhcp.msg.origin ignore_missing: true - rename: + tag: rename_zeek_dhcp_client_software_to_zeek_dhcp_software_client_fa809645 field: zeek.dhcp.client_software target_field: zeek.dhcp.software.client ignore_missing: true - rename: + tag: rename_zeek_dhcp_server_software_to_zeek_dhcp_software_server_504efd65 field: zeek.dhcp.server_software target_field: zeek.dhcp.software.server ignore_missing: true - rename: + tag: rename_zeek_dhcp_circuit_id_to_zeek_dhcp_id_circuit_03003383 field: zeek.dhcp.circuit_id target_field: zeek.dhcp.id.circuit ignore_missing: true - rename: + tag: rename_zeek_dhcp_agent_remote_id_to_zeek_dhcp_id_remote_agent_f8086215 field: zeek.dhcp.agent_remote_id target_field: zeek.dhcp.id.remote_agent ignore_missing: true - rename: + tag: rename_zeek_dhcp_subscriber_id_to_zeek_dhcp_id_subscriber_6c4c2115 field: zeek.dhcp.subscriber_id target_field: zeek.dhcp.id.subscriber ignore_missing: true - rename: + tag: rename_zeek_dhcp_client_port_to_source_port_c7362953 field: zeek.dhcp.client_port target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_dhcp_server_port_to_destination_port_e87a6270 field: zeek.dhcp.server_port target_field: destination.port ignore_missing: true - set: + tag: set_network_name_e9b2c463 field: network.name copy_from: zeek.dhcp.domain if: ctx.zeek?.dhcp?.domain != null - set: + tag: set_source_port_76b2113c field: source.port value: 68 if: ctx.source?.port == null - set: + tag: set_destination_port_605ef611 field: destination.port value: 67 if: ctx.destination?.port == null - set: + tag: set_source_address_0f28001b field: source.address copy_from: zeek.dhcp.address.client ignore_empty_value: true - set: + tag: set_client_address_7d576639 field: client.address copy_from: zeek.dhcp.address.client ignore_empty_value: true - set: + tag: set_source_ip_eb9e3caa field: source.ip copy_from: zeek.dhcp.address.client ignore_empty_value: true - set: + tag: set_destination_address_f04decb8 field: destination.address copy_from: zeek.dhcp.address.server ignore_empty_value: true - set: + tag: set_destination_ip_e3283d57 field: destination.ip copy_from: zeek.dhcp.address.server ignore_empty_value: true - set: + tag: set_server_address_e00746b9 field: server.address copy_from: zeek.dhcp.address.server ignore_empty_value: true - date: + tag: date_zeek_dhcp_ts_4ed7aefa field: zeek.dhcp.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_dhcp_ts_b8519076 field: zeek.dhcp.ts - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - community_id: + tag: community_id_612651e3 target_field: network.community_id on_failure: - set: @@ -183,4 +230,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml index 42e461859a4..d35ea0508c9 100644 --- a/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml @@ -2,135 +2,172 @@ description: Pipeline for normalizing Zeek dnp3.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_dnp3_f3bbeed3 field: _temp_ target_field: zeek.dnp3 # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_556db83d field: network.protocol value: dnp3 - dot_expander: + tag: dot_expander_id_orig_p_3fb4f661 path: zeek.dnp3 field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_0c7f29f9 path: zeek.dnp3 field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_e064cf26 path: zeek.dnp3 field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_fa23a5fe path: zeek.dnp3 field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_dnp3_id_orig_h_to_source_address_5e4c214e field: zeek.dnp3.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_dnp3_id_orig_p_to_source_port_bb206617 field: zeek.dnp3.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_dnp3_id_resp_h_to_destination_address_47fcc200 field: zeek.dnp3.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_dnp3_id_resp_p_to_destination_port_f51d5651 field: zeek.dnp3.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_dnp3_uid_to_zeek_session_id_de1cebab field: zeek.dnp3.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_dnp3_fc_request_to_zeek_dnp3_function_request_e5edb712 field: zeek.dnp3.fc_request target_field: zeek.dnp3.function.request ignore_missing: true - rename: + tag: rename_zeek_dnp3_fc_reply_to_zeek_dnp3_function_reply_d41a0722 field: zeek.dnp3.fc_reply target_field: zeek.dnp3.function.reply ignore_missing: true - rename: + tag: rename_zeek_dnp3_iin_to_zeek_dnp3_id_5c4211cd field: zeek.dnp3.iin target_field: zeek.dnp3.id ignore_missing: true - date: + tag: date_zeek_dnp3_ts_6e7f418e field: zeek.dnp3.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_dnp3_ts_b1c53d72 field: zeek.dnp3.ts - set: + tag: set_event_action_0a97089b field: event.action copy_from: zeek.dnp3.function.request if: "ctx.zeek?.dnp3?.function?.request != null" - set: + tag: set_event_action_a60d02eb field: event.action copy_from: zeek.dnp3.function.reply if: "ctx.zeek?.dnp3?.function?.reply != null" - lowercase: + tag: lowercase_event_action_9334b869 field: event.action ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_geo_394bcdb5 field: destination.ip target_field: destination.geo - geoip: + tag: geoip_source_ip_to_source_geo_0494c6b1 field: source.ip target_field: source.geo - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -139,6 +176,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -147,34 +185,42 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: ctx.source?.ip != null allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: ctx.destination?.ip != null allow_duplicates: false - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_d1acf53b field: - zeek.dnp3.id ignore_missing: true @@ -184,4 +230,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml index 643dd527c05..97a85639539 100644 --- a/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml @@ -2,149 +2,188 @@ description: Pipeline for Filebeat Zeek dns.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_dns_45ec9173 field: _temp_ target_field: zeek.dns # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_network_protocol_96a840d1 field: network.protocol value: dns - dot_expander: + tag: dot_expander_id_orig_p_1f7ee71d path: zeek.dns field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_49fe96a5 path: zeek.dns field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_8fb6b820 path: zeek.dns field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_b5b65aa8 path: zeek.dns field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_dns_id_orig_h_to_source_address_3832f9c6 field: zeek.dns.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_dns_id_orig_p_to_source_port_b500365f field: zeek.dns.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_dns_id_resp_h_to_destination_address_167cc768 field: zeek.dns.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_dns_id_resp_p_to_destination_port_db36d389 field: zeek.dns.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_dns_uid_to_zeek_session_id_03b41443 field: zeek.dns.uid target_field: zeek.session_id ignore_missing: true - rename: + tag: rename_zeek_dns_proto_to_network_transport_607a912b field: zeek.dns.proto target_field: network.transport ignore_missing: true - set: + tag: set_event_id_aebf373d field: event.id copy_from: zeek.session_id ignore_empty_value: true - set: + tag: set_source_ip_d4d6cdcc field: source.ip copy_from: source.address ignore_empty_value: true - set: + tag: set_destination_ip_cc9671de field: destination.ip copy_from: destination.address ignore_empty_value: true - append: + tag: append_dns_header_flags_1b67509e field: dns.header_flags value: AA if: ctx.zeek?.dns?.AA == true - append: + tag: append_dns_header_flags_2d081654 field: dns.header_flags value: TC if: ctx.zeek?.dns?.TC == true - append: + tag: append_dns_header_flags_c9df60fe field: dns.header_flags value: RD if: ctx.zeek?.dns?.RD == true - append: + tag: append_dns_header_flags_f1069d14 field: dns.header_flags value: RA if: ctx.zeek?.dns?.RA == true - set: + tag: set_dns_question_class_0affa25a field: dns.question.class value: IN if: ctx.zeek?.dns?.qclass == 1 - set: + tag: set_dns_question_class_d12a46c0 field: dns.question.class value: CH if: ctx.zeek?.dns?.qclass == 3 - set: + tag: set_dns_question_class_860a8e8b field: dns.question.class value: HS if: ctx.zeek?.dns?.qclass == 4 - set: + tag: set_dns_question_class_8d7fb955 field: dns.question.class value: NONE if: ctx.zeek?.dns?.qclass == 254 - set: + tag: set_dns_question_class_257a90da field: dns.question.class value: ANY if: ctx.zeek?.dns?.qclass == 255 - set: + tag: set_dns_type_badb9d79 field: dns.type value: answer if: ctx.zeek?.dns?.rcode_name != null - set: + tag: set_dns_type_c4fb6b6c field: dns.type value: query if: ctx.dns?.type == null - script: + tag: script_ad41a13c lang: painless source: >- ctx.event.duration = ctx.zeek.dns.rtt * 1000000000L; if: "ctx.zeek?.dns?.rtt != null" - script: + tag: script_2f27409e lang: painless source: >- def answers = ctx.zeek.dns.answers; @@ -165,6 +204,7 @@ processors: ctx.dns.answers = lst; if: "ctx.zeek?.dns?.answers != null && ctx.zeek?.dns?.TTLs != null" - foreach: + tag: foreach_dns_answers_4353c0cc field: dns.answers processor: convert: @@ -175,6 +215,7 @@ processors: ignore_missing: true if: 'ctx.dns?.answers != null && !ctx.dns?.answers.isEmpty()' - script: + tag: script_f5b5a715 lang: painless source: >- def answers = ctx.dns.answers; @@ -188,55 +229,68 @@ processors: ctx.dns.resolved_ip = iplist; if: 'ctx.dns?.answers != null && !ctx.dns?.answers.isEmpty()' - set: + tag: set_event_outcome_d97e22aa field: event.outcome value: success if: ctx.dns?.rcode == 0 - set: + tag: set_event_outcome_d8aaefc7 field: event.outcome value: success if: ctx.event?.outcome == null - convert: + tag: convert_zeek_dns_trans_id_1062c3ea field: zeek.dns.trans_id type: string ignore_missing: true - set: + tag: set_dns_id_73821fb7 field: dns.id copy_from: zeek.dns.trans_id ignore_empty_value: true - set: + tag: set_dns_question_type_f72e2fc9 field: dns.question.type copy_from: zeek.dns.qtype_name ignore_empty_value: true - set: + tag: set_dns_response_code_69b53f00 field: dns.response_code copy_from: zeek.dns.rcode_name ignore_empty_value: true - registered_domain: + tag: registered_domain_zeek_dns_query_to_dns_question_dd5b6f3a field: zeek.dns.query target_field: dns.question ignore_missing: true - rename: + tag: rename_dns_question_domain_to_dns_question_name_88a92ba9 field: dns.question.domain target_field: dns.question.name ignore_missing: true - date: + tag: date_zeek_dns_ts_b73b43f2 field: zeek.dns.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_dns_ts_ba88abfe field: zeek.dns.ts # IP Geolocation Lookup - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true # IP Autonomous System (AS) Lookup - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -245,6 +299,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -253,39 +308,48 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - community_id: + tag: community_id_612651e3 target_field: network.community_id - append: + tag: append_related_ip_bf444bca field: related.ip value: "{{{source.ip}}}" if: ctx.source?.address != null allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: ctx.destination?.ip != null allow_duplicates: false - rename: + tag: rename_message_to_event_original_56a77272 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_3d30813a field: - zeek.dns.Z - zeek.dns.auth @@ -298,4 +362,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml index 5a5b982e616..ab30689485e 100644 --- a/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml @@ -2,109 +2,138 @@ description: Pipeline for normalizing Zeek dpd.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_dpd_6935811e field: _temp_ target_field: zeek.dpd # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_8a66ccaa field: event.type value: info - dot_expander: + tag: dot_expander_id_orig_p_bb928a88 path: zeek.dpd field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_3221b7e0 path: zeek.dpd field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_e4d795c9 path: zeek.dpd field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_c215e161 path: zeek.dpd field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_dpd_id_orig_h_to_source_address_6fd8d8eb field: zeek.dpd.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_dpd_id_orig_p_to_source_port_45569a58 field: zeek.dpd.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_dpd_id_resp_h_to_destination_address_c46c69a7 field: zeek.dpd.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_dpd_id_resp_p_to_destination_port_c46df0f4 field: zeek.dpd.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_dpd_proto_to_network_transport_8344ecf4 field: zeek.dpd.proto target_field: network.transport ignore_missing: true - rename: + tag: rename_zeek_dpd_uid_to_zeek_session_id_2568bb14 field: zeek.dpd.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - date: + tag: date_zeek_dpd_ts_867668bf field: zeek.dpd.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_dpd_ts_8100932d field: zeek.dpd.ts - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -113,6 +142,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -121,34 +151,42 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_5c633780 field: - zeek.dpd.id ignore_missing: true @@ -158,4 +196,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml index b93ce63f936..8eba5b5f4bf 100644 --- a/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml @@ -2,83 +2,105 @@ description: Pipeline for normalizing Zeek files.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_files_e8c3167b field: _temp_ target_field: zeek.files # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_423f7188 field: event.category value: file - append: + tag: append_event_type_8a66ccaa field: event.type value: info - rename: + tag: rename_zeek_files_conn_uids_to_zeek_files_session_ids_b2d7b297 field: zeek.files.conn_uids target_field: zeek.files.session_ids ignore_missing: true - set: + tag: set_file_mime_type_c4fa33ed field: file.mime_type copy_from: zeek.files.mime_type if: ctx.zeek?.files?.mime_type != null - rename: + tag: rename_zeek_files_filename_to_file_name_eada5108 field: zeek.files.filename target_field: file.name ignore_missing: true - rename: + tag: rename_zeek_files_total_bytes_to_file_size_e7c3737b field: zeek.files.total_bytes target_field: file.size ignore_missing: true - set: + tag: set_file_hash_md5_068fed34 field: file.hash.md5 copy_from: zeek.files.md5 if: ctx.zeek?.files?.md5 != null - set: + tag: set_file_hash_sha1_e40df607 field: file.hash.sha1 copy_from: zeek.files.sha1 if: ctx.zeek?.files?.sha1 != null - set: + tag: set_file_hash_sha256_7afacdb1 field: file.hash.sha256 copy_from: zeek.files.sha256 if: ctx.zeek?.files?.sha256 != null - date: + tag: date_zeek_files_ts_fd2cf6ca field: zeek.files.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_files_ts_023e62ae field: zeek.files.ts - script: + tag: script_d61d0d05 lang: painless source: ctx.zeek.session_id = ctx.zeek.files.session_ids[0]; if: ctx.zeek.files.session_ids != null ignore_failure: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - foreach: + tag: foreach_zeek_files_tx_hosts_bc57860a field: zeek.files.tx_hosts processor: append: @@ -86,14 +108,17 @@ processors: value: "{{{_ingest._value}}}" ignore_missing: true - script: + tag: script_86142fdd lang: painless source: ctx.zeek.files.tx_host = ctx.zeek.files.tx_hosts[0]; ctx.zeek.files.remove('tx_hosts'); ignore_failure: true - set: + tag: set_server_ip_f98d4ea4 field: server.ip copy_from: zeek.files.tx_host if: "ctx.zeek?.files?.tx_host != null" - foreach: + tag: foreach_zeek_files_rx_hosts_1e8660c8 field: zeek.files.rx_hosts processor: append: @@ -101,29 +126,35 @@ processors: value: "{{{_ingest._value}}}" ignore_missing: true - script: + tag: script_0641ec4b lang: painless source: ctx.zeek.files.rx_host = ctx.zeek.files.rx_hosts[0]; ctx.zeek.files.remove('rx_hosts'); ignore_failure: true - set: + tag: set_client_ip_ad58129d field: client.ip value: "{{{zeek.files.rx_host}}}" if: "ctx.zeek?.files?.rx_host != null" - append: + tag: append_related_hash_6af42da3 field: related.hash value: "{{{file.hash.md5}}}" if: "ctx.file?.hash?.md5 != null" allow_duplicates: false - append: + tag: append_related_hash_b56c7961 field: related.hash value: "{{{file.hash.sha1}}}" if: "ctx.file?.hash?.sha1 != null" allow_duplicates: false - append: + tag: append_related_hash_017aef51 field: related.hash value: "{{{file.hash.sha256}}}" if: "ctx.file?.hash?.sha256 != null" allow_duplicates: false - remove: + tag: remove_8d1033a0 field: - zeek.files.x509 ignore_missing: true @@ -133,4 +164,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml index 2d1f881855b..bdd00131689 100644 --- a/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml @@ -2,190 +2,240 @@ description: Pipeline for normalizing Zeek ftp.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_ftp_f1b8a7e0 field: _temp_ target_field: zeek.ftp # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_68bba8e2 field: network.protocol value: ftp - dot_expander: + tag: dot_expander_id_orig_p_998a029e path: zeek.ftp field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_83bd2016 path: zeek.ftp field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_1c040e57 path: zeek.ftp field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_6e80e90f path: zeek.ftp field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_ftp_id_orig_h_to_source_address_4bddfd2d field: zeek.ftp.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_ftp_id_orig_p_to_source_port_bf53a82a field: zeek.ftp.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_ftp_id_resp_h_to_destination_address_a57e8949 field: zeek.ftp.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_ftp_id_resp_p_to_destination_port_ee61e826 field: zeek.ftp.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_ftp_uid_to_zeek_session_id_5ae0938a field: zeek.ftp.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - set: + tag: set_user_name_28e99f60 field: user.name copy_from: zeek.ftp.user if: ctx.zeek?.ftp?.user != null - set: + tag: set_event_action_21526a46 field: event.action copy_from: zeek.ftp.command if: ctx.zeek?.ftp?.command != null - rename: + tag: rename_zeek_ftp_file_size_to_file_size_573f6b0e field: zeek.ftp.file_size target_field: file.size ignore_missing: true - rename: + tag: rename_zeek_ftp_mime_type_to_file_mime_type_6df9c3c1 field: zeek.ftp.mime_type target_field: file.mime_type ignore_missing: true - rename: + tag: rename_zeek_ftp_fuid_to_zeek_ftp_file_uid_a05d9fd6 field: zeek.ftp.fuid target_field: zeek.ftp.file.uid ignore_missing: true - rename: + tag: rename_zeek_ftp_reply_code_to_zeek_ftp_reply_code_fd9ec1c9 field: zeek.ftp.reply_code target_field: zeek.ftp.reply.code ignore_missing: true - rename: + tag: rename_zeek_ftp_reply_msg_to_zeek_ftp_reply_msg_762b8d97 field: zeek.ftp.reply_msg target_field: zeek.ftp.reply.msg ignore_missing: true - dot_expander: + tag: dot_expander_data_channel_orig_h_32f90dbb path: zeek.ftp field: data_channel.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_data_channel_orig_p_58d7fdd3 path: zeek.ftp field: data_channel.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_data_channel_resp_h_734963f2 path: zeek.ftp field: data_channel.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_data_channel_resp_p_09d0ef6a path: zeek.ftp field: data_channel.resp_p ignore_failure: true - rename: + tag: rename_zeek_ftp_data_channel_orig_h_to_zeek_ftp_data_channel_originating_host_71b0fb32 field: zeek.ftp.data_channel.orig_h target_field: zeek.ftp.data_channel.originating_host ignore_missing: true - rename: + tag: rename_zeek_ftp_data_channel_orig_p_to_zeek_ftp_data_channel_originating_port_5eac5a99 field: zeek.ftp.data_channel.orig_p target_field: zeek.ftp.data_channel.originating_port ignore_missing: true - rename: + tag: rename_zeek_ftp_data_channel_resp_h_to_zeek_ftp_data_channel_response_host_2ad86ba3 field: zeek.ftp.data_channel.resp_h target_field: zeek.ftp.data_channel.response_host ignore_missing: true - rename: + tag: rename_zeek_ftp_data_channel_resp_p_to_zeek_ftp_data_channel_response_port_766cfd7c field: zeek.ftp.data_channel.resp_p target_field: zeek.ftp.data_channel.response_port ignore_missing: true - date: + tag: date_zeek_ftp_ts_adec15fd field: zeek.ftp.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_ftp_ts_de09002f field: zeek.ftp.ts - dot_expander: + tag: dot_expander_data_channel_passive_c24de6eb field: data_channel.passive path: zeek.ftp - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - append: + tag: append_related_user_837e080f field: related.user value: "{{{user.name}}}" if: "ctx.user?.name != null" allow_duplicates: false - geoip: + tag: geoip_destination_ip_to_destination_geo_394bcdb5 field: destination.ip target_field: destination.geo - geoip: + tag: geoip_source_ip_to_source_geo_0494c6b1 field: source.ip target_field: source.geo - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -194,6 +244,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -202,24 +253,30 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_1cd9b74a field: - zeek.ftp.id ignore_missing: true @@ -229,4 +286,8 @@ on_failure: value: pipeline_error - set: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml index eb89ec02996..8ac4b7c9fb9 100644 --- a/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml @@ -2,187 +2,237 @@ description: Pipeline for normalizing Zeek http.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_http_4aa6e6f2 field: _temp_ target_field: zeek.http # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_category_4595ee28 field: event.category value: web - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - dot_expander: + tag: dot_expander_id_orig_p_6ab66410 path: zeek.http field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_738cdb48 path: zeek.http field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_5f200537 path: zeek.http field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_01779d8f path: zeek.http field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_http_id_orig_h_to_source_address_1501a19f field: zeek.http.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_http_id_orig_p_to_source_port_6625b1fc field: zeek.http.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_http_id_resp_h_to_destination_address_1527d85b field: zeek.http.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_http_id_resp_p_to_destination_port_8cf05b08 field: zeek.http.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_http_uid_to_zeek_session_id_49e2f200 field: zeek.http.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_http_method_to_http_request_method_d300e4aa field: zeek.http.method target_field: http.request.method ignore_missing: true - rename: + tag: rename_zeek_http_referrer_to_http_request_referrer_3922cfe2 field: zeek.http.referrer target_field: http.request.referrer ignore_missing: true - rename: + tag: rename_zeek_http_status_code_to_http_response_status_code_1681ad88 field: zeek.http.status_code target_field: http.response.status_code ignore_missing: true - rename: + tag: rename_zeek_http_version_to_http_version_e841b821 field: zeek.http.version target_field: http.version ignore_missing: true - rename: + tag: rename_zeek_http_request_body_len_to_http_request_body_bytes_e6857151 field: zeek.http.request_body_len target_field: http.request.body.bytes ignore_missing: true - rename: + tag: rename_zeek_http_response_body_len_to_http_response_body_bytes_34ea4413 field: zeek.http.response_body_len target_field: http.response.body.bytes ignore_missing: true - uri_parts: + tag: uri_parts_zeek_http_uri_44969687 if: ctx.zeek?.http?.uri != null field: zeek.http.uri on_failure: - set: + tag: set_url_original_65c79f89 field: url.original copy_from: zeek.http.uri ignore_failure: true - append: + tag: append_tags_42ba1fd5 field: tags value: _zeek_http_url_parse_failure - remove: + tag: remove_zeek_http_uri_f50a6c07 field: zeek.http.uri ignore_missing: true - remove: + tag: remove_url_domain_e9f80361 field: url.domain ignore_missing: true if: "ctx.url?.domain == null || ctx.url?.domain.isEmpty()" - remove: + tag: remove_url_scheme_1ff54ede field: url.scheme ignore_missing: true if: "ctx.url?.scheme == null || ctx.url?.scheme.isEmpty()" - rename: + tag: rename_zeek_http_host_to_url_domain_e92aff84 field: zeek.http.host target_field: url.domain ignore_missing: true if: ctx.zeek?.http?.host != null && ctx.url?.domain == null - remove: + tag: remove_zeek_http_host_584ee44d field: zeek.http.host ignore_missing: true if: ctx.zeek?.http?.host == ctx.url?.domain - rename: + tag: rename_zeek_http_username_to_url_username_248cbd1c field: zeek.http.username target_field: url.username ignore_missing: true - rename: + tag: rename_zeek_http_password_to_url_password_ec8e79c0 field: zeek.http.password target_field: url.password ignore_missing: true - rename: + tag: rename_zeek_http_user_agent_to_user_agent_original_dc1f0122 field: zeek.http.user_agent target_field: user_agent.original ignore_missing: true - set: + tag: set_event_action_25de2e5c field: event.action copy_from: http.request.method if: ctx.http?.request?.method != null - set: + tag: set_user_name_35160f21 field: user.name copy_from: url.username if: ctx.url?.username != null - date: + tag: date_zeek_http_ts_0ba9a207 field: zeek.http.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_http_ts_4fc0d8bd field: zeek.http.ts - geoip: + tag: geoip_destination_ip_to_destination_geo_394bcdb5 field: destination.ip target_field: destination.geo - geoip: + tag: geoip_source_ip_to_source_geo_0494c6b1 field: source.ip target_field: source.geo - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -191,6 +241,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -199,50 +250,62 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - user_agent: + tag: user_agent_user_agent_original_b5325863 field: user_agent.original ignore_missing: true - set: + tag: set_event_outcome_b50e8224 field: event.outcome value: success if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400" - set: + tag: set_event_outcome_75086854 field: event.outcome value: failure if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code >= 400" - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - append: + tag: append_related_user_4822ea5b field: related.user value: "{{{url.username}}}" if: "ctx.url?.username != null" allow_duplicates: false - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_55474e4a field: - zeek.http.id ignore_missing: true @@ -252,4 +315,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml index 4e5937e2186..2f3c651d6d0 100644 --- a/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml @@ -2,131 +2,165 @@ description: Pipeline for normalizing Zeek intel.log. processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_intel_89c4bf82 field: _temp_ target_field: zeek.intel # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_a131107f field: event.kind value: enrichment - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_4cc22c32 field: event.category value: threat - append: + tag: append_event_type_7779ef85 field: event.type value: indicator - dot_expander: + tag: dot_expander_id_orig_p_99458db8 path: zeek.intel field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_6ca59120 path: zeek.intel field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_3364a8e9 path: zeek.intel field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_c5ac6911 path: zeek.intel field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_intel_id_orig_h_to_source_address_34d19377 field: zeek.intel.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_intel_id_orig_p_to_source_port_78111064 field: zeek.intel.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_intel_id_resp_h_to_destination_address_fe7d7363 field: zeek.intel.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_intel_id_resp_p_to_destination_port_2e62e420 field: zeek.intel.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_intel_uid_to_zeek_session_id_a32e12d8 field: zeek.intel.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - dot_expander: + tag: dot_expander_seen_indicator_e4bf7099 path: zeek.intel field: seen.indicator ignore_failure: true - dot_expander: + tag: dot_expander_seen_indicator_type_8d517aaa path: zeek.intel field: seen.indicator_type ignore_failure: true - dot_expander: + tag: dot_expander_seen_host_90148a4a path: zeek.intel field: seen.host ignore_failure: true - dot_expander: + tag: dot_expander_seen_where_2cc0e863 path: zeek.intel field: seen.where ignore_failure: true - dot_expander: + tag: dot_expander_seen_node_d7c28130 path: zeek.intel field: seen.node ignore_failure: true - dot_expander: + tag: dot_expander_seen_conn_2a4f49d2 path: zeek.intel field: seen.conn ignore_failure: true - dot_expander: + tag: dot_expander_seen_uid_45500be4 path: zeek.intel field: seen.uid ignore_failure: true - dot_expander: + tag: dot_expander_seen_f_787585c4 path: zeek.intel field: seen.f ignore_failure: true - dot_expander: + tag: dot_expander_seen_fuid_7551a450 path: zeek.intel field: seen.fuid ignore_failure: true - date: + tag: date_zeek_intel_ts_2a754abf field: zeek.intel.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_intel_ts_df212a2d field: zeek.intel.ts # IP Geolocation Lookup - geoip: + tag: geoip_source_ip_to_source_geo_cfbe207d if: ctx.source?.geo == null field: source.ip target_field: source.geo @@ -140,6 +174,7 @@ processors: - region_iso_code - region_name - geoip: + tag: geoip_destination_ip_to_destination_geo_f4263b5c if: ctx.destination?.geo == null field: destination.ip target_field: destination.geo @@ -154,6 +189,7 @@ processors: - region_name # IP Autonomous System (AS) Lookup - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -162,6 +198,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -170,40 +207,49 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: "related.ip" value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: "related.ip" value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - community_id: + tag: community_id_612651e3 target_field: network.community_id - rename: + tag: rename_message_to_event_original_56a77273 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null # Add threat indicators. - convert: + tag: convert_zeek_intel_seen_indicator_to_threat_indicator_ip_4f1135c2 target_field: threat.indicator.ip field: zeek.intel.seen.indicator type: ip @@ -211,14 +257,17 @@ processors: ignore_missing: true ignore_failure: true - set: + tag: set_threat_indicator_type_795ab288 field: threat.indicator.type value: ipv4-addr if: ctx.zeek?.intel?.seen?.indicator_type == "Intel::ADDR" - set: + tag: set_threat_indicator_type_672fbc23 field: threat.indicator.type value: ipv6-addr if: ctx.threat?.indicator?.ip?.contains(':') == true - geoip: + tag: geoip_threat_indicator_ip_to_threat_indicator_as_22b400d3 database_file: GeoLite2-ASN.mmdb field: threat.indicator.ip target_field: threat.indicator.as @@ -227,16 +276,19 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_threat_indicator_as_asn_to_threat_indicator_as_number_4cf53b68 field: threat.indicator.as.asn target_field: threat.indicator.as.number ignore_missing: true ignore_failure: true - rename: + tag: rename_threat_indicator_as_organization_name_to_threat_indicator_as_organization_name_46c670c0 field: threat.indicator.as.organization_name target_field: threat.indicator.as.organization.name ignore_missing: true ignore_failure: true - geoip: + tag: geoip_threat_indicator_ip_to_threat_indicator_geo_345473f0 field: threat.indicator.ip target_field: threat.indicator.geo ignore_missing: true @@ -250,15 +302,18 @@ processors: - region_name - timezone - set: + tag: set_threat_indicator__url_a4b97d62 field: threat.indicator._url copy_from: zeek.intel.seen.indicator if: ctx.zeek?.intel?.seen?.indicator_type == "Intel::URL" ignore_failure: true - set: + tag: set_threat_indicator_type_fb479854 field: threat.indicator.type value: url if: ctx.zeek?.intel?.seen?.indicator_type == "Intel::URL" - uri_parts: + tag: uri_parts_threat_indicator__url_to_threat_indicator_url_5fcf7b95 field: threat.indicator._url target_field: threat.indicator.url ignore_failure: true @@ -266,24 +321,29 @@ processors: remove_if_successful: true if: ctx.threat?.indicator?._url != null - set: + tag: set_threat_indicator_email_address_4fbc3673 field: threat.indicator.email.address copy_from: zeek.intel.seen.indicator if: ctx.zeek?.intel?.seen?.indicator_type == "Intel::EMAIL" ignore_failure: true - set: + tag: set_threat_indicator_type_abd36834 field: threat.indicator.type value: email-addr if: ctx.zeek?.intel?.seen?.indicator_type == "Intel::EMAIL" - set: + tag: set_threat_indicator_file_name_f35b0e6a field: threat.indicator.file.name copy_from: zeek.intel.seen.indicator if: ctx.zeek?.intel?.seen?.indicator_type == "Intel::FILE_NAME" ignore_failure: true - set: + tag: set_threat_indicator_type_137190ee field: threat.indicator.type value: file if: ctx.zeek?.intel?.seen?.indicator_type == "Intel::FILE_NAME" - remove: + tag: remove_028bd494 field: - zeek.intel.id ignore_missing: true @@ -293,4 +353,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml index bd6af836380..b831ddc0cef 100644 --- a/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml @@ -2,147 +2,186 @@ description: Pipeline for normalizing Zeek irc.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_irc_89a797f6 field: _temp_ target_field: zeek.irc # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_6ca148e4 field: network.protocol value: irc - dot_expander: + tag: dot_expander_id_orig_p_b880a3d0 path: zeek.irc field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_bbb0e138 path: zeek.irc field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_28247bdd path: zeek.irc field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_c751a7a5 path: zeek.irc field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_irc_id_orig_h_to_source_address_ed9ce593 field: zeek.irc.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_irc_id_orig_p_to_source_port_8bba7890 field: zeek.irc.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_irc_id_resp_h_to_destination_address_be37515f field: zeek.irc.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_irc_id_resp_p_to_destination_port_7e73391c field: zeek.irc.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_irc_uid_to_zeek_session_id_e0368bfc field: zeek.irc.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_irc_dcc_file_name_to_file_name_930b7675 field: zeek.irc.dcc_file_name target_field: file.name ignore_missing: true - rename: + tag: rename_zeek_irc_dcc_mime_type_to_file_mime_type_1b263d08 field: zeek.irc.dcc_mime_type target_field: file.mime_type ignore_missing: true - rename: + tag: rename_zeek_irc_dcc_file_size_to_file_size_89378daf field: zeek.irc.dcc.file.size target_field: file.size ignore_missing: true - rename: + tag: rename_zeek_irc_user_to_user_name_ecbee06e field: zeek.irc.user target_field: user.name ignore_missing: true - set: + tag: set_event_action_5dd80ec2 field: event.action copy_from: zeek.irc.command if: ctx.zeek?.irc?.command != null - date: + tag: date_zeek_irc_ts_76b35db7 field: zeek.irc.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_irc_ts_2fb03b35 field: zeek.irc.ts - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - append: + tag: append_related_user_837e080f field: related.user value: "{{{user.name}}}" if: "ctx.user?.name != null" allow_duplicates: false - geoip: + tag: geoip_destination_ip_to_destination_geo_394bcdb5 field: destination.ip target_field: destination.geo - geoip: + tag: geoip_source_ip_to_source_geo_0494c6b1 field: source.ip target_field: source.geo - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -151,6 +190,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -159,24 +199,30 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_b70f4094 field: - zeek.irc.id ignore_missing: true @@ -186,4 +232,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml index 517b86b3a70..1c4858b634f 100644 --- a/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml @@ -2,198 +2,250 @@ description: Pipeline for normalizing Zeek kerberos.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_kerberos_47ac0029 field: _temp_ target_field: zeek.kerberos # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_70727d9c field: event.category value: ["network", "authentication"] - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - append: + tag: append_event_type_f8289914 field: event.type value: access - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_10e639d3 field: network.protocol value: kerberos - dot_expander: + tag: dot_expander_id_orig_p_5de9dc0f path: zeek.kerberos field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_be7bf4f7 path: zeek.kerberos field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_037b3bdc path: zeek.kerberos field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_80966784 path: zeek.kerberos field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_kerberos_id_orig_h_to_source_address_37d8f2a4 field: zeek.kerberos.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_kerberos_id_orig_p_to_source_port_e87f37b5 field: zeek.kerberos.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_kerberos_id_resp_h_to_destination_address_af377016 field: zeek.kerberos.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_kerberos_id_resp_p_to_destination_port_7a8832cf field: zeek.kerberos.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_kerberos_uid_to_zeek_session_id_8c9cc765 field: zeek.kerberos.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_client_address_0aa293ce field: client.address copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - set: + tag: set_server_address_3c18622e field: server.address copy_from: destination.address if: ctx.destination?.address != null - set: + tag: set_event_action_898159a2 field: event.action copy_from: zeek.kerberos.request_type if: ctx.zeek?.kerberos?.request_type != null - rename: + tag: rename_zeek_kerberos_till_to_zeek_kerberos_valid_until_b6c50fab field: zeek.kerberos.till target_field: zeek.kerberos.valid.until ignore_missing: true - rename: + tag: rename_zeek_kerberos_from_to_zeek_kerberos_valid_from_ccc374b0 field: zeek.kerberos.from target_field: zeek.kerberos.valid.from ignore_missing: true - rename: + tag: rename_zeek_kerberos_error_code_to_zeek_kerberos_error_code_eafb0d0d field: zeek.kerberos.error_code target_field: zeek.kerberos.error.code ignore_missing: true - rename: + tag: rename_zeek_kerberos_error_msg_to_zeek_kerberos_error_msg_90fc02d1 field: zeek.kerberos.error_msg target_field: zeek.kerberos.error.msg ignore_missing: true - dot_expander: + tag: dot_expander_cert_client_3ec3efaf path: zeek.kerberos field: cert.client ignore_failure: true - dot_expander: + tag: dot_expander_cert_client_subject_99470160 path: zeek.kerberos field: cert.client_subject ignore_failure: true - dot_expander: + tag: dot_expander_cert_client_fuid_954cf900 path: zeek.kerberos field: cert.client_fuid ignore_failure: true - dot_expander: + tag: dot_expander_cert_server_87f2bd3b path: zeek.kerberos field: cert.server ignore_failure: true - dot_expander: + tag: dot_expander_cert_server_subject_4cbeec3c path: zeek.kerberos field: cert.server_subject ignore_failure: true - dot_expander: + tag: dot_expander_cert_server_fuid_be77a144 path: zeek.kerberos field: cert.server_fuid ignore_failure: true - rename: + tag: rename_zeek_kerberos_cert_client_to_zeek_kerberos_cert_client_value_f0b21b19 field: zeek.kerberos.cert.client target_field: zeek.kerberos.cert.client.value ignore_missing: true - rename: + tag: rename_zeek_kerberos_cert_client_subject_to_zeek_kerberos_cert_client_subject_5cbaf46d field: zeek.kerberos.cert.client_subject target_field: zeek.kerberos.cert.client.subject ignore_missing: true - rename: + tag: rename_zeek_kerberos_cert_client_fuid_to_zeek_kerberos_cert_client_fuid_ec36d35d field: zeek.kerberos.cert.client_fuid target_field: zeek.kerberos.cert.client.fuid ignore_missing: true - rename: + tag: rename_zeek_kerberos_cert_server_to_zeek_kerberos_cert_server_value_cc147761 field: zeek.kerberos.cert.server target_field: zeek.kerberos.cert.server.value ignore_missing: true - rename: + tag: rename_zeek_kerberos_cert_server_subject_to_zeek_kerberos_cert_server_subject_62348fa5 field: zeek.kerberos.cert.server_subject target_field: zeek.kerberos.cert.server.subject ignore_missing: true - rename: + tag: rename_zeek_kerberos_cert_server_fuid_to_zeek_kerberos_cert_server_fuid_4d50cc6d field: zeek.kerberos.cert.server_fuid target_field: zeek.kerberos.cert.server.fuid ignore_missing: true - rename: + tag: rename_zeek_kerberos_auth_ticket_to_zeek_kerberos_ticket_auth_8c6a721d field: zeek.kerberos.auth_ticket target_field: zeek.kerberos.ticket.auth ignore_missing: true - rename: + tag: rename_zeek_kerberos_new_ticket_to_zeek_kerberos_ticket_new_81c387a1 field: zeek.kerberos.new_ticket target_field: zeek.kerberos.ticket.new ignore_missing: true - dissect: + tag: dissect_zeek_kerberos_client_cda99d91 field: zeek.kerberos.client pattern: "%{user.name}/%{user.domain}" ignore_missing: true if: ctx.zeek?.kerberos?.client?.contains('/') == true - date: + tag: date_zeek_kerberos_ts_0d0cae3c field: zeek.kerberos.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_kerberos_ts_29671b14 field: zeek.kerberos.ts - script: + tag: script_21306b89 source: "ctx.zeek.kerberos.valid.days = Math.round( (ctx.zeek.kerberos.valid.until - ctx.zeek.kerberos.valid.from) / 86400 )" if: "ctx.zeek.kerberos.valid?.from != null && ctx.zeek.kerberos.valid?.until != null" - date: + tag: date_zeek_kerberos_valid_until_to_zeek_kerberos_valid_until_a7efbd79 field: zeek.kerberos.valid.until target_field: zeek.kerberos.valid.until formats: @@ -201,6 +253,7 @@ processors: - ISO8601 if: ctx.zeek.kerberos.valid?.until != null - date: + tag: date_zeek_kerberos_valid_from_to_zeek_kerberos_valid_from_a98354a1 field: zeek.kerberos.valid.from target_field: zeek.kerberos.valid.from formats: @@ -208,22 +261,27 @@ processors: - ISO8601 if: ctx.zeek.kerberos.valid?.from != null - set: + tag: set_event_outcome_d42cf482 field: event.outcome value: success if: "ctx.zeek?.kerberos?.success == true" - set: + tag: set_event_outcome_e58eeaec field: event.outcome value: failure if: "ctx.zeek?.kerberos?.success == false" - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -232,6 +290,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -240,115 +299,142 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - append: + tag: append_related_user_837e080f field: related.user value: "{{{user.name}}}" if: "ctx.user?.name != null" allow_duplicates: false - gsub: + tag: gsub_zeek_kerberos_cert_client_subject_daf776ee field: zeek.kerberos.cert.client.subject pattern: \\, replacement: "" ignore_missing: true - kv: + tag: kv_zeek_kerberos_cert_client_subject_to_zeek_kerberos_cert_client_kv_sub_6e94f2f1 field: zeek.kerberos.cert.client.subject field_split: "," value_split: "=" target_field: zeek.kerberos.cert.client.kv_sub ignore_missing: true - set: + tag: set_tls_client_x509_subject_country_043dc380 field: tls.client.x509.subject.country value: ['{{{zeek.kerberos.cert.client.kv_sub.C}}}'] if: ctx.zeek?.kerberos?.cert?.client?.kv_sub?.C instanceof String - set: + tag: set_tls_client_x509_subject_common_name_a8030e2d field: tls.client.x509.subject.common_name value: ['{{{zeek.kerberos.cert.client.kv_sub.CN}}}'] if: ctx.zeek?.kerberos?.cert?.client?.kv_sub?.CN instanceof String - set: + tag: set_tls_client_x509_subject_locality_5bbac95d field: tls.client.x509.subject.locality value: ['{{{zeek.kerberos.cert.client.kv_sub.L}}}'] if: ctx.zeek?.kerberos?.cert?.client?.kv_sub?.L instanceof String - set: + tag: set_tls_client_x509_subject_organization_aec12ae5 field: tls.client.x509.subject.organization value: ['{{{zeek.kerberos.cert.client.kv_sub.O}}}'] if: ctx.zeek?.kerberos?.cert?.client?.kv_sub?.O instanceof String - set: + tag: set_tls_client_x509_subject_organizational_unit_ca9ed45d field: tls.client.x509.subject.organizational_unit value: ['{{{zeek.kerberos.cert.client.kv_sub.OU}}}'] if: ctx.zeek?.kerberos?.cert?.client?.kv_sub?.OU instanceof String - set: + tag: set_tls_client_x509_subject_state_or_province_5acbe8ce field: tls.client.x509.subject.state_or_province value: ['{{{zeek.kerberos.cert.client.kv_sub.ST}}}'] if: ctx.zeek?.kerberos?.cert?.client?.kv_sub?.ST instanceof String - remove: + tag: remove_zeek_kerberos_cert_client_kv_sub_78a04ec1 field: zeek.kerberos.cert.client.kv_sub ignore_missing: true - gsub: + tag: gsub_zeek_kerberos_cert_server_subject_beb9616a field: zeek.kerberos.cert.server.subject pattern: \\, replacement: "" ignore_missing: true - kv: + tag: kv_zeek_kerberos_cert_server_subject_to_zeek_kerberos_cert_server_kv_sub_7fb0a791 field: zeek.kerberos.cert.server.subject field_split: "," value_split: "=" target_field: zeek.kerberos.cert.server.kv_sub ignore_missing: true - set: + tag: set_tls_server_x509_subject_country_b8eb0604 field: tls.server.x509.subject.country value: ['{{{zeek.kerberos.cert.server.kv_sub.C}}}'] if: ctx.zeek?.kerberos?.cert?.server?.kv_sub?.C instanceof String - set: + tag: set_tls_server_x509_subject_common_name_3f777e89 field: tls.server.x509.subject.common_name value: ['{{{zeek.kerberos.cert.server.kv_sub.CN}}}'] if: ctx.zeek?.kerberos?.cert?.server?.kv_sub?.CN instanceof String - set: + tag: set_tls_server_x509_subject_locality_ecbbcad1 field: tls.server.x509.subject.locality value: ['{{{zeek.kerberos.cert.server.kv_sub.L}}}'] if: ctx.zeek?.kerberos?.cert?.server?.kv_sub?.L instanceof String - set: + tag: set_tls_server_x509_subject_organization_52afd979 field: tls.server.x509.subject.organization value: ['{{{zeek.kerberos.cert.server.kv_sub.O}}}'] if: ctx.zeek?.kerberos?.cert?.server?.kv_sub?.O instanceof String - set: + tag: set_tls_server_x509_subject_organizational_unit_a15c6b91 field: tls.server.x509.subject.organizational_unit value: ['{{{zeek.kerberos.cert.server.kv_sub.OU}}}'] if: ctx.zeek?.kerberos?.cert?.server?.kv_sub?.OU instanceof String - set: + tag: set_tls_server_x509_subject_state_or_province_b42b5bda field: tls.server.x509.subject.state_or_province value: ['{{{zeek.kerberos.cert.server.kv_sub.ST}}}'] if: ctx.zeek?.kerberos?.cert?.server?.kv_sub?.ST instanceof String - remove: + tag: remove_zeek_kerberos_cert_server_kv_sub_0fc052f5 field: zeek.kerberos.cert.server.kv_sub ignore_missing: true - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_b6951cdc field: - message - json @@ -360,4 +446,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/known_certs/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/known_certs/elasticsearch/ingest_pipeline/default.yml index 549dd8561fd..102e1feea8f 100644 --- a/packages/zeek/data_stream/known_certs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/known_certs/elasticsearch/ingest_pipeline/default.yml @@ -2,74 +2,92 @@ description: Pipeline for normalizing Zeek known_certs.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: + tag: drop_5c081c54 description: Drop if no timestamp (invalid json) if: 'ctx.json?.ts == null' # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_fe6ef82e field: event.category value: - network - file - set: + tag: set_event_type_ec95f7f2 field: event.type value: - info - date: + tag: date_json_ts_e63f15b8 field: json.ts formats: - UNIX - ISO8601 - rename: + tag: rename_json_host_to_host_ip_ea8bb3f3 field: json.host target_field: host.ip ignore_missing: true - set: + tag: set_network_type_f877b344 field: network.type value: ipv4 if: ctx.host?.ip?.contains('.') == true - set: + tag: set_network_type_d9396dce field: network.type value: ipv6 if: ctx.host?.ip?.contains(':') == true - append: + tag: append_related_ip_54415bdd field: related.ip value: "{{{host.ip}}}" if: ctx.host?.ip != null allow_duplicates: false - geoip: + tag: geoip_host_ip_to_host_geo_d8a29594 field: host.ip target_field: host.geo ignore_missing: true - set: + tag: set_host_ip_1d72afcb field: host.ip value: ['{{{host.ip}}}'] if: ctx.host?.ip instanceof String - set: + tag: set_server_56c66a6c field: server copy_from: host ignore_empty_value: true - rename: + tag: rename_json_port_num_to_server_port_d5e2b762 field: json.port_num target_field: server.port ignore_missing: true - geoip: + tag: geoip_server_ip_to_server_as_ed2798db database_file: GeoLite2-ASN.mmdb field: server.ip target_field: server.as @@ -78,26 +96,32 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_server_as_asn_to_server_as_number_f46ba339 field: server.as.asn target_field: server.as.number ignore_missing: true - rename: + tag: rename_server_as_organization_name_to_server_as_organization_name_a7e512d7 field: server.as.organization_name target_field: server.as.organization.name ignore_missing: true - rename: + tag: rename_json_subject_to_tls_server_x509_subject_distinguished_name_557c8652 field: json.subject target_field: tls.server.x509.subject.distinguished_name ignore_missing: true - rename: + tag: rename_json_issuer_subject_to_tls_server_x509_issuer_distinguished_name_44a442f7 field: json.issuer_subject target_field: tls.server.x509.issuer.distinguished_name ignore_missing: true - rename: + tag: rename_json_serial_to_tls_server_x509_serial_number_40ac0cf8 field: json.serial target_field: tls.server.x509.serial_number ignore_missing: true - grok: + tag: grok_tls_server_x509_subject_distinguished_name_cfd51228 field: tls.server.x509.subject.distinguished_name ignore_missing: true patterns: @@ -106,10 +130,12 @@ processors: CN: '[^,]+' if: ctx.tls?.server?.x509?.subject?.distinguished_name != null && ctx.tls.server.x509.subject.distinguished_name.contains('CN=') - set: + tag: set_tls_server_x509_subject_common_name_f606deee field: tls.server.x509.subject.common_name value: ['{{{tls.server.x509.subject.common_name}}}'] if: ctx.tls?.server?.x509?.subject?.common_name instanceof String - grok: + tag: grok_tls_server_x509_issuer_distinguished_name_b9d4b532 field: tls.server.x509.issuer.distinguished_name ignore_missing: true patterns: @@ -118,18 +144,22 @@ processors: CN: '[^,]+' if: ctx.tls?.server?.x509?.issuer?.distinguished_name != null && ctx.tls.server.x509.issuer.distinguished_name.contains('CN=') - set: + tag: set_tls_server_x509_issuer_common_name_3372f307 field: tls.server.x509.issuer.common_name value: ['{{{tls.server.x509.issuer.common_name}}}'] if: ctx.tls?.server?.x509?.issuer?.common_name instanceof String - set: + tag: set_tls_server_issuer_b59c17a7 field: tls.server.issuer copy_from: tls.server.x509.issuer.distinguished_name ignore_empty_value: true - set: + tag: set_tls_server_subject_bf762dc5 field: tls.server.subject copy_from: tls.server.x509.subject.distinguished_name ignore_empty_value: true - remove: + tag: remove_e5a80934 field: - json ignore_missing: true @@ -139,4 +169,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/known_hosts/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/known_hosts/elasticsearch/ingest_pipeline/default.yml index 8d20c0fbe74..6f99b3b498c 100644 --- a/packages/zeek/data_stream/known_hosts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/known_hosts/elasticsearch/ingest_pipeline/default.yml @@ -2,66 +2,82 @@ description: Pipeline for normalizing Zeek known_hosts.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: + tag: drop_5c081c54 description: Drop if no timestamp (invalid json) if: 'ctx.json?.ts == null' # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_8f48329e field: event.category value: - network - host - set: + tag: set_event_type_ec95f7f2 field: event.type value: - info - date: + tag: date_json_ts_e63f15b8 field: json.ts formats: - UNIX - ISO8601 - rename: + tag: rename_json_host_to_host_ip_ea8bb3f3 field: json.host target_field: host.ip ignore_missing: true - set: + tag: set_network_type_f877b344 field: network.type value: ipv4 if: ctx.host?.ip?.contains('.') == true - set: + tag: set_network_type_d9396dce field: network.type value: ipv6 if: ctx.host?.ip?.contains(':') == true - append: + tag: append_related_ip_54415bdd field: related.ip value: "{{{host.ip}}}" if: ctx.host?.ip != null allow_duplicates: false - geoip: + tag: geoip_host_ip_to_host_geo_d8a29594 field: host.ip target_field: host.geo ignore_missing: true - set: + tag: set_host_ip_1d72afcb field: host.ip value: ['{{{host.ip}}}'] if: ctx.host?.ip instanceof String - remove: + tag: remove_e5a80934 field: - json ignore_missing: true @@ -71,4 +87,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/known_services/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/known_services/elasticsearch/ingest_pipeline/default.yml index 4b2932c81f0..ae4882ef382 100644 --- a/packages/zeek/data_stream/known_services/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/known_services/elasticsearch/ingest_pipeline/default.yml @@ -2,73 +2,91 @@ description: Pipeline for normalizing Zeek known_services.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: + tag: drop_5c081c54 description: Drop if no timestamp (invalid json) if: 'ctx.json?.ts == null' # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_dbab8a4e field: event.category value: - network - set: + tag: set_event_type_ec95f7f2 field: event.type value: - info - date: + tag: date_json_ts_e63f15b8 field: json.ts formats: - UNIX - ISO8601 - rename: + tag: rename_json_host_to_host_ip_ea8bb3f3 field: json.host target_field: host.ip ignore_missing: true - set: + tag: set_network_type_f877b344 field: network.type value: ipv4 if: ctx.host?.ip?.contains('.') == true - set: + tag: set_network_type_d9396dce field: network.type value: ipv6 if: ctx.host?.ip?.contains(':') == true - append: + tag: append_related_ip_54415bdd field: related.ip value: "{{{host.ip}}}" if: ctx.host?.ip != null allow_duplicates: false - geoip: + tag: geoip_host_ip_to_host_geo_d8a29594 field: host.ip target_field: host.geo ignore_missing: true - set: + tag: set_host_ip_1d72afcb field: host.ip value: ['{{{host.ip}}}'] if: ctx.host?.ip instanceof String - set: + tag: set_server_56c66a6c field: server copy_from: host ignore_empty_value: true - rename: + tag: rename_json_port_num_to_server_port_d5e2b762 field: json.port_num target_field: server.port ignore_missing: true - geoip: + tag: geoip_server_ip_to_server_as_ed2798db database_file: GeoLite2-ASN.mmdb field: server.ip target_field: server.as @@ -77,22 +95,27 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_server_as_asn_to_server_as_number_f46ba339 field: server.as.asn target_field: server.as.number ignore_missing: true - rename: + tag: rename_server_as_organization_name_to_server_as_organization_name_a7e512d7 field: server.as.organization_name target_field: server.as.organization.name ignore_missing: true - rename: + tag: rename_json_port_proto_to_network_transport_55b7cbab field: json.port_proto target_field: network.transport ignore_missing: true - rename: + tag: rename_json_service_to_network_application_d2d9b3cd field: json.service target_field: network.application ignore_missing: true - remove: + tag: remove_e5a80934 field: - json ignore_missing: true @@ -102,4 +125,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml index 3c73c0a4270..eb21411cf8a 100644 --- a/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml @@ -2,137 +2,173 @@ description: Pipeline for normalizing Zeek modbus.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_modbus_b99a6948 field: _temp_ target_field: zeek.modbus # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_82dd6b06 field: network.protocol value: modbus - dot_expander: + tag: dot_expander_id_orig_p_c8b0866a path: zeek.modbus field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_35789a12 path: zeek.modbus field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_1537fb5d path: zeek.modbus field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_20a7d4c5 path: zeek.modbus field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_modbus_id_orig_h_to_source_address_35f5221d field: zeek.modbus.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_modbus_id_orig_p_to_source_port_c7e20b9a field: zeek.modbus.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_modbus_id_resp_h_to_destination_address_31986a99 field: zeek.modbus.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_modbus_id_resp_p_to_destination_port_19c6f7b6 field: zeek.modbus.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_modbus_uid_to_zeek_session_id_dbad981a field: zeek.modbus.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - rename: + tag: rename_zeek_modbus_func_to_zeek_modbus_function_df78faf6 field: zeek.modbus.func target_field: zeek.modbus.function ignore_missing: true - set: + tag: set_event_action_5d33a8e7 field: event.action copy_from: zeek.modbus.function if: ctx.source?.address != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - set: + tag: set_event_outcome_80535223 field: event.outcome value: failure if: ctx.zeek?.modbus?.exception != null - set: + tag: set_event_outcome_d8aaefc7 field: event.outcome value: success if: ctx.event?.outcome == null - date: + tag: date_zeek_modbus_ts_f1632bd1 field: zeek.modbus.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_modbus_ts_3dc67a9b field: zeek.modbus.ts - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -141,6 +177,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -149,24 +186,30 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_70c9f504 field: - zeek.modbus.id ignore_missing: true @@ -176,4 +219,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml index a79b57ed75e..6f80865e574 100644 --- a/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml @@ -2,136 +2,172 @@ description: Pipeline for normalizing Zeek mysql.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_mysql_3ba21d40 field: _temp_ target_field: zeek.mysql # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_category_f4543241 field: event.category value: database - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_0d3e22c2 field: network.protocol value: mysql - dot_expander: + tag: dot_expander_id_orig_p_5e05401e path: zeek.mysql field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_3245cf26 path: zeek.mysql field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_79bdcdef path: zeek.mysql field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_dd283497 path: zeek.mysql field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_mysql_id_orig_h_to_source_address_8e5e5a7d field: zeek.mysql.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_mysql_id_orig_p_to_source_port_337cb4fa field: zeek.mysql.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_mysql_id_resp_h_to_destination_address_a05e5379 field: zeek.mysql.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_mysql_id_resp_p_to_destination_port_55085a16 field: zeek.mysql.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_mysql_uid_to_zeek_session_id_864b76fa field: zeek.mysql.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - set: + tag: set_event_action_c1f068a8 field: event.action copy_from: zeek.mysql.cmd if: ctx.zeek?.mysql?.cmd != null - set: + tag: set_event_outcome_54b8a853 field: event.outcome value: success if: ctx.zeek?.mysql?.success == true - set: + tag: set_event_outcome_737ee7cc field: event.outcome value: failure if: ctx.event?.outcome == null - date: + tag: date_zeek_mysql_ts_e1b6ae0d field: zeek.mysql.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_mysql_ts_66d5c5bf field: zeek.mysql.ts - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -140,6 +176,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -148,48 +185,60 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_event_type_14dbb200 field: event.type value: access if: "ctx.zeek?.mysql?.cmd != null && (ctx.zeek.mysql.cmd == 'connect' || ctx.zeek.mysql.cmd == 'connect_out')" - append: + tag: append_event_type_9fbc0ce1 field: event.type value: change if: "ctx.zeek?.mysql?.cmd != null && (ctx.zeek.mysql.cmd == 'init_db' || ctx.zeek.mysql.cmd == 'change_user' || ctx.zeek.mysql.cmd == 'set_option' || ctx.zeek.mysql.cmd == 'drop_db' || ctx.zeek.mysql.cmd == 'create_db' || ctx.zeek.mysql.cmd == 'process_kill' || ctx.zeek.mysql.cmd == 'delayed_insert')" - append: + tag: append_event_type_b2abae3f field: event.type value: info if: "ctx.zeek?.mysql?.cmd != null && ctx.zeek.mysql.cmd != 'init_db' && ctx.zeek.mysql.cmd != 'change_user' && ctx.zeek.mysql.cmd != 'set_option' && ctx.zeek.mysql.cmd != 'drop_db' && ctx.zeek.mysql.cmd != 'create_db' && ctx.zeek.mysql.cmd != 'process_kill' && ctx.zeek.mysql.cmd != 'delayed_insert' && ctx.zeek.mysql.cmd != 'connect' && ctx.zeek.mysql.cmd != 'connect_out'" - append: + tag: append_event_type_43289dea field: event.type value: start if: "ctx.zeek?.mysql?.cmd != null && ctx.zeek.mysql.cmd == 'connect'" - append: + tag: append_event_type_453386ca field: event.type value: end if: "ctx.zeek?.mysql?.cmd != null && ctx.zeek.mysql.cmd == 'connect_out'" - append: + tag: append_event_category_6b98c19e field: event.category value: session if: "ctx.zeek?.mysql?.cmd != null && (ctx.zeek.mysql.cmd == 'connect' || ctx.zeek.mysql.cmd == 'connect_out')" - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_aa875ffa field: - zeek.mysql.id ignore_missing: true @@ -199,4 +248,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml index 2f813494391..01304726a71 100644 --- a/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml @@ -2,209 +2,262 @@ description: Pipeline for normalizing Zeek notice.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_notice_ced9612a field: _temp_ target_field: zeek.notice # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_39295792 field: event.kind value: alert - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_8468ef55 field: event.category value: intrusion_detection - append: + tag: append_event_type_8a66ccaa field: event.type value: info - dot_expander: + tag: dot_expander_id_orig_p_771c30a4 path: zeek.notice field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_4d3533dc path: zeek.notice field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_73ce8e9b path: zeek.notice field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_79f2c713 path: zeek.notice field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_notice_id_orig_h_to_source_address_0f4b7007 field: zeek.notice.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_notice_id_orig_p_to_source_port_7bfaf134 field: zeek.notice.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_notice_id_resp_h_to_destination_address_f84523d3 field: zeek.notice.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_notice_id_resp_p_to_destination_port_ab9daf50 field: zeek.notice.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_notice_uid_to_zeek_session_id_e8178a08 field: zeek.notice.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - rename: + tag: rename_zeek_notice_src_to_source_address_b077831b field: zeek.notice.src target_field: source.address ignore_missing: true if: ctx.source?.address == null - remove: + tag: remove_zeek_notice_src_b00ceeb3 field: zeek.notice.src ignore_missing: true if: ctx.zeek?.notice?.src == ctx.source?.address - rename: + tag: rename_zeek_notice_dst_to_destination_address_a77477be field: zeek.notice.dst target_field: destination.address ignore_missing: true if: ctx.destination?.address == null - remove: + tag: remove_zeek_notice_dst_c31f6468 field: zeek.notice.dst ignore_missing: true if: ctx.zeek?.notice?.dst == ctx.destination?.address - rename: + tag: rename_zeek_notice_p_to_destination_port_9c2290f5 field: zeek.notice.p target_field: destination.port ignore_missing: true if: ctx.destination?.port == null - remove: + tag: remove_zeek_notice_p_98520dd9 field: zeek.notice.p ignore_missing: true if: ctx.zeek?.notice?.p == ctx.destination?.port - rename: + tag: rename_zeek_notice_conn_to_zeek_notice_connnection_id_46e587e0 field: zeek.notice.conn target_field: zeek.notice.connnection_id ignore_missing: true - rename: + tag: rename_zeek_notice_iconn_to_zeek_notice_icmp_id_dc084a98 field: zeek.notice.iconn target_field: zeek.notice.icmp_id ignore_missing: true - rename: + tag: rename_zeek_notice_proto_to_network_transport_2ba367f0 field: zeek.notice.proto target_field: network.transport ignore_missing: true - dot_expander: + tag: dot_expander_f_id_132a7d16 path: zeek.notice field: f.id ignore_failure: true - dot_expander: + tag: dot_expander_f_parent_id_50efad07 path: zeek.notice field: f.parent_id ignore_failure: true - dot_expander: + tag: dot_expander_f_source_0250d97e path: zeek.notice field: f.source ignore_failure: true - dot_expander: + tag: dot_expander_f_is_orig_0699696f path: zeek.notice field: f.is_orig ignore_failure: true - dot_expander: + tag: dot_expander_f_seen_bytes_e6d2b228 path: zeek.notice field: f.seen_bytes ignore_failure: true - dot_expander: + tag: dot_expander_f_total_bytes_deb811d5 path: zeek.notice field: f.total_bytes ignore_failure: true - rename: + tag: rename_zeek_notice_f_id_to_zeek_notice_file_id_c0d1ef86 field: zeek.notice.f.id target_field: zeek.notice.file.id ignore_missing: true - rename: + tag: rename_zeek_notice_f_parent_id_to_zeek_notice_file_parent_id_32687788 field: zeek.notice.f.parent_id target_field: zeek.notice.file.parent_id ignore_missing: true - rename: + tag: rename_zeek_notice_f_source_to_zeek_notice_file_source_06066806 field: zeek.notice.f.source target_field: zeek.notice.file.source ignore_missing: true - rename: + tag: rename_zeek_notice_f_is_orig_to_zeek_notice_file_is_orig_7da49f78 field: zeek.notice.f.is_orig target_field: zeek.notice.file.is_orig ignore_missing: true - rename: + tag: rename_zeek_notice_f_seen_bytes_to_zeek_notice_file_seen_bytes_e797ddc6 field: zeek.notice.f.seen_bytes target_field: zeek.notice.file.seen_bytes ignore_missing: true - rename: + tag: rename_zeek_notice_f_total_bytes_to_zeek_notice_file_total_bytes_255a3ac0 field: zeek.notice.f.total_bytes target_field: zeek.notice.file.total_bytes ignore_missing: true - rename: + tag: rename_zeek_notice_file_mime_type_to_zeek_notice_file_mime_type_ebaf5f83 field: zeek.notice.file_mime_type target_field: zeek.notice.file.mime_type ignore_missing: true - set: + tag: set_file_size_01bbf14a field: file.size copy_from: zeek.notice.file.total_bytes if: ctx.zeek?.notice?.file?.total_bytes != null - set: + tag: set_file_mime_type_2bdf2b1e field: file.mime_type copy_from: zeek.notice.file.mime_type if: ctx.zeek?.notice?.file?.mime_type != null - set: + tag: set_rule_name_fcdcbaff field: rule.name copy_from: zeek.notice.note if: ctx.zeek?.notice?.note != null - set: + tag: set_rule_description_d45883f2 field: rule.description copy_from: zeek.notice.msg if: ctx.zeek?.notice?.msg != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - date: + tag: date_zeek_notice_ts_bc595063 field: zeek.notice.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_notice_ts_be7c7e69 field: zeek.notice.ts - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -213,6 +266,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -221,42 +275,52 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - append: + tag: append_event_type_731bd0d2 field: event.type value: allowed if: "ctx.zeek?.notice?.dropped == false" - append: + tag: append_event_type_3c1900dc field: event.type value: denied if: "ctx.zeek?.notice?.dropped == true" - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_0150510b field: - zeek.notice.action - zeek.notice.remote_location @@ -268,4 +332,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml index 47de58031a2..5435b337347 100644 --- a/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml @@ -2,161 +2,203 @@ description: Pipeline for normalizing Zeek ntlm.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_ntlm_f1b30b79 field: _temp_ target_field: zeek.ntlm # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_category_109b4d72 field: event.category value: authentication - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_bac05bff field: network.protocol value: ntlm - dot_expander: + tag: dot_expander_id_orig_p_5aef7b63 path: zeek.ntlm field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_430b961b path: zeek.ntlm field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_c1239960 path: zeek.ntlm field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_c39649b8 path: zeek.ntlm field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_ntlm_id_orig_h_to_source_address_aa333fcc field: zeek.ntlm.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_ntlm_id_orig_p_to_source_port_32abec2d field: zeek.ntlm.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_ntlm_id_resp_h_to_destination_address_bb67e36e field: zeek.ntlm.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_ntlm_id_resp_p_to_destination_port_af5a37d7 field: zeek.ntlm.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_ntlm_uid_to_zeek_session_id_3574430d field: zeek.ntlm.uid target_field: zeek.session_id ignore_missing: true - rename: + tag: rename_zeek_ntlm_domainname_to_zeek_ntlm_domain_9cb88dad field: zeek.ntlm.domainname target_field: zeek.ntlm.domain ignore_missing: true - rename: + tag: rename_zeek_ntlm_server_dns_computer_name_to_zeek_ntlm_server_name_dns_d434172e field: zeek.ntlm.server_dns_computer_name target_field: zeek.ntlm.server.name.dns ignore_missing: true - rename: + tag: rename_zeek_ntlm_server_nb_computer_name_to_zeek_ntlm_server_name_netbios_5cec406a field: zeek.ntlm.server_nb_computer_name target_field: zeek.ntlm.server.name.netbios ignore_missing: true - rename: + tag: rename_zeek_ntlm_server_tree_name_to_zeek_ntlm_server_name_tree_d7892468 field: zeek.ntlm.server_tree_name target_field: zeek.ntlm.server.name.tree ignore_missing: true - set: + tag: set_user_name_9476b790 field: user.name copy_from: zeek.ntlm.username if: ctx.zeek?.ntlm?.username != null - set: + tag: set_user_domain_a1d4f40d field: user.domain copy_from: zeek.ntlm.domain if: ctx.zeek?.ntlm?.domain != null - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - set: + tag: set_event_outcome_3cc97eae field: event.outcome value: success if: ctx.zeek?.ntlm?.success == true - set: + tag: set_event_outcome_c6bf3f00 field: event.outcome value: failure if: ctx.zeek?.ntlm?.success == false - date: + tag: date_zeek_ntlm_ts_1f5818d8 field: zeek.ntlm.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_ntlm_ts_6a966ff8 field: zeek.ntlm.ts - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - append: + tag: append_related_user_837e080f field: related.user value: "{{{user.name}}}" if: "ctx.user?.name != null" allow_duplicates: false - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -165,6 +207,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -173,24 +216,30 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_9d474128 field: - message - json @@ -202,4 +251,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/default.yml index 7edb94a734a..239182a00a9 100644 --- a/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/default.yml @@ -2,37 +2,47 @@ description: Pipeline for normalizing Zeek conn.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_ntp_e67478c9 field: _temp_ target_field: zeek.ntp ignore_failure: true # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - append: + tag: append_event_type_c741503d field: event.type value: - connection @@ -40,118 +50,145 @@ processors: - info allow_duplicates: false - dot_expander: + tag: dot_expander_id_orig_p_42d83246 path: zeek.ntp field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_12df582e path: zeek.ntp field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_92112f4f path: zeek.ntp field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_f5faf217 path: zeek.ntp field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_ntp_id_orig_h_to_source_address_4fce83a5 field: zeek.ntp.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_ntp_id_orig_p_to_source_port_4c3a7a72 field: zeek.ntp.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_ntp_id_resp_h_to_destination_address_fa105611 field: zeek.ntp.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_ntp_id_resp_p_to_destination_port_6bf11cde field: zeek.ntp.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_ntp_uid_to_zeek_session_id_8b247422 field: zeek.ntp.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - set: + tag: set_network_transport_a6cfd568 field: network.transport value: udp - set: + tag: set_network_protocol_dfb6e17a field: network.protocol value: ntp - set: + tag: set_network_type_279a1289 field: network.type value: ipv4 if: ctx.source?.ip?.contains('.') == true - set: + tag: set_network_type_a3d39523 field: network.type value: ipv6 if: ctx.source?.ip?.contains(':') == true - community_id: + tag: community_id_0d5b55bd ignore_missing: true - date: + tag: date_zeek_ntp_ts_d4f3f512 field: zeek.ntp.ts formats: - UNIX - ISO8601 if: ctx.zeek?.ntp?.ts != null - date: + tag: date_zeek_ntp_ref_time_to_zeek_ntp_ref_time_c8406f7c field: zeek.ntp.ref_time target_field: zeek.ntp.ref_time formats: - UNIX if: ctx.zeek?.ntp?.ref_time != null - date: + tag: date_zeek_ntp_org_time_to_zeek_ntp_org_time_0578965b field: zeek.ntp.org_time target_field: zeek.ntp.org_time formats: - UNIX if: ctx.zeek?.ntp?.org_time != null - date: + tag: date_zeek_ntp_rec_time_to_zeek_ntp_rec_time_e910a993 field: zeek.ntp.rec_time target_field: zeek.ntp.rec_time formats: - UNIX if: ctx.zeek?.ntp?.rec_time != null - date: + tag: date_zeek_ntp_xmt_time_to_zeek_ntp_xmt_time_d02ba450 field: zeek.ntp.xmt_time target_field: zeek.ntp.xmt_time formats: - UNIX if: ctx.zeek?.ntp?.xmt_time != null - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: ctx.source?.ip != null allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: ctx.destination?.ip != null allow_duplicates: false - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -160,6 +197,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -168,22 +206,27 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - remove: + tag: remove_a86cf076 field: - zeek.ntp.id - zeek.ntp.ts @@ -194,4 +237,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/ntp/manifest.yml b/packages/zeek/data_stream/ntp/manifest.yml index 3b34b7b3a3a..44077575bd2 100644 --- a/packages/zeek/data_stream/ntp/manifest.yml +++ b/packages/zeek/data_stream/ntp/manifest.yml @@ -39,3 +39,4 @@ streams: show_user: false description: > Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details. + diff --git a/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml index a8d0dbd4886..4fc8ad8da34 100644 --- a/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml @@ -2,83 +2,105 @@ description: Pipeline for normalizing Zeek ocsp.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_ocsp_8ed330a5 field: _temp_ target_field: zeek.ocsp # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - rename: + tag: rename_zeek_ocsp_id_to_zeek_ocsp_file_id_2b5f73eb field: zeek.ocsp.id target_field: zeek.ocsp.file_id ignore_missing: true - rename: + tag: rename_zeek_ocsp_hashAlgorithm_to_zeek_ocsp_hash_algorithm_7d8be280 field: zeek.ocsp.hashAlgorithm target_field: zeek.ocsp.hash.algorithm ignore_missing: true - rename: + tag: rename_zeek_ocsp_issuerNameHash_to_zeek_ocsp_hash_issuer_name_b0b72b0a field: zeek.ocsp.issuerNameHash target_field: zeek.ocsp.hash.issuer.name ignore_missing: true - rename: + tag: rename_zeek_ocsp_issuerKeyHash_to_zeek_ocsp_hash_issuer_key_69333f14 field: zeek.ocsp.issuerKeyHash target_field: zeek.ocsp.hash.issuer.key ignore_missing: true - rename: + tag: rename_zeek_ocsp_serialNumber_to_zeek_ocsp_serial_number_8d256231 field: zeek.ocsp.serialNumber target_field: zeek.ocsp.serial_number ignore_missing: true - rename: + tag: rename_zeek_ocsp_certStatus_to_zeek_ocsp_status_f80bfb66 field: zeek.ocsp.certStatus target_field: zeek.ocsp.status ignore_missing: true - rename: + tag: rename_zeek_ocsp_revoketime_to_zeek_ocsp_revoke_date_68ef90bb field: zeek.ocsp.revoketime target_field: zeek.ocsp.revoke.date ignore_missing: true - rename: + tag: rename_zeek_ocsp_revokereason_to_zeek_ocsp_revoke_reason_616744fe field: zeek.ocsp.revokereason target_field: zeek.ocsp.revoke.reason ignore_missing: true - rename: + tag: rename_zeek_ocsp_thisUpdate_to_zeek_ocsp_update_this_ca5751ce field: zeek.ocsp.thisUpdate target_field: zeek.ocsp.update.this ignore_missing: true - rename: + tag: rename_zeek_ocsp_nextUpdate_to_zeek_ocsp_update_next_d970be34 field: zeek.ocsp.nextUpdate target_field: zeek.ocsp.update.next ignore_missing: true - date: + tag: date_zeek_ocsp_ts_5008d814 field: zeek.ocsp.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_ocsp_ts_706fc3bc field: zeek.ocsp.ts - date: + tag: date_zeek_ocsp_revoke_date_to_zeek_ocsp_revoke_date_8461a543 field: zeek.ocsp.revoke.date target_field: zeek.ocsp.revoke.date formats: @@ -86,6 +108,7 @@ processors: - ISO8601 if: ctx.zeek.ocsp.revoke?.date != null - date: + tag: date_zeek_ocsp_update_this_to_zeek_ocsp_update_this_41b3dbec field: zeek.ocsp.update.this target_field: zeek.ocsp.update.this formats: @@ -93,6 +116,7 @@ processors: - ISO8601 if: ctx.zeek.ocsp.update?.this != null - date: + tag: date_zeek_ocsp_update_next_to_zeek_ocsp_update_next_06288895 field: zeek.ocsp.update.next target_field: zeek.ocsp.update.next formats: @@ -100,11 +124,13 @@ processors: - ISO8601 if: ctx.zeek.ocsp.update?.next != null - append: + tag: append_related_hash_f67072c7 field: related.hash value: "{{{zeek.ocsp.issuerNameHash}}}" if: "ctx.zeek?.ocsp?.issuerNameHash != null" allow_duplicates: false - append: + tag: append_related_hash_066a4fc1 field: related.hash value: "{{{zeek.ocsp.issuerKeyHash}}}" if: "ctx.zeek?.ocsp?.issuerKeyHash != null" @@ -115,4 +141,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml index 218312646ba..a529a585828 100644 --- a/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml @@ -2,50 +2,64 @@ description: Pipeline for normalizing Zeek pe.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_pe_d228fd8f field: _temp_ target_field: zeek.pe # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_423f7188 field: event.category value: file - append: + tag: append_event_type_8a66ccaa field: event.type value: info - rename: + tag: rename_zeek_pe_compile_ts_to_zeek_pe_compile_time_7bb142ac field: zeek.pe.compile_ts target_field: zeek.pe.compile_time ignore_missing: true - date: + tag: date_zeek_pe_ts_48a775b2 field: zeek.pe.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_pe_ts_26d6873e field: zeek.pe.ts - date: + tag: date_zeek_pe_compile_time_to_zeek_pe_compile_time_1197e017 field: zeek.pe.compile_time target_field: zeek.pe.compile_time formats: @@ -58,4 +72,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml index fdd62c3149b..25cf077ed5e 100644 --- a/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml @@ -2,132 +2,167 @@ description: Pipeline for normalizing Zeek radius.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_radius_1f5ceae0 field: _temp_ target_field: zeek.radius # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_category_109b4d72 field: event.category value: authentication - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_network_transport_a6cfd568 field: network.transport value: udp - set: + tag: set_network_protocol_0602cf22 field: network.protocol value: radius - dot_expander: + tag: dot_expander_id_orig_p_53623bd6 path: zeek.radius field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_38dcea4e path: zeek.radius field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_9498806d path: zeek.radius field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_bff5c925 path: zeek.radius field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_radius_id_orig_h_to_source_address_aca003e5 field: zeek.radius.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_radius_id_orig_p_to_source_port_91d003b2 field: zeek.radius.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_radius_id_resp_h_to_destination_address_9fb86d51 field: zeek.radius.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_radius_id_resp_p_to_destination_port_7fc1361e field: zeek.radius.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_radius_uid_to_zeek_session_id_f1061ee2 field: zeek.radius.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - set: + tag: set_user_name_9bdc2a4a field: user.name copy_from: zeek.radius.username if: ctx.zeek?.radius?.username != null - set: + tag: set_event_outcome_48bd97bc field: event.outcome copy_from: zeek.radius.result if: ctx.zeek?.radius?.result != null - date: + tag: date_zeek_radius_ts_74cdf9d5 field: zeek.radius.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_radius_ts_b87d5bcf field: zeek.radius.ts - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -136,6 +171,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -144,29 +180,36 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_user_837e080f field: related.user value: "{{{user.name}}}" if: "ctx.user?.name != null" allow_duplicates: false - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_aecb2340 field: - zeek.radius.id ignore_missing: true @@ -176,4 +219,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml index 7f979d125ec..af89706b7fe 100644 --- a/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml @@ -2,160 +2,202 @@ description: Pipeline for normalizing Zeek rdp.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_rdp_804aa82c field: _temp_ target_field: zeek.rdp # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_93d80746 field: network.protocol value: rdp - dot_expander: + tag: dot_expander_id_orig_p_ee5f03fa path: zeek.rdp field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_91ff4112 path: zeek.rdp field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_88c1e803 path: zeek.rdp field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_7690dcfb path: zeek.rdp field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_rdp_id_orig_h_to_source_address_4e0a58c9 field: zeek.rdp.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_rdp_id_orig_p_to_source_port_ca761d46 field: zeek.rdp.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_rdp_id_resp_h_to_destination_address_bb5ef065 field: zeek.rdp.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_rdp_id_resp_p_to_destination_port_f9d3b3c2 field: zeek.rdp.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_rdp_uid_to_zeek_session_id_22ac539e field: zeek.rdp.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_rdp_client_build_to_zeek_rdp_client_build_122d7037 field: zeek.rdp.client_build target_field: zeek.rdp.client.build ignore_missing: true - rename: + tag: rename_zeek_rdp_client_name_to_zeek_rdp_client_name_0e27923d field: zeek.rdp.client_name target_field: zeek.rdp.client.name ignore_missing: true - rename: + tag: rename_zeek_rdp_client_dig_product_id_to_zeek_rdp_client_product_id_f46d8116 field: zeek.rdp.client_dig_product_id target_field: zeek.rdp.client.product_id ignore_missing: true - rename: + tag: rename_zeek_rdp_desktop_width_to_zeek_rdp_desktop_width_efbe00ab field: zeek.rdp.desktop_width target_field: zeek.rdp.desktop.width ignore_missing: true - rename: + tag: rename_zeek_rdp_desktop_height_to_zeek_rdp_desktop_height_58d10ef1 field: zeek.rdp.desktop_height target_field: zeek.rdp.desktop.height ignore_missing: true - rename: + tag: rename_zeek_rdp_requested_color_depth_to_zeek_rdp_desktop_color_depth_98a32143 field: zeek.rdp.requested_color_depth target_field: zeek.rdp.desktop.color_depth ignore_missing: true - rename: + tag: rename_zeek_rdp_cert_type_to_zeek_rdp_cert_type_65dd0939 field: zeek.rdp.cert_type target_field: zeek.rdp.cert.type ignore_missing: true - rename: + tag: rename_zeek_rdp_cert_count_to_zeek_rdp_cert_count_2ff2926f field: zeek.rdp.cert_count target_field: zeek.rdp.cert.count ignore_missing: true - rename: + tag: rename_zeek_rdp_cert_permanent_to_zeek_rdp_cert_permanent_6f086199 field: zeek.rdp.cert_permanent target_field: zeek.rdp.cert.permanent ignore_missing: true - rename: + tag: rename_zeek_rdp_encryption_level_to_zeek_rdp_encryption_level_84b90f87 field: zeek.rdp.encryption_level target_field: zeek.rdp.encryption.level ignore_missing: true - rename: + tag: rename_zeek_rdp_encryption_method_to_zeek_rdp_encryption_method_df94a4ed field: zeek.rdp.encryption_method target_field: zeek.rdp.encryption.method ignore_missing: true - date: + tag: date_zeek_rdp_ts_f9bf5871 field: zeek.rdp.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_rdp_ts_51f336cb field: zeek.rdp.ts - convert: + tag: convert_zeek_rdp_ssl_to_tls_established_63435a23 field: zeek.rdp.ssl target_field: tls.established type: boolean ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -164,6 +206,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -172,34 +215,42 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_923e5396 field: - zeek.rdp.id ignore_missing: true @@ -209,4 +260,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml index 035d958716c..2393e1fd824 100644 --- a/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml @@ -2,145 +2,183 @@ description: Pipeline for normalizing Zeek rfb.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_rfb_27a8d640 field: _temp_ target_field: zeek.rfb # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_ec79d932 field: network.protocol value: rfb - dot_expander: + tag: dot_expander_id_orig_p_4700d5e6 path: zeek.rfb field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_de27049e path: zeek.rfb field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_181d9c5b path: zeek.rfb field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_322ba253 path: zeek.rfb field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_rfb_id_orig_h_to_source_address_21477f35 field: zeek.rfb.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_rfb_id_orig_p_to_source_port_9565d2c2 field: zeek.rfb.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_rfb_id_resp_h_to_destination_address_fd08aac1 field: zeek.rfb.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_rfb_id_resp_p_to_destination_port_c41f550e field: zeek.rfb.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_rfb_uid_to_zeek_session_id_bbc1bb52 field: zeek.rfb.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_rfb_client_major_version_to_zeek_rfb_version_client_major_a2ad2020 field: zeek.rfb.client_major_version target_field: zeek.rfb.version.client.major ignore_missing: true - rename: + tag: rename_zeek_rfb_client_minor_version_to_zeek_rfb_version_client_minor_be828228 field: zeek.rfb.client_minor_version target_field: zeek.rfb.version.client.minor ignore_missing: true - rename: + tag: rename_zeek_rfb_server_major_version_to_zeek_rfb_version_server_major_c78fa248 field: zeek.rfb.server_major_version target_field: zeek.rfb.version.server.major ignore_missing: true - rename: + tag: rename_zeek_rfb_server_minor_version_to_zeek_rfb_version_server_minor_94078700 field: zeek.rfb.server_minor_version target_field: zeek.rfb.version.server.minor ignore_missing: true - rename: + tag: rename_zeek_rfb_auth_to_zeek_rfb_auth_success_cd235687 field: zeek.rfb.auth target_field: zeek.rfb.auth.success ignore_missing: true - rename: + tag: rename_zeek_rfb_authentication_method_to_zeek_rfb_auth_method_7935dc3f field: zeek.rfb.authentication_method target_field: zeek.rfb.auth.method ignore_missing: true - date: + tag: date_zeek_rfb_ts_03f18825 field: zeek.rfb.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_rfb_ts_c684d7df field: zeek.rfb.ts - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -149,6 +187,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -157,24 +196,30 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_655cf52e field: - zeek.rfb.id ignore_missing: true @@ -184,4 +229,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/default.yml index 2c9ab128fa1..33e5647ca11 100644 --- a/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/default.yml @@ -2,108 +2,135 @@ description: Pipeline for normalizing Zeek conn.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_signature_8d1f93d5 field: _temp_ target_field: zeek.signature ignore_failure: true # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - set: + tag: set_event_kind_39295792 field: event.kind value: alert - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - rename: + tag: rename_zeek_signature_src_addr_to_source_address_e3f36094 field: zeek.signature.src_addr target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_signature_src_port_to_source_port_001cc181 field: zeek.signature.src_port target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_signature_dst_addr_to_destination_address_37f0744a field: zeek.signature.dst_addr target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_signature_dst_port_to_destination_port_48a5579f field: zeek.signature.dst_port target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_signature_uid_to_zeek_session_id_476d35ca field: zeek.signature.uid target_field: zeek.session_id ignore_missing: true - rename: + tag: rename_zeek_signature_sig_id_to_rule_id_a952149f field: zeek.signature.sig_id target_field: rule.id ignore_missing: true - rename: + tag: rename_zeek_signature_event_msg_to_rule_description_eb78b435 field: zeek.signature.event_msg target_field: rule.description ignore_missing: true - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - date: + tag: date_zeek_signature_ts_7411cc39 field: zeek.signature.ts formats: - UNIX - ISO8601 - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_network_type_279a1289 field: network.type value: ipv4 if: ctx.source?.ip?.contains('.') == true - set: + tag: set_network_type_a3d39523 field: network.type value: ipv6 if: ctx.source?.ip?.contains(':') == true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: ctx.source?.ip != null allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: ctx.destination?.ip != null allow_duplicates: false - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -112,6 +139,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -120,22 +148,27 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - remove: + tag: remove_dcc5048a field: - zeek.signature.ts ignore_missing: true @@ -145,4 +178,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/signature/manifest.yml b/packages/zeek/data_stream/signature/manifest.yml index 83d77336ca1..31f83d25a02 100644 --- a/packages/zeek/data_stream/signature/manifest.yml +++ b/packages/zeek/data_stream/signature/manifest.yml @@ -39,3 +39,4 @@ streams: show_user: false description: > Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details. + diff --git a/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml index 2e152359378..bb1da7429b5 100644 --- a/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml @@ -2,175 +2,221 @@ description: Pipeline for normalizing Zeek sip.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_sip_cfae06fa field: _temp_ target_field: zeek.sip # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - set: + tag: set_network_transport_a6cfd568 field: network.transport value: udp - set: + tag: set_network_protocol_cb49071c field: network.protocol value: sip - dot_expander: + tag: dot_expander_id_orig_p_2388e638 path: zeek.sip field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_5b08c540 path: zeek.sip field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_59f033fd path: zeek.sip field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_20f2f305 path: zeek.sip field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_sip_id_orig_h_to_source_address_51ee29cf field: zeek.sip.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_sip_id_orig_p_to_source_port_6d77b6ac field: zeek.sip.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_sip_id_resp_h_to_destination_address_e89fbd6b field: zeek.sip.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_sip_id_resp_p_to_destination_port_52476558 field: zeek.sip.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_sip_uid_to_zeek_session_id_7ec4c510 field: zeek.sip.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_fa496343 field: event.id copy_from: zeek.session_id if: ctx.zeek.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_sip_trans_depth_to_zeek_sip_transaction_depth_dba79866 field: zeek.sip.trans_depth target_field: zeek.sip.transaction_depth ignore_missing: true - rename: + tag: rename_zeek_sip_method_to_zeek_sip_sequence_method_8281af9b field: zeek.sip.method target_field: zeek.sip.sequence.method ignore_missing: true - rename: + tag: rename_zeek_sip_request_from_to_zeek_sip_request_from_5816705d field: zeek.sip.request_from target_field: zeek.sip.request.from ignore_missing: true - rename: + tag: rename_zeek_sip_request_to_to_zeek_sip_request_to_d6cd43cf field: zeek.sip.request_to target_field: zeek.sip.request.to ignore_missing: true - rename: + tag: rename_zeek_sip_request_path_to_zeek_sip_request_path_f06fc6ab field: zeek.sip.request_path target_field: zeek.sip.request.path ignore_missing: true - rename: + tag: rename_zeek_sip_request_body_len_to_zeek_sip_request_body_length_5751e0f4 field: zeek.sip.request_body_len target_field: zeek.sip.request.body_length ignore_missing: true - rename: + tag: rename_zeek_sip_response_from_to_zeek_sip_response_from_ae4cb861 field: zeek.sip.response_from target_field: zeek.sip.response.from ignore_missing: true - rename: + tag: rename_zeek_sip_response_to_to_zeek_sip_response_to_3f9297ed field: zeek.sip.response_to target_field: zeek.sip.response.to ignore_missing: true - rename: + tag: rename_zeek_sip_response_path_to_zeek_sip_response_path_67fe3871 field: zeek.sip.response_path target_field: zeek.sip.response.path ignore_missing: true - rename: + tag: rename_zeek_sip_response_body_len_to_zeek_sip_response_body_length_662e0154 field: zeek.sip.response_body_len target_field: zeek.sip.response.body_length ignore_missing: true - rename: + tag: rename_zeek_sip_status_code_to_zeek_sip_status_code_07a86089 field: zeek.sip.status_code target_field: zeek.sip.status.code ignore_missing: true - rename: + tag: rename_zeek_sip_status_msg_to_zeek_sip_status_msg_eb7517c7 field: zeek.sip.status_msg target_field: zeek.sip.status.msg ignore_missing: true - set: + tag: set_event_action_ad385fd1 field: event.action copy_from: zeek.sip.sequence.method if: ctx.zeek?.sip?.sequence?.method != null - set: + tag: set_url_full_c1f35404 field: url.full copy_from: zeek.sip.uri if: ctx.zeek?.sip?.uri != null - date: + tag: date_zeek_sip_ts_25937cbf field: zeek.sip.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_sip_ts_b42570ad field: zeek.sip.ts - grok: + tag: grok_zeek_sip_seq_539bf75f field: zeek.sip.seq patterns: - "%{NUMBER:zeek.sip.sequence.number}" ignore_missing: true - remove: + tag: remove_zeek_sip_seq_97234fb8 field: zeek.sip.seq ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -179,6 +225,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -187,42 +234,52 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - set: + tag: set_event_outcome_3eec7c87 field: event.outcome value: failure if: "ctx.zeek?.sip?.status?.code != null && ctx.zeek.sip.status.code >= 400" - set: + tag: set_event_outcome_19c1dc35 field: event.outcome value: success if: "ctx.zeek?.sip?.status?.code != null && ctx.zeek.sip.status.code < 400" - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_f00413d0 field: - zeek.sip.id ignore_missing: true @@ -232,4 +289,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml index fb0e440ca74..5053931b401 100644 --- a/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml @@ -2,104 +2,132 @@ description: Pipeline for normalizing Zeek smb_cmd.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_smb_cmd_9efc3f11 field: _temp_ target_field: zeek.smb_cmd # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_5063935e field: network.protocol value: smb - dot_expander: + tag: dot_expander_referenced_file_ts_bf65e33d path: zeek.smb_cmd field: referenced_file.ts ignore_failure: true - dot_expander: + tag: dot_expander_referenced_file_id_orig_p_998556af path: zeek.smb_cmd field: referenced_file.id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_referenced_file_id_resp_p_52647186 path: zeek.smb_cmd field: referenced_file.id.resp_p ignore_failure: true - dot_expander: + tag: dot_expander_referenced_file_size_dce41459 path: zeek.smb_cmd field: referenced_file.size ignore_failure: true - dot_expander: + tag: dot_expander_referenced_file_times_modified_6a9169e3 path: zeek.smb_cmd field: referenced_file.times.modified ignore_failure: true - dot_expander: + tag: dot_expander_referenced_file_times_accessed_b4328807 path: zeek.smb_cmd field: referenced_file.times.accessed ignore_failure: true - dot_expander: + tag: dot_expander_referenced_file_times_created_5b18cc5e path: zeek.smb_cmd field: referenced_file.times.created ignore_failure: true - dot_expander: + tag: dot_expander_referenced_file_times_changed_1be8d47a path: zeek.smb_cmd field: referenced_file.times.changed ignore_failure: true - dot_expander: + tag: dot_expander_referenced_file_uid_553c18ec path: zeek.smb_cmd field: referenced_file.uid ignore_failure: true - dot_expander: + tag: dot_expander_referenced_file_id_orig_h_57655457 path: zeek.smb_cmd field: referenced_file.id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_referenced_file_id_resp_h_b41be3ae path: zeek.smb_cmd field: referenced_file.id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_referenced_file_action_aeba3234 path: zeek.smb_cmd field: referenced_file.action ignore_failure: true - dot_expander: + tag: dot_expander_referenced_file_name_d4e8c757 path: zeek.smb_cmd field: referenced_file.name ignore_failure: true - dot_expander: + tag: dot_expander_referenced_file_path_bf047bc3 path: zeek.smb_cmd field: referenced_file.path ignore_failure: true - remove: + tag: remove_aba57fb3 field: - zeek.smb_cmd.referenced_file.ts - zeek.smb_cmd.referenced_file.id.orig_p @@ -111,6 +139,7 @@ processors: - zeek.smb_cmd.referenced_file.times.changed ignore_missing: true - remove: + tag: remove_dcd3f021 field: - zeek.smb_cmd.referenced_file.uid - zeek.smb_cmd.referenced_file.id.orig_h @@ -118,104 +147,130 @@ processors: ignore_missing: true if: ctx.zeek?.smb_cmd?.referenced_file?.action == null - dot_expander: + tag: dot_expander_id_orig_p_55700ccb path: zeek.smb_cmd field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_9469c793 path: zeek.smb_cmd field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_aad63452 path: zeek.smb_cmd field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_12f03d7a path: zeek.smb_cmd field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_smb_cmd_id_orig_h_to_source_address_7414d02c field: zeek.smb_cmd.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_smb_cmd_id_orig_p_to_source_port_738a768d field: zeek.smb_cmd.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_smb_cmd_id_resp_h_to_destination_address_d9a2064e field: zeek.smb_cmd.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_smb_cmd_id_resp_p_to_destination_port_1b5d32b7 field: zeek.smb_cmd.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_smb_cmd_uid_to_zeek_session_id_1d7e04ed field: zeek.smb_cmd.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_smb_cmd_referenced_file_uid_to_zeek_smb_cmd_file_uid_cf867e58 field: zeek.smb_cmd.referenced_file.uid target_field: zeek.smb_cmd.file.uid ignore_missing: true - rename: + tag: rename_zeek_smb_cmd_referenced_file_id_orig_h_to_zeek_smb_cmd_file_host_tx_a237e1d3 field: zeek.smb_cmd.referenced_file.id.orig_h target_field: zeek.smb_cmd.file.host.tx ignore_missing: true - rename: + tag: rename_zeek_smb_cmd_referenced_file_id_resp_h_to_zeek_smb_cmd_file_host_rx_7f9f706e field: zeek.smb_cmd.referenced_file.id.resp_h target_field: zeek.smb_cmd.file.host.rx ignore_missing: true - rename: + tag: rename_zeek_smb_cmd_referenced_file_name_to_zeek_smb_cmd_file_name_c028bbc6 field: zeek.smb_cmd.referenced_file.name target_field: zeek.smb_cmd.file.name ignore_missing: true - rename: + tag: rename_zeek_smb_cmd_referenced_file_path_to_zeek_smb_cmd_file_path_fe44b302 field: zeek.smb_cmd.referenced_file.path target_field: zeek.smb_cmd.file.path ignore_missing: true - rename: + tag: rename_zeek_smb_cmd_referenced_file_action_to_zeek_smb_cmd_file_action_e5f9c756 field: zeek.smb_cmd.referenced_file.action target_field: zeek.smb_cmd.file.action ignore_missing: true - set: + tag: set_event_action_b4ba9f7e field: event.action copy_from: zeek.smb_cmd.command if: ctx.zeek?.smb_cmd?.command != null - set: + tag: set_user_name_981d2182 field: user.name copy_from: zeek.smb_cmd.username if: ctx.zeek?.smb_cmd?.username != null - date: + tag: date_zeek_smb_cmd_ts_5b739710 field: zeek.smb_cmd.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_smb_cmd_ts_5185fb98 field: zeek.smb_cmd.ts - remove: + tag: remove_zeek_smb_cmd_referenced_file_72c51fc4 field: zeek.smb_cmd.referenced_file ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -224,6 +279,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -232,51 +288,63 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - append: + tag: append_related_user_837e080f field: related.user value: "{{{user.name}}}" if: "ctx.user?.name != null" allow_duplicates: false - append: + tag: append_event_type_f482b432 field: event.type value: error if: "ctx.zeek?.smb_cmd?.status != null && ctx.zeek.smb_cmd.status.equalsIgnoreCase('success') == false" - set: + tag: set_event_outcome_8d82926e field: event.outcome value: success if: "ctx.zeek?.smb_cmd?.status != null && ctx.zeek.smb_cmd.status.equalsIgnoreCase('success') == true" - set: + tag: set_event_outcome_d7a9fcc0 field: event.outcome value: failure if: "ctx.zeek?.smb_cmd?.status != null && ctx.zeek.smb_cmd.status.equalsIgnoreCase('success') == false" - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_b769ea73 field: - zeek.smb_cmd.id ignore_missing: true @@ -286,4 +354,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml index 874fccf4f18..2055a552cc1 100644 --- a/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml @@ -2,130 +2,166 @@ description: Pipeline for normalizing Zeek smb_files.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_smb_files_625e4070 field: _temp_ target_field: zeek.smb_files # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_category_423f7188 field: event.category value: file - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_5063935e field: network.protocol value: smb - dot_expander: + tag: dot_expander_id_orig_p_d01db476 path: zeek.smb_files field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_9b28fc0e path: zeek.smb_files field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_d9b70173 path: zeek.smb_files field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_7dc7662b path: zeek.smb_files field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_smb_files_id_orig_h_to_source_address_57f3b985 field: zeek.smb_files.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_smb_files_id_orig_p_to_source_port_21e44cd2 field: zeek.smb_files.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_smb_files_id_resp_h_to_destination_address_600a5cf1 field: zeek.smb_files.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_smb_files_id_resp_p_to_destination_port_58c890be field: zeek.smb_files.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_smb_files_uid_to_zeek_session_id_ef624582 field: zeek.smb_files.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - set: + tag: set_event_action_25b08d2e field: event.action copy_from: zeek.smb_files.action if: ctx.zeek?.smb_files?.action != null - set: + tag: set_file_name_1b79738b field: file.name copy_from: zeek.smb_files.name if: ctx.zeek?.smb_files?.name != null - set: + tag: set_file_size_8862dd41 field: file.size copy_from: zeek.smb_files.size if: ctx.zeek?.smb_files?.size != null - date: + tag: date_zeek_smb_files_ts_c1854bb5 field: zeek.smb_files.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_smb_files_ts_12dcbd1f field: zeek.smb_files.ts - dot_expander: + tag: dot_expander_times_accessed_cc3f3a67 field: times.accessed path: zeek.smb_files - dot_expander: + tag: dot_expander_times_changed_21aabd9a field: times.changed path: zeek.smb_files - dot_expander: + tag: dot_expander_times_created_12ca106e field: times.created path: zeek.smb_files - dot_expander: + tag: dot_expander_times_modified_16fcc7bb field: times.modified path: zeek.smb_files - date: + tag: date_zeek_smb_files_times_accessed_to_zeek_smb_files_times_accessed_b64d7419 field: zeek.smb_files.times.accessed target_field: zeek.smb_files.times.accessed formats: @@ -133,10 +169,12 @@ processors: - ISO8601 if: ctx.zeek.smb_files.times?.accessed != null - set: + tag: set_file_accessed_fc7494d3 field: file.accessed value: "{{{zeek.smb_files.times.accessed}}}" if: "ctx.zeek?.smb_files?.times?.accessed != null" - date: + tag: date_zeek_smb_files_times_changed_to_zeek_smb_files_times_changed_58cb884b field: zeek.smb_files.times.changed target_field: zeek.smb_files.times.changed formats: @@ -144,10 +182,12 @@ processors: - ISO8601 if: ctx.zeek.smb_files.times?.accessed != null - set: + tag: set_file_ctime_fab12906 field: file.ctime value: "{{{zeek.smb_files.times.changed}}}" if: "ctx.zeek?.smb_files?.times?.changed != null" - date: + tag: date_zeek_smb_files_times_created_to_zeek_smb_files_times_created_8afef51b field: zeek.smb_files.times.created target_field: zeek.smb_files.times.created formats: @@ -155,10 +195,12 @@ processors: - ISO8601 if: ctx.zeek.smb_files.times?.accessed != null - set: + tag: set_file_created_575ca75e field: file.created value: "{{{zeek.smb_files.times.created}}}" if: "ctx.zeek?.smb_files?.times?.created != null" - date: + tag: date_zeek_smb_files_times_modified_to_zeek_smb_files_times_modified_b7930d01 field: zeek.smb_files.times.modified target_field: zeek.smb_files.times.modified formats: @@ -166,18 +208,22 @@ processors: - ISO8601 if: ctx.zeek.smb_files.times?.accessed != null - set: + tag: set_file_mtime_a9a8bf3e field: file.mtime value: "{{{zeek.smb_files.times.modified}}}" if: "ctx.zeek?.smb_files?.times?.modified != null" - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -186,6 +232,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -194,55 +241,68 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - append: + tag: append_related_user_837e080f field: related.user value: "{{{user.name}}}" if: "ctx.user?.name != null" allow_duplicates: false - set: + tag: set_file_path_d6e3a1fe field: file.path value: "{{{zeek.smb_files.path}}}\\{{{zeek.smb_files.name}}}" if: "ctx.zeek?.smb_files?.path != null && ctx.zeek?.smb_files?.name != null" - append: + tag: append_event_type_87d3ed7b field: event.type value: deletion if: "ctx.zeek?.smb_files?.action == 'SMB::FILE_DELETE'" - append: + tag: append_event_type_282df796 field: event.type value: change if: "ctx.zeek?.smb_files?.action == 'SMB::FILE_RENAME' || ctx.zeek?.smb_files?.action == 'SMB::FILE_SET_ATTRIBUTE'" - append: + tag: append_event_type_132185a5 field: event.type value: info if: "ctx.zeek?.smb_files?.action != null && ctx.zeek.smb_files != 'SMB::FILE_DELETE' && ctx.zeek.smb_files != 'SMB::FILE_RENAME' && ctx.zeek.smb_files != 'SMB::FILE_SET_ATTRIBUTE'" - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_286eab46 field: - zeek.smb_files.id ignore_missing: true @@ -252,4 +312,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml index a48ffc59492..e47b594dc2a 100644 --- a/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml @@ -2,111 +2,141 @@ description: Pipeline for normalizing Zeek smb_mapping.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_smb_mapping_42b501fb field: _temp_ target_field: zeek.smb_mapping # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_5063935e field: network.protocol value: smb - dot_expander: + tag: dot_expander_id_orig_p_92b806f1 path: zeek.smb_mapping field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_4aa15f19 path: zeek.smb_mapping field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_4f78ed24 path: zeek.smb_mapping field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_f182064c path: zeek.smb_mapping field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_smb_mapping_id_orig_h_to_source_address_2d4e0926 field: zeek.smb_mapping.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_smb_mapping_id_orig_p_to_source_port_8eaf363f field: zeek.smb_mapping.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_smb_mapping_id_resp_h_to_destination_address_2a20f9c8 field: zeek.smb_mapping.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_smb_mapping_id_resp_p_to_destination_port_bbe9ae69 field: zeek.smb_mapping.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_smb_mapping_uid_to_zeek_session_id_278adea3 field: zeek.smb_mapping.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - date: + tag: date_zeek_smb_mapping_ts_11502b8e field: zeek.smb_mapping.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_smb_mapping_ts_d3425b02 field: zeek.smb_mapping.ts - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -115,6 +145,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -123,34 +154,42 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_d9c0c691 field: - zeek.smb_mapping.id ignore_missing: true @@ -160,4 +199,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml index cb21514b6d7..5a080719f6b 100644 --- a/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml @@ -2,120 +2,152 @@ description: Pipeline for normalizing Zeek smtp.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_smtp_19f39c5e field: _temp_ target_field: zeek.smtp # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_0c6fa334 field: network.protocol value: smtp - dot_expander: + tag: dot_expander_id_orig_p_1944d418 path: zeek.smtp field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_99060030 path: zeek.smtp field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_403f9e2b path: zeek.smtp field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_511ee243 path: zeek.smtp field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_smtp_id_orig_h_to_source_address_500b8fd3 field: zeek.smtp.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_smtp_id_orig_p_to_source_port_cbbab3d0 field: zeek.smtp.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_smtp_id_resp_h_to_destination_address_346f119f field: zeek.smtp.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_smtp_id_resp_p_to_destination_port_1b6e995c field: zeek.smtp.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_smtp_uid_to_zeek_session_id_e71225bc field: zeek.smtp.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_smtp_trans_depth_to_zeek_smtp_transaction_depth_0f52389e field: zeek.smtp.trans_depth target_field: zeek.smtp.transaction_depth ignore_missing: true - rename: + tag: rename_zeek_smtp_mailfrom_to_zeek_smtp_mail_from_308aa887 field: zeek.smtp.mailfrom target_field: zeek.smtp.mail_from ignore_missing: true - rename: + tag: rename_zeek_smtp_rcptto_to_zeek_smtp_rcpt_to_229168a5 field: zeek.smtp.rcptto target_field: zeek.smtp.rcpt_to ignore_missing: true - convert: + tag: convert_zeek_smtp_tls_to_tls_established_cf0eaeb4 field: zeek.smtp.tls target_field: tls.established type: boolean if: ctx.zeek?.smtp?.tls != null - date: + tag: date_zeek_smtp_ts_ae12ab0f field: zeek.smtp.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_smtp_ts_f52f0af5 field: zeek.smtp.ts - date: + tag: date_zeek_smtp_date_to_zeek_smtp_date_71fb0b24 field: zeek.smtp.date target_field: zeek.smtp.date formats: @@ -123,14 +155,17 @@ processors: - d MMM yyyy HH:mm:ss Z if: ctx.zeek.smtp.date != null - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -139,6 +174,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -147,39 +183,48 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_d5e719bd field: - zeek.smtp.fuids ignore_missing: true if: 'ctx.zeek?.smtp?.fuids == null || ctx.zeek?.smtp?.isEmpty()' - remove: + tag: remove_4b7e6f92 field: - zeek.smtp.id ignore_missing: true @@ -189,4 +234,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml index ebb7781db53..6fe6e50b120 100644 --- a/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml @@ -2,119 +2,151 @@ description: Pipeline for normalizing Zeek snmp.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_snmp_90b8e3e2 field: _temp_ target_field: zeek.snmp # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - set: + tag: set_network_transport_a6cfd568 field: network.transport value: udp - set: + tag: set_network_protocol_61c210d4 field: network.protocol value: snmp - dot_expander: + tag: dot_expander_id_orig_p_6d108f88 path: zeek.snmp field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_bdda1a70 path: zeek.snmp field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_18b486bb path: zeek.snmp field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_fbcc74a3 path: zeek.snmp field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_snmp_id_orig_h_to_source_address_5ac391bf field: zeek.snmp.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_snmp_id_orig_p_to_source_port_08b95c1c field: zeek.snmp.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_snmp_id_resp_h_to_destination_address_566f837b field: zeek.snmp.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_snmp_id_resp_p_to_destination_port_b5b43f28 field: zeek.snmp.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_snmp_uid_to_zeek_session_id_269b4d20 field: zeek.snmp.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_snmp_get_requests_to_zeek_snmp_get_requests_e08b99a1 field: zeek.snmp.get_requests target_field: zeek.snmp.get.requests ignore_missing: true - rename: + tag: rename_zeek_snmp_get_bulk_requests_to_zeek_snmp_get_bulk_requests_af99f037 field: zeek.snmp.get_bulk_requests target_field: zeek.snmp.get.bulk_requests ignore_missing: true - rename: + tag: rename_zeek_snmp_get_responses_to_zeek_snmp_get_responses_033f7569 field: zeek.snmp.get_responses target_field: zeek.snmp.get.responses ignore_missing: true - rename: + tag: rename_zeek_snmp_set_requests_to_zeek_snmp_set_requests_4193e8b9 field: zeek.snmp.set_requests target_field: zeek.snmp.set.requests ignore_missing: true - date: + tag: date_zeek_snmp_ts_2a2344ef field: zeek.snmp.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_snmp_ts_12e502a5 field: zeek.snmp.ts - date: + tag: date_zeek_snmp_up_since_to_zeek_snmp_up_since_a312950d field: zeek.snmp.up_since target_field: zeek.snmp.up_since formats: @@ -122,14 +154,17 @@ processors: - ISO8601 if: ctx.zeek.snmp.up_since != null - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -138,6 +173,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -146,34 +182,42 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_e2fb06b6 field: - zeek.snmp.id ignore_missing: true @@ -183,4 +227,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml index a56ed87a2bc..36a0a8c7a01 100644 --- a/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml @@ -2,134 +2,170 @@ description: Pipeline for normalizing Zeek socks.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_socks_267c7389 field: _temp_ target_field: zeek.socks # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_275c009b field: network.protocol value: socks - dot_expander: + tag: dot_expander_id_orig_p_253e4d8f path: zeek.socks field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_291dfa47 path: zeek.socks field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_ff8bb026 path: zeek.socks field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_4de0e21e path: zeek.socks field: id.resp_p ignore_failure: true - dot_expander: + tag: dot_expander_request_name_430cd39c path: zeek.socks field: request.name ignore_failure: true - rename: + tag: rename_zeek_socks_id_orig_h_to_source_address_114d0f2c field: zeek.socks.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_socks_id_orig_p_to_source_port_7a0cd38d field: zeek.socks.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_socks_id_resp_h_to_destination_address_4df3034e field: zeek.socks.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_socks_id_resp_p_to_destination_port_e37509b7 field: zeek.socks.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_socks_uid_to_zeek_session_id_afbfd9ed field: zeek.socks.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_socks_request_name_to_zeek_socks_request_host_d09d96d9 field: zeek.socks.request.name target_field: zeek.socks.request.host ignore_missing: true - rename: + tag: rename_zeek_socks_request_p_to_zeek_socks_request_port_03678a44 field: zeek.socks.request_p target_field: zeek.socks.request.port ignore_missing: true - rename: + tag: rename_zeek_socks_bound_p_to_zeek_socks_bound_port_43e928c8 field: zeek.socks.bound_p target_field: zeek.socks.bound.port ignore_missing: true - set: + tag: set_user_name_3e97ca90 field: user.name copy_from: zeek.socks.user if: ctx.zeek?.socks?.user != null - date: + tag: date_zeek_socks_ts_1a7e132c field: zeek.socks.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_socks_ts_9dd7264c field: zeek.socks.ts - dot_expander: + tag: dot_expander_bound_host_00fdf54b field: bound.host path: zeek.socks - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -138,6 +174,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -146,51 +183,63 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - append: + tag: append_related_user_837e080f field: related.user value: "{{{user.name}}}" if: "ctx.user?.name != null" allow_duplicates: false - append: + tag: append_event_type_b66bb519 field: event.type value: error if: "ctx.zeek?.socks?.status != null && ctx.zeek.socks.status != 'succeeded'" - set: + tag: set_event_outcome_c9f20558 field: event.outcome value: success if: "ctx.zeek?.socks?.status != null && ctx.zeek.socks.status == 'succeeded'" - set: + tag: set_event_outcome_bf13ad2b field: event.outcome value: failure if: "ctx.zeek?.socks?.status != null && ctx.zeek.socks.status != 'succeeded'" - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_86d3317b field: - zeek.socks.id ignore_missing: true @@ -200,4 +249,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/software/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/software/elasticsearch/ingest_pipeline/default.yml index dc6dd80b16d..de8aadbe69e 100644 --- a/packages/zeek/data_stream/software/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/software/elasticsearch/ingest_pipeline/default.yml @@ -2,97 +2,122 @@ description: Pipeline for normalizing Zeek software.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_software_22eff672 field: _temp_ target_field: zeek.software ignore_failure: true # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_fe6ef82e field: event.category value: - network - file - set: + tag: set_event_type_ec95f7f2 field: event.type value: - info - rename: + tag: rename_zeek_software_host_to_host_ip_98d7889f field: zeek.software.host target_field: host.ip ignore_missing: true - date: + tag: date_zeek_software_ts_e751c958 field: zeek.software.ts formats: - UNIX - ISO8601 - set: + tag: set_network_type_f877b344 field: network.type value: ipv4 if: ctx.host?.ip?.contains('.') == true - set: + tag: set_network_type_d9396dce field: network.type value: ipv6 if: ctx.host?.ip?.contains(':') == true - append: + tag: append_related_ip_54415bdd field: related.ip value: "{{{host.ip}}}" if: ctx.host?.ip != null allow_duplicates: false - geoip: + tag: geoip_host_ip_to_host_geo_d8a29594 field: host.ip target_field: host.geo ignore_missing: true - set: + tag: set_host_ip_1d72afcb field: host.ip value: ['{{{host.ip}}}'] if: ctx.host?.ip instanceof String - rename: + tag: rename_zeek_software_software_type_to_zeek_software_type_df395308 field: zeek.software.software_type target_field: zeek.software.type ignore_missing: true - rename: + tag: rename_zeek_software_unparsed_version_to_zeek_software_version_full_ff80081e field: zeek.software.unparsed_version target_field: zeek.software.version.full ignore_missing: true - dot_expander: + tag: dot_expander_version_major_a0cc91f8 field: version.major path: zeek.software - dot_expander: + tag: dot_expander_version_minor_d1d211d4 field: version.minor path: zeek.software - dot_expander: + tag: dot_expander_version_minor2_b36525dc field: version.minor2 path: zeek.software - dot_expander: + tag: dot_expander_version_minor3_1416e11f field: version.minor3 path: zeek.software - dot_expander: + tag: dot_expander_version_addl_d05e2230 field: version.addl path: zeek.software - rename: + tag: rename_zeek_software_version_addl_to_zeek_software_version_additional_102deba0 field: zeek.software.version.addl target_field: zeek.software.version.additional ignore_missing: true - remove: + tag: remove_efd3ef9d field: - zeek.software.ts ignore_missing: true @@ -102,4 +127,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml index 05be9a636d7..4b2767d7e1d 100644 --- a/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml @@ -2,139 +2,176 @@ description: Pipeline for normalizing Zeek ssh.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_ssh_bbd88850 field: _temp_ target_field: zeek.ssh # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - set: + tag: set_network_protocol_ad65f9d6 field: network.protocol value: ssh - dot_expander: + tag: dot_expander_id_orig_p_2136ec12 path: zeek.ssh field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_dafea1fa path: zeek.ssh field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_77be268b path: zeek.ssh field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_21176db3 path: zeek.ssh field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_ssh_id_orig_h_to_source_address_aa4b4e5d field: zeek.ssh.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_ssh_id_orig_p_to_source_port_423242da field: zeek.ssh.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_ssh_id_resp_h_to_destination_address_14026ed9 field: zeek.ssh.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_ssh_id_resp_p_to_destination_port_c9b11ef6 field: zeek.ssh.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_ssh_uid_to_zeek_session_id_899da9da field: zeek.ssh.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_ssh_auth_attempts_to_zeek_ssh_auth_attempts_8286f825 field: zeek.ssh.auth_attempts target_field: zeek.ssh.auth.attempts ignore_missing: true - rename: + tag: rename_zeek_ssh_auth_success_to_zeek_ssh_auth_success_bf11a64f field: zeek.ssh.auth_success target_field: zeek.ssh.auth.success ignore_missing: true - rename: + tag: rename_zeek_ssh_cipher_alg_to_zeek_ssh_algorithm_cipher_10131c2c field: zeek.ssh.cipher_alg target_field: zeek.ssh.algorithm.cipher ignore_missing: true - rename: + tag: rename_zeek_ssh_mac_alg_to_zeek_ssh_algorithm_mac_e20dc176 field: zeek.ssh.mac_alg target_field: zeek.ssh.algorithm.mac ignore_missing: true - rename: + tag: rename_zeek_ssh_compression_alg_to_zeek_ssh_algorithm_compression_c5cbf1f6 field: zeek.ssh.compression_alg target_field: zeek.ssh.algorithm.compression ignore_missing: true - rename: + tag: rename_zeek_ssh_kex_alg_to_zeek_ssh_algorithm_key_exchange_446324a9 field: zeek.ssh.kex_alg target_field: zeek.ssh.algorithm.key_exchange ignore_missing: true - rename: + tag: rename_zeek_ssh_host_key_alg_to_zeek_ssh_algorithm_host_key_fd8d294e field: zeek.ssh.host_key_alg target_field: zeek.ssh.algorithm.host_key ignore_missing: true - date: + tag: date_zeek_ssh_ts_0697aa39 field: zeek.ssh.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_ssh_ts_880c29cb field: zeek.ssh.ts - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -143,6 +180,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -151,42 +189,52 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - set: + tag: set_event_outcome_52434906 field: event.outcome value: failure if: "ctx.zeek?.ssh?.auth?.success != null && ctx.zeek.ssh.auth.success == false" - set: + tag: set_event_outcome_9a231218 field: event.outcome value: success if: "ctx.zeek?.ssh?.auth?.success != null && ctx.zeek.ssh.auth.success == true" - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_ecdac60a field: - zeek.ssh.id ignore_missing: true @@ -196,4 +244,8 @@ on_failure: value: pipeline_error - set: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml index 86658904fe9..9384d799974 100644 --- a/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml @@ -2,144 +2,182 @@ description: Pipeline for normalizing Zeek ssl.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_ssl_d15f060c field: _temp_ target_field: zeek.ssl # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7ca1b382 field: event.type value: protocol - set: + tag: set_network_transport_a8ca8f4c field: network.transport value: tcp - dot_expander: + tag: dot_expander_id_orig_p_09347256 path: zeek.ssl field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_c2fc283e path: zeek.ssl field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_274cd80f path: zeek.ssl field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_0914f3f7 path: zeek.ssl field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_ssl_id_orig_h_to_source_address_1fb29429 field: zeek.ssl.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_ssl_id_orig_p_to_source_port_dd5bf3a6 field: zeek.ssl.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_ssl_id_resp_h_to_destination_address_1dcff545 field: zeek.ssl.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_ssl_id_resp_p_to_destination_port_afdbf422 field: zeek.ssl.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_ssl_uid_to_zeek_session_id_95fa737e field: zeek.ssl.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_client_address_0aa293ce field: client.address copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - set: + tag: set_server_address_3c18622e field: server.address copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_ssl_server_name_to_zeek_ssl_server_name_e0158fa9 field: zeek.ssl.server_name target_field: zeek.ssl.server.name ignore_missing: true - rename: + tag: rename_zeek_ssl_cert_chain_to_zeek_ssl_server_cert_chain_2eb43fad field: zeek.ssl.cert_chain target_field: zeek.ssl.server.cert_chain ignore_missing: true - rename: + tag: rename_zeek_ssl_cert_chain_fuids_to_zeek_ssl_server_cert_chain_fuids_d9c0a60d field: zeek.ssl.cert_chain_fuids target_field: zeek.ssl.server.cert_chain_fuids ignore_missing: true - rename: + tag: rename_zeek_ssl_client_cert_chain_to_zeek_ssl_client_cert_chain_47adc831 field: zeek.ssl.client_cert_chain target_field: zeek.ssl.client.cert_chain ignore_missing: true - rename: + tag: rename_zeek_ssl_client_cert_chain_fuids_to_zeek_ssl_client_cert_chain_fuids_fcd9dc01 field: zeek.ssl.client_cert_chain_fuids target_field: zeek.ssl.client.cert_chain_fuids ignore_missing: true - rename: + tag: rename_zeek_ssl_validation_status_to_zeek_ssl_validation_status_70a5ff85 field: zeek.ssl.validation_status target_field: zeek.ssl.validation.status ignore_missing: true - rename: + tag: rename_zeek_ssl_validation_code_to_zeek_ssl_validation_code_1880a6ad field: zeek.ssl.validation_code target_field: zeek.ssl.validation.code ignore_missing: true - date: + tag: date_zeek_ssl_ts_6084dae5 field: zeek.ssl.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_ssl_ts_e7fc812f field: zeek.ssl.ts - rename: + tag: rename_zeek_ssl_not_valid_after_to_tls_server_not_after_5c3b792a field: zeek.ssl.not_valid_after target_field: tls.server.not_after ignore_missing: true - rename: + tag: rename_zeek_ssl_not_valid_before_to_tls_server_not_before_92fa49c6 field: zeek.ssl.not_valid_before target_field: tls.server.not_before ignore_missing: true - date: + tag: date_tls_server_not_before_to_tls_server_not_before_1478f3a9 if: ctx.tls?.server?.not_before != null field: tls.server.not_before target_field: tls.server.not_before @@ -147,6 +185,7 @@ processors: - UNIX - ISO8601 - date: + tag: date_tls_server_not_after_to_tls_server_not_after_bd9ed62a if: ctx.tls?.server?.not_after != null field: tls.server.not_after target_field: tls.server.not_after @@ -154,14 +193,17 @@ processors: - UNIX - ISO8601 - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -170,6 +212,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -178,302 +221,374 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - remove: + tag: remove_zeek_ssl_client_cert_chain_fuids_fddcdc08 field: zeek.ssl.client.cert_chain_fuids if: ctx.zeek.ssl.client?.cert_chain_fuids?.length == 0 ignore_missing: true - gsub: + tag: gsub_zeek_ssl_issuer_e992ff57 field: zeek.ssl.issuer pattern: \\, replacement: "" ignore_missing: true - kv: + tag: kv_zeek_ssl_issuer_to_zeek_ssl_server_issuer_14120838 field: zeek.ssl.issuer field_split: "," value_split: "=" target_field: zeek.ssl.server.issuer ignore_missing: true - rename: + tag: rename_zeek_ssl_issuer_to_tls_server_issuer_c9eaffbd field: zeek.ssl.issuer target_field: tls.server.issuer ignore_missing: true - rename: + tag: rename_zeek_ssl_resp_certificate_sha1_to_tls_server_hash_sha1_ed218666 field: zeek.ssl.resp_certificate_sha1 target_field: tls.server.hash.sha1 ignore_missing: true - uppercase: + tag: uppercase_tls_server_hash_sha1_b04b0ad7 field: tls.server.hash.sha1 ignore_missing: true - set: + tag: set_tls_server_x509_issuer_country_d3673720 field: tls.server.x509.issuer.country value: ["{{{zeek.ssl.server.issuer.C}}}"] if: ctx.zeek?.ssl?.server?.issuer?.C instanceof String - rename: + tag: rename_zeek_ssl_server_issuer_C_to_zeek_ssl_server_issuer_country_2c0aede5 field: zeek.ssl.server.issuer.C target_field: zeek.ssl.server.issuer.country ignore_missing: true - set: + tag: set_tls_server_x509_issuer_common_name_0ac7715f field: tls.server.x509.issuer.common_name value: ["{{{zeek.ssl.server.issuer.CN}}}"] if: ctx.zeek?.ssl?.server?.issuer?.CN instanceof String - rename: + tag: rename_zeek_ssl_server_issuer_CN_to_zeek_ssl_server_issuer_common_name_9c19e154 field: zeek.ssl.server.issuer.CN target_field: zeek.ssl.server.issuer.common_name ignore_missing: true - set: + tag: set_tls_server_x509_issuer_locality_d89082f1 field: tls.server.x509.issuer.locality value: ["{{{zeek.ssl.server.issuer.L}}}"] if: ctx.zeek?.ssl?.server?.issuer?.L instanceof String - rename: + tag: rename_zeek_ssl_server_issuer_L_to_zeek_ssl_server_issuer_locality_a8fe340b field: zeek.ssl.server.issuer.L target_field: zeek.ssl.server.issuer.locality ignore_missing: true - set: + tag: set_tls_server_x509_issuer_organization_c3169a19 field: tls.server.x509.issuer.organization value: ["{{{zeek.ssl.server.issuer.O}}}"] if: ctx.zeek?.ssl?.server?.issuer?.O instanceof String - rename: + tag: rename_zeek_ssl_server_issuer_O_to_zeek_ssl_server_issuer_organization_022ad0fc field: zeek.ssl.server.issuer.O target_field: zeek.ssl.server.issuer.organization ignore_missing: true - set: + tag: set_tls_server_x509_issuer_organizational_unit_9e2f0213 field: tls.server.x509.issuer.organizational_unit value: ["{{{zeek.ssl.server.issuer.OU}}}"] if: ctx.zeek?.ssl?.server?.issuer?.OU instanceof String - rename: + tag: rename_zeek_ssl_server_issuer_OU_to_zeek_ssl_server_issuer_organizational_unit_4279fe67 field: zeek.ssl.server.issuer.OU target_field: zeek.ssl.server.issuer.organizational_unit ignore_missing: true - set: + tag: set_tls_server_x509_issuer_state_or_province_f656ed42 field: tls.server.x509.issuer.state_or_province value: ["{{{zeek.ssl.server.issuer.ST}}}"] if: ctx.zeek?.ssl?.server?.issuer?.ST instanceof String - rename: + tag: rename_zeek_ssl_server_issuer_ST_to_zeek_ssl_server_issuer_state_c732835e field: zeek.ssl.server.issuer.ST target_field: zeek.ssl.server.issuer.state ignore_missing: true - gsub: + tag: gsub_zeek_ssl_subject_55de8332 field: zeek.ssl.subject pattern: \\, replacement: "" ignore_missing: true - kv: + tag: kv_zeek_ssl_subject_to_zeek_ssl_server_subject_c1866c44 field: zeek.ssl.subject field_split: "," value_split: "=" target_field: zeek.ssl.server.subject ignore_missing: true - rename: + tag: rename_zeek_ssl_subject_to_tls_server_subject_aae8c93d field: zeek.ssl.subject target_field: tls.server.subject ignore_missing: true - set: + tag: set_tls_server_x509_subject_country_75f44ca5 field: tls.server.x509.subject.country value: ["{{{zeek.ssl.server.subject.C}}}"] if: ctx.zeek?.ssl?.server?.subject?.C instanceof String - rename: + tag: rename_zeek_ssl_server_subject_C_to_zeek_ssl_server_subject_country_20d4db6d field: zeek.ssl.server.subject.C target_field: zeek.ssl.server.subject.country ignore_missing: true - set: + tag: set_tls_server_x509_subject_common_name_6312f06c field: tls.server.x509.subject.common_name value: ["{{{zeek.ssl.server.subject.CN}}}"] if: ctx.zeek?.ssl?.server?.subject?.CN instanceof String - rename: + tag: rename_zeek_ssl_server_subject_CN_to_zeek_ssl_server_subject_common_name_c6002c56 field: zeek.ssl.server.subject.CN target_field: zeek.ssl.server.subject.common_name ignore_missing: true - set: + tag: set_tls_server_x509_subject_locality_742f7630 field: tls.server.x509.subject.locality value: ["{{{zeek.ssl.server.subject.L}}}"] if: ctx.zeek?.ssl?.server?.subject?.L instanceof String - rename: + tag: rename_zeek_ssl_server_subject_L_to_zeek_ssl_server_subject_locality_b2784909 field: zeek.ssl.server.subject.L target_field: zeek.ssl.server.subject.locality ignore_missing: true - set: + tag: set_tls_server_x509_subject_organization_12ae9abe field: tls.server.x509.subject.organization value: ["{{{zeek.ssl.server.subject.O}}}"] if: ctx.zeek?.ssl?.server?.subject?.O instanceof String - rename: + tag: rename_zeek_ssl_server_subject_O_to_zeek_ssl_server_subject_organization_438f5454 field: zeek.ssl.server.subject.O target_field: zeek.ssl.server.subject.organization ignore_missing: true - set: + tag: set_tls_server_x509_subject_organizational_unit_ed63af56 field: tls.server.x509.subject.organizational_unit value: ["{{{zeek.ssl.server.subject.OU}}}"] if: ctx.zeek?.ssl?.server?.subject?.OU instanceof String - rename: + tag: rename_zeek_ssl_server_subject_OU_to_zeek_ssl_server_subject_organizational_unit_7c8e51cf field: zeek.ssl.server.subject.OU target_field: zeek.ssl.server.subject.organizational_unit ignore_missing: true - set: + tag: set_tls_server_x509_subject_state_or_province_d02b3fed field: tls.server.x509.subject.state_or_province value: ["{{{zeek.ssl.server.subject.ST}}}"] if: ctx.zeek?.ssl?.server?.subject?.ST instanceof String - rename: + tag: rename_zeek_ssl_server_subject_ST_to_zeek_ssl_server_subject_state_452d401c field: zeek.ssl.server.subject.ST target_field: zeek.ssl.server.subject.state ignore_missing: true - gsub: + tag: gsub_zeek_ssl_client_issuer_0eba0d6d field: zeek.ssl.client_issuer pattern: \\, replacement: "" ignore_missing: true - kv: + tag: kv_zeek_ssl_client_issuer_to_zeek_ssl_client_issuer_e4784266 field: zeek.ssl.client_issuer field_split: "," value_split: "=" target_field: zeek.ssl.client.issuer ignore_missing: true - rename: + tag: rename_zeek_ssl_client_issuer_to_tls_client_issuer_5e7725c1 field: zeek.ssl.client_issuer target_field: tls.client.issuer ignore_missing: true - set: + tag: set_tls_client_x509_issuer_country_89927f1c field: tls.client.x509.issuer.country value: ["{{{zeek.ssl.client.issuer.C}}}"] if: ctx.zeek?.ssl?.client?.issuer?.C instanceof String - rename: + tag: rename_zeek_ssl_client_issuer_C_to_zeek_ssl_client_issuer_country_1e9cd32d field: zeek.ssl.client.issuer.C target_field: zeek.ssl.client.issuer.country ignore_missing: true - set: + tag: set_tls_client_x509_issuer_common_name_ffed7edb field: tls.client.x509.issuer.common_name value: ["{{{zeek.ssl.client.issuer.CN}}}"] if: ctx.zeek?.ssl?.client?.issuer?.CN instanceof String - rename: + tag: rename_zeek_ssl_client_issuer_CN_to_zeek_ssl_client_issuer_common_name_9712d304 field: zeek.ssl.client.issuer.CN target_field: zeek.ssl.client.issuer.common_name ignore_missing: true - set: + tag: set_tls_client_x509_issuer_locality_bda95a55 field: tls.client.x509.issuer.locality value: ["{{{zeek.ssl.client.issuer.L}}}"] if: ctx.zeek?.ssl?.client?.issuer?.L instanceof String - rename: + tag: rename_zeek_ssl_client_issuer_L_to_zeek_ssl_client_issuer_locality_b9ce4fab field: zeek.ssl.client.issuer.L target_field: zeek.ssl.client.issuer.locality ignore_missing: true - set: + tag: set_tls_client_x509_issuer_organization_d4e32b7d field: tls.client.x509.issuer.organization value: ["{{{zeek.ssl.client.issuer.O}}}"] if: ctx.zeek?.ssl?.client?.issuer?.O instanceof String - rename: + tag: rename_zeek_ssl_client_issuer_O_to_zeek_ssl_client_issuer_organization_8752efc4 field: zeek.ssl.client.issuer.O target_field: zeek.ssl.client.issuer.organization ignore_missing: true - set: + tag: set_tls_client_x509_issuer_organizational_unit_34f1a98f field: tls.client.x509.issuer.organizational_unit value: ["{{{zeek.ssl.client.issuer.OU}}}"] if: ctx.zeek?.ssl?.client?.issuer?.OU instanceof String - rename: - field: zeek.ssl.client.subject.OU - target_field: zeek.ssl.client.subject.organizational_unit + tag: rename_zeek_ssl_issuer_subject_OU_to_zeek_ssl_issuer_subject_organizational_unit_1bd2945b + field: zeek.ssl.issuer.subject.OU + target_field: zeek.ssl.issuer.subject.organizational_unit ignore_missing: true - set: + tag: set_tls_client_x509_issuer_state_or_province_72078606 field: tls.client.x509.issuer.state_or_province value: ["{{{zeek.ssl.client.issuer.ST}}}"] if: ctx.zeek?.ssl?.client?.issuer?.ST instanceof String - rename: + tag: rename_zeek_ssl_client_issuer_ST_to_zeek_ssl_client_issuer_state_30bc3ef6 field: zeek.ssl.client.issuer.ST target_field: zeek.ssl.client.issuer.state ignore_missing: true - gsub: + tag: gsub_zeek_ssl_client_subject_83c6b1f0 field: zeek.ssl.client_subject pattern: \\, replacement: "" ignore_missing: true - kv: + tag: kv_zeek_ssl_client_subject_to_zeek_ssl_client_subject_0687e01a field: zeek.ssl.client_subject field_split: "," value_split: "=" target_field: zeek.ssl.client.subject ignore_missing: true - remove: + tag: remove_zeek_ssl_client_subject_94220763 field: zeek.ssl.client_subject ignore_missing: true - set: + tag: set_tls_client_x509_subject_country_2c3d7381 field: tls.client.x509.subject.country value: ["{{{zeek.ssl.client.subject.C}}}"] if: ctx.zeek?.ssl?.client?.subject?.C instanceof String - rename: + tag: rename_zeek_ssl_client_subject_C_to_zeek_ssl_client_subject_country_848c13ed field: zeek.ssl.client.subject.C target_field: zeek.ssl.client.subject.country ignore_missing: true - set: + tag: set_tls_client_x509_subject_common_name_536e4198 field: tls.client.x509.subject.common_name value: ["{{{zeek.ssl.client.subject.CN}}}"] if: ctx.zeek?.ssl?.client?.subject?.CN instanceof String - rename: + tag: rename_zeek_ssl_client_subject_CN_to_zeek_ssl_client_subject_common_name_803e000e field: zeek.ssl.client.subject.CN target_field: zeek.ssl.client.subject.common_name ignore_missing: true - set: + tag: set_tls_client_x509_subject_locality_597b68bc field: tls.client.x509.subject.locality value: ["{{{zeek.ssl.client.subject.L}}}"] if: ctx.zeek?.ssl?.client?.subject?.L instanceof String - rename: + tag: rename_zeek_ssl_client_subject_L_to_zeek_ssl_client_subject_locality_c92a9321 field: zeek.ssl.client.subject.L target_field: zeek.ssl.client.subject.locality ignore_missing: true - set: + tag: set_tls_client_x509_subject_organization_056ba2e2 field: tls.client.x509.subject.organization value: ["{{{zeek.ssl.client.subject.O}}}"] if: ctx.zeek?.ssl?.client?.subject?.O instanceof String - rename: + tag: rename_zeek_ssl_client_subject_O_to_zeek_ssl_client_subject_organization_5113c774 field: zeek.ssl.client.subject.O target_field: zeek.ssl.client.subject.organization ignore_missing: true - set: + tag: set_tls_client_x509_subject_organizational_unit_30163e82 field: tls.client.x509.subject.organizational_unit value: ["{{{zeek.ssl.client.subject.OU}}}"] if: ctx.zeek?.ssl?.client?.subject?.OU instanceof String - rename: + tag: rename_zeek_ssl_client_subject_OU_to_zeek_ssl_client_subject_organizational_unit_09aac317 field: zeek.ssl.client.subject.OU target_field: zeek.ssl.client.subject.organizational_unit ignore_missing: true - set: + tag: set_tls_client_x509_subject_state_or_province_3bd70991 field: tls.client.x509.subject.state_or_province value: ["{{{zeek.ssl.client.subject.ST}}}"] if: ctx.zeek?.ssl?.client?.subject?.ST instanceof String - rename: + tag: rename_zeek_ssl_client_subject_ST_to_zeek_ssl_client_subject_state_3710e104 field: zeek.ssl.client.subject.ST target_field: zeek.ssl.client.subject.state ignore_missing: true - set: + tag: set_tls_cipher_ed1b0773 field: tls.cipher value: "{{{zeek.ssl.cipher}}}" if: ctx.zeek?.ssl?.cipher != null - set: + tag: set_tls_curve_8dd65eb1 field: tls.curve value: "{{{zeek.ssl.curve}}}" if: ctx.zeek?.ssl?.curve != null - convert: + tag: convert_zeek_ssl_established_to_tls_established_cffff9df target_field: tls.established field: zeek.ssl.established type: boolean ignore_missing: true - convert: + tag: convert_zeek_ssl_resumed_to_tls_resumed_b448c671 target_field: tls.resumed field: zeek.ssl.resumed type: boolean ignore_missing: true - script: + tag: script_b5425ae8 lang: painless if: ctx.zeek?.ssl?.version != null source: >- @@ -488,40 +603,49 @@ processors: } ctx.tls.version_protocol = parts[0].toLowerCase(); - rename: + tag: rename_zeek_ssl_ja3_to_tls_client_ja3_7448e66d field: zeek.ssl.ja3 target_field: tls.client.ja3 ignore_missing: true - rename: + tag: rename_zeek_ssl_ja3s_to_tls_server_ja3s_716bf205 field: zeek.ssl.ja3s target_field: tls.server.ja3s ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - append: + tag: append_related_hash_62160678 field: related.hash value: "{{{tls.server.ja3s}}}" if: "ctx.tls?.server?.ja3s != null" - append: + tag: append_related_hash_c186cc1b field: related.hash value: "{{{tls.client.ja3}}}" if: "ctx.tls?.client?.ja3 != null" allow_duplicates: false - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_b9073500 field: - zeek.ssl.client ignore_missing: true if: 'ctx.zeek?.ssl?.client == null || ctx.zeek?.ssl?.client.isEmpty()' - remove: + tag: remove_ddd0b326 field: - zeek.ssl.id ignore_missing: true @@ -531,4 +655,8 @@ on_failure: value: pipeline_error - set: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml index dff4ac660f1..fc5be6052f9 100644 --- a/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml @@ -2,136 +2,171 @@ description: Pipeline for normalizing Zeek stats.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_stats_e9e3c737 field: _temp_ target_field: zeek.stats # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - rename: + tag: rename_zeek_stats_mem_to_zeek_stats_memory_5eea29cc field: zeek.stats.mem target_field: zeek.stats.memory ignore_missing: true - rename: + tag: rename_zeek_stats_pkts_proc_to_zeek_stats_packets_processed_c2d168aa field: zeek.stats.pkts_proc target_field: zeek.stats.packets.processed ignore_missing: true - rename: + tag: rename_zeek_stats_pkts_dropped_to_zeek_stats_packets_dropped_87a141e6 field: zeek.stats.pkts_dropped target_field: zeek.stats.packets.dropped ignore_missing: true - rename: + tag: rename_zeek_stats_pkts_link_to_zeek_stats_packets_received_8b974b5f field: zeek.stats.pkts_link target_field: zeek.stats.packets.received ignore_missing: true - rename: + tag: rename_zeek_stats_bytes_recv_to_zeek_stats_bytes_received_19b81516 field: zeek.stats.bytes_recv target_field: zeek.stats.bytes.received ignore_missing: true - rename: + tag: rename_zeek_stats_tcp_conns_to_zeek_stats_connections_tcp_count_e53e554a field: zeek.stats.tcp_conns target_field: zeek.stats.connections.tcp.count ignore_missing: true - rename: + tag: rename_zeek_stats_active_tcp_conns_to_zeek_stats_connections_tcp_active_ecb1bbec field: zeek.stats.active_tcp_conns target_field: zeek.stats.connections.tcp.active ignore_missing: true - rename: + tag: rename_zeek_stats_udp_conns_to_zeek_stats_connections_udp_count_21ed7c52 field: zeek.stats.udp_conns target_field: zeek.stats.connections.udp.count ignore_missing: true - rename: + tag: rename_zeek_stats_active_udp_conns_to_zeek_stats_connections_udp_active_0be22094 field: zeek.stats.active_udp_conns target_field: zeek.stats.connections.udp.active ignore_missing: true - rename: + tag: rename_zeek_stats_icmp_conns_to_zeek_stats_connections_icmp_count_83c93208 field: zeek.stats.icmp_conns target_field: zeek.stats.connections.icmp.count ignore_missing: true - rename: + tag: rename_zeek_stats_active_icmp_conns_to_zeek_stats_connections_icmp_active_ee863a26 field: zeek.stats.active_icmp_conns target_field: zeek.stats.connections.icmp.active ignore_missing: true - rename: + tag: rename_zeek_stats_events_proc_to_zeek_stats_events_processed_68c0e8bf field: zeek.stats.events_proc target_field: zeek.stats.events.processed ignore_missing: true - rename: + tag: rename_zeek_stats_events_queued_to_zeek_stats_events_queued_6d5d31d9 field: zeek.stats.events_queued target_field: zeek.stats.events.queued ignore_missing: true - rename: + tag: rename_zeek_stats_timers_to_zeek_stats_timers_count_fdb32dd9 field: zeek.stats.timers target_field: zeek.stats.timers.count ignore_missing: true - rename: + tag: rename_zeek_stats_active_timers_to_zeek_stats_timers_active_1329bafd field: zeek.stats.active_timers target_field: zeek.stats.timers.active ignore_missing: true - rename: + tag: rename_zeek_stats_files_to_zeek_stats_files_count_987f58f3 field: zeek.stats.files target_field: zeek.stats.files.count ignore_missing: true - rename: + tag: rename_zeek_stats_active_files_to_zeek_stats_files_active_1d70a1c3 field: zeek.stats.active_files target_field: zeek.stats.files.active ignore_missing: true - rename: + tag: rename_zeek_stats_dns_requests_to_zeek_stats_dns_requests_count_659f8fbd field: zeek.stats.dns_requests target_field: zeek.stats.dns_requests.count ignore_missing: true - rename: + tag: rename_zeek_stats_active_dns_requests_to_zeek_stats_dns_requests_active_5488f5b1 field: zeek.stats.active_dns_requests target_field: zeek.stats.dns_requests.active ignore_missing: true - rename: + tag: rename_zeek_stats_reassem_tcp_size_to_zeek_stats_reassembly_size_tcp_a71e32ec field: zeek.stats.reassem_tcp_size target_field: zeek.stats.reassembly_size.tcp ignore_missing: true - rename: + tag: rename_zeek_stats_reassem_file_size_to_zeek_stats_reassembly_size_file_6ff374d6 field: zeek.stats.reassem_file_size target_field: zeek.stats.reassembly_size.file ignore_missing: true - rename: + tag: rename_zeek_stats_reassem_frag_size_to_zeek_stats_reassembly_size_frag_5cd516fa field: zeek.stats.reassem_frag_size target_field: zeek.stats.reassembly_size.frag ignore_missing: true - rename: + tag: rename_zeek_stats_reassem_unknown_size_to_zeek_stats_reassembly_size_unknown_f227f27e field: zeek.stats.reassem_unknown_size target_field: zeek.stats.reassembly_size.unknown ignore_missing: true - rename: + tag: rename_zeek_stats_pkt_lag_to_zeek_stats_timestamp_lag_1aa78195 field: zeek.stats.pkt_lag target_field: zeek.stats.timestamp_lag ignore_missing: true - date: + tag: date_zeek_stats_ts_c0b13aa6 field: zeek.stats.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_stats_ts_cd142fc2 field: zeek.stats.ts - set: + tag: set_event_kind_495d69f0 field: event.kind value: metric on_failure: @@ -140,4 +175,8 @@ on_failure: value: pipeline_error - set: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml index eba4619c773..a7ba234ff71 100644 --- a/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml @@ -2,119 +2,150 @@ description: Pipeline for normalizing Zeek syslog.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_syslog_bf286a35 field: _temp_ target_field: zeek.syslog # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - set: + tag: set_network_protocol_3aef0a77 field: network.protocol value: syslog - dot_expander: + tag: dot_expander_id_orig_p_e0d6aa63 path: zeek.syslog field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_a52a97db path: zeek.syslog field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_1ecdaf50 path: zeek.syslog field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_4e393ba8 path: zeek.syslog field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_syslog_id_orig_h_to_source_address_1c2d48d8 field: zeek.syslog.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_syslog_id_orig_p_to_source_port_85a0f1b9 field: zeek.syslog.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_syslog_id_resp_h_to_destination_address_f71fa59a field: zeek.syslog.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_syslog_id_resp_p_to_destination_port_bccf8453 field: zeek.syslog.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_syslog_uid_to_zeek_session_id_1f56c0e1 field: zeek.syslog.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_syslog_proto_to_network_transport_d8c17765 field: zeek.syslog.proto target_field: network.transport ignore_missing: true - rename: + tag: rename_zeek_syslog_message_to_zeek_syslog_msg_e2814016 field: zeek.syslog.message target_field: zeek.syslog.msg ignore_missing: true - set: + tag: set_message_49477aab field: message copy_from: zeek.syslog.msg ignore_empty_value: true - set: + tag: set_log_syslog_facility_name_36b65333 field: log.syslog.facility.name copy_from: zeek.syslog.facility if: ctx.zeek?.syslog?.facility != null - set: + tag: set_log_syslog_severity_name_7c85a099 field: log.syslog.severity.name copy_from: zeek.syslog.severity if: ctx.zeek?.syslog?.severity != null - date: + tag: date_zeek_syslog_ts_470b8c98 field: zeek.syslog.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_syslog_ts_978e0410 field: zeek.syslog.ts - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -123,6 +154,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -131,32 +163,39 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - community_id: + tag: community_id_612651e3 target_field: network.community_id on_failure: - set: @@ -164,4 +203,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml index 4193da4127d..1d70b04ae85 100644 --- a/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml @@ -2,74 +2,94 @@ description: Pipeline for normalizing Zeek traceroute.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_traceroute_5c28060c field: _temp_ target_field: zeek.traceroute # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_8a66ccaa field: event.type value: info - rename: + tag: rename_zeek_traceroute_src_to_source_address_50bb6d1e field: zeek.traceroute.src target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_traceroute_dst_to_destination_address_40efd7be field: zeek.traceroute.dst target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_traceroute_proto_to_network_transport_ba359662 field: zeek.traceroute.proto target_field: network.transport ignore_missing: true - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - date: + tag: date_zeek_traceroute_ts_41d92629 field: zeek.traceroute.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_traceroute_ts_9ae62343 field: zeek.traceroute.ts - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -78,6 +98,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -86,32 +107,39 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - remove: + tag: remove_dd7b7815 field: - zeek.traceroute ignore_missing: true @@ -122,4 +150,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml index 98d6e31f11f..ecaba5920ee 100644 --- a/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml @@ -2,110 +2,139 @@ description: Pipeline for normalizing Zeek tunnel.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_tunnel_82d53914 field: _temp_ target_field: zeek.tunnel # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - dot_expander: + tag: dot_expander_id_orig_p_11a584b2 path: zeek.tunnel field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_68042bea path: zeek.tunnel field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_09190059 path: zeek.tunnel field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_223253f1 path: zeek.tunnel field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_tunnel_id_orig_h_to_source_address_d9589101 field: zeek.tunnel.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_tunnel_id_orig_p_to_source_port_0b36b5ae field: zeek.tunnel.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_tunnel_id_resp_h_to_destination_address_67eda40d field: zeek.tunnel.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_tunnel_id_resp_p_to_destination_port_6755f9da field: zeek.tunnel.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_tunnel_uid_to_zeek_session_id_a87cccd6 field: zeek.tunnel.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_tunnel_tunnel_type_to_zeek_tunnel_type_9cfb799d field: zeek.tunnel.tunnel_type target_field: zeek.tunnel.type ignore_missing: true - set: + tag: set_event_action_113b688e field: event.action copy_from: zeek.tunnel.action if: ctx.zeek?.tunnel?.action != null - date: + tag: date_zeek_tunnel_ts_3d1b72e9 field: zeek.tunnel.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_tunnel_ts_3b951cc3 field: zeek.tunnel.ts - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -114,6 +143,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -122,34 +152,42 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - community_id: + tag: community_id_612651e3 target_field: network.community_id - remove: + tag: remove_8aa1a424 field: - zeek.tunnel.id ignore_missing: true @@ -159,4 +197,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml index a8cd26f3410..e5ca7c78db1 100644 --- a/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml @@ -2,110 +2,139 @@ description: Pipeline for normalizing Zeek weird.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_weird_dc4cdff1 field: _temp_ target_field: zeek.weird # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_8a66ccaa field: event.type value: info - dot_expander: + tag: dot_expander_id_orig_p_8b3e4e63 path: zeek.weird field: id.orig_p ignore_failure: true - dot_expander: + tag: dot_expander_id_orig_h_dd341e6b path: zeek.weird field: id.orig_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_h_fccd0ee2 path: zeek.weird field: id.resp_h ignore_failure: true - dot_expander: + tag: dot_expander_id_resp_p_a0f57c0a path: zeek.weird field: id.resp_p ignore_failure: true - rename: + tag: rename_zeek_weird_id_orig_h_to_source_address_bdedba24 field: zeek.weird.id.orig_h target_field: source.address ignore_missing: true - rename: + tag: rename_zeek_weird_id_orig_p_to_source_port_34195e35 field: zeek.weird.id.orig_p target_field: source.port ignore_missing: true - rename: + tag: rename_zeek_weird_id_resp_h_to_destination_address_b8dfe696 field: zeek.weird.id.resp_h target_field: destination.address ignore_missing: true - rename: + tag: rename_zeek_weird_id_resp_p_to_destination_port_ef21064f field: zeek.weird.id.resp_p target_field: destination.port ignore_missing: true - rename: + tag: rename_zeek_weird_uid_to_zeek_session_id_407fe9e5 field: zeek.weird.uid target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - set: + tag: set_source_ip_e17d09df field: source.ip copy_from: source.address if: ctx.source?.address != null - set: + tag: set_destination_ip_3d6f6004 field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - rename: + tag: rename_zeek_weird_addl_to_zeek_weird_additional_info_a20fb6b3 field: zeek.weird.addl target_field: zeek.weird.additional_info ignore_missing: true - set: + tag: set_rule_name_fe26024b field: rule.name copy_from: zeek.weird.name if: ctx.weird?.name != null - date: + tag: date_zeek_weird_ts_cac0ee48 field: zeek.weird.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_weird_ts_a42f78f8 field: zeek.weird.ts - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -114,6 +143,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -122,32 +152,39 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - remove: + tag: remove_34686103 field: - zeek.weird.id ignore_missing: true @@ -157,4 +194,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml index e43198e87e1..262d334d678 100644 --- a/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml @@ -2,167 +2,205 @@ description: Pipeline for normalizing Zeek x509.log processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ - pipeline: + tag: pipeline_c69a6660 if: ctx?._temp_?.result != null name: '{{ IngestPipeline "third-party" }}' - drop: + tag: drop_c70c3de1 description: Drop if no timestamp (invalid json) if: 'ctx?._temp_?.ts == null' - rename: + tag: rename__temp__to_zeek_x509_598fdc86 field: _temp_ target_field: zeek.x509 # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - append: + tag: append_event_type_8a66ccaa field: event.type value: info - dot_expander: + tag: dot_expander_certificate_version_5083ac0e path: zeek.x509 field: certificate.version ignore_failure: true - dot_expander: + tag: dot_expander_certificate_serial_01dc8028 path: zeek.x509 field: certificate.serial ignore_failure: true - dot_expander: + tag: dot_expander_certificate_subject_6e404b6a path: zeek.x509 field: certificate.subject ignore_failure: true - dot_expander: + tag: dot_expander_certificate_issuer_2d8e56ab path: zeek.x509 field: certificate.issuer ignore_failure: true - dot_expander: + tag: dot_expander_certificate_not_valid_before_2aa647ca path: zeek.x509 field: certificate.not_valid_before ignore_failure: true - dot_expander: + tag: dot_expander_certificate_not_valid_after_85dc08f7 path: zeek.x509 field: certificate.not_valid_after ignore_failure: true - dot_expander: + tag: dot_expander_certificate_key_alg_fde685f0 path: zeek.x509 field: certificate.key_alg ignore_failure: true - dot_expander: + tag: dot_expander_certificate_sig_alg_b1726db2 path: zeek.x509 field: certificate.sig_alg ignore_failure: true - dot_expander: + tag: dot_expander_certificate_key_type_d7adb3a8 path: zeek.x509 field: certificate.key_type ignore_failure: true - dot_expander: + tag: dot_expander_certificate_key_length_f246e250 path: zeek.x509 field: certificate.key_length ignore_failure: true - dot_expander: + tag: dot_expander_certificate_exponent_68e8e903 path: zeek.x509 field: certificate.exponent ignore_failure: true - dot_expander: + tag: dot_expander_certificate_cn_5485b055 path: zeek.x509 field: certificate.cn ignore_failure: true - dot_expander: + tag: dot_expander_zeek_x509_basic_constraints_ca_525a9213 path: zeek.x509 field: zeek.x509.basic_constraints.ca ignore_failure: true - dot_expander: + tag: dot_expander_basic_constraints_path_len_050dd025 path: zeek.x509 field: basic_constraints.path_len ignore_failure: true - rename: + tag: rename_zeek_x509_id_to_zeek_session_id_fb9653af field: zeek.x509.id target_field: zeek.session_id ignore_missing: true - set: + tag: set_event_id_3119a372 field: event.id copy_from: zeek.session_id if: ctx.zeek?.session_id != null - rename: + tag: rename_zeek_x509_certificate_not_valid_before_to_zeek_x509_certificate_valid_from_09d379aa field: zeek.x509.certificate.not_valid_before target_field: zeek.x509.certificate.valid.from ignore_missing: true - rename: + tag: rename_zeek_x509_certificate_not_valid_after_to_zeek_x509_certificate_valid_until_4c244863 field: zeek.x509.certificate.not_valid_after target_field: zeek.x509.certificate.valid.until ignore_missing: true - rename: + tag: rename_zeek_x509_basic_constraints_ca_to_zeek_x509_basic_constraints_certificate_authority_fbeb87ff field: zeek.x509.basic_constraints.ca target_field: zeek.x509.basic_constraints.certificate_authority ignore_missing: true - rename: + tag: rename_zeek_x509_basic_constraints_path_len_to_zeek_x509_basic_constraints_path_length_9da5e6d7 field: zeek.x509.basic_constraints.path_len target_field: zeek.x509.basic_constraints.path_length ignore_missing: true - rename: - field: zeek.x509.basic_constraints.path_len - target_field: zeek.x509.basic_constraints.path_length - ignore_missing: true - - rename: + tag: rename_zeek_x509_certificate_cn_to_zeek_x509_certificate_common_name_5866d37a field: zeek.x509.certificate.cn target_field: zeek.x509.certificate.common_name ignore_missing: true - rename: + tag: rename_zeek_x509_certificate_issuer_to_zeek_x509_certificate_iss_15ddb1d0 field: zeek.x509.certificate.issuer target_field: zeek.x509.certificate.iss ignore_missing: true - rename: + tag: rename_zeek_x509_certificate_subject_to_zeek_x509_certificate_sub_d3411c30 field: zeek.x509.certificate.subject target_field: zeek.x509.certificate.sub ignore_missing: true - rename: + tag: rename_zeek_x509_certificate_key_alg_to_zeek_x509_certificate_key_algorithm_7997e97c field: zeek.x509.certificate.key_alg target_field: zeek.x509.certificate.key.algorithm ignore_missing: true - rename: + tag: rename_zeek_x509_certificate_key_length_to_zeek_x509_certificate_key_length_a50ab535 field: zeek.x509.certificate.key_length target_field: zeek.x509.certificate.key.length ignore_missing: true - rename: + tag: rename_zeek_x509_certificate_key_type_to_zeek_x509_certificate_key_type_6215876d field: zeek.x509.certificate.key_type target_field: zeek.x509.certificate.key.type ignore_missing: true - rename: + tag: rename_zeek_x509_certificate_sig_alg_to_zeek_x509_certificate_signature_algorithm_9817da8a field: zeek.x509.certificate.sig_alg target_field: zeek.x509.certificate.signature_algorithm ignore_missing: true - rename: + tag: rename_zeek_x509_logcert_to_zeek_x509_log_cert_b5804d43 field: zeek.x509.logcert target_field: zeek.x509.log_cert ignore_missing: true - date: + tag: date_zeek_x509_ts_34d12623 field: zeek.x509.ts formats: - UNIX - ISO8601 - remove: + tag: remove_zeek_x509_ts_c7501221 field: zeek.x509.ts - set: + tag: set_event_id_80b01e72 field: event.id value: "{{{zeek.session_id}}}" if: ctx.zeek.session_id != null - set: + tag: set_file_x509_signature_algorithm_2f464450 field: file.x509.signature_algorithm value: "{{{zeek.x509.certificate.signature_algorithm}}}" ignore_empty_value: true - script: + tag: script_1aff3302 lang: painless params: "md2WithRSAEncryption": MD2-RSA @@ -185,40 +223,50 @@ processors: } if: ctx.file?.x509?.signature_algorithm != null - set: + tag: set_file_x509_public_key_algorithm_62b55ca7 field: file.x509.public_key_algorithm value: "{{{zeek.x509.certificate.key.algorithm}}}" ignore_empty_value: true - convert: + tag: convert_zeek_x509_certificate_key_length_to_file_x509_public_key_size_cf406a93 field: zeek.x509.certificate.key.length target_field: file.x509.public_key_size type: long ignore_missing: true - dot_expander: + tag: dot_expander_certificate_exponent_ba4f3304 field: certificate.exponent path: zeek.x509 - convert: + tag: convert_zeek_x509_certificate_exponent_to_file_x509_public_key_exponent_1035b6d1 field: zeek.x509.certificate.exponent target_field: file.x509.public_key_exponent type: long ignore_missing: true - dot_expander: + tag: dot_expander_certificate_serial_9326a2f3 field: certificate.serial path: zeek.x509 - set: + tag: set_file_x509_serial_number_4877ce26 field: file.x509.serial_number value: "{{{zeek.x509.certificate.serial}}}" ignore_empty_value: true - dot_expander: + tag: dot_expander_certificate_version_ebec668d field: certificate.version path: zeek.x509 - set: + tag: set_file_x509_version_number_c2e37dd2 field: file.x509.version_number value: "{{{zeek.x509.certificate.version}}}" ignore_empty_value: true - dot_expander: + tag: dot_expander_san_dns_e327ed4f field: san.dns path: zeek.x509 - foreach: + tag: foreach_zeek_x509_san_dns_5312a024 field: zeek.x509.san.dns ignore_missing: true processor: @@ -226,9 +274,11 @@ processors: field: file.x509.alternative_names value: "{{{_ingest._value}}}" - dot_expander: + tag: dot_expander_san_uri_7a553e36 field: san.uri path: zeek.x509 - foreach: + tag: foreach_zeek_x509_san_uri_788c08d7 field: zeek.x509.san.uri ignore_missing: true processor: @@ -236,9 +286,11 @@ processors: field: file.x509.alternative_names value: "{{{_ingest._value}}}" - dot_expander: + tag: dot_expander_san_email_40317b68 field: san.email path: zeek.x509 - foreach: + tag: foreach_zeek_x509_san_email_bcb9759d field: zeek.x509.san.email ignore_missing: true processor: @@ -246,9 +298,11 @@ processors: field: file.x509.alternative_names value: "{{{_ingest._value}}}" - dot_expander: + tag: dot_expander_san_ip_f895e3bd field: san.ip path: zeek.x509 - foreach: + tag: foreach_zeek_x509_san_ip_63f6cf26 field: zeek.x509.san.ip ignore_missing: true processor: @@ -256,9 +310,11 @@ processors: field: file.x509.alternative_names value: "{{{_ingest._value}}}" - dot_expander: + tag: dot_expander_san_other_fields_604c920a field: san.other_fields path: zeek.x509 - foreach: + tag: foreach_zeek_x509_san_other_fields_8f24523b field: zeek.x509.san.other_fields ignore_missing: true processor: @@ -266,6 +322,7 @@ processors: field: file.x509.alternative_names value: "{{{_ingest._value}}}" - date: + tag: date_zeek_x509_certificate_valid_from_to_zeek_x509_certificate_valid_from_d81d6a64 field: zeek.x509.certificate.valid.from target_field: zeek.x509.certificate.valid.from formats: @@ -273,10 +330,12 @@ processors: - ISO8601 if: ctx.zeek.x509.certificate?.valid?.from != null - set: + tag: set_file_x509_not_before_ea4f1c11 field: file.x509.not_before value: "{{{zeek.x509.certificate.valid.from}}}" ignore_empty_value: true - date: + tag: date_zeek_x509_certificate_valid_until_to_zeek_x509_certificate_valid_until_c2ad96d8 field: zeek.x509.certificate.valid.until target_field: zeek.x509.certificate.valid.until formats: @@ -284,130 +343,161 @@ processors: - ISO8601 if: ctx.zeek.x509.certificate?.valid?.until != null - set: + tag: set_file_x509_not_after_797c0380 field: file.x509.not_after value: "{{{zeek.x509.certificate.valid.until}}}" ignore_empty_value: true - gsub: + tag: gsub_zeek_x509_certificate_iss_349885d6 field: zeek.x509.certificate.iss pattern: \\, replacement: "" ignore_missing: true - kv: + tag: kv_zeek_x509_certificate_iss_to_zeek_x509_certificate_issuer_fa72cbc1 field: zeek.x509.certificate.iss field_split: "," value_split: "=" target_field: zeek.x509.certificate.issuer ignore_missing: true - remove: + tag: remove_zeek_x509_certificate_iss_aa25f739 field: zeek.x509.certificate.iss ignore_missing: true - set: + tag: set_file_x509_issuer_country_0e7fc00a field: file.x509.issuer.country value: ["{{{zeek.x509.certificate.issuer.C}}}"] if: ctx.zeek?.x509?.certificate?.issuer?.C instanceof String - rename: + tag: rename_zeek_x509_certificate_issuer_C_to_zeek_x509_certificate_issuer_country_de6f4b7d field: zeek.x509.certificate.issuer.C target_field: zeek.x509.certificate.issuer.country ignore_missing: true - set: + tag: set_file_x509_issuer_common_name_c1b64639 field: file.x509.issuer.common_name value: ["{{{zeek.x509.certificate.issuer.CN}}}"] if: ctx.zeek?.x509?.certificate?.issuer?.CN instanceof String - rename: + tag: rename_zeek_x509_certificate_issuer_CN_to_zeek_x509_certificate_issuer_common_name_f26fece4 field: zeek.x509.certificate.issuer.CN target_field: zeek.x509.certificate.issuer.common_name ignore_missing: true - set: + tag: set_file_x509_issuer_locality_4489d94b field: file.x509.issuer.locality value: ["{{{zeek.x509.certificate.issuer.L}}}"] if: ctx.zeek?.x509?.certificate?.issuer?.L instanceof String - rename: + tag: rename_zeek_x509_certificate_issuer_L_to_zeek_x509_certificate_issuer_locality_e2096533 field: zeek.x509.certificate.issuer.L target_field: zeek.x509.certificate.issuer.locality ignore_missing: true - set: + tag: set_file_x509_issuer_organization_4520f5db field: file.x509.issuer.organization value: ["{{{zeek.x509.certificate.issuer.O}}}"] if: ctx.zeek?.x509?.certificate?.issuer?.O instanceof String - rename: + tag: rename_zeek_x509_certificate_issuer_O_to_zeek_x509_certificate_issuer_organization_278d0d74 field: zeek.x509.certificate.issuer.O target_field: zeek.x509.certificate.issuer.organization ignore_missing: true - set: + tag: set_file_x509_issuer_organizational_unit_78e08f85 field: file.x509.issuer.organizational_unit value: ["{{{zeek.x509.certificate.issuer.OU}}}"] if: ctx.zeek?.x509?.certificate?.issuer?.OU instanceof String - rename: + tag: rename_zeek_x509_certificate_issuer_OU_to_zeek_x509_certificate_issuer_organizational_unit_a605e237 field: zeek.x509.certificate.issuer.OU target_field: zeek.x509.certificate.issuer.organizational_unit ignore_missing: true - set: + tag: set_file_x509_issuer_state_or_province_c6af40f8 field: file.x509.issuer.state_or_province value: ["{{{zeek.x509.certificate.issuer.ST}}}"] if: ctx.zeek?.x509?.certificate?.issuer?.ST instanceof String - rename: + tag: rename_zeek_x509_certificate_issuer_ST_to_zeek_x509_certificate_issuer_state_c22aa4fa field: zeek.x509.certificate.issuer.ST target_field: zeek.x509.certificate.issuer.state ignore_missing: true - gsub: + tag: gsub_zeek_x509_certificate_sub_446b0941 field: zeek.x509.certificate.sub pattern: \\, replacement: "" ignore_missing: true - kv: + tag: kv_zeek_x509_certificate_sub_to_zeek_x509_certificate_subject_d5b2dfe9 field: zeek.x509.certificate.sub field_split: "," value_split: "=" target_field: zeek.x509.certificate.subject ignore_missing: true - remove: + tag: remove_zeek_x509_certificate_sub_52c2a7d4 field: zeek.x509.certificate.sub ignore_missing: true - set: + tag: set_file_x509_subject_country_07786ab3 field: file.x509.subject.country value: ["{{{zeek.x509.certificate.subject.C}}}"] if: ctx.zeek?.x509?.certificate?.subject?.C instanceof String - rename: + tag: rename_zeek_x509_certificate_subject_C_to_zeek_x509_certificate_subject_country_992b66c9 field: zeek.x509.certificate.subject.C target_field: zeek.x509.certificate.subject.country ignore_missing: true - set: + tag: set_file_x509_subject_common_name_abff9222 field: file.x509.subject.common_name value: ["{{{zeek.x509.certificate.subject.CN}}}"] if: ctx.zeek?.x509?.certificate?.subject?.CN instanceof String - rename: + tag: rename_zeek_x509_certificate_subject_CN_to_zeek_x509_certificate_subject_common_name_746cfd1a field: zeek.x509.certificate.subject.CN target_field: zeek.x509.certificate.subject.common_name ignore_missing: true - set: + tag: set_file_x509_subject_locality_127aece6 field: file.x509.subject.locality value: ["{{{zeek.x509.certificate.subject.L}}}"] if: ctx.zeek?.x509?.certificate?.subject?.L instanceof String - rename: + tag: rename_zeek_x509_certificate_subject_L_to_zeek_x509_certificate_subject_locality_62cbedd5 field: zeek.x509.certificate.subject.L target_field: zeek.x509.certificate.subject.locality ignore_missing: true - set: + tag: set_file_x509_subject_organization_f52f7b8c field: file.x509.subject.organization value: ["{{{zeek.x509.certificate.subject.O}}}"] if: ctx.zeek?.x509?.certificate?.subject?.O instanceof String - rename: + tag: rename_zeek_x509_certificate_subject_O_to_zeek_x509_certificate_subject_organization_d18ac700 field: zeek.x509.certificate.subject.O target_field: zeek.x509.certificate.subject.organization ignore_missing: true - set: + tag: set_file_x509_subject_organizational_unit_fe4eb674 field: file.x509.subject.organizational_unit value: ["{{{zeek.x509.certificate.subject.OU}}}"] if: ctx.zeek?.x509?.certificate?.subject?.OU instanceof String - rename: + tag: rename_zeek_x509_certificate_subject_OU_to_zeek_x509_certificate_subject_organizational_unit_fa4a788b field: zeek.x509.certificate.subject.OU target_field: zeek.x509.certificate.subject.organizational_unit ignore_missing: true - set: + tag: set_file_x509_subject_state_or_province_2b505aff field: file.x509.subject.state_or_province value: ["{{{zeek.x509.certificate.subject.ST}}}"] if: ctx.zeek?.x509?.certificate?.subject?.ST instanceof String - rename: + tag: rename_zeek_x509_certificate_subject_ST_to_zeek_x509_certificate_subject_state_4aa73304 field: zeek.x509.certificate.subject.ST target_field: zeek.x509.certificate.subject.state ignore_missing: true @@ -417,4 +507,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/third-party.yml index e7e7e2a3288..d12dfa8e796 100644 --- a/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/third-party.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Zeek logs from third party api processors: - fingerprint: + tag: fingerprint_8e6b679f fields: - _temp_.result._cd - _temp_.result._indextime @@ -12,21 +13,26 @@ processors: target_field: '_id' ignore_missing: true - set: + tag: set_event_original_5bbfe67e field: event.original copy_from: _temp_.result._raw ignore_empty_value: true - set: + tag: set_host_name_d99bf8df field: host.name copy_from: _temp_.result.host ignore_empty_value: true - set: + tag: set_log_file_path_76cd6132 copy_from: _temp_.result.source field: log.file.path ignore_empty_value: true - remove: + tag: remove__temp__fc21405e field: _temp_ ignore_missing: true - json: + tag: json_event_original_to__temp__7408cf8c field: event.original target_field: _temp_ on_failure: @@ -36,7 +42,7 @@ on_failure: - append: field: error.message value: >- - error in third party api pipeline: - error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} - with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} - {{{ _ingest.on_failure_message }}} + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/zeek/manifest.yml b/packages/zeek/manifest.yml index 126a339d303..18118532712 100644 --- a/packages/zeek/manifest.yml +++ b/packages/zeek/manifest.yml @@ -1,6 +1,6 @@ name: zeek title: Zeek -version: "3.0.2" +version: "3.0.3" description: Collect logs from Zeek with Elastic Agent. type: integration icons: @@ -9,7 +9,7 @@ icons: size: 214x203 type: image/svg+xml format_version: "3.0.3" -categories: +categories: - network - security conditions: