prisma_cloud: Add misconfiguration and vulnerability data streams for CDR

packages/prisma_cloud/data_stream/misconfiguration/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,4 @@
+fields:
+  tags:
+    - preserve_original_event
+    - preserve_duplicate_custom_fields

packages/prisma_cloud/data_stream/misconfiguration/_dev/test/system/test-cel-config.yml

@@ -0,0 +1,13 @@
+input: cel
+service: prisma_cloud
+vars:
+  username: xxxx
+  password: xxxx
+data_stream:
+  vars:
+    url: http://{{Hostname}}:{{Port}}
+    batch_size: 2
+    preserve_original_event: true
+    preserve_duplicate_custom_fields: true
+assert:
+  hit_count: 4

packages/prisma_cloud/data_stream/misconfiguration/agent/stream/cel.yml.hbs

@@ -0,0 +1,144 @@
+config_version: 2
+interval: {{interval}}
+resource.tracer:
+  enabled: {{enable_request_tracer}}
+  filename: "../../logs/cel/http-request-trace-*.ndjson"
+  maxbackups: 5
+{{#if proxy_url}}
+resource.proxy_url: {{proxy_url}}
+{{/if}}
+{{#if ssl}}
+resource.ssl: {{ssl}}
+{{/if}}
+{{#if http_client_timeout}}
+resource.timeout: {{http_client_timeout}}
+{{/if}}
+resource.url: {{url}}
+state:
+  user: {{username}}
+  password: {{password}}
+  batch_size: {{batch_size}}
+redact:
+  fields:
+    - user
+    - password
+    - next.access_token
+program: |
+  (
+    state.want_more ?
+      state
+    :
+      // Generate auth token
+      post_request(
+        state.url.trim_right("/") + "/login",
+        "application/json",
+        {
+          "username": state.user,
+          "password": state.password,
+        }.encode_json()
+      ).do_request().as(resp, resp.StatusCode == 200 ?
+          resp.Body.decode_json().as(body, {
+            "next": {
+              "access_token": body.token,
+            },
+            "url": state.url,
+            "user": state.user,
+            "password": state.password,
+            "batch_size": state.batch_size,
+          })
+        :
+          {
+            "events": {
+              "error": {
+                "code": string(resp.StatusCode),
+                "id": string(resp.Status),
+                "message": "POST " + state.url.trim_right("/") + "/authenticate: " + (
+                  size(resp.Body) != 0 ?
+                    string(resp.Body)
+                  :
+                    string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+                ),
+              },
+            },
+            "want_more": false,
+            "batch_size": state.batch_size,
+            "url": state.url,
+            "user": state.user,
+            "password": state.password,
+          }
+      )
+  ).as(state,
+    // Exit if there is an error
+    has(state.?events.error) ?
+    	state
+    :
+      // Fetch scan info which contains the resources with misconfiguration details
+      request("GET",
+        state.url.trim_right("/") + "/resource/scan_info?" + {
+          "scan.status": ["all"],
+          "policy.complianceStandard": ["*"],
+          "limit": [string(state.batch_size)],
+          ?"pageToken": state.?next.pageToken.optMap(v, [v]),
+        }.format_query()
+      ).with({
+        "Header":{
+          "x-redlock-auth": [state.next.access_token],
+        }
+      }).do_request().as(resp, resp.StatusCode == 200 ?
+          resp.Body.decode_json().as(body, {
+            "events": (has(body.resources) && body.resources.size() > 0 ?
+              body.resources.map(e, {
+                "message": e.encode_json(),
+              })
+            :
+              []
+            ),
+            "next": {
+              ?"access_token": has(body.nextPageToken) ? optional.of(state.next.access_token) : optional.none(),
+              ?"pageToken": has(body.nextPageToken) ? optional.of(body.nextPageToken) : optional.none(),
+            },
+            "batch_size": state.batch_size,
+            "url": state.url,
+            "user": state.user,
+            "password": state.password,
+            "want_more": has(body.nextPageToken),
+          })
+        :
+          {
+            "events": {
+              "error": {
+                "code": string(resp.StatusCode),
+                "id": string(resp.Status),
+                "message": "GET " + state.url.trim_right("/") + "/resource/scan_info: " + (
+                  size(resp.Body) != 0 ?
+                    string(resp.Body)
+                  :
+                    string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+                ),
+              },
+            },
+            "want_more": false,
+            "batch_size": state.batch_size,
+            "url": state.url,
+            "user": state.user,
+            "password": state.password,
+          }
+      )
+  )
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#if preserve_duplicate_custom_fields}}
+  - preserve_duplicate_custom_fields
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/prisma_cloud/data_stream/misconfiguration/fields/base-fields.yml

@@ -0,0 +1,20 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: event.module
+  type: constant_keyword
+  description: Event module.
+  value: prisma_cloud
+- name: event.dataset
+  type: constant_keyword
+  description: Event dataset.
+  value: prisma_cloud.misconfiguration
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

packages/prisma_cloud/data_stream/misconfiguration/fields/beats.yml

@@ -0,0 +1,6 @@
+- name: input.type
+  type: keyword
+  description: Type of filebeat input.
+- name: log.offset
+  type: long
+  description: Log offset.

packages/prisma_cloud/data_stream/misconfiguration/manifest.yml

@@ -0,0 +1,93 @@
+title: Collect Incident Audit logs from Prisma Cloud Workload Protection.
+type: logs
+streams:
+  - input: cel
+    title: Incident Audit Logs
+    description: Collect Incident Audit logs from Prisma Cloud Workload Protection.
+    template_path: cel.yml.hbs
+    enabled: false
+    vars:
+      - name: url
+        type: text
+        title: URL
+        description: Base URL of the Prisma Cloud Server API, in the form `https://<CONSOLE>/api/v<VERSION>`.
+        required: true
+        show_user: true
+      - name: initial_interval
+        type: text
+        title: Initial Interval
+        multi: false
+        required: true
+        show_user: true
+        default: 24h
+        description: How far back to pull the Prisma Cloud Incident Audit logs from the API. Supported units for this parameter are h/m/s.
+      - name: interval
+        type: text
+        title: Interval
+        description: Interval between two REST API calls. Supported units for this parameter are h/m/s.
+        default: 5m
+        multi: false
+        required: true
+        show_user: true
+      - name: batch_size
+        type: integer
+        title: Batch Size
+        description: Number of events to retrieve in a page. The maximum limit is 100.
+        default: 10000
+        multi: false
+        required: true
+        show_user: false
+      - name: http_client_timeout
+        type: text
+        title: HTTP Client Timeout
+        description: Duration before declaring that the HTTP client connection has timed out. Supported time units are ns, us, ms, s, m, h.
+        multi: false
+        required: true
+        show_user: false
+        default: 60s
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        default: false
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          The request tracer logs requests and responses to the agent's local file-system for debugging configurations.
+          Enabling this request tracing compromises security and should only be used for debugging. Disabling the request
+          tracer will delete any stored traces.
+          See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_resource_tracer_enable)
+          for details.
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        required: true
+        show_user: false
+        default:
+          - forwarded
+          - prisma_cloud-misconfiguration
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`.
+        type: bool
+        multi: false
+        default: false
+      - name: preserve_duplicate_custom_fields
+        required: true
+        show_user: false
+        title: Preserve duplicate custom fields
+        description: Preserve prisma_cloud.misconfiguration fields that were copied to Elastic Common Schema (ECS) fields.
+        type: bool
+        multi: false
+        default: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.

packages/prisma_cloud/data_stream/vulnerability/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,4 @@
+fields:
+  tags:
+    - preserve_original_event
+    - preserve_duplicate_custom_fields

packages/prisma_cloud/data_stream/vulnerability/_dev/test/system/test-cel-config.yml

@@ -0,0 +1,13 @@
+input: cel
+service: prisma_cloud
+vars:
+  username: xxxx
+  password: xxxx
+data_stream:
+  vars:
+    url: http://{{Hostname}}:{{Port}}
+    batch_size: 2
+    preserve_original_event: true
+    preserve_duplicate_custom_fields: true
+assert:
+  hit_count: 4