[Managed OTLP] Add Host/K8s onboarding configurations (#6630)

* feat: remove apm-processing from gateway collector

* remove elasticsearch exporter in favor of managed otlp

* update darwin configurations

* refactor: move motel values to motel dir

* add ApiKey prefix for motel exporter headers

* refactor: move motel configurations to motel subdir

* test: add motel configuration integration test case

* chore: add fragments entry

* refactor: use managed_otlp directory

* fix: mock a valid ingest endpoint url

* chore: remove MOtel in favor of Managed OTLP Input

* increase gateway resource limits

* use ELASTIC_OTLP_ENDPOINT as secret and env variable

* never split metrics requests in batchprocessor

* update env variable on host configuration

* gateway: use autoscaler instead of fixed replicas

changelog/fragments/1739544282-add-MOtel-sample-configurations.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: add MOtel sample configurations
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: "elastic-agent"
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/6630
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

internal/pkg/otel/samples/darwin/managed_otlp/logs_metrics_traces.yml

@@ -0,0 +1,112 @@
+receivers:
+  # Receiver for platform specific log files
+  filelog/platformlogs:
+    include: [ /var/log/*.log ]
+    retry_on_failure:
+      enabled: true
+    start_at: end
+    storage: file_storage
+#    start_at: beginning
+
+  # Receiver for CPU, Disk, Memory, and Filesystem metrics
+  hostmetrics/system:
+    collection_interval: 30s
+    scrapers:
+      filesystem:
+      memory:
+        metrics:
+          system.memory.utilization:
+            enabled: true
+      process:
+        mute_process_exe_error: true
+        mute_process_io_error: true
+        mute_process_user_error: true
+        metrics:
+          process.threads:
+            enabled: true
+          process.open_file_descriptors:
+            enabled: true
+          process.memory.utilization:
+            enabled: true
+          process.disk.operations:
+            enabled: true
+      network:
+      processes:
+      load:
+
+  # Receiver for logs, traces, and metrics from SDKs
+  otlp/fromsdk:
+    protocols:
+      grpc:
+      http:
+
+extensions:
+  file_storage:
+    directory: ${env:STORAGE_DIR}
+
+processors:
+  resourcedetection:
+    detectors: ["system"]
+    system:
+      hostname_sources: ["os"]
+      resource_attributes:
+        host.name:
+          enabled: true
+        host.id:
+          enabled: false
+        host.arch:
+          enabled: true
+        host.ip:
+          enabled: true
+        host.mac:
+          enabled: true
+        host.cpu.vendor.id:
+          enabled: true
+        host.cpu.family:
+          enabled: true
+        host.cpu.model.id:
+          enabled: true
+        host.cpu.model.name:
+          enabled: true
+        host.cpu.stepping:
+          enabled: true
+        host.cpu.cache.l2.size:
+          enabled: true
+        os.description:
+          enabled: true
+        os.type:
+          enabled: true
+
+exporters:
+  otlp/ingest:
+    endpoint: ${env:ELASTIC_OTLP_ENDPOINT}
+    headers:
+      Authorization: ${env:ELASTIC_API_KEY}
+
+service:
+  extensions: [file_storage]
+  pipelines:
+    traces/fromsdk:
+      receivers: [otlp/fromsdk]
+      processors: []
+      exporters: [otlp/ingest]
+
+    metrics/fromsdk:
+      receivers: [otlp/fromsdk]
+      processors: []
+      exporters: [otlp/ingest]
+
+    logs/fromsdk:
+      receivers: [otlp/fromsdk]
+      processors: []
+      exporters: [otlp/ingest]
+
+    metrics/hostmetrics:
+      receivers: [hostmetrics/system]
+      processors: [resourcedetection]
+      exporters: [otlp/ingest]
+
+    logs/platformlogs:
+      receivers: [filelog/platformlogs]
+      processors: [resourcedetection]
+      exporters: [otlp/ingest]

internal/pkg/otel/samples/darwin/managed_otlp/platformlogs.yml

@@ -0,0 +1,67 @@
+receivers:
+  # Receiver for platform specific log files
+  filelog/platformlogs:
+    include: [ /var/log/*.log ]
+    retry_on_failure:
+      enabled: true
+    start_at: end
+    storage: file_storage
+#   start_at: beginning
+
+extensions:
+  file_storage:
+    directory: ${env:STORAGE_DIR}
+
+processors:
+  resourcedetection:
+    detectors: ["system"]
+    system:
+      hostname_sources: ["os"]
+      resource_attributes:
+        host.name:
+          enabled: true
+        host.id:
+          enabled: false
+        host.arch:
+          enabled: true
+        host.ip:
+          enabled: true
+        host.mac:
+          enabled: true
+        host.cpu.vendor.id:
+          enabled: true
+        host.cpu.family:
+          enabled: true
+        host.cpu.model.id:
+          enabled: true
+        host.cpu.model.name:
+          enabled: true
+        host.cpu.stepping:
+          enabled: true
+        host.cpu.cache.l2.size:
+          enabled: true
+        os.description:
+          enabled: true
+        os.type:
+          enabled: true
+
+exporters:
+  # Exporter to print the first 5 logs/metrics and then every 1000th
+  debug:
+    verbosity: detailed
+    sampling_initial: 5
+    sampling_thereafter: 1000
+
+# Exporter to send logs and metrics to Elasticsearch Managed OTLP Input
+  otlp/ingest:
+    endpoint: ${env:ELASTIC_OTLP_ENDPOINT}
+    headers:
+      Authorization: ${env:ELASTIC_API_KEY}
+
+service:
+  extensions: [file_storage]
+  pipelines:
+    logs/platformlogs:
+      receivers: [filelog/platformlogs]
+      processors: [resourcedetection]
+      exporters: [debug, otlp/ingest]

internal/pkg/otel/samples/darwin/managed_otlp/platformlogs_hostmetrics.yml

@@ -0,0 +1,91 @@
+receivers:
+  # Receiver for platform specific log files
+  filelog/platformlogs:
+    include: [ /var/log/*.log ]
+    retry_on_failure:
+      enabled: true
+    start_at: end
+    storage: file_storage
+#   start_at: beginning
+
+  # Receiver for CPU, Disk, Memory, and Filesystem metrics
+  hostmetrics/system:
+    collection_interval: 30s
+    scrapers:
+      filesystem:
+      memory:
+        metrics:
+          system.memory.utilization:
+            enabled: true
+      process:
+        mute_process_exe_error: true
+        mute_process_io_error: true
+        mute_process_user_error: true
+        metrics:
+          process.threads:
+            enabled: true
+          process.open_file_descriptors:
+            enabled: true
+          process.memory.utilization:
+            enabled: true
+          process.disk.operations:
+            enabled: true
+      network:
+      processes:
+      load:
+
+extensions:
+  file_storage:
+    directory: ${env:STORAGE_DIR}
+
+processors:
+  resourcedetection:
+    detectors: ["system"]
+    system:
+      hostname_sources: ["os"]
+      resource_attributes:
+        host.name:
+          enabled: true
+        host.id:
+          enabled: false
+        host.arch:
+          enabled: true
+        host.ip:
+          enabled: true
+        host.mac:
+          enabled: true
+        host.cpu.vendor.id:
+          enabled: true
+        host.cpu.family:
+          enabled: true
+        host.cpu.model.id:
+          enabled: true
+        host.cpu.model.name:
+          enabled: true
+        host.cpu.stepping:
+          enabled: true
+        host.cpu.cache.l2.size:
+          enabled: true
+        os.description:
+          enabled: true
+        os.type:
+          enabled: true
+
+exporters:
+  # Exporter to send logs and metrics to Elasticsearch Managed OTLP Input
+  otlp/ingest:
+    endpoint: ${env:ELASTIC_OTLP_ENDPOINT}
+    headers:
+      Authorization: ${env:ELASTIC_API_KEY}
+
+service:
+  extensions: [file_storage]
+  pipelines:
+    metrics/hostmetrics:
+      receivers: [hostmetrics/system]
+      processors: [resourcedetection]
+      exporters: [otlp/ingest]
+    logs/platformlogs:
+      receivers: [filelog/platformlogs]
+      processors: [resourcedetection]
+      exporters: [otlp/ingest]