docker-compose.yml

# Observability Stack
# Docker Compose configuration for local development
# WARNING: This configuration is for development/testing only - not production-ready

include:
  - path: ${INCLUDE_COMPOSE_EXAMPLES:-docker-compose/util/docker-compose.empty.yml}
  - path: ${INCLUDE_COMPOSE_OTEL_DEMO:-docker-compose/util/docker-compose.empty.yml}
  - path: ${INCLUDE_COMPOSE_LOCAL_OPENSEARCH:-docker-compose/util/docker-compose.empty.yml}
  - path: ${INCLUDE_COMPOSE_LOCAL_OPENSEARCH_DASHBOARDS:-docker-compose/util/docker-compose.empty.yml}

x-default-logging: &logging
  driver: "json-file"
  options:
    max-size: "5m"
    max-file: "2"
    tag: "{{.Name}}"

networks:
  observability-stack-network:
    name: observability-stack-network
    driver: bridge

volumes:
  prometheus-data:
    driver: local

services:
  # OpenTelemetry Collector - Receives telemetry data via OTLP protocol
  otel-collector:
    image: otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION}
    container_name: otel-collector
    pull_policy: always
    command: ["--config=/etc/otelcol-config.yml"]
    volumes:
      - ./docker-compose/otel-collector/config.yaml:/etc/otelcol-config.yml
    ports:
      # OTLP gRPC receiver - high-performance binary protocol
      - "${OTEL_COLLECTOR_PORT_GRPC}:4317"
      # OTLP HTTP receiver - easier debugging and browser compatibility
      - "${OTEL_COLLECTOR_PORT_HTTP}:4318"
      # Metrics endpoint for collector self-monitoring
      - "${OTEL_COLLECTOR_METRICS_PORT}:8888"
    networks:
      - observability-stack-network
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: ${OTEL_COLLECTOR_MEMORY_LIMIT}
    environment:
      - OTEL_COLLECTOR_HOST=${OTEL_COLLECTOR_HOST}
      - OTEL_COLLECTOR_PORT_GRPC=${OTEL_COLLECTOR_PORT_GRPC}
      - OTEL_COLLECTOR_PORT_HTTP=${OTEL_COLLECTOR_PORT_HTTP}
      - OPENSEARCH_HOST=${OPENSEARCH_HOST}
      - OPENSEARCH_PORT=${OPENSEARCH_PORT}
      - GOMEMLIMIT=160MiB
    depends_on:
      opensearch:
        condition: service_healthy
        required: false
    logging: *logging

  # Data Prepper - Transforms and enriches logs/traces before OpenSearch ingestion
  data-prepper:
    image: ${DATA_PREPPER_DOCKER_REPO}/opensearch-data-prepper:${DATA_PREPPER_VERSION}
    container_name: data-prepper
    pull_policy: always
    platform: linux/amd64
    command: >
      /bin/bash -c "
      cp /tmp/pipelines.template.yaml /tmp/pipelines.yaml &&
      chmod +w /tmp/pipelines.yaml &&
      sed -i 's|OPENSEARCH_USER|${OPENSEARCH_USER}|g' /tmp/pipelines.yaml &&
      sed -i 's|OPENSEARCH_PASSWORD|${OPENSEARCH_PASSWORD}|g' /tmp/pipelines.yaml &&
      sed -i 's|OPENSEARCH_PROTOCOL|${OPENSEARCH_PROTOCOL}|g' /tmp/pipelines.yaml &&
      sed -i 's|OPENSEARCH_HOST|${OPENSEARCH_HOST}|g' /tmp/pipelines.yaml &&
      sed -i 's|OPENSEARCH_PORT|${OPENSEARCH_PORT}|g' /tmp/pipelines.yaml &&
      mv /tmp/pipelines.yaml /usr/share/data-prepper/pipelines/pipelines.yaml &&
      exec /usr/share/data-prepper/bin/data-prepper"
    volumes:
      - ./docker-compose/data-prepper/pipelines.template.yaml:/tmp/pipelines.template.yaml
      - ./docker-compose/data-prepper/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml
    ports:
      # OTLP gRPC receiver from OpenTelemetry Collector
      - "${DATA_PREPPER_OTLP_PORT}:21890"
      # OTLP HTTP receiver
      - "${DATA_PREPPER_HTTP_PORT}:21892"
    environment:
      - OPENSEARCH_HOST=${OPENSEARCH_HOST}
      - OPENSEARCH_PORT=${OPENSEARCH_PORT}
      - OPENSEARCH_USER=${OPENSEARCH_USER}
      - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
    depends_on:
      opensearch:
        condition: service_healthy
        required: false
    networks:
      - observability-stack-network
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: ${DATA_PREPPER_MEMORY_LIMIT}
    logging: *logging

  # Prometheus - Time-series database for metrics storage
  prometheus:
    image: prom/prometheus:${PROMETHEUS_VERSION}
    container_name: prometheus
    pull_policy: always
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      # Retention period from environment variable
      - '--storage.tsdb.retention.time=${PROMETHEUS_RETENTION}'
      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
      - '--web.console.templates=/usr/share/prometheus/consoles'
      # Enable remote write receiver for OpenTelemetry Collector
      - '--web.enable-remote-write-receiver'
      - '--web.enable-lifecycle'
      - '--web.route-prefix=/'
      - '--enable-feature=exemplar-storage'
      - '--web.enable-otlp-receiver'
    volumes:
      - ./docker-compose/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      # Persist metrics data across container restarts
      - prometheus-data:/prometheus
    ports:
      # Web UI and API endpoint
      - "${PROMETHEUS_PORT}:9090"
    networks:
      - observability-stack-network
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: ${PROMETHEUS_MEMORY_LIMIT}
    logging: *logging

  # OpenSearch Prometheus Exporter - Exposes OpenSearch metrics for Prometheus scraping
  opensearch-exporter:
    image: prometheuscommunity/elasticsearch-exporter:v1.10.0
    container_name: opensearch-exporter
    command:
      - --es.uri=${OPENSEARCH_PROTOCOL}://${OPENSEARCH_HOST}:${OPENSEARCH_PORT}
      - --es.ssl-skip-verify
      - --es.all
      - --es.indices
      - --es.shards
    environment:
      - ES_USERNAME=${OPENSEARCH_USER}
      - ES_PASSWORD=${OPENSEARCH_PASSWORD}
    ports:
      - "9114:9114"
    networks:
      - observability-stack-network
    depends_on:
      opensearch:
        condition: service_healthy
        required: false
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 128M
    logging: *logging

  # OpenSearch Dashboards Initialization - Creates workspace, index patterns, and saved queries
  opensearch-dashboards-init:
    image: python:3.11-alpine
    container_name: opensearch-dashboards-init
    command: sh -c "pip install requests pyyaml && python /init.py"
    environment:
      - OPENSEARCH_USER=${OPENSEARCH_USER}
      - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
      - OPENSEARCH_HOST=${OPENSEARCH_HOST}
      - OPENSEARCH_PORT=${OPENSEARCH_PORT}
      - OPENSEARCH_PROTOCOL=${OPENSEARCH_PROTOCOL}
      - OPENSEARCH_DASHBOARDS_HOST=${OPENSEARCH_DASHBOARDS_HOST}
      - OPENSEARCH_DASHBOARDS_PORT=${OPENSEARCH_DASHBOARDS_PORT}
      - OPENSEARCH_DASHBOARDS_PROTOCOL=${OPENSEARCH_DASHBOARDS_PROTOCOL}
      - PROMETHEUS_HOST=${PROMETHEUS_HOST}
      - PROMETHEUS_PORT=${PROMETHEUS_PORT}
    volumes:
      - ./docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py:/init.py
      - ./docker-compose/opensearch-dashboards/saved-queries-traces.yaml:/config/saved-queries-traces.yaml
      - ./docker-compose/opensearch-dashboards/saved-queries-metrics.yaml:/config/saved-queries-metrics.yaml
      - ./docker-compose/opensearch-dashboards/dashboard-opensearch-health.yaml:/config/dashboard-opensearch-health.yaml
      - ./docker-compose/opensearch-dashboards/dashboard-pipeline-health.yaml:/config/dashboard-pipeline-health.yaml
      - ./docker-compose/opensearch-dashboards/init/architecture.png:/config/architecture.png
    networks:
      - observability-stack-network
    restart: "no"
    logging: *logging