Oil_data_back/docker-compose.yml at main · GLec1er/Oil_data_back · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
name: oil-data

x-pipeline: &pipeline
  build:
    context: .
    dockerfile: Dockerfile.pipeline
  image: oil-data-pipeline:local
  environment:
    KAFKA_BOOTSTRAP: kafka:9092
    TELEMETRY_TOPIC: telemetry.raw
    DATA_DIR: /data
    TRIGGER_SECONDS: "10"
  volumes:
    - pipeline-data:/data

services:
  kafka:
    image: apache/kafka:3.7.0
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
      KAFKA_LOG_RETENTION_HOURS: 168
    volumes:
      - kafka-data:/var/lib/kafka/data
    healthcheck:
      test: ["CMD-SHELL", "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1"]
      interval: 5s
      timeout: 10s
      retries: 30

  kafka-init:
    image: apache/kafka:3.7.0
    depends_on:
      kafka:
        condition: service_healthy
    entrypoint: ["/bin/sh", "-c"]
    command:
      - >
        /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:9092 --create
        --if-not-exists --topic telemetry.raw --partitions 1 --replication-factor 1
        --config retention.ms=604800000
    restart: "no"

  raw:
    <<: *pipeline
    command: ["python", "-m", "pipeline.raw_stream"]
    depends_on:
      kafka-init:
        condition: service_completed_successfully
    restart: unless-stopped

  materializer:
    <<: *pipeline
    command: ["python", "-m", "pipeline.materializer"]
    depends_on:
      kafka-init:
        condition: service_completed_successfully
    restart: unless-stopped

  api:
    build: .
    image: oil-data-api:local
    environment:
      MARTS_DIR: /data/marts
    volumes:
      - pipeline-data:/data:ro  # read-only: the API never writes marts
    ports:
      - "${API_PORT:-8000}:8000"
    restart: unless-stopped

  cli:
    <<: *pipeline
    profiles: ["tools"]
    entrypoint: ["python", "-m", "pipeline"]
    depends_on:
      kafka-init:
        condition: service_completed_successfully

volumes:
  kafka-data:
  pipeline-data: