Update Docker compose

Author: VuBacktracking

kafka-docker-compose.yaml

@@ -34,22 +34,17 @@ services:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
-      # Define how clients connect to brokers
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      # Schema Registry URL for storing and managing Avro schemas
       KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081
-      # Confluent Metrics Reporter for Control Center Cluster Monitoring
       KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
       CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: "broker:9092"
       CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
       CONFLUENT_METRICS_ENABLE: "true"
-      # For fixing the bug replication factor 3 > the number of node
       KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
 
-  # For managing Avro schemas
   schema-registry:
     image: confluentinc/cp-schema-registry:7.5.0
     container_name: streaming-schema-registry
@@ -67,44 +62,42 @@ services:
       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "broker:29092"
       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
 
-# For connecting to offline store (Connect Kafka to database)
-# connect:
-  image: confluentinc/cp-kafka-connect:7.5.0
-  container_name: streaming-connect
-  depends_on:
-    broker:
-      condition: service_healthy
-    schema-registry:
-      condition: service_healthy
-    zookeeper:
-      condition: service_healthy
-  ports:
-    - "8083:8083"
-  environment:
-    CONNECT_BOOTSTRAP_SERVERS: "broker:29092"
-    CONNECT_REST_ADVERTISED_HOST_NAME: connect
-    CONNECT_REST_PORT: 8083
-    CONNECT_GROUP_ID: compose-connect-group
-    CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
-    CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
-    CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
-    CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
-    CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
-    CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
-    CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
-    CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
-    CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: false
-    CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
-    CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: true
-    # CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
-    # CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
-    # CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
-    # CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
-    CONNECT_PLUGIN_PATH: "/usr/share/java,/etc/kafka-connect/jars"
-  volumes:
-    - $PWD/data_ingestion/kafka_connect/jars/:/etc/kafka-connect/jars
+  connect:
+    image: confluentinc/cp-kafka-connect:7.5.0
+    container_name: streaming-connect
+    depends_on:
+      broker:
+        condition: service_healthy
+      schema-registry:
+        condition: service_healthy
+      zookeeper:
+        condition: service_healthy
+    ports:
+      - "8083:8083"
+    environment:
+      CONNECT_BOOTSTRAP_SERVERS: "broker:29092"
+      CONNECT_REST_ADVERTISED_HOST_NAME: connect
+      CONNECT_REST_PORT: 8083
+      CONNECT_GROUP_ID: compose-connect-group
+      CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
+      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
+      CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
+      CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
+      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
+      CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
+      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
+      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
+      CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: false
+      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
+      CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: true
+      # CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
+      # CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
+      # CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
+      # CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
+      CONNECT_PLUGIN_PATH: "/usr/share/java,/etc/kafka-connect/jars"
+    volumes:
+      - $PWD/data_ingestion/kafka_connect/jars/:/etc/kafka-connect/jars
 
-  # Confluent control center to manage Kafka
   control-center:
     image: confluentinc/cp-enterprise-control-center:7.5.0
     container_name: streaming-control-center
@@ -127,7 +120,7 @@ services:
       CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
       CONTROL_CENTER_CONNECT_HEALTHCHECK_ENDPOINT: "/connectors"
       CONFLUENT_METRICS_TOPIC_REPLICATION: 1
- # CDC platform to capture changes in DB and stream them to Kafka
+
   debezium:
     image: debezium/connect:1.9
     container_name: streaming-debezium
@@ -154,22 +147,19 @@ services:
       timeout: 5s
       retries: 5
     ports:
-      - "8083:8083"
+      - "8086:8083"
     environment:
       BOOTSTRAP_SERVERS: broker:29092
       GROUP_ID: 1
       CONFIG_STORAGE_TOPIC: connect_configs
       OFFSET_STORAGE_TOPIC: connect_offsets
-      # Set to Avro for higher performance
-      # KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
-      # VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
       KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
       VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
       CONNECT_TOPIC_CREATION_ENABLE: true
 
-  # Debezium UI
+
   debezium-ui:
     image: debezium/debezium-ui:latest
     container_name: debezium-ui
@@ -183,8 +173,6 @@ services:
       KAFKA_CONNECT_URIS: http://debezium:8083
 
   postgresql:
-    # Set wal_level from replica (default) to logical
-    # so that Debezium can capture change
     image: postgres:latest
     command: ["postgres", "-c", "wal_level=logical"]
     container_name: streaming-postgresql
@@ -196,11 +184,11 @@ services:
     ports:
       - "5432:5432"
     environment:
-      - POSTGRES_DB=v9
-      - POSTGRES_USER=v9
-      - POSTGRES_PASSWORD=v9
+      - POSTGRES_DB=my_database
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
     volumes:
       - cdc_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  cdc_postgres_data:
+  cdc_postgres_data:

storage-docker-compose.yaml

@@ -1,39 +1,33 @@
-version: "3.7"
+version: "3.8"
 services:
-  # We use PostgreSQL to store Hive metadata about
-  # how the datafile are mapped to schemas and tables
   metastore_db:
     container_name: metastoredb-datalake
     image: postgres:11
     hostname: metastore_db
     ports:
-      - "5433:5432" # Access via Thrift protocol
+      - "5433:5432"
     environment:
       POSTGRES_USER: hive
       POSTGRES_PASSWORD: hive
       POSTGRES_DB: metastore
 
-  # Expose service to get metadata, which is a repository of metadata about the tables,
-  # such as database names, table names, schema and data location of each table
   hive-metastore:
     container_name: datalake-hive-metastore
     image: "starburstdata/hive:3.1.2-e.18"
     hostname: hive-metastore
     ports:
-      - "9083:9083" # Access via Thrift protocol
+      - "9083:9083"
     environment:
       HIVE_METASTORE_DRIVER: org.postgresql.Driver
       HIVE_METASTORE_JDBC_URL: jdbc:postgresql://metastore_db:5432/metastore
       HIVE_METASTORE_USER: hive
       HIVE_METASTORE_PASSWORD: hive
-      HIVE_METASTORE_WAREHOUSE_DIR: s3://datalake/ # HDFS config, we don't need it
-      HIVE_METASTORE_USERS_IN_ADMIN_ROLE: "admin" # We also don't need it
+      HIVE_METASTORE_WAREHOUSE_DIR: s3://datalake/
+      HIVE_METASTORE_USERS_IN_ADMIN_ROLE: "admin"
       S3_ENDPOINT: http://minio:9000
       S3_ACCESS_KEY: minio_access_key
       S3_SECRET_KEY: minio_secret_key
       S3_PATH_STYLE_ACCESS: "true"
-      # Below arguments exist for no reasons, but
-      # we can not live without it
       REGION: ""
       GOOGLE_CLOUD_KEY_FILE_PATH: ""
       AZURE_ADL_CLIENT_ID: ""
@@ -52,19 +46,19 @@ services:
       - metastore_db
 
   trino:
-    ports:
-      - "8084:8080"
     container_name: trinodb-datalake
     image: "trinodb/trino:410"
     hostname: trino
+    ports:
+      - "8084:8080"
     volumes:
-      - ./trino/etc:usr/lib/trino/rtc:ro
-      - ./trino/catalog:etc/trino/catalog
+      - /home/vuphan/stream-data-processing/trino/etc:/usr/lib/trino/etc:ro
+      - /home/vuphan/stream-data-processing/trino/catalog:/etc/trino/catalog
     depends_on:
       - hive-metastore
 
   minio:
-    image: mino/minio
+    image: minio/minio
     container_name: minio-datalake
     hostname: minio
     ports:
@@ -73,11 +67,9 @@ services:
     volumes:
       - minio_storage:/data
     environment:
-      - MINIO_ACCESS_KEY: minio_access_key
-      - MINIO_SECRET_KEY: minio_secret_key
+      - MINIO_ACCESS_KEY=minio_access_key
+      - MINIO_SECRET_KEY=minio_secret_key
     command: server --console-address ":9001" /data
 
 volumes:
   minio_storage:
-  data:
-    driver: local