Claude/explain codebase mmljhjn4c3wypjfr kt7 lf

docker/playground/compose-debug.yml

@@ -0,0 +1,106 @@
+# Minimal debug deployment for Compass
+# Usage:
+#   1. Build: mvn clean package -DskipTests -Pdist
+#   2. Copy:  cp dist/compass-*.tar.gz docker/playground/
+#   3. Start deps:   docker compose -f compose-debug.yml --profile dependencies up -d
+#   4. Start compass: docker compose -f compose-debug.yml --profile compass-debug up -d --build
+#   5. Web UI: http://127.0.0.1:7075/compass  (compass/compass)
+#
+# Debug ports (JPDA remote debug):
+#   - task-portal:   5005
+#   - task-collector: 5006
+#   - task-analyzer: 5007
+#
+# Minimal services started:
+#   - task-portal    (Web UI + REST API)
+#   - task-collector (data collection: merged task-application + task-metadata)
+#   - task-analyzer  (diagnosis: merged task-detect + task-parser)
+#
+# Skipped services (not needed for basic debugging):
+#   - task-canal / task-canal-adapter (CDC from scheduler DB)
+#   - task-syncer (scheduler data sync)
+#   - task-flink  (Flink diagnostics)
+#   - task-gpt    (ChatGPT integration)
+
+services:
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-12}
+    profiles: ["dependencies"]
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: compass
+    container_name: compass-debug-postgres
+    hostname: postgres
+    ports:
+      - 5432:5432
+    volumes:
+      - ./script/compass_postgresql.sql:/docker-entrypoint-initdb.d/compass_postgresql.sql
+
+  kafka:
+    image: bitnami/kafka:3.6.1
+    profiles: ["dependencies"]
+    environment:
+      - KAFKA_CFG_NODE_ID=1
+      - KAFKA_CFG_PROCESS_ROLES=broker,controller
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+      - ALLOW_PLAINTEXT_LISTENER=yes
+      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
+    container_name: compass-debug-kafka
+    hostname: kafka
+    ports:
+      - 9092:9092
+
+  redis:
+    image: redis:7.2.1
+    profiles: ["dependencies"]
+    container_name: compass-debug-redis
+    hostname: redis
+    ports:
+      - 6379:6379
+
+  opensearch:
+    image: opensearchproject/opensearch:2.13.0
+    profiles: ["dependencies"]
+    environment:
+      - cluster.name=opensearch-cluster
+      - node.name=opensearch
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - "OPENSEARCH_JAVA_OPTS=-Xms256m -Xmx256m"
+      - "DISABLE_INSTALL_DEMO_CONFIG=true"
+      - "DISABLE_SECURITY_PLUGIN=true"
+    container_name: compass-debug-opensearch
+    hostname: opensearch
+    ports:
+      - 9200:9200
+
+  compass-debug:
+    build:
+      context: ./
+      dockerfile: dockerfiles/compass-debug.Dockerfile
+    ports:
+      - "7075:7075"   # portal web UI
+      - "7070:7070"   # collector
+      - "7071:7071"   # analyzer
+      - "5005:5005"   # debug: portal
+      - "5006:5006"   # debug: collector
+      - "5007:5007"   # debug: analyzer
+    environment:
+      - TASK_CANAL_ENABLE=False
+      - COMPASS_DEBUG=true
+      - JAVA_DEBUG_PORT_PORTAL=5005
+      - JAVA_DEBUG_PORT_COLLECTOR=5006
+      - JAVA_DEBUG_PORT_ANALYZER=5007
+    tty: true
+    container_name: compass-debug
+    profiles: ["compass-debug"]
+    depends_on:
+      - postgres
+      - kafka
+      - redis
+      - opensearch

docker/playground/compose.yml

@@ -14,66 +14,39 @@ services:
       - ./script/compass_postgresql.sql:/docker-entrypoint-initdb.d/compass_postgresql.sql
       - ./script/ds_postgresql.sql:/docker-entrypoint-initdb.d/ds_postgresql.sql
 
-  zookeeper:
-    image: zookeeper:${ZOOKEEPER_VERSION}
-    profiles: ["dependencies"]
-    ports:
-      - 2181
-    container_name: zookeeper
-    hostname: zookeeper
+  # ZooKeeper removed: Kafka now uses KRaft mode (built-in Raft, no external coordinator needed)
 
   kafka:
-    image: bitnami/kafka:latest
+    image: bitnami/kafka:3.6.1
     profiles: ["dependencies"]
     environment:
-      - KAFKA_CFG_NODE_ID=0
-      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
-      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
+      # KRaft mode: broker + controller roles in one process, no ZooKeeper dependency
+      - KAFKA_CFG_NODE_ID=1
+      - KAFKA_CFG_PROCESS_ROLES=broker,controller
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
       - ALLOW_PLAINTEXT_LISTENER=yes
       - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
     container_name: kafka
     hostname: kafka
     ports:
       - 9092:9092
-    depends_on:
-      - zookeeper
 
-  redis1:
+  # Redis cluster (3 nodes) replaced with single-node Redis.
+  # Cluster mode was only needed for Redis-as-queue (detect→parser handoff).
+  # After merging task-detect+task-parser into task-analyzer, Redis is used
+  # only for Portal query caching — single node is sufficient.
+  redis:
     image: redis:7.2.1
     profiles: ["dependencies"]
-    command: ["sh", "-c", "redis-server --port 6379 --cluster-enabled yes --cluster-config-file nodes.conf"]
-    container_name: redis1
-    hostname: redis1
+    container_name: redis
+    hostname: redis
     ports:
       - 6379:6379
 
-  redis2:
-    image: redis:7.2.1
-    profiles: ["dependencies"]
-    command: ["sh", "-c", "redis-server --port 6380 --cluster-enabled yes --cluster-config-file nodes.conf"]
-    container_name: redis2
-    hostname: redis2
-    ports:
-      - 6380:6380
-
-  redis3:
-    image: redis:7.2.1
-    profiles: ["dependencies"]
-    command: ["sh", "-c", "redis-server --port 6381 --cluster-enabled yes --cluster-config-file nodes.conf"]
-    container_name: redis3
-    hostname: redis3
-    ports:
-      - 6381:6381
-
-  redis-cluster:
-    image: redis:7.2.1
-    profiles: ["dependencies"]
-    command: ["sh", "-c", "sleep 10 && redis-cli -h redis1 -p 6379 --cluster create redis1:6379 redis2:6380 redis3:6381 --cluster-replicas 0 --cluster-yes"]
-    depends_on:
-      - redis1
-      - redis2
-      - redis3
-
   # elasticsearch:
   #   image: elasticsearch:8.10.2
   #   profiles: ["dependencies"]
@@ -88,7 +61,7 @@ services:
   #     - 9200:9200
 
   opensearch:
-    image: opensearchproject/opensearch:1.3.12
+    image: opensearchproject/opensearch:2.13.0
     profiles: ["dependencies"]
     environment:
       - cluster.name=opensearch-cluster

docker/playground/conf/compass_env.sh

@@ -22,8 +22,10 @@ export SPRING_DATASOURCE_PASSWORD="postgres"
 # Kafka
 export SPRING_KAFKA_BOOTSTRAPSERVERS="kafka:9092"
 
-# Redis
-export SPRING_REDIS_CLUSTER_NODES="redis1:6379,redis2:6380,redis3:6381"
+# Redis (single node, matching compose.yml)
+export REDIS_HOST="redis"
+export REDIS_PORT="6379"
+export SPRING_REDIS_CLUSTER_NODES="redis:6379"
 # Optional
 export SPRING_REDIS_PASSWORD=""
 

docker/playground/conf/compass_env_debug.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# =============================================================================
+# Minimal Debug Deployment Configuration
+# Only essential services: portal, collector, analyzer
+# Canal/Syncer/Flink/GPT disabled
+# =============================================================================
+
+# Scheduler type (not actively used in debug mode without Canal)
+export SCHEDULER="custom"
+export SPRING_PROFILES_ACTIVE="hadoop"
+
+# Compass database (PostgreSQL)
+export DATASOURCE_TYPE="postgresql"
+export COMPASS_DATASOURCE_ADDRESS="postgres:5432"
+export COMPASS_DATASOURCE_DB="compass"
+export SPRING_DATASOURCE_URL="jdbc:postgresql://${COMPASS_DATASOURCE_ADDRESS}/${COMPASS_DATASOURCE_DB}"
+export SPRING_DATASOURCE_USERNAME="postgres"
+export SPRING_DATASOURCE_PASSWORD="postgres"
+
+# Kafka (KRaft, single broker)
+export SPRING_KAFKA_BOOTSTRAPSERVERS="kafka:9092"
+
+# Redis (single node)
+export REDIS_HOST="redis"
+export REDIS_PORT="6379"
+export SPRING_REDIS_CLUSTER_NODES="redis:6379"
+export SPRING_REDIS_PASSWORD=""
+
+# OpenSearch (single node, security disabled)
+export SPRING_OPENSEARCH_NODES="opensearch:9200"
+export SPRING_OPENSEARCH_USERNAME=""
+export SPRING_OPENSEARCH_PASSWORD=""
+export SPRING_OPENSEARCH_TRUSTSTORE=""
+export SPRING_OPENSEARCH_TRUSTSTOREPASSWORD=""
+
+# Spark compression codec (empty = no compression)
+export SPARK_IO_COMPRESSION_CODEC=""
+
+# ChatGPT disabled
+export CHATGPT_ENABLE=false
+
+# Canal disabled (no scheduler DB sync in debug mode)
+export TASK_CANAL_ENABLE="False"
+
+# =============================================================================
+# Environment variables for new merged services
+# =============================================================================
+# task-collector (merged task-application + task-metadata)
+export DB_URL="${SPRING_DATASOURCE_URL}"
+export DB_USERNAME="${SPRING_DATASOURCE_USERNAME}"
+export DB_PASSWORD="${SPRING_DATASOURCE_PASSWORD}"
+export KAFKA_BOOTSTRAP_SERVERS="${SPRING_KAFKA_BOOTSTRAPSERVERS}"
+export OPENSEARCH_NODES="${SPRING_OPENSEARCH_NODES}"
+export OPENSEARCH_USERNAME="${SPRING_OPENSEARCH_USERNAME}"
+export OPENSEARCH_PASSWORD="${SPRING_OPENSEARCH_PASSWORD}"
+
+# JWT secret for portal (debug only, not for production)
+export JWT_SECRET="compass-debug-secret-key-minimum-32-chars"

docker/playground/conf/start_debug.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+# Minimal debug startup script
+# Only starts: task-portal, task-collector, task-analyzer
+# Enables JPDA remote debugging on ports 5005/5006/5007
+
+HOME_DIR=$(cd $(dirname $0)/.. && pwd)
+ENV_SH=$(dirname $0)/compass_env.sh
+
+if [ -f ${ENV_SH} ]; then
+  source ${ENV_SH}
+  for dir in ${HOME_DIR}/task-portal ${HOME_DIR}/task-collector ${HOME_DIR}/task-analyzer; do
+    if [ -d $dir ]; then
+      cp ${ENV_SH} $dir/bin/
+    fi
+  done
+fi
+
+# Copy hadoop conf to modules that need it
+for dir in task-collector task-analyzer; do
+  if [ -d ${HOME_DIR}/${dir}/conf ] && [ -f ${HOME_DIR}/conf/application-hadoop.yml ]; then
+    cp ${HOME_DIR}/conf/application-hadoop.yml ${HOME_DIR}/${dir}/conf/
+  fi
+done
+
+# Wait for dependencies to be ready
+echo "Waiting for PostgreSQL..."
+retry -t 30 -d 2 -- bash -c "echo 'SELECT 1' | busybox nc -w 1 postgres 5432 >/dev/null 2>&1" 2>/dev/null || echo "PostgreSQL may not be ready, continuing..."
+
+echo "Waiting for Kafka..."
+retry -t 15 -d 2 -- bash -c "busybox nc -w 1 kafka 9092 >/dev/null 2>&1" 2>/dev/null || echo "Kafka may not be ready, continuing..."
+
+echo "Waiting for Redis..."
+retry -t 15 -d 2 -- bash -c "busybox nc -w 1 redis 6379 >/dev/null 2>&1" 2>/dev/null || echo "Redis may not be ready, continuing..."
+
+echo "Waiting for OpenSearch..."
+retry -t 30 -d 2 -- bash -c "busybox nc -w 1 opensearch 9200 >/dev/null 2>&1" 2>/dev/null || echo "OpenSearch may not be ready, continuing..."
+
+echo ""
+echo "============================================"
+echo " Starting Compass (Minimal Debug Mode)"
+echo "============================================"
+
+# Debug JVM options (JPDA remote debug, suspend=n so services start immediately)
+JAVA_DEBUG_OPTS_PORTAL="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:${JAVA_DEBUG_PORT_PORTAL:-5005}"
+JAVA_DEBUG_OPTS_COLLECTOR="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:${JAVA_DEBUG_PORT_COLLECTOR:-5006}"
+JAVA_DEBUG_OPTS_ANALYZER="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:${JAVA_DEBUG_PORT_ANALYZER:-5007}"
+
+JAVA_OPTS_COMMON="-Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom"
+JAVA_OPTS_GC="-server -XX:+UseG1GC -XX:G1HeapRegionSize=8m"
+
+start_service() {
+  local name=$1
+  local main_class=$2
+  local debug_opts=$3
+  local dir=${HOME_DIR}/${name}
+
+  if [ ! -d ${dir} ]; then
+    echo "[SKIP] ${name}: directory not found"
+    return
+  fi
+
+  mkdir -p ${dir}/logs
+
+  echo "[START] ${name} (debug: ${debug_opts})"
+  nohup java \
+    -DappName=${name} \
+    ${JAVA_OPTS_COMMON} \
+    ${JAVA_OPTS_GC} \
+    ${debug_opts} \
+    -cp "${dir}/conf":"${dir}/lib/*" \
+    ${main_class} \
+    >${dir}/logs/stdout.log 2>&1 &
+
+  local pid=$!
+  echo $pid > ${dir}/tpid
+  echo "[OK] ${name} started (pid=${pid})"
+}
+
+# Start only the 3 core services
+start_service "task-portal"    "com.oppo.cloud.portal.TaskPortalApplication"       "${JAVA_DEBUG_OPTS_PORTAL}"
+start_service "task-collector" "com.oppo.cloud.collector.TaskCollectorApplication"  "${JAVA_DEBUG_OPTS_COLLECTOR}"
+start_service "task-analyzer"  "com.oppo.cloud.analyzer.TaskAnalyzerApplication"   "${JAVA_DEBUG_OPTS_ANALYZER}"
+
+echo ""
+echo "============================================"
+echo " Compass Debug Mode Started"
+echo "--------------------------------------------"
+echo " Web UI:          http://localhost:7075/compass"
+echo " Login:           compass / compass"
+echo "--------------------------------------------"
+echo " Debug Ports (JPDA):"
+echo "   task-portal:    5005"
+echo "   task-collector: 5006"
+echo "   task-analyzer:  5007"
+echo "============================================"