diff --git a/docker/playground/compose-debug.yml b/docker/playground/compose-debug.yml new file mode 100644 index 000000000..76c8596f7 --- /dev/null +++ b/docker/playground/compose-debug.yml @@ -0,0 +1,106 @@ +# Minimal debug deployment for Compass +# Usage: +# 1. Build: mvn clean package -DskipTests -Pdist +# 2. Copy: cp dist/compass-*.tar.gz docker/playground/ +# 3. Start deps: docker compose -f compose-debug.yml --profile dependencies up -d +# 4. Start compass: docker compose -f compose-debug.yml --profile compass-debug up -d --build +# 5. Web UI: http://127.0.0.1:7075/compass (compass/compass) +# +# Debug ports (JPDA remote debug): +# - task-portal: 5005 +# - task-collector: 5006 +# - task-analyzer: 5007 +# +# Minimal services started: +# - task-portal (Web UI + REST API) +# - task-collector (data collection: merged task-application + task-metadata) +# - task-analyzer (diagnosis: merged task-detect + task-parser) +# +# Skipped services (not needed for basic debugging): +# - task-canal / task-canal-adapter (CDC from scheduler DB) +# - task-syncer (scheduler data sync) +# - task-flink (Flink diagnostics) +# - task-gpt (ChatGPT integration) + +services: + postgres: + image: postgres:${POSTGRES_VERSION:-12} + profiles: ["dependencies"] + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: compass + container_name: compass-debug-postgres + hostname: postgres + ports: + - 5432:5432 + volumes: + - ./script/compass_postgresql.sql:/docker-entrypoint-initdb.d/compass_postgresql.sql + + kafka: + image: bitnami/kafka:3.6.1 + profiles: ["dependencies"] + environment: + - KAFKA_CFG_NODE_ID=1 + - KAFKA_CFG_PROCESS_ROLES=broker,controller + - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 + - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093 + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER + - ALLOW_PLAINTEXT_LISTENER=yes + - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true + container_name: compass-debug-kafka + hostname: kafka + ports: + - 9092:9092 + + redis: + image: redis:7.2.1 + profiles: ["dependencies"] + container_name: compass-debug-redis + hostname: redis + ports: + - 6379:6379 + + opensearch: + image: opensearchproject/opensearch:2.13.0 + profiles: ["dependencies"] + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch + - discovery.type=single-node + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms256m -Xmx256m" + - "DISABLE_INSTALL_DEMO_CONFIG=true" + - "DISABLE_SECURITY_PLUGIN=true" + container_name: compass-debug-opensearch + hostname: opensearch + ports: + - 9200:9200 + + compass-debug: + build: + context: ./ + dockerfile: dockerfiles/compass-debug.Dockerfile + ports: + - "7075:7075" # portal web UI + - "7070:7070" # collector + - "7071:7071" # analyzer + - "5005:5005" # debug: portal + - "5006:5006" # debug: collector + - "5007:5007" # debug: analyzer + environment: + - TASK_CANAL_ENABLE=False + - COMPASS_DEBUG=true + - JAVA_DEBUG_PORT_PORTAL=5005 + - JAVA_DEBUG_PORT_COLLECTOR=5006 + - JAVA_DEBUG_PORT_ANALYZER=5007 + tty: true + container_name: compass-debug + profiles: ["compass-debug"] + depends_on: + - postgres + - kafka + - redis + - opensearch diff --git a/docker/playground/compose.yml b/docker/playground/compose.yml index 5e1a5a08f..ff9af177f 100644 --- a/docker/playground/compose.yml +++ b/docker/playground/compose.yml @@ -14,66 +14,39 @@ services: - ./script/compass_postgresql.sql:/docker-entrypoint-initdb.d/compass_postgresql.sql - ./script/ds_postgresql.sql:/docker-entrypoint-initdb.d/ds_postgresql.sql - zookeeper: - image: zookeeper:${ZOOKEEPER_VERSION} - profiles: ["dependencies"] - ports: - - 2181 - container_name: zookeeper - hostname: zookeeper + # ZooKeeper removed: Kafka now uses KRaft mode (built-in Raft, no external coordinator needed) kafka: - image: bitnami/kafka:latest + image: bitnami/kafka:3.6.1 profiles: ["dependencies"] environment: - - KAFKA_CFG_NODE_ID=0 - - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092 + # KRaft mode: broker + controller roles in one process, no ZooKeeper dependency + - KAFKA_CFG_NODE_ID=1 + - KAFKA_CFG_PROCESS_ROLES=broker,controller + - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 + - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093 + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true container_name: kafka hostname: kafka ports: - 9092:9092 - depends_on: - - zookeeper - redis1: + # Redis cluster (3 nodes) replaced with single-node Redis. + # Cluster mode was only needed for Redis-as-queue (detect→parser handoff). + # After merging task-detect+task-parser into task-analyzer, Redis is used + # only for Portal query caching — single node is sufficient. + redis: image: redis:7.2.1 profiles: ["dependencies"] - command: ["sh", "-c", "redis-server --port 6379 --cluster-enabled yes --cluster-config-file nodes.conf"] - container_name: redis1 - hostname: redis1 + container_name: redis + hostname: redis ports: - 6379:6379 - redis2: - image: redis:7.2.1 - profiles: ["dependencies"] - command: ["sh", "-c", "redis-server --port 6380 --cluster-enabled yes --cluster-config-file nodes.conf"] - container_name: redis2 - hostname: redis2 - ports: - - 6380:6380 - - redis3: - image: redis:7.2.1 - profiles: ["dependencies"] - command: ["sh", "-c", "redis-server --port 6381 --cluster-enabled yes --cluster-config-file nodes.conf"] - container_name: redis3 - hostname: redis3 - ports: - - 6381:6381 - - redis-cluster: - image: redis:7.2.1 - profiles: ["dependencies"] - command: ["sh", "-c", "sleep 10 && redis-cli -h redis1 -p 6379 --cluster create redis1:6379 redis2:6380 redis3:6381 --cluster-replicas 0 --cluster-yes"] - depends_on: - - redis1 - - redis2 - - redis3 - # elasticsearch: # image: elasticsearch:8.10.2 # profiles: ["dependencies"] @@ -88,7 +61,7 @@ services: # - 9200:9200 opensearch: - image: opensearchproject/opensearch:1.3.12 + image: opensearchproject/opensearch:2.13.0 profiles: ["dependencies"] environment: - cluster.name=opensearch-cluster diff --git a/docker/playground/conf/compass_env.sh b/docker/playground/conf/compass_env.sh index 54b7baae2..2e71d94b3 100644 --- a/docker/playground/conf/compass_env.sh +++ b/docker/playground/conf/compass_env.sh @@ -22,8 +22,10 @@ export SPRING_DATASOURCE_PASSWORD="postgres" # Kafka export SPRING_KAFKA_BOOTSTRAPSERVERS="kafka:9092" -# Redis -export SPRING_REDIS_CLUSTER_NODES="redis1:6379,redis2:6380,redis3:6381" +# Redis (single node, matching compose.yml) +export REDIS_HOST="redis" +export REDIS_PORT="6379" +export SPRING_REDIS_CLUSTER_NODES="redis:6379" # Optional export SPRING_REDIS_PASSWORD="" diff --git a/docker/playground/conf/compass_env_debug.sh b/docker/playground/conf/compass_env_debug.sh new file mode 100644 index 000000000..e938ee836 --- /dev/null +++ b/docker/playground/conf/compass_env_debug.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# ============================================================================= +# Minimal Debug Deployment Configuration +# Only essential services: portal, collector, analyzer +# Canal/Syncer/Flink/GPT disabled +# ============================================================================= + +# Scheduler type (not actively used in debug mode without Canal) +export SCHEDULER="custom" +export SPRING_PROFILES_ACTIVE="hadoop" + +# Compass database (PostgreSQL) +export DATASOURCE_TYPE="postgresql" +export COMPASS_DATASOURCE_ADDRESS="postgres:5432" +export COMPASS_DATASOURCE_DB="compass" +export SPRING_DATASOURCE_URL="jdbc:postgresql://${COMPASS_DATASOURCE_ADDRESS}/${COMPASS_DATASOURCE_DB}" +export SPRING_DATASOURCE_USERNAME="postgres" +export SPRING_DATASOURCE_PASSWORD="postgres" + +# Kafka (KRaft, single broker) +export SPRING_KAFKA_BOOTSTRAPSERVERS="kafka:9092" + +# Redis (single node) +export REDIS_HOST="redis" +export REDIS_PORT="6379" +export SPRING_REDIS_CLUSTER_NODES="redis:6379" +export SPRING_REDIS_PASSWORD="" + +# OpenSearch (single node, security disabled) +export SPRING_OPENSEARCH_NODES="opensearch:9200" +export SPRING_OPENSEARCH_USERNAME="" +export SPRING_OPENSEARCH_PASSWORD="" +export SPRING_OPENSEARCH_TRUSTSTORE="" +export SPRING_OPENSEARCH_TRUSTSTOREPASSWORD="" + +# Spark compression codec (empty = no compression) +export SPARK_IO_COMPRESSION_CODEC="" + +# ChatGPT disabled +export CHATGPT_ENABLE=false + +# Canal disabled (no scheduler DB sync in debug mode) +export TASK_CANAL_ENABLE="False" + +# ============================================================================= +# Environment variables for new merged services +# ============================================================================= +# task-collector (merged task-application + task-metadata) +export DB_URL="${SPRING_DATASOURCE_URL}" +export DB_USERNAME="${SPRING_DATASOURCE_USERNAME}" +export DB_PASSWORD="${SPRING_DATASOURCE_PASSWORD}" +export KAFKA_BOOTSTRAP_SERVERS="${SPRING_KAFKA_BOOTSTRAPSERVERS}" +export OPENSEARCH_NODES="${SPRING_OPENSEARCH_NODES}" +export OPENSEARCH_USERNAME="${SPRING_OPENSEARCH_USERNAME}" +export OPENSEARCH_PASSWORD="${SPRING_OPENSEARCH_PASSWORD}" + +# JWT secret for portal (debug only, not for production) +export JWT_SECRET="compass-debug-secret-key-minimum-32-chars" diff --git a/docker/playground/conf/start_debug.sh b/docker/playground/conf/start_debug.sh new file mode 100755 index 000000000..079579047 --- /dev/null +++ b/docker/playground/conf/start_debug.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Minimal debug startup script +# Only starts: task-portal, task-collector, task-analyzer +# Enables JPDA remote debugging on ports 5005/5006/5007 + +HOME_DIR=$(cd $(dirname $0)/.. && pwd) +ENV_SH=$(dirname $0)/compass_env.sh + +if [ -f ${ENV_SH} ]; then + source ${ENV_SH} + for dir in ${HOME_DIR}/task-portal ${HOME_DIR}/task-collector ${HOME_DIR}/task-analyzer; do + if [ -d $dir ]; then + cp ${ENV_SH} $dir/bin/ + fi + done +fi + +# Copy hadoop conf to modules that need it +for dir in task-collector task-analyzer; do + if [ -d ${HOME_DIR}/${dir}/conf ] && [ -f ${HOME_DIR}/conf/application-hadoop.yml ]; then + cp ${HOME_DIR}/conf/application-hadoop.yml ${HOME_DIR}/${dir}/conf/ + fi +done + +# Wait for dependencies to be ready +echo "Waiting for PostgreSQL..." +retry -t 30 -d 2 -- bash -c "echo 'SELECT 1' | busybox nc -w 1 postgres 5432 >/dev/null 2>&1" 2>/dev/null || echo "PostgreSQL may not be ready, continuing..." + +echo "Waiting for Kafka..." +retry -t 15 -d 2 -- bash -c "busybox nc -w 1 kafka 9092 >/dev/null 2>&1" 2>/dev/null || echo "Kafka may not be ready, continuing..." + +echo "Waiting for Redis..." +retry -t 15 -d 2 -- bash -c "busybox nc -w 1 redis 6379 >/dev/null 2>&1" 2>/dev/null || echo "Redis may not be ready, continuing..." + +echo "Waiting for OpenSearch..." +retry -t 30 -d 2 -- bash -c "busybox nc -w 1 opensearch 9200 >/dev/null 2>&1" 2>/dev/null || echo "OpenSearch may not be ready, continuing..." + +echo "" +echo "============================================" +echo " Starting Compass (Minimal Debug Mode)" +echo "============================================" + +# Debug JVM options (JPDA remote debug, suspend=n so services start immediately) +JAVA_DEBUG_OPTS_PORTAL="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:${JAVA_DEBUG_PORT_PORTAL:-5005}" +JAVA_DEBUG_OPTS_COLLECTOR="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:${JAVA_DEBUG_PORT_COLLECTOR:-5006}" +JAVA_DEBUG_OPTS_ANALYZER="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:${JAVA_DEBUG_PORT_ANALYZER:-5007}" + +JAVA_OPTS_COMMON="-Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom" +JAVA_OPTS_GC="-server -XX:+UseG1GC -XX:G1HeapRegionSize=8m" + +start_service() { + local name=$1 + local main_class=$2 + local debug_opts=$3 + local dir=${HOME_DIR}/${name} + + if [ ! -d ${dir} ]; then + echo "[SKIP] ${name}: directory not found" + return + fi + + mkdir -p ${dir}/logs + + echo "[START] ${name} (debug: ${debug_opts})" + nohup java \ + -DappName=${name} \ + ${JAVA_OPTS_COMMON} \ + ${JAVA_OPTS_GC} \ + ${debug_opts} \ + -cp "${dir}/conf":"${dir}/lib/*" \ + ${main_class} \ + >${dir}/logs/stdout.log 2>&1 & + + local pid=$! + echo $pid > ${dir}/tpid + echo "[OK] ${name} started (pid=${pid})" +} + +# Start only the 3 core services +start_service "task-portal" "com.oppo.cloud.portal.TaskPortalApplication" "${JAVA_DEBUG_OPTS_PORTAL}" +start_service "task-collector" "com.oppo.cloud.collector.TaskCollectorApplication" "${JAVA_DEBUG_OPTS_COLLECTOR}" +start_service "task-analyzer" "com.oppo.cloud.analyzer.TaskAnalyzerApplication" "${JAVA_DEBUG_OPTS_ANALYZER}" + +echo "" +echo "============================================" +echo " Compass Debug Mode Started" +echo "--------------------------------------------" +echo " Web UI: http://localhost:7075/compass" +echo " Login: compass / compass" +echo "--------------------------------------------" +echo " Debug Ports (JPDA):" +echo " task-portal: 5005" +echo " task-collector: 5006" +echo " task-analyzer: 5007" +echo "============================================" diff --git a/docker/playground/dockerfiles/compass-debug.Dockerfile b/docker/playground/dockerfiles/compass-debug.Dockerfile new file mode 100644 index 000000000..062246775 --- /dev/null +++ b/docker/playground/dockerfiles/compass-debug.Dockerfile @@ -0,0 +1,36 @@ +FROM eclipse-temurin:8-focal + +RUN set -x && \ + ln -snf /usr/bin/bash /usr/bin/sh && \ + apt-get update -q && \ + apt-get install -y tzdata && \ + apt-get install -yq retry busybox && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir /opt/busybox && \ + busybox --install /opt/busybox + +ENV PATH=/opt/java/openjdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/busybox + +ENV TASK_CANAL_ENABLE="False" +ENV COMPASS_DEBUG="true" +ENV TZ=Asia/Shanghai + +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo $TZ > /etc/timezone + +COPY compass-*.tar.gz /opt + +# Extract compass distribution +RUN tar -xzf /opt/compass-*.tar.gz -C /opt && \ + rm /opt/compass-*.tar.gz && \ + ln -s /opt/compass-* /opt/compass && \ + rm /opt/compass/bin/compass_env.sh + +COPY conf/compass_env_debug.sh /opt/compass/bin/compass_env.sh +COPY conf/start_debug.sh /opt/compass/bin/start_debug.sh +RUN chmod +x /opt/compass/bin/start_debug.sh + +# Expose service ports and debug ports +EXPOSE 7070 7071 7075 5005 5006 5007 + +CMD /opt/compass/bin/start_debug.sh && /bin/bash diff --git a/document/manual/architecture.md b/document/manual/architecture.md index e3a43e39a..649add477 100644 --- a/document/manual/architecture.md +++ b/document/manual/architecture.md @@ -19,11 +19,11 @@ The overall architecture is divided into 3 layers: Specific module process stages: -- Data Collection Stage: The task-canal/adapter module subscribes to synchronize the user, DAG, job, execution record and other workflow metadata of the scheduling system to the diagnostic platform; the task-metadata module periodically synchronizes Yarn ResourceManager, Spark HistoryServer App metadata to the diagnostic system, associating log storage paths, as a basis for subsequent data processing stages; +- Data Collection Stage: The task-canal/adapter module subscribes to synchronize the user, DAG, job, execution record and other workflow metadata of the scheduling system to the diagnostic platform; the task-collector module periodically synchronizes Yarn ResourceManager and Spark HistoryServer App metadata to the diagnostic system, associating log storage paths and task applicationIds with HDFS log paths, as a basis for subsequent data processing stages; -- Data Association and Model Standardization Stage: The task-syncer module standardizes the synchronized data into User, Project, Flow, Task, TaskInstance models; the task-application module associates workflow layer and engine layer metadata; +- Data Association and Model Standardization Stage: The task-syncer module standardizes the synchronized data into User, Project, Flow, Task, TaskInstance models; -- Workflow Layer & Engine Layer Anomaly Detection Stage: At this point, the standard data model has been obtained, and further Workflow anomaly detection processes are carried out for the standard model. The task-detect module performs workflow layer anomaly task detection, such as `Failed task`, `Abnormal time-elapsed task`, etc.; the task-parser module performs engine layer anomaly task detection, such as `Failed SQL task`, `Shuffle failed task`, etc.; the task-flink module performs flink job resource and anomaly detection, such as `Low CPU usage`, `Low memory usage`, etc.; +- Workflow Layer & Engine Layer Anomaly Detection Stage: At this point, the standard data model has been obtained, and further Workflow anomaly detection processes are carried out for the standard model. The task-analyzer module performs both workflow layer anomaly detection (e.g. `Failed task`, `Abnormal time-elapsed task`) and engine layer log parsing and anomaly detection (e.g. `Failed SQL task`, `Shuffle failed task`); the task-flink module performs flink job resource and anomaly detection, such as `Low CPU usage`, `Low memory usage`, etc.; - Business View: The task-portal module provides user report overview, one-click diagnosis, workflow layer task diagnosis, engine layer job Application diagnosis, diagnostic suggestions and detailed reports, whitelist and other functions. diff --git a/document/manual/architecture_zh.md b/document/manual/architecture_zh.md index 888a44fa3..a1c6c2e3c 100644 --- a/document/manual/architecture_zh.md +++ b/document/manual/architecture_zh.md @@ -24,11 +24,11 @@ 具体模块流程阶段： -- 数据采集阶段：task-canal/adapter模块订阅同步调度系统的用户、DAG、作业、执行记录等工作流元数据同步至诊断平台；task-metadata模块定时同步Yarn ResourceManager、Spark HistoryServer App元数据至诊断系统，关联日志存储路径，为后续数据处理阶段作基础； +- 数据采集阶段：task-canal/adapter模块订阅同步调度系统的用户、DAG、作业、执行记录等工作流元数据同步至诊断平台；task-collector模块定时同步Yarn ResourceManager、Spark HistoryServer App元数据至诊断系统，关联日志存储路径及任务applicationId与HDFS日志路径，为后续数据处理阶段作基础； -- 数据关联与模型标准化阶段：task-syncer模块将同步的数据标准化为User、Project、Flow、Task、TaskInstance模型；task-application模块将工作流层与引擎层元数据关联； +- 数据关联与模型标准化阶段：task-syncer模块将同步的数据标准化为User、Project、Flow、Task、TaskInstance模型； -- 工作流层&引擎层异常检测阶段：至此已经获得数据标准模型，针对标准模型进一步Workflow异常检测流程。task-detect模块进行工作流层异常任务检测，例如运行失败、基线耗时异常等；task-parser模块进行引擎层异常任务检测，例如SQL失败、Shuffle失败等；task-flink模块进行flink作业资源及异常检测，例如cpu利用率低，内存利用率低等； +- 工作流层&引擎层异常检测阶段：至此已经获得数据标准模型，针对标准模型进一步Workflow异常检测流程。task-analyzer模块同时进行工作流层异常任务检测（例如运行失败、基线耗时异常）和引擎层日志解析与异常检测（例如SQL失败、Shuffle失败）；task-flink模块进行flink作业资源及异常检测，例如cpu利用率低，内存利用率低等； - 业务视图：task-portal模块提供用户报告总览、一键诊断、工作流层任务诊断、引擎层作业Application诊断、诊断建议和详细报告、白名单等功能。 diff --git a/document/manual/deployment.md b/document/manual/deployment.md index 7bbc1c4cb..c911f86f4 100644 --- a/document/manual/deployment.md +++ b/document/manual/deployment.md @@ -146,7 +146,7 @@ fi ``` hadoop: - # Configuration dependency for task-application & task-parser modules + # Configuration dependency for task-collector & task-analyzer modules namenodes: - nameservices: logs-hdfs # dfs.nameservices value namenodesAddr: [ "machine1.example.com", "machine2.example.com" ] # dfs.namenode.rpc-address.[nameservice ID].[name node ID] value @@ -154,7 +154,7 @@ hadoop: user: hdfs # user password: # password，empty if no password port: 8020 # port - matchPathKeys: [ "flume" ] # Usage of the task-application module, keywords for the HDFS path of the scheduling platform log + matchPathKeys: [ "flume" ] # Used by task-collector: keywords for the HDFS path of the scheduling platform log # kerberos enableKerberos: false # /etc/krb5.conf @@ -166,7 +166,7 @@ hadoop: # /var/kerberos/krb5kdc/admin.keytab keytabPath: "" - # Configuration dependency for the task-metadata module + # Configuration dependency for the task-collector module yarn: - clusterName: "bigdata" resourceManager: [ "ip:port" ] # yarn.resourcemanager.webapp.address diff --git a/document/manual/deployment_detail.md b/document/manual/deployment_detail.md index fa9c45c66..81de75227 100644 --- a/document/manual/deployment_detail.md +++ b/document/manual/deployment_detail.md @@ -10,12 +10,10 @@ compass │   └── stop_all.sh stop script ├── conf │   └── application-hadoop.yml hadoop configuration -├── task-application Associated Scheduler Instance, Spark Instance (ApplicationId), Log Path +├── task-collector Collect scheduler task instances, associate ApplicationId and HDFS log paths, sync Yarn/Spark metadata +├── task-analyzer Workflow anomaly detection and engine-layer log parsing (Spark/MR diagnosis) ├── task-canal Synchronize scheduler metadata to Compass as a diagnostic event. ├── task-canal-adapter Synchronize scheduler metadata to Compass, save the original table, and perform data-assisted queries. -├── task-detect Detect abnormalities in scheduler tasks. -├── task-metadata Synchronize Hadoop and Spark metadata, including Spark application and YARN application metadata, to Compass and save it -├── task-parser Parse the scheduler log, Spark application event log, and executor log for abnormalities. ├── task-portal Display diagnostic and analytical results for Spark, Flink, MapReduce, and the scheduler. ├── task-flink Flink resource diagnostic module ├── task-flink-core Flink diagnostic rules @@ -322,28 +320,28 @@ task_instance table： queries: [ "select t2.schedule_time as execution_time, t3.name as flow_name, t4.name as project_name from t_ds_task_instance as t1 inner join t_ds_process_instance as t2 on t1.process_instance_id = t2.id inner join t_ds_process_definition as t3 on t2.process_definition_code = t3.code inner join t_ds_project as t4 on t3.project_code=t4.code where t1.id=${id}" ] ``` -## task-application +## task-collector -The task-application module associates task_name, applicationId, and hdfs_log_path. This module needs to read the scheduling platform logs, and it is recommended to collect them to HDFS using Flume for the convenience of unified log diagnosis and analysis. +The task-collector module consolidates data collection responsibilities: it associates task_name, applicationId, and hdfs_log_path from scheduler logs (formerly task-application), and periodically synchronizes Yarn ResourceManager and Spark HistoryServer App metadata (formerly task-metadata). ``` -task-application/ +task-collector/ ├── bin -│   ├── compass_env.sh -│   ├── startup.sh -│   └── stop.sh +│ ├── compass_env.sh +│ ├── startup.sh +│ └── stop.sh ├── conf -│   ├── application-airflow.yml -│   ├── application-dolphinscheduler.yml -│   ├── application-hadoop.yml -│   ├── application.yml -│   └── logback.xml +│ ├── application-airflow.yml +│ ├── application-dolphinscheduler.yml +│ ├── application-hadoop.yml +│ ├── application.yml +│ └── logback.xml ├── lib ``` ### Configuration -conf/application-hadoop.yml +conf/application-hadoop.yml — HDFS namenode and Yarn/Spark cluster configuration: ``` hadoop: @@ -355,64 +353,7 @@ hadoop: password: port: 8020 matchPathKeys: [ "flume" ] -``` - -`conf/application-dolphinscheduler/airflow/custom.yml` - -This configuration involves the concatenation of log path rules, specifically determining the absolute path of the logs. Taking the example of collecting dolphinscheduler logs to HDFS using Flume, the same logic applies to Airflow. - -The table t_ds_task_instance records the log path (log_path). However, this is the directory on the worker host, and the directory changes when uploading to HDFS. - -For example: - -scheduler worker log_path: /home/service/app/dolphinscheduler/logs/8590950992992_2/33552/33934.log - -hdfs log_path: hdfs://log-hdfs:8020/flume/dolphinscheduler/2023-03-30/8590950992992_2/33552/xxx - -Therefore, based on the above relationship changes, the absolute path is determined through step-by-step directory identification, and then the relationship between task_name, application_id, and hdfs_log_path is finally determined and stored in the task_application table. - -``` -custom: - # Execute parsing to the task's applicationId. - rules: - - logPathDep: - # Variable dependency query - query: "select CASE WHEN end_time IS NOT NULL THEN DATE_ADD(end_time, INTERVAL 1 second) ELSE start_time END as end_time,log_path from t_ds_task_instance where id=${id}" # 查询, id 是 task-instance表的id - logPathJoins: - # end_time: 2023-02-18 01:43:11 - # log_path: ../logs/6354680786144_1/3/4.log - - { "column": "", "data": "/flume/dolphinscheduler" } # Configuration for storing scheduling logs in the root directory of HDFS - - { "column": "end_time", "regex": "^.*(?\\d{4}-\\d{2}-\\d{2}).+$", "name": "date" } - - { "column": "log_path", "regex": "^.*logs/(?.*)$", "name": "logpath" } - extractLog: # Parse logs based on the log_path - regex: "^.*Submitted application (?application_[0-9]+_[0-9]+).*$" - name: "applicationId" # Match the text name, must have applicationId at the end. -``` - -Note: The native Flume-taildir-source plugin does not support recursively traversing subdirectory files, and requires modification. If your logs have already been collected, you can ignore this. - -If you have not collected logs, you can modify the TaildirMatcher.getMatchingFilesNoCache() method to implement this function. If you are using Airflow, the generated log directory may contain characters that do not comply with the HDFS directory rules. When sinking to HDFS, you need to modify and replace directory special characters with underscores ('_'). -## task-metadata - -The task-metadata module is used to synchronize the Yarn and Spark task applicationId lists and associate the storage paths of driver, executor, and eventlog logs with applicationId. -``` -task-metadata -├── bin -│   ├── compass_env.sh -│   ├── startup.sh -│   └── stop.sh -├── conf -│   ├── application.yml -│   └── logback.xml -├── lib -``` - -### Configuration - -conf/application.yml -``` -hadoop: yarn: - clusterName: "bigdata" resourceManager: [ "ip:port" ] @@ -422,51 +363,41 @@ hadoop: sparkHistoryServer: [ "ip:port" ] ``` -## task-detect - -The task-detect module is designed for abnormal detection at the workflow level. The types of abnormalities include running failure, baseline time exception, baseline duration exception, first-time failure, long-term failure, and long running time. -``` -task-detect -├── bin -│   ├── compass_env.sh -│   ├── startup.sh -│   └── stop.sh -├── conf -│   ├── application.yml -├── lib -``` - -### Configuration - -conf/application.yml +`conf/application-dolphinscheduler/airflow/custom.yml` — log path assembly rules for extracting applicationId from scheduler logs. The scheduler worker log_path (e.g. `/home/.../logs/xxx/33552/33934.log`) is mapped to the HDFS path (e.g. `hdfs://log-hdfs:8020/flume/dolphinscheduler/2023-03-30/xxx/33552/...`): ``` custom: - detectionRule: - # unit: hour - durationWarning: 2 - # unit: day - alwaysFailedWarning: 10 + rules: + - logPathDep: + query: "select CASE WHEN end_time IS NOT NULL THEN DATE_ADD(end_time, INTERVAL 1 second) ELSE start_time END as end_time,log_path from t_ds_task_instance where id=${id}" + logPathJoins: + - { "column": "", "data": "/flume/dolphinscheduler" } + - { "column": "end_time", "regex": "^.*(?\\d{4}-\\d{2}-\\d{2}).+$", "name": "date" } + - { "column": "log_path", "regex": "^.*logs/(?.*)$", "name": "logpath" } + extractLog: + regex: "^.*Submitted application (?application_[0-9]+_[0-9]+).*$" + name: "applicationId" ``` -## task-parser +Note: The native Flume-taildir-source plugin does not support recursively traversing subdirectory files and requires modification. -The task-parser module is designed to parse and diagnose Spark tasks and related logs. The types of exceptions include SQL failure, Shuffle failure, memory overflow, memory waste, CPU waste, large table scans, OOM warnings, data skewness, abnormal Job duration, abnormal Stage duration, long tail of tasks, HDFS lag, excessive delayed execution of tasks, and global sorting anomalies. +## task-analyzer + +The task-analyzer module consolidates both workflow-level anomaly detection (formerly task-detect) and engine-layer log parsing and diagnosis (formerly task-parser). It handles: running failure, baseline exceptions, SQL failure, Shuffle failure, memory overflow, CPU/memory waste, data skewness, and other Spark/MR anomalies. ``` -task-parser +task-analyzer/ ├── bin -│   ├── compass_env.sh -│   ├── startup.sh -│   └── stop.sh +│ ├── compass_env.sh +│ ├── startup.sh +│ └── stop.sh ├── conf -│   ├── applicationbk.yml -│   ├── application-hadoop.yml -│   ├── application.yml -│   ├── logback.xml -│   ├── rules.json -│   └── scripts -│   └── logRecordConsumer.lua +│ ├── application-hadoop.yml +│ ├── application.yml +│ ├── logback.xml +│ ├── rules.json +│ └── scripts +│ └── logRecordConsumer.lua ├── lib ``` @@ -475,13 +406,13 @@ task-parser The `conf/rules.json` configuration is used to write log parsing rules. The fields are defined as following: -**logType**: scheduler/driver/executor/yarn +**logType**: scheduler/driver/executor/yarn **action**: Define the name of each matching rule. -**desc**： Description for action +**desc**: Description for action -**category**： Definition of rule types, such as shuffleFailed/sqlFailed, etc. +**category**: Definition of rule types, such as shuffleFailed/sqlFailed, etc. **step**: order of matching action. @@ -489,19 +420,9 @@ The fields are defined as following: **parserTemplate**: Text parsing templates consist of the first line, middle lines, and ending lines. -If only simple line matching is required, it is sufficient to fill in **parserTemplate.heads**. - -If text block matching is required, such as for exception stacks, it is necessary to fill in **parserTemplate.heads** and **parserTemplate.tails** to determine the rules for the first line and the ending line. - -If a specific line needs to be matched within a text block, the middle line rule in **parserTemplate.middles** must be filled in. - -**groupNames**: Extracting values of named capturing groups in regular expression matching by users. - -**children**: Used for nested rules, for example, when there are multiple identical exception stacks in the text (with the same start and end markers), if it is necessary to differentiate them into different actions, nested rules can be used to achieve this - For example: ``` - { + { "logType": "scheduler", "actions": [ { @@ -512,9 +433,7 @@ For example: "skip": false, "parserType": "DEFAULT", "parserTemplate": { - "heads": [ - "^.+ERROR.+$" - ], + "heads": [ "^.+ERROR.+$" ], "middles": [], "tails": [] }, @@ -525,16 +444,15 @@ For example: } ``` -`conf/application.yml` - -"custom.detector" is used to configure custom detectors for monitoring Spark event logs, such as detecting abnormalities related to Spark environment variables, memory wastage, large table scans, etc. +`conf/application.yml` — detection rules and Spark event log detectors: ``` custom: + detectionRule: + durationWarning: 2 # unit: hour + alwaysFailedWarning: 10 # unit: day detector: sparkEnvironmentConfig: - jvmInformation: - - Java Version sparkProperties: - spark.driver.memoryOverhead - spark.driver.memory @@ -544,11 +462,10 @@ custom: - spark.dynamicAllocation.maxExecutors - spark.default.parallelism - spark.sql.shuffle.partitions - systemProperties: - - sun.java.command ... ``` + ## task-gpt The task-gpt module is used to aggregate log templates and use ChatGPT to provide solutions for the templates. @@ -599,7 +516,7 @@ task-portal ## Offline task metadata reporting diagnosis. -Supports third-party reporting of Spark/MapReduce task application metadata for diagnosis. If you don't need to synchronize scheduling platform metadata and logs, simply start the task-portal and task-parser modules. +Supports third-party reporting of Spark/MapReduce task application metadata for diagnosis. If you don't need to synchronize scheduling platform metadata and logs, simply start the task-portal and task-analyzer modules. Request API：http://[compass_host]/compass/openapi/offline/app/metadata @@ -655,5 +572,5 @@ For example： ## One-Click diagnosis -Offline diagnosis supports one-click diagnosis for all Spark/MapReduce tasks, including those not submitted to the scheduling platform. If you only want to experience this function, simply start the task-portal, task-metadata, and task-parser modules. +Offline diagnosis supports one-click diagnosis for all Spark/MapReduce tasks, including those not submitted to the scheduling platform. If you only want to experience this function, simply start the task-portal, task-collector, and task-analyzer modules. diff --git a/document/manual/deployment_zh.md b/document/manual/deployment_zh.md index 3c93d3e03..34f65a40b 100644 --- a/document/manual/deployment_zh.md +++ b/document/manual/deployment_zh.md @@ -146,7 +146,7 @@ fi ``` hadoop: - # task-applicaiton & task-parser 模块配置依赖 + # task-collector & task-analyzer 模块配置依赖 namenodes: - nameservices: logs-hdfs # dfs.nameservices 属性值 namenodesAddr: [ "machine1.example.com", "machine2.example.com" ] # dfs.namenode.rpc-address.[nameservice ID].[name node ID] 属性值 @@ -154,7 +154,7 @@ hadoop: user: hdfs # 用户 password: # 密码，如果没开启鉴权，则不需要 port: 8020 # 端口 - matchPathKeys: [ "flume" ] # task-application模块使用，调度平台日志hdfs路径关键字 + matchPathKeys: [ "flume" ] # task-collector模块使用，调度平台日志hdfs路径关键字 # kerberos enableKerberos: false # /etc/krb5.conf @@ -166,7 +166,7 @@ hadoop: # /var/kerberos/krb5kdc/admin.keytab keytabPath: "" - # task-metadata 模块配置依赖 + # task-collector 模块配置依赖 yarn: - clusterName: "bigdata" resourceManager: [ "ip:port" ] # yarn.resourcemanager.webapp.address 属性值 diff --git a/document/manual/deployment_zh_detail.md b/document/manual/deployment_zh_detail.md index 15db4d67f..3e077fc57 100644 --- a/document/manual/deployment_zh_detail.md +++ b/document/manual/deployment_zh_detail.md @@ -10,12 +10,10 @@ compass │   └── stop_all.sh 停止脚本 ├── conf │   └── application-hadoop.yml hadoop相关配置 -├── task-application 关联任务实例、applicationId、hdfs_log_path +├── task-collector 采集任务实例、关联applicationId及HDFS日志路径、同步Yarn/Spark元数据 ├── task-canal 订阅调度平台MySQL表元数据到Kafka ├── task-canal-adapter 同步调度平台MySQL表元数据Compass平台 -├── task-detect 工作流层异常类型检测 -├── task-metadata 同步Yarn、Spark任务元数据到OpenSearch -├── task-parser 日志解析和Spark任务异常检测 +├── task-analyzer 工作流层异常检测和引擎层日志解析诊断 ├── task-portal 异常任务的可视化服务 ├── task-flink Flink任务资源及异常诊断 ├── task-flink-core Flink任务诊断规则逻辑 @@ -327,28 +325,28 @@ task_instance表映射： queries: [ "select t2.schedule_time as execution_time, t3.name as flow_name, t4.name as project_name from t_ds_task_instance as t1 inner join t_ds_process_instance as t2 on t1.process_instance_id = t2.id inner join t_ds_process_definition as t3 on t2.process_definition_code = t3.code inner join t_ds_project as t4 on t3.project_code=t4.code where t1.id=${id}" ] ``` -## task-application +## task-collector -task-application模块关联task_name、applicationId、hdfs_log_path，该模块需要读取调度平台日志，推荐使用flume收集到hdfs，方便统一做日志诊断和分析。 +task-collector模块整合了数据采集职责：关联task_name、applicationId、hdfs_log_path（原task-application功能），并定时同步Yarn ResourceManager、Spark HistoryServer App元数据（原task-metadata功能）。 ``` -task-application/ +task-collector/ ├── bin -│   ├── compass_env.sh -│   ├── startup.sh -│   └── stop.sh +│ ├── compass_env.sh +│ ├── startup.sh +│ └── stop.sh ├── conf -│   ├── application-airflow.yml -│   ├── application-dolphinscheduler.yml -│   ├── application-hadoop.yml -│   ├── application.yml -│   └── logback.xml +│ ├── application-airflow.yml +│ ├── application-dolphinscheduler.yml +│ ├── application-hadoop.yml +│ ├── application.yml +│ └── logback.xml ├── lib ``` -### 核心配置 +### 配置 -conf/application-hadoop.yml +conf/application-hadoop.yml — HDFS namenode及Yarn/Spark集群配置： ``` hadoop: @@ -360,66 +358,7 @@ hadoop: password: port: 8020 matchPathKeys: [ "flume" ] -``` - -conf/application-dolphinscheduler/airflow/custom.yml - -该配置涉及日志路径规则的拼接，即日志绝对路径的确定。以flume收集dolphinscheduler到hdfs为例，airflow等同理。 -表t_ds_task_instance记录了日志路径log_path,但这个是worker主机中的目录，上传到hdfs的目录有所变化。 - -例如: - -scheduler worker log_path: /home/service/app/dolphinscheduler/logs/8590950992992_2/33552/33934.log - -hdfs log_path: hdfs://log-hdfs:8020/flume/dolphinscheduler/2023-03-30/8590950992992_2/33552/xxx - -因此需要根据上面的变化关系，通过逐级目录确定绝对路径，然后最终确定 task_name,application_id,hdfs_log_path -之间的关系存储到表task_application中。 - -``` -custom: - # 从上到下串行执行解析到任务的applicationId - rules: - - logPathDep: - # 变量依赖查询 - query: "select CASE WHEN end_time IS NOT NULL THEN DATE_ADD(end_time, INTERVAL 1 second) ELSE start_time END as end_time,log_path from t_ds_task_instance where id=${id}" # 查询, id 是 task-instance表的id - logPathJoins: - # end_time: 2023-02-18 01:43:11 - # log_path: ../logs/6354680786144_1/3/4.log - - { "column": "", "data": "/flume/dolphinscheduler" } # 配置存储调度日志的hdfs根目录 - - { "column": "end_time", "regex": "^.*(?\\d{4}-\\d{2}-\\d{2}).+$", "name": "date" } - - { "column": "log_path", "regex": "^.*logs/(?.*)$", "name": "logpath" } - extractLog: # 根据组装的日志路径解析日志 - regex: "^.*Submitted application (?application_[0-9]+_[0-9]+).*$" # 匹配规则 - name: "applicationId" # 匹配文本名，最后必须有applicationId -``` - -注意：原生flume-taildir-source插件是不支持递归遍历子目录文件的，需要进行改造。如果您日志已经收集，可忽略。 -如果您还没有收集，可修改TaildirMatcher.getMatchingFilesNoCache()方法实现。 -如果你使用的是Airflow，生成的日志目录可能包含不符合hdfs目录规则，sink to hdfs时需要修改替换目录特殊字符为下划线‘_’。 - -## task-metadata - -task-metadata模块是用于同步Yarn、Spark任务applicationId列表，关联applicationId的driver、executor、eventlog日志存储路径 - -``` -task-metadata -├── bin -│   ├── compass_env.sh -│   ├── startup.sh -│   └── stop.sh -├── conf -│   ├── application.yml -│   └── logback.xml -├── lib -``` - -### 核心配置 -conf/application.yml - -``` -hadoop: yarn: - clusterName: "bigdata" resourceManager: [ "ip:port" ] @@ -429,119 +368,55 @@ hadoop: sparkHistoryServer: [ "ip:port" ] ``` -## task-detect - -task-detect模块是针对工作流层异常检测，异常类型包括运行失败、基线时间异常、基线耗时异常、首次失败、长期失败、运行耗时长 - -``` -task-detect -├── bin -│   ├── compass_env.sh -│   ├── startup.sh -│   └── stop.sh -├── conf -│   ├── application.yml -├── lib -``` - -### 核心配置 - -conf/application.yml +`conf/application-dolphinscheduler/airflow/custom.yml` — 日志路径拼接规则，用于从调度日志中提取applicationId： ``` custom: - detectionRule: - # 运行耗时长配置，单位小时 - durationWarning: 2 - # 长期失败配置，单位天 - alwaysFailedWarning: 10 + rules: + - logPathDep: + query: "select CASE WHEN end_time IS NOT NULL THEN DATE_ADD(end_time, INTERVAL 1 second) ELSE start_time END as end_time,log_path from t_ds_task_instance where id=${id}" + logPathJoins: + - { "column": "", "data": "/flume/dolphinscheduler" } + - { "column": "end_time", "regex": "^.*(?\\d{4}-\\d{2}-\\d{2}).+$", "name": "date" } + - { "column": "log_path", "regex": "^.*logs/(?.*)$", "name": "logpath" } + extractLog: + regex: "^.*Submitted application (?application_[0-9]+_[0-9]+).*$" + name: "applicationId" ``` -## task-parser +## task-analyzer -task-parser模块是针对Spark任务和相关日志进行解析诊断，异常类型包括：SQL失败、Shuffle失败、内存溢出、内存浪费、CPU浪费、大表扫描、OOM预警、 -数据倾斜、Job耗时异常、Stage耗时异常、Task长尾、HDFS卡顿、推迟执行Task过多、全局排序异常等 +task-analyzer模块整合了工作流层异常检测（原task-detect）和引擎层日志解析诊断（原task-parser）。支持：运行失败、基线耗时异常、SQL失败、Shuffle失败、内存溢出、CPU/内存浪费、数据倾斜等Spark/MR异常。 ``` -task-parser +task-analyzer/ ├── bin -│   ├── compass_env.sh -│   ├── startup.sh -│   └── stop.sh +│ ├── compass_env.sh +│ ├── startup.sh +│ └── stop.sh ├── conf -│   ├── applicationbk.yml -│   ├── application-hadoop.yml -│   ├── application.yml -│   ├── logback.xml -│   ├── rules.json -│   └── scripts -│   └── logRecordConsumer.lua +│ ├── application-hadoop.yml +│ ├── application.yml +│ ├── logback.xml +│ ├── rules.json +│ └── scripts +│ └── logRecordConsumer.lua ├── lib ``` -### 核心配置 - -conf/rules.json 该配置是用于编写日志解析规则 - -logType: scheduler/driver/executor/yarn 日志类型，若有其他日志，可自行实现 - -action: 定义每个匹配规则名称 - -desc： action描述 - -category： 定义规则类型，例如shuffleFailed/sqlFailed等 - -step: 匹配顺序，默认升序 - -parserType: 匹配类型，默认 DEFAULT(按行或者块匹配)，JOIN(把结果合并成一行再匹配) - -parserTemplate: 文本解析模板，由首行、中间行和结束行组成。 +### 配置 -如果只是简单按行匹配，则只需要填写parserTemplate.heads即可； +conf/rules.json — 日志解析规则，支持scheduler/driver/executor/yarn日志类型。 -如果需要按文本块匹配，例如异常栈，则需要填写parserTemplate.heads和parserTemplate.tails确定首行和结束行规则； - -如果需要在文本块中匹配某一行，则需要填写parserTemplate.middles中间行规则。 - -groupNames：用户提取正则匹配分组名称的值 - -children: 用于嵌套规则，例如文本中有多个相同的异常栈(开始和结束标志一样)，如果需要区分成不同的action，那么就可以嵌套规则实现 - -``` - { - "logType": "scheduler", - "actions": [ - { - "action": "otherError", - "desc": "其他错误信息", - "category": "otherException", - "step": 1, - "skip": false, - "parserType": "DEFAULT", - "parserTemplate": { - "heads": [ - "^.+ERROR.+$" - ], - "middles": [], - "tails": [] - }, - "groupNames": [], - "children": [] - } - ] - } -``` - -conf/application.yml - -custom.detector用于配置检测Spark Event Log，比如Spark环境变量、内存浪费、大表扫描等异常检测类型 +conf/application.yml — 检测规则及Spark事件日志检测器： ``` custom: + detectionRule: + durationWarning: 2 # 单位：小时 + alwaysFailedWarning: 10 # 单位：天 detector: sparkEnvironmentConfig: - jvmInformation: - - Java Version sparkProperties: - spark.driver.memoryOverhead - spark.driver.memory @@ -549,13 +424,11 @@ custom: - spark.executor.memory - spark.executor.cores - spark.dynamicAllocation.maxExecutors - - spark.default.parallelism - spark.sql.shuffle.partitions - systemProperties: - - sun.java.command ... ``` + ## task-gpt task-gpt模块用于聚合日志模板，并使用chatgpt给模板解决方案 @@ -607,7 +480,7 @@ task-portal ## 离线任务上报元数据诊断 -支持第三方上报Spark/MapReduce任务application元数据进行诊断，如果不需要同步调度平台元数据和日志，只要启动task-portal和task-parser模块。 +支持第三方上报Spark/MapReduce任务application元数据进行诊断，如果不需要同步调度平台元数据和日志，只要启动task-portal和task-analyzer模块。 请求接口：http://[compass_host]/compass/openapi/offline/app/metadata @@ -661,4 +534,4 @@ task-portal ## 一键诊断功能 -离线诊断支持全量(包含非调度平台提交任务)Spark/MapReduce任务进行一键诊断，如果仅需要体验该功能，只要启动task-portal、task-metadata和task-parser模块。 +离线诊断支持全量(包含非调度平台提交任务)Spark/MapReduce任务进行一键诊断，如果仅需要体验该功能，只要启动task-portal、task-collector和task-analyzer模块。 diff --git a/pom.xml b/pom.xml index c47417e3f..9814d039f 100644 --- a/pom.xml +++ b/pom.xml @@ -10,27 +10,35 @@ pom ${revision} + task-mbg task-common - task-parser - task-portal - task-metadata - task-detect + + + task-analyzer + task-collector task-syncer - task-application - task-canal - task-canal-adapter + task-portal + task-gpt + + task-flink-core task-flink + + + task-canal + task-canal-adapter + + task-ui - task-gpt task-assembly + org.springframework.boot spring-boot-starter-parent - 2.7.8 + 3.2.4 @@ -48,7 +56,7 @@ 2.4.5 3.1.0 0.9.0 - 4.0.1 + 6.0.0 5.8.2 2.11.1 3.12.0 @@ -57,9 +65,11 @@ 2.19.1 3.1.0 4.2.0 + 3.7.1 3.4.0 - 2.0.23 + 3.5.6 + 1.12.4 31.1-jre 3.3.4 2.17.2 @@ -67,7 +77,7 @@ 4.3.1 3.0.0 2.27.2 - 1.3.12 + 2.13.0 4.2.2 true 10.14.2.0 @@ -110,18 +120,6 @@ ${revision} - - com.oppo.cloud - task-detect - ${revision} - - - - com.oppo.cloud - task-metadata - ${revision} - - com.oppo.cloud task-portal @@ -140,11 +138,6 @@ ${revision} - - com.oppo.cloud - task-parser - ${revision} - com.oppo.cloud task-flink-core @@ -236,10 +229,18 @@ ${spring-kafka.version} + + + + com.baomidou + mybatis-plus-boot-starter + ${mybatis-plus.version} + + - com.alibaba.fastjson2 - fastjson2 - ${fastjson2.version} + io.micrometer + micrometer-registry-prometheus + ${micrometer-prometheus.version} @@ -255,9 +256,9 @@ - javax.servlet - javax.servlet-api - ${javax-servlet.version} + jakarta.servlet + jakarta.servlet-api + ${jakarta-servlet.version} @@ -523,9 +524,7 @@ org.apache.maven.plugins maven-surefire-plugin ${maven-surefire-plugin.version} - - true - + org.apache.maven.plugins diff --git a/task-parser/pom.xml b/task-analyzer/pom.xml similarity index 63% rename from task-parser/pom.xml rename to task-analyzer/pom.xml index 38d6107a9..5e4e0d205 100644 --- a/task-parser/pom.xml +++ b/task-analyzer/pom.xml @@ -1,29 +1,30 @@ - - - 4.0.0 + + compass com.oppo.cloud ${revision} - task-parser + 4.0.0 - task-parser + task-analyzer + com.oppo.cloud + ${revision} + task-analyzer - 1.8 + 17 - - - org.apache.derby - derby - test - - + com.oppo.cloud task-mbg @@ -35,6 +36,13 @@ + + + org.springframework.kafka + spring-kafka + + + org.apache.hadoop hadoop-common @@ -47,18 +55,6 @@ javax.servlet servlet-api - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - com.sun.xml.bind - jaxb-impl - org.slf4j slf4j-reload4j @@ -81,92 +77,56 @@ javax.servlet servlet-api - - com.google.code.findbugs - jsr305 - + - org.mockito - mockito-inline - test - - - org.mockito - mockito-core - - + com.mysql + mysql-connector-j + - org.testcontainers - testcontainers - test + org.lz4 + lz4-java - - mysql - mysql-connector-java + com.github.luben + zstd-jni + - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - test + org.springframework.boot + spring-boot-starter-actuator - - org.apache.hadoop - hadoop-hdfs - test-jar - ${hadoop.version} - test + io.micrometer + micrometer-registry-prometheus + - org.apache.hadoop - hadoop-common - ${hadoop.version} - tests + org.apache.derby + derby test - - - org.slf4j - * - - - - org.springframework.kafka - spring-kafka - - - - org.lz4 - lz4-java + org.testcontainers + testcontainers + test - - com.github.luben - zstd-jni + org.mockito + mockito-inline + test - ${project.artifactId} - - org.apache.maven.plugins - maven-surefire-plugin - - true - - org.apache.maven.plugins maven-assembly-plugin @@ -184,14 +144,13 @@ false - com.oppo.cloud.parser.TaskParserApplication + com.oppo.cloud.analyzer.TaskAnalyzerApplication src/assembly/assembly.xml - diff --git a/task-detect/src/assembly/assembly.xml b/task-analyzer/src/assembly/assembly.xml similarity index 98% rename from task-detect/src/assembly/assembly.xml rename to task-analyzer/src/assembly/assembly.xml index 96285c1db..a55a92089 100644 --- a/task-detect/src/assembly/assembly.xml +++ b/task-analyzer/src/assembly/assembly.xml @@ -27,4 +27,4 @@ lib - \ No newline at end of file + diff --git a/task-application/src/bin/startup.sh b/task-analyzer/src/bin/startup.sh old mode 100644 new mode 100755 similarity index 90% rename from task-application/src/bin/startup.sh rename to task-analyzer/src/bin/startup.sh index ec9e0aefd..4b7fc1d80 --- a/task-application/src/bin/startup.sh +++ b/task-analyzer/src/bin/startup.sh @@ -1,7 +1,7 @@ #!/bin/bash HOME_DIR=$(cd $(dirname $0)/.. && pwd) -APP_NAME="task-application" +APP_NAME="task-analyzer" PID_FILE=${HOME_DIR}/tpid if [ -f $(dirname $0)/compass_env.sh ]; then @@ -29,7 +29,7 @@ start() { JAVA_OPTS="-DappName=${APP_NAME} -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom" JAVA_OPTS_GC="-server -XX:+UseG1GC -XX:G1HeapRegionSize=8m -verbose:GC -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${HOME_DIR}/logs/dump.hprof -Xloggc:${HOME_DIR}/logs/gc-${APP_NAME}.log" - nohup java $JAVA_OPTS $JAVA_OPTS_GC -cp "${HOME_DIR}/conf":"${HOME_DIR}/lib/*" com.oppo.cloud.application.TaskApplication >/dev/null 2>&1 & + nohup java $JAVA_OPTS $JAVA_OPTS_GC -cp "${HOME_DIR}/conf":"${HOME_DIR}/lib/*" com.oppo.cloud.analyzer.TaskAnalyzerApplication >/dev/null 2>&1 & pid=$! echo $pid diff --git a/task-analyzer/src/bin/stop.sh b/task-analyzer/src/bin/stop.sh new file mode 100755 index 000000000..cd3ce1593 --- /dev/null +++ b/task-analyzer/src/bin/stop.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +HOME_DIR=$(cd $(dirname $0)/.. && pwd) +PID_FILE=${HOME_DIR}/tpid + +if [ -f ${PID_FILE} ]; then + pid=$(cat ${PID_FILE}) + if kill -0 ${pid} 2>/dev/null; then + kill ${pid} + echo "task-analyzer stopped (pid=${pid})" + else + echo "task-analyzer is not running" + fi + rm -f ${PID_FILE} +else + echo "PID file not found" +fi diff --git a/task-analyzer/src/main/java/com/oppo/cloud/analyzer/TaskAnalyzerApplication.java b/task-analyzer/src/main/java/com/oppo/cloud/analyzer/TaskAnalyzerApplication.java new file mode 100644 index 000000000..f6c323e18 --- /dev/null +++ b/task-analyzer/src/main/java/com/oppo/cloud/analyzer/TaskAnalyzerApplication.java @@ -0,0 +1,54 @@ +/* + * Copyright 2023 OPPO. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.oppo.cloud.analyzer; + +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; + +/** + * task-analyzer: merged from task-detect + task-parser. + * + *

Eliminates the Redis queue intermediary between detection and parsing by directly + * calling the parser (JobManager) after detection completes. Data flow: + * + * 

+ *   Kafka[task-instance]
+ *       → DetectedTask (Kafka consumer, batch=50)
+ *       → DetectService.detect() [6 workflow detectors]
+ *       → AnalyzerBridge.analyze(logRecord)  ← NEW: replaces Redis push
+ *       → JobManager.run(logRecord)           ← was in separate process
+ *       → Kafka[task-gpt]
+ *
+ */ +@SpringBootApplication(scanBasePackages = { + "com.oppo.cloud.analyzer", + "com.oppo.cloud.detect", + "com.oppo.cloud.parser", + "com.oppo.cloud.common" +}) +@MapperScan({"com.oppo.cloud.mapper", "com.oppo.cloud.detect.mapper", "com.oppo.cloud.parser.mapper"}) +@EnableScheduling +@EnableAsync +public class TaskAnalyzerApplication { + + public static void main(String[] args) { + SpringApplication.run(TaskAnalyzerApplication.class, args); + } +} diff --git a/task-analyzer/src/main/java/com/oppo/cloud/analyzer/bridge/AnalyzerBridge.java b/task-analyzer/src/main/java/com/oppo/cloud/analyzer/bridge/AnalyzerBridge.java new file mode 100644 index 000000000..691a9ee93 --- /dev/null +++ b/task-analyzer/src/main/java/com/oppo/cloud/analyzer/bridge/AnalyzerBridge.java @@ -0,0 +1,82 @@ +/* + * Copyright 2023 OPPO. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.oppo.cloud.analyzer.bridge; + +import com.oppo.cloud.common.domain.job.LogRecord; +import com.oppo.cloud.parser.service.job.JobManager; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.concurrent.Executor; +import java.util.concurrent.Semaphore; + +/** + * AnalyzerBridge replaces the Redis queue handoff between task-detect and task-parser. + * + *

Previously, task-detect pushed {@link LogRecord} to a Redis List, and task-parser + * polled that list every 2 seconds via a Lua script. This caused: + *

    + *
  • 2–5 second additional latency per task diagnosis
    • + *
  • Potential message loss on Redis restart (no persistence guarantee)
    • + *
  • Complex Lua scripts for distributed deduplication
    • + *
  • Requirement for Redis Cluster (3 nodes) just for this queue
    • + *
+ * + *

After merging into task-analyzer, detection and parsing run in the same JVM. + * The bridge directly calls {@link JobManager#run(LogRecord)} on a thread pool, + * preserving concurrency control via a {@link Semaphore} (same logic as before). + */ +@Slf4j +@Component +public class AnalyzerBridge { + + private final JobManager jobManager; + private final Executor parserExecutorPool; + private final Semaphore semaphore; + + public AnalyzerBridge(JobManager jobManager, + Executor parserExecutorPool, + int maxConcurrentParsers) { + this.jobManager = jobManager; + this.parserExecutorPool = parserExecutorPool; + this.semaphore = new Semaphore(maxConcurrentParsers); + } + + /** + * Asynchronously parse the log record. + * Replaces: {@code redisService.lLeftPush(logRecordQueue, JSONObject.toJSONString(logRecord))} + */ + public void analyze(LogRecord logRecord) { + try { + semaphore.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error("Interrupted while acquiring parser semaphore for logRecord: {}", logRecord.getId()); + return; + } + parserExecutorPool.execute(() -> { + try { + log.info("analyzeLogRecord id={}", logRecord.getId()); + jobManager.run(logRecord); + } catch (Exception e) { + log.error("Parser failed for logRecord id={}: ", logRecord.getId(), e); + } finally { + semaphore.release(); + } + }); + } +} diff --git a/task-analyzer/src/main/java/com/oppo/cloud/analyzer/config/AnalyzerConfig.java b/task-analyzer/src/main/java/com/oppo/cloud/analyzer/config/AnalyzerConfig.java new file mode 100644 index 000000000..baf0a5a7b --- /dev/null +++ b/task-analyzer/src/main/java/com/oppo/cloud/analyzer/config/AnalyzerConfig.java @@ -0,0 +1,74 @@ +/* + * Copyright 2023 OPPO. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.oppo.cloud.analyzer.config; + +import com.oppo.cloud.analyzer.bridge.AnalyzerBridge; +import com.oppo.cloud.parser.service.job.JobManager; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.Executor; + +@Configuration +public class AnalyzerConfig { + + /** Thread pool for running workflow detectors (was task-detect's DETECT_EXECUTOR_POOL) */ + public static final String DETECT_EXECUTOR_POOL = "detectExecutorPool"; + + /** Thread pool for running log parsers (was task-parser's REDIS_CONSUMER_THREAD_POOL) */ + public static final String PARSER_EXECUTOR_POOL = "parserExecutorPool"; + + @Value("${custom.detect.threadPoolSize:10}") + private int detectThreadPoolSize; + + @Value("${custom.parser.maxConcurrentParsers:10}") + private int maxConcurrentParsers; + + @Bean(name = DETECT_EXECUTOR_POOL) + public Executor detectExecutorPool() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(detectThreadPoolSize); + executor.setMaxPoolSize(detectThreadPoolSize * 2); + executor.setQueueCapacity(500); + executor.setThreadNamePrefix("detect-"); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(30); + executor.initialize(); + return executor; + } + + @Bean(name = PARSER_EXECUTOR_POOL) + public Executor parserExecutorPool() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(maxConcurrentParsers); + executor.setMaxPoolSize(maxConcurrentParsers); + executor.setQueueCapacity(200); + executor.setThreadNamePrefix("parser-"); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(60); + executor.initialize(); + return executor; + } + + @Bean + public AnalyzerBridge analyzerBridge(JobManager jobManager, + @org.springframework.beans.factory.annotation.Qualifier(PARSER_EXECUTOR_POOL) Executor parserExecutorPool) { + return new AnalyzerBridge(jobManager, parserExecutorPool, maxConcurrentParsers); + } +} diff --git a/task-parser/src/main/resources/application.yml b/task-analyzer/src/main/resources/application.yml similarity index 51% rename from task-parser/src/main/resources/application.yml rename to task-analyzer/src/main/resources/application.yml index b518d41e9..068d2125e 100644 --- a/task-parser/src/main/resources/application.yml +++ b/task-analyzer/src/main/resources/application.yml @@ -1,62 +1,120 @@ server: - port: 7073 + port: 7071 spring: + application: + name: task-analyzer profiles: active: hadoop jackson: time-zone: GMT+8 date-format: yyyy-MM-dd HH:mm:ss datasource: - url: jdbc:mysql://localhost:33066/compass?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai - username: root - password: root + url: ${DB_URL:jdbc:mysql://localhost:3306/compass?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai} + username: ${DB_USERNAME:} + password: ${DB_PASSWORD:} druid: initial-size: 5 min-idle: 10 max-active: 20 + # Redis: single node (no cluster). Used only for task-detect's delayed queue. + # RedisConsumer (parser) is disabled; parsing is now done in-process via AnalyzerBridge. redis: - cluster: - nodes: localhost:6379 - max-redirects: 3 - password: + host: ${REDIS_HOST:localhost} + port: ${REDIS_PORT:6379} + password: ${REDIS_PASSWORD:} lettuce: pool: - max-active: 32 - max-idle: 16 - min-idle: 8 + max-active: 8 + max-idle: 4 + min-idle: 1 opensearch: - nodes: localhost:19527 - username: - password: + nodes: ${OPENSEARCH_NODES:localhost:9200} + username: ${OPENSEARCH_USERNAME:} + password: ${OPENSEARCH_PASSWORD:} truststore: truststore-password: + # OpenSearch index prefixes (from task-parser) log-prefix: "compass-log-summary-" detector-prefix: "compass-detector-app-" gc-prefix: "compass-gc-log-" job-prefix: "compass-job-analysis-" task-app-prefix: "compass-task-app-" kafka: - bootstrap-servers: "localhost:9092" + bootstrap-servers: ${KAFKA_BOOTSTRAP_SERVERS:localhost:9092} + listener: + type: batch + ack-mode: MANUAL_IMMEDIATE + consumer: + enable-auto-commit: false + max-poll-records: 50 # increased from 1 for throughput + # topic for publishing parsed log results to task-gpt topic: "task-gpt" +mybatis: + mapper-locations: + - classpath:dao/*.xml + - classpath*:com/**/mapper/*.xml + configuration: + log-impl: org.apache.ibatis.logging.slf4j.Slf4jImpl + +management: + endpoints: + web: + exposure: + include: health,info,prometheus,metrics + endpoint: + health: + show-details: when_authorized + custom: - redisConsumer: + schedulerType: ${SCHEDULER:dolphinscheduler} + + # Detect settings (from task-detect) + kafka: + consumer: + group-id: cp-analyzer-task + topic-name: task-instance + auto: + start: true + + detect: + threadPoolSize: 10 + detectionRule: + durationWarning: 2 # hours + alwaysFailedWarning: 10 # days + + # Delayed task queue for retrying tasks with pending applicationIds + delayedTaskQueue: enable: true - maxThreadPoolSize: 10 - logRecordList: "{lua}:log:record" - processingHash: "{lua}:parser:processing" - scheduled: - cron: "0/2 * * * * ?" - limitTime: 2 # Unit: hour - limitCount: 2 # number of count + delayedSeconds: 10 + tryTimes: 5 + redis: + delayedQueue: "{lua}:delayed:task" + processing: "{lua}:detected:processing" + + # Parser settings (from task-parser) + parser: + maxConcurrentParsers: 10 + + # RedisConsumer DISABLED: parsing is now done in-process via AnalyzerBridge. + # Setting enable=false prevents RedisConsumer from starting. + redisConsumer: + enable: false + + opensearch: + yarn-app-index: "compass-yarn-app" + spark-app-index: "compass-spark-app" + job-index: "compass-job-analysis" + app-index: "compass-task-app" + job-instance-index: "compass-job-instance" + jvmType: - Java HotSpot - OpenJDK - # Spark event log check parameters - # disable default: false, set it true to disable this detection item. + + # Detector thresholds (from task-parser) detector: - # spark sparkEnvironmentConfig: jvmInformation: - Java Version @@ -74,103 +132,62 @@ custom: speculativeTaskConfig: disable: false threshold: 20 - # app duration(ms) duration: 300000 memWasteConfig: disable: false threshold: 70 - # app duration(ms) duration: 300000 cpuWasteConfig: disable: false - # app duration(ms) duration: 300000 driverThreshold: 50 executorThreshold: 50 jobDurationConfig: disable: false threshold: 30 - # job duration(ms) duration: 300000 stageDurationConfig: disable: false threshold: 30 - # stage duration(ms) duration: 300000 taskDurationConfig: disable: false threshold: 10 - # app duration(ms) duration: 300000 hdfsStuckConfig: disable: false threshold: 10 - # MB taskSize: 10 - # task duration(ms) duration: 300000 globalSortConfig: disable: false taskCount: 1 records: 50000000 - # task duration duration: 1200000 largeTableScanConfig: disable: false scanTable: 50000000000 - # app duration(ms) duration: 1200000 oomWarnConfig: disable: false oom: 40 - # app duration(ms) duration: 1200000 broadcastRows: 5000000 broadcastRowsOom: 30 dataSkewConfig: disable: false - # app duration(ms) duration: 1200000 - # stage耗时/任务总耗时 stageDurationPercentage: 20 interval: - # 0w-5w - - start: 0 - end: 50000 - threshold: 0 - # 5w-10w - - start: 50000 - end: 100000 - threshold: 100 - # 10w-100w - - start: 100000 - end: 1000000 - threshold: 50 - # 100w-500w - - start: 1000000 - end: 5000000 - threshold: 10 - # 500w-2000w - - start: 5000000 - end: 20000000 - threshold: 5 - # 2000w-3000w - - start: 20000000 - end: 30000000 - threshold: 3.5 - # 3000w-4000w - - start: 30000000 - end: 40000000 - threshold: 3 - # 4000w-5000w - - start: 40000000 - end: 50000000 - threshold: 2.25 - # 5000w - - start: 50000000 - end: 0 - threshold: 2 - # mapreduce + - {start: 0, end: 50000, threshold: 0} + - {start: 50000, end: 100000, threshold: 100} + - {start: 100000, end: 1000000, threshold: 50} + - {start: 1000000, end: 5000000, threshold: 10} + - {start: 5000000, end: 20000000, threshold: 5} + - {start: 20000000, end: 30000000, threshold: 3.5} + - {start: 30000000, end: 40000000, threshold: 3} + - {start: 40000000, end: 50000000, threshold: 2.25} + - {start: 50000000, end: 0, threshold: 2} mrEnvironmentConfig: keys: - mapreduce.map.cpu.vcores @@ -182,7 +199,6 @@ custom: disable: false mapThreshold: 80 reduceThreshold: 80 - # task duration(ms) duration: 1200000 mrLargeTableScanConfig: disable: false @@ -190,13 +206,10 @@ custom: duration: 1200000 mrDataSkewConfig: disable: false - # MB taskSize: 500 - # task duration(ms) taskDuration: 900000 mapThreshold: 2 reduceThreshold: 2 - # app duration(ms) duration: 1200000 mrSpeculativeTaskConfig: disable: false @@ -204,15 +217,17 @@ custom: duration: 1200000 mrTaskDurationConfig: disable: false - # task duration(ms) taskDuration: 900000 mapThreshold: 2 reduceThreshold: 2 - # app duration(ms) duration: 1200000 mrGCConfig: disable: false mapThreshold: 0.1 reduceThreshold: 0.1 - # app duration(ms) duration: 1200000 + +spark: + io: + compression: + codec: "" diff --git a/task-application/src/bin/stop.sh b/task-application/src/bin/stop.sh deleted file mode 100644 index a86ad6397..000000000 --- a/task-application/src/bin/stop.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -HOME_DIR=$(cd $(dirname $0)/.. && pwd) -APP_NAME="task-application" -PID_FILE=${HOME_DIR}/tpid - -stop() { - if [ -f ${PID_FILE} ]; then - local pid=$(cat ${PID_FILE}) - echo $APP_NAME $pid - command="ps --pid ${pid}" - if [[ $(uname) == "Darwin" ]]; then - command="ps -p ${pid}" - fi - if eval ${command} >/dev/null; then - kill $pid && rm -f $PID_FILE - fi - sleep 5 - - if eval ${command} >/dev/null; then - kill -9 $pid - fi - - else - ps -ef | grep $APP_NAME | grep -v grep | awk '{print $2}' | xargs kill -9 - fi - rm -f $PID_FILE -} - -stop \ No newline at end of file diff --git a/task-application/src/main/java/com/oppo/cloud/application/TaskApplication.java b/task-application/src/main/java/com/oppo/cloud/application/TaskApplication.java deleted file mode 100644 index 508f7393b..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/TaskApplication.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.scheduling.annotation.EnableScheduling; - -@SpringBootApplication(scanBasePackages = "com.oppo.cloud") -@ComponentScan(basePackages = "com.oppo.cloud") -@EnableScheduling -public class TaskApplication { - - public static void main(String[] args) { - SpringApplication.run(TaskApplication.class, args); - } -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/config/CustomConfig.java b/task-application/src/main/java/com/oppo/cloud/application/config/CustomConfig.java deleted file mode 100644 index 4248b1dfb..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/config/CustomConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.config; - -import com.oppo.cloud.application.domain.Rule; -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; - -import java.util.List; - -@Configuration -@ConfigurationProperties(prefix = "custom") -@Data -public class CustomConfig { - - /** - * Log parsing rules - */ - private List rules; -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/config/HadoopConfig.java b/task-application/src/main/java/com/oppo/cloud/application/config/HadoopConfig.java deleted file mode 100644 index 20ec7ae04..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/config/HadoopConfig.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.config; - - -import com.oppo.cloud.common.domain.cluster.hadoop.NameNodeConf; -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Hadoop Namenode cluster information - */ -@Configuration -@ConfigurationProperties(prefix = "hadoop") -@Data -@EnableConfigurationProperties(HadoopConfig.class) -public class HadoopConfig { - /** - * Hadoop Namenode cluster information - */ - private List namenodes; -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/config/KafkaConfig.java b/task-application/src/main/java/com/oppo/cloud/application/config/KafkaConfig.java deleted file mode 100644 index 2029ca0dd..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/config/KafkaConfig.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.config; - -import lombok.Data; -import org.apache.kafka.clients.admin.AdminClient; -import org.apache.kafka.clients.admin.AdminClientConfig; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.RoundRobinAssignor; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.annotation.EnableKafka; -import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.core.ConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.listener.ContainerProperties; - -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Kafka configuration - */ -@Configuration -@EnableKafka -@Data -public class KafkaConfig { - - /** - * Consume topic - */ - @Value("${spring.kafka.topics}") - private String topics; - /** - * Topic to save application metadata - */ - @Value("${spring.kafka.producer.topics:task-application}") - private String producerTopics; - /** - * Consume group - */ - @Value("${spring.kafka.consumer.group-id}") - private String groupId; - /** - * Consume mode: lastest, earliest - */ - @Value("${spring.kafka.consumer.auto-offset-reset}") - private String autoOffsetReset; - /** - * kafka broker cluster address - */ - @Value("${spring.kafka.bootstrap-servers}") - private String bootstrapServers; - /** - * Maximum time interval between two consecutive consume - */ - @Value("${spring.kafka.consumer.max-poll-interval-ms}") - private String maxPollIntervalMs; - - /** - * Create consumer factory - */ - @Bean - public ConsumerFactory consumerFactory() { - return new DefaultKafkaConsumerFactory<>(consumerConfig(), new StringDeserializer(), new StringDeserializer()); - } - - /** - * Configure consumer - */ - public Map consumerConfig() { - return Stream.of( - new AbstractMap.SimpleEntry<>(ConsumerConfig.CLIENT_ID_CONFIG, consumerClientId()), - new AbstractMap.SimpleEntry<>(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers), - new AbstractMap.SimpleEntry<>(ConsumerConfig.GROUP_ID_CONFIG, groupId), - new AbstractMap.SimpleEntry<>(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset), - new AbstractMap.SimpleEntry<>(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"), - new AbstractMap.SimpleEntry<>(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, maxPollIntervalMs), - new AbstractMap.SimpleEntry<>(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, - RoundRobinAssignor.class.getName()), - new AbstractMap.SimpleEntry<>(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class), - new AbstractMap.SimpleEntry<>(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - /** - * Configure consume listener - */ - @Bean - public ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory(ConsumerFactory consumerFactory) { - - ConcurrentKafkaListenerContainerFactory factory = - new ConcurrentKafkaListenerContainerFactory<>(); - - factory.setConsumerFactory(consumerFactory()); - factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE); - return factory; - } - - /** - * Get clientId: Manifested as Kafka memberId prefix - */ - @Bean(name = "consumerId") - public String consumerClientId() { - return String.format("client-%d-%d", Thread.currentThread().getId(), new Random().nextInt()); - } - - /** - * Get consume topics - */ - @Bean(name = "topics") - public String getTopics() { - return this.topics; - } - - /** - * Get brokers - */ - @Bean(name = "bootstrapServers") - public String getBootstrapServers() { - return this.bootstrapServers; - } - - /** - * Get groupId - */ - @Bean(name = "groupId") - public String getGroupId() { - return this.groupId; - } - - /** - * Using Kafka Admin Client API for operations - */ - @Bean(name = "kafkaAdminClient") - public AdminClient kafkaAdminClient() { - Properties properties = new Properties(); - properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); - return AdminClient.create(properties); - } -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/config/MyBatisConfig.java b/task-application/src/main/java/com/oppo/cloud/application/config/MyBatisConfig.java deleted file mode 100644 index c1b661718..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/config/MyBatisConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.config; - -import org.mybatis.spring.annotation.MapperScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.transaction.annotation.EnableTransactionManagement; - -/** - * Related database configuration - */ -@Configuration -@EnableTransactionManagement -@MapperScan({"com.oppo.cloud.mapper", "com.oppo.cloud.application.dao"}) -public class MyBatisConfig { -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/config/RedisConfig.java b/task-application/src/main/java/com/oppo/cloud/application/config/RedisConfig.java deleted file mode 100644 index 899d63636..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/config/RedisConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.config; - -import com.oppo.cloud.common.config.RedisTemplateConfig; -import org.springframework.context.annotation.Bean; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.Resource; -import org.springframework.data.redis.core.script.RedisScript; -import org.springframework.stereotype.Component; - -/** - * Redis configuration - */ -@Component -public class RedisConfig extends RedisTemplateConfig { - - @Bean - public RedisScript delayedTaskScript() { - Resource scriptSource = new ClassPathResource("scripts/delayedTask.lua"); - return RedisScript.of(scriptSource, Object.class); - } - -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/config/ThreadPoolConfig.java b/task-application/src/main/java/com/oppo/cloud/application/config/ThreadPoolConfig.java deleted file mode 100644 index 785d395fa..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/config/ThreadPoolConfig.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.config; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; - -import java.util.concurrent.Executor; -import java.util.concurrent.ThreadPoolExecutor; - -@Configuration -public class ThreadPoolConfig { - - public static final String DELAYED_QUEUE_EXECUTOR_POOL = "delayedQueueExecutorPool"; - - /** - * Delayed task thread pool - */ - @Bean(name = DELAYED_QUEUE_EXECUTOR_POOL) - public Executor delayQueueExecutorPool() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(4); - executor.setMaxPoolSize(8); - executor.setKeepAliveSeconds(120); - executor.setQueueCapacity(500); - executor.setThreadNamePrefix("delayed-queue-executor-"); - executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); - executor.setWaitForTasksToCompleteOnShutdown(true); - executor.setAwaitTerminationSeconds(60); - executor.initialize(); - return executor; - } -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/constant/RetCode.java b/task-application/src/main/java/com/oppo/cloud/application/constant/RetCode.java deleted file mode 100644 index 70a575be5..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/constant/RetCode.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.constant; - -/** - * Return codes for log processing results - */ -public enum RetCode { - RET_OK, RET_DATA_NOT_EXIST, RET_LOG_NOT_EXIST, RET_EXCEPTION, RET_SKIP, RET_OP_NEED_RETRY -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/consumer/ConsumerMessage.java b/task-application/src/main/java/com/oppo/cloud/application/consumer/ConsumerMessage.java deleted file mode 100644 index 852f1807f..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/consumer/ConsumerMessage.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.consumer; - -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.TypeReference; -import com.oppo.cloud.application.constant.RetCode; -import com.oppo.cloud.application.domain.DelayedTaskInfo; -import com.oppo.cloud.application.domain.ParseRet; -import com.oppo.cloud.application.service.DelayedTaskService; -import com.oppo.cloud.application.service.LogParserService; -import com.oppo.cloud.common.domain.syncer.TableMessage; -import com.oppo.cloud.model.TaskInstance; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.Consumer; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.kafka.annotation.KafkaListener; -import org.springframework.kafka.support.Acknowledgment; -import org.springframework.kafka.support.KafkaHeaders; -import org.springframework.messaging.handler.annotation.Header; -import org.springframework.messaging.handler.annotation.Payload; -import org.springframework.stereotype.Component; - -import java.util.Map; -import java.util.UUID; - -/** - * Consume task-instance task instances. - */ -@Slf4j -@Component -public class ConsumerMessage { - - /** - * Log parsing service - */ - @Autowired - private LogParserService logParserService; - - /** - * Delayed processing of task logs - */ - @Autowired - private DelayedTaskService delayedTaskService; - - /** - * Log consumption - */ - @KafkaListener(topics = "${spring.kafka.topics}", containerFactory = "kafkaListenerContainerFactory") - public void receive(@Payload String message, @Header(KafkaHeaders.RECEIVED_PARTITION_ID) int partition, - @Header(KafkaHeaders.RECEIVED_TOPIC) String topic, Consumer consumer, - Acknowledgment ack) throws Exception { - log.debug(String.format("%d, From partition %d: %s", consumer.hashCode(), partition, message)); - - TableMessage tableMessage = JSON.parseObject(message, TableMessage.class); - TaskInstance taskInstance = JSON.parseObject(tableMessage.getBody(), TaskInstance.class); - Map rawData = - JSON.parseObject(tableMessage.getRawData(), new TypeReference>() {}); - try { - ParseRet parseRet = logParserService.handle(taskInstance, rawData); - // Adding delay for retry - if (parseRet.getRetCode() == RetCode.RET_OP_NEED_RETRY) { - delayedTaskService - .pushDelayedQueue(new DelayedTaskInfo(UUID.randomUUID().toString(), 1, taskInstance, rawData)); - } - } catch (Exception e) { - log.error(e.getMessage()); - } - - consumer.commitSync(); - } -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/dao/TaskApplicationExtendMapper.java b/task-application/src/main/java/com/oppo/cloud/application/dao/TaskApplicationExtendMapper.java deleted file mode 100644 index 7a8df2597..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/dao/TaskApplicationExtendMapper.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.dao; - -import com.oppo.cloud.mapper.TaskApplicationMapper; -import com.oppo.cloud.model.TaskApplication; - -public interface TaskApplicationExtendMapper extends TaskApplicationMapper { - - int save(TaskApplication record); - -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/domain/DelayedTaskInfo.java b/task-application/src/main/java/com/oppo/cloud/application/domain/DelayedTaskInfo.java deleted file mode 100644 index 4c958e362..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/domain/DelayedTaskInfo.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.domain; - -import com.oppo.cloud.model.TaskInstance; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.Map; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class DelayedTaskInfo { - - /** - * Cache key - */ - private String key; - /** - * Number of retries - */ - private Integer tryTimes; - /** - * Task instance - */ - private TaskInstance taskInstance; - - /** - * Raw data for task execution - */ - Map rawData; -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/domain/ExtractLog.java b/task-application/src/main/java/com/oppo/cloud/application/domain/ExtractLog.java deleted file mode 100644 index 9ac95fa23..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/domain/ExtractLog.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.domain; - -import lombok.Data; - -/** - * Keyword extraction - */ -@Data -public class ExtractLog { - - /** - * match regular expression - */ - private String regex; - /** - * content name - */ - private String name; -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/domain/LogPathDep.java b/task-application/src/main/java/com/oppo/cloud/application/domain/LogPathDep.java deleted file mode 100644 index d75ae8691..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/domain/LogPathDep.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.domain; - -import lombok.Data; - -/** - * log path dependent on database field query - */ -@Data -public class LogPathDep { - - /** - * SQL query database - */ - private String query; -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/domain/LogPathJoin.java b/task-application/src/main/java/com/oppo/cloud/application/domain/LogPathJoin.java deleted file mode 100644 index ac17636b7..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/domain/LogPathJoin.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.domain; - -import lombok.Data; - -/** - * log path composition - */ -@Data -public class LogPathJoin { - - /** - * dependent data columns - */ - private String column; - /** - * parse log column with regular expression - */ - private String regex; - /** - * match log name - */ - private String name; - /** - * static data, no need for regular expression matching - */ - private String data; -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/domain/NameNode.java b/task-application/src/main/java/com/oppo/cloud/application/domain/NameNode.java deleted file mode 100644 index 7a3c88196..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/domain/NameNode.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.domain; - -import lombok.Data; - -import java.io.Serializable; - -/** - * NameNode configuration - */ -@Data -public class NameNode implements Serializable { - - /** - * name services - */ - private String nameservices; - /** - * namenode addresses - */ - private String[] namenodesAddr; - /** - * names of namenode - */ - private String[] namenodes; - /** - * user - */ - private String user; -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/domain/ParseRet.java b/task-application/src/main/java/com/oppo/cloud/application/domain/ParseRet.java deleted file mode 100644 index 42ca54c24..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/domain/ParseRet.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.domain; - -import com.oppo.cloud.application.constant.RetCode; -import com.oppo.cloud.model.TaskInstance; -import lombok.Data; - -import java.util.Map; - -/** - * Parsing result - */ -@Data -public class ParseRet { - - /** - * Return code - */ - private RetCode retCode; - /** - * TaskInstance related with applicationId in log - */ - private TaskInstance taskInstance; - /** - * raw column dependent data - */ - private Map rawData; - - public ParseRet(RetCode retCode, TaskInstance taskInstance) { - this.retCode = retCode; - this.taskInstance = taskInstance; - } -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/domain/RealtimeTaskInstance.java b/task-application/src/main/java/com/oppo/cloud/application/domain/RealtimeTaskInstance.java deleted file mode 100644 index 2c4820374..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/domain/RealtimeTaskInstance.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.domain; - -import com.oppo.cloud.model.TaskInstance; -import lombok.Data; - -@Data -public class RealtimeTaskInstance { - - TaskInstance taskInstance; - String applicationId; -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/domain/Rule.java b/task-application/src/main/java/com/oppo/cloud/application/domain/Rule.java deleted file mode 100644 index 925b1f0c2..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/domain/Rule.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.domain; - -import lombok.Data; - -import java.util.List; - -/** - * text parsing rules - */ -@Data -public class Rule { - - /** - * dependent query path - */ - private LogPathDep logPathDep; - /** - * log path field composition - */ - private List logPathJoins; - /** - * extraction rule - */ - private ExtractLog extractLog; -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/producer/MessageProducer.java b/task-application/src/main/java/com/oppo/cloud/application/producer/MessageProducer.java deleted file mode 100644 index 6bfa9d7b5..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/producer/MessageProducer.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.producer; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.stereotype.Component; -import org.springframework.util.concurrent.ListenableFutureCallback; - -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -/** - * producer consumer message - */ -@Component -public class MessageProducer { - - @Autowired - private KafkaTemplate kafkaTemplate; - - /** - * synchronous message sending - */ - public void sendMessageSync(String topic, - String message) throws ExecutionException, InterruptedException, TimeoutException { - kafkaTemplate.send(topic, message).get(10, TimeUnit.SECONDS); - } - - /** - * asynchronous message sending - */ - public void sendMessageAsync(String topic, String message) { - kafkaTemplate.send(topic, message).addCallback(new ListenableFutureCallback() { - - @Override - public void onFailure(Throwable ex) { - // todo: log - } - - @Override - public void onSuccess(Object result) { - // todo: log - } - }); - } -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/service/DelayedTaskService.java b/task-application/src/main/java/com/oppo/cloud/application/service/DelayedTaskService.java deleted file mode 100644 index 57c5851de..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/service/DelayedTaskService.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.service; - -import com.oppo.cloud.application.domain.DelayedTaskInfo; - -import java.util.List; - -public interface DelayedTaskService { - - /** - * add delayed queue task - */ - void pushDelayedQueue(DelayedTaskInfo delayedTaskInfo); - - /** - * "get delayed queue retry task - */ - List getDelayedTasks(); -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/service/LogParserService.java b/task-application/src/main/java/com/oppo/cloud/application/service/LogParserService.java deleted file mode 100644 index 86bb6da9f..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/service/LogParserService.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.service; - -import com.oppo.cloud.application.domain.ParseRet; -import com.oppo.cloud.model.TaskInstance; - -import java.util.Map; - -/** - * log parsing service - */ -public interface LogParserService { - - /** - * specific processing function - */ - ParseRet handle(TaskInstance taskInstance, Map rawData) throws Exception; - -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/service/impl/DelayedTaskServiceImpl.java b/task-application/src/main/java/com/oppo/cloud/application/service/impl/DelayedTaskServiceImpl.java deleted file mode 100644 index 0ccb4916e..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/service/impl/DelayedTaskServiceImpl.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.service.impl; - -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONArray; -import com.alibaba.fastjson2.JSONObject; -import com.oppo.cloud.application.domain.DelayedTaskInfo; -import com.oppo.cloud.application.service.DelayedTaskService; -import com.oppo.cloud.common.service.RedisService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.redis.core.script.RedisScript; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -@Slf4j -@Service -public class DelayedTaskServiceImpl implements DelayedTaskService { - - @Value("${custom.delayedTask.queue}") - private String queue; - - @Value("${custom.delayedTask.processing}") - private String processingKey; - - @Value("${custom.delayedTask.delayedSeconds}") - private Integer delayedSeconds; - - @Autowired - private RedisScript delayedTaskScript; - - @Autowired - private RedisService redisService; - - /** - * add delayed queue task - */ - @Override - public void pushDelayedQueue(DelayedTaskInfo delayedTaskInfo) { - redisService.zSetAdd(queue, JSON.toJSONString(delayedTaskInfo), System.currentTimeMillis() + delayedSeconds * 1000); - log.info("pushDelayedQueue:{},{}", queue, JSON.toJSONString(delayedTaskInfo)); - } - - /** - * get delayed queue retry task - */ - @Override - public List getDelayedTasks() { - Object delayTasks = redisService.executeScript(delayedTaskScript, Arrays.asList(queue, processingKey), - String.valueOf(System.currentTimeMillis())); - if (delayTasks == null) { - return null; - } - JSONArray data = (JSONArray) JSON.parse((String) delayTasks); - List taskList = new ArrayList<>(); - for (Object obj : data.toArray()) { - DelayedTaskInfo delayedTaskInfo = JSONObject.parseObject((String) obj, DelayedTaskInfo.class); - taskList.add(delayedTaskInfo); - } - return taskList; - } -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/service/impl/LogParserServiceImpl.java b/task-application/src/main/java/com/oppo/cloud/application/service/impl/LogParserServiceImpl.java deleted file mode 100644 index d7dcd9917..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/service/impl/LogParserServiceImpl.java +++ /dev/null @@ -1,482 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.service.impl; - -import com.alibaba.fastjson2.JSONObject; -import com.alibaba.fastjson2.JSON; -import com.oppo.cloud.application.config.CustomConfig; -import com.oppo.cloud.application.config.HadoopConfig; -import com.oppo.cloud.application.config.KafkaConfig; -import com.oppo.cloud.application.constant.RetCode; -import com.oppo.cloud.application.dao.TaskApplicationExtendMapper; -import com.oppo.cloud.application.domain.LogPathJoin; -import com.oppo.cloud.application.domain.ParseRet; -import com.oppo.cloud.application.domain.Rule; -import com.oppo.cloud.application.producer.MessageProducer; -import com.oppo.cloud.application.service.LogParserService; -import com.oppo.cloud.application.util.EscapePathUtil; -import com.oppo.cloud.application.util.HDFSUtil; -import com.oppo.cloud.application.util.StringUtil; -import com.oppo.cloud.common.domain.cluster.hadoop.NameNodeConf; -import com.oppo.cloud.model.TaskApplication; -import com.oppo.cloud.model.TaskInstance; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.DuplicateKeyException; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.stereotype.Service; - -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * log parsing service - */ -@Service -@Slf4j -public class LogParserServiceImpl implements LogParserService { - - /** - * Hadoop node resources - */ - @Autowired - private HadoopConfig hadoopConfig; - - @Autowired - private CustomConfig customConfig; - - @Autowired - private KafkaConfig kafkaConfig; - - @Autowired - private MessageProducer messageProducer; - - /** - * native SQL query - */ - @Autowired - private JdbcTemplate jdbcTemplate; - - /** - * task application table management - */ - @Autowired - private TaskApplicationExtendMapper taskApplicationExtendMapper; - /** - * Hadoop file read node configuration - */ - private Map nameNodeMap; - /** - * task success status - */ - private static final String TASK_STATE_SUCCESS = "success"; - - /** - * task failure status - */ - private static final String TASK_STATE_FAIL = "fail"; - - /** - * other task status - */ - private static final String TASK_STATE_OTHER = "other"; - - /** - * Flink Task Type - */ - private static final String TASK_TYPE_FLINK = "FLINK"; - - /** - * Spark Application Id - */ - private static final String APPLICATION_ID = "applicationId"; - - /** - * log address storage - */ - private final static String LOG_PATH_KEY = "__log_path"; - - /** - * Get Hadoop cluster configuration information - */ - public synchronized Map getNameNodeMap() { - if (nameNodeMap == null) { - nameNodeMap = initNameNode(); - } - return nameNodeMap; - } - - /** - * initialize configuration information - */ - public Map initNameNode() { - Map m = new HashMap<>(); - if (hadoopConfig.getNamenodes() == null) { - return m; - } - - for (NameNodeConf nameNodeConf : hadoopConfig.getNamenodes()) { - m.put(nameNodeConf.getNameservices(), nameNodeConf); - } - return m; - } - - /** - * filter task instance without matching rule. - */ - public boolean skipTaskInstance(TaskInstance taskInstance) { - // The instance status is either empty or not a success or failure status - if (StringUtils.isBlank(taskInstance.getTaskState())) { - return true; - } - if (taskInstance.getTaskState().equals(TASK_STATE_OTHER)) { - if (StringUtils.isBlank(taskInstance.getTaskType()) - || !taskInstance.getTaskType().equals(TASK_TYPE_FLINK)) { - return true; - } - } - return false; - } - - /** - * task processing - */ - @Override - public ParseRet handle(TaskInstance taskInstance, Map rawData) throws Exception { - log.debug("Receiving task instance,{}", taskInstance); - if (skipTaskInstance(taskInstance)) { - return new ParseRet(RetCode.RET_SKIP, null); - } - // retrieve complete data - Map data; - String sql = null; - Object[] args = null; - try { - if (taskInstance.getId() != null && taskInstance.getId() != 0) { - sql = "SELECT * FROM task_instance WHERE id = ?"; - args = new Object[]{taskInstance.getId()}; - } else { - sql = "SELECT * FROM task_instance WHERE flow_name = ? and task_name = ? and execution_time = ?"; - args = new Object[]{taskInstance.getFlowName(), taskInstance.getTaskName(), - taskInstance.getExecutionTime()}; - } - data = jdbcTemplate.queryForMap(sql, args); - } catch (Exception e) { - log.error("exception:" + e + ", sql=" + sql + ", args=" + Arrays.toString(args) + ",taskInstance=" - + taskInstance); - return new ParseRet(RetCode.RET_EXCEPTION, taskInstance); - } - // supplement other data dependencies - if (rawData != null) { - for (Map.Entry map : rawData.entrySet()) { - if (!data.containsKey(map.getKey())) { - data.put(map.getKey(), map.getValue()); - } - } - } - // get all information of task instance - taskInstance = new JSONObject(data).toJavaObject(TaskInstance.class); - - int count = 0; - List logPathList = new ArrayList<>(); - ParseRet parseRet = new ParseRet(RetCode.RET_SKIP, null); - - for (Rule rule : customConfig.getRules()) { - LogParser logParser = new LogParser(data, rule, count, taskInstance.getTaskType()); - try { - RetCode retCode = logParser.extract(); - parseRet = new ParseRet(retCode, taskInstance); - data = logParser.getData(); - logPathList.add((String) data.get(LOG_PATH_KEY)); - if (retCode != RetCode.RET_OK) { - break; - } - count++; // counting - } catch (Exception e) { - log.error("parseError:", e); - parseRet = new ParseRet(RetCode.RET_EXCEPTION, taskInstance); - break; - } - } - - log.debug("parseRet==>" + parseRet); - // return of parsing that is not successful - if (parseRet.getRetCode() != RetCode.RET_OK && parseRet.getRetCode() != RetCode.RET_DATA_NOT_EXIST) { - return parseRet; - } - - String logPath = String.join(",", logPathList); - // save applicationId - Object applicationId = data.get(APPLICATION_ID); - if (applicationId instanceof List) { - // remove duplicate applicationId - Set setId = new HashSet<>(); - for (Object appId : (List) applicationId) { - if (setId.add((String) appId)) { - addTaskApplication((String) appId, taskInstance, logPath); - } - } - } else { - addTaskApplication((String) applicationId, taskInstance, logPath); - } - log.info("project: {}, process:{}, task:{}, execute_time: {}, parse applicationId done!", - taskInstance.getProjectName(), taskInstance.getFlowName(), taskInstance.getTaskName(), - taskInstance.getExecutionTime()); - parseRet.setRetCode(RetCode.RET_OK); - return parseRet; - } - - /** - * add task application ID - */ - public void addTaskApplication(String applicationId, TaskInstance taskInstance, String logPath) { - // write data back to Kafka subscription - log.debug("application save: applicationId=" + applicationId + " task_instance=" + taskInstance + ",lopPath=" - + logPath); - TaskApplication taskApplication = getTaskApplication(applicationId, taskInstance, logPath); - try { - taskApplicationExtendMapper.save(taskApplication); - } catch (DuplicateKeyException e) { - return; - // duplicate key with return - } catch (Exception e) { - log.error("insertErr:" + e.getMessage()); - } - - try { - messageProducer.sendMessageSync(kafkaConfig.getProducerTopics(), - JSON.toJSONString(taskApplication)); - } catch (Exception ex) { - log.error("failed to send insert data to kafka, err: " + ex.getMessage()); - } - } - - private static TaskApplication getTaskApplication(String applicationId, TaskInstance taskInstance, String logPath) { - TaskApplication taskApplication = new TaskApplication(); - taskApplication.setApplicationId(applicationId); - taskApplication.setProjectName(taskInstance.getProjectName()); - taskApplication.setTaskName(taskInstance.getTaskName()); - taskApplication.setFlowName(taskInstance.getFlowName()); - taskApplication.setExecuteTime(taskInstance.getExecutionTime()); - taskApplication.setRetryTimes(taskInstance.getRetryTimes()); - taskApplication.setLogPath(logPath); - taskApplication.setTaskType(taskInstance.getTaskType()); - taskApplication.setCreateTime(new Date()); - taskApplication.setUpdateTime(new Date()); - return taskApplication; - } - - /** - * log parsing - */ - class LogParser { - - /** - * intermediate data storage - */ - private Map data; - /** - * log parsing rules - */ - private Rule rule; - - /** - * parsing sequence - */ - private final int index; - - /** - * delay time for reading Hadoop files - */ - private final int[] SLEEP_TIME = new int[]{20, 40, 60, 80, 100}; - - private static final String TMP_EXTENSION = ".tmp"; - - /** - * task type - */ - private String taskType = ""; - - public LogParser(Map data, Rule rule, int index, String taskType) { - this.data = data; - this.rule = rule; - this.index = index; - - if (taskType != null) { - this.taskType = taskType; - } - - } - - /** - * log extraction - */ - public RetCode extract() throws Exception { - if (!StringUtils.isBlank(rule.getLogPathDep().getQuery())) { - String sql = StringUtil.replaceParams(rule.getLogPathDep().getQuery(), data); - log.info("extract SQL:{}, data:{}", sql, data); - Map depData = null; - try { - depData = jdbcTemplate.queryForMap(sql); - } catch (Exception e) { - log.error(e.getMessage()); - return RetCode.RET_EXCEPTION; - } - - for (String key : depData.keySet()) { - data.put(key, depData.get(key)); - } - } - - String logPath = this.getLogPath(); - if (StringUtils.isBlank(logPath)) { - return RetCode.RET_DATA_NOT_EXIST; - } - - log.info("logPath:{}", logPath); - NameNodeConf nameNodeConf = HDFSUtil.getNameNode(getNameNodeMap(), logPath); - if (nameNodeConf == null) { - log.error("logPath: {} does not have hadoop config", logPath); - return RetCode.RET_EXCEPTION; - } - List filePaths; - try { - filePaths = HDFSUtil.filesPattern(nameNodeConf, String.format("%s*", logPath)); - } catch (Exception e) { - log.error("filesPattern {} error:", logPath, e); - return RetCode.RET_OP_NEED_RETRY; - } - if (filePaths.size() == 0) { - log.warn("logPath: {} does not exist", logPath); - return RetCode.RET_OP_NEED_RETRY; - } - String tmpFile = getTmpFile(filePaths); - if (tmpFile != null) { - log.warn("tmp file: {}", tmpFile); - return RetCode.RET_OP_NEED_RETRY; - } - - // record log path - logPath = String.join(",", filePaths); - data.put(LOG_PATH_KEY, logPath); - - boolean hasApplicationIds = false; - int countFileIfHasContent = 0; - Pattern pattern = Pattern.compile(rule.getExtractLog().getRegex()); - - for (String filePath : filePaths) { - String[] lines = HDFSUtil.readLines(nameNodeConf, filePath); - log.info(filePath + " has no log content"); - if (lines.length > 0) { - countFileIfHasContent += 1; - } - // extract keywords - for (String line : lines) { - Matcher matcher = pattern.matcher(line); - if (matcher.matches()) { - String matchVal = matcher.group(rule.getExtractLog().getName()); - - if (this.data.get(rule.getExtractLog().getName()) != null) { // The value already exists, and the original value becomes a list type. - Object val = this.data.get(rule.getExtractLog().getName()); - // If it is application ID, there may be multiple - if (val instanceof List) { - ((List) val).add(matchVal); - } else { - List l = new ArrayList(); - l.add(val); - l.add(matchVal); - val = l; - } - this.data.put(rule.getExtractLog().getName(), val); - } else { - this.data.put(rule.getExtractLog().getName(), matchVal); - } - hasApplicationIds = true; - } - } - } - - if (hasApplicationIds) { - return RetCode.RET_OK; - } - - // There may not be any logs - if (taskType.equals(TASK_TYPE_FLINK)) { - log.info("filePaths Count=" + filePaths.size() + ", hasContentCount=" + countFileIfHasContent); - return RetCode.RET_OP_NEED_RETRY; - } - - return RetCode.RET_DATA_NOT_EXIST; - } - - /** - * retrieve log path - */ - public String getLogPath() { - List paths = new ArrayList<>(); - for (LogPathJoin logPathJoin : rule.getLogPathJoins()) { - if (StringUtils.isBlank(logPathJoin.getColumn())) { - paths.add(logPathJoin.getData()); - } else { - log.info("logPathJoin:{}, data:{}", logPathJoin, data); - Object columnDataObj = this.data.get(logPathJoin.getColumn()); - if (columnDataObj == null) { - log.error("getColumnData value null, key=" + logPathJoin.getColumn() + ",data=" + this.data); - return ""; - } - String columnData = columnDataObj.toString(); - Pattern pattern = Pattern.compile(logPathJoin.getRegex()); - Matcher matcher = pattern.matcher(columnData); - if (matcher.matches()) { - String matchedData = matcher.group(logPathJoin.getName()); - matchedData = EscapePathUtil.escape(matchedData); - if (StringUtils.isNotBlank(logPathJoin.getData())) { - paths.add(logPathJoin.getData() + matchedData); - } else { - paths.add(matchedData); - } - } else { - log.error("`{}` does not match `{}`", logPathJoin.getRegex(), columnData); - } - } - } - - return String.join("/", paths); - } - - /** - * retrieve data - */ - public Map getData() { - return this.data; - } - - /** - * If it contains temporary file, need to retry and wait for completion. - */ - private String getTmpFile(List filePaths) { - for (String filePath : filePaths) { - if (filePath.endsWith(TMP_EXTENSION)) { - return filePath; - } - } - return null; - } - } -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/task/DelayedTask.java b/task-application/src/main/java/com/oppo/cloud/application/task/DelayedTask.java deleted file mode 100644 index b62c1bb85..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/task/DelayedTask.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.task; - -import com.oppo.cloud.application.config.ThreadPoolConfig; -import com.oppo.cloud.application.constant.RetCode; -import com.oppo.cloud.application.domain.DelayedTaskInfo; -import com.oppo.cloud.application.domain.ParseRet; -import com.oppo.cloud.application.service.DelayedTaskService; -import com.oppo.cloud.application.service.LogParserService; -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.model.TaskInstance; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.CommandLineRunner; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Configuration; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import javax.annotation.Resource; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executor; - -/** - * process log delay task - */ -@Slf4j -@Component -@Configuration -@ConditionalOnProperty(prefix = "custom.delayedTask", name = "enable", havingValue = "true") -public class DelayedTask implements CommandLineRunner { - - @Value("${custom.delayedTask.queue}") - private String queue; - - @Value("${custom.delayedTask.processing}") - private String processingKey; - - @Value("${custom.delayedTask.delayedSeconds}") - private Integer delayedSeconds; - - @Value("${custom.delayedTask.tryTimes}") - private Integer tryTimes; - - @Resource(name = ThreadPoolConfig.DELAYED_QUEUE_EXECUTOR_POOL) - private Executor delayedQueueExecutorPool; - - @Autowired - private DelayedTaskService delayedTaskService; - - @Autowired - private LogParserService logParserService; - - @Autowired - private RedisService redisService; - - @PostConstruct - void init() { - // load tasks interrupted due to restart - Map processingMap = null; - try { - processingMap = redisService.hGetAll(processingKey); - } catch (Exception e) { - log.error("get processing key err:", e); - } - if (processingMap != null) { - log.info("initProcessingTaskSize:{}", processingMap.size()); - processingMap.forEach((k, v) -> { - log.info("initProcessingTaskData,k:{},v:{}", k, v); - redisService.zSetAdd(queue, v, System.currentTimeMillis()); - redisService.hDel(processingKey, k); - }); - } - } - - @Override - public void run(String... args) throws Exception { - while (true) { - try { - List delayedTaskInfoList = delayedTaskService.getDelayedTasks(); - if (delayedTaskInfoList == null) { - Thread.sleep(delayedSeconds * 1000); - continue; - } - for (DelayedTaskInfo delayedTaskInfo : delayedTaskInfoList) { - delayedQueueExecutorPool.execute(() -> handleDelayTask(delayedTaskInfo)); - } - } catch (Exception e) { - log.error("Exception:", e); - } - } - } - - public void handleDelayTask(DelayedTaskInfo delayedTaskInfo) { - log.info("delayProcessTask:{}", delayedTaskInfo); - TaskInstance instance = delayedTaskInfo.getTaskInstance(); - if (instance == null) { - redisService.hDel(processingKey, delayedTaskInfo.getKey()); - return; - } - Map rawData = delayedTaskInfo.getRawData(); - try { - ParseRet parseRet = logParserService.handle(instance, rawData); - // retry unsuccessful - if (parseRet.getRetCode() != RetCode.RET_OK) { - if (delayedTaskInfo.getTryTimes() > tryTimes) { - log.error("discard delay task:{}", delayedTaskInfo); - redisService.hDel(processingKey, delayedTaskInfo.getKey()); - return; - } - delayedTaskInfo.setTryTimes(delayedTaskInfo.getTryTimes() + 1); - delayedTaskService.pushDelayedQueue(delayedTaskInfo); - } else { - // successfully delete the cache - redisService.hDel(processingKey, delayedTaskInfo.getKey()); - } - } catch (Exception e) { - log.error("delay task retry err: ", e); - } - } -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/util/EscapePathUtil.java b/task-application/src/main/java/com/oppo/cloud/application/util/EscapePathUtil.java deleted file mode 100644 index 20722d279..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/util/EscapePathUtil.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.util; - -public class EscapePathUtil { - - /** - * escape special characters - */ - public static String escape(String str) { - if (str != null) { -// return str.replaceAll("\\s+|:|\\.+|~", "_"); - return str.replaceAll("\\s+|:|~", "_"); - } - return null; - } -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/util/HDFSUtil.java b/task-application/src/main/java/com/oppo/cloud/application/util/HDFSUtil.java deleted file mode 100644 index 43d120a1d..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/util/HDFSUtil.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.util; - -import com.oppo.cloud.common.constant.Constant; -import com.oppo.cloud.common.domain.cluster.hadoop.NameNodeConf; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.security.UserGroupInformation; - -import java.io.ByteArrayOutputStream; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * HDFS utility - */ -public class HDFSUtil { - - /** - * Get NameNode, based on whether the configuration matchPathKeys is included in the path keywords or not - */ - public static NameNodeConf getNameNode(Map nameNodeMap, String filePath) { - for (String nameService : nameNodeMap.keySet()) { - NameNodeConf nameNodeConf = nameNodeMap.get(nameService); - for (String pathKey : nameNodeConf.getMatchPathKeys()) { - if (filePath.contains(pathKey)) { - return nameNodeConf; - } - } - } - return null; - } - - /** - * get FileSystem - */ - private static FileSystem getFileSystem(NameNodeConf nameNodeConf) throws Exception { - Configuration conf = new Configuration(false); - conf.setBoolean("fs.hdfs.impl.disable.cache", true); - - if (nameNodeConf.getNamenodes().length == 1) { - String defaultFs = - String.format("%s%s:%s", Constant.HDFS_SCHEME, nameNodeConf.getNamenodesAddr()[0], nameNodeConf.getPort()); - conf.set("fs.defaultFS", defaultFs); - if (nameNodeConf.isEnableKerberos()) { - return getAuthenticationFileSystem(nameNodeConf, conf); - } - URI uri = new URI(defaultFs); - return FileSystem.get(uri, conf); - } - - conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); - - String nameservices = nameNodeConf.getNameservices(); - - conf.set("fs.defaultFS", Constant.HDFS_SCHEME + nameservices); - conf.set("dfs.nameservices", nameservices); - conf.set("dfs.client.failover.proxy.provider." + nameservices, - "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"); - - for (int i = 0; i < nameNodeConf.getNamenodes().length; i++) { - String r = nameNodeConf.getNamenodes()[i]; - conf.set("dfs.namenode.rpc-address." + nameNodeConf.getNameservices() + "." + r, - nameNodeConf.getNamenodesAddr()[i] + ":" + nameNodeConf.getPort()); - } - - String nameNodes = String.join(",", nameNodeConf.getNamenodes()); - conf.set("dfs.ha.namenodes." + nameNodeConf.getNameservices(), nameNodes); - URI uri = new URI(Constant.HDFS_SCHEME + nameservices); - if (StringUtils.isNotBlank(nameNodeConf.getUser())) { - System.setProperty("HADOOP_USER_NAME", nameNodeConf.getUser()); - } - if (StringUtils.isNotBlank(nameNodeConf.getPassword())) { - System.setProperty("HADOOP_USER_PASSWORD", nameNodeConf.getPassword()); - } - if (nameNodeConf.isEnableKerberos()) { - return getAuthenticationFileSystem(nameNodeConf, conf); - } - return FileSystem.get(uri, conf); - } - - private static FileSystem getAuthenticationFileSystem(NameNodeConf nameNodeConf, Configuration conf) throws Exception { - conf.set("hadoop.security.authorization", "true"); - conf.set("hadoop.security.authentication", "kerberos"); - System.setProperty("java.security.krb5.conf", nameNodeConf.getKrb5Conf()); - conf.set("dfs.namenode.kerberos.principal.pattern", nameNodeConf.getPrincipalPattern()); - UserGroupInformation.setConfiguration(conf); - UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI( - nameNodeConf.getLoginUser(), nameNodeConf.getKeytabPath()); - return ugi.doAs((PrivilegedExceptionAction) () -> FileSystem.get(conf)); - } - - /** - * read file and return log content - */ - public static String[] readLines(NameNodeConf nameNodeConf, String filePath) throws Exception { - FSDataInputStream fsDataInputStream = null; - try { - FileSystem fs = HDFSUtil.getFileSystem(nameNodeConf); - fsDataInputStream = fs.open(new Path(filePath)); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - // 64kb - byte[] buffer = new byte[65536]; - int byteRead; - - while ((byteRead = fsDataInputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, byteRead); - } - - byte[] bytes = outputStream.toByteArray(); - String datas = new String(bytes, StandardCharsets.UTF_8); - return datas.split("\n"); - } catch (Exception e) { - throw new Exception(String.format("failed to read file: %s, err: %s", filePath, e.getMessage())); - } finally { - if (Objects.nonNull(fsDataInputStream)) { - fsDataInputStream.close(); - } - } - } - - /** - * get file list with wildcard using * - */ - public static List filesPattern(NameNodeConf nameNodeConf, String filePath) throws Exception { - filePath = checkLogPath(nameNodeConf, filePath); - FileSystem fs = HDFSUtil.getFileSystem(nameNodeConf); - FileStatus[] fileStatuses = fs.globStatus(new Path(filePath)); - List result = new ArrayList<>(); - if (fileStatuses == null) { - return result; - } - - for (FileStatus fileStatus : fileStatuses) { - if (fs.exists(fileStatus.getPath())) { - result.add(fileStatus.getPath().toString()); - } - } - return result; - } - - private static String checkLogPath(NameNodeConf nameNode, String logPath) { - if (logPath.contains(Constant.HDFS_SCHEME)) { - return logPath; - } - return String.format("%s%s%s", Constant.HDFS_SCHEME, nameNode.getNameservices(), logPath); - } -} diff --git a/task-application/src/main/java/com/oppo/cloud/application/util/StringUtil.java b/task-application/src/main/java/com/oppo/cloud/application/util/StringUtil.java deleted file mode 100644 index 757046005..000000000 --- a/task-application/src/main/java/com/oppo/cloud/application/util/StringUtil.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.util; - -import lombok.extern.slf4j.Slf4j; - -import java.util.List; -import java.util.Map; - -/** - * string processing tool - */ -@Slf4j -public class StringUtil { - - /** - * replace template parameters - */ - public static String replaceParams(String template, Map params) { - for (String key : params.keySet()) { - if (params.get(key) == null) { - continue; - } - if (params.get(key) instanceof List) { - log.error("Wrong DataType for replaceParams, data: {} ", params.get(key)); - continue; - } - template = template.replace("${" + key + "}", params.get(key).toString()); - } - return template; - } -} diff --git a/task-application/src/main/resources/application-airflow.yml b/task-application/src/main/resources/application-airflow.yml deleted file mode 100644 index 950f15fa2..000000000 --- a/task-application/src/main/resources/application-airflow.yml +++ /dev/null @@ -1,47 +0,0 @@ -spring: - datasource: - url: jdbc:mysql://localhost:33066/compass?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai - username: root - password: root - druid: - initial-size: 5 - min-idle: 10 - max-active: 20 - kafka: - bootstrap-servers: "localhost:9095" - topics: "task-instance" - consumer: - group-id: "cp-task-application" - auto-offset-reset: "earliest" - max-poll-interval-ms: 300000 - redis: - cluster: - nodes: localhost:6379 - max-redirects: 3 - password: - lettuce: - pool: - max-active: 32 - max-idle: 16 - min-idle: 8 - -custom: - delayedTask: - enable: true - queue: "{lua}:task:application" - processing: "{lua}:task-processing" - delayedSeconds: 5 - tryTimes: 20 - # parse through to the task's Application ID serially from top to bottom - rules: - - logPathDep: # log dependency query - query: "" # variable dependency query - logPathJoins: # concatenate log absolute path - - { "column": "", "data": "/flume/airflow" } # configure the HDFS root directory for storing scheduled logs - - { "column": "flow_name",data: "dag_id=", "regex": "(?.*)","name": "flowName" } - - { "column": "run_id",data: "run_id=", "regex": "(?.*)", "name": "runId" } - - { "column": "task_name",data: "task_id=","regex": "(?.*)", "name": "taskName" } - - { "column": "retry_times",data: "attempt=", "regex": "(?.*)", "name": "fileName" } - extractLog: # parse logs based on the assembled log path - regex: ".*(?application_[0-9]+_[0-9]+).*$" # matching rules - name: "applicationId" # match text name, must end with application ID diff --git a/task-application/src/main/resources/application-custom.yml b/task-application/src/main/resources/application-custom.yml deleted file mode 100644 index 988cf31e9..000000000 --- a/task-application/src/main/resources/application-custom.yml +++ /dev/null @@ -1,47 +0,0 @@ -spring: - datasource: - url: jdbc:mysql://localhost:33066/compass?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai - username: root - password: root - druid: - initial-size: 5 - min-idle: 10 - max-active: 20 - kafka: - bootstrap-servers: "localhost:9095" - topics: "task-instance" - consumer: - group-id: "cp-task-application" - auto-offset-reset: "earliest" - max-poll-interval-ms: 300000 - producer: # save task application metadata - topics: "task-application" - redis: - cluster: - nodes: localhost:6379 - max-redirects: 3 - password: - lettuce: - pool: - max-active: 32 - max-idle: 16 - min-idle: 8 - -custom: - delayedTask: - enable: true - queue: "{lua}:task:application" - processing: "{lua}:task-processing" - delayedSeconds: 5 - tryTimes: 20 - # parse through to the task's application ID serially from top to bottom - rules: - - logPathDep: # log dependency query - query: "select CASE WHEN end_time IS NOT NULL THEN DATE_ADD(end_time, INTERVAL 1 second) ELSE start_time END as end_time,log_path from t_ds_task_instance where id=${id}" # 查询, id 是 task-instance表的id - logPathJoins: # concatenate log absolute path - - { "column": "", "data": "/flume/custom" } # configure the HDFS root directory for storing scheduled logs - - { "column": "end_time", "regex": "^.*(?\\d{4}-\\d{2}-\\d{2}).+$", "name": "date" } - - { "column": "log_path", "regex": "^.*logs/(?.*)$", "name": "logpath" } - extractLog: # parse logs based on the assembled log path - regex: ".*(?application_[0-9]+_[0-9]+).*$" # matching rules - name: "applicationId" # match text name, must end with application ID diff --git a/task-application/src/main/resources/application-dolphinscheduler.yml b/task-application/src/main/resources/application-dolphinscheduler.yml deleted file mode 100644 index 25591a3c5..000000000 --- a/task-application/src/main/resources/application-dolphinscheduler.yml +++ /dev/null @@ -1,48 +0,0 @@ -spring: - datasource: - url: jdbc:mysql://localhost:33066/compass?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai - username: root - password: root - druid: - initial-size: 5 - min-idle: 10 - max-active: 20 - kafka: - bootstrap-servers: "localhost:9095" - topics: "task-instance" - consumer: - group-id: "cp-task-application" - auto-offset-reset: "earliest" - max-poll-interval-ms: 300000 - taskApplicationTopic: "task-application" - redis: - cluster: - nodes: localhost:6379 - max-redirects: 3 - password: - lettuce: - pool: - max-active: 32 - max-idle: 16 - min-idle: 8 - -custom: - delayedTask: - enable: true - queue: "{lua}:task:application" - processing: "{lua}:task-processing" - delayedSeconds: 5 - tryTimes: 20 - # parse through to the task's application ID serially from top to bottom - rules: - - logPathDep: # log dependency query - query: "SELECT CASE WHEN end_time IS NOT NULL THEN end_time ELSE start_time END as end_time,log_path FROM t_ds_task_instance WHERE id=${id}" # task-instance id - logPathJoins: # compose log path - # end_time: 2023-02-18 01:43:11 - # log_path: ../logs/6354680786144_1/3/4.log - - { "column": "", "data": "/flume/dolphinscheduler" } # configure the HDFS root directory for storing scheduled logs - - { "column": "end_time", "regex": "^.*(?\\d{4}-\\d{2}-\\d{2}).+$", "name": "date" } - - { "column": "log_path", "regex": "^.*logs/(?.*)$", "name": "logpath" } - extractLog: # parse logs based on the assembled log path - regex: ".*(?application_[0-9]+_[0-9]+).*$" # matching rules - name: "applicationId" # match a text name that must end with the application ID diff --git a/task-application/src/main/resources/application-hadoop.yml b/task-application/src/main/resources/application-hadoop.yml deleted file mode 100644 index 50036dcb0..000000000 --- a/task-application/src/main/resources/application-hadoop.yml +++ /dev/null @@ -1,21 +0,0 @@ -hadoop: - namenodes: - - nameservices: logs-hdfs - namenodesAddr: [ "host1", "host2" ] - namenodes: ["namenode1", "namenode2"] - # Optional - user: hdfs - # Optional - password: - port: 8020 - matchPathKeys: [ "flume" ] - # kerberos - enableKerberos: false - # /etc/krb5.conf - krb5Conf: "" - # hdfs/*@EXAMPLE.COM - principalPattern: "" - # admin - loginUser: "" - # /var/kerberos/krb5kdc/admin.keytab - keytabPath: "" diff --git a/task-application/src/main/resources/application.yml b/task-application/src/main/resources/application.yml deleted file mode 100644 index f27e3a01c..000000000 --- a/task-application/src/main/resources/application.yml +++ /dev/null @@ -1,15 +0,0 @@ -server: - port: 7070 - -spring: - application: - name: task-application - profiles: - active: dolphinscheduler,hadoop - -mybatis: - mapper-locations: - - classpath:dao/*.xml - - classpath*:com/**/mapper/*.xml - configuration: - log-impl: org.apache.ibatis.logging.stdout.StdOutImpl diff --git a/task-application/src/main/resources/dao/TaskApplicationExtendMapper.xml b/task-application/src/main/resources/dao/TaskApplicationExtendMapper.xml deleted file mode 100644 index 9e0728b96..000000000 --- a/task-application/src/main/resources/dao/TaskApplicationExtendMapper.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - insert into task_application (application_id, task_name, flow_name, - project_name, execute_time, retry_times, - create_time, update_time, log_path - ) - values (#{applicationId,jdbcType=VARCHAR}, #{taskName,jdbcType=VARCHAR}, #{flowName,jdbcType=VARCHAR}, - #{projectName,jdbcType=VARCHAR}, #{executeTime,jdbcType=TIMESTAMP}, #{retryTimes,jdbcType=INTEGER}, - #{createTime,jdbcType=TIMESTAMP}, #{updateTime,jdbcType=TIMESTAMP}, #{logPath,jdbcType=LONGVARCHAR} - ) - - diff --git a/task-application/src/main/resources/logback.xml b/task-application/src/main/resources/logback.xml deleted file mode 100644 index 4b4a952ad..000000000 --- a/task-application/src/main/resources/logback.xml +++ /dev/null @@ -1,105 +0,0 @@ - - - - - logback - - - - - - - info - - - ${LOG_PATTERN_COLOR} - - - - - - logs/task-application.info.%d{yyyy-MM-dd}.%i.log - 7 - - 500MB - - 2GB - - - INFO - ACCEPT - DENY - - - ${LOG_PATTERN} - - - - - - logs/task-application.error.%d{yyyy-MM-dd}.%i.log - 7 - - 500MB - - 2GB - - - ERROR - ACCEPT - DENY - - - ${LOG_PATTERN} - - - - - - logs/task-application.warn.%d{yyyy-MM-dd}.%i.log - 7 - - 500MB - - 2GB - - - WARN - ACCEPT - DENY - - - ${LOG_PATTERN} - - - - - - logs/task-application.debug.%d{yyyy-MM-dd}.%i.log - 7 - - 500MB - - 2GB - - - debug - ACCEPT - DENY - - - ${LOG_PATTERN} - - - - - - - - - - - - - - \ No newline at end of file diff --git a/task-application/src/main/resources/scripts/delayedTask.lua b/task-application/src/main/resources/scripts/delayedTask.lua deleted file mode 100644 index d7d4e4de3..000000000 --- a/task-application/src/main/resources/scripts/delayedTask.lua +++ /dev/null @@ -1,16 +0,0 @@ -redis.replicate_commands() -local delayedQueue= KEYS[1] -local processingKey= KEYS[2] -local tasks = redis.call('ZRANGEBYSCORE', delayedQueue, '0', ARGV[1]) -if next(tasks) ~= nil then - for _, task in ipairs(tasks) do - local msg = cjson.decode(task) - if msg.key ~= nil then - redis.call("HSET", processingKey, msg.key, task) - end - redis.call('ZREM', delayedQueue, task) - end - return cjson.encode(tasks) -end -return nil - diff --git a/task-application/src/test/java/com/oppo/cloud/application/config/TestCustomConfig.java b/task-application/src/test/java/com/oppo/cloud/application/config/TestCustomConfig.java deleted file mode 100644 index c7b76416e..000000000 --- a/task-application/src/test/java/com/oppo/cloud/application/config/TestCustomConfig.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.config; - -import com.alibaba.fastjson2.JSON; -import com.oppo.cloud.application.domain.LogPathJoin; -import com.oppo.cloud.application.domain.Rule; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.jdbc.core.JdbcTemplate; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -@SpringBootConfiguration -@SpringBootTest -@ComponentScan(basePackages = "com.oppo.cloud") -public class TestCustomConfig { - - @Autowired - private CustomConfig customConfig; - - @Autowired - private HadoopConfig hadoopConfig; - - @Autowired - private JdbcTemplate jdbcTemplate; - - - @Test - public void testParseNamenodeConfig() { - Assertions.assertTrue(hadoopConfig.getNamenodes() != null, "namenode config should not be null"); - Assertions.assertTrue(hadoopConfig.getNamenodes().size() > 0, "namenode config should not be empty"); - } - - @Test - public void testParseRuleConfig() { - Assertions.assertTrue(customConfig.getRules() != null, "rules should not be null"); - Assertions.assertTrue(customConfig.getRules().size() > 0, "rules should not be empty"); - } - - @Test - public void testParseRuleLog() { - String path = "/data1_1T/dolphinscheduler/logs/4563211890432_1/271/284.log"; - String regex = "^.*(?logs/.*)$"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(path); - - Assertions.assertTrue(matcher.matches(), "path should be matched"); - } - - @Test - public void testParseRuleLog2() { - String dateString = "2022-02-18 01:43:11"; - String regex = "^.*(?\\d{4}-\\d{2}-\\d{2}).+$"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(dateString); - Assertions.assertTrue(matcher.matches(), "date should be matched"); - Assertions.assertEquals(matcher.group("date"), "2022-02-18"); - } - - @Disabled - @Test - public void testParseRuleLog3() { - Map m = jdbcTemplate.queryForMap("select * from t_ds_task_instance where id=284"); - System.out.println(m); - - List paths = new ArrayList<>(); - for (Rule rule : customConfig.getRules()) { - for (LogPathJoin logPathJoin : rule.getLogPathJoins()) { - if (logPathJoin.getColumn() == null || logPathJoin.getColumn().isEmpty()) { - paths.add(logPathJoin.getData()); - } else { - String columnData = m.get(logPathJoin.getColumn()).toString(); - Pattern pattern = Pattern.compile(logPathJoin.getRegex()); - Matcher matcher = pattern.matcher(columnData); - if (matcher.matches()) { - String matchedData = matcher.group(logPathJoin.getName()); - paths.add(matchedData); - } - } - } - break; - } - System.out.println(paths); - String path = String.join("/", paths); - System.out.println(path); - } - - @Test - public void testParseString2Map() { - String str = "{\"age\": 24, \"name\": \"Bob\"}"; - Map m = JSON.parseObject(str); - System.out.println(m); - } - - @Test - public void testParseMatchTaskId() { - String message = - "[INFO] 2022-02-22 11:02:27.417 - [taskAppId=TASK-4574789155328_1-2413-2607]:[90] - execId: exec_id018013linkis-cg-entrancebjht3929:9104IDE_hdfs_spark_4, taskId: 2859"; - String regex = "^\\[INFO\\].+execId.+taskId:\\s*(?\\d+).*$"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(message); - Assertions.assertTrue(matcher.matches(), "taskId should be matched"); - Assertions.assertEquals(matcher.group("taskId"), "2859"); - } - - @Test - public void testParseMatchApplicationId() { - String message = "2022-02-22 11:02:28.002 INFO yarn application id: application_1642582961937_0382"; - String regex = "^.*(?application_[0-9]+_[0-9]+).*$"; - Pattern pattern = Pattern.compile(regex); - - Matcher matcher = pattern.matcher(message); - Assertions.assertTrue(matcher.matches(), "applicationId should be matched"); - Assertions.assertEquals(matcher.group("applicationId"), "application_1642582961937_0382"); - } - - @Test - public void testConvertDataType() { - Map m = new HashMap<>(); - String a = (String) m.get("a"); - System.out.println(a); - } -} diff --git a/task-application/src/test/java/com/oppo/cloud/application/service/impl/DelayedTaskServiceImplTest.java b/task-application/src/test/java/com/oppo/cloud/application/service/impl/DelayedTaskServiceImplTest.java deleted file mode 100644 index 936843900..000000000 --- a/task-application/src/test/java/com/oppo/cloud/application/service/impl/DelayedTaskServiceImplTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.service.impl; - -import com.oppo.cloud.application.domain.DelayedTaskInfo; -import com.oppo.cloud.application.service.DelayedTaskService; -import com.oppo.cloud.test.redis.WithRedisServer; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; - -import java.util.List; -import java.util.UUID; - -@Slf4j -@SpringBootTest -class DelayedTaskServiceImplTest implements WithRedisServer { - - @Autowired - private DelayedTaskService delayedTaskService; - - @Test - void pushDelayQueue() { - DelayedTaskInfo delayedTaskInfo = new DelayedTaskInfo(); - delayedTaskInfo.setKey(UUID.randomUUID().toString()); - delayedTaskService.pushDelayedQueue(delayedTaskInfo); - } - - @Test - void getDelayTasks() { - List delayedTaskInfoList = delayedTaskService.getDelayedTasks(); - log.info("delayTaskService:{}", delayedTaskInfoList); - } -} diff --git a/task-application/src/test/java/com/oppo/cloud/application/util/TestKafkaClient.java b/task-application/src/test/java/com/oppo/cloud/application/util/TestKafkaClient.java deleted file mode 100644 index 3f54561ca..000000000 --- a/task-application/src/test/java/com/oppo/cloud/application/util/TestKafkaClient.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.util; - -import org.apache.kafka.clients.admin.*; -import org.junit.jupiter.api.Test; - -import java.util.*; - -public class TestKafkaClient implements WithKafkaServer { - - @Test - public void testGetConsumers() { - Properties props = new Properties(); - props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaServer.getBootstrapServers()); - try (AdminClient client = AdminClient.create(props)) { - // List groups = client.listConsumerGroups().all().get() - // .stream().map(s -> s.groupId()).collect(Collectors.toList()); - List groups = new ArrayList<>(); - groups.add("task-application"); - Map descriptionMap = client.describeConsumerGroups(groups).all().get(); - System.out.println(descriptionMap); - for (String groupId : groups) { - ConsumerGroupDescription description = descriptionMap.get(groupId); - Collection members = description.members(); - System.out.print(groupId + " - "); - System.out.println(members); - } - } catch (Exception e) { - e.printStackTrace(); - } - - } -} diff --git a/task-application/src/test/java/com/oppo/cloud/application/util/TestStringUtil.java b/task-application/src/test/java/com/oppo/cloud/application/util/TestStringUtil.java deleted file mode 100644 index bea8efba7..000000000 --- a/task-application/src/test/java/com/oppo/cloud/application/util/TestStringUtil.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.util; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class TestStringUtil { - - @Test - public void testReplaceParams() { - String template = "This is ${name}'s template"; - Map params = new HashMap<>(); - params.put("name", "Bob"); - - String result = StringUtil.replaceParams(template, params); - Assertions.assertEquals(result, "This is Bob's template"); - } - - @Test - public void testCheckObjectType() { - Object o = new ArrayList(); - Assertions.assertTrue(o instanceof ArrayList); - Assertions.assertTrue(o instanceof List); - Assertions.assertFalse(o instanceof String); - - Map m = new HashMap<>(); - m.put("a", "b"); - - Object val = m.get("a"); - - List l = new ArrayList(); - l.add("c"); - l.add(val); - val = l; - m.put("a", val); - - Assertions.assertTrue(((List) m.get("a")).size() == 2); - - Object val2 = m.get("a"); - ((List) val2).add("d"); - Assertions.assertTrue(((List) m.get("a")).size() == 3); - } - - @Test - public void testEndwith() { - String s = - "hdfs://ansible-hdfs:8020/flume/dolphinscheduler/2022-03-18/logs/4839779932416_5/2996/3396.log.1647591885843.tmp"; - System.out.println(s.endsWith(".tmp")); - } -} diff --git a/task-application/src/test/java/com/oppo/cloud/application/util/WithKafkaServer.java b/task-application/src/test/java/com/oppo/cloud/application/util/WithKafkaServer.java deleted file mode 100644 index 1e2456999..000000000 --- a/task-application/src/test/java/com/oppo/cloud/application/util/WithKafkaServer.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.application.util; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.testcontainers.containers.KafkaContainer; -import org.testcontainers.utility.DockerImageName; - -public interface WithKafkaServer { - - String KAFKA_IMAGE_NAME = "confluentinc/cp-kafka:7.5.0"; - KafkaContainer kafkaServer = new KafkaContainer(DockerImageName.parse(KAFKA_IMAGE_NAME)); - - @BeforeAll - static void startKafkaServer() { - kafkaServer.start(); - } - - @AfterAll - static void stopKafkaServer() { - kafkaServer.stop(); - } -} - diff --git a/task-application/src/test/resources/application.yml b/task-application/src/test/resources/application.yml deleted file mode 100644 index a560d3256..000000000 --- a/task-application/src/test/resources/application.yml +++ /dev/null @@ -1,46 +0,0 @@ -server: - port: 7070 - -spring: - application: - name: task-application - profiles: - active: hadoop - datasource: - url: jdbc:derby:memory:diagnose_data;create=true - druid: - initial-size: 5 # 连接池初始化大小 - min-idle: 10 # 最小空闲连接数 - max-active: 20 # 最大连接数 - kafka: - bootstrap-servers: "localhost:9095" - topics: "task-instance" - consumer: - group-id: "cp-task-application" - auto-offset-reset: "earliest" # 从提交offset开始消费，无提交offset,从头开始消费 - max-poll-interval-ms: 300000 # 两次消费活跃时间间隔5min - taskApplicationTopic: "task-application" - redis: - host: localhost - port: 6379 - -custom: - delayedTask: - enable: false - queue: "{lua}:task:application" - processing: "{lua}:task-processing" - delayedSeconds: 5 - tryTimes: 20 - # 从上到下串行执行解析到任务的applicationId - rules: - - logPathDep: # 日志依赖查询 - query: "select CASE WHEN end_time IS NOT NULL THEN DATE_ADD(end_time, INTERVAL 1 second) ELSE start_time END as end_time,log_path from t_ds_task_instance where id=${id}" # 查询, id 是 task-instance表的id - logPathJoins: # 组成日志路径 - # end_time: 2023-02-18 01:43:11 - # log_path: ../logs/6354680786144_1/3/4.log - - { "column": "", "data": "/flume/dolphinscheduler" } # 配置存储调度日志的hdfs根目录 - - { "column": "end_time", "regex": "^.*(?\\d{4}-\\d{2}-\\d{2}).+$", "name": "date" } - - { "column": "log_path", "regex": "^.*logs/(?.*)$", "name": "logpath" } - extractLog: # 根据组装的日志路径解析日志 - regex: "^.*Submitted application (?application_[0-9]+_[0-9]+).*$" # 匹配规则 - name: "applicationId" # 匹配文本名，最后必须有applicationId \ No newline at end of file diff --git a/task-application/src/test/resources/log/2607.log b/task-application/src/test/resources/log/2607.log deleted file mode 100644 index 5b84cbc63..000000000 --- a/task-application/src/test/resources/log/2607.log +++ /dev/null @@ -1,10 +0,0 @@ -[INFO] 2022-02-22 11:02:27.148 - [taskAppId=TASK-4574789155328_1-2413-2607]:[42] - spark task params {"appName":"","deployMode":"client","driverCores":1,"driverMemory":"512M","executorCores":2,"executorMemory":"2G","mainArgs":"","mainClass":"","mainJar":{"id":45,"res":"20220218155540.py","resourceName":"/20220218155540.py"},"numExecutors":2,"others":"","programType":"PYTHON","resourceFilesList":[],"resourceList":[],"sparkVersion":"SPARK2"} -[INFO] 2022-02-22 11:02:27.152 - [taskAppId=TASK-4574789155328_1-2413-2607]:[128] - getResourceFile, filePath /tmp/dolphinscheduler_201/exec/process/4551996174208/4574789155328_1/2413/2607//20220218155540.py -[INFO] 2022-02-22 11:02:27.153 - [taskAppId=TASK-4574789155328_1-2413-2607]:[133] - getResourceFile, filePath /tmp/dolphinscheduler_201/exec/process/4551996174208/4574789155328_1/2413/2607//20220218155540.py, content rdd = sc.parallelize([1,2,3,4,5]) -print(rdd.getNumPartitions()) - -[INFO] 2022-02-22 11:02:27.156 - [taskAppId=TASK-4574789155328_1-2413-2607]:[61] - user : hdfs, content : [rdd = sc.parallelize([1,2,3,4,5]) -print(rdd.getNumPartitions()) -] -[INFO] 2022-02-22 11:02:27.417 - [taskAppId=TASK-4574789155328_1-2413-2607]:[90] - execId: exec_id018013linkis-cg-entrancebjht3929:9104IDE_hdfs_spark_4, taskId: 2859 -[INFO] 2022-02-22 11:02:31.669 - [taskAppId=TASK-4574789155328_1-2413-2607]:[105] - fileContents: [[2]] diff --git a/task-application/src/test/resources/log/2859.log b/task-application/src/test/resources/log/2859.log deleted file mode 100644 index 1f6f23bcf..000000000 --- a/task-application/src/test/resources/log/2859.log +++ /dev/null @@ -1,29 +0,0 @@ -2022-02-22 11:02:27.002 INFO Program is substituting variables for you -2022-02-22 11:02:27.002 INFO Variables substitution ended successfully -Job with jobId : IDE_hdfs_spark_4 and execID : IDE_hdfs_spark_4 submitted -2022-02-22 11:02:27.002 INFO You have submitted a new jobAnalysis, script code (after variable substitution) is -************************************SCRIPT CODE************************************ -rdd = sc.parallelize([1,2,3,4,5]) -print(rdd.getNumPartitions()) -************************************SCRIPT CODE************************************ -2022-02-22 11:02:27.002 INFO Your jobAnalysis is accepted, jobID is IDE_hdfs_spark_4 and taskID is 2859 in ServiceInstance(linkis-cg-entrance, bjht3929:9104). Please wait it to be scheduled -jobAnalysis is scheduled. -2022-02-22 11:02:27.002 INFO Your jobAnalysis is Scheduled. Please wait it to run. -Your jobAnalysis is being scheduled by orchestrator. -2022-02-22 11:02:28.002 INFO jobAnalysis is running. -2022-02-22 11:02:28.002 INFO Your jobAnalysis is Running now. Please wait it to complete. -Job with jobGroupId : 2859 and subJobId : 2846 was submitted to Orchestrator. -2022-02-22 11:02:28.002 INFO Background is starting a new engine for you, it may take several seconds, please wait -2022-02-22 11:02:28.002 INFO EngineConn local log path: ServiceInstance(linkis-cg-engineconn, bjht3929:27505) /home/service/app/dss-linkis/data/hdfs/workDir/801b248a-6c45-4a79-bca4-3450165dabea/logs -2022-02-22 11:02:28.002 INFO yarn application id: application_1642582961937_0382 -bjht3929:27505 >> rdd = sc.parallelize([1,2,3,4,5]) -print(rdd.getNumPartitions()) -2022-02-22 11:02:28.492 WARN [Thread-112] org.apache.hadoop.conf.Configuration 2670 loadProperty - /home/service/app/dss-linkis/deps/hadoop-2.7.2/etc/hadoop/core-site.xml:an attempt to override final parameter: hadoop.security.authorization; Ignoring. -2022-02-22 11:02:28.529 WARN [Thread-112] org.apache.hadoop.conf.Configuration 2670 loadProperty - /home/service/app/dss-linkis/deps/hadoop-2.7.2/etc/hadoop/core-site.xml:an attempt to override final parameter: hadoop.security.authorization; Ignoring. -2022-02-22 11:02:29.660 WARN [Linkis-Default-Scheduler-Thread-12] org.apache.hadoop.conf.Configuration 2670 loadProperty - /home/service/app/dss-linkis/deps/hadoop-2.7.2/etc/hadoop/core-site.xml:an attempt to override final parameter: hadoop.security.authorization; Ignoring. -Your subjob : 2846 execue with state succeed, has 1 resultsets. -Congratuaions! Your jobAnalysis : IDE_hdfs_spark_4 executed with status succeed and 0 results. -2022-02-22 11:02:30.002 INFO jobAnalysis is completed. -2022-02-22 11:02:30.002 INFO Task creation time(任务创建时间): 2022-02-22 11:02:27, Task scheduling time(任务调度时间): 2022-02-22 11:02:27, Task start time(任务开始时间): 2022-02-22 11:02:28, Mission end time(任务结束时间): 2022-02-22 11:02:30 -2022-02-22 11:02:30.002 INFO Your mission(您的任务) 2859 The total time spent is(总耗时时间为): 3.3 秒 -2022-02-22 11:02:30.002 INFO Congratulations. Your jobAnalysis completed with status Success. \ No newline at end of file diff --git a/task-assembly/assembly/assembly.xml b/task-assembly/assembly/assembly.xml index bef5cfe09..a4723d666 100644 --- a/task-assembly/assembly/assembly.xml +++ b/task-assembly/assembly/assembly.xml @@ -19,44 +19,35 @@ conf + - ${project.parent.basedir}/task-application/target/task-application - task-application + ${project.parent.basedir}/task-collector/target/task-collector + task-collector - ${project.parent.basedir}/task-canal/target/task-canal - task-canal + ${project.parent.basedir}/task-analyzer/target/task-analyzer + task-analyzer - ${project.parent.basedir}/task-canal-adapter/target/task-canal-adapter - task-canal-adapter - - - - ${project.parent.basedir}/task-detect/target/task-detect - task-detect - - - - ${project.parent.basedir}/task-metadata/target/task-metadata - task-metadata + ${project.parent.basedir}/task-portal/target/task-portal + task-portal - ${project.parent.basedir}/task-parser/target/task-parser - task-parser + ${project.parent.basedir}/task-syncer/target/task-syncer + task-syncer - ${project.parent.basedir}/task-portal/target/task-portal - task-portal + ${project.parent.basedir}/task-canal/target/task-canal + task-canal - ${project.parent.basedir}/task-syncer/target/task-syncer - task-syncer + ${project.parent.basedir}/task-canal-adapter/target/task-canal-adapter + task-canal-adapter diff --git a/task-assembly/bin/start_all.sh b/task-assembly/bin/start_all.sh index e948407d6..b7ed99544 100644 --- a/task-assembly/bin/start_all.sh +++ b/task-assembly/bin/start_all.sh @@ -13,11 +13,15 @@ if [ -f ${ENV_SH} ]; then done fi -# copy hadoop conf -cp /${HOME_DIR}/conf/application-hadoop.yml ${HOME_DIR}/task-application/conf -cp /${HOME_DIR}/conf/application-hadoop.yml ${HOME_DIR}/task-metadata/conf -cp /${HOME_DIR}/conf/application-hadoop.yml ${HOME_DIR}/task-parser/conf -cp /${HOME_DIR}/conf/application-hadoop.yml ${HOME_DIR}/task-flink/conf +# copy hadoop conf to modules that need it +HADOOP_CONF="${HOME_DIR}/conf/application-hadoop.yml" +if [ -f "${HADOOP_CONF}" ]; then + for target_dir in task-collector task-analyzer task-flink; do + if [ -d "${HOME_DIR}/${target_dir}/conf" ]; then + cp "${HADOOP_CONF}" "${HOME_DIR}/${target_dir}/conf/" + fi + done +fi TASK_CANAL_ENABLE=${TASK_CANAL_ENABLE:-"True"} diff --git a/task-application/pom.xml b/task-collector/pom.xml similarity index 73% rename from task-application/pom.xml rename to task-collector/pom.xml index 3b3e12df5..bc4b2689d 100644 --- a/task-application/pom.xml +++ b/task-collector/pom.xml @@ -1,5 +1,10 @@ - + @@ -10,10 +15,17 @@ 4.0.0 - task-application + task-collector + com.oppo.cloud ${revision} + task-collector + + + 17 + + com.oppo.cloud task-mbg @@ -24,14 +36,32 @@ + + + + com.oppo.cloud + task-common + + + io.springfox + springfox-boot-starter + + + + + org.springframework.kafka spring-kafka + + org.springframework.boot spring-boot-starter-jdbc + + org.apache.hadoop hadoop-common @@ -46,50 +76,44 @@ org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - com.sun.xml.bind - jaxb-impl + slf4j-reload4j - - org.slf4j - slf4j-reload4j - org.apache.hadoop hadoop-client - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - javax.servlet servlet-api - - com.google.code.findbugs - jsr305 - + + + + com.mysql + mysql-connector-j + + + org.apache.kafka kafka-clients - + + + org.springframework.boot + spring-boot-starter-actuator + + + io.micrometer + micrometer-registry-prometheus + + + org.apache.derby derby @@ -100,19 +124,6 @@ testcontainers test - - com.oppo.cloud - task-common - ${project.version} - test-jar - test - - - io.springfox - springfox-boot-starter - - - org.springframework.kafka spring-kafka-test @@ -124,16 +135,10 @@ test + ${project.artifactId} - - org.apache.maven.plugins - maven-surefire-plugin - - true - - org.apache.maven.plugins maven-assembly-plugin @@ -151,7 +156,7 @@ false - com.oppo.cloud.application.TaskApplication + com.oppo.cloud.collector.TaskCollectorApplication @@ -161,4 +166,5 @@ + diff --git a/task-application/src/assembly/assembly.xml b/task-collector/src/assembly/assembly.xml similarity index 89% rename from task-application/src/assembly/assembly.xml rename to task-collector/src/assembly/assembly.xml index 0f64079c2..c971dec6a 100644 --- a/task-application/src/assembly/assembly.xml +++ b/task-collector/src/assembly/assembly.xml @@ -1,3 +1,4 @@ + @@ -7,22 +8,22 @@ false + + ${project.basedir}/src/bin + ./bin + 0755 + ${project.basedir}/src/main/resources - **/scripts/** + **/com/** conf - - ${project.basedir}/src/bin - bin - 0755 - lib - \ No newline at end of file + diff --git a/task-detect/src/bin/startup.sh b/task-collector/src/bin/startup.sh old mode 100644 new mode 100755 similarity index 89% rename from task-detect/src/bin/startup.sh rename to task-collector/src/bin/startup.sh index de22f4166..d2967bc93 --- a/task-detect/src/bin/startup.sh +++ b/task-collector/src/bin/startup.sh @@ -1,7 +1,7 @@ -#!/bin/sh +#!/bin/bash HOME_DIR=$(cd $(dirname $0)/.. && pwd) -APP_NAME="task-detect" +APP_NAME="task-collector" PID_FILE=${HOME_DIR}/tpid if [ -f $(dirname $0)/compass_env.sh ]; then @@ -29,7 +29,7 @@ start() { JAVA_OPTS="-DappName=${APP_NAME} -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom" JAVA_OPTS_GC="-server -XX:+UseG1GC -XX:G1HeapRegionSize=8m -verbose:GC -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${HOME_DIR}/logs/dump.hprof -Xloggc:${HOME_DIR}/logs/gc-${APP_NAME}.log" - nohup java $JAVA_OPTS $JAVA_OPTS_GC -cp "${HOME_DIR}/conf":"${HOME_DIR}/lib/*" com.oppo.cloud.detect.TaskDetectApplication >/dev/null 2>&1 & + nohup java $JAVA_OPTS $JAVA_OPTS_GC -cp "${HOME_DIR}/conf":"${HOME_DIR}/lib/*" com.oppo.cloud.collector.TaskCollectorApplication >/dev/null 2>&1 & pid=$! echo $pid @@ -37,4 +37,3 @@ start() { } start - diff --git a/task-collector/src/bin/stop.sh b/task-collector/src/bin/stop.sh new file mode 100755 index 000000000..ca18b1f33 --- /dev/null +++ b/task-collector/src/bin/stop.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +HOME_DIR=$(cd $(dirname $0)/.. && pwd) +PID_FILE=${HOME_DIR}/tpid + +if [ -f ${PID_FILE} ]; then + pid=$(cat ${PID_FILE}) + if kill -0 ${pid} 2>/dev/null; then + kill ${pid} + echo "task-collector stopped (pid=${pid})" + else + echo "task-collector is not running" + fi + rm -f ${PID_FILE} +else + echo "PID file not found" +fi diff --git a/task-collector/src/main/java/com/oppo/cloud/collector/TaskCollectorApplication.java b/task-collector/src/main/java/com/oppo/cloud/collector/TaskCollectorApplication.java new file mode 100644 index 000000000..36c1bcf4f --- /dev/null +++ b/task-collector/src/main/java/com/oppo/cloud/collector/TaskCollectorApplication.java @@ -0,0 +1,55 @@ +/* + * Copyright 2023 OPPO. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.oppo.cloud.collector; + +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; + +/** + * task-collector: merged from task-application + task-metadata. + * + *

Unifies two services that both deal with "discovering YARN/Spark application IDs + * and their log paths": + * + *

    + *
  • task-application (event-driven): Kafka[task-instance] consumer → + * reads HDFS scheduler logs → extracts applicationId → publishes Kafka[task-application]
    • + *
  • task-metadata (scheduled polling): Spark HistoryServer + YARN RM REST APIs → + * writes app metadata to OpenSearch
    • + *
+ * + *

Both scan base packages are included so existing Spring components need no code changes. + * The two entry-point modes (Kafka listener + @Scheduled jobs) run concurrently in the same JVM. + */ +@SpringBootApplication(scanBasePackages = { + "com.oppo.cloud.collector", + "com.oppo.cloud.application", + "com.oppo.cloud.meta", + "com.oppo.cloud.common" +}) +@MapperScan({"com.oppo.cloud.mapper", "com.oppo.cloud.application.dao"}) +@EnableScheduling +@EnableAsync +public class TaskCollectorApplication { + + public static void main(String[] args) { + SpringApplication.run(TaskCollectorApplication.class, args); + } +} diff --git a/task-collector/src/main/resources/application-dolphinscheduler.yml b/task-collector/src/main/resources/application-dolphinscheduler.yml new file mode 100644 index 000000000..622f0c39e --- /dev/null +++ b/task-collector/src/main/resources/application-dolphinscheduler.yml @@ -0,0 +1,14 @@ +# DolphinScheduler integration profile (inherited from task-application) +custom: + rules: + - logPathDep: + # IMPORTANT: ${id} is replaced via PreparedStatement parameter binding (not string interpolation) + # to prevent SQL injection. See LogParserServiceImpl for implementation. + query: "SELECT CASE WHEN end_time IS NOT NULL THEN end_time ELSE start_time END as end_time,log_path FROM t_ds_task_instance WHERE id=?" + logPathJoins: + - { "column": "", "data": "/flume/dolphinscheduler" } + - { "column": "end_time", "regex": "^.*(?\\d{4}-\\d{2}-\\d{2}).+$", "name": "date" } + - { "column": "log_path", "regex": "^.*logs/(?.*)$", "name": "logpath" } + extractLog: + regex: ".*(?application_[0-9]+_[0-9]+).*$" + name: "applicationId" diff --git a/task-collector/src/main/resources/application-hadoop.yml b/task-collector/src/main/resources/application-hadoop.yml new file mode 100644 index 000000000..7b95dccd7 --- /dev/null +++ b/task-collector/src/main/resources/application-hadoop.yml @@ -0,0 +1,18 @@ +# Hadoop cluster configuration (configure your actual cluster details here) +hadoop: + namenodes: + - nameservices: logs-hdfs + namenodesAddr: [ "host1", "host2" ] + namenodes: [ "namenode1", "namenode2" ] + user: ${HDFS_USER:hdfs} + password: ${HDFS_PASSWORD:} + port: 8020 + matchPathKeys: [ "flume" ] + + yarn: + - clusterName: "bigdata" + resourceManager: [ "${YARN_RM_HOST:localhost}:8088" ] + jobHistoryServer: "${JOB_HISTORY_HOST:localhost}:19888" + + spark: + sparkHistoryServer: [ "${SPARK_HISTORY_HOST:localhost}:18018" ] diff --git a/task-collector/src/main/resources/application.yml b/task-collector/src/main/resources/application.yml new file mode 100644 index 000000000..f4dbfcee6 --- /dev/null +++ b/task-collector/src/main/resources/application.yml @@ -0,0 +1,85 @@ +server: + port: 7070 + +spring: + application: + name: task-collector + # Activate scheduler-specific profile (dolphinscheduler|airflow|custom) + hadoop profile + profiles: + active: dolphinscheduler,hadoop + task: + scheduling: + pool: + size: 10 + datasource: + url: ${DB_URL:jdbc:mysql://localhost:3306/compass?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai} + username: ${DB_USERNAME:} + password: ${DB_PASSWORD:} + druid: + initial-size: 5 + min-idle: 10 + max-active: 20 + kafka: + bootstrap-servers: ${KAFKA_BOOTSTRAP_SERVERS:localhost:9092} + topics: "task-instance" + consumer: + group-id: "cp-task-collector" + auto-offset-reset: "earliest" + max-poll-interval-ms: 300000 + taskApplicationTopic: "task-application" + redis: + host: ${REDIS_HOST:localhost} + port: ${REDIS_PORT:6379} + password: ${REDIS_PASSWORD:} + lettuce: + pool: + max-active: 8 + max-idle: 4 + min-idle: 1 + opensearch: + nodes: ${OPENSEARCH_NODES:localhost:9200} + username: ${OPENSEARCH_USERNAME:} + password: ${OPENSEARCH_PASSWORD:} + truststore: + truststore-password: + yarn-app-prefix: "compass-yarn-app-" + spark-app-prefix: "compass-spark-app-" + +mybatis: + mapper-locations: + - classpath:dao/*.xml + - classpath*:com/**/mapper/*.xml + configuration: + log-impl: org.apache.ibatis.logging.slf4j.Slf4jImpl + +management: + endpoints: + web: + exposure: + include: health,info,prometheus,metrics + endpoint: + health: + show-details: when_authorized + +# Delayed task queue (from task-application): retry tasks with pending applicationIds +custom: + delayedTask: + enable: true + queue: "{lua}:task:application" + processing: "{lua}:task-processing" + delayedSeconds: 5 + tryTimes: 20 + +# Scheduled metadata polling (from task-metadata) +scheduler: + clusterMeta: + enable: true + cron: "0 */30 * * * ?" + yarnMeta: + enable: true + cron: "0/30 * * * * ?" + startedTimeBegin: 6 # hours + sparkMeta: + enable: true + cron: "0/30 * * * * ?" + limitCount: 1000 diff --git a/task-detect/pom.xml b/task-detect/pom.xml deleted file mode 100644 index c8b067dbd..000000000 --- a/task-detect/pom.xml +++ /dev/null @@ -1,84 +0,0 @@ - - - - - compass - com.oppo.cloud - ${revision} - - 4.0.0 - - task-detect - com.oppo.cloud - ${revision} - - - - com.oppo.cloud - task-mbg - - - io.springfox - springfox-boot-starter - - - - - org.springframework.kafka - spring-kafka - - - - mysql - mysql-connector-java - - - - - org.apache.derby - derby - test - - - - - ${project.artifactId} - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - org.apache.maven.plugins - maven-assembly-plugin - - - make-assembly - package - - single - - - - - ${assembly.skip} - false - - - com.oppo.cloud.detect.TaskDetectApplication - - - - src/assembly/assembly.xml - - - - - - - \ No newline at end of file diff --git a/task-detect/src/bin/stop.sh b/task-detect/src/bin/stop.sh deleted file mode 100644 index 329623b50..000000000 --- a/task-detect/src/bin/stop.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -HOME_DIR=$(cd $(dirname $0)/.. && pwd) -APP_NAME="task-detect" -PID_FILE=${HOME_DIR}/tpid - - -stop() { - if [ -f ${PID_FILE} ]; then - local pid=$(cat ${PID_FILE}) - echo $APP_NAME $pid - command="ps --pid ${pid}" - if [[ $(uname) == "Darwin" ]]; then - command="ps -p ${pid}" - fi - if eval ${command} >/dev/null; then - kill $pid && rm -f $PID_FILE - fi - sleep 5 - - if eval ${command} >/dev/null; then - kill -9 $pid - fi - - else - ps -ef | grep $APP_NAME | grep -v grep | awk '{print $2}' | xargs kill -9 - fi - rm -f $PID_FILE -} - -stop \ No newline at end of file diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/TaskDetectApplication.java b/task-detect/src/main/java/com/oppo/cloud/detect/TaskDetectApplication.java deleted file mode 100644 index 0df3d3e04..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/TaskDetectApplication.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.scheduling.annotation.EnableScheduling; - -/** - * Task diagnosis startup. - */ -@SpringBootApplication(scanBasePackages = "com.oppo.cloud") -@ComponentScan(basePackages = "com.oppo.cloud") -@EnableScheduling -public class TaskDetectApplication { - - public static void main(String[] args) { - SpringApplication.run(TaskDetectApplication.class, args); - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/config/MyBatisConfig.java b/task-detect/src/main/java/com/oppo/cloud/detect/config/MyBatisConfig.java deleted file mode 100644 index e2558861d..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/config/MyBatisConfig.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.config; - -import org.mybatis.spring.annotation.MapperScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.transaction.annotation.EnableTransactionManagement; - -/** - * MyBatis Configuration - */ -@Configuration -@EnableTransactionManagement -@MapperScan({"com.oppo.cloud.mapper", "com.oppo.cloud.detect.mapper"}) -public class MyBatisConfig { - -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/config/OpenSearchConfig.java b/task-detect/src/main/java/com/oppo/cloud/detect/config/OpenSearchConfig.java deleted file mode 100644 index f1327497a..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/config/OpenSearchConfig.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.config; - -import com.oppo.cloud.common.util.opensearch.OpenSearchClient; -import org.opensearch.client.RestHighLevelClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; - -/** - * OpenSearch Config - */ -@Configuration -public class OpenSearchConfig { - - @Value("${spring.opensearch.nodes}") - private String nodes; - @Value("${spring.opensearch.username}") - private String username; - @Value("${spring.opensearch.password}") - private String password; - @Value("${spring.opensearch.truststore}") - private String truststore; - @Value("${spring.opensearch.truststore-password}") - private String truststorePassword; - - @Primary - @Bean("opensearch") - public RestHighLevelClient restHighLevelClient() { - return OpenSearchClient.create(nodes, username, password, truststore, truststorePassword); - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/config/RedisConfig.java b/task-detect/src/main/java/com/oppo/cloud/detect/config/RedisConfig.java deleted file mode 100644 index 0a578e847..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/config/RedisConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.config; - -import com.oppo.cloud.common.config.RedisTemplateConfig; -import org.springframework.context.annotation.Bean; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.Resource; -import org.springframework.data.redis.core.script.RedisScript; -import org.springframework.stereotype.Component; - -/** - * redis配置 - */ -@Component -public class RedisConfig extends RedisTemplateConfig { - - @Bean - public RedisScript delayTaskedScript() { - Resource scriptSource = new ClassPathResource("scripts/delayedTask.lua"); - return RedisScript.of(scriptSource, Object.class); - } - -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/config/ThreadPoolConfig.java b/task-detect/src/main/java/com/oppo/cloud/detect/config/ThreadPoolConfig.java deleted file mode 100644 index da8d3f967..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/config/ThreadPoolConfig.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.config; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; - -import java.util.concurrent.Executor; -import java.util.concurrent.ThreadPoolExecutor; - -/** - * Executor Thread Pool Configuration - */ -@Configuration -public class ThreadPoolConfig { - - public static final String DETECT_EXECUTOR_POOL = "detectExecutorPool"; - public static final String DELAY_QUEUE_EXECUTOR_POOL = "delayQueueExecutorPool"; - - /** - * Parsing task thread pool - */ - @Bean(name = DETECT_EXECUTOR_POOL) - public Executor detectExecutorPool() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(4); - executor.setMaxPoolSize(8); - executor.setKeepAliveSeconds(120); - executor.setQueueCapacity(500); - executor.setThreadNamePrefix("detect-executor-"); - executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); - executor.setWaitForTasksToCompleteOnShutdown(true); - executor.setAwaitTerminationSeconds(60); - executor.initialize(); - return executor; - } - - /** - * Delayed Task Thread Pool - */ - @Bean(name = DELAY_QUEUE_EXECUTOR_POOL) - public Executor delayQueueExecutorPool() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(4); - executor.setMaxPoolSize(8); - executor.setKeepAliveSeconds(120); - executor.setQueueCapacity(500); - executor.setThreadNamePrefix("delay-queue-executor-"); - executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); - executor.setWaitForTasksToCompleteOnShutdown(true); - executor.setAwaitTerminationSeconds(60); - executor.initialize(); - return executor; - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/detector/AlwaysFailed.java b/task-detect/src/main/java/com/oppo/cloud/detect/detector/AlwaysFailed.java deleted file mode 100644 index 34621225d..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/detector/AlwaysFailed.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.detector; - -import com.oppo.cloud.common.constant.JobCategoryEnum; -import com.oppo.cloud.common.constant.TaskStateEnum; -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.util.DateUtil; -import com.oppo.cloud.detect.domain.TaskStateHistory; -import com.oppo.cloud.detect.service.TaskInstanceService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Service; - -import java.util.Date; -import java.util.List; - -/** - * Long-term failure detector. - */ -@Order(3) -@Service -@Slf4j -public class AlwaysFailed extends DetectServiceImpl { - - @Value("${custom.detectionRule.alwaysFailedWarning}") - private Integer alwaysFailedWarning; - - @Autowired - private TaskInstanceService taskInstanceService; - - @Override - public void detect(JobAnalysis jobAnalysis) throws Exception { - if (jobAnalysis.getTaskState().equals(TaskStateEnum.success.name())) { - return; - } - Date endExecutionTime = DateUtil.getOffsetDate(jobAnalysis.getExecutionDate(), -60); - // Query the execution status of this task in the past two months - List taskStateHistories = - taskInstanceService.searchTaskStateHistory(jobAnalysis.getProjectName(), - jobAnalysis.getFlowName(), jobAnalysis.getTaskName(), jobAnalysis.getExecutionDate(), - endExecutionTime, 60); - log.debug("taskName:{}, executionTime:{}, taskStateHistories:{}", jobAnalysis.getTaskName(), - jobAnalysis.getEndTime(), taskStateHistories); - if (taskStateHistories == null || taskStateHistories.size() == 0) { - return; - } - Date lastSuccessDate = DateUtil.getOffsetDate(jobAnalysis.getExecutionDate(), -alwaysFailedWarning); - if (taskStateHistories.get(taskStateHistories.size() - 1).getExecutionTime().getTime() > lastSuccessDate - .getTime()) { - // Filter out tasks that have not been executed for less than 10 days - return; - } - long recentSuccess = 0L; - for (TaskStateHistory taskStateHistory : taskStateHistories) { - long executionTime = taskStateHistory.getExecutionTime().getTime() / 1000; - String state = taskStateHistory.getState(); - // Success tasks completed within a specified time. - if (state.equals(TaskStateEnum.success.name()) - && ((executionTime > jobAnalysis.getExecutionDate().getTime() / 1000 - - alwaysFailedWarning * 24 * 3600))) { - return; - } - // Record the most recent successful time within a time frame. - if (state.equals(TaskStateEnum.success.name())) { - recentSuccess = executionTime; - break; - } - } - if (recentSuccess != 0L) { - jobAnalysis.setSuccessExecutionDay(DateUtil.format(new Date(recentSuccess * 1000))); - jobAnalysis.setSuccessDays( - String.valueOf((jobAnalysis.getExecutionDate().getTime() / 1000 - recentSuccess) / (24 * 3600))); - } - jobAnalysis.getCategories().add(JobCategoryEnum.alwaysFailed.name()); - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/detector/DetectServiceImpl.java b/task-detect/src/main/java/com/oppo/cloud/detect/detector/DetectServiceImpl.java deleted file mode 100644 index c98e1ca04..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/detector/DetectServiceImpl.java +++ /dev/null @@ -1,284 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.detector; - -import com.alibaba.fastjson2.JSONObject; -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.domain.opensearch.SimpleUser; -import com.oppo.cloud.common.domain.job.App; -import com.oppo.cloud.common.domain.job.LogRecord; -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.common.util.DateUtil; -import com.oppo.cloud.detect.domain.AbnormalTaskAppInfo; -import com.oppo.cloud.detect.service.*; -import com.oppo.cloud.detect.util.DetectorUtil; -import com.oppo.cloud.mapper.*; -import com.oppo.cloud.model.*; -import lombok.extern.slf4j.Slf4j; -import org.apache.logging.log4j.util.Strings; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; - -import java.util.*; - -/** - * Job Detection Service - */ -@Slf4j -public abstract class DetectServiceImpl implements DetectService { - - @Autowired - private OpenSearchService openSearchService; - - @Value("${custom.opensearch.job-index}") - private String jobIndex; - - @Autowired - public TaskService taskService; - - @Autowired - private UserInfoMapper userMapper; - - @Autowired - private LogRecordService logRecordService; - - @Autowired - private TaskAppService taskAppService; - - @Autowired - private DelayedTaskService delayTaskService; - - @Autowired - private AbnormalJobService abnormalJobService; - - @Autowired - private RedisService redisService; - - @Autowired - public TaskInstanceService taskInstanceService; - - @Autowired - private JobInstanceService jobInstanceService; - - /** - * Parsing message transmission through a Redis queue. - */ - @Value("${custom.redis.logRecord}") - private String logRecordQueue; - - /** - * Task diagnosis - */ - @Override - public abstract void detect(JobAnalysis detectJobAnalysis) throws Exception; - - /** - * Normal task processing. - */ - @Override - public void handleNormalJob(JobAnalysis detectJobAnalysis) throws Exception { - // Update user information. - updateUserInfo(detectJobAnalysis); - // Query the appIds under this task. - AbnormalTaskAppInfo abnormalTaskAppInfo = taskAppService.getAbnormalTaskAppsInfo(detectJobAnalysis, null); - if (!"".equals(abnormalTaskAppInfo.getExceptionInfo())) { - // Send after constructed it. - delayTaskService.pushDelayedQueue(detectJobAnalysis, abnormalTaskAppInfo.getHandleApps(), - abnormalTaskAppInfo.getExceptionInfo()); - return; - } - // An appId is required for engine-level diagnostic. - if (abnormalTaskAppInfo.getTaskAppList().size() != 0) { - // Generate parsing log message body logRecord. - abnormalJobService.updateResource(detectJobAnalysis, abnormalTaskAppInfo.getTaskAppList()); - LogRecord logRecord = this.genLogRecord(abnormalTaskAppInfo, detectJobAnalysis); - this.sendLogRecordMsg(logRecord); - } - // Save jobInstance information. - jobInstanceService.insertOrUpdate(detectJobAnalysis); - } - - /** - * Exceptional task processing - */ - @Override - public void handleAbnormalJob(JobAnalysis detectJobAnalysis) throws Exception { - // Update user information. - updateUserInfo(detectJobAnalysis); - sendAbnormalJobApp(detectJobAnalysis); - // Save abnormal tasks. - this.addOrUpdate(detectJobAnalysis); - } - - - public void sendAbnormalJobApp(JobAnalysis detectJobAnalysis) throws Exception { - // Query the appIds under this task. - AbnormalTaskAppInfo abnormalTaskAppInfo = taskAppService.getAbnormalTaskAppsInfo(detectJobAnalysis, null); - if (abnormalTaskAppInfo.getTaskAppList().size() != 0) { - taskAppService.insertTaskApps(abnormalTaskAppInfo.getTaskAppList()); - // Update vcoreSeconds and memorySeconds - abnormalJobService.updateResource(detectJobAnalysis, abnormalTaskAppInfo.getTaskAppList()); - } - // Generate parsing log message body logRecord. - LogRecord logRecord = this.genLogRecord(abnormalTaskAppInfo, detectJobAnalysis); - - // Push to the delay queue and retry if there is exception existing. - if (!"".equals(abnormalTaskAppInfo.getExceptionInfo())) { - delayTaskService.pushDelayedQueue(detectJobAnalysis, abnormalTaskAppInfo.getHandleApps(), - abnormalTaskAppInfo.getExceptionInfo()); - } - // There is no message to send. - if (logRecord.getApps().size() == 0) { - return; - } - // Send a parsing message. - this.sendLogRecordMsg(logRecord); - // Save jobInstance information. - jobInstanceService.insertOrUpdate(detectJobAnalysis); - } - - - /** - * Generate LogRecord - */ - public LogRecord genLogRecord(AbnormalTaskAppInfo abnormalTaskAppInfo, JobAnalysis detectJobAnalysis) { - LogRecord logRecord = new LogRecord(); - logRecord.setId(UUID.randomUUID().toString()); - logRecord.setIsOneClick(false); - logRecord.setJobAnalysis(detectJobAnalysis); - logRecord.toTaskAppMap(abnormalTaskAppInfo.getTaskAppList()); - List appLogPath = logRecordService.getAppLog(abnormalTaskAppInfo.getTaskAppList()); - List schedulerLogApp = logRecordService.getSchedulerLog(detectJobAnalysis); - appLogPath.addAll(schedulerLogApp); - logRecord.setApps(appLogPath); - if (schedulerLogApp.size() != 0) { - // 更新已处理的事件信息【记录调度日志已成功发送】 - abnormalTaskAppInfo.setHandleApps(abnormalTaskAppInfo.getHandleApps() + "scheduler" + ";"); - } - return logRecord; - } - - /** - * Send LogRecord - */ - public void sendLogRecordMsg(LogRecord logRecord) { - Long size = redisService.lLeftPush(logRecordQueue, JSONObject.toJSONString(logRecord)); - log.info("send logRecord: key:{}, size:{}, data:{}", logRecordQueue, size, logRecord); - } - - /** - * Save exception task data - */ - public void addOrUpdate(JobAnalysis detectJobAnalysis) throws Exception { - JobAnalysis esJobAnalysis = abnormalJobService.searchJob(detectJobAnalysis); - if (esJobAnalysis != null) { - // Update operation - esJobAnalysis.getCategories().addAll(detectJobAnalysis.getCategories()); - if (Strings.isNotBlank(detectJobAnalysis.getSuccessExecutionDay())) { - esJobAnalysis.setSuccessExecutionDay(detectJobAnalysis.getSuccessExecutionDay()); - } - if (Strings.isBlank(detectJobAnalysis.getSuccessDays())) { - esJobAnalysis.setSuccessDays(detectJobAnalysis.getSuccessDays()); - } - if (Strings.isBlank(detectJobAnalysis.getDurationBaseline())) { - esJobAnalysis.setDurationBaseline(detectJobAnalysis.getDurationBaseline()); - } - if (Strings.isBlank(detectJobAnalysis.getEndTimeBaseline())) { - esJobAnalysis.setEndTimeBaseline(detectJobAnalysis.getEndTimeBaseline()); - } - esJobAnalysis.setUpdateTime(new Date()); - openSearchService.insertOrUpDate(esJobAnalysis.getIndex(), esJobAnalysis.getDocId(), - esJobAnalysis.genDoc()); - } else { - // Add operation - detectJobAnalysis.setCreateTime(new Date()); - detectJobAnalysis.setUpdateTime(new Date()); - String index = detectJobAnalysis.genIndex(jobIndex); - String docId = detectJobAnalysis.genDocId(); - openSearchService.insertOrUpDate(index, docId, detectJobAnalysis.genDoc()); - // Record index information and docId - detectJobAnalysis.setIndex(index); - detectJobAnalysis.setDocId(docId); - } - } - - - /** - * Update user information of the task - */ - public void updateUserInfo(JobAnalysis detectJobAnalysis) { - Task task = taskService.getTask(detectJobAnalysis.getProjectName(), detectJobAnalysis.getFlowName(), - detectJobAnalysis.getTaskName()); - if (task == null) { - log.error("get task null:{}", detectJobAnalysis); - return; - } - detectJobAnalysis.setTaskId(task.getId()); - detectJobAnalysis.setProjectId(task.getProjectId()); - detectJobAnalysis.setFlowId(task.getFlowId()); - UserInfoExample userExample = new UserInfoExample(); - userExample.createCriteria().andUserIdEqualTo(task.getUserId()); - List users = userMapper.selectByExample(userExample); - if (users.size() > 0) { - UserInfo user = users.get(0); - SimpleUser simpleUser = new SimpleUser(); - simpleUser.setUserId(user.getUserId()); - simpleUser.setUsername(user.getUsername()); - detectJobAnalysis.setUsers(Collections.singletonList(simpleUser)); - } - - } - - public double[] getEndTimeBaseline(JobAnalysis detectJobAnalysis) throws Exception { - Date startTime = DateUtil.getOffsetDate(detectJobAnalysis.getExecutionDate(), -30); - // Get data within the last month - List relativeEndDateHistory = taskInstanceService.searchTaskRelativeEndTime( - detectJobAnalysis.getProjectName(), detectJobAnalysis.getFlowName(), - detectJobAnalysis.getTaskName(), detectJobAnalysis.getExecutionDate(), startTime, 20); - // If the sample value is less than 10, do not perform anomaly detection - if (relativeEndDateHistory.size() < 10) { - return null; - } - Double[] relativeEndDate = relativeEndDateHistory.toArray(new Double[0]); - long executionTimestamp = detectJobAnalysis.getExecutionDate().getTime() / 1000; - double[] relativeMedian = DetectorUtil.boxplotValue(relativeEndDate); - double relativeEndDateStart = relativeMedian[0]; - double relativeEndDateEnd = relativeMedian[4]; - double normalEndDateBegin = relativeEndDateStart + executionTimestamp; - double normalEndDateEnd = relativeEndDateEnd + executionTimestamp; - return new double[]{normalEndDateBegin, normalEndDateEnd}; - } - - public double[] getDurationBaseline(JobAnalysis detectJobAnalysis) throws Exception { - Date startTime = DateUtil.getOffsetDate(detectJobAnalysis.getExecutionDate(), 30); - // Query the historical data of the last month - List durationHistory = taskInstanceService.searchTaskDurationHistory(detectJobAnalysis.getProjectName(), - detectJobAnalysis.getFlowName(), detectJobAnalysis.getTaskName(), detectJobAnalysis.getExecutionDate(), - startTime, 20); - // If the sample value is less than 10, do not perform anomaly detection - if (durationHistory.size() < 10) { - return null; - } - Double[] durationData = durationHistory.toArray(new Double[0]); - // Box plot method - double[] durationBeginAndEnd = DetectorUtil.boxplotValue(durationData); - // Extreme outliers - double normalDurationBegin = durationBeginAndEnd[0]; - double normalDurationEnd = durationBeginAndEnd[4]; - return new double[]{normalDurationBegin, normalDurationEnd}; - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/detector/DurationAbnormal.java b/task-detect/src/main/java/com/oppo/cloud/detect/detector/DurationAbnormal.java deleted file mode 100644 index 268ecd9ae..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/detector/DurationAbnormal.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.detector; - -import com.oppo.cloud.common.constant.JobCategoryEnum; -import com.oppo.cloud.common.constant.TaskStateEnum; -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.detect.service.TaskInstanceService; -import com.oppo.cloud.detect.util.DetectorUtil; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Service; - -/** - * Run the runtime duration anomaly detector. - */ -@Order(1) -@Service -@Slf4j -public class DurationAbnormal extends DetectServiceImpl { - - @Autowired - private TaskInstanceService taskInstanceService; - - @Override - public void detect(JobAnalysis jobAnalysis) throws Exception { - // Failed tasks are not subject to runtime duration detection. - if (jobAnalysis.getTaskState().equals(TaskStateEnum.fail.name())) { - return; - } - double[] durationBeginAndEnd = getDurationBaseline(jobAnalysis); - if (durationBeginAndEnd == null) { - return; - } - // Extremely abnormal values. - double normalDurationBegin = durationBeginAndEnd[0]; - double normalDurationEnd = durationBeginAndEnd[4]; - double tailAvg = jobAnalysis.getDuration(); - String normalDurationBeginStr = DetectorUtil.transferSecond(normalDurationBegin); - String normalDurationEndStr = DetectorUtil.transferSecond(normalDurationEnd); - jobAnalysis.setDurationBaseline(normalDurationEndStr); - if ((tailAvg > normalDurationEnd || tailAvg < normalDurationBegin)) { - jobAnalysis.getCategories().add(JobCategoryEnum.durationAbnormal.name()); - } - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/detector/DurationLong.java b/task-detect/src/main/java/com/oppo/cloud/detect/detector/DurationLong.java deleted file mode 100644 index b410a965b..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/detector/DurationLong.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.detector; - -import com.oppo.cloud.common.constant.JobCategoryEnum; -import com.oppo.cloud.common.constant.TaskStateEnum; -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Service; - -/** - * Run the long runtime duration detector - */ -@Order(4) -@Service -public class DurationLong extends DetectServiceImpl { - - @Value("${custom.detectionRule.durationWarning}") - private Float durationWarning; - - @Override - public void detect(JobAnalysis jobAnalysis) throws Exception { - // Failed tasks are not subject to the long runtime detection. - if (jobAnalysis.getTaskState().equals(TaskStateEnum.fail.name())) { - return; - } - if (jobAnalysis.getDuration() >= durationWarning * 3600) { - jobAnalysis.getCategories().add(JobCategoryEnum.durationLong.name()); - } - - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/detector/EndTimeAbnormal.java b/task-detect/src/main/java/com/oppo/cloud/detect/detector/EndTimeAbnormal.java deleted file mode 100644 index e0ba28d11..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/detector/EndTimeAbnormal.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.detector; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.oppo.cloud.common.constant.JobCategoryEnum; -import com.oppo.cloud.common.constant.TaskStateEnum; -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.domain.job.Datum; -import com.oppo.cloud.common.util.DateUtil; -import com.oppo.cloud.detect.util.DetectorUtil; -import com.oppo.cloud.mapper.TaskDatumMapper; -import com.oppo.cloud.model.*; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Service; - -import java.util.*; - -/** - * End time detector. - */ -@Order(2) -@Slf4j -@Service -public class EndTimeAbnormal extends DetectServiceImpl { - - - @Autowired - private ObjectMapper objectMapper; - - @Autowired - private TaskDatumMapper taskDatumMapper; - - @Override - public void detect(JobAnalysis jobAnalysis) throws Exception { - // Failed tasks are not subject to runtime duration detection. - if (jobAnalysis.getTaskState().equals(TaskStateEnum.fail.name())) { - return; - } - double[] normalValue = getEndTimeBaseline(jobAnalysis); - if (normalValue == null) { - return; - } - double normalEndDateBegin = normalValue[0]; - double normalEndDateEnd = normalValue[1]; - long endTimeTimestamp = jobAnalysis.getEndTime().getTime() / 1000; - String normalEndDateBeginStr = DetectorUtil.timeStampToStr((long) normalEndDateBegin, "yyyy-MM-dd HH:mm:ss"); - String normalEndDateEndStr = DetectorUtil.timeStampToStr((long) normalEndDateEnd, "yyyy-MM-dd HH:mm:ss"); - jobAnalysis.setEndTimeBaseline(normalEndDateEndStr); - if (endTimeTimestamp > normalEndDateEnd || endTimeTimestamp < normalEndDateBegin) { - jobAnalysis.getCategories().add(JobCategoryEnum.endTimeAbnormal.name()); - genBaselineTree(jobAnalysis); - } - } - - public void genBaselineTree(JobAnalysis detectJobAnalysis) throws Exception { - Datum datum = getTotalBaselineTree(detectJobAnalysis); - String baselineTreeStr = objectMapper.writeValueAsString(datum); - TaskDatum taskDatum = new TaskDatum(); - taskDatum.setProjectName(detectJobAnalysis.getProjectName()); - taskDatum.setFlowName(detectJobAnalysis.getFlowName()); - taskDatum.setTaskName(detectJobAnalysis.getTaskName()); - taskDatum.setExecutionDate(detectJobAnalysis.getExecutionDate()); - taskDatum.setBaseline(baselineTreeStr); - taskDatum.setCreateTime(new Date()); - taskDatumMapper.insert(taskDatum); - } - - /** - * Get the baseline tree of the upstream task. - * - * @param detectJobAnalysis - * @return - * @throws Exception - */ - public Datum getTotalBaselineTree(JobAnalysis detectJobAnalysis) throws Exception { - Task task = taskService.getTask(detectJobAnalysis.getProjectName(), detectJobAnalysis.getFlowName(), - detectJobAnalysis.getTaskName()); - Integer taskId = task.getId(); - Datum datum = new Datum(); - Set nodeList = new HashSet<>(); - List vergeList = new ArrayList<>(); - Datum.Node node = this.getNode(detectJobAnalysis); - node.setId(taskId); - nodeList.add(node); - datum.setNodeList(nodeList); - datum.setVergeList(vergeList); - return datum; - } - - /** - * Construct a baseline node. - * - * @param detectJobAnalysis - * @return - * @throws Exception - */ - public Datum.Node getNode(JobAnalysis detectJobAnalysis) throws Exception { - Datum.Node node = new Datum.Node(); - node.setFlowName(detectJobAnalysis.getFlowName()); - node.setTaskName(detectJobAnalysis.getTaskName()); - node.setDuration(detectJobAnalysis.getDuration() == 0 ? "-" - : DetectorUtil.transferSecond(detectJobAnalysis.getDuration())); - node.setExecutionDate(DateUtil.format(detectJobAnalysis.getExecutionDate(), "yyyy-MM-dd HH:mm:ss")); - node.setStartTime(detectJobAnalysis.getStartTime() == null ? "-" - : DateUtil.format(detectJobAnalysis.getStartTime(), "yyyy-MM-dd HH:mm:ss")); - node.setEndTime(DateUtil.format(detectJobAnalysis.getEndTime(), "yyyy-MM-dd HH:mm:ss")); - node.setPeriod(String.format("%s~%s", - detectJobAnalysis.getStartTime() == null ? "00:00" - : DateUtil.format(detectJobAnalysis.getStartTime(), "HH:mm:ss"), - detectJobAnalysis.getEndTime() == null ? "00:00" - : DateUtil.format(detectJobAnalysis.getEndTime(), "HH:mm:ss"))); - node.setTaskState(detectJobAnalysis.getTaskState()); - node.setEndTimeBaseLine("-"); - node.setDurationBaseLine("-"); - if (detectJobAnalysis.getEndTime() != null) { - double[] normalEndTimeValue = getEndTimeBaseline(detectJobAnalysis); - if (normalEndTimeValue != null) { - node.setEndTimeBaseLine( - DateUtil.format(new Date((long) (normalEndTimeValue[1]) * 1000L), "yyyy-MM-dd HH:mm:ss")); - if (detectJobAnalysis.getEndTime().getTime() > normalEndTimeValue[1] * 1000L) { - node.setEndTimeAbnormal(true); - } - } - } - if (detectJobAnalysis.getDuration() != 0) { - double[] normalDurationValue = getDurationBaseline(detectJobAnalysis); - if (normalDurationValue != null) { - node.setDurationBaseLine(DetectorUtil.transferSecond(normalDurationValue[1])); - if (detectJobAnalysis.getDuration() > normalDurationValue[1]) { - node.setDurationAbnormal(true); - } - } - } - return node; - } - -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/detector/ExecuteFailed.java b/task-detect/src/main/java/com/oppo/cloud/detect/detector/ExecuteFailed.java deleted file mode 100644 index 09c36af40..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/detector/ExecuteFailed.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.detector; - -import com.oppo.cloud.common.constant.JobCategoryEnum; -import com.oppo.cloud.common.constant.TaskStateEnum; -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import lombok.extern.slf4j.Slf4j; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Service; - -/** - * Execute the failure detector. - */ -@Order(0) -@Service -@Slf4j -public class ExecuteFailed extends DetectServiceImpl { - - @Override - public void detect(JobAnalysis jobAnalysis) throws Exception { - if (TaskStateEnum.fail.name().equals(jobAnalysis.getTaskState())) { - jobAnalysis.getCategories().add(JobCategoryEnum.executionFailed.name()); - } - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/detector/FirstFailed.java b/task-detect/src/main/java/com/oppo/cloud/detect/detector/FirstFailed.java deleted file mode 100644 index 2246e5266..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/detector/FirstFailed.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.detector; - -import com.oppo.cloud.common.constant.JobCategoryEnum; -import com.oppo.cloud.common.constant.TaskStateEnum; -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Service; - -/** - * Task first failure detector. - */ -@Order(4) -@Service -public class FirstFailed extends DetectServiceImpl { - - @Override - public void detect(JobAnalysis jobAnalysis) throws Exception { - // Failed tasks are not subject to runtime duration detection. - if (jobAnalysis.getTaskState().equals(TaskStateEnum.fail.name())) { - return; - } - if (jobAnalysis.getRetryTimes() > 1) { - jobAnalysis.getCategories().add(JobCategoryEnum.firstFailed.name()); - } - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/domain/AbnormalTaskAppInfo.java b/task-detect/src/main/java/com/oppo/cloud/detect/domain/AbnormalTaskAppInfo.java deleted file mode 100644 index 66e6f5405..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/domain/AbnormalTaskAppInfo.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.domain; - -import com.oppo.cloud.common.domain.opensearch.TaskApp; -import lombok.Data; - -import java.util.List; - -/** - * AbnormalTaskApp Information - */ -@Data -public class AbnormalTaskAppInfo { - - private List taskAppList; - private String exceptionInfo; - private String handleApps; -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/domain/DelayedTaskInfo.java b/task-detect/src/main/java/com/oppo/cloud/detect/domain/DelayedTaskInfo.java deleted file mode 100644 index c99bbf0d8..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/domain/DelayedTaskInfo.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.domain; - -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import lombok.Data; - -import java.util.Date; -import java.util.UUID; - -@Data -public class DelayedTaskInfo { - - /** - * Cache key - */ - private String key; - /** - * Handle the number of retries for delayed tasks. - */ - private Integer processRetries; - /** - * Delayed task information. - */ - private JobAnalysis jobAnalysis; - /** - * Set of taskApps that have been processed and completed. - */ - private String handledApps; - - /** - * Status (processing, success, failed) - */ - private String processState; - - /** - * Create time - */ - private Date createTime; - - /** - * Update time - */ - private Date updateTime; - - /** - * Exception information - */ - private String exceptionInfo; - - public DelayedTaskInfo() { - - } - - public void setDelayProcessTask(JobAnalysis detectJobAnalysis, String handledApps, String exception) { - this.key = UUID.randomUUID().toString(); - this.jobAnalysis = detectJobAnalysis; - this.handledApps = handledApps; - this.exceptionInfo = exception; - this.createTime = new Date(); - this.updateTime = new Date(); - this.processRetries = 1; - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/domain/TaskStateHistory.java b/task-detect/src/main/java/com/oppo/cloud/detect/domain/TaskStateHistory.java deleted file mode 100644 index 827b42c08..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/domain/TaskStateHistory.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.domain; - -import lombok.Data; - -import java.util.Date; - -/** - * Task execution history status information - */ -@Data -public class TaskStateHistory { - - /** - * The status of the task execution. - */ - private String state; - /** - * The period of the task execution. - */ - private Date executionTime; - /** - * The running data of the task execution. - */ - private Double value; -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/mapper/TaskInstanceExtendMapper.java b/task-detect/src/main/java/com/oppo/cloud/detect/mapper/TaskInstanceExtendMapper.java deleted file mode 100644 index e99528203..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/mapper/TaskInstanceExtendMapper.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.mapper; - -import com.oppo.cloud.detect.domain.TaskStateHistory; -import com.oppo.cloud.mapper.TaskInstanceMapper; -import com.oppo.cloud.model.TaskInstanceExample; - -import java.util.List; - -public interface TaskInstanceExtendMapper extends TaskInstanceMapper { - - /** - * Query the historical data of task execution status. - */ - List searchTaskState(TaskInstanceExample example); - - /** - * Query the historical data of task execution duration. - */ - List searchTaskDuration(TaskInstanceExample example); - - /** - * Query the historical data of the end time of task execution relative to the running cycle. - */ - List searchTaskRelativeEndTime(TaskInstanceExample example); -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/AbnormalJobService.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/AbnormalJobService.java deleted file mode 100644 index 7dc2ccdd6..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/AbnormalJobService.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service; - -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.domain.opensearch.TaskApp; - -import java.util.List; - - -public interface AbnormalJobService { - - /** - * Update memory data and Vcore data. - */ - void updateVcoreAndMemory(JobAnalysis jobAnalysis) throws Exception; - - /** - * Incrementally update memory data and vCore data. - */ - void updateResource(JobAnalysis jobAnalysis, List taskAppList); - - /** - * Search job - */ - JobAnalysis searchJob(JobAnalysis jobAnalysis) throws Exception; - - /** - * Insert or update job instances. - */ - void insertOrUpdate(JobAnalysis jobAnalysis) throws Exception; - -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/BlocklistService.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/BlocklistService.java deleted file mode 100644 index abd546b19..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/BlocklistService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service; - -/** - * Block list service. - */ -public interface BlocklistService { - - /** - * Check if the task to be detected is in the block list. - */ - Boolean isBlocklistTask(String projectName, String flowName, String taskName); - -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/DelayedTaskService.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/DelayedTaskService.java deleted file mode 100644 index 49f9cbbbb..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/DelayedTaskService.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service; - -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.detect.domain.DelayedTaskInfo; - -import java.util.List; - -/** - * Delayed processing task service. - */ -public interface DelayedTaskService { - - /** - * Add delayed queue task. - */ - void pushDelayedQueue(JobAnalysis detectJobAnalysis, String handledApps, String exception); - - /** - * Re-add the delayed queue task. - */ - void rePushDelayedQueue(DelayedTaskInfo delayedTaskInfo); - /** - * Get the retried tasks from the delayed queue. - */ - List getDelayedTasks(); -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/DetectService.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/DetectService.java deleted file mode 100644 index 66e976f60..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/DetectService.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service; - -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; - -/** - * Job diagnosis service. - */ -public interface DetectService { - - /** - * Job diagnosis - * @param jobAnalysis - * @return - * @throws Exception - */ - void detect(JobAnalysis jobAnalysis) throws Exception; - - /** - * Exceptional job task processing - * @param jobAnalysis - * @throws Exception - */ - void handleAbnormalJob(JobAnalysis jobAnalysis) throws Exception; - - /** - * Normal job task processing - * @param jobAnalysis - * @throws Exception - */ - void handleNormalJob(JobAnalysis jobAnalysis) throws Exception; -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/JobInstanceService.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/JobInstanceService.java deleted file mode 100644 index 329982a59..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/JobInstanceService.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service; - -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; - -/** - * @date 2022/11/1 20:23 - */ -public interface JobInstanceService { - - /** - * Add or update jobInstance - * @param jobAnalysis - */ - void insertOrUpdate(JobAnalysis jobAnalysis) throws Exception; -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/LogRecordService.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/LogRecordService.java deleted file mode 100644 index 524ffb023..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/LogRecordService.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service; - -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.domain.opensearch.TaskApp; -import com.oppo.cloud.common.domain.job.App; - -import java.util.List; - -/** - * Log parsing message interface. - */ -public interface LogRecordService { - - /** - * Get the scheduling log of the task. - */ - List getSchedulerLog(JobAnalysis detectJobAnalysis); - - /** - * Get the log of the appId. - */ - List getAppLog(List taskAppList); -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/OpenSearchService.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/OpenSearchService.java deleted file mode 100644 index 7aa085131..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/OpenSearchService.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service; - -import com.oppo.cloud.common.domain.cluster.spark.SparkApp; -import com.oppo.cloud.common.domain.cluster.yarn.YarnApp; -import com.oppo.cloud.common.domain.opensearch.OpenSearchInfo; -import org.opensearch.action.update.UpdateResponse; -import org.opensearch.search.SearchHits; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.search.sort.SortOrder; - -import java.util.List; -import java.util.Map; - -/** - * OpenSearchService - */ -public interface OpenSearchService { - - - /** - * Construct a general query condition. - */ - SearchSourceBuilder genSearchBuilder(Map termQuery, Map rangeConditions, - Map sort, - Map or); - - /** - * Query records by condition, and query by page. - */ - SearchHits find(SearchSourceBuilder builder, String... indexes) throws Exception; - - /** - * Query YarnApp data. - */ - YarnApp searchYarnApp(String applicationId) throws Exception; - - /** - * Query SparkApp data. - */ - SparkApp searchSparkApp(String applicationId) throws Exception; - - /** - * Insert or update. - */ - UpdateResponse insertOrUpDate(String index, String id, Object document) throws Exception; - - /** - * Query data. - */ - List find(Class itemType, SearchSourceBuilder builder, String... indexes) throws Exception; -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/SchedulerLogService.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/SchedulerLogService.java deleted file mode 100644 index d53a851f0..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/SchedulerLogService.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service; - -import java.util.Date; -import java.util.List; - -/** - * Scheduling log acquisition interface. - */ -public interface SchedulerLogService { - - /** - * Get Dolphin's scheduling log from task_application. - */ - List getSchedulerLog(String projectName, String flowName, String taskName, - Date executionDate, Integer tryNum); -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/TaskAppService.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/TaskAppService.java deleted file mode 100644 index e0219846e..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/TaskAppService.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service; - -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.domain.opensearch.TaskApp; -import com.oppo.cloud.detect.domain.AbnormalTaskAppInfo; - -import java.util.List; -import java.util.Map; - -/** - * Task taskApp interface. - */ -public interface TaskAppService { - - /** - * Get the result information of exception task Apps (including exception information). - */ - AbnormalTaskAppInfo getAbnormalTaskAppsInfo(JobAnalysis jobAnalysis, String handledApps); - - /** - * Get all AbnormalTaskApp data under the task, including data without taskApp. - */ - Map> getAbnormalTaskApps(JobAnalysis jobAnalysis); - - /** - * Insert the app information of exception tasks. - */ - void insertTaskApps(List taskAppList) throws Exception; - - /** - * Get the app information of exception tasks. - */ - List searchTaskApps(JobAnalysis jobAnalysis) throws Exception; -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/TaskDependencyService.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/TaskDependencyService.java deleted file mode 100644 index 5175d64f8..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/TaskDependencyService.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service; - -import java.util.List; - - -public interface TaskDependencyService { - - /** - * Get the direct upstream list of the task. - */ - List getDirectUpStreamTask(Integer taskId); -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/TaskInstanceService.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/TaskInstanceService.java deleted file mode 100644 index 26df11c68..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/TaskInstanceService.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service; - -import com.oppo.cloud.detect.domain.TaskStateHistory; -import com.oppo.cloud.model.TaskInstance; - -import java.util.Date; -import java.util.List; - -/** - * Task execution instance interface. - */ -public interface TaskInstanceService { - - /** - * Query the data of recent task execution. - */ - List searchTaskStateHistory(String projectName, String flowName, String taskName, - Date executionTime, Date endExecutionTime, Integer sampleNum); - - /** - * Query the recent execution duration data of tasks. - */ - List searchTaskDurationHistory(String projectName, String flowName, String taskName, - Date executionTime, Date endExecutionTime, Integer sampleNum); - - /** - * Query the data of the difference between the end time of task execution and the running cycle in recent times. - */ - List searchTaskRelativeEndTime(String projectName, String flowName, String taskName, - Date executionTime, Date endExecutionTime, Integer sampleNum); - - /** - * Query aggregated tasks for a specific execution cycle. - */ - TaskInstance searchTaskSum(String projectName, String flowName, String taskName, - Date executionTime); -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/TaskService.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/TaskService.java deleted file mode 100644 index 848a0053d..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/TaskService.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service; - -import com.oppo.cloud.model.Task; - - -public interface TaskService { - - /** - * Get task metadata. - */ - Task getTask(String projectName, String flowName, String taskName); - - /** - * Get task metadata. - */ - Task getTask(Integer id); -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/AbnormalJobServiceImpl.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/AbnormalJobServiceImpl.java deleted file mode 100644 index e6f383a03..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/AbnormalJobServiceImpl.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service.impl; - -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.domain.opensearch.TaskApp; -import com.oppo.cloud.common.util.DateUtil; -import com.oppo.cloud.detect.service.AbnormalJobService; -import com.oppo.cloud.detect.service.TaskAppService; -import com.oppo.cloud.detect.service.OpenSearchService; -import lombok.extern.slf4j.Slf4j; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import java.util.*; - -/** - * Exception job data processing service. - */ -@Slf4j -@Service -public class AbnormalJobServiceImpl implements AbnormalJobService { - - @Autowired - private OpenSearchService openSearchService; - - @Autowired - private TaskAppService taskAppService; - - @Value("${custom.opensearch.job-index}") - private String jobIndex; - - /** - * Update the memory data and Vcore data. - */ - @Override - public void updateVcoreAndMemory(JobAnalysis jobAnalysis) throws Exception { - List taskAppList = taskAppService.searchTaskApps(jobAnalysis); - double vcoreSeconds = 0; - double memorySeconds = 0; - for (TaskApp taskApp : taskAppList) { - vcoreSeconds += taskApp.getVcoreSeconds() == null ? 0 : taskApp.getVcoreSeconds(); - memorySeconds += taskApp.getMemorySeconds() == null ? 0 : taskApp.getMemorySeconds(); - } - JobAnalysis jobAnalysisEs = this.searchJob(jobAnalysis); - jobAnalysis.setVcoreSeconds(vcoreSeconds); - jobAnalysis.setMemorySeconds(memorySeconds); - openSearchService.insertOrUpDate(jobAnalysisEs.getIndex(), jobAnalysisEs.getDocId(), - jobAnalysisEs.genDoc()); - } - - /** - * Incrementally update memory data and vCore data - */ - @Override - public void updateResource(JobAnalysis jobAnalysis, List taskAppList) { - double memorySecond = jobAnalysis.getMemorySeconds() == null ? 0.0 : jobAnalysis.getMemorySeconds(); - double vcoreSecond = jobAnalysis.getVcoreSeconds() == null ? 0.0 : jobAnalysis.getVcoreSeconds(); - for (TaskApp taskApp : taskAppList) { - memorySecond += taskApp.getMemorySeconds(); - vcoreSecond += taskApp.getVcoreSeconds(); - } - jobAnalysis.setMemorySeconds(memorySecond); - jobAnalysis.setVcoreSeconds(vcoreSecond); - } - - @Override - public JobAnalysis searchJob(JobAnalysis jobAnalysis) throws Exception { - HashMap termQuery = new HashMap<>(); - termQuery.put("projectName.keyword", jobAnalysis.getProjectName()); - termQuery.put("flowName.keyword", jobAnalysis.getFlowName()); - termQuery.put("taskName.keyword", jobAnalysis.getTaskName()); - termQuery.put("executionDate", DateUtil.timestampToUTCDate(jobAnalysis.getExecutionDate().getTime())); - SearchSourceBuilder searchSourceBuilder = openSearchService.genSearchBuilder(termQuery, null, null, null); - List jobAnalysisList = - openSearchService.find(JobAnalysis.class, searchSourceBuilder, jobIndex + "-*"); - if (jobAnalysisList.size() != 0) { - return jobAnalysisList.get(0); - } else { - return null; - } - } - - @Override - public void insertOrUpdate(JobAnalysis jobAnalysis) throws Exception { - Map termQuery = new HashMap<>(); - termQuery.put("projectName.keyword", jobAnalysis.getProjectName()); - termQuery.put("flowName.keyword", jobAnalysis.getFlowName()); - termQuery.put("taskName.keyword", jobAnalysis.getTaskName()); - termQuery.put("executionDate", DateUtil.timestampToUTCDate(jobAnalysis.getExecutionDate().getTime())); - SearchSourceBuilder searchSourceBuilder = openSearchService.genSearchBuilder(termQuery, null, null, null); - List jobAnalysisList = - openSearchService.find(JobAnalysis.class, searchSourceBuilder, jobIndex + "-*"); - if (jobAnalysisList.size() != 0) { - JobAnalysis jobAnalysisEs = jobAnalysisList.get(0); - jobAnalysis.setDocId(jobAnalysisEs.getDocId()); - jobAnalysis.setIndex(jobAnalysisEs.getIndex()); - jobAnalysis.setCategories(jobAnalysisEs.getCategories()); - } - openSearchService.insertOrUpDate(jobAnalysis.genIndex(jobIndex), jobAnalysis.genDocId(), - jobAnalysis.genDoc()); - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/BlocklistServiceImpl.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/BlocklistServiceImpl.java deleted file mode 100644 index b96bc7de7..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/BlocklistServiceImpl.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service.impl; - -import com.oppo.cloud.detect.service.BlocklistService; -import com.oppo.cloud.mapper.BlocklistMapper; -import com.oppo.cloud.model.Blocklist; -import com.oppo.cloud.model.BlocklistExample; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.List; - -/** - * Block list - */ -@Slf4j -@Service -public class BlocklistServiceImpl implements BlocklistService { - - @Autowired - private BlocklistMapper blocklistMapper; - - /** - * Check if the task to be detected is in the block list. - */ - @Override - public Boolean isBlocklistTask(String projectName, String flowName, String taskName) { - BlocklistExample blocklistExample = new BlocklistExample(); - BlocklistExample.Criteria criteria = blocklistExample.createCriteria(); - criteria.andTaskNameEqualTo(taskName).andDeletedEqualTo(0); - if (StringUtils.isNotBlank(projectName)) { - criteria.andProjectNameEqualTo(projectName); - } - if (StringUtils.isNotBlank(flowName)) { - criteria.andFlowNameEqualTo(flowName); - } - List blocklists = blocklistMapper.selectByExample(blocklistExample); - return blocklists.size() > 0; - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/DelayedTaskServiceImpl.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/DelayedTaskServiceImpl.java deleted file mode 100644 index 5dc8b3120..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/DelayedTaskServiceImpl.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service.impl; - -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONArray; -import com.alibaba.fastjson2.JSONObject; -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.detect.domain.DelayedTaskInfo; -import com.oppo.cloud.detect.service.DelayedTaskService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.redis.core.script.RedisScript; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * Delayed task processing service. - */ -@Slf4j -@Service -public class DelayedTaskServiceImpl implements DelayedTaskService { - - @Value("${custom.redis.delayedQueue}") - private String delayQueue; - - @Value("${custom.redis.processing}") - private String processingKey; - - @Value("${custom.delayedTaskQueue.delayedSeconds}") - private Integer delayedSeconds; - - @Autowired - private RedisService redisService; - - @Autowired - private RedisScript delayTaskedScript; - - /** - * Push job to delayed queue. - */ - @Override - public void pushDelayedQueue(JobAnalysis detectJobAnalysis, String handledApps, String exception) { - DelayedTaskInfo delayedTaskInfo = new DelayedTaskInfo(); - delayedTaskInfo.setDelayProcessTask(detectJobAnalysis, handledApps, exception); - redisService.zSetAdd(delayQueue, JSON.toJSONString(delayedTaskInfo), - System.currentTimeMillis() + delayedSeconds * 1000); - log.info("pushDelayQueue:{},{}", delayQueue, JSON.toJSONString(delayedTaskInfo)); - } - - /** - * Re-delay Queue Task - * - */ - @Override - public void rePushDelayedQueue(DelayedTaskInfo delayedTaskInfo) { - redisService.zSetAdd(delayQueue, JSON.toJSONString(delayedTaskInfo), - System.currentTimeMillis() + delayedSeconds * 1000); - log.info("rePushDelayQueue:{},{}", delayQueue, JSON.toJSONString(delayedTaskInfo)); - } - - @Override - public List getDelayedTasks() { - Object delayTasks = redisService.executeScript(delayTaskedScript, Arrays.asList(delayQueue, processingKey), - String.valueOf(System.currentTimeMillis())); - if (delayTasks == null) { - return null; - } - JSONArray data = (JSONArray) JSON.parse((String) delayTasks); - List taskList = new ArrayList<>(); - for (Object obj : data.toArray()) { - DelayedTaskInfo delayedTaskInfo = JSONObject.parseObject((String) obj, DelayedTaskInfo.class); - taskList.add(delayedTaskInfo); - } - return taskList; - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/JobInstanceServiceImpl.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/JobInstanceServiceImpl.java deleted file mode 100644 index 76ff6be0d..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/JobInstanceServiceImpl.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service.impl; - -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.domain.opensearch.JobInstance; -import com.oppo.cloud.common.util.DateUtil; -import com.oppo.cloud.detect.service.OpenSearchService; -import com.oppo.cloud.detect.service.JobInstanceService; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Job instance processing service. - */ - -@Service -public class JobInstanceServiceImpl implements JobInstanceService { - - @Autowired - private OpenSearchService openSearchService; - - @Value("${custom.opensearch.job-instance-index}") - private String jonInstanceIndex; - - @Override - public void insertOrUpdate(JobAnalysis jobAnalysis) throws Exception { - JobInstance jobInstance = new JobInstance(); - BeanUtils.copyProperties(jobAnalysis, jobInstance); - jobInstance.setIndex(null); - jobInstance.setDocId(null); - Map termQuery = new HashMap<>(); - termQuery.put("projectName.keyword", jobAnalysis.getProjectName()); - termQuery.put("flowName.keyword", jobAnalysis.getFlowName()); - termQuery.put("taskName.keyword", jobAnalysis.getTaskName()); - termQuery.put("executionDate", DateUtil.timestampToUTCDate(jobAnalysis.getExecutionDate().getTime())); - SearchSourceBuilder searchSourceBuilder = openSearchService.genSearchBuilder(termQuery, null, null, null); - List jobInstanceList = - openSearchService.find(JobInstance.class, searchSourceBuilder, jonInstanceIndex + "-*"); - if (jobInstanceList.size() != 0) { - JobInstance jobInstanceEs = jobInstanceList.get(0); - jobInstance.setDocId(jobInstanceEs.getDocId()); - jobInstance.setIndex(jobInstanceEs.getIndex()); - } - openSearchService.insertOrUpDate(jobInstance.genIndex(jonInstanceIndex), jobInstance.genDocId(), - jobInstance.genDoc()); - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/LogRecordServiceImpl.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/LogRecordServiceImpl.java deleted file mode 100644 index ce9221588..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/LogRecordServiceImpl.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service.impl; - -import com.oppo.cloud.common.domain.job.App; -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.domain.opensearch.TaskApp; -import com.oppo.cloud.detect.service.LogRecordService; -import com.oppo.cloud.detect.service.SchedulerLogService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.List; - -/** - * Log parsing message service. - */ -@Service -@Slf4j -public class LogRecordServiceImpl implements LogRecordService { - - @Autowired - SchedulerLogService schedulerLogService; - - - @Override - public List getSchedulerLog(JobAnalysis detectJobAnalysis) { - List apps = new ArrayList<>(); - // Construct scheduling logs based on the number of times the task is retried. - for (int i = 0; i <= detectJobAnalysis.getRetryTimes(); i++) { - List logPaths = schedulerLogService.getSchedulerLog(detectJobAnalysis.getProjectName(), - detectJobAnalysis.getFlowName(), detectJobAnalysis.getTaskName(), - detectJobAnalysis.getExecutionDate(), i); - if (logPaths != null && logPaths.size() != 0) { - App app = new App(); - app.formatSchedulerLog(logPaths, i); - apps.add(app); - } - } - // Successfully constructing the scheduling log for each retry. - if (apps.size() < detectJobAnalysis.getRetryTimes() + 1) { - apps = new ArrayList<>(); - } - return apps; - } - - @Override - public List getAppLog(List taskAppList) { - List apps = new ArrayList<>(); - // If there is a complete appId, send it together. - for (TaskApp taskApp : taskAppList) { - App app = new App(); - app.formatAppLog(taskApp); - apps.add(app); - } - return apps; - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/OpenSearchServiceImp.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/OpenSearchServiceImp.java deleted file mode 100644 index 40edabcbb..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/OpenSearchServiceImp.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service.impl; - -import com.alibaba.fastjson2.JSON; -import com.oppo.cloud.common.constant.YarnAppFinalStatus; -import com.oppo.cloud.common.domain.cluster.spark.SparkApp; -import com.oppo.cloud.common.domain.cluster.yarn.YarnApp; -import com.oppo.cloud.common.domain.opensearch.OpenSearchInfo; -import com.oppo.cloud.common.util.DateUtil; -import com.oppo.cloud.detect.service.OpenSearchService; -import lombok.extern.slf4j.Slf4j; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.support.WriteRequest; -import org.opensearch.action.update.UpdateRequest; -import org.opensearch.action.update.UpdateResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.index.query.RangeQueryBuilder; -import org.opensearch.search.SearchHit; -import org.opensearch.search.SearchHits; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.search.sort.SortOrder; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Service -@Slf4j -public class OpenSearchServiceImp implements OpenSearchService { - - @Value("${custom.opensearch.yarn-app-index}") - private String yarnAppIndex; - - @Value("${custom.opensearch.spark-app-index}") - private String sparkAppIndex; - - @Autowired - private RestHighLevelClient restHighLevelClient; - - - /** - * Build general query conditions. - */ - @Override - public SearchSourceBuilder genSearchBuilder(Map termQuery, Map rangeConditions, - Map sort, - Map or) { - SearchSourceBuilder builder = new SearchSourceBuilder(); - BoolQueryBuilder boolQuery = new BoolQueryBuilder(); - // Term query - for (String key : termQuery.keySet()) { - Object value = termQuery.get(key); - if (value == null) { - // null value query - boolQuery.mustNot(QueryBuilders.existsQuery(key)); - } else if ("".equals(value)) { - // Does not match any valid string. - boolQuery.mustNot(QueryBuilders.wildcardQuery(key, "*")); - } else if (value instanceof java.util.List) { - // Field query - boolQuery.filter(QueryBuilders.termsQuery(key, (List) value)); - } else { - // Single string query. - boolQuery.filter(QueryBuilders.termsQuery(key, value)); - } - } - // Or conditional query [xx and (a=1 or c=2)] - if (or != null) { - BoolQueryBuilder orQuery = new BoolQueryBuilder(); - for (String key : or.keySet()) { - Object value = or.get(key); - if (value != null) { - orQuery.should(QueryBuilders.termQuery(key, value)); - } - } - boolQuery.must(orQuery); - } - // Range query. - if (rangeConditions != null) { - for (String key : rangeConditions.keySet()) { - RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(key); - Object[] queryValue = rangeConditions.get(key); - if (queryValue[0] != null) { - rangeQueryBuilder.gte(queryValue[0]); - } - if (queryValue[1] != null) { - rangeQueryBuilder.lte(queryValue[1]); - } - boolQuery.filter(rangeQueryBuilder); - } - } - // sort - if (sort != null) { - for (String key : sort.keySet()) { - builder.sort(key, sort.get(key)); - } - } - builder.query(boolQuery); - return builder; - } - - /** - * Query raw data based on the query condition builder and indexes. - * - * @param builder query condition - * @param indexes query indexes - * @return - * @throws Exception - */ - @Override - public SearchHits find(SearchSourceBuilder builder, String... indexes) throws Exception { - SearchRequest searchRequest = new SearchRequest().indices(indexes).source(builder); - Long startTime = System.currentTimeMillis(); - SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); - Long endTime = System.currentTimeMillis(); - log.info("indexes:{}, duration:{} ,condition:{}", indexes, (endTime - startTime) / 1000, builder.toString()); - return searchResponse.getHits(); - } - - /** - * Query YarnApp data. - */ - @Override - public YarnApp searchYarnApp(String applicationId) throws Exception { - YarnApp yarnApp = null; - HashMap termQuery = new HashMap<>(); - termQuery.put("id.keyword", applicationId); - SearchSourceBuilder searchSourceBuilder = this.genSearchBuilder(termQuery, null, null, null); - SearchHits searchHits = this.find(searchSourceBuilder, yarnAppIndex + "-*"); - if (searchHits.getHits().length == 0) { - throw new Exception(String.format("can not find this appId from yarnApp, appId:%s", applicationId)); - } - for (SearchHit hit : searchHits) { - yarnApp = JSON.parseObject(hit.getSourceAsString(), YarnApp.class); - } - if (yarnApp == null) { - throw new Exception(String.format("yarnApp is null, appId:%s", applicationId)); - } - if (yarnApp.getFinalStatus().equals(YarnAppFinalStatus.SUCCEEDED.toString()) || - yarnApp.getFinalStatus().equals(YarnAppFinalStatus.FAILED.toString()) || - yarnApp.getFinalStatus().equals(YarnAppFinalStatus.KILLED.toString())) { - return yarnApp; - } - throw new Exception(String.format("yarnApp state:%s, finalStatus:%s, appId:%s", yarnApp.getState(), - yarnApp.getFinalStatus(), applicationId)); - } - - - /** - * Query SparkApp data. - */ - @Override - public SparkApp searchSparkApp(String applicationId) throws Exception { - SparkApp sparkApp = null; - HashMap termQuery = new HashMap<>(); - termQuery.put("appId.keyword", applicationId); - SearchSourceBuilder searchSourceBuilder = this.genSearchBuilder(termQuery, null, null, null); - SearchHits searchHits = this.find(searchSourceBuilder, sparkAppIndex + "-*"); - if (searchHits.getHits().length == 0) { - throw new Exception(String.format("can not find this appId from sparkAppIndex, appId:%s", applicationId)); - } - for (SearchHit hit : searchHits) { - sparkApp = JSON.parseObject(hit.getSourceAsString(), SparkApp.class); - break; - } - return sparkApp; - } - - @Override - public UpdateResponse insertOrUpDate(String index, String id, Object document) throws Exception { - String json = ""; - try { - json = JSON.toJSONString(document); - } catch (Exception e) { - throw new Exception(String.format("insertOrUpDate writeValueAsString failed:%s", e.getMessage())); - } - UpdateResponse updateResponse; - try { - UpdateRequest request = - new UpdateRequest(index, id).doc(json, XContentType.JSON).upsert(json, XContentType.JSON); - request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - updateResponse = restHighLevelClient.update(request, RequestOptions.DEFAULT); - } catch (Exception e) { - throw new Exception(String.format("insertOrUpDate update failed:%s", e.getMessage())); - } - return updateResponse; - } - - /** - * Query data based on the query conditions and index. - */ - @Override - public List find(Class itemType, SearchSourceBuilder builder, - String... indexes) throws Exception { - List items = new ArrayList<>(); - SearchHits hits; - try { - hits = this.find(builder, indexes); - for (SearchHit hit : hits) { - try { - T value = JSON.parseObject(hit.getSourceAsString(), itemType, DateUtil.getUTCContext()); - value.setDocId(hit.getId()); - value.setIndex(hit.getIndex()); - items.add(value); - } catch (Exception e) { - log.error("parse JSON : {}, exception: ", hit.getSourceAsString(), e); - throw new Exception(String.format("parse JSON exception: %s", e.getMessage())); - } - } - } catch (Exception e) { - log.error("find OpenSearch exception: ", e); - throw new Exception(String.format("find OpenSearch exception: %s", e.getMessage())); - } - return items; - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/SchedulerLogServiceImpl.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/SchedulerLogServiceImpl.java deleted file mode 100644 index 19e996d78..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/SchedulerLogServiceImpl.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service.impl; - -import com.oppo.cloud.common.util.ui.TryNumberUtil; -import com.oppo.cloud.detect.service.SchedulerLogService; -import com.oppo.cloud.mapper.TaskApplicationMapper; -import com.oppo.cloud.model.TaskApplication; -import com.oppo.cloud.model.TaskApplicationExample; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import java.util.Arrays; -import java.util.Date; -import java.util.List; - -/** - * Implementation of scheduling log interface. - */ -@Slf4j -@Service -public class SchedulerLogServiceImpl implements SchedulerLogService { - - @Value("${custom.schedulerType}") - private String schedulerType; - - @Autowired - private TaskApplicationMapper taskApplicationMapper; - - - @Override - public List getSchedulerLog(String projectName, String flowName, String taskName, Date executionDate, - Integer tryNum) { - TaskApplicationExample taskApplicationExample = new TaskApplicationExample(); - taskApplicationExample.createCriteria() - .andProjectNameEqualTo(projectName) - .andFlowNameEqualTo(flowName) - .andTaskNameEqualTo(taskName) - .andExecuteTimeEqualTo(executionDate); - List taskApplicationList = - taskApplicationMapper.selectByExampleWithBLOBs(taskApplicationExample); - if (taskApplicationList.size() != 0) { - TaskApplication taskApplication = null; - for (TaskApplication temp : taskApplicationList) { - temp.setRetryTimes(TryNumberUtil.updateTryNumber(temp.getRetryTimes(), schedulerType)); - if (temp.getRetryTimes().equals(tryNum)) { - taskApplication = temp; - break; - } - } - // The number of retries for tasks without a scheduling cycle is set to 0 by default. - if (taskApplication == null) { - taskApplication = taskApplicationList.get(0); - } - if (!StringUtils.isEmpty(taskApplication.getLogPath())) { - return Arrays.asList(taskApplication.getLogPath().split(",")); - } - } - log.error( - "can not find scheduler log from task_application,taskName:{},flowName:{}, executionDate:{}, tryNum:{}", - taskName, flowName, executionDate, tryNum); - return null; - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/TaskAppServiceImpl.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/TaskAppServiceImpl.java deleted file mode 100644 index ae89303ef..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/TaskAppServiceImpl.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service.impl; - -import com.oppo.cloud.common.constant.ApplicationType; -import com.oppo.cloud.common.domain.cluster.spark.SparkApp; -import com.oppo.cloud.common.domain.cluster.yarn.YarnApp; -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.domain.opensearch.TaskApp; -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.common.util.DateUtil; -import com.oppo.cloud.common.util.ui.TryNumberUtil; -import com.oppo.cloud.detect.domain.AbnormalTaskAppInfo; -import com.oppo.cloud.detect.service.OpenSearchService; -import com.oppo.cloud.detect.service.TaskAppService; -import com.oppo.cloud.mapper.TaskApplicationMapper; -import com.oppo.cloud.model.TaskApplication; -import com.oppo.cloud.model.TaskApplicationExample; -import lombok.extern.slf4j.Slf4j; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import java.util.*; - -/** - * Exception task App interface class. - */ -@Service -@Slf4j -public class TaskAppServiceImpl implements TaskAppService { - - @Autowired - private TaskApplicationMapper taskApplicationMapper; - - @Autowired - private OpenSearchService openSearchService; - - @Autowired - private RedisService redisService; - - @Value("${custom.opensearch.app-index}") - private String appIndex; - - @Value("${custom.schedulerType}") - private String schedulerType; - - @Value(value = "${spark.io.compression.codec}") - private String sparkCompressionCodec; - - /** - * Get the list information of exception task Apps. - */ - @Override - public AbnormalTaskAppInfo getAbnormalTaskAppsInfo(JobAnalysis jobAnalysis, String handledApps) { - AbnormalTaskAppInfo abnormalTaskAppInfo = new AbnormalTaskAppInfo(); - List taskAppList = new ArrayList<>(); - // Collect exception information for each appId. - StringBuilder exceptionInfo = new StringBuilder(); - // Newly processed taskApps in this round. - StringBuilder handledAppsNew = new StringBuilder(); - // Determine whether the appId of each retry of the task has been found. - Map needed = new HashMap<>(); - for (int i = 0; i <= jobAnalysis.getRetryTimes(); i++) { - needed.put(i, false); - } - - List taskApplicationList = getTaskApplications(jobAnalysis.getProjectName(), - jobAnalysis.getFlowName(), jobAnalysis.getTaskName(), jobAnalysis.getExecutionDate()); - - for (TaskApplication taskApplication : taskApplicationList) { - taskApplication.setRetryTimes(TryNumberUtil.updateTryNumber(taskApplication.getRetryTimes(), schedulerType)); - try { - if (handledApps != null && handledApps.contains(taskApplication.getApplicationId())) { - // This appId has already been processed. - needed.put(taskApplication.getRetryTimes(), true); - continue; - } - if (needed.containsKey(taskApplication.getRetryTimes())) { - needed.put(taskApplication.getRetryTimes(), true); - } else { - // Compatible with manually executed tasks, all retries are treated as the first retry of a different cycle. - taskApplication.setRetryTimes(0); - needed.put(0, true); - } - // Construct a TaskApp (including related log paths) based on the appId. - TaskApp taskApp = this.buildAbnormalTaskApp(taskApplication); - // Update the metadata information to the taskApp. - taskApp.setTaskId(jobAnalysis.getTaskId()); - taskApp.setFlowId(jobAnalysis.getFlowId()); - taskApp.setProjectId(jobAnalysis.getProjectId()); - taskApp.setUsers(jobAnalysis.getUsers()); - taskAppList.add(taskApp); - handledAppsNew.append(taskApp.getApplicationId()).append(";"); - } catch (Exception e) { - exceptionInfo.append(e.getMessage()).append(";"); - } - } - List notFound = new ArrayList<>(); - // Mark the number of retries for the appId that has been queried. - for (Integer taskTryNum : needed.keySet()) { - if (!needed.get(taskTryNum)) { - notFound.add(String.valueOf(taskTryNum)); - } - } - // Determine whether all appIds under all retry counts have been found. - if (notFound.size() > 0) { - exceptionInfo.append(String.format("can not find appId by tryNum: %s", String.join(",", notFound))); - } - abnormalTaskAppInfo.setTaskAppList(taskAppList); - abnormalTaskAppInfo.setExceptionInfo(exceptionInfo.toString()); - abnormalTaskAppInfo.setHandleApps(handledAppsNew.toString()); - return abnormalTaskAppInfo; - } - - /** - * Get the appId under the exception task, including those with or without. - */ - - @Override - public Map> getAbnormalTaskApps(JobAnalysis jobAnalysis) { - Map> res = new HashMap<>(); - List taskApplicationList = getTaskApplications(jobAnalysis.getProjectName(), - jobAnalysis.getFlowName(), jobAnalysis.getTaskName(), jobAnalysis.getExecutionDate()); - // Construct all retry records based on the number of retries. - for (int i = 0; i <= jobAnalysis.getRetryTimes(); i++) { - List temp = new ArrayList<>(); - res.put(i, temp); - } - // Query all appIds. - for (TaskApplication taskApplication : taskApplicationList) { - TaskApp taskApp = this.tryBuildAbnormalTaskApp(taskApplication); - List temp; - if (res.containsKey(taskApplication.getRetryTimes())) { - temp = res.get(taskApplication.getRetryTimes()); - temp.add(taskApp); - res.put(taskApplication.getRetryTimes(), temp); - } else { - // If not included, place this appId in the first retry. - temp = new ArrayList<>(); - taskApp.setRetryTimes(0); - temp.add(taskApp); - res.put(0, temp); - } - } - // Construct an empty one if there is no appId. - for (Integer tryTime : res.keySet()) { - List temp = res.get(tryTime); - if (temp.size() == 0) { - TaskApp taskApp = new TaskApp(); - taskApp.setRetryTimes(tryTime); - taskApp.setTaskName(jobAnalysis.getTaskName()); - taskApp.setFlowName(jobAnalysis.getFlowName()); - taskApp.setProjectName(jobAnalysis.getProjectName()); - taskApp.setExecutionDate(jobAnalysis.getExecutionDate()); - temp.add(taskApp); - res.put(tryTime, temp); - } - } - return res; - } - - @Override - public void insertTaskApps(List taskAppList) throws Exception { - for (TaskApp taskApp : taskAppList) { - String index = taskApp.genIndex(appIndex); - log.info("insertTaskApp {},{},{}", index, taskApp.getApplicationId(), taskApp.genDoc()); - openSearchService.insertOrUpDate(index, taskApp.genDocId(), taskApp.genDoc()); - } - } - - @Override - public List searchTaskApps(JobAnalysis jobAnalysis) throws Exception { - HashMap termCondition = new HashMap<>(); - termCondition.put("projectName.keyword", jobAnalysis.getProjectName()); - termCondition.put("flowName.keyword", jobAnalysis.getFlowName()); - termCondition.put("taskName.keyword", jobAnalysis.getTaskName()); - termCondition.put("executionDate", DateUtil.timestampToUTCDate(jobAnalysis.getExecutionDate().getTime())); - SearchSourceBuilder searchSourceBuilder = - openSearchService.genSearchBuilder(termCondition, null, null, null); - return openSearchService.find(TaskApp.class, searchSourceBuilder, appIndex + "-*"); - } - - /** - * Construct an AbnormalTaskApp based on the basic appId information. If there is an exception, exit directly and throw an exception. - */ - public TaskApp buildAbnormalTaskApp(TaskApplication taskApplication) throws Exception { - TaskApp taskApp = new TaskApp(); - BeanUtils.copyProperties(taskApplication, taskApp); - taskApp.setExecutionDate(taskApplication.getExecuteTime()); - - YarnApp yarnApp = openSearchService.searchYarnApp(taskApplication.getApplicationId()); - - SparkApp sparkApp = null; - if (ApplicationType.SPARK.getValue().equals(yarnApp.getApplicationType())) { - sparkApp = openSearchService.searchSparkApp(taskApplication.getApplicationId()); - } - - taskApp.updateTaskApp(yarnApp, sparkApp, redisService, sparkCompressionCodec); - - return taskApp; - } - - - public TaskApp tryBuildAbnormalTaskApp(TaskApplication taskApplication) { - TaskApp taskApp = new TaskApp(); - BeanUtils.copyProperties(taskApplication, taskApp); - taskApp.setExecutionDate(taskApplication.getExecuteTime()); - taskApp.setRetryTimes(taskApplication.getRetryTimes()); - try { - YarnApp yarnApp = openSearchService.searchYarnApp(taskApplication.getApplicationId()); - taskApp.setStartTime(new Date(yarnApp.getStartedTime())); - taskApp.setFinishTime(new Date(yarnApp.getFinishedTime())); - taskApp.setElapsedTime((double) yarnApp.getElapsedTime()); - taskApp.setClusterName(yarnApp.getClusterName()); - taskApp.setApplicationType(yarnApp.getApplicationType()); - taskApp.setQueue(yarnApp.getQueue()); - taskApp.setDiagnoseResult(yarnApp.getDiagnostics()); - taskApp.setExecuteUser(yarnApp.getUser()); - taskApp.setVcoreSeconds((double) yarnApp.getVcoreSeconds()); - taskApp.setMemorySeconds((double) Math.round(yarnApp.getMemorySeconds())); - taskApp.setTaskAppState(yarnApp.getState()); - String[] amHost = yarnApp.getAmHostHttpAddress().split(":"); - if (amHost.length != 0) { - taskApp.setAmHost(amHost[0]); - } - } catch (Exception e) { - log.error("try complete yarn info failed, msg:", e); - } - try { - SparkApp sparkApp = openSearchService.searchSparkApp(taskApplication.getApplicationId()); - taskApp.setEventLogPath(sparkApp.getEventLogDirectory() + "/" + taskApplication.getApplicationId()); - } catch (Exception e) { - log.error("try complete spark info failed, msg:", e); - } - return taskApp; - } - - - public List getTaskApplications(String projectName, String flowName, String taskName, - Date executionTime) { - TaskApplicationExample taskApplicationExample = new TaskApplicationExample(); - taskApplicationExample.createCriteria() - .andProjectNameEqualTo(projectName) - .andFlowNameEqualTo(flowName) - .andTaskNameEqualTo(taskName) - .andExecuteTimeEqualTo(executionTime) - .andApplicationIdIsNotNull(); - return taskApplicationMapper.selectByExample(taskApplicationExample); - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/TaskInstanceServiceImpl.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/TaskInstanceServiceImpl.java deleted file mode 100644 index b812a6998..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/TaskInstanceServiceImpl.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service.impl; - -import com.github.pagehelper.PageHelper; -import com.github.pagehelper.PageInfo; -import com.oppo.cloud.common.constant.TaskStateEnum; -import com.oppo.cloud.detect.domain.TaskStateHistory; -import com.oppo.cloud.detect.mapper.TaskInstanceExtendMapper; -import com.oppo.cloud.detect.service.TaskInstanceService; -import com.oppo.cloud.model.TaskInstance; -import com.oppo.cloud.model.TaskInstanceExample; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.List; - -/** - * Task execution instance interface. - */ -@Service -public class TaskInstanceServiceImpl implements TaskInstanceService { - - @Autowired - private TaskInstanceExtendMapper taskInstanceExtendMapper; - - /** - * Query the data of the most recent task execution. - */ - @Override - public List searchTaskStateHistory(String projectName, String flowName, String taskName, - Date executionTime, Date endExecutionTime, Integer sampleNum) { - TaskInstanceExample taskInstanceExample = new TaskInstanceExample(); - // Take data from the past two months. - taskInstanceExample.createCriteria() - .andProjectNameEqualTo(projectName) - .andFlowNameEqualTo(flowName) - .andTaskNameEqualTo(taskName) - .andExecutionTimeLessThan(executionTime) - .andExecutionTimeGreaterThan(endExecutionTime); - return taskInstanceExtendMapper.searchTaskState(taskInstanceExample); - } - - /** - * Query the recent execution duration data of tasks. - */ - @Override - public List searchTaskDurationHistory(String projectName, String flowName, String taskName, - Date executionTime, Date endExecutionTime, Integer sampleNum) { - List result = new ArrayList<>(); - PageHelper.startPage(1, sampleNum, false); - TaskInstanceExample taskInstanceExample = new TaskInstanceExample(); - taskInstanceExample.createCriteria() - .andProjectNameEqualTo(projectName) - .andFlowNameEqualTo(flowName) - .andTaskNameEqualTo(taskName) - .andTaskStateEqualTo(TaskStateEnum.success.name()) - .andExecutionTimeLessThan(executionTime) - .andExecutionTimeGreaterThan(endExecutionTime); - List taskStateHistoryList = taskInstanceExtendMapper.searchTaskDuration(taskInstanceExample); - for (TaskStateHistory taskStateHistory : taskStateHistoryList) { - if (TaskStateEnum.success.name().equals(taskStateHistory.getState())) { - result.add(taskStateHistory.getValue()); - } - } - return result; - } - - /** - * Query the data of the difference between the end time of task execution and the running cycle in the past month. - */ - @Override - public List searchTaskRelativeEndTime(String projectName, String flowName, String taskName, - Date executionTime, Date endExecutionTime, Integer sampleNum) { - List result = new ArrayList<>(); - PageHelper.startPage(1, sampleNum, false); - TaskInstanceExample taskInstanceExample = new TaskInstanceExample(); - taskInstanceExample.createCriteria() - .andProjectNameEqualTo(projectName) - .andFlowNameEqualTo(flowName) - .andTaskNameEqualTo(taskName) - .andTaskStateEqualTo(TaskStateEnum.success.name()) - .andExecutionTimeLessThan(executionTime) - .andExecutionTimeGreaterThan(endExecutionTime); - List taskStateHistoryList = - taskInstanceExtendMapper.searchTaskRelativeEndTime(taskInstanceExample); - for (TaskStateHistory taskStateHistory : taskStateHistoryList) { - if (taskStateHistory.getValue() != null) { - result.add(taskStateHistory.getValue()); - } - } - return result; - } - - /** - * Query specific execution cycles of tasks. - */ - @Override - public TaskInstance searchTaskSum(String projectName, String flowName, String taskName, Date executionTime) { - TaskInstanceExample taskInstanceExample = new TaskInstanceExample(); - taskInstanceExample.setOrderByClause("retry_times asc"); - taskInstanceExample.createCriteria().andProjectNameEqualTo(projectName) - .andFlowNameEqualTo(flowName) - .andTaskNameEqualTo(taskName) - .andExecutionTimeEqualTo(executionTime); - List taskInstances = taskInstanceExtendMapper.selectByExample(taskInstanceExample); - if (taskInstances.size() != 0) { - Date startTime = taskInstances.get(0).getStartTime(); - TaskInstance taskInstance = taskInstances.get(taskInstances.size() - 1); - taskInstance.setStartTime(startTime); - return taskInstance; - } else { - return null; - } - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/TaskServiceImpl.java b/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/TaskServiceImpl.java deleted file mode 100644 index 15765c28d..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/service/impl/TaskServiceImpl.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service.impl; - -import com.oppo.cloud.detect.service.TaskService; -import com.oppo.cloud.mapper.TaskMapper; -import com.oppo.cloud.model.Task; -import com.oppo.cloud.model.TaskExample; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.List; - -@Service -public class TaskServiceImpl implements TaskService { - - @Autowired - private TaskMapper taskMapper; - - @Override - public Task getTask(String projectName, String flowName, String taskName) { - TaskExample taskExample = new TaskExample(); - taskExample.createCriteria().andTaskNameEqualTo(taskName) - .andFlowNameEqualTo(flowName) - .andProjectNameEqualTo(projectName); - List tasks = taskMapper.selectByExample(taskExample); - if (tasks.size() > 0) { - return tasks.get(0); - } - return null; - } - - @Override - public Task getTask(Integer id) { - TaskExample taskExample = new TaskExample(); - taskExample.createCriteria().andIdEqualTo(id); - List tasks = taskMapper.selectByExample(taskExample); - if (tasks.size() > 0) { - return tasks.get(0); - } - return null; - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/task/DelayedTask.java b/task-detect/src/main/java/com/oppo/cloud/detect/task/DelayedTask.java deleted file mode 100644 index aff40f27d..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/task/DelayedTask.java +++ /dev/null @@ -1,302 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.task; - -import com.alibaba.fastjson2.JSONObject; -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.domain.opensearch.TaskApp; -import com.oppo.cloud.common.domain.job.App; -import com.oppo.cloud.common.domain.job.LogRecord; -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.detect.config.ThreadPoolConfig; -import com.oppo.cloud.detect.domain.AbnormalTaskAppInfo; -import com.oppo.cloud.detect.domain.DelayedTaskInfo; -import com.oppo.cloud.detect.service.*; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.CommandLineRunner; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Configuration; - -import javax.annotation.PostConstruct; -import javax.annotation.Resource; -import java.util.*; -import java.util.concurrent.Executor; - -/** - * Delayed task scheduled scanning and processing service. - */ -@Slf4j -@Configuration -@ConditionalOnProperty(prefix = "custom.delayedTaskQueue", name = "enable", havingValue = "true") -public class DelayedTask implements CommandLineRunner { - - @Value("${custom.redis.logRecord}") - private String logRecordQueue; - - @Value("${custom.redis.delayedQueue}") - private String delayedQueue; - - @Value("${custom.redis.processing}") - private String processingKey; - - @Value("${custom.delayedTaskQueue.delayedSeconds}") - private Integer delaySeconds; - - @Value("${custom.delayedTaskQueue.tryTimes}") - private Integer tryTimes; - - @Resource(name = ThreadPoolConfig.DELAY_QUEUE_EXECUTOR_POOL) - private Executor delayQueueExecutorPool; - - @Autowired - private DelayedTaskService delayedTaskService; - - @Autowired - private LogRecordService logRecordService; - - @Autowired - private RedisService redisService; - - @Autowired - private TaskAppService taskAppService; - - @Autowired - private AbnormalJobService abnormalJobService; - - @Autowired - private JobInstanceService jobInstanceService; - - - @PostConstruct - void init() { - // Resume tasks interrupted by restart. - Map processingMap = null; - try { - processingMap = redisService.hGetAll(processingKey); - } catch (Exception e) { - log.error("get processing key err:", e); - } - if (processingMap != null) { - log.info("initProcessingTaskSize:{}", processingMap.size()); - processingMap.forEach((k, v) -> { - log.info("initProcessingTaskData,k:{},v:{}", k, v); - redisService.zSetAdd(delayedQueue, v, System.currentTimeMillis()); - redisService.hDel(processingKey, k); - }); - } - } - - @Override - public void run(String... args) throws Exception { - while (true) { - try { - List delayedTaskInfoList = delayedTaskService.getDelayedTasks(); - if (delayedTaskInfoList == null) { - Thread.sleep(delaySeconds * 1000); - continue; - } - for (DelayedTaskInfo delayedTaskInfo : delayedTaskInfoList) { - delayQueueExecutorPool.execute(() -> handleDelayTask(delayedTaskInfo)); - } - } catch (Exception e) { - log.error("Exception:", e); - } - } - } - - public void handleDelayTask(DelayedTaskInfo delayedTaskInfo) { - log.info("delayProcessTask:{}", delayedTaskInfo); - try { - if (delayedTaskInfo.getJobAnalysis().getCategories().size() == 0) { - dealWithNormalTask(delayedTaskInfo); - } else { - dealWithAbnormalTask(delayedTaskInfo); - } - } catch (Exception e) { - log.error("dealWithDelayTask failed, msg: ", e); - } - } - - /** - * Processing retries for failed tasks - */ - public void dealWithAbnormalTask(DelayedTaskInfo delayedTaskInfo) throws Exception { - // If the number of retries exceeds the limit, give up the retry and save the relevant appId. - // TODO: Possibly send AM logs directly if Spark tasks have not started. - if (delayedTaskInfo.getProcessRetries() >= tryTimes) { - try { - saveAllAbnormalTaskApp(delayedTaskInfo); - } catch (Exception e) { - log.error("saveAllAbnormalTaskApp failed ", e); - } - redisService.hDel(processingKey, delayedTaskInfo.getKey()); - log.error("delay task retry failed:{}", delayedTaskInfo); - return; - } - - AbnormalTaskAppInfo abnormalTaskAppInfo = - taskAppService.getAbnormalTaskAppsInfo(delayedTaskInfo.getJobAnalysis(), - delayedTaskInfo.getHandledApps()); - - // save appIds - if (abnormalTaskAppInfo.getTaskAppList().size() != 0) { - taskAppService.insertTaskApps(abnormalTaskAppInfo.getTaskAppList()); - abnormalJobService.updateResource(delayedTaskInfo.getJobAnalysis(), abnormalTaskAppInfo.getTaskAppList()); - abnormalJobService.insertOrUpdate(delayedTaskInfo.getJobAnalysis()); - jobInstanceService.insertOrUpdate(delayedTaskInfo.getJobAnalysis()); - } - - LogRecord logRecord = buildLogRecord(delayedTaskInfo, abnormalTaskAppInfo); - - if (!delayedTaskInfo.getHandledApps().contains("scheduler")) { - // The first sending needs to be accompanied by the scheduling log. - List schedulerLogApps = logRecordService.getSchedulerLog(delayedTaskInfo.getJobAnalysis()); - if (schedulerLogApps.size() != 0) { - logRecord.getApps().addAll(schedulerLogApps); - delayedTaskInfo.setHandledApps(delayedTaskInfo.getHandledApps() + "scheduler;"); - } - } - // Process delayed task information. - handleTryTask(abnormalTaskAppInfo, delayedTaskInfo); - - if (logRecord.getApps().size() == 0) { - return; - } - // Send a message for log analysis - String logRecordJson = JSONObject.toJSONString(logRecord); - Long size = redisService.lLeftPush(logRecordQueue, logRecordJson); - log.info("pushLogRecord: key:{}, size:{}, data:{}", logRecordQueue, size, logRecordJson); - } - - /** - * Handling of retries for regular tasks - */ - public void dealWithNormalTask(DelayedTaskInfo delayedTaskInfo) throws Exception { - // If the retry count is greater than the specified limit, give up retrying. - if (delayedTaskInfo.getProcessRetries() >= tryTimes) { - log.warn("discard retry task:{}", delayedTaskInfo); - redisService.hDel(processingKey, delayedTaskInfo.getKey()); - return; - } - - AbnormalTaskAppInfo abnormalTaskAppInfo = taskAppService - .getAbnormalTaskAppsInfo(delayedTaskInfo.getJobAnalysis(), delayedTaskInfo.getHandledApps()); - - if (!"".equals(abnormalTaskAppInfo.getExceptionInfo())) { - // Construct the complete appId information before sending, or retain the fallback information for the last retry. - if (delayedTaskInfo.getProcessRetries() != tryTimes - 1) { - delayedTaskInfo.setProcessRetries(delayedTaskInfo.getProcessRetries() + 1); - delayedTaskService.rePushDelayedQueue(delayedTaskInfo); - return; - } - } - if (abnormalTaskAppInfo.getTaskAppList().size() != 0) { - // Update job memorySeconds and vcoreSeconds - abnormalJobService.updateResource(delayedTaskInfo.getJobAnalysis(), abnormalTaskAppInfo.getTaskAppList()); - } - // Save job instance - jobInstanceService.insertOrUpdate(delayedTaskInfo.getJobAnalysis()); - - LogRecord logRecord = buildLogRecord(delayedTaskInfo, abnormalTaskAppInfo); - - boolean firstSend = StringUtils.isEmpty(delayedTaskInfo.getHandledApps()); - if (firstSend) { - // The first sending needs to be accompanied by the scheduling log. - List schedulerLogApps = logRecordService.getSchedulerLog(delayedTaskInfo.getJobAnalysis()); - if (schedulerLogApps.size() != 0) { - logRecord.getApps().addAll(schedulerLogApps); - delayedTaskInfo.setHandledApps(delayedTaskInfo.getHandledApps() + "scheduler;"); - } else { - log.warn("cant found schedulerLogApps"); - } - } - - // Process delayed task information. - handleTryTask(abnormalTaskAppInfo, delayedTaskInfo); - - if (logRecord.getApps().size() == 0) { - return; - } - String logRecordJson = JSONObject.toJSONString(logRecord); - Long size = redisService.lLeftPush(logRecordQueue, logRecordJson); - log.info("pushLogRecord: key:{}, size:{}, data:{}", logRecordQueue, size, logRecordJson); - } - - /** - * Process delayed task information. - */ - public void handleTryTask(AbnormalTaskAppInfo abnormalTaskAppInfo, DelayedTaskInfo delayedTaskInfo) { - if ("".equals(abnormalTaskAppInfo.getExceptionInfo())) { - delayedTaskInfo.setHandledApps(delayedTaskInfo.getHandledApps() + abnormalTaskAppInfo.getHandleApps()); - redisService.hDel(processingKey, delayedTaskInfo.getKey()); - log.info("retry delay task success:{}", delayedTaskInfo); - } else { - delayedTaskInfo.setExceptionInfo(abnormalTaskAppInfo.getExceptionInfo()); - delayedTaskInfo.setHandledApps(delayedTaskInfo.getHandledApps() + abnormalTaskAppInfo.getHandleApps()); - delayedTaskInfo.setProcessRetries(delayedTaskInfo.getProcessRetries() + 1); - delayedTaskInfo.setUpdateTime(new Date()); - delayedTaskService.rePushDelayedQueue(delayedTaskInfo); - - } - } - - /** - * Construct a LogRecord. - */ - - public LogRecord buildLogRecord(DelayedTaskInfo delayedTaskInfo, AbnormalTaskAppInfo abnormalTaskAppInfo) { - LogRecord logRecord = new LogRecord(); - logRecord.setId(UUID.randomUUID().toString()); - logRecord.setJobAnalysis(delayedTaskInfo.getJobAnalysis()); - logRecord.toTaskAppMap(abnormalTaskAppInfo.getTaskAppList()); - List appLogPath = logRecordService.getAppLog(abnormalTaskAppInfo.getTaskAppList()); - logRecord.setApps(appLogPath); - return logRecord; - } - - /** - * Finally, check and save all appIds. - */ - public void saveAllAbnormalTaskApp(DelayedTaskInfo delayedTaskInfo) throws Exception { - Map> allAbnormalTaskApp = - taskAppService.getAbnormalTaskApps(delayedTaskInfo.getJobAnalysis()); - List needInsertTaskApp = new ArrayList<>(); - for (int i = 0; i <= delayedTaskInfo.getJobAnalysis().getRetryTimes(); i++) { - List abnormalTaskAppList = allAbnormalTaskApp.get(i); - for (TaskApp taskApp : abnormalTaskAppList) { - if (taskApp.getApplicationId() != null) { - if (!delayedTaskInfo.getHandledApps().contains(taskApp.getApplicationId())) { - // Already processed appId. - needInsertTaskApp.add(taskApp); - } - } - } - } - if (needInsertTaskApp.size() != 0) { - taskAppService.insertTaskApps(needInsertTaskApp); - abnormalJobService.updateResource(delayedTaskInfo.getJobAnalysis(), needInsertTaskApp); - JobAnalysis jobAnalysis = delayedTaskInfo.getJobAnalysis(); - abnormalJobService.insertOrUpdate(jobAnalysis); - jobInstanceService.insertOrUpdate(delayedTaskInfo.getJobAnalysis()); - } - } - - -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/task/DetectedTask.java b/task-detect/src/main/java/com/oppo/cloud/detect/task/DetectedTask.java deleted file mode 100644 index 6aebfc9bd..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/task/DetectedTask.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.task; - -import com.alibaba.fastjson2.JSON; -import com.oppo.cloud.common.constant.TaskStateEnum; -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.domain.syncer.TableMessage; -import com.oppo.cloud.common.util.ui.TryNumberUtil; -import com.oppo.cloud.detect.config.ThreadPoolConfig; -import com.oppo.cloud.detect.service.BlocklistService; -import com.oppo.cloud.detect.service.DetectService; -import com.oppo.cloud.detect.service.TaskInstanceService; -import com.oppo.cloud.model.TaskInstance; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.kafka.annotation.KafkaListener; -import org.springframework.kafka.support.Acknowledgment; -import org.springframework.messaging.handler.annotation.Payload; -import org.springframework.stereotype.Component; - -import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Executor; - -/** - * Detected Task Service - */ -@Component -@Slf4j -public class DetectedTask { - - @Value("${custom.schedulerType}") - private String schedulerType; - @Resource - private List abnormalDetects; - - @Resource(name = ThreadPoolConfig.DETECT_EXECUTOR_POOL) - private Executor detectExecutorPool; - - @Autowired - private TaskInstanceService taskInstanceService; - - @Autowired - private BlocklistService blocklistService; - - @KafkaListener(topics = "${custom.kafka.consumer.topic-name}", groupId = "${custom.kafka.consumer.group-id}", autoStartup = "${custom.kafka.consumer.auto.start}") - public void consumerTask(@Payload List tableChangeMessages, Acknowledgment ack) { - for (String message : tableChangeMessages) { - // Filter non-final state task data - if (preFilter(message)) { - log.info("message:{}", message); - TableMessage tableMessage; - try { - tableMessage = JSON.parseObject(message, TableMessage.class); - } catch (Exception e) { - log.error("parse kafka message failed, error msg:{}, kafka message:{}", e.getMessage(), message); - continue; - } - TaskInstance taskInstance; - try { - taskInstance = JSON.parseObject(tableMessage.getBody(), TaskInstance.class); - } catch (Exception e) { - log.error("parse taskInstance message failed, error msg:{}, kafka message:{}", e.getMessage(), - tableMessage.getBody()); - continue; - } - if (judgeTaskFinished(taskInstance)) { - detectExecutorPool.execute(() -> detectTask(taskInstance)); - } - } - } - ack.acknowledge(); - } - - /** - * Check if it is the final state - */ - public boolean preFilter(String message) { - // Only detect non-deletion operations in the task_instance table. - if (message.contains("\"table\":\"task_instance\"") && !message.contains("\"eventType\":\"DELETE\"")) { - if (message.contains("\\\"taskState\\\":\\\"success\\\"") - || message.contains("\\\"taskState\\\":\\\"fail\\\"")) { - return true; - } - } - return false; - } - - /** - * Determine whether the task has already ended. - */ - - public boolean judgeTaskFinished(TaskInstance taskInstance) { - // Successful tasks are detected directly. - if (TaskStateEnum.success.name().equals(taskInstance.getTaskState())) { - return true; - } - if (TaskStateEnum.fail.name().equals(taskInstance.getTaskState())) { - // Manually executed retries are treated as a single execution cycle. - if ("manual".equals(taskInstance.getTriggerType())) { - return true; - } else { - // Non-manual executions are judged whether the retries are completed. - Integer tryNumber = TryNumberUtil.updateTryNumber(taskInstance.getRetryTimes(), schedulerType); - log.info("schedulerType:{},{},{}, {}", schedulerType, taskInstance.getRetryTimes(), tryNumber, taskInstance.getMaxRetryTimes()); - return tryNumber.equals(taskInstance.getMaxRetryTimes()); - } - } - return false; - } - - - /** - * Diagnose each task. - */ - public void detectTask(TaskInstance taskInstance) { - if (taskInstance.getProjectName() == null || taskInstance.getFlowName() == null) { - log.warn("instance projectName or flowName is null:{}", taskInstance); - return; - } - // Filter out whitelisted tasks. - if (blocklistService.isBlocklistTask(taskInstance.getProjectName(), taskInstance.getFlowName(), - taskInstance.getTaskName())) { - log.info("find blocklist task, taskInstance:{}", taskInstance); - return; - } - JobAnalysis jobAnalysis = new JobAnalysis(); - TaskInstance taskInstanceSum; - if ("manual".equals(taskInstance.getTriggerType())) { - // Manually executed retries are treated as a single execution cycle. - taskInstance.setRetryTimes(0); - taskInstance.setMaxRetryTimes(0); - taskInstanceSum = taskInstance; - } else { - // Update the start/end time of the task. - taskInstanceSum = taskInstanceService.searchTaskSum(taskInstance.getProjectName(), - taskInstance.getFlowName(), taskInstance.getTaskName(), taskInstance.getExecutionTime()); - } - try { - BeanUtils.copyProperties(taskInstanceSum, jobAnalysis); - } catch (Exception e) { - log.error("taskInstanceSum:{}, taskInstance:{}, exception:{}", taskInstanceSum, taskInstance, e.getMessage()); - return; - } - jobAnalysis.setExecutionDate(taskInstanceSum.getExecutionTime()); - jobAnalysis.setDuration((double) (taskInstanceSum.getEndTime().getTime() / 1000 - - taskInstanceSum.getStartTime().getTime() / 1000)); - jobAnalysis.setCategories(new ArrayList<>()); - - jobAnalysis.setRetryTimes(TryNumberUtil.updateTryNumber(jobAnalysis.getRetryTimes(),schedulerType)); - - // Exception task detection. - for (DetectService detectService : abnormalDetects) { - try { - detectService.detect(jobAnalysis); - } catch (Exception e) { - log.error("detect task failed: ", e); - } - } - - try { - if (jobAnalysis.getCategories().size() == 0) { - // Normal job task processing. - abnormalDetects.get(0).handleNormalJob(jobAnalysis); - } else { - // Exception job task processing. - abnormalDetects.get(0).handleAbnormalJob(jobAnalysis); - } - } catch (Exception e) { - log.error("handle job failed: ", e); - } - - } -} diff --git a/task-detect/src/main/java/com/oppo/cloud/detect/util/DetectorUtil.java b/task-detect/src/main/java/com/oppo/cloud/detect/util/DetectorUtil.java deleted file mode 100644 index 6b6ff2807..000000000 --- a/task-detect/src/main/java/com/oppo/cloud/detect/util/DetectorUtil.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.util; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.text.DecimalFormat; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; - -/** - * Diagnostic tool. - */ -@Slf4j -public class DetectorUtil { - - /** - * Box plot algorithm: Judging whether a number is an anomalous data based on the lower quartile, median, - * and upper quartile of a set of historical data, and calculating the median, mildly anomalous value range, - * and extremely anomalous value range of the box plot. - */ - public static double[] boxplotValue(Double[] simpleData) throws Exception { - double[] res = new double[5]; - double q1, q2, q3, iQR; - try { - double[] medianValue = sampleMedian(simpleData); - q1 = medianValue[0]; - q3 = medianValue[2]; - q2 = medianValue[1]; - iQR = q3 - q1; - } catch (Exception e) { - throw new Exception(String.format("Failed to calculate the median value：%s, Sample：%s", Arrays.toString(e.getStackTrace()), - Arrays.toString(simpleData))); - } - res[0] = q1 - 3 * iQR; - // If it is less than zero, take the minimum value - res[0] = res[0] > 0 ? res[0] : simpleData[0]; - res[1] = q1 - 1.5 * iQR; - res[2] = q2; - res[3] = q3 + 1.5 * iQR; - res[4] = q3 + 3 * iQR; - return res; - } - - /** - * Get the lower quartile, median, and upper quartile of a set of data. - */ - public static double[] sampleMedian(Double[] data) { - double[] res = new double[3]; - Arrays.sort(data); - int len = (data.length - 1); - if (len % 4 == 0) { - res[0] = data[len / 4]; - } else { - res[0] = (data[len / 4] + data[len / 4 + 1]) / 2; - } - if (len % 2 == 0) { - res[1] = data[len / 2]; - } else { - res[1] = (data[len / 2] + data[len / 2 + 1]) / 2; - } - if (len * 3 % 4 == 0) { - res[2] = data[len * 3 / 4]; - } else { - res[2] = (data[len * 3 / 4] + data[len * 3 / 4 + 1]) / 2; - } - - return res; - } - - /** - * Dynamically convert the timestamp to a string of seconds, minutes, and hours. - */ - - public static String transferSecond(double timestamp) { - DecimalFormat decimalFormat = new DecimalFormat("0.00"); - if (timestamp < 60) { - return decimalFormat.format(timestamp) + "s"; - } else if (timestamp >= 60 && timestamp < 3600) { - return decimalFormat.format(timestamp / 60) + "m"; - } else if (timestamp >= 3600 && timestamp < (24 * 3600)) { - return decimalFormat.format(timestamp / 3600) + "h"; - } else { - return decimalFormat.format(timestamp / (24 * 3600)) + "d"; - } - } - - /** - * Convert the timestamp to a time string according to the given time format. - */ - - public static String timeStampToStr(long timeUnix, String format) { - try { - if (StringUtils.isBlank(format)) { - format = "yyyy-MM-dd'T'HH:mm:ss"; - } - SimpleDateFormat timeFormat = new SimpleDateFormat(format); - return timeFormat.format(new Date(timeUnix * 1000)); - } catch (Exception e) { - log.error(e.getMessage()); - } - return ""; - } - -} diff --git a/task-detect/src/main/resources/application.yml b/task-detect/src/main/resources/application.yml deleted file mode 100644 index 8a0a0ac12..000000000 --- a/task-detect/src/main/resources/application.yml +++ /dev/null @@ -1,79 +0,0 @@ -server: - port: 7071 - -spring: - application: - name: "task-detect" - profiles: - active: - jackson: - time-zone: GMT+8 - date-format: yyyy-MM-dd HH:mm:ss - datasource: - url: jdbc:mysql://localhost:33066/compass?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai - username: root - password: root - druid: - initial-size: 5 - min-idle: 10 - max-active: 20 - redis: - cluster: - nodes: localhost:6379 - max-redirects: 3 - password: - lettuce: - pool: - max-active: 32 - max-idle: 16 - min-idle: 8 - opensearch: - nodes: localhost:9200 - username: - password: - truststore: - truststore-password: - - kafka: - bootstrap-servers: localhost:9095 - listener: - type: batch - ack-mode: MANUAL_IMMEDIATE - consumer: - enable-auto-commit: false - max-poll-records: 1 - -custom: - schedulerType: ${SCHEDULER:dolphinscheduler} - redis: - logRecord: "{lua}:log:record" - delayedQueue: "{lua}:delayed:task" - processing: "{lua}:detected:processing" - delayedTaskQueue: - enable: true - delayedSeconds: 10 - tryTimes: 5 - opensearch: - yarn-app-index: "compass-yarn-app" - spark-app-index: "compass-spark-app" - job-index: "compass-job-analysis" - app-index: "compass-task-app" - job-instance-index: "compass-job-instance" - - kafka: - consumer: - group-id: cp-detection-task - topic-name: task-instance - auto: - start: true - - detectionRule: - # Configuration for long running time, in hours. - durationWarning: 2 - # Configuration for long-term failure, in days. - alwaysFailedWarning: 10 - -spark: - io: - compression: - codec: "" diff --git a/task-detect/src/main/resources/com/oppo/cloud/detect/mapper/TaskInstanceExtendMapper.xml b/task-detect/src/main/resources/com/oppo/cloud/detect/mapper/TaskInstanceExtendMapper.xml deleted file mode 100644 index 8d1af38f5..000000000 --- a/task-detect/src/main/resources/com/oppo/cloud/detect/mapper/TaskInstanceExtendMapper.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/task-detect/src/main/resources/logback.xml b/task-detect/src/main/resources/logback.xml deleted file mode 100644 index b0f9a526e..000000000 --- a/task-detect/src/main/resources/logback.xml +++ /dev/null @@ -1,105 +0,0 @@ - - - - - logback - - - - - - - info - - - ${LOG_PATTERN_COLOR} - - - - - - logs/task-detect.info.%d{yyyy-MM-dd}.%i.log - 7 - - 500MB - - 2GB - - - INFO - ACCEPT - DENY - - - ${LOG_PATTERN} - - - - - - logs/task-detect.error.%d{yyyy-MM-dd}.%i.log - 7 - - 500MB - - 2GB - - - ERROR - ACCEPT - DENY - - - ${LOG_PATTERN} - - - - - - logs/task-detect.warn.%d{yyyy-MM-dd}.%i.log - 7 - - 500MB - - 2GB - - - WARN - ACCEPT - DENY - - - ${LOG_PATTERN} - - - - - - logs/task-detect.debug.%d{yyyy-MM-dd}.%i.log - 7 - - 500MB - - 2GB - - - debug - ACCEPT - DENY - - - ${LOG_PATTERN} - - - - - - - - - - - - - - \ No newline at end of file diff --git a/task-detect/src/main/resources/scripts/delayedTask.lua b/task-detect/src/main/resources/scripts/delayedTask.lua deleted file mode 100644 index 487a50f79..000000000 --- a/task-detect/src/main/resources/scripts/delayedTask.lua +++ /dev/null @@ -1,16 +0,0 @@ -redis.replicate_commands() -local delayQueue= KEYS[1] -local processingKey= KEYS[2] -local tasks = redis.call('ZRANGEBYSCORE', delayQueue, '0', ARGV[1]) -if next(tasks) ~= nil then - for _, task in ipairs(tasks) do - local msg = cjson.decode(task) - if msg.key ~= nil then - redis.call("HSET", processingKey, msg.key, task) - end - redis.call('ZREM', delayQueue, task) - end - return cjson.encode(tasks) -end -return nil - diff --git a/task-detect/src/test/java/com/oppo/cloud/detect/service/impl/BlocklistServiceImplTest.java b/task-detect/src/test/java/com/oppo/cloud/detect/service/impl/BlocklistServiceImplTest.java deleted file mode 100644 index fef325ecd..000000000 --- a/task-detect/src/test/java/com/oppo/cloud/detect/service/impl/BlocklistServiceImplTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service.impl; - -import com.oppo.cloud.detect.service.BlocklistService; -import com.oppo.cloud.mapper.BlocklistMapper; -import com.oppo.cloud.model.Blocklist; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.boot.test.mock.mockito.SpyBean; - -import java.util.ArrayList; -import java.util.List; - - -@SpringBootTest(classes = BlocklistServiceImpl.class) -class BlocklistServiceImplTest { - - @MockBean(name = "blocklistMapper") - BlocklistMapper blocklistMapper; - - @SpyBean - BlocklistService blocklistService; - - @Test - void isBlocklistTask() { - List res = new ArrayList<>(); - Mockito.when(blocklistMapper.selectByExample(Mockito.any())).thenReturn(res); - boolean result = blocklistService.isBlocklistTask("project", "flow", "task"); - Assertions.assertFalse(result); - } -} diff --git a/task-detect/src/test/java/com/oppo/cloud/detect/service/impl/SchedulerLogServiceImplTest.java b/task-detect/src/test/java/com/oppo/cloud/detect/service/impl/SchedulerLogServiceImplTest.java deleted file mode 100644 index 053026e51..000000000 --- a/task-detect/src/test/java/com/oppo/cloud/detect/service/impl/SchedulerLogServiceImplTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service.impl; - -import com.oppo.cloud.detect.service.SchedulerLogService; -import com.oppo.cloud.mapper.TaskApplicationMapper; -import com.oppo.cloud.model.TaskApplication; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.springframework.jdbc.core.JdbcTemplate; - -import java.util.*; - - -@SpringBootTest(classes = SchedulerLogServiceImpl.class) -class SchedulerLogServiceImplTest { - - @MockBean(name = "taskApplicationMapper") - TaskApplicationMapper taskApplicationMapper; - - @MockBean - private JdbcTemplate jdbcTemplate; - - @SpyBean - private SchedulerLogService schedulerLogService; - - @Test - void getSchedulerLog() { - List taskApplications = new ArrayList<>(); - TaskApplication taskApp = new TaskApplication(); - taskApp.setId(10); - taskApp.setRetryTimes(1); - taskApp.setLogPath("/home/service/logs/2045/2013.log"); - taskApplications.add(taskApp); - Mockito.when(taskApplicationMapper.selectByExampleWithBLOBs(Mockito.any())).thenReturn(taskApplications); - - List schedulerLog = schedulerLogService.getSchedulerLog("project", "flow", "task", new Date(), 1); - Assertions.assertTrue(schedulerLog.size() == 1); - } - -} diff --git a/task-detect/src/test/java/com/oppo/cloud/detect/service/impl/TaskInstanceServiceImplTest.java b/task-detect/src/test/java/com/oppo/cloud/detect/service/impl/TaskInstanceServiceImplTest.java deleted file mode 100644 index 8cdcc6270..000000000 --- a/task-detect/src/test/java/com/oppo/cloud/detect/service/impl/TaskInstanceServiceImplTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.detect.service.impl; - -import com.oppo.cloud.detect.domain.TaskStateHistory; -import com.oppo.cloud.detect.mapper.TaskInstanceExtendMapper; -import com.oppo.cloud.detect.service.TaskInstanceService; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.boot.test.mock.mockito.SpyBean; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - - -@SpringBootTest(classes = TaskInstanceServiceImpl.class) -class TaskInstanceServiceImplTest { - - @MockBean(name = "taskInstanceExtendMapper") - TaskInstanceExtendMapper taskInstanceExtendMapper; - - @SpyBean - TaskInstanceService taskInstanceService; - - @Test - void searchTaskStateHistory() { - List taskStateHistories = new ArrayList<>(); - Mockito.when(taskInstanceExtendMapper.searchTaskState(Mockito.any())).thenReturn(taskStateHistories); - List res = - taskInstanceService.searchTaskStateHistory("project", "flow", "task", new Date(), new Date(), 2); - Assertions.assertNotNull(res); - } - - @Test - void searchTaskDurationHistory() { - List taskStateHistories = new ArrayList<>(); - TaskStateHistory taskStateHistory = new TaskStateHistory(); - taskStateHistory.setExecutionTime(new Date()); - taskStateHistory.setState("success"); - taskStateHistory.setValue(100d); - taskStateHistories.add(taskStateHistory); - Mockito.when(taskInstanceExtendMapper.searchTaskDuration(Mockito.any())).thenReturn(taskStateHistories); - List res = - taskInstanceService.searchTaskDurationHistory("project", "flow", "task", new Date(), new Date(), 2); - Assertions.assertNotNull(res); - - } - - @Test - void searchTaskRelativeEndTime() { - List taskStateHistories = new ArrayList<>(); - TaskStateHistory taskStateHistory = new TaskStateHistory(); - taskStateHistory.setExecutionTime(new Date()); - taskStateHistory.setState("success"); - taskStateHistory.setValue(100d); - taskStateHistories.add(taskStateHistory); - Mockito.when(taskInstanceExtendMapper.searchTaskRelativeEndTime(Mockito.any())).thenReturn(taskStateHistories); - List res = - taskInstanceService.searchTaskRelativeEndTime("project", "flow", "task", new Date(), new Date(), 2); - Assertions.assertNotNull(res); - } -} diff --git a/task-detect/src/test/resources/application.yml b/task-detect/src/test/resources/application.yml deleted file mode 100644 index 0deb5b1c8..000000000 --- a/task-detect/src/test/resources/application.yml +++ /dev/null @@ -1,65 +0,0 @@ -server: - port: 7071 - -spring: - application: - name: "task-detect" - profiles: - active: - jackson: - time-zone: GMT+8 - date-format: yyyy-MM-dd HH:mm:ss - datasource: - url: jdbc:derby:memory:compass;create=true - druid: - initial-size: 5 - min-idle: 10 - max-active: 20 - redis: - host: localhost - port: 6379 - opensearch: - nodes: localhost:9200 - username: - password: - truststore: - truststore-password: - - kafka: - bootstrap-servers: localhost:9092 - listener: - type: batch - ack-mode: MANUAL_IMMEDIATE - consumer: - enable-auto-commit: false - max-poll-records: 1 - -custom: - schedulerType: ${SCHEDULER:dolphinscheduler} - redis: - logRecord: "{lua}:log:record" - delayedQueue: "{lua}:delayed:task" - processing: "{lua}:detected:processing" - delayedTaskQueue: - enable: true - delayedSeconds: 10 - tryTimes: 5 - opensearch: - yarn-app-index: "compass-yarn-app" - spark-app-index: "compass-spark-app" - job-index: "compass-job-analysis" - app-index: "compass-task-app" - job-instance-index: "compass-job-instance" - - kafka: - consumer: - group-id: cp-detection-task - topic-name: task-instance - auto: - start: true - - detectionRule: - # 运行耗时长配置，单位小时 - durationWarning: 2 - # 长期失败配置，单位天 - alwaysFailedWarning: 10 \ No newline at end of file diff --git a/task-flink/pom.xml b/task-flink/pom.xml index d517efdf5..9674eb17c 100644 --- a/task-flink/pom.xml +++ b/task-flink/pom.xml @@ -62,13 +62,7 @@ ${project.artifactId} - - org.apache.maven.plugins - maven-surefire-plugin - - true - - + org.apache.maven.plugins maven-assembly-plugin diff --git a/task-flink/src/main/resources/application.yml b/task-flink/src/main/resources/application.yml index 714823357..a92be86ef 100644 --- a/task-flink/src/main/resources/application.yml +++ b/task-flink/src/main/resources/application.yml @@ -40,7 +40,7 @@ mybatis: - classpath:dao/*.xml - classpath*:com/**/mapper/*.xml configuration: # print sql - log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + log-impl: org.apache.ibatis.logging.slf4j.Slf4jImpl diagnosis: flinkPrometheusHost: http://localhost:9090 diff --git a/task-gpt/pom.xml b/task-gpt/pom.xml index fd3b577e4..5e9c72f8d 100644 --- a/task-gpt/pom.xml +++ b/task-gpt/pom.xml @@ -73,13 +73,7 @@ ${project.artifactId} - - org.apache.maven.plugins - maven-surefire-plugin - - true - - + org.apache.maven.plugins maven-assembly-plugin diff --git a/task-gpt/src/main/resources/application.yml b/task-gpt/src/main/resources/application.yml index 1e94fcd3b..a785a4d9f 100644 --- a/task-gpt/src/main/resources/application.yml +++ b/task-gpt/src/main/resources/application.yml @@ -40,7 +40,7 @@ mybatis: - classpath:dao/*.xml - classpath*:com/**/mapper/*.xml configuration: - log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + log-impl: org.apache.ibatis.logging.slf4j.Slf4jImpl # drain config for aggregating log. drain: diff --git a/task-metadata/pom.xml b/task-metadata/pom.xml deleted file mode 100644 index b9ba09598..000000000 --- a/task-metadata/pom.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - - - compass - com.oppo.cloud - ${revision} - - 4.0.0 - - task-metadata - ${revision} - - - 8 - 8 - - - - - com.oppo.cloud - task-common - - - io.springfox - springfox-boot-starter - - - - - - - org.testcontainers - testcontainers - test - - - com.oppo.cloud - task-common - ${project.version} - test-jar - test - - - io.springfox - springfox-boot-starter - - - - - - - - ${project.artifactId} - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - maven-assembly-plugin - - - make-assembly - package - - single - - - - - ${assembly.skip} - false - - - com.oppo.cloud.meta.TaskMetaApplication - - - - src/assembly/assembly.xml - - - - - - - \ No newline at end of file diff --git a/task-metadata/src/assembly/assembly.xml b/task-metadata/src/assembly/assembly.xml deleted file mode 100644 index dd65ecc84..000000000 --- a/task-metadata/src/assembly/assembly.xml +++ /dev/null @@ -1,25 +0,0 @@ - - release - - dir - - false - - - ${project.basedir}/src/bin - bin - 0755 - - - ${project.basedir}/src/main/resources - conf - - - - - lib - - - \ No newline at end of file diff --git a/task-metadata/src/bin/startup.sh b/task-metadata/src/bin/startup.sh deleted file mode 100644 index 1eba3cecf..000000000 --- a/task-metadata/src/bin/startup.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh - -HOME_DIR=$(cd $(dirname $0)/.. && pwd) -APP_NAME="task-metadata" -PID_FILE=${HOME_DIR}/tpid - -if [ -f $(dirname $0)/compass_env.sh ]; then - source $(dirname $0)/compass_env.sh -fi - -check() { - if [ -f ${PID_FILE} ]; then - local pid=$(cat ${PID_FILE}) - echo $pid - command="ps --pid ${pid}" - if [[ $(uname) == "Darwin" ]]; then - command="ps -p ${pid}" - fi - if eval ${command} >/dev/null; then - echo "${APP_NAME} is running, pid=${pid}. Please stop first!" - exit 1 - fi - fi -} - -start() { - check - - JAVA_OPTS="-DappName=${APP_NAME} -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom" - JAVA_OPTS_GC="-server -XX:+UseG1GC -XX:G1HeapRegionSize=8m -verbose:GC -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${HOME_DIR}/logs/dump.hprof -Xloggc:${HOME_DIR}/logs/gc-${APP_NAME}.log" - - nohup java $JAVA_OPTS $JAVA_OPTS_GC -cp "${HOME_DIR}/conf":"${HOME_DIR}/lib/*" com.oppo.cloud.meta.TaskMetaApplication >/dev/null 2>&1 & - - pid=$! - echo $pid - echo $pid >$PID_FILE -} - -start \ No newline at end of file diff --git a/task-metadata/src/bin/stop.sh b/task-metadata/src/bin/stop.sh deleted file mode 100644 index ee56c546e..000000000 --- a/task-metadata/src/bin/stop.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -HOME_DIR=$(cd $(dirname $0)/.. && pwd) -APP_NAME="task-metadata" -PID_FILE=${HOME_DIR}/tpid - - -stop() { - if [ -f ${PID_FILE} ]; then - local pid=$(cat ${PID_FILE}) - echo $APP_NAME $pid - command="ps --pid ${pid}" - if [[ $(uname) == "Darwin" ]]; then - command="ps -p ${pid}" - fi - if eval ${command} >/dev/null; then - kill $pid && rm -f $PID_FILE - fi - sleep 5 - - if eval ${command} >/dev/null; then - kill -9 $pid - fi - - else - ps -ef | grep $APP_NAME | grep -v grep | awk '{print $2}' | xargs kill -9 - fi - rm -f $PID_FILE -} - -stop \ No newline at end of file diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/TaskMetaApplication.java b/task-metadata/src/main/java/com/oppo/cloud/meta/TaskMetaApplication.java deleted file mode 100644 index fd3c2c073..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/TaskMetaApplication.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.scheduling.annotation.EnableScheduling; - -@ComponentScan(basePackages = "com.oppo.cloud.*") -@EnableScheduling -@SpringBootApplication -public class TaskMetaApplication { - - public static void main(String[] args) { - SpringApplication.run(TaskMetaApplication.class, args); - } - -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/config/HadoopConfig.java b/task-metadata/src/main/java/com/oppo/cloud/meta/config/HadoopConfig.java deleted file mode 100644 index 1c9d36fef..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/config/HadoopConfig.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.config; - -import com.oppo.cloud.common.domain.cluster.hadoop.HadoopConf; -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; - -/** - * hadoop config - */ -@Configuration -@ConfigurationProperties(prefix = "hadoop") -@Data -public class HadoopConfig extends HadoopConf { -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/config/OpenSearchConfig.java b/task-metadata/src/main/java/com/oppo/cloud/meta/config/OpenSearchConfig.java deleted file mode 100644 index a8ef1df3c..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/config/OpenSearchConfig.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.config; - -import com.oppo.cloud.common.util.opensearch.OpenSearchClient; -import lombok.extern.slf4j.Slf4j; - -import org.opensearch.client.RestHighLevelClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; - -/** - * OpenSearch Config - */ -@Configuration -@Slf4j -public class OpenSearchConfig { - - @Value("${spring.opensearch.nodes}") - private String nodes; - @Value("${spring.opensearch.username}") - private String username; - @Value("${spring.opensearch.password}") - private String password; - @Value("${spring.opensearch.truststore}") - private String truststore; - @Value("${spring.opensearch.truststore-password}") - private String truststorePassword; - - @Primary - @Bean("opensearch") - public RestHighLevelClient restHighLevelClient() { - return OpenSearchClient.create(nodes, username, password, truststore, truststorePassword); - } - -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/config/RedisConfig.java b/task-metadata/src/main/java/com/oppo/cloud/meta/config/RedisConfig.java deleted file mode 100644 index 7ad0325c7..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/config/RedisConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.config; - -import com.oppo.cloud.common.config.RedisTemplateConfig; -import org.springframework.context.annotation.Bean; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.Resource; -import org.springframework.data.redis.core.script.RedisScript; -import org.springframework.stereotype.Component; - -/** - * redis config - */ -@Component -public class RedisConfig extends RedisTemplateConfig { - - @Bean - public RedisScript releaseLockScript() { - Resource scriptSource = new ClassPathResource("scripts/releaseLock.lua"); - return RedisScript.of(scriptSource, Object.class); - } -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/config/ThreadPoolConfig.java b/task-metadata/src/main/java/com/oppo/cloud/meta/config/ThreadPoolConfig.java deleted file mode 100644 index 311b48652..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/config/ThreadPoolConfig.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.config; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; - -import java.util.concurrent.Executor; -import java.util.concurrent.ThreadPoolExecutor; - -/** - * Executor Thread Pool Configuration - */ -@Configuration -public class ThreadPoolConfig { - - @Bean("yarnMetaExecutor") - public Executor yarnAppsExecutor() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(4); - executor.setMaxPoolSize(8); - executor.setKeepAliveSeconds(120); - executor.setQueueCapacity(500); - executor.setThreadNamePrefix("yarn-request-"); - executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); - executor.setWaitForTasksToCompleteOnShutdown(true); - executor.setAwaitTerminationSeconds(60); - executor.initialize(); - return executor; - } - - @Bean("sparkMetaExecutor") - public Executor sparkAppsExecutor() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(4); - executor.setMaxPoolSize(8); - executor.setKeepAliveSeconds(120); - executor.setQueueCapacity(500); - executor.setThreadNamePrefix("spark-request-"); - executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); - executor.setWaitForTasksToCompleteOnShutdown(true); - executor.setAwaitTerminationSeconds(60); - executor.initialize(); - return executor; - } -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/domain/HistoryInfo.java b/task-metadata/src/main/java/com/oppo/cloud/meta/domain/HistoryInfo.java deleted file mode 100644 index a3c278f0c..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/domain/HistoryInfo.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.oppo.cloud.meta.domain; - -import lombok.Data; - -@Data -public class HistoryInfo { - - private String hadoopVersion; -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/domain/HistoryInfoProperties.java b/task-metadata/src/main/java/com/oppo/cloud/meta/domain/HistoryInfoProperties.java deleted file mode 100644 index 318d1b816..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/domain/HistoryInfoProperties.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.oppo.cloud.meta.domain; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class HistoryInfoProperties { - @JsonProperty("historyInfo") - private HistoryInfo historyInfo; -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/domain/Properties.java b/task-metadata/src/main/java/com/oppo/cloud/meta/domain/Properties.java deleted file mode 100644 index 1fb57e610..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/domain/Properties.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.domain; - -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import lombok.Data; -import org.springframework.util.Assert; - -@Data -public class Properties { - - private String key; - private String value; - - public String getFinalValue(List properties) { - String variableName = findVariableName(value); - if (variableName == null) { - return value; - } - - for (Properties property : properties) { - if (!variableName.equals(property.key)) { - continue; - } - - String variableValue = property.getFinalValue(properties); - return value.replace("${" + variableName + "}", variableValue); - } - - return value; - } - - private String findVariableName(String originalValue) { - Pattern VAR_PATTERN = Pattern.compile("\\$\\{([^{}\\s\\r\\n]+)}"); // format of variables, for example: ${var_name} - - Matcher matcher = VAR_PATTERN.matcher(originalValue); - if (matcher.find()) { - return matcher.group(1); - } - - return null; - } - - public static void main(String[] args) { - List properties = new ArrayList<>(); - Properties prop_1 = new Properties(); - prop_1.key = "yarn.timeline-service.webapp.address"; - prop_1.value = "${yarn.timeline-service.hostname}:8188"; - - Properties prop_2 = new Properties(); - prop_2.key = "dfs.webhdfs.user.provider.user.pattern"; - prop_2.value = "^[A-Za-z_][A-Za-z0-9._-]*[$]?$"; - - Properties prop_3 = new Properties(); - prop_3.key = "yarn.scheduler.configuration.fs.path"; - prop_3.value = "file://${hadoop.tmp.dir}/yarn/system/schedconf"; - - Properties prop_4 = new Properties(); - prop_4.key = "yarn.timeline-service.reader.webapp.address"; - prop_4.value = "${yarn.timeline-service.webapp.address}"; - - Properties prop_5 = new Properties(); - prop_5.key = "yarn.timeline-service.hostname"; - prop_5.value = "127.0.0.1"; - - properties.add(prop_1); - properties.add(prop_2); - properties.add(prop_3); - properties.add(prop_4); - properties.add(prop_5); - - Assert.isTrue("127.0.0.1:8188".equals(prop_1.getFinalValue(properties))); - Assert.isTrue("127.0.0.1:8188".equals(prop_4.getFinalValue(properties))); - Assert.isTrue("file://${hadoop.tmp.dir}/yarn/system/schedconf".equals(prop_3.getFinalValue(properties))); - Assert.isTrue("^[A-Za-z_][A-Za-z0-9._-]*[$]?$".equals(prop_2.getFinalValue(properties))); - Assert.isTrue("127.0.0.1".equals(prop_5.getFinalValue(properties))); - } -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/domain/YarnConfProperties.java b/task-metadata/src/main/java/com/oppo/cloud/meta/domain/YarnConfProperties.java deleted file mode 100644 index 16da240c1..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/domain/YarnConfProperties.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.domain; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.List; - -@Data -public class YarnConfProperties { - - @JsonProperty("properties") - private List properties; -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/domain/YarnPathInfo.java b/task-metadata/src/main/java/com/oppo/cloud/meta/domain/YarnPathInfo.java deleted file mode 100644 index a731d70a0..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/domain/YarnPathInfo.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.domain; - -import lombok.Data; - -@Data -public class YarnPathInfo { - /** - * fs.defaultFS - */ - private String defaultFS; - - /** - * yarn.nodemanager.remote-app-log-dir - */ - private String remoteDir; - - /** - * yarn.app.mapreduce.am.staging-dir - */ - private String mapreduceStagingDir; - - /** - * mapreduce.jobhistory.done-dir - */ - private String mapreduceDoneDir; - - /** - * mapreduce.jobhistory.intermediate-done-dir - */ - private String mapreduceIntermediateDoneDir; -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/enums/ClusterRole.java b/task-metadata/src/main/java/com/oppo/cloud/meta/enums/ClusterRole.java deleted file mode 100644 index 0623d4644..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/enums/ClusterRole.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.enums; - -/** - * Cluster Role Enumeration - */ -public enum ClusterRole { - resourcemanager, - jobhistoryserver, - historyserver; -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/enums/ModuleType.java b/task-metadata/src/main/java/com/oppo/cloud/meta/enums/ModuleType.java deleted file mode 100644 index 6fea00dab..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/enums/ModuleType.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.enums; - -/** - * Module Enumeration - */ -public enum ModuleType { - spark, - yarn; -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/scheduler/ClusterConfigScheduler.java b/task-metadata/src/main/java/com/oppo/cloud/meta/scheduler/ClusterConfigScheduler.java deleted file mode 100644 index db3a3788b..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/scheduler/ClusterConfigScheduler.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.scheduler; - -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.meta.service.IClusterConfigService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.redis.core.script.RedisScript; -import org.springframework.scheduling.annotation.Scheduled; - -import javax.annotation.PostConstruct; -import javax.annotation.Resource; -import java.util.Collections; -import java.util.UUID; - -/** - * Synchronization of YARN and Spark Cluster Address Information - */ -@Slf4j -@Configuration -@ConditionalOnProperty(prefix = "scheduler.clusterMeta", name = "enable", havingValue = "true") -public class ClusterConfigScheduler { - - /** - * synchronize cluster config redis distributed lock - */ - private static final String LOCK_KEY = "compass:metadata:cluster"; - - @Resource - private RedisService redisService; - - @Resource - private RedisScript releaseLockScript; - - @Resource - private IClusterConfigService iClusterConfigService; - - @PostConstruct - void init() { - iClusterConfigService.updateClusterConfig(); - } - - @Scheduled(cron = "${scheduler.clusterMeta.cron}") - private void run() { - try { - syncer(); - } catch (Exception e) { - log.error(e.getMessage()); - } - } - - private void syncer() throws Exception { - String lockValue = UUID.randomUUID().toString(); - // Only one instance of the application can run the synchronization process at the same time. - Boolean acquire = redisService.acquireLock(LOCK_KEY, lockValue, 5L); - if (!acquire) { - log.info("can not get the lock: {}", LOCK_KEY); - return; - } - try { - log.info("lockKey: {}, lockValue: {}", LOCK_KEY, lockValue); - iClusterConfigService.updateClusterConfig(); - } catch (Exception e) { - log.error("Exception: ", e); - } finally { - Object result = redisService.executeScript(releaseLockScript, Collections.singletonList(LOCK_KEY), lockValue); - log.info("release {}, result: {}", LOCK_KEY, result); - } - } - -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/scheduler/SparkMetaScheduler.java b/task-metadata/src/main/java/com/oppo/cloud/meta/scheduler/SparkMetaScheduler.java deleted file mode 100644 index cb32e90ca..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/scheduler/SparkMetaScheduler.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.scheduler; - -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.meta.service.ITaskSyncerMetaService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.redis.core.script.RedisScript; -import org.springframework.scheduling.annotation.Scheduled; - -import javax.annotation.Resource; -import java.util.Collections; -import java.util.UUID; - -/** - * Synchronization of Spark Job Application List Data - */ -@Slf4j -@Configuration -@ConditionalOnProperty(prefix = "scheduler.sparkMeta", name = "enable", havingValue = "true") -public class SparkMetaScheduler { - - /** - * synchronize spark app redis distributed lock - */ - private static final String LOCK_KEY = "compass:metadata:spark"; - - @Resource - private RedisService redisService; - - @Resource - private RedisScript releaseLockScript; - - @Resource(name = "SparkMetaServiceImpl") - private ITaskSyncerMetaService spark; - - @Scheduled(cron = "${scheduler.sparkMeta.cron}") - private void run() { - try { - syncer(); - } catch (Exception e) { - log.error("Exception: ", e); - } - } - - private void syncer() throws Exception { - String lockValue = UUID.randomUUID().toString(); - // Only one instance of the application can run the synchronization process at the same time. - Boolean acquire = redisService.acquireLock(LOCK_KEY, lockValue, 5L); - if (!acquire) { - log.info("can not get the lock: {}", LOCK_KEY); - return; - } - try { - log.info("lockKey: {}, lockValue: {}", LOCK_KEY, lockValue); - spark.syncer(); - } catch (Exception e) { - log.error("Exception: ", e); - } finally { - Object result = redisService.executeScript(releaseLockScript, Collections.singletonList(LOCK_KEY), lockValue); - log.info("release {}, result: {}", LOCK_KEY, result); - } - } - -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/scheduler/YarnMetaScheduler.java b/task-metadata/src/main/java/com/oppo/cloud/meta/scheduler/YarnMetaScheduler.java deleted file mode 100644 index 97a23ae9d..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/scheduler/YarnMetaScheduler.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.scheduler; - -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.meta.service.ITaskSyncerMetaService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.redis.core.script.RedisScript; -import org.springframework.scheduling.annotation.Scheduled; - -import javax.annotation.Resource; -import java.util.Collections; -import java.util.UUID; - -/** - * Synchronization of YARN Job Application List Data - */ -@Slf4j -@Configuration -@ConditionalOnProperty(prefix = "scheduler.yarnMeta", name = "enable", havingValue = "true") -public class YarnMetaScheduler { - - /** - * synchronize yarn app redis distributed lock - */ - private static final String LOCK_KEY = "compass:metadata:yarn"; - - @Resource - private RedisService redisService; - - @Resource - private RedisScript releaseLockScript; - - @Resource(name = "YarnMetaServiceImpl") - private ITaskSyncerMetaService yarn; - - @Scheduled(cron = "${scheduler.yarnMeta.cron}") - private void run() { - try { - syncer(); - } catch (Exception e) { - log.error("Exception:", e); - } - } - - private void syncer() throws Exception { - String lockValue = UUID.randomUUID().toString(); - // Only one instance of the application can run the synchronization process at the same time. - Boolean acquire = redisService.acquireLock(LOCK_KEY, lockValue, 5L); - if (!acquire) { - log.info("can not get the lock: {}", LOCK_KEY); - return; - } - try { - log.info("lockKey: {}, lockValue: {}", LOCK_KEY, lockValue); - yarn.syncer(); - } catch (Exception e) { - log.error("Exception: ", e); - } finally { - Object result = redisService.executeScript(releaseLockScript, Collections.singletonList(LOCK_KEY), lockValue); - log.info("release {}, result: {}", LOCK_KEY, result); - } - } - -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/service/IClusterConfigService.java b/task-metadata/src/main/java/com/oppo/cloud/meta/service/IClusterConfigService.java deleted file mode 100644 index 6a6e1aefa..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/service/IClusterConfigService.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.service; - -import java.util.List; -import java.util.Map; - -/** - * YARN and Spark Cluster Address Configuration Information - */ -public interface IClusterConfigService { - - /** - * Obtaining the Spark History Server List - */ - List getSparkHistoryServers(); - - /** - * Obtaining the YARN Resource Manager List - */ - Map getYarnClusters(); - - /** - * Updating Cluster Information - */ - void updateClusterConfig(); -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/service/ITaskSyncerMetaService.java b/task-metadata/src/main/java/com/oppo/cloud/meta/service/ITaskSyncerMetaService.java deleted file mode 100644 index b0f272699..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/service/ITaskSyncerMetaService.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.service; - -/** - * Synchronization of Cluster Task Metadata - */ -public interface ITaskSyncerMetaService { - - /** - * Data Synchronization - */ - void syncer(); -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/service/impl/ClusterConfigServiceImpl.java b/task-metadata/src/main/java/com/oppo/cloud/meta/service/impl/ClusterConfigServiceImpl.java deleted file mode 100644 index e69e3e4c2..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/service/impl/ClusterConfigServiceImpl.java +++ /dev/null @@ -1,337 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.service.impl; - -import com.alibaba.fastjson2.JSON; -import com.oppo.cloud.common.constant.Constant; -import com.oppo.cloud.common.domain.cluster.hadoop.YarnConf; -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.common.util.YarnUtil; -import com.oppo.cloud.meta.config.HadoopConfig; -import com.oppo.cloud.meta.domain.HistoryInfoProperties; -import com.oppo.cloud.meta.domain.YarnPathInfo; -import com.oppo.cloud.meta.domain.Properties; -import com.oppo.cloud.meta.domain.YarnConfProperties; -import com.oppo.cloud.meta.service.IClusterConfigService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.http.HttpHeaders; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestTemplate; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; - -import javax.annotation.Resource; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import java.io.ByteArrayInputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * YARN and Spark Cluster Address Configuration Information - */ -@Slf4j -@Service -public class ClusterConfigServiceImpl implements IClusterConfigService { - - @Resource - private RedisService redisService; - - @Resource - private HadoopConfig config; - - @Resource(name = "restTemplate") - private RestTemplate restTemplate; - - private static final String YARN_CONF = "http://%s/conf"; - - private static final String HISTORY_INFO_API = "http://%s/ws/v1/history/info"; - - private static final String DEFAULT_FS = "fs.defaultFS"; - - private static final String YARN_REMOTE_APP_LOG_DIR = "yarn.nodemanager.remote-app-log-dir"; - - private static final String YARN_MAPREDUCE_STAGING_DIR = "yarn.app.mapreduce.am.staging-dir"; - - private static final String MARREDUCE_DONE_DIR = "mapreduce.jobhistory.done-dir"; - - private static final String MARREDUCE_INTERMEDIATE_DONE_DIR = "mapreduce.jobhistory.intermediate-done-dir"; - - /** - * Get spark history servers - */ - @Override - public List getSparkHistoryServers() { - return config.getSpark().getSparkHistoryServer(); - } - - /** - * Get Yarn clusters - */ - @Override - public Map getYarnClusters() { - List yarnConfList = config.getYarn(); - return YarnUtil.getYarnClusters(yarnConfList); - } - - - /** - * Update cluster config - */ - @Override - public void updateClusterConfig() { - - log.info("clusterConfig:{}", config); - // cache spark history server - List sparkHistoryServerList = config.getSpark().getSparkHistoryServer(); - log.info("{}:{}", Constant.SPARK_HISTORY_SERVERS, sparkHistoryServerList); - redisService.set(Constant.SPARK_HISTORY_SERVERS, JSON.toJSONString(sparkHistoryServerList)); - - // cache yarn server - List yarnConfList = config.getYarn(); - // the jobHistoryServer corresponding to resourceManager - Map rmJhsMap = new HashMap<>(); - yarnConfList.forEach(clusterInfo -> clusterInfo.getResourceManager() - .forEach(rm -> rmJhsMap.put(rm, clusterInfo.getJobHistoryServer()))); - redisService.set(Constant.YARN_CLUSTERS, JSON.toJSONString(sparkHistoryServerList)); - log.info("{}:{}", Constant.YARN_CLUSTERS, yarnConfList); - redisService.set(Constant.RM_JHS_MAP, JSON.toJSONString(rmJhsMap)); - log.info("{}:{}", Constant.RM_JHS_MAP, rmJhsMap); - updateJHSConfig(yarnConfList); - } - - /** - * Update the jobhistoryserver hdfs path information in the configuration - */ - public void updateJHSConfig(List list) { - for (YarnConf yarnClusterInfo : list) { - String host = yarnClusterInfo.getJobHistoryServer(); - YarnPathInfo yarnPathInfo = getYarnPathInfo(host); - if (yarnPathInfo == null) { - log.error("get {}, hdfsPath empty", host); - continue; - } - String remotePathKey = Constant.JHS_HDFS_PATH + host; - String suffixPathKey = Constant.JHS_HDFS_SUFFIX_PATH + host; - String mapreduceStagingPathKey = Constant.JHS_MAPREDUCE_STAGING_PATH + host; - String mapreduceDonePathKey = Constant.JHS_MAPREDUCE_DONE_PATH + host; - String mapreduceIntermediateDonePathKey = Constant.JHS_MAPREDUCE_INTERMEDIATE_DONE_PATH + host; - String suffixPath = getDifferentHadoopSuffixDir(host); - log.info("cache yarnPathInfo:{},{}", remotePathKey, yarnPathInfo.getRemoteDir()); - log.info("cache yarnPathInfo:{},{}", suffixPathKey, suffixPath); - log.info("cache yarnPathInfo:{},{}", mapreduceStagingPathKey, yarnPathInfo.getMapreduceStagingDir()); - log.info("cache yarnPathInfo:{},{}", mapreduceDonePathKey, yarnPathInfo.getMapreduceDoneDir()); - log.info("cache yarnPathInfo:{},{}", mapreduceIntermediateDonePathKey, yarnPathInfo.getMapreduceIntermediateDoneDir()); - redisService.set(remotePathKey, yarnPathInfo.getRemoteDir()); - redisService.set(suffixPathKey, suffixPath); - redisService.set(mapreduceStagingPathKey, yarnPathInfo.getMapreduceStagingDir()); - redisService.set(mapreduceDonePathKey, yarnPathInfo.getMapreduceDoneDir()); - redisService.set(mapreduceIntermediateDonePathKey, yarnPathInfo.getMapreduceIntermediateDoneDir()); - } - } - - /** - * Get jobhistoryserver hdfs path information - */ - public YarnPathInfo getYarnPathInfo(String ip) { - String url = String.format(YARN_CONF, ip); - log.info("getHDFSPath:{}", url); - ResponseEntity responseEntity; - try { - responseEntity = restTemplate.getForEntity(url, String.class); - } catch (Exception e) { - log.error("getHDFSPathErr:{},{}", url, e.getMessage()); - return null; - } - - if (responseEntity.getBody() == null) { - log.error("getHDFSPathErr:{}", url); - return null; - } - List contentType = responseEntity.getHeaders().get(HttpHeaders.CONTENT_TYPE); - if (contentType == null) { - log.error("getContentTypeErr:{}", url); - return null; - } - - YarnConfProperties yarnConfProperties = new YarnConfProperties(); - - try { - if (contentType.toString().contains("json")) { - yarnConfProperties = JSON.parseObject(responseEntity.getBody(), YarnConfProperties.class); - } else if (contentType.toString().contains("xml")) { - parseXML(responseEntity.getBody(), yarnConfProperties); - } else { - log.error("unsupported type: {},{}", url, contentType); - return null; - } - } catch (Exception e) { - log.error("Exception:", e); - return null; - } - - String remoteDir = ""; - String defaultFS = ""; - String mapreduceStagingDir = ""; - String mapreduceDoneDir = ""; - String mapreduceIntermediateDoneDir = ""; - - if (yarnConfProperties != null && yarnConfProperties.getProperties() != null) { - for (Properties properties : yarnConfProperties.getProperties()) { - String key = properties.getKey(); - String value = properties.getFinalValue(yarnConfProperties.getProperties()); - if (YARN_REMOTE_APP_LOG_DIR.equals(key)) { - log.info("yarnConfProperties key: {}, value: {}", YARN_REMOTE_APP_LOG_DIR, value); - remoteDir = value; - } - if (DEFAULT_FS.equals(key)) { - log.info("yarnConfProperties key: {}, value: {}", DEFAULT_FS, value); - defaultFS = value; - } - if (MARREDUCE_DONE_DIR.equals(key)) { - log.info("yarnConfProperties key: {}, value: {}", MARREDUCE_DONE_DIR, value); - mapreduceDoneDir = value; - } - if (MARREDUCE_INTERMEDIATE_DONE_DIR.equals(key)) { - log.info("yarnConfProperties key: {}, value: {}", MARREDUCE_INTERMEDIATE_DONE_DIR, value); - mapreduceIntermediateDoneDir = value; - } - if (YARN_MAPREDUCE_STAGING_DIR.equals(key)) { - log.info("yarnConfProperties key: {}, value: {}", YARN_MAPREDUCE_STAGING_DIR, value); - mapreduceStagingDir = value; - } - } - } - if (StringUtils.isEmpty(defaultFS)) { - log.error("defaultFSEmpty:{}", url); - return null; - } - if (StringUtils.isEmpty(remoteDir)) { - log.error("remoteDirEmpty:{}", url); - return null; - } - if (StringUtils.isEmpty(mapreduceDoneDir)) { - log.error("mapreduceDoneDirEmpty:{}", url); - return null; - } - if (!remoteDir.contains(Constant.HDFS_SCHEME)) { - remoteDir = defaultFS + remoteDir; - } - if (!mapreduceStagingDir.contains(Constant.HDFS_SCHEME)) { - mapreduceStagingDir = defaultFS + mapreduceStagingDir; - } - if (!mapreduceDoneDir.contains(Constant.HDFS_SCHEME)) { - mapreduceDoneDir = defaultFS + mapreduceDoneDir; - } - if (!mapreduceIntermediateDoneDir.contains(Constant.HDFS_SCHEME)) { - mapreduceIntermediateDoneDir = defaultFS + mapreduceIntermediateDoneDir; - } - - YarnPathInfo yarnPathInfo = new YarnPathInfo(); - yarnPathInfo.setDefaultFS(defaultFS); - yarnPathInfo.setRemoteDir(remoteDir); - yarnPathInfo.setMapreduceStagingDir(mapreduceStagingDir); - yarnPathInfo.setMapreduceDoneDir(mapreduceDoneDir); - yarnPathInfo.setMapreduceIntermediateDoneDir(mapreduceIntermediateDoneDir); - log.info("yarnPathInfo: {}, {}", url, yarnPathInfo); - return yarnPathInfo; - } - - /** - * Parse xml format - */ - public void parseXML(String xml, YarnConfProperties yarnConfProperties) throws Exception { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - Document document = builder.parse(new ByteArrayInputStream(xml.getBytes())); - Element root = document.getDocumentElement(); - NodeList nodeList = root.getElementsByTagName("property"); - List properties = new ArrayList<>(); - for (int i = 0; i < nodeList.getLength(); i++) { - Element element = (Element) nodeList.item(i); - String name = element.getElementsByTagName("name").item(0).getTextContent(); - String value = element.getElementsByTagName("value").item(0).getTextContent(); - Properties property = new Properties(); - property.setKey(name); - property.setValue(value); - properties.add(property); - } - yarnConfProperties.setProperties(properties); - } - - /** - * Get the log directory suffix - */ - public String getDifferentHadoopSuffixDir(String ip) { - String version = getHadoopVersion(ip); - log.info("hadoop version:{},{}", ip, version); - if (StringUtils.isBlank(version)) { - return "logs"; - } - // Compatible with x.x.x or x.x.x-xxx format, for example 2.6.0 or 2.6.0-cdh5.14.4 etc. - if (version.startsWith("2.")) { - return "logs"; - } else if (version.startsWith("3.0") || version.startsWith("3.1") || version.startsWith("3.2.0")) { - return "logs"; - } else if (version.startsWith("3.2")) { - return "logs-tfile"; - } else if (version.startsWith("3.3")) { - return "bucket-logs-tfile"; - } else { - return "logs"; - } - } - - /** - * Get hadoop version by history server api - */ - public String getHadoopVersion(String ip) { - String url = String.format(HISTORY_INFO_API, ip); - log.info("get history info: {}", url); - ResponseEntity responseEntity; - try { - responseEntity = restTemplate.getForEntity(url, String.class); - } catch (Exception e) { - log.error("Exception:{}", url, e); - return null; - } - - if (responseEntity.getBody() == null) { - log.error("get history info err: {}", url); - return null; - } - - HistoryInfoProperties historyInfo = null; - try { - historyInfo = JSON.parseObject(responseEntity.getBody(), HistoryInfoProperties.class); - } catch (Exception e) { - log.error("Exception:{}", url, e); - return null; - } - if (historyInfo != null && historyInfo.getHistoryInfo() != null) { - return historyInfo.getHistoryInfo().getHadoopVersion(); - } - return null; - } - - -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/service/impl/SparkMetaServiceImpl.java b/task-metadata/src/main/java/com/oppo/cloud/meta/service/impl/SparkMetaServiceImpl.java deleted file mode 100644 index 55416c9c7..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/service/impl/SparkMetaServiceImpl.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.service.impl; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.type.TypeFactory; -import com.oppo.cloud.common.constant.Constant; -import com.oppo.cloud.common.domain.cluster.spark.SparkApp; -import com.oppo.cloud.common.domain.cluster.spark.SparkApplication; -import com.oppo.cloud.common.domain.cluster.yarn.Attempt; -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.common.util.DateUtil; -import com.oppo.cloud.common.util.opensearch.BulkApi; -import com.oppo.cloud.meta.config.HadoopConfig; -import com.oppo.cloud.meta.service.IClusterConfigService; -import com.oppo.cloud.meta.service.ITaskSyncerMetaService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.opensearch.action.bulk.BulkItemResponse; -import org.opensearch.action.bulk.BulkResponse; -import org.opensearch.client.RestHighLevelClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestClientException; -import org.springframework.web.client.RestTemplate; - -import javax.annotation.Resource; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Synchronization of Spark Application Metadata - */ -@Slf4j -@Service("SparkMetaServiceImpl") -public class SparkMetaServiceImpl implements ITaskSyncerMetaService { - - @Value("${scheduler.sparkMeta.limitCount}") - private long limitCount; - - @Value("${spring.opensearch.spark-app-prefix}") - private String sparkAppPrefix; - @Resource - private HadoopConfig config; - - @Resource - private IClusterConfigService iClusterConfigService; - - @Resource(name = "restTemplate") - private RestTemplate restTemplate; - - @Resource - private RedisService redisService; - - @Resource - private Executor sparkMetaExecutor; - - @Resource - private ObjectMapper objectMapper; - - @Resource - private RestHighLevelClient client; - - private final Pattern hdfsPattern = Pattern.compile(".*?(?hdfs://.*).*", Pattern.DOTALL); - - private static final String SPARK_HOME_URL = "http://%s/"; - - private static final String SPARK_APPS_URL = "http://%s/api/v1/applications?limit=%d&minDate=%s"; - - @Override - public void syncer() { - List clusters = iClusterConfigService.getSparkHistoryServers(); - log.info("sparkClusters:{}", clusters); - if (clusters == null) { - return; - } - CompletableFuture[] array = new CompletableFuture[clusters.size()]; - for (int i = 0; i < clusters.size(); i++) { - int finalI = i; - array[i] = CompletableFuture.supplyAsync(() -> { - try { - pull(clusters.get(finalI)); - } catch (Exception e) { - log.error("Exception: ", e); - } - return null; - }, sparkMetaExecutor); - } - try { - CompletableFuture.allOf(array).get(); - } catch (Exception e) { - log.error(e.getMessage()); - } - } - - /** - * Synchronization of Task Data - */ - public void pull(String shs) { - log.info("start to pull spark tasks:{}", shs); - String eventLogDirectory; - try { - eventLogDirectory = getEventLogDirectory(shs); - } catch (Exception e) { - log.error("sparkMetaErr:eventLogDirectory:{}", shs, e); - return; - } - if (StringUtils.isBlank(eventLogDirectory)) { - log.info("sparkMetaErr:eventLogDirectory:{}", shs); - return; - } - - List apps = sparkRequest(shs); - if (apps == null || apps.size() == 0) { - log.error("sparkMetaErr:appsNull:{}", shs); - return; - } - log.info("sparkApps:{},{}", shs, apps.size()); - Map> sparkAppMap = new HashMap<>(); - - for (SparkApplication info : apps) { - if (info.getAttempts() == null || info.getAttempts().size() == 0) { - log.error("sparkHistoryInfoAttemptSizeZero {},{}", shs, info); - continue; - } - Attempt attempt = info.getAttempts().get(0); - try { - SparkApp sparkApp = new SparkApp(info.getId(), eventLogDirectory, attempt, shs); - log.info("sparkApp:{}", sparkApp); - String id = sparkApp.getSparkHistoryServer() + "_" + sparkApp.getAppId(); - sparkAppMap.put(id, sparkApp.getSparkAppMap()); - } catch (Exception e) { - log.error("saveSparkAppsErr:{},{},{}", shs, e.getMessage(), e); - } - } - - BulkResponse response; - try { - response = BulkApi.bulkByIds(client, sparkAppPrefix + DateUtil.getDay(0), sparkAppMap); - } catch (Exception e) { - log.error("bulkSparkAppsErr:", e); - return; - } - BulkItemResponse[] responses = response.getItems(); - - for (BulkItemResponse r : responses) { - if (r.isFailed()) { - log.error("failedInsertApp:{},{}", r.getId(), r.status()); - } - } - - log.info("saveSparkAppCount:{},{}", shs, sparkAppMap.size()); - } - - /** - * Obtaining Spark Jobs - */ - public List sparkRequest(String shs) { - String url = String.format(SPARK_APPS_URL, shs, limitCount, DateUtil.getDay(-1)); - log.info("sparkUrl:{}", url); - ResponseEntity responseEntity = null; - try { - responseEntity = restTemplate.getForEntity(url, String.class); - } catch (RestClientException e) { - log.error("sparkRequestErr:{},{}", shs, e.getMessage()); - return null; - } - if (responseEntity.getBody() == null) { - log.error("sparkRequestErr:{}", shs); - return null; - } - List value; - try { - value = objectMapper.readValue(responseEntity.getBody(), - TypeFactory.defaultInstance().constructCollectionType(List.class, SparkApplication.class)); - } catch (JsonProcessingException e) { - log.error("sparkRequestErr:{},{}", shs, e.getMessage()); - return null; - } - return value; - } - - /** - * Obtaining SparkHistoryServer Event log directory: hdfs://ip:port/spark/ - */ - public String getEventLogDirectory(String ip) throws Exception { - String key = Constant.SPARK_EVENT_LOG_DIRECTORY + ip; - String cacheResult = (String) redisService.get(key); - if (StringUtils.isNotBlank(cacheResult)) { - log.info("getEventLogDirectoryFromCache:{},{}", key, cacheResult); - return cacheResult; - } - ResponseEntity responseEntity; - try { - responseEntity = restTemplate.getForEntity(String.format(SPARK_HOME_URL, ip), - String.class); - } catch (Exception e) { - log.error("getEventLogDirectoryErr:{},{}", ip, e.getMessage()); - return ""; - } - if (responseEntity.getBody() != null) { - Matcher m = hdfsPattern.matcher(responseEntity.getBody()); - if (m.matches()) { - String path = m.group("hdfs"); - if (StringUtils.isNotBlank(path)) { - log.info("cacheEventLogDirectory:{},{}", key, path); - redisService.set(key, path, Constant.DAY_SECONDS); - } - return path; - } - } - return ""; - } - -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/service/impl/YarnMetaServiceImpl.java b/task-metadata/src/main/java/com/oppo/cloud/meta/service/impl/YarnMetaServiceImpl.java deleted file mode 100644 index 066ba5927..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/service/impl/YarnMetaServiceImpl.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.service.impl; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.oppo.cloud.common.constant.Constant; -import com.oppo.cloud.common.domain.cluster.yarn.YarnApp; -import com.oppo.cloud.common.domain.cluster.yarn.YarnResponse; -import com.oppo.cloud.common.util.DateUtil; -import com.oppo.cloud.common.util.opensearch.BulkApi; -import com.oppo.cloud.meta.config.HadoopConfig; -import com.oppo.cloud.meta.service.IClusterConfigService; -import com.oppo.cloud.meta.service.ITaskSyncerMetaService; -import lombok.extern.slf4j.Slf4j; -import org.opensearch.action.bulk.BulkItemResponse; -import org.opensearch.action.bulk.BulkResponse; -import org.opensearch.client.RestHighLevelClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestClientException; -import org.springframework.web.client.RestTemplate; - -import javax.annotation.Resource; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; - -/** - * Synchronization of YARN Application Metadata - */ -@Slf4j -@Service("YarnMetaServiceImpl") -public class YarnMetaServiceImpl implements ITaskSyncerMetaService { - - @Resource - private HadoopConfig config; - - @Value("${scheduler.yarnMeta.startedTimeBegin}") - private long startedTimeBegin; - - @Value("${spring.opensearch.yarn-app-prefix}") - private String yarnAppPrefix; - - @Resource - private Executor yarnMetaExecutor; - - @Resource(name = "restTemplate") - private RestTemplate restTemplate; - - @Resource - private IClusterConfigService iClusterConfigService; - - @Resource - private ObjectMapper objectMapper; - - @Resource - private RestHighLevelClient client; - /** - * Specifying a Start Time Timestamp - */ - private static final String YARN_APPS_URL = "http://%s/ws/v1/cluster/apps?startedTimeBegin=%d"; - - /** - * Cluster Concurrent Synchronization - */ - @Override - public void syncer() { - Map yarnClusters = iClusterConfigService.getYarnClusters(); - log.info("yarnClusters:{}", yarnClusters); - if (yarnClusters == null || yarnClusters.size() == 0) { - log.error("yarnClusters empty"); - return; - } - - CompletableFuture[] array = new CompletableFuture[yarnClusters.size()]; - int i = 0; - for (Map.Entry yarnCluster : yarnClusters.entrySet()) { - array[i] = CompletableFuture.supplyAsync(() -> { - try { - pull(yarnCluster.getKey(), yarnCluster.getValue()); - } catch (Exception e) { - log.error("Exception: ", e); - } - return null; - }, yarnMetaExecutor); - i++; - } - - try { - CompletableFuture.allOf(array).get(); - } catch (Exception e) { - log.error("Exception: ", e); - } - } - - /** - * Pulling Cluster Application Metadata - */ - public void pull(String ip, String clusterName) { - log.info("start to pull yarn tasks:{}", ip); - List apps = yarnRequest(ip); - if (apps == null) { - log.error("yarnMetaErr:appsNull:{}", ip); - return; - } - - Map> yarnAppMap = new HashMap<>(); - for (YarnApp app : apps) { - String id = ip + "_" + app.getId(); - app.setCreateTime(System.currentTimeMillis()); - app.setIp(ip); - app.setClusterName(clusterName); - yarnAppMap.put(id, app.getYarnAppMap()); - log.info("yarnApp-->{},{},{},{}", ip, app.getId(), app.getFinishedTime(), app.getFinalStatus()); - } - BulkResponse response; - try { - response = BulkApi.bulkByIds(client, yarnAppPrefix + DateUtil.getDay(0), yarnAppMap); - } catch (IOException e) { - log.error("bulkYarnAppsErr:{}", e.getMessage()); - return; - } - BulkItemResponse[] responses = response.getItems(); - - for (BulkItemResponse r : responses) { - if (r.isFailed()) { - log.error("failedInsertApp:{},{}", r.getId(), r.status()); - } - } - - log.info("saveYarnAppCount:{},{}", ip, yarnAppMap.size()); - } - - /** - * Obtaining YARN Jobs - */ - public List yarnRequest(String ip) { - long begin = System.currentTimeMillis() - startedTimeBegin * Constant.HOUR_MS; - String url = String.format(YARN_APPS_URL, ip, begin); - log.info("yarnUrl:{}", url); - ResponseEntity responseEntity; - try { - responseEntity = restTemplate.getForEntity(url, String.class); - } catch (RestClientException e) { - log.error("yarnRequestErr:{},{}", ip, e.getMessage()); - return null; - } - if (responseEntity.getBody() == null) { - log.error("yarnRequestErr:{}", ip); - return null; - } - YarnResponse value; - try { - value = objectMapper.readValue(responseEntity.getBody(), YarnResponse.class); - } catch (JsonProcessingException e) { - log.error(e.getMessage()); - return null; - } - if (value == null || value.getApps() == null || value.getApps().getApp() == null - || value.getApps().getApp().size() == 0) { - log.error("yarnRequestErr:null"); - return null; - } - return value.getApps().getApp(); - } - -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/utils/MatcherUtil.java b/task-metadata/src/main/java/com/oppo/cloud/meta/utils/MatcherUtil.java deleted file mode 100644 index 5789bc186..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/utils/MatcherUtil.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.utils; - -import lombok.extern.slf4j.Slf4j; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Matching Utility - */ -@Slf4j -public class MatcherUtil { - - /** - * Obtaining Data for a single Grouping within the Group - */ - public static String getGroupData(String content, Pattern pattern, String name) { - Matcher matcher = pattern.matcher(content); - if (!matcher.find()) { - log.error("matcher not found {} :", name); - return ""; - } - String data; - try { - data = matcher.group(name); - } catch (Exception e) { - log.error(e.getMessage()); - return ""; - } - return data; - } -} diff --git a/task-metadata/src/main/java/com/oppo/cloud/meta/utils/RestTemplateUtil.java b/task-metadata/src/main/java/com/oppo/cloud/meta/utils/RestTemplateUtil.java deleted file mode 100644 index 65dceaad5..000000000 --- a/task-metadata/src/main/java/com/oppo/cloud/meta/utils/RestTemplateUtil.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.utils; - -import org.springframework.context.annotation.Bean; -import org.springframework.http.client.SimpleClientHttpRequestFactory; -import org.springframework.stereotype.Component; -import org.springframework.web.client.RestTemplate; - -/** - * HTTP Client - */ -@Component -public class RestTemplateUtil { - - @Bean("restTemplate") - public RestTemplate restTemplate() { - SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); - requestFactory.setConnectTimeout(15000); - requestFactory.setReadTimeout(30000); - RestTemplate restTemplate = new RestTemplate(); - restTemplate.setRequestFactory(requestFactory); - return restTemplate; - } -} diff --git a/task-metadata/src/main/resources/application-hadoop.yml b/task-metadata/src/main/resources/application-hadoop.yml deleted file mode 100644 index cf80a4167..000000000 --- a/task-metadata/src/main/resources/application-hadoop.yml +++ /dev/null @@ -1,17 +0,0 @@ -hadoop: - namenodes: - - nameservices: logs-hdfs - namenodesAddr: [ "host1", "host2" ] - namenodes: [ "namenode1", "namenode2" ] - user: hdfs - password: - port: 8020 - matchPathKeys: [ "flume" ] - - yarn: - - clusterName: "bigdata" - resourceManager: [ "localhost:8088" ] - jobHistoryServer: "localhost:19888" - - spark: - sparkHistoryServer: [ "localhost:18018" ] \ No newline at end of file diff --git a/task-metadata/src/main/resources/application.yml b/task-metadata/src/main/resources/application.yml deleted file mode 100644 index f4274f53d..000000000 --- a/task-metadata/src/main/resources/application.yml +++ /dev/null @@ -1,37 +0,0 @@ -server: - port: 7072 - -spring: - profiles: - active: hadoop - task: - scheduling: - pool: - size: 10 - redis: - cluster: - nodes: localhost:6379 - password: - - opensearch: - nodes: localhost:19527 - username: - password: - truststore: - truststore-password: - yarn-app-prefix: "compass-yarn-app-" - spark-app-prefix: "compass-spark-app-" - -scheduler: - clusterMeta: - enable: true - cron: "0 */30 * * * ?" - yarnMeta: - enable: true - cron: "0/30 * * * * ?" - # hour - startedTimeBegin: 6 - sparkMeta: - enable: true - cron: "0/30 * * * * ?" - limitCount: 1000 diff --git a/task-metadata/src/main/resources/logback.xml b/task-metadata/src/main/resources/logback.xml deleted file mode 100644 index 21ed0c9a0..000000000 --- a/task-metadata/src/main/resources/logback.xml +++ /dev/null @@ -1,105 +0,0 @@ - - - - - logback - - - - - - - info - - - ${LOG_PATTERN_COLOR} - - - - - - logs/task-metadata.info.%d{yyyy-MM-dd}.%i.log - 7 - - 500MB - - 2GB - - - INFO - ACCEPT - DENY - - - ${LOG_PATTERN} - - - - - - logs/task-metadata.error.%d{yyyy-MM-dd}.%i.log - 7 - - 500MB - - 2GB - - - ERROR - ACCEPT - DENY - - - ${LOG_PATTERN} - - - - - - logs/task-metadata.warn.%d{yyyy-MM-dd}.%i.log - 7 - - 500MB - - 2GB - - - WARN - ACCEPT - DENY - - - ${LOG_PATTERN} - - - - - - logs/task-metadata.debug.%d{yyyy-MM-dd}.%i.log - 7 - - 500MB - - 2GB - - - debug - ACCEPT - DENY - - - ${LOG_PATTERN} - - - - - - - - - - - - - - \ No newline at end of file diff --git a/task-metadata/src/main/resources/scripts/releaseLock.lua b/task-metadata/src/main/resources/scripts/releaseLock.lua deleted file mode 100644 index 904d7f648..000000000 --- a/task-metadata/src/main/resources/scripts/releaseLock.lua +++ /dev/null @@ -1,6 +0,0 @@ -redis.replicate_commands() -if redis.call("GET", KEYS[1]) == ARGV[1] then - return tostring(redis.call("DEL", KEYS[1])) -else - return nil -end \ No newline at end of file diff --git a/task-metadata/src/test/java/com/oppo/cloud/meta/service/ITaskSyncerMetaServiceTest.java b/task-metadata/src/test/java/com/oppo/cloud/meta/service/ITaskSyncerMetaServiceTest.java deleted file mode 100644 index 1e75ecf1d..000000000 --- a/task-metadata/src/test/java/com/oppo/cloud/meta/service/ITaskSyncerMetaServiceTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.service; - -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; - -import javax.annotation.Resource; - -@Slf4j -@SpringBootTest -class ITaskSyncerMetaServiceTest { - - @Resource(name = "SparkMetaServiceImpl") - ITaskSyncerMetaService spark; - @Resource(name = "YarnMetaServiceImpl") - ITaskSyncerMetaService yarn; - - @Test - void spark() { - spark.syncer(); - } - - @Test - void yarn() { - yarn.syncer(); - } - -} diff --git a/task-metadata/src/test/java/com/oppo/cloud/meta/service/impl/ClusterMetaServiceImplTest.java b/task-metadata/src/test/java/com/oppo/cloud/meta/service/impl/ClusterMetaServiceImplTest.java deleted file mode 100644 index 03281274f..000000000 --- a/task-metadata/src/test/java/com/oppo/cloud/meta/service/impl/ClusterMetaServiceImplTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.meta.service.impl; - -import com.oppo.cloud.common.constant.Constant; -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.meta.service.IClusterConfigService; -import com.oppo.cloud.test.redis.WithRedisServer; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; - -import javax.annotation.Resource; - -@Slf4j -@SpringBootTest -class ClusterMetaServiceImplTest implements WithRedisServer { - - @Resource - RedisService redisService; - - @Resource - IClusterConfigService clusterMetaService; - - - @Test - void getCache() { - clusterMetaService.updateClusterConfig(); - String sparkHistoryServer = (String) redisService.get(Constant.SPARK_HISTORY_SERVERS); - log.info("sparkHistoryServer:{}", sparkHistoryServer); - String yarnClusterInfo = (String) redisService.get(Constant.YARN_CLUSTERS); - log.info("yarnClusterInfo:{}", yarnClusterInfo); - String rmJhsMap = (String) redisService.get(Constant.RM_JHS_MAP); - log.info("rmJhsMap:{}", rmJhsMap); - } - -} diff --git a/task-metadata/src/test/resources/application.yml b/task-metadata/src/test/resources/application.yml deleted file mode 100644 index 94fdb3b0e..000000000 --- a/task-metadata/src/test/resources/application.yml +++ /dev/null @@ -1,36 +0,0 @@ -server: - port: 7072 - -spring: - profiles: - active: hadoop - task: - scheduling: - pool: - size: 10 - redis: - host: localhost - port: 6379 - - opensearch: - nodes: localhost:19527 - username: - password: - truststore: - truststore-password: - yarn-app-prefix: "compass-yarn-app-" - spark-app-prefix: "compass-spark-app-" - -scheduler: - clusterMeta: - enable: true - cron: "0 */30 * * * ?" - yarnMeta: - enable: true - cron: "0/30 * * * * ?" - # hour - startedTimeBegin: 6 - sparkMeta: - enable: true - cron: "0/30 * * * * ?" - limitCount: 1000 diff --git a/task-parser/src/assembly/assembly.xml b/task-parser/src/assembly/assembly.xml deleted file mode 100644 index 8a45d5775..000000000 --- a/task-parser/src/assembly/assembly.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - release - - dir - - false - ${project.artifactId} - - - ${project.basedir}/src/bin - bin - 0755 - - - ${project.basedir}/src/main/resources - - **/scripts/** - - conf - - - - - lib - - - \ No newline at end of file diff --git a/task-parser/src/bin/startup.sh b/task-parser/src/bin/startup.sh deleted file mode 100644 index 83af62024..000000000 --- a/task-parser/src/bin/startup.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh - -HOME_DIR=$(cd $(dirname $0)/.. && pwd) -APP_NAME="task-parser" -PID_FILE=${HOME_DIR}/tpid - -if [ -f $(dirname $0)/compass_env.sh ]; then - source $(dirname $0)/compass_env.sh -fi - -check() { - if [ -f ${PID_FILE} ]; then - local pid=$(cat ${PID_FILE}) - echo $pid - command="ps --pid ${pid}" - if [[ $(uname) == "Darwin" ]]; then - command="ps -p ${pid}" - fi - if eval ${command} >/dev/null; then - echo "${APP_NAME} is running, pid=${pid}. Please stop first!" - exit 1 - fi - fi -} - - -start() { - check - - JAVA_OPTS="-DappName=${APP_NAME} -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom" - JAVA_OPTS_GC="-server -XX:+UseG1GC -XX:G1HeapRegionSize=8m -verbose:GC -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${HOME_DIR}/logs/dump.hprof -Xloggc:${HOME_DIR}/logs/gc-${APP_NAME}.log" - - nohup java $JAVA_OPTS $JAVA_OPTS_GC -cp "${HOME_DIR}/conf":"${HOME_DIR}/lib/*" com.oppo.cloud.parser.TaskParserApplication >/dev/null 2>&1 & - - pid=$! - echo $pid - echo $pid >$PID_FILE -} - -start \ No newline at end of file diff --git a/task-parser/src/bin/stop.sh b/task-parser/src/bin/stop.sh deleted file mode 100644 index 0887ab2f2..000000000 --- a/task-parser/src/bin/stop.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh - -HOME_DIR=$(cd $(dirname $0)/.. && pwd) -APP_NAME="task-parser" -PID_FILE=${HOME_DIR}/tpid - -stop() { - if [ -f ${PID_FILE} ]; then - local pid=$(cat ${PID_FILE}) - echo $APP_NAME $pid - command="ps --pid ${pid}" - if [[ $(uname) == "Darwin" ]]; then - command="ps -p ${pid}" - fi - if eval ${command} >/dev/null; then - kill $pid && rm -f $PID_FILE - fi - sleep 5 - - if eval ${command} >/dev/null; then - kill -9 $pid - fi - - else - ps -ef | grep $APP_NAME | grep -v grep | awk '{print $2}' | xargs kill -9 - fi - rm -f $PID_FILE -} - -stop \ No newline at end of file diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/TaskParserApplication.java b/task-parser/src/main/java/com/oppo/cloud/parser/TaskParserApplication.java deleted file mode 100644 index 58e0960c3..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/TaskParserApplication.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.scheduling.annotation.EnableAsync; -import org.springframework.scheduling.annotation.EnableScheduling; - -@ComponentScan(basePackages = "com.oppo.cloud.*") -@EnableAsync -@EnableScheduling -@SpringBootApplication -public class TaskParserApplication { - - public static void main(String[] args) { - SpringApplication.run(TaskParserApplication.class, args); - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/config/CustomConfig.java b/task-parser/src/main/java/com/oppo/cloud/parser/config/CustomConfig.java deleted file mode 100644 index 449c0bf7e..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/config/CustomConfig.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.config; - -import com.oppo.cloud.common.domain.eventlog.config.DetectorConfig; -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.util.List; - -/** - * custom yml config - */ -@Configuration -@ConfigurationProperties(prefix = "custom") -@Data -public class CustomConfig { - - /** - * opensearch index prefix - */ - @Value("${spring.opensearch.log-prefix}") - private String logSummaryPrefix; - - @Value("${spring.opensearch.detector-prefix}") - private String detectorPrefix; - - @Value("${spring.opensearch.gc-prefix}") - private String gcPrefix; - - @Value("${spring.opensearch.job-prefix}") - private String jobPrefix; - - @Value("${spring.opensearch.task-app-prefix}") - private String taskAppPrefix; - - @Value("${custom.redisConsumer.enable}") - private Boolean enableRedisConsumer; - - @Value("${custom.redisConsumer.maxThreadPoolSize}") - private Integer maxThreadPoolSize; - /** - * logRecord list - */ - @Value("${custom.redisConsumer.logRecordList}") - private String logRecordList; - - /** - * logRecord processing - */ - @Value("${custom.redisConsumer.processingHash}") - private String processingHash; - - - private List jvmType; - - public static final String GC_CONFIG = "gcConfig"; - - @Bean(name = GC_CONFIG) - public List loadGCConfig() { - return jvmType; - } - - @Bean - @ConfigurationProperties(prefix = "custom.detector") - public DetectorConfig eventLogDetectorConfig() { - return new DetectorConfig(); - } - - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/config/DiagnosisConfig.java b/task-parser/src/main/java/com/oppo/cloud/parser/config/DiagnosisConfig.java deleted file mode 100644 index 7bb7ec6b2..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/config/DiagnosisConfig.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.config; - -import com.alibaba.fastjson2.JSON; -import com.oppo.cloud.common.domain.eventlog.config.DetectorConfig; -import com.oppo.cloud.common.util.textparser.ParserAction; -import com.oppo.cloud.common.util.textparser.ParserActionUtil; -import lombok.extern.slf4j.Slf4j; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Diagnostic rule configuration - */ -@Slf4j -public class DiagnosisConfig { - - /** - * configuration instance - */ - private static final DiagnosisConfig INSTANCE = new DiagnosisConfig(); - - /** - * text type diagnostic rule - */ - public Map ruleConfig = new HashMap<>(); - - /** - * spark eventLog diagnostic configuration - */ - public DetectorConfig detectorConfig; - - - public static DiagnosisConfig getInstance() { - return INSTANCE; - } - - public void setRuleMap(Map ruleConfig) { - this.ruleConfig = ruleConfig; - } - - public void setDetectorConfig(DetectorConfig detectorConfig) { - this.detectorConfig = detectorConfig; - } - - public DiagnosisConfig() { - - } - - public List getActions(String category) { - List actions = new ArrayList<>(); - try { - actions = JSON.parseArray(this.ruleConfig.get(category),ParserAction.class); - } catch (Exception e) { - log.error(e.getMessage()); - } - actions = ParserActionUtil.verifyParserActions(actions); - return actions; - } - - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/config/HadoopConfig.java b/task-parser/src/main/java/com/oppo/cloud/parser/config/HadoopConfig.java deleted file mode 100644 index 9ef993ffb..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/config/HadoopConfig.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.config; - - -import com.oppo.cloud.common.domain.cluster.hadoop.NameNodeConf; -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * hadoop hdfs NameNode config - */ -@Configuration -@ConfigurationProperties(prefix = "hadoop") -@Data -@EnableConfigurationProperties(HadoopConfig.class) -public class HadoopConfig { - - private List namenodes; - - public static final String NAME_NODE_MAP = "nameNodeMap"; - - @Bean(name = NAME_NODE_MAP) - public Map loadNameNodeMap() { - Map nameNodeMap = new HashMap<>(); - for (NameNodeConf nameNode : namenodes) { - nameNodeMap.put(nameNode.getNameservices(), nameNode); - } - return nameNodeMap; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/config/KafkaConfig.java b/task-parser/src/main/java/com/oppo/cloud/parser/config/KafkaConfig.java deleted file mode 100644 index 342e62c2c..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/config/KafkaConfig.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.config; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.core.KafkaTemplate; - -import javax.annotation.Resource; -import java.util.List; -import java.util.concurrent.TimeUnit; - - -/** - * Kafka configuration - */ -@Configuration -@Data -public class KafkaConfig { - - /** - * Kafka topic - */ - @Value("${spring.kafka.topic}") - private String topic; - - - @Resource - KafkaTemplate kafkaTemplate; - - public void sendMessages(List messages) throws Exception { - for (String msg : messages) { - kafkaTemplate.send(topic, msg).get(10, TimeUnit.SECONDS); - } - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/config/MyBatisConfig.java b/task-parser/src/main/java/com/oppo/cloud/parser/config/MyBatisConfig.java deleted file mode 100644 index 0159b44fe..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/config/MyBatisConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.config; - -import org.mybatis.spring.annotation.MapperScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.transaction.annotation.EnableTransactionManagement; - -/** - * MyBatis config - */ -@Configuration -@EnableTransactionManagement -@MapperScan({"com.oppo.cloud.mapper"}) -public class MyBatisConfig { -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/config/OpenSearchConfig.java b/task-parser/src/main/java/com/oppo/cloud/parser/config/OpenSearchConfig.java deleted file mode 100644 index 1abe4f792..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/config/OpenSearchConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.config; - -import com.oppo.cloud.common.util.opensearch.OpenSearchClient; -import lombok.extern.slf4j.Slf4j; -import org.opensearch.client.RestHighLevelClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; - -/** - * OpenSearch Config - */ -@Configuration -@Slf4j -public class OpenSearchConfig { - - @Value("${spring.opensearch.nodes}") - private String nodes; - @Value("${spring.opensearch.username}") - private String username; - @Value("${spring.opensearch.password}") - private String password; - @Value("${spring.opensearch.truststore}") - private String truststore; - @Value("${spring.opensearch.truststore-password}") - private String truststorePassword; - - - public final static String SEARCH_CLIENT = "searchClient"; - - @Primary - @Bean(SEARCH_CLIENT) - public RestHighLevelClient restHighLevelClient() { - return OpenSearchClient.create(nodes, username, password, truststore, truststorePassword); - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/config/RedisConfig.java b/task-parser/src/main/java/com/oppo/cloud/parser/config/RedisConfig.java deleted file mode 100644 index 43233d9f8..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/config/RedisConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.config; - -import com.oppo.cloud.common.config.RedisTemplateConfig; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.Resource; -import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.data.redis.core.script.RedisScript; -import org.springframework.stereotype.Component; - -import java.util.List; - -/** - * redis config - */ -@Component -public class RedisConfig extends RedisTemplateConfig { - - @Bean - public RedisScript logRecordConsumerScript() { - Resource scriptSource = new ClassPathResource("scripts/logRecordConsumer.lua"); - return RedisScript.of(scriptSource, Object.class); - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/config/ThreadPoolConfig.java b/task-parser/src/main/java/com/oppo/cloud/parser/config/ThreadPoolConfig.java deleted file mode 100644 index e18151251..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/config/ThreadPoolConfig.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.config; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; - -import javax.annotation.Resource; -import java.util.concurrent.Executor; -import java.util.concurrent.ThreadPoolExecutor; - -/** - * Executor pool config - */ -@Configuration -public class ThreadPoolConfig { - - public static final String JOB_THREAD_POOL = "jobThreadPool"; - - public static final String TASK_THREAD_POOL = "taskThreadPool"; - - public static final String DETECTOR_THREAD_POOL = "detectorThreadPool"; - - public static final String PARSER_THREAD_POOL = "parserThreadPool"; - - public static final String REDIS_CONSUMER_THREAD_POOL = "redisConsumerThreadPool"; - - @Resource - private CustomConfig config; - - @Bean(name = JOB_THREAD_POOL) - public Executor jobThreadPool() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(config.getMaxThreadPoolSize()); - executor.setMaxPoolSize(config.getMaxThreadPoolSize()); - executor.setKeepAliveSeconds(120); - executor.setQueueCapacity(10000); - executor.setThreadNamePrefix("job-thread-"); - executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); - executor.setWaitForTasksToCompleteOnShutdown(true); - executor.setAwaitTerminationSeconds(60); - executor.initialize(); - return executor; - } - - @Bean(name = TASK_THREAD_POOL) - public Executor taskThreadPool() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(config.getMaxThreadPoolSize()); - executor.setMaxPoolSize(config.getMaxThreadPoolSize()); - executor.setKeepAliveSeconds(120); - executor.setQueueCapacity(10000); - executor.setThreadNamePrefix("task-thread-"); - executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); - executor.setWaitForTasksToCompleteOnShutdown(true); - executor.setAwaitTerminationSeconds(60); - executor.initialize(); - return executor; - } - - @Bean(name = PARSER_THREAD_POOL) - public Executor parserThreadPool() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(config.getMaxThreadPoolSize()); - executor.setMaxPoolSize(config.getMaxThreadPoolSize()); - executor.setKeepAliveSeconds(120); - executor.setQueueCapacity(10000); - executor.setThreadNamePrefix("parser-thread-"); - executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); - executor.setWaitForTasksToCompleteOnShutdown(true); - executor.setAwaitTerminationSeconds(60); - executor.initialize(); - return executor; - } - - @Bean(name = DETECTOR_THREAD_POOL) - public Executor detectorThreadPool() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(2); - executor.setMaxPoolSize(4); - executor.setKeepAliveSeconds(120); - executor.setQueueCapacity(10000); - executor.setThreadNamePrefix("detector-thread-"); - executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); - executor.setWaitForTasksToCompleteOnShutdown(true); - executor.setAwaitTerminationSeconds(60); - executor.initialize(); - return executor; - } - - - @Bean(name = REDIS_CONSUMER_THREAD_POOL) - public Executor redisConsumerThreadPool() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(config.getMaxThreadPoolSize()); - executor.setMaxPoolSize(config.getMaxThreadPoolSize()); - executor.setKeepAliveSeconds(120); - executor.setQueueCapacity(10000); - executor.setThreadNamePrefix("redis-thread-"); - executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); - executor.setWaitForTasksToCompleteOnShutdown(true); - executor.setAwaitTerminationSeconds(60); - executor.initialize(); - return executor; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/CommonResult.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/CommonResult.java deleted file mode 100644 index ac5a463be..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/CommonResult.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.job; - -import com.oppo.cloud.common.constant.LogType; -import lombok.Data; - -@Data -public class CommonResult { - - private LogType logType; - - private T result; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/DetectorParam.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/DetectorParam.java deleted file mode 100644 index 1eb1787b9..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/DetectorParam.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.job; - -import com.oppo.cloud.common.constant.ApplicationType; -import com.oppo.cloud.common.domain.eventlog.config.DetectorConfig; -import com.oppo.cloud.parser.domain.mr.MRAppInfo; -import com.oppo.cloud.parser.utils.ReplaySparkEventLogs; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.Date; - -@Data -@NoArgsConstructor -public class DetectorParam { - - /** - * flow name - */ - private String flowName; - - /** - * project name - */ - private String projectName; - - /** - * task name - */ - private String taskName; - - /** - * execution time - */ - private Date executionTime; - - /** - * try number - */ - private Integer tryNumber; - - /** - * applicationId - */ - private String appId; - - /** - * application type - */ - private ApplicationType appType; - - /** - * app duration(ms) - */ - private long appDuration; - - private String logPath; - - private DetectorConfig config; - - private boolean isOneClick; - - private ReplaySparkEventLogs replayEventLogs; - - private MRAppInfo mrAppInfo; - - public DetectorParam(String flowName, String projectName, String taskName, Date executionTime, Integer tryNumber, - String appId, ApplicationType appType, long appDuration, String logPath, DetectorConfig config, - boolean isOneClick) { - this.flowName = flowName; - this.projectName = projectName; - this.taskName = taskName; - this.executionTime = executionTime; - this.tryNumber = tryNumber; - this.appId = appId; - this.appType = appType; - this.appDuration = appDuration; - this.logPath = logPath; - this.config = config; - this.isOneClick = isOneClick; - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/ExecutorTimeSpan.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/ExecutorTimeSpan.java deleted file mode 100644 index 0a6f48df6..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/ExecutorTimeSpan.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.job; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class ExecutorTimeSpan { - - private Long startTime; - private Long endTime; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/MemoryCalculateParam.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/MemoryCalculateParam.java deleted file mode 100644 index 625bb155e..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/MemoryCalculateParam.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.job; - -import lombok.Data; - -import java.util.Map; - -@Data -public class MemoryCalculateParam { - - private Long appTotalTime; - private Long driverMemory; - private Long executorMemory; - private Map executorRuntimeMap; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/ParserParam.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/ParserParam.java deleted file mode 100644 index b879c8cf1..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/ParserParam.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.job; - -import com.oppo.cloud.common.constant.LogType; -import com.oppo.cloud.common.domain.job.App; -import com.oppo.cloud.common.domain.job.LogPath; -import com.oppo.cloud.common.domain.job.LogRecord; -import com.oppo.cloud.common.util.textparser.ParserAction; -import lombok.Data; - -import java.util.List; - -@Data -public class ParserParam { - - private LogType logType; - private LogRecord logRecord; - private App app; - private List logPaths; - - public ParserParam() { - - } - - public ParserParam(String logType, LogRecord logRecord, App app, List logPaths) { - this.logType = LogType.get(logType); - this.logRecord = logRecord; - this.app = app; - this.logPaths = logPaths; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/SparkEventLogParserResult.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/SparkEventLogParserResult.java deleted file mode 100644 index c0e5ad22d..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/SparkEventLogParserResult.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.job; - -import com.oppo.cloud.common.domain.eventlog.DetectorStorage; -import lombok.Data; - -@Data -public class SparkEventLogParserResult { - - private DetectorStorage detectorStorage; - private MemoryCalculateParam memoryCalculateParam; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/SparkExecutorLogParserResult.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/SparkExecutorLogParserResult.java deleted file mode 100644 index 5f3a23609..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/SparkExecutorLogParserResult.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.job; - -import com.oppo.cloud.common.domain.gc.GCReport; -import com.oppo.cloud.common.util.textparser.ParserAction; -import lombok.Data; - -import java.util.List; -import java.util.Map; - - -@Data -public class SparkExecutorLogParserResult { - - /** - * log path - */ - private String logPath; - /** - * parser result - */ - private Map actionMap; - /** - * gc reports - */ - private List gcReports; - /** - * abnormal categories - */ - private List categories; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/StageDetectionInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/StageDetectionInfo.java deleted file mode 100644 index d1d37b78d..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/StageDetectionInfo.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.job; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class StageDetectionInfo { - - private Integer id; - private Integer attemptId; - /** - * The submission time for a stage is misleading because it counts the time. - * the stage waits to be launched. (SPARK-10930) - * refer to https://github.com/apache/spark/pull/9051 - */ - private Long firstLaunchTime; - private Long submissionTime; - private Long completeTime; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/TaskDetectionInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/TaskDetectionInfo.java deleted file mode 100644 index 7b43f9d63..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/TaskDetectionInfo.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.job; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class TaskDetectionInfo { - - private Integer jobId; - private Integer stageId; - private Integer attemptId; - private Long taskId; - private Long firstLaunchTime; - private Long finishTime; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/TaskParam.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/TaskParam.java deleted file mode 100644 index 5e53c69c8..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/TaskParam.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.job; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.oppo.cloud.common.constant.LogGroupType; -import com.oppo.cloud.common.domain.job.App; -import com.oppo.cloud.common.domain.job.LogInfo; -import com.oppo.cloud.common.domain.job.LogRecord; -import com.oppo.cloud.common.domain.opensearch.TaskApp; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -/** - * Task Param Information - */ -@Slf4j -@Data -public class TaskParam { - - /** - * log group type - */ - private LogGroupType category; - /** - * LogRecord information - */ - private LogRecord logRecord; - /** - * app information - */ - private App app; - /** - * log information - */ - private LogInfo logInfo; - - /** - * task retry number - */ - private int retry; - - private ObjectMapper objectMapper = new ObjectMapper(); - - public TaskParam() { - - } - - public TaskParam(String category, LogRecord logRecord, App app, LogInfo logInfo) { - this.category = LogGroupType.get(category); - this.logRecord = logRecord; - this.app = app; - this.logInfo = logInfo; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/TaskResult.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/TaskResult.java deleted file mode 100644 index 40a7fd311..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/job/TaskResult.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.job; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -/** - * Task abnormal result - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class TaskResult { - - private String appId; - - private List categories; - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/CounterInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/CounterInfo.java deleted file mode 100644 index ab5e0913a..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/CounterInfo.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr; - -public class CounterInfo { - - public enum MRConfiguration { - /** - * map cpu vcores - */ - MAP_CPU_VCORES("mapreduce.map.cpu.vcores"), - MAP_MEMORY_MB("mapreduce.map.memory.mb"), - REDUCE_CPU_VCORES("mapreduce.reduce.cpu.vcores"), - REDUCE_MEMORY_MB("mapreduce.reduce.memory.mb"), - YARN_INCREMENT_VCORES("yarn.scheduler.increment-allocation-vcores"), - YARN_INCREMENT_MEMORY_MB("yarn.scheduler.increment-allocation-mb"), - HIVE_QUERY_STRING("hive.query.string"); - - private final String key; - - MRConfiguration(String key) { - this.key = key; - } - - public String getKey() { - return key; - } - - } - - public enum CounterGroupName { - /** - * File System Counters - */ - FILE_SYSTEM_COUNTER("org.apache.hadoop.mapreduce.FileSystemCounter"), - JOB_COUNTER("org.apache.hadoop.mapreduce.JobCounter"), - TASK_COUNTER("org.apache.hadoop.mapreduce.TaskCounter"), - HIVE("HIVE"), - SHUFFLE_ERRORS("Shuffle Errors"), - FILE_INPUT_FORMAT_COUNTER("org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter"), - FILE_OUTPUT_FORMAT_COUNTER("org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter"); - - private final String counterGroupName; - - CounterGroupName(String counterGroupName) { - this.counterGroupName = counterGroupName; - } - - public String getCounterGroupName() { - return counterGroupName; - } - - } - - public enum CounterName { - /** - * FileSystemCounter - */ - FILE_LARGE_READ_OPS("FILE_LARGE_READ_OPS"), - FILE_WRITE_OPS("FILE_WRITE_OPS"), - HDFS_READ_OPS("HDFS_READ_OPS"), - HDFS_BYTES_READ("HDFS_BYTES_READ"), - HDFS_LARGE_READ_OPS("HDFS_LARGE_READ_OPS"), - FILE_READ_OPS("FILE_READ_OPS"), - FILE_BYTES_WRITTEN("FILE_BYTES_WRITTEN"), - FILE_BYTES_READ("FILE_BYTES_READ"), - HDFS_WRITE_OPS("HDFS_WRITE_OPS"), - HDFS_BYTES_WRITTEN("HDFS_BYTES_WRITTEN"), - - /** - * JobCounter - */ - TOTAL_LAUNCHED_MAPS("TOTAL_LAUNCHED_MAPS"), - VCORES_MILLIS_REDUCES("VCORES_MILLIS_REDUCES"), - TOTAL_LAUNCHED_REDUCES("TOTAL_LAUNCHED_REDUCES"), - NUM_KILLED_MAPS("NUM_KILLED_MAPS"), - OTHER_LOCAL_MAPS("OTHER_LOCAL_MAPS"), - NUM_KILLED_REDUCES("NUM_KILLED_REDUCES"), - DATA_LOCAL_MAPS("DATA_LOCAL_MAPS"), - MB_MILLIS_MAPS("MB_MILLIS_MAPS"), - SLOTS_MILLIS_REDUCES("SLOTS_MILLIS_REDUCES"), - VCORES_MILLIS_MAPS("VCORES_MILLIS_MAPS"), - MB_MILLIS_REDUCES("MB_MILLIS_REDUCES"), - SLOTS_MILLIS_MAPS("SLOTS_MILLIS_MAPS"), - RACK_LOCAL_MAPS("RACK_LOCAL_MAPS"), - MILLIS_REDUCES("MILLIS_REDUCES"), - MILLIS_MAPS("MILLIS_MAPS"), - - /** - * TaskCounter - */ - MAP_OUTPUT_MATERIALIZED_BYTES("MAP_OUTPUT_MATERIALIZED_BYTES"), - REDUCE_INPUT_RECORDS("REDUCE_INPUT_RECORDS"), - SPILLED_RECORDS("SPILLED_RECORDS"), - MERGED_MAP_OUTPUTS("MERGED_MAP_OUTPUTS"), - VIRTUAL_MEMORY_BYTES("VIRTUAL_MEMORY_BYTES"), - MAP_INPUT_RECORDS("MAP_INPUT_RECORDS"), - SPLIT_RAW_BYTES("SPLIT_RAW_BYTES"), - FAILED_SHUFFLE("FAILED_SHUFFLE"), - MAP_OUTPUT_BYTES("MAP_OUTPUT_BYTES"), - REDUCE_SHUFFLE_BYTES("REDUCE_SHUFFLE_BYTES"), - PHYSICAL_MEMORY_BYTES("PHYSICAL_MEMORY_BYTES"), - GC_TIME_MILLIS("GC_TIME_MILLIS"), - REDUCE_INPUT_GROUPS("REDUCE_INPUT_GROUPS"), - COMBINE_OUTPUT_RECORDS("COMBINE_OUTPUT_RECORDS"), - SHUFFLED_MAPS("SHUFFLED_MAPS"), - REDUCE_OUTPUT_RECORDS("REDUCE_OUTPUT_RECORDS"), - MAP_OUTPUT_RECORDS("MAP_OUTPUT_RECORDS"), - COMBINE_INPUT_RECORDS("COMBINE_INPUT_RECORDS"), - CPU_MILLISECONDS("CPU_MILLISECONDS"), - COMMITTED_HEAP_BYTES("COMMITTED_HEAP_BYTES"), - - - /** - * HIVE - */ - CREATED_FILES("CREATED_FILES"), - RECORDS_IN("RECORDS_IN"), - RECORDS_OUT_0("RECORDS_OUT_0"), - RECORDS_OUT_INTERMEDIATE("RECORDS_OUT_INTERMEDIATE"), - - /** - * Shuffle Errors - */ - CONNECTION("CONNECTION"), - WRONG_LENGTH("WRONG_LENGTH"), - BAD_ID("BAD_ID"), - WRONG_MAP("WRONG_MAP"), - WRONG_REDUCE("WRONG_REDUCE"), - IO_ERROR("IO_ERROR"), - - /** - * FileOutPutFormatCounter - */ - BYTES_WRITTEN("BYTES_WRITTEN"), - - /** - * FileInPutFormatCounter - */ - BYTES_READ("BYTES_READ"); - - private final String counterName; - - CounterName(String counterName) { - this.counterName = counterName; - } - - public String getCounterName() { - return counterName; - } - - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/JobHistoryFileInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/JobHistoryFileInfo.java deleted file mode 100644 index 329f70794..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/JobHistoryFileInfo.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr; - -import com.oppo.cloud.parser.domain.reader.ReaderObject; -import lombok.AllArgsConstructor; -import lombok.Data; - -@Data -@AllArgsConstructor -public class JobHistoryFileInfo { - private ReaderObject jobHistoryReader; - private ReaderObject confReader; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/MRAppInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/MRAppInfo.java deleted file mode 100644 index bf1248f15..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/MRAppInfo.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr; - -import com.oppo.cloud.parser.domain.mr.event.TaskInfo; -import lombok.Data; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Data -public class MRAppInfo { - private String jobId; - private String errorInfo; - private String username; - private String jobName; - private String jobQueueName; - private Long submitTime; - private Long launchTime; - private Integer totalMaps; - private Integer totalReduces; - private Integer failedMaps; - private Integer failedReduces; - private Integer succeededMaps; - private Integer succeededReduces; - private Integer killedMaps; - private Integer killedReduces; - private Long finishTime; - private Long elapsedTime; - private String jobStatus; - private Map confMap; - private Map tasksMap; - private Map> totalCounters; - private List mapList; - private List reduceList; - private SpeculationInfo speculationInfo; - - public MRAppInfo() { - this.tasksMap = new HashMap<>(); - this.totalCounters = new HashMap<>(); - this.mapList = new ArrayList<>(); - this.reduceList = new ArrayList<>(); - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/MRTaskAttemptInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/MRTaskAttemptInfo.java deleted file mode 100644 index cb5258059..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/MRTaskAttemptInfo.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr; - -import lombok.Data; - -import java.util.Map; - -@Data -public class MRTaskAttemptInfo { - - private int taskId; - - private String attemptId; - - private String taskStatus; - private long startTime; - - private long finishTime; - - private long shuffleFishTime; - - private long sortFinishTime; - - private long elapsedTime; - - private String error; - - private Map> counters; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/SpeculationInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/SpeculationInfo.java deleted file mode 100644 index e7b0994f4..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/SpeculationInfo.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr; - -import lombok.Data; - -import java.util.ArrayList; -import java.util.List; - -@Data -public class SpeculationInfo { - - private List elapsedTime; - - private List taskAttemptIds; - - public SpeculationInfo() { - this.elapsedTime = new ArrayList<>(); - this.taskAttemptIds = new ArrayList<>(); - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/AMStarted.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/AMStarted.java deleted file mode 100644 index 874a8be01..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/AMStarted.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class AMStarted { - private String applicationAttemptId; - private Long startTime; - private String containerId; - private String nodeManagerHost; - private Integer nodeManagerPort; - private Integer nodeManagerHttpPort; - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/Event.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/Event.java deleted file mode 100644 index 30c1b9e6b..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/Event.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class Event { - EventType type; - String event; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/EventType.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/EventType.java deleted file mode 100644 index db1da7897..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/EventType.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -public enum EventType { - /** - * MR event type - */ - JOB_SUBMITTED, - JOB_INITED, - JOB_FINISHED, - JOB_PRIORITY_CHANGED, - JOB_STATUS_CHANGED, - JOB_QUEUE_CHANGED, - JOB_FAILED, - JOB_KILLED, - JOB_ERROR, - JOB_INFO_CHANGED, - TASK_STARTED, - TASK_FINISHED, - TASK_FAILED, - TASK_UPDATED, - NORMALIZED_RESOURCE, - MAP_ATTEMPT_STARTED, - MAP_ATTEMPT_FINISHED, - MAP_ATTEMPT_FAILED, - MAP_ATTEMPT_KILLED, - REDUCE_ATTEMPT_STARTED, - REDUCE_ATTEMPT_FINISHED, - REDUCE_ATTEMPT_FAILED, - REDUCE_ATTEMPT_KILLED, - SETUP_ATTEMPT_STARTED, - SETUP_ATTEMPT_FINISHED, - SETUP_ATTEMPT_FAILED, - SETUP_ATTEMPT_KILLED, - CLEANUP_ATTEMPT_STARTED, - CLEANUP_ATTEMPT_FINISHED, - CLEANUP_ATTEMPT_FAILED, - CLEANUP_ATTEMPT_KILLED, - AM_STARTED -} \ No newline at end of file diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JhCounter.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JhCounter.java deleted file mode 100644 index 68fbf576c..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JhCounter.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class JhCounter { - private String name; - private String displayName; - private Long value; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JhCounterGroup.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JhCounterGroup.java deleted file mode 100644 index e23ee6e3a..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JhCounterGroup.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -import java.util.List; - -@Data -public class JhCounterGroup { - private String name; - private String displayName; - private List counts; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JhCounters.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JhCounters.java deleted file mode 100644 index 5cbb4165c..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JhCounters.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -import java.util.List; - -@Data -public class JhCounters { - private String name; - private List groups; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobFinished.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobFinished.java deleted file mode 100644 index b1227cbea..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobFinished.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class JobFinished { - private String jobid; - private Long finishTime; - private Integer finishedMaps; - private Integer finishedReduces; - private Integer failedMaps; - private Integer failedReduces; - private JhCounters totalCounters; - private JhCounters mapCounters; - private JhCounters reduceCounters; - private Integer killedMaps; - private Integer killedReduces; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobInfoChange.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobInfoChange.java deleted file mode 100644 index 09ef6e017..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobInfoChange.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class JobInfoChange { - private String jobid; - private Long submitTime; - private Long launchTime; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobInit.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobInit.java deleted file mode 100644 index acaf5e4d4..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobInit.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class JobInit { - private String jobid; - private Long launchTime; - private Integer totalMaps; - private Integer totalReduces; - private String jobStatus; - private Boolean uberized; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobPriorityChange.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobPriorityChange.java deleted file mode 100644 index b9fa1f69a..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobPriorityChange.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class JobPriorityChange { - private String jobid; - private String priority; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobQueueChange.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobQueueChange.java deleted file mode 100644 index 78e23bb67..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobQueueChange.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class JobQueueChange { - private String jobid; - private String jobQueueName; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobStatus.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobStatus.java deleted file mode 100644 index 8f818b5d1..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobStatus.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -public enum JobStatus { - RUNNING,SUCCEEDED,FAILED,PREP,KILLED -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobSubmitted.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobSubmitted.java deleted file mode 100644 index 9a099f200..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobSubmitted.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -import java.util.Map; - -@Data -public class JobSubmitted { - private String jobid; - private String jobName; - private String userName; - private Long submitTime; - private String jobConfPath; - private Map acls; - private String jobQueueName; - private String workflowId; - private String workflowName; - private String workflowNodeName; - private String workflowAdjacencies; - private String workflowTags; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobUnsuccessfulCompletion.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobUnsuccessfulCompletion.java deleted file mode 100644 index 2c4fd656f..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/JobUnsuccessfulCompletion.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class JobUnsuccessfulCompletion { - private String jobid; - private Long finishTime; - private Integer finishedMaps; - private Integer finishedReduces; - private String jobStatus; - private String diagnostics; - private Integer failedMaps; - private Integer failedReduces; - private Integer killedMaps; - private Integer killedReduces; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/MapAttemptFinished.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/MapAttemptFinished.java deleted file mode 100644 index 15cce0e35..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/MapAttemptFinished.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -import java.util.List; - -@Data -public class MapAttemptFinished { - private String taskid; - private String attemptId; - private String taskType; - private String taskStatus; - private Long mapFinishTime; - private Long finishTime; - private String hostname; - private Integer port; - private String rackname; - private String state; - private JhCounters counters; - private List clockSplits; - private List cpuUsages; - private List vMemKbytes; - private List physMemKbytes; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/ReduceAttemptFinished.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/ReduceAttemptFinished.java deleted file mode 100644 index 7b582b5b5..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/ReduceAttemptFinished.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -import java.util.List; - -@Data -public class ReduceAttemptFinished { - private String taskid; - private String attemptId; - private String taskType; - private String taskStatus; - private Long shuffleFinishTime; - private Long sortFinishTime; - private Long finishTime; - private String hostname; - private Integer port; - private String rackname; - private String state; - private JhCounters counters; - private List clockSplits; - private List cpuUsages; - private List vMemKbytes; - private List physMemKbytes; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskAttemptFinished.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskAttemptFinished.java deleted file mode 100644 index 37a055374..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskAttemptFinished.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class TaskAttemptFinished { - private String taskid; - private String attemptId; - private String taskType; - private String taskStatus; - private Long finishTime; - private String rackname; - private String hostname; - private String state; - private JhCounters jhCounters; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskAttemptInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskAttemptInfo.java deleted file mode 100644 index b58c6bca2..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskAttemptInfo.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class TaskAttemptInfo { - private String attemptId; - private Long startTime; - private Long finishTime; - private Long shuffleFinishTime; - private Long sortFinishTime; - private Long mapFinishTime; - private String error; - private String status; - private String state; - private String taskType; - private String trackerName; - private JhCounters counters; - private int httpPort; - private int shufflePort; - private String hostname; - private int port; - private String rackname; - private String containerId; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskAttemptStarted.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskAttemptStarted.java deleted file mode 100644 index 72a5d513d..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskAttemptStarted.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class TaskAttemptStarted { - private String taskid; - private String taskType; - private String attemptId; - private Long startTime; - private String trackerName; - private Integer httpPort; - private Integer shufflePort; - private String containerId; - private String locality; - private String avataar; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskAttemptUnsuccessfulCompletion.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskAttemptUnsuccessfulCompletion.java deleted file mode 100644 index e6c049a8f..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskAttemptUnsuccessfulCompletion.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -import java.util.List; - -@Data -public class TaskAttemptUnsuccessfulCompletion { - private String taskid; - private String taskType; - private String attemptId; - private Long finishTime; - private String hostname; - private Integer port; - private String rackname; - private String status; - private String error; - private JhCounters counters; - private List clockSplits; - private List cpuUsages; - private List vMemKbytes; - private List physMemKbytes; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskFailed.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskFailed.java deleted file mode 100644 index 5590fb543..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskFailed.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class TaskFailed { - private String taskid; - private String taskType; - private Long finishTime; - private String error; - private String failedDueToAttempt; - private String status; - private JhCounters counters; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskFinished.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskFinished.java deleted file mode 100644 index b2b8eef72..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskFinished.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class TaskFinished { - private String taskid; - private String taskType; - private Long finishTime; - private String status; - private JhCounters counters; - private String successfulAttemptId; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskInfo.java deleted file mode 100644 index 2e74b6945..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskInfo.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -import java.util.Map; - -@Data -public class TaskInfo { - private String taskId; - - private long startTime; - - private long finishTime; - - private String taskType; - - private String splitLocations; - - private JhCounters counters; - - private String status; - - private String error; - - private String failedDueToAttemptId; - - private String successfulAttemptId; - - private Map attemptsMap; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskStarted.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskStarted.java deleted file mode 100644 index 4e8c83659..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskStarted.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class TaskStarted { - private String taskid; - private String taskType; - private Long startTime; - private String splitLocations; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskType.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskType.java deleted file mode 100644 index 3cc74079a..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskType.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -public enum TaskType { - MAP, REDUCE -} \ No newline at end of file diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskUpdated.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskUpdated.java deleted file mode 100644 index f40a7c883..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/mr/event/TaskUpdated.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.mr.event; - -import lombok.Data; - -@Data -public class TaskUpdated { - private String taskid; - private Long finishTime; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/reader/ReaderObject.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/reader/ReaderObject.java deleted file mode 100644 index 03a79eacd..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/reader/ReaderObject.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.reader; - -import com.github.luben.zstd.ZstdInputStream; -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import net.jpountz.lz4.LZ4BlockInputStream; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.xerial.snappy.SnappyInputStream; - -import java.io.*; -import java.util.Locale; - -@Slf4j -public class ReaderObject { - - @Getter - @Setter - private String logPath; - private BufferedReader bufferedReader; - @Getter - @Setter - private InputStream inputStream; - @Getter - @Setter - private FileSystem fs; - - public BufferedReader getBufferedReader(String compressCodec) throws IOException { - if (bufferedReader != null) { - return bufferedReader; - } - InputStream decompressedInputStream; - switch (compressCodec.toLowerCase(Locale.ROOT)) { - case "lz4": - decompressedInputStream = new LZ4BlockInputStream(inputStream, false); - break; - case "snappy": - decompressedInputStream = new SnappyInputStream(inputStream); - break; - case "zstd": - decompressedInputStream = new BufferedInputStream(new ZstdInputStream(inputStream), 32 * 1023); - break; - default: - decompressedInputStream = inputStream; - break; - } - bufferedReader = new BufferedReader(new InputStreamReader(decompressedInputStream)); - return bufferedReader; - } - - public BufferedReader getBufferedReader() throws IOException { - if (bufferedReader != null) { - return bufferedReader; - } - return getBufferedReader("none"); - } - - public void close() { - try { - if (inputStream != null) { - inputStream.close(); - } - if (bufferedReader != null) { - bufferedReader.close(); - } - if (fs != null) { - // TODO: should close fs ? the fs may be a share object if we use FileSystem.get internal cache mechanisam - fs.close(); - } - } catch (IOException e) { - log.error("close file: {}, exception: ", logPath, e); - } - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/rule/Rules.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/rule/Rules.java deleted file mode 100644 index ca52f8d9c..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/rule/Rules.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.rule; - -import com.oppo.cloud.common.util.textparser.ParserAction; -import lombok.Data; - -import java.util.List; - -/** - * Diagnostic rule - */ -@Data -public class Rules { - - /** - * log type - */ - private String logType; - /** - * parser action - */ - private List actions; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/Accumulable.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/Accumulable.java deleted file mode 100644 index 89c3791c6..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/Accumulable.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import lombok.Data; - -@Data -public class Accumulable { - - private Long id; - - private R initialValue; - - private String name; - - private AccumulatorParam param; - - private Boolean countFailedValues; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/AccumulableInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/AccumulableInfo.java deleted file mode 100644 index d8e1c49e6..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/AccumulableInfo.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class AccumulableInfo { - - @JsonProperty("ID") - private Long id; - - @JsonProperty("Name") - private String name; - - @JsonProperty("Update") - private Object update; - - @JsonProperty("Value") - private Object value; - - @JsonProperty("Internal") - private boolean internal; - - @JsonProperty("Count Failed Values") - private boolean countFailedValues; - - @JsonProperty("Metadata") - private String metadata; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/Accumulator.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/Accumulator.java deleted file mode 100644 index ef5c4251b..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/Accumulator.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import lombok.Data; - -@Data -public class Accumulator extends Accumulable { -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/AccumulatorParam.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/AccumulatorParam.java deleted file mode 100644 index 23eec1439..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/AccumulatorParam.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import lombok.Data; - -@Data -public class AccumulatorParam { -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/BlockManagerId.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/BlockManagerId.java deleted file mode 100644 index d18912bf9..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/BlockManagerId.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class BlockManagerId { - - @JsonProperty("Executor ID") - private String executorId; - - @JsonProperty("Host") - private String host; - - @JsonProperty("Port") - private String port; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/BlockStatus.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/BlockStatus.java deleted file mode 100644 index 69f972603..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/BlockStatus.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class BlockStatus { - - @JsonProperty("Storage Level") - public StorageLevel storageLevel; - - @JsonProperty("Memory Size") - public Long memSize; - - @JsonProperty("Disk Size") - public Long diskSize; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/DetectResult.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/DetectResult.java deleted file mode 100644 index 00ab3e99f..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/DetectResult.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import lombok.Data; - -@Data -public class DetectResult { - - private Long maxResult = 0L; - private SparkPlanInfo sparkPlanInfo; - - /** - * @return raw node name： Scan orc union_os_dw.dim_model_info - */ - public String getScanNodeName() { - if (this.sparkPlanInfo != null) { - return this.sparkPlanInfo.getNodeName(); - } - return "scan table"; - } - - /** - * @return node table name: union_os_dw.dim_model_info - */ - public String getScanNodeTable() { - if (this.sparkPlanInfo == null) { - return "scan table"; - } - - String[] fields = this.sparkPlanInfo.getNodeName().split(" "); - if (fields.length > 0) { - return fields[fields.length - 1]; - } - return "scan table"; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/ExecutorInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/ExecutorInfo.java deleted file mode 100644 index 19d9f0576..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/ExecutorInfo.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.Map; - -@Data -public class ExecutorInfo { - - @JsonProperty("Host") - private String executorHost; - - @JsonProperty("Total Cores") - private Integer totalCores; - - @JsonProperty("Log Urls") - private Map logUrlMap; - - /** - * The version 2.4.0 does not exist - */ - @JsonProperty("Attributes") - private Map attributes; - - /** - * The version 2.4.0 does not exist - */ - @JsonProperty("Resources") - private Map resourceInfo; - - /** - * The version 2.4.0 does not exist - */ - @JsonProperty("Resource Profile Id") - private Integer resourceProfileId; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/InputMetrics.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/InputMetrics.java deleted file mode 100644 index dd22beaef..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/InputMetrics.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class InputMetrics { - - @JsonProperty("Bytes Read") - private Long bytesRead; - - @JsonProperty("Records Read") - private Long recordsRead; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/JobResult.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/JobResult.java deleted file mode 100644 index 8fd8750e0..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/JobResult.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class JobResult { - - @JsonProperty("Job Result") - private String result; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/OutputMetrics.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/OutputMetrics.java deleted file mode 100644 index 2c2bd0030..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/OutputMetrics.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class OutputMetrics { - - @JsonProperty("Bytes Written") - private Long bytesWritten; - @JsonProperty("Records Written") - private Long recordsWritten; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/RDDInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/RDDInfo.java deleted file mode 100644 index 74b4e1cbb..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/RDDInfo.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import lombok.Data; - -import java.util.List; - -@Data -public class RDDInfo { - - @JsonProperty("RDD ID") - private Integer id; - - @JsonProperty("Name") - private String name; - - @JsonProperty("Number of Partitions") - private Integer numPartitions; - - @JsonProperty("Storage Level") - private StorageLevel storageLevel; - - @JsonProperty("Parent IDs") - private List parentIds; - - @JsonProperty("Callsite") - private String callSite; - - @JsonDeserialize(converter = RDDScopeConverter.class) - @JsonProperty("Scope") - private RDDOperationScope scope; - - @JsonProperty("Number of Cached Partitions") - private Integer numCachedPartitions; - - @JsonProperty("Memory Size") - private Long memSize; - - @JsonProperty("Disk Size") - private Long diskSize; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/RDDOperationScope.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/RDDOperationScope.java deleted file mode 100644 index 0a7f19386..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/RDDOperationScope.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import lombok.Data; - -@Data -@JsonPropertyOrder({"id", "name", "parent"}) -public class RDDOperationScope { - - @JsonProperty("name") - private String name; - - /** - * option - */ - @JsonProperty("parent") - private RDDOperationScope parent; - - @JsonProperty("id") - private String id; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/RDDScopeConverter.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/RDDScopeConverter.java deleted file mode 100644 index 1a86281e6..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/RDDScopeConverter.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.util.StdConverter; - -public class RDDScopeConverter extends StdConverter { - - @Override - public RDDOperationScope convert(String value) { - try { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - return objectMapper.readValue(value, RDDOperationScope.class); - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/ResourceInformation.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/ResourceInformation.java deleted file mode 100644 index ca51ae01b..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/ResourceInformation.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.List; - -@Data -public class ResourceInformation { - - @JsonProperty("name") - private String name; - - @JsonProperty("addresses") - private List addresses; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SQLMetricInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SQLMetricInfo.java deleted file mode 100644 index cf136c605..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SQLMetricInfo.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SQLMetricInfo { - - @JsonProperty("name") - private String nodeName; - - @JsonProperty("accumulatorId") - private Long accumulatorId; - - @JsonProperty("metricType") - private String metricType; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SQLPlanMetric.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SQLPlanMetric.java deleted file mode 100644 index 96713e153..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SQLPlanMetric.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SQLPlanMetric { - - @JsonProperty("name") - private String name; - - @JsonProperty("accumulatorId") - private Long accumulatorId; - - @JsonProperty("metricType") - private String metricType; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/ShuffleReadMetrics.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/ShuffleReadMetrics.java deleted file mode 100644 index 5933ff005..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/ShuffleReadMetrics.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class ShuffleReadMetrics { - - @JsonProperty("Remote Blocks Fetched") - private Long remoteBlocksFetched; - - @JsonProperty("Local Blocks Fetched") - private Long localBlocksFetched; - - @JsonProperty("Remote Bytes Read") - private Long remoteBytesRead; - - @JsonProperty("Remote Bytes Read To Disk") - private Long remoteBytesReadToDisk; - - @JsonProperty("Local Bytes Read") - private Long localBytesRead; - - @JsonProperty("Fetch Wait Time") - private Long fetchWaitTime; - - /** - * total record read - */ - @JsonProperty("Total Records Read") - private Long recordsRead; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/ShuffleWriteMetrics.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/ShuffleWriteMetrics.java deleted file mode 100644 index d8163dc34..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/ShuffleWriteMetrics.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class ShuffleWriteMetrics { - - @JsonProperty("Shuffle Bytes Writte") - private Long bytesWritten; - - @JsonProperty("Shuffle Records Written") - private Long recordsWritten; - - @JsonProperty("Shuffle Write Time") - private Long writeTime; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkApplication.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkApplication.java deleted file mode 100644 index f7a9a4e58..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkApplication.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.oppo.cloud.parser.utils.UnitUtil; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -@Slf4j -@Data -public class SparkApplication { - - private Long appDuration; - private Long appStartTimestamp; - private Long appEndTimestamp; - private Properties jvmInformation; - private Properties sparkProperties; - private Properties systemProperties; - private Properties classpathEntries; - - private Long executorMemory; - - private Long driverMemory; - - private String sparkExecutorCores; - - public SparkApplication() { - - } - - /** - * SparkListenerEnvironmentUpdate属性 - */ - public void setSparkApplication(SparkListenerEnvironmentUpdate env) { - this.jvmInformation = env.getJvmInformation(); - this.sparkProperties = env.getSparkProperties(); - this.systemProperties = env.getSystemProperties(); - this.classpathEntries = env.getClasspathEntries(); - this.executorMemory = UnitUtil.toBytes(env.getSparkProperty("spark.executor.memory")); - this.driverMemory = UnitUtil.toBytes(env.getSparkProperty("spark.driver.memory")); - this.sparkExecutorCores = env.getSparkProperty("spark.executor.cores"); - } - - public Long getAppDuration() { - if (this.appEndTimestamp == null) { - this.appEndTimestamp = System.currentTimeMillis(); - } - return this.appEndTimestamp - this.appStartTimestamp; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkBlockManager.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkBlockManager.java deleted file mode 100644 index f641f9e8d..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkBlockManager.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import lombok.Data; - -@Data -public class SparkBlockManager { - - private Long maxMem; - private Long addTimestamp; - private String executorId; - - public SparkBlockManager(SparkListenerBlockManagerAdded added) { - this.maxMem = added.getMaxMem(); - this.addTimestamp = added.getTime(); - this.executorId = added.getBlockManagerId().getExecutorId(); - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkExecutor.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkExecutor.java deleted file mode 100644 index cda451a52..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkExecutor.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import lombok.Data; - -import java.util.ArrayList; -import java.util.List; - -@Data -public class SparkExecutor { - - /** - * executor ID - */ - private String Id; - private String host; - private List blockManagers; - private Long startTimestamp; - private Integer totalCores; - private List tasks; - private String removeReason; - private Long removeTimestamp; - - public SparkExecutor(SparkListenerExecutorAdded added) { - this.blockManagers = new ArrayList<>(); - this.tasks = new ArrayList<>(); - this.Id = added.getExecutorId(); - this.host = added.getExecutorInfo().getExecutorHost(); - this.startTimestamp = added.getTime(); - this.totalCores = added.getExecutorInfo().getTotalCores(); - this.removeTimestamp = 0L; - } - - public void addBlockManager(SparkBlockManager blockManager) { - this.blockManagers.add(blockManager); - } - - public void remove(SparkListenerExecutorRemoved removed) { - this.removeReason = removed.getReason(); - if (removed.getTime() != null) { - this.removeTimestamp = removed.getTime(); - } - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkJob.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkJob.java deleted file mode 100644 index 63c0168da..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkJob.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import lombok.Data; - -import java.util.ArrayList; -import java.util.List; - -@Data -public class SparkJob { - - private Integer jobId; - private List stages; - private Long submissionTime; - private Long endTime; - private JobResult jobResult; - /** - * executor run time - */ - private Long executorRunTime = 0L; - - public SparkJob() { - this.stages = new ArrayList<>(); - } - - public SparkJob(SparkListenerJobStart jobStart) { - this.stages = new ArrayList<>(); - this.jobId = jobStart.getJobId(); - for (StageInfo stageInfo : jobStart.getStageInfos()) { - this.stages.add(new SparkStage(stageInfo)); - } - this.submissionTime = jobStart.getTime(); - } - - public void complete(SparkListenerJobEnd jobEnd) { - this.jobResult = jobEnd.getJobResult(); - this.endTime = jobEnd.getTime(); - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerApplicationEnd.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerApplicationEnd.java deleted file mode 100644 index c260bd4d5..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerApplicationEnd.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SparkListenerApplicationEnd extends SparkListenerEvent { - - @JsonProperty("Timestamp") - private Long time; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerApplicationStart.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerApplicationStart.java deleted file mode 100644 index 1151d60c5..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerApplicationStart.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.Map; - -@Data -public class SparkListenerApplicationStart extends SparkListenerEvent { - - @JsonProperty("App Name") - private String appName; - @JsonProperty("App ID") - private String appId; - @JsonProperty("Timestamp") - private Long time; - @JsonProperty("User") - private String sparkUser; - @JsonProperty("App Attempt ID") - private String appAttemptId; - @JsonProperty("Driver Logs") - private Map driverLogs; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerBlockManagerAdded.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerBlockManagerAdded.java deleted file mode 100644 index 0dd27462f..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerBlockManagerAdded.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SparkListenerBlockManagerAdded extends SparkListenerEvent { - - @JsonProperty("Timestamp") - private Long time; - - @JsonProperty("Block Manager ID") - private BlockManagerId blockManagerId; - - @JsonProperty("Maximum Memory") - private Long maxMem; - - @JsonProperty("Maximum Onheap Memory") - private Long maxOnHeapMem; - - @JsonProperty("Maximum Offheap Memory") - private Long maxOffHeapMem; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerBlockManagerRemoved.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerBlockManagerRemoved.java deleted file mode 100644 index df4ab9142..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerBlockManagerRemoved.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SparkListenerBlockManagerRemoved extends SparkListenerEvent { - - @JsonProperty("Timestamp") - private Long time; - - @JsonProperty("Block Manager ID") - private BlockManagerId blockManagerId; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerDriverAccumUpdates.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerDriverAccumUpdates.java deleted file mode 100644 index d80bbf4e5..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerDriverAccumUpdates.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.List; - -/** - * * @param executionId The execution id for a query, so we can find the query plan. - * * @param accumUpdates Map from accumulator id to the metric value (metrics are always 64-bit ints). - */ -@Data -public class SparkListenerDriverAccumUpdates extends SparkListenerEvent { - - @JsonProperty("executionId") - private Long executionId; - - @JsonProperty("accumUpdates") - private List> accumUpdates; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerEnvironmentUpdate.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerEnvironmentUpdate.java deleted file mode 100644 index 698fa5ca6..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerEnvironmentUpdate.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Properties; - -@EqualsAndHashCode(callSuper = true) -@Data -public class SparkListenerEnvironmentUpdate extends SparkListenerEvent { - - @JsonProperty("JVM Information") - private Properties jvmInformation; - @JsonProperty("Spark Properties") - private Properties sparkProperties; - @JsonProperty("System Properties") - private Properties systemProperties; - @JsonProperty("Classpath Entries") - private Properties classpathEntries; - - public String getJvmInformationProperty(String key) { - return this.jvmInformation == null ? "" : this.jvmInformation.getProperty(key, ""); - } - - public String getSparkProperty(String key) { - return this.sparkProperties == null ? "" : this.sparkProperties.getProperty(key, ""); - } - - public String getSystemProperty(String key) { - return this.systemProperties == null ? "" : this.systemProperties.getProperty(key, ""); - } - - public String getClasspathEntry(String key) { - return this.classpathEntries == null ? "" : this.classpathEntries.getProperty(key, ""); - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerEvent.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerEvent.java deleted file mode 100644 index 625770234..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerEvent.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.io.Serializable; - -@Data -public class SparkListenerEvent implements Serializable { - - @JsonProperty("Event") - private String event; - - public boolean logEvent() { - return true; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerExecutorAdded.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerExecutorAdded.java deleted file mode 100644 index 404b41922..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerExecutorAdded.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SparkListenerExecutorAdded extends SparkListenerEvent { - - @JsonProperty("Timestamp") - private Long time; - - @JsonProperty("Executor ID") - private String executorId; - - @JsonProperty("Executor Info") - private ExecutorInfo executorInfo; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerExecutorRemoved.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerExecutorRemoved.java deleted file mode 100644 index f9374cd93..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerExecutorRemoved.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SparkListenerExecutorRemoved extends SparkListenerEvent { - - @JsonProperty("Timestamp") - private Long time; - - @JsonProperty("Executor ID") - private String executorId; - - @JsonProperty("Removed Reason") - private String reason; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerJobEnd.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerJobEnd.java deleted file mode 100644 index a0fcd04b3..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerJobEnd.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SparkListenerJobEnd extends SparkListenerEvent { - - @JsonProperty("Job ID") - private Integer jobId; - - @JsonProperty("Completion Time") - private Long time; - - @JsonProperty("Job Result") - private JobResult jobResult; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerJobStart.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerJobStart.java deleted file mode 100644 index 320d57796..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerJobStart.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.List; -import java.util.Properties; - -@Data -public class SparkListenerJobStart extends SparkListenerEvent { - - @JsonProperty("Job ID") - private Integer jobId; - - @JsonProperty("Submission Time") - private Long time; - - @JsonProperty("Stage Infos") - private List stageInfos; - - @JsonProperty("Properties") - private Properties properties; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerLogStart.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerLogStart.java deleted file mode 100644 index 562ec5960..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerLogStart.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SparkListenerLogStart extends SparkListenerEvent { - - @JsonProperty("Spark Version") - private String sparkVersion; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerSQLAdaptiveExecutionUpdate.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerSQLAdaptiveExecutionUpdate.java deleted file mode 100644 index dbbb95a71..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerSQLAdaptiveExecutionUpdate.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -/** - * SparkListenerSQLAdaptiveExecutionUpdate,the version 2.4.0 does not exist - */ -@Data -public class SparkListenerSQLAdaptiveExecutionUpdate extends SparkListenerEvent { - - @JsonProperty("executionId") - private Long executionId; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerSQLAdaptiveSQLMetricUpdates.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerSQLAdaptiveSQLMetricUpdates.java deleted file mode 100644 index dd7b9491f..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerSQLAdaptiveSQLMetricUpdates.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.List; - -@Data -public class SparkListenerSQLAdaptiveSQLMetricUpdates extends SparkListenerEvent { - - @JsonProperty("executionId") - private Long executionId; - - @JsonProperty("sqlPlanMetrics") - private List sqlPlanMetrics; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerSQLExecutionEnd.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerSQLExecutionEnd.java deleted file mode 100644 index f638174ad..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerSQLExecutionEnd.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SparkListenerSQLExecutionEnd extends SparkListenerEvent { - - @JsonProperty("executionId") - private Long executionId; - - @JsonProperty("time") - private Long time; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerSQLExecutionStart.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerSQLExecutionStart.java deleted file mode 100644 index be2e6b565..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerSQLExecutionStart.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SparkListenerSQLExecutionStart extends SparkListenerEvent { - - @JsonProperty("executionId") - private Long executionId; - - @JsonProperty("description") - private String description; - - @JsonProperty("details") - private String detail; - - @JsonProperty("physicalPlanDescription") - private String physicalPlanDescription; - - @JsonProperty("sparkPlanInfo") - private SparkPlanInfo sparkPlanInfo; - - @JsonProperty("time") - private Long time; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerStageCompleted.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerStageCompleted.java deleted file mode 100644 index cc7cc9072..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerStageCompleted.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SparkListenerStageCompleted extends SparkListenerEvent { - - @JsonProperty("Stage Info") - private StageInfo stageInfo; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerStageSubmitted.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerStageSubmitted.java deleted file mode 100644 index 20b8ee325..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerStageSubmitted.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.Properties; - -@Data -public class SparkListenerStageSubmitted extends SparkListenerEvent { - - @JsonProperty("Stage Info") - private StageInfo stageInfo; - - @JsonProperty("Properties") - private Properties properties; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerTaskEnd.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerTaskEnd.java deleted file mode 100644 index 480f0331c..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerTaskEnd.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SparkListenerTaskEnd extends SparkListenerEvent { - - @JsonProperty("Stage ID") - private Integer stageId; - - @JsonProperty("Stage Attempt ID") - private Integer stageAttemptId; - - @JsonProperty("Task Type") - private String taskType; - - @JsonProperty("Task Info") - private TaskInfo taskInfo; - - @JsonProperty("Task End Reason") - private TaskEndReason reason; - - @JsonProperty("Task Metrics") - private TaskMetrics taskMetrics; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerTaskGettingResult.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerTaskGettingResult.java deleted file mode 100644 index 9bcd0a163..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerTaskGettingResult.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SparkListenerTaskGettingResult extends SparkListenerEvent { - - @JsonProperty("Task Info") - private TaskInfo taskInfo; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerTaskStart.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerTaskStart.java deleted file mode 100644 index 0cacfd074..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkListenerTaskStart.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class SparkListenerTaskStart extends SparkListenerEvent { - - @JsonProperty("Stage ID") - private Integer stageId; - - @JsonProperty("Stage Attempt ID") - private Integer stageAttemptId; - - @JsonProperty("Task Info") - private TaskInfo taskInfo; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkPlanInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkPlanInfo.java deleted file mode 100644 index 526cb68c7..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkPlanInfo.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.List; -import java.util.Map; - -@Data -public class SparkPlanInfo { - - @JsonProperty("nodeName") - private String nodeName; - - @JsonProperty("simpleString") - private String simpleString; - - @JsonProperty("metadata") - private Map metadata; - - @JsonProperty("children") - private List children; - - @JsonProperty("metrics") - private List metrics; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkRDD.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkRDD.java deleted file mode 100644 index 28f5a7ae1..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkRDD.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -public class SparkRDD { - - private Integer id; - private Long diskSize; - private Long memSize; - private String name; - private Integer numPartitions; - private Integer replication; - - public SparkRDD(RDDInfo rddInfo) { - this.id = rddInfo.getId(); - this.diskSize = rddInfo.getDiskSize(); - this.memSize = rddInfo.getMemSize(); - this.name = rddInfo.getName(); - this.numPartitions = rddInfo.getNumPartitions(); - this.replication = rddInfo.getStorageLevel().getReplication(); - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkStage.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkStage.java deleted file mode 100644 index 969daa930..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkStage.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Slf4j -@Data -@NoArgsConstructor -public class SparkStage { - - private Integer stageId; - private String details; - private Integer numTasks; - private List RDDs; - private String name; - private Map> tasksMap; - /** - * Stage with retry - */ - private Map failed; - private Map completeTimeMap; - private Map submissionTimeMap; - private List accumulableInfos; - - public SparkStage(StageInfo stageInfo) { - this.stageId = stageInfo.getStageId(); - this.details = stageInfo.getDetails(); - this.numTasks = stageInfo.getNumTasks(); - this.RDDs = new ArrayList<>(); - this.tasksMap = new HashMap<>(); - this.failed = new HashMap<>(); - this.completeTimeMap = new HashMap<>(); - this.submissionTimeMap = new HashMap<>(); - - for (RDDInfo rddInfo : stageInfo.getRddInfos()) { - this.RDDs.add(new SparkRDD(rddInfo)); - } - this.name = stageInfo.getName(); - } - - public void complete(SparkListenerStageCompleted completed) { - this.accumulableInfos = completed.getStageInfo().getAccumulables(); - - this.completeTimeMap.put(completed.getStageInfo().getAttemptNumber(), - completed.getStageInfo().getCompleteTime()); - this.submissionTimeMap.put(completed.getStageInfo().getAttemptNumber(), - completed.getStageInfo().getSubmissionTime()); - if (!StringUtils.isEmpty(completed.getStageInfo().getFailureReason())) { - failed.put(completed.getStageInfo().getAttemptNumber(), true); - } - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkTask.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkTask.java deleted file mode 100644 index ff5c3bcb5..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/SparkTask.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -public class SparkTask { - - private Long taskId; - private Integer stageId; - private Integer stageAttemptId; - private String executorId; - private Long launchTime; - private String taskLocality; - private Boolean speculative; - - private String endReason = ""; - public String failedMessage = ""; - public String fullStackTrace = ""; - public String lossReason = ""; - private Boolean failed = false; - private Long finishTime = 0L; - private Long gettingResultTime = 0L; - private Integer index = 0; - private String taskType = ""; - private Boolean hasMetric = false; - private Long diskBytesSpilled = 0L; - private Long memoryBytesSpilled = 0L; - private Long executorDeserializeTime = 0L; - private Long executorRunTime = 0L; - private Long jvmGCTime = 0L; - private Long resultSerializationTime = 0L; - private Long resultSize = 0L; - /** - * Total Shuffle Read = Remote Bytes Read + Local Bytes Read - */ - private Long totalShuffleReadBytes = 0L; - private Long totalRecordsRead = 0L; - /** - * task InputMetrics - */ - private Long bytesRead = 0L; - - public SparkTask(SparkListenerTaskStart start) { - this.taskId = start.getTaskInfo().getTaskId(); - this.stageId = start.getStageId(); - this.stageAttemptId = start.getStageAttemptId(); - this.executorId = start.getTaskInfo().getExecutorId(); - this.launchTime = start.getTaskInfo().getLaunchTime(); - this.taskLocality = start.getTaskInfo().getTaskLocality(); - this.speculative = start.getTaskInfo().getSpeculative(); - } - - public void finish(SparkListenerTaskEnd end) { - if (end.getReason() != null) { - this.endReason = end.getReason().getReason(); - this.failedMessage = end.getReason().getMessage(); - this.fullStackTrace = end.getReason().getFullStackTrace(); - this.lossReason = end.getReason().getLossReason(); - } - if (end.getTaskInfo() != null) { - this.failed = end.getTaskInfo().getFailed(); - this.finishTime = end.getTaskInfo().getFinishTime(); - this.gettingResultTime = end.getTaskInfo().getGettingResultTime(); - this.index = end.getTaskInfo().getIndex(); - } - this.taskType = end.getTaskType(); - if (end.getTaskMetrics() != null) { - this.hasMetric = true; - this.diskBytesSpilled = end.getTaskMetrics().getDiskBytesSpilled(); - this.memoryBytesSpilled = end.getTaskMetrics().getMemoryBytesSpilled(); - this.executorDeserializeTime = end.getTaskMetrics().getExecutorDeserializeTime(); - this.executorRunTime = end.getTaskMetrics().getExecutorRunTime(); - this.jvmGCTime = end.getTaskMetrics().getJvmGCTime(); - this.resultSerializationTime = end.getTaskMetrics().getResultSerializationTime(); - this.resultSize = end.getTaskMetrics().getResultSize(); - if (end.getTaskMetrics().getShuffleReadMetrics() != null) { - totalShuffleReadBytes = end.getTaskMetrics().getShuffleReadMetrics().getRemoteBytesRead() + - end.getTaskMetrics().getShuffleReadMetrics().getLocalBytesRead(); - totalRecordsRead = end.getTaskMetrics().getShuffleReadMetrics().getRecordsRead(); - } - if (end.getTaskMetrics().getInputMetrics() != null) { - this.bytesRead = end.getTaskMetrics().getInputMetrics().getBytesRead(); - } - - } - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/StackTraceElement.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/StackTraceElement.java deleted file mode 100644 index 7f96bd1be..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/StackTraceElement.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.io.Serializable; - -@Data -public class StackTraceElement implements Serializable { - - @JsonProperty("Declaring Class") - private String declaringClass; - - @JsonProperty("Method Name") - private String methodName; - - @JsonProperty("File Name") - private String fileName; - - @JsonProperty("Line Number") - private Integer lineNumber; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/StageInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/StageInfo.java deleted file mode 100644 index c295b1161..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/StageInfo.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.List; - -@Data -public class StageInfo { - - @JsonProperty("Stage ID") - private Integer stageId; - - @JsonProperty("Stage Attempt ID") - private Integer attemptNumber; - - @JsonProperty("Stage Name") - private String name; - - @JsonProperty("Number of Tasks") - private Integer numTasks; - - @JsonProperty("RDD Info") - private List rddInfos; - - @JsonProperty("Parent IDs") - private List parentIds; - - @JsonProperty("Details") - private String details; - - @JsonProperty("Task Metrics") - private TaskMetrics taskMetrics; - - @JsonProperty("Submission Time") - private Long submissionTime; - - @JsonProperty("Completion Time") - private Long completeTime; - - @JsonProperty("Failure Reason") - private String failureReason; - - @JsonProperty("Accumulables") - private List accumulables; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/StorageLevel.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/StorageLevel.java deleted file mode 100644 index f5fb3094c..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/StorageLevel.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class StorageLevel { - - @JsonProperty("Use Disk") - private boolean useDisk; - - @JsonProperty("Use Memory") - private boolean useMemory; - - /** - * The version 2.4.0 does not exist - */ - @JsonProperty("Use OffHeap") - private boolean useOffHeap; - - @JsonProperty("Deserialized") - private boolean deserialized; - - @JsonProperty("Replication") - private Integer replication; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/TaskEndReason.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/TaskEndReason.java deleted file mode 100644 index bc68c9f32..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/TaskEndReason.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.List; - -@Data -public class TaskEndReason { - - /** - * Success - */ - @JsonProperty("Reason") - private String Reason; - - /** - * TaskFailedReason - * FetchFailed - */ - @JsonProperty("Block Manager Address") - private BlockManagerId blockManagerAddress; - - @JsonProperty("Shuffle ID") - private Integer shuffleId; - - @JsonProperty("Map ID") - private Integer mapId; - - @JsonProperty("Reduce ID") - private Integer reduceId; - - @JsonProperty("Message") - private String message; - - /** - * ExceptionFailure - */ - @JsonProperty("Class Name") - private String className; - - @JsonProperty("Description") - private String description; - - @JsonProperty("Stack Trace") - private List stackTrace; - - @JsonProperty("Full Stack Trace") - private String fullStackTrace; - - @JsonProperty("Accumulator Updates") - private List accumUpdates; - /** - * TaskCommitDenied - */ - @JsonProperty("Job ID") - private Integer jobID; - - @JsonProperty("Partition ID") - private Integer partitionID; - - @JsonProperty("Attempt Number") - private Integer attemptNumber; - - /** - * ExecutorLostFailure - */ - @JsonProperty("Executor ID") - private String execId; - - @JsonProperty("Exit Caused By App") - private boolean exitCausedByApp; - - @JsonProperty("Loss Reason") - private String lossReason; - - /** - * TaskKilled - */ - @JsonProperty("Kill Reason") - private String killReason; - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/TaskInfo.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/TaskInfo.java deleted file mode 100644 index a0a2c7b03..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/TaskInfo.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.List; - -@Data -public class TaskInfo { - - @JsonProperty("Task ID") - private Long taskId; - - @JsonProperty("Index") - private Integer index; - - @JsonProperty("Attempt") - private Integer attemptNumber; - - @JsonProperty("Launch Time") - private Long launchTime; - - @JsonProperty("Executor ID") - private String executorId; - - @JsonProperty("Host") - private String host; - - @JsonProperty("Locality") - private String taskLocality; - - @JsonProperty("Speculative") - private Boolean speculative; - - @JsonProperty("Getting Result Time") - private Long gettingResultTime; - - @JsonProperty("Finish Time") - private Long finishTime; - - @JsonProperty("Failed") - private Boolean failed; - - @JsonProperty("Killed") - private Boolean killed; - - @JsonProperty("Accumulables") - private List accumulables; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/TaskMetrics.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/TaskMetrics.java deleted file mode 100644 index bd18dcc77..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/TaskMetrics.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.List; - -@Data -public class TaskMetrics { - - @JsonProperty("Executor Deserialize Time") - private Long executorDeserializeTime; - - @JsonProperty("Executor Deserialize CPU Time") - private Long executorDeserializeCpuTime; - - @JsonProperty("Executor Run Time") - private Long executorRunTime; - - @JsonProperty("Executor CPU Time") - private Long executorCpuTime; - - @JsonProperty("Result Size") - private Long resultSize; - - @JsonProperty("JVM GC Time") - private Long jvmGCTime; - - @JsonProperty("Result Serialization Time") - private Long resultSerializationTime; - - @JsonProperty("Memory Bytes Spilled") - private Long memoryBytesSpilled; - - @JsonProperty("Disk Bytes Spilled") - private Long diskBytesSpilled; - /** - * The version 2.4.0 does not exist - */ - @JsonProperty("peakExecutionMemory") - private Long peakExecutionMemory; - - @JsonProperty("Shuffle Read Metrics") - private ShuffleReadMetrics shuffleReadMetrics; - - @JsonProperty("Shuffle Write Metrics") - private ShuffleWriteMetrics shuffleWriteMetrics; - - @JsonProperty("Input Metrics") - private InputMetrics inputMetrics; - - @JsonProperty("Output Metrics") - private OutputMetrics outputMetrics; - - @JsonProperty("Updated Blocks") - private List updatedBlockStatues; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/UpdateBlockStatus.java b/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/UpdateBlockStatus.java deleted file mode 100644 index 21885e77d..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/domain/spark/eventlog/UpdateBlockStatus.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.domain.spark.eventlog; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -@Data -public class UpdateBlockStatus { - - @JsonProperty("Block ID") - private String blockId; - - @JsonProperty("Status") - private BlockStatus blockStatus; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/consumer/RedisConsumer.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/consumer/RedisConsumer.java deleted file mode 100644 index 714f0b31a..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/consumer/RedisConsumer.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.consumer; - -import com.alibaba.fastjson2.JSONObject; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.oppo.cloud.common.domain.job.LogRecord; -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.parser.config.CustomConfig; -import com.oppo.cloud.parser.config.ThreadPoolConfig; -import com.oppo.cloud.parser.service.job.JobManager; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.CommandLineRunner; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.redis.core.script.RedisScript; - -import javax.annotation.PostConstruct; -import javax.annotation.Resource; -import java.util.Arrays; -import java.util.Map; -import java.util.concurrent.Executor; -import java.util.concurrent.Semaphore; - -/** - * consume redis list data - */ -@Slf4j -@Configuration -@ConditionalOnProperty(prefix = "custom.redisConsumer", name = "enable", havingValue = "true") -public class RedisConsumer implements CommandLineRunner { - - @Resource - private CustomConfig config; - - @Resource - private RedisService redisService; - - @Autowired - private RedisScript logRecordConsumerScript; - - @Resource - private JobManager jobManager; - - @Resource - private ObjectMapper objectMapper; - - @Resource(name = ThreadPoolConfig.REDIS_CONSUMER_THREAD_POOL) - private Executor redisConsumerExecutorPool; - - @PostConstruct - void init() { - // Reload interrupted task - Map processingMap = null; - try { - processingMap = redisService.hGetAll(config.getProcessingHash()); - } catch (Exception e) { - log.error("Exception:", e); - } - if (processingMap != null) { - log.info("initProcessingJobSize:{}", processingMap.size()); - processingMap.forEach((k, v) -> { - log.info("initProcessingJobData,id:{},pushLogRecord:{}", k, v); - redisService.lRightPush(config.getLogRecordList(), v); - redisService.hDel(config.getProcessingHash(), k); - }); - } - } - - @Override - public void run(String... args) throws Exception { - log.info("maxThreadPoolSize:{}", config.getMaxThreadPoolSize()); - Semaphore semaphore = new Semaphore(config.getMaxThreadPoolSize()); - while (true) { - try { - String msg = (String) redisService.executeScript(logRecordConsumerScript, - Arrays.asList(config.getLogRecordList(), config.getProcessingHash())); - if (msg == null) { - Thread.sleep(5000); - continue; - } - log.info("consumeLogRecord:{}", msg); - semaphore.acquire(); - LogRecord logRecord = JSONObject.parseObject(msg, LogRecord.class); - redisConsumerExecutorPool.execute(() -> consume(logRecord, semaphore)); - } catch (Exception e) { - log.error("Exception:", e); - } - } - } - - private void consume(LogRecord logRecord, Semaphore semaphore) { - try { - jobManager.run(logRecord); - } catch (Exception e) { - log.error("Exception:", e); - }finally { - semaphore.release(); - long size = redisService.hDel(config.getProcessingHash(), logRecord.getId()); - if (size == 0) { - log.error("delete redis cache err:{}", logRecord); - } - } - - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/JobManager.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/JobManager.java deleted file mode 100644 index 996589656..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/JobManager.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job; - -import com.oppo.cloud.common.constant.LogGroupType; -import com.oppo.cloud.common.domain.job.App; -import com.oppo.cloud.common.domain.job.LogInfo; -import com.oppo.cloud.common.domain.job.LogRecord; -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.parser.config.CustomConfig; -import com.oppo.cloud.parser.config.ThreadPoolConfig; -import com.oppo.cloud.parser.domain.job.TaskParam; -import com.oppo.cloud.parser.domain.job.TaskResult; -import com.oppo.cloud.parser.service.job.task.Task; -import com.oppo.cloud.parser.service.job.task.TaskFactory; -import com.oppo.cloud.parser.service.writer.OpenSearchWriter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; -import java.util.concurrent.Future; - -/** - * Job manager - */ -@Slf4j -@Service -public class JobManager { - - @Resource - private CustomConfig customConfig; - - @Resource - private RedisService redisService; - - @Resource(name = ThreadPoolConfig.JOB_THREAD_POOL) - private Executor jobExecutorPool; - - public JobManager() { - - } - - private List createTasks(LogRecord logRecord) { - List tasks = new ArrayList<>(); - if (logRecord.getApps() != null) { - for (App app : logRecord.getApps()) { - for (LogInfo logInfo : app.getLogInfoList()) { - Task task = TaskFactory.create(new TaskParam(logInfo.getLogGroup(), logRecord, app, logInfo)); - if (task == null) { - log.warn("unsupported log group:{}", logInfo.getLogGroup()); - continue; - } - tasks.add(task); - } - tasks.add(TaskFactory.create(new TaskParam(LogGroupType.YARN.getName(), logRecord, app, null))); - } - } - return tasks; - } - - public List> createFutures(List tasks, Executor taskThreadPool) { - - List> futures = new ArrayList<>(); - - for (Task task : tasks) { - CompletableFuture future = CompletableFuture.supplyAsync(() -> { - TaskResult taskResult = null; - try { - taskResult = task.run(); - } catch (Exception e) { - log.error("Exception:", e); - } - return taskResult; - }, taskThreadPool); - futures.add(future); - } - - return futures; - } - - - public void run(LogRecord logRecord) throws Exception { - log.info("start job logRecord:{}", logRecord.getId()); - - List tasks = createTasks(logRecord); - - if (tasks.size() == 0) { - return; - } - - long start = System.currentTimeMillis(); - - List> futures = createFutures(tasks, jobExecutorPool); - - List taskResults = new ArrayList<>(); - - for (Future result : futures) { - TaskResult taskResult; - try { - taskResult = result.get(); - if (taskResult != null) { - taskResults.add(taskResult); - } - } catch (Exception e) { - log.error("Exception:", e); - } - } - - try { - OpenSearchWriter.getInstance().saveTaskResults(logRecord, taskResults); - } catch (Exception e) { - log.error("Exception:", e); - } - - long end = System.currentTimeMillis(); - log.info("finished job({}) elapsed time :{}(ms)", logRecord.getId(), end - start); - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/DetectorManager.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/DetectorManager.java deleted file mode 100644 index 3f1b70717..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/DetectorManager.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector; - -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.DetectorStorage; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; - -@Slf4j -public class DetectorManager implements IManager { - - @Override - public List createDetectors(DetectorParam param) { - return new DetectorRegister(param).registerDetectors(); - } - - @Override - public DetectorStorage run(DetectorParam param) { - List detectors = this.createDetectors(param); - DetectorStorage detectorStorage = new DetectorStorage( - param.getFlowName(), param.getProjectName(), - param.getTaskName(), param.getExecutionTime(), - param.getTryNumber(), param.getAppId(), - param.getLogPath(), param.getConfig()); - - for (IDetector detector : detectors) { - DetectorResult result; - try { - result = detector.detect(); - } catch (Exception e) { - log.error("Exception:{},", param.getAppId(), e); - continue; - } - if (result == null) { - continue; - } - if (result.getAbnormal()) { - detectorStorage.setAbnormal(true); - log.info("DetectorResult:{},{}", param.getAppId(), result.getAppCategory()); - } - detectorStorage.addDetectorResult(result); - } - - return detectorStorage; - } - - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/DetectorRegister.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/DetectorRegister.java deleted file mode 100644 index 3763fd20c..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/DetectorRegister.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector; - -import com.oppo.cloud.common.domain.eventlog.config.*; -import com.oppo.cloud.common.domain.mr.config.*; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.service.job.detector.mr.*; -import com.oppo.cloud.parser.service.job.detector.spark.*; - -import java.util.ArrayList; -import java.util.List; - -public class DetectorRegister { - - private final DetectorParam param; - - private final DetectorConfig config; - - private List detectors; - - public DetectorRegister(DetectorParam param) { - this.param = param; - this.config = param.getConfig(); - detectors = new ArrayList<>(); - } - - public List registerDetectors() { - switch (this.param.getAppType()) { - case SPARK: - registerSparkDetectors(); - break; - case MAPREDUCE: - registerMapReduceDetectors(); - break; - default: - break; - } - return this.detectors; - } - - private void registerSparkDetectors() { - registerCpuWasteDetector(); - registerDataSkewDetector(); - registerGlobalSortDetector(); - registerHdfsStuckDetector(); - registerJobDurationDetector(); - registerLargeTableScanDetector(); - registerOOMWarnDetector(); - registerSpeculativeTaskDetector(); - registerStageDurationDetector(); - registerTaskDurationDetector(); - } - - private void registerMapReduceDetectors() { - registerMRLargeTableScanDetector(); - registerMRMemoryWasteDetector(); - registerMRDataSkewDetector(); - registerMRSpeculativeTaskDetector(); - registerMRTaskDurationDetector(); - registerMRGCDetector(); - } - - private void registerCpuWasteDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new CpuWasteDetector(this.param)); - return; - } - CpuWasteConfig cpuWasteConfig = this.config.getCpuWasteConfig(); - if (!cpuWasteConfig.getDisable() && durationFilter(cpuWasteConfig.getDuration())) { - this.detectors.add(new CpuWasteDetector(this.param)); - } - } - - private void registerDataSkewDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new DataSkewDetector(this.param)); - return; - } - DataSkewConfig dataSkewConfig = this.config.getDataSkewConfig(); - if (!dataSkewConfig.getDisable() && durationFilter(dataSkewConfig.getDuration())) { - this.detectors.add(new DataSkewDetector(this.param)); - } - } - - private void registerGlobalSortDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new GlobalSortDetector(this.param)); - return; - } - GlobalSortConfig globalSortConfig = this.config.getGlobalSortConfig(); - if (!globalSortConfig.getDisable()) { - this.detectors.add(new GlobalSortDetector(this.param)); - } - } - - private void registerHdfsStuckDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new HdfsStuckDetector(this.param)); - return; - } - HdfsStuckConfig hdfsStuckConfig = this.config.getHdfsStuckConfig(); - if (!hdfsStuckConfig.getDisable()) { - this.detectors.add(new HdfsStuckDetector(this.param)); - } - } - - private void registerJobDurationDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new JobDurationDetector(this.param)); - return; - } - JobDurationConfig jobDurationConfig = this.config.getJobDurationConfig(); - if (!jobDurationConfig.getDisable()) { - this.detectors.add(new JobDurationDetector(this.param)); - } - } - - private void registerLargeTableScanDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new LargeTableScanDetector(this.param)); - return; - } - LargeTableScanConfig largeTableScanConfig = this.config.getLargeTableScanConfig(); - if (!largeTableScanConfig.getDisable() && durationFilter(largeTableScanConfig.getDuration())) { - this.detectors.add(new LargeTableScanDetector(this.param)); - } - } - - private void registerOOMWarnDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new OOMWarnDetector(this.param)); - return; - } - OOMWarnConfig oomWarnConfig = this.config.getOomWarnConfig(); - if (!oomWarnConfig.getDisable() && durationFilter(oomWarnConfig.getDuration())) { - this.detectors.add(new OOMWarnDetector(this.param)); - } - } - - private void registerSpeculativeTaskDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new SpeculativeTaskDetector(this.param)); - return; - } - SpeculativeTaskConfig speculativeTaskConfig = this.config.getSpeculativeTaskConfig(); - if (!speculativeTaskConfig.getDisable() && durationFilter(speculativeTaskConfig.getDuration())) { - this.detectors.add(new SpeculativeTaskDetector(this.param)); - } - } - - private void registerStageDurationDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new StageDurationDetector(this.param)); - return; - } - StageDurationConfig stageDurationConfig = this.config.getStageDurationConfig(); - if (!stageDurationConfig.getDisable()) { - this.detectors.add(new StageDurationDetector(param)); - } - } - - private void registerTaskDurationDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new TaskDurationDetector(this.param)); - return; - } - TaskDurationConfig taskDurationConfig = this.config.getTaskDurationConfig(); - if (!this.config.getTaskDurationConfig().getDisable() && durationFilter(taskDurationConfig.getDuration())) { - this.detectors.add(new TaskDurationDetector(this.param)); - } - } - - private void registerMRLargeTableScanDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new MRLargeTableScanDetector(this.param)); - return; - } - MRLargeTableScanConfig largeTableScanConfig = this.config.getMrLargeTableScanConfig(); - if (!largeTableScanConfig.getDisable() && durationFilter(largeTableScanConfig.getDuration())) { - this.detectors.add(new MRLargeTableScanDetector(this.param)); - } - } - - private void registerMRMemoryWasteDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new MRMemoryWasteDetector(this.param)); - return; - } - MRMemWasteConfig memWasteConfig = this.config.getMrMemWasteConfig(); - if (!memWasteConfig.getDisable() && durationFilter(memWasteConfig.getDuration())) { - this.detectors.add(new MRMemoryWasteDetector(this.param)); - } - } - - private void registerMRDataSkewDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new MRDataSkewDetector(this.param)); - return; - } - MRDataSkewConfig dataSkewConfig = this.config.getMrDataSkewConfig(); - if (!dataSkewConfig.getDisable() && durationFilter(dataSkewConfig.getDuration())) { - this.detectors.add(new MRDataSkewDetector(this.param)); - } - } - - private void registerMRSpeculativeTaskDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new MRSpeculativeTaskDetector(this.param)); - return; - } - MRSpeculativeTaskConfig speculativeTaskConfig = this.config.getMrSpeculativeTaskConfig(); - if (!speculativeTaskConfig.getDisable() && durationFilter(speculativeTaskConfig.getDuration())) { - this.detectors.add(new MRSpeculativeTaskDetector(this.param)); - } - } - - private void registerMRTaskDurationDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new MRTaskDurationDetector(this.param)); - return; - } - MRTaskDurationConfig taskDurationConfig = this.config.getMrTaskDurationConfig(); - if (!taskDurationConfig.getDisable() && durationFilter(taskDurationConfig.getDuration())) { - this.detectors.add(new MRTaskDurationDetector(this.param)); - } - } - - private void registerMRGCDetector() { - if (this.param.isOneClick()) { - this.detectors.add(new MRGCDetector(this.param)); - return; - } - MRGCConfig gcConfig = this.config.getMrGCConfig(); - if (!gcConfig.getDisable() && durationFilter(gcConfig.getDuration())) { - this.detectors.add(new MRGCDetector(this.param)); - } - } - - private boolean durationFilter(Long duration) { - return this.param.getAppDuration() >= duration; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/IDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/IDetector.java deleted file mode 100644 index 886bfd363..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/IDetector.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector; - -import com.oppo.cloud.common.domain.eventlog.DetectorResult; - -public interface IDetector { - - DetectorResult detect(); -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/IManager.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/IManager.java deleted file mode 100644 index 214c8aac0..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/IManager.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector; - -import com.oppo.cloud.common.domain.eventlog.DetectorStorage; -import com.oppo.cloud.parser.domain.job.DetectorParam; - -import java.util.List; - -public interface IManager { - - List createDetectors(DetectorParam param); - - - DetectorStorage run(DetectorParam param); - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRDataSkewDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRDataSkewDetector.java deleted file mode 100644 index fb4e37437..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRDataSkewDetector.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.mr; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.constant.MRTaskType; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.GraphType; -import com.oppo.cloud.common.domain.mr.MRDataSkewAbnormal; -import com.oppo.cloud.common.domain.mr.MRDataSkewGraph; -import com.oppo.cloud.common.domain.mr.config.MRDataSkewConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.mr.CounterInfo; -import com.oppo.cloud.parser.domain.mr.MRAppInfo; -import com.oppo.cloud.parser.domain.mr.MRTaskAttemptInfo; -import com.oppo.cloud.parser.service.job.detector.IDetector; -import com.oppo.cloud.parser.utils.UnitUtil; -import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; - -import java.util.*; - -public class MRDataSkewDetector implements IDetector { - - private final DetectorParam param; - - private final MRDataSkewConfig config; - - public MRDataSkewDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getMrDataSkewConfig(); - } - - @Override - public DetectorResult detect() { - DetectorResult> detectorResult = - new DetectorResult<>(AppCategoryEnum.MR_DATA_SKEW.getCategory(), false); - - List data = new ArrayList<>(); - MRAppInfo mrAppInfo = param.getMrAppInfo(); - - MRDataSkewAbnormal mapResult = judgeDataSkew(mrAppInfo.getMapList(), MRTaskType.MAP); - MRDataSkewAbnormal reduceResult = judgeDataSkew(mrAppInfo.getReduceList(), MRTaskType.REDUCE); - if (mapResult.getAbnormal() || reduceResult.getAbnormal()) { - detectorResult.setAbnormal(true); - data.add(mapResult); - data.add(reduceResult); - } - detectorResult.setData(data); - return detectorResult; - } - - - private MRDataSkewAbnormal judgeDataSkew(List lists, MRTaskType taskType) { - MRDataSkewAbnormal mrDataSkewAbnormal = new MRDataSkewAbnormal(); - mrDataSkewAbnormal.setAbnormal(false); - mrDataSkewAbnormal.setTaskType(taskType.getName()); - List dataSkewGraphList = new ArrayList<>(); - - double[] dataList = new double[lists.size()]; - long elapsedTime = 0L; - long maxTaskSize = 0L; - for (int i = 0; i < lists.size(); i++) { - MRTaskAttemptInfo task = lists.get(i); - Long dataSize = 0L; - switch (taskType) { - case MAP: - String group = CounterInfo.CounterGroupName.FILE_SYSTEM_COUNTER.getCounterGroupName(); - Map metrics = task.getCounters().get(group); - dataSize = metrics.getOrDefault(CounterInfo.CounterName.HDFS_BYTES_READ.getCounterName(), 0L); - break; - case REDUCE: - group = CounterInfo.CounterGroupName.TASK_COUNTER.getCounterGroupName(); - metrics = task.getCounters().get(group); - dataSize = metrics.getOrDefault(CounterInfo.CounterName.REDUCE_SHUFFLE_BYTES.getCounterName(), 0L); - break; - default: - break; - - } - dataList[i] = dataSize; - dataSkewGraphList.add(new MRDataSkewGraph( - (long) task.getTaskId(), - dataSize, - GraphType.normal.toString()) - ); - if (dataSize > maxTaskSize) { - maxTaskSize = dataSize; - elapsedTime = task.getElapsedTime(); - } - } - mrDataSkewAbnormal.setElapsedTime(elapsedTime); - DescriptiveStatistics statistics = new DescriptiveStatistics(dataList); - double median = statistics.getPercentile(50); - double ratio = maxTaskSize / median; - - boolean skew = false; - switch (taskType) { - case MAP: - skew = ratio > config.getMapThreshold() && maxTaskSize > UnitUtil.MBToByte(config.getTaskSize()) && - elapsedTime > config.getTaskDuration(); - break; - case REDUCE: - skew = ratio > config.getReduceThreshold() && maxTaskSize > UnitUtil.MBToByte(config.getTaskSize()) && - elapsedTime > config.getTaskDuration(); - break; - default: - break; - } - mrDataSkewAbnormal.setAbnormal(skew); - mrDataSkewAbnormal.setRatio(ratio); - if(dataSkewGraphList.size() > 0){ - mrDataSkewAbnormal.setGraphList(handleSkewGraph(dataSkewGraphList)); - } - return mrDataSkewAbnormal; - } - - private List handleSkewGraph(List dataSkewGraphList) { - dataSkewGraphList.sort(Comparator.comparing(MRDataSkewGraph::getDataSize)); - Map statisticsMap = getStatisticsMap(dataSkewGraphList); - dataSkewGraphList.forEach(data -> { - MRDataSkewGraph cache = statisticsMap.get(data.getTaskId()); - if (cache != null) { - data.setGraphType(cache.getGraphType()); - statisticsMap.remove(cache.getTaskId()); - } - }); - statisticsMap.forEach((k, v) -> dataSkewGraphList.add(v)); - return dataSkewGraphList; - } - - private Map getStatisticsMap(List dataSkewGraphs) { - Map statisticsMap = new HashMap<>(); - int middleIndex = dataSkewGraphs.size() / 2; - if (dataSkewGraphs.size() % 2 == 0) { - - MRDataSkewGraph pre = dataSkewGraphs.get(middleIndex - 1); - statisticsMap.put( - pre.getTaskId(), - new MRDataSkewGraph( - pre.getTaskId(), - pre.getDataSize(), - GraphType.median.toString() - ) - ); - - MRDataSkewGraph post = dataSkewGraphs.get(middleIndex); - statisticsMap.put( - post.getTaskId(), - new MRDataSkewGraph( - post.getTaskId(), - post.getDataSize(), - GraphType.median.toString() - ) - ); - - } else { - MRDataSkewGraph mediaData = dataSkewGraphs.get(middleIndex); - statisticsMap.put( - mediaData.getTaskId(), - new MRDataSkewGraph( - mediaData.getTaskId(), - mediaData.getDataSize(), - GraphType.median.toString() - ) - ); - } - MRDataSkewGraph maxGraph = dataSkewGraphs.get(dataSkewGraphs.size() - 1); - maxGraph.setGraphType(GraphType.max.toString()); - statisticsMap.put(maxGraph.getTaskId(), maxGraph); - return statisticsMap; - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRGCDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRGCDetector.java deleted file mode 100644 index e3e0c41ff..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRGCDetector.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.mr; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.constant.MRTaskType; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.GraphType; -import com.oppo.cloud.common.domain.mr.MRGCAbnormal; -import com.oppo.cloud.common.domain.mr.MRGCGraph; -import com.oppo.cloud.common.domain.mr.config.MRGCConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.mr.CounterInfo; -import com.oppo.cloud.parser.domain.mr.MRAppInfo; -import com.oppo.cloud.parser.domain.mr.MRTaskAttemptInfo; -import com.oppo.cloud.parser.service.job.detector.IDetector; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Slf4j -public class MRGCDetector implements IDetector { - - private final DetectorParam param; - - private final MRGCConfig config; - - public MRGCDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getMrGCConfig(); - } - - @Override - public DetectorResult detect() { - DetectorResult> detectorResult = - new DetectorResult<>(AppCategoryEnum.MR_GC_ABNORMAL.getCategory(), false); - - MRAppInfo mrAppInfo = param.getMrAppInfo(); - MRGCAbnormal mapGCInfo = judgeGCAbnormal(mrAppInfo.getMapList(), MRTaskType.MAP); - MRGCAbnormal reduceGCInfo = judgeGCAbnormal(mrAppInfo.getReduceList(), MRTaskType.REDUCE); - List data = new ArrayList<>(); - if (mapGCInfo.getAbnormal() || reduceGCInfo.getAbnormal()) { - detectorResult.setAbnormal(true); - data.add(mapGCInfo); - data.add(reduceGCInfo); - } - detectorResult.setData(data); - return detectorResult; - } - - private MRGCAbnormal judgeGCAbnormal(List lists, MRTaskType taskType) { - MRGCAbnormal gcInfo = new MRGCAbnormal(); - List graphList = new ArrayList<>(); - - double[] cpuTimes = new double[lists.size()]; - double[] gcTimes = new double[lists.size()]; - - for (int i = 0; i < lists.size(); i++) { - MRTaskAttemptInfo task = lists.get(i); - String group = CounterInfo.CounterGroupName.TASK_COUNTER.getCounterGroupName(); - Map metrics = task.getCounters().get(group); - long cpuMs = metrics.getOrDefault(CounterInfo.CounterName.CPU_MILLISECONDS.getCounterName(), 0L); - long gcMs = metrics.getOrDefault(CounterInfo.CounterName.GC_TIME_MILLIS.getCounterName(), 0L); - graphList.add(new MRGCGraph( - (long) task.getTaskId(), - (double) cpuMs, - (double) gcMs, - GraphType.normal.toString())); - cpuTimes[i] = cpuMs; - gcTimes[i] = gcMs; - } - - DescriptiveStatistics cpuTimeStatistics = new DescriptiveStatistics(cpuTimes); - double cpuTimeMedian = cpuTimeStatistics.getMean(); - - DescriptiveStatistics gcTimeStatistics = new DescriptiveStatistics(gcTimes); - double gcTimeMedian = gcTimeStatistics.getMean(); - - double ratio = gcTimeMedian / cpuTimeMedian; - - boolean abnormal = false; - switch (taskType) { - case MAP: - abnormal = ratio > config.getMapThreshold(); - break; - case REDUCE: - abnormal = ratio > config.getReduceThreshold(); - break; - default: - break; - } - gcInfo.setAbnormal(abnormal); - gcInfo.setCpuTimeMedian(cpuTimeMedian); - gcInfo.setGcTimeMedian(gcTimeMedian); - gcInfo.setRatio(ratio); - gcInfo.setTaskType(taskType.getName()); - gcInfo.setGraphList(graphList); - return gcInfo; - - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRLargeTableScanDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRLargeTableScanDetector.java deleted file mode 100644 index ed9c299aa..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRLargeTableScanDetector.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.mr; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.mr.MRLargeTableScanAbnormal; -import com.oppo.cloud.common.domain.mr.config.MRLargeTableScanConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.mr.CounterInfo; -import com.oppo.cloud.parser.service.job.detector.IDetector; - -import java.util.Map; - -public class MRLargeTableScanDetector implements IDetector { - - private final DetectorParam param; - - private final MRLargeTableScanConfig config; - - public MRLargeTableScanDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getMrLargeTableScanConfig(); - } - - @Override - public DetectorResult detect() { - DetectorResult detectorResult = - new DetectorResult<>(AppCategoryEnum.MR_LARGE_TABLE_SCAN.getCategory(), false); - - MRLargeTableScanAbnormal largeTableScanAbnormal = new MRLargeTableScanAbnormal(); - largeTableScanAbnormal.setAbnormal(false); - - Map group = param.getMrAppInfo().getTotalCounters().get(CounterInfo.CounterGroupName.HIVE.getCounterGroupName()); - if (group == null) { - return null; - } - Long records = group.get(CounterInfo.CounterName.RECORDS_IN.getCounterName()); - if (records != null && records > config.getThreshold()) { - largeTableScanAbnormal.setAbnormal(true); - detectorResult.setAbnormal(true); - } - largeTableScanAbnormal.setRecords(records); - detectorResult.setData(largeTableScanAbnormal); - - return detectorResult; - } - - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRMemoryWasteDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRMemoryWasteDetector.java deleted file mode 100644 index e67e08dfe..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRMemoryWasteDetector.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.mr; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.constant.MRTaskType; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.mr.MRMemWasteAbnormal; -import com.oppo.cloud.common.domain.mr.MRTaskMemPeak; -import com.oppo.cloud.common.domain.mr.config.MRMemWasteConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.mr.CounterInfo; -import com.oppo.cloud.parser.domain.mr.MRAppInfo; -import com.oppo.cloud.parser.domain.mr.MRTaskAttemptInfo; -import com.oppo.cloud.parser.service.job.detector.IDetector; -import lombok.extern.slf4j.Slf4j; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Slf4j -public class MRMemoryWasteDetector implements IDetector { - - private final DetectorParam param; - - private final MRMemWasteConfig config; - - public MRMemoryWasteDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getMrMemWasteConfig(); - } - - @Override - public DetectorResult detect() { - DetectorResult> detectorResult = - new DetectorResult<>(AppCategoryEnum.MR_MEMORY_WASTE.getCategory(), false); - - List data = new ArrayList<>(); - MRAppInfo mrAppInfo = param.getMrAppInfo(); - Map confMap = param.getMrAppInfo().getConfMap(); - - long mapMemoryMB = Long.parseLong(confMap.getOrDefault(CounterInfo.MRConfiguration.MAP_MEMORY_MB.getKey(), "0")); - long reduceMemoryMB = Long.parseLong(confMap.getOrDefault(CounterInfo.MRConfiguration.REDUCE_MEMORY_MB.getKey(), "0")); - if (mapMemoryMB == 0 || reduceMemoryMB == 0) { - log.error("appId：{},mapMemoryMB:{},reduceMemoryMB:{}", param.getAppId(), - confMap.get(CounterInfo.MRConfiguration.MAP_MEMORY_MB.getKey()), - confMap.get(CounterInfo.MRConfiguration.REDUCE_MEMORY_MB.getKey())); - return detectorResult; - } - - MRMemWasteAbnormal mapResult = calculateMemoryWaste(detectorResult, mrAppInfo.getMapList(), mapMemoryMB, MRTaskType.MAP.getName()); - MRMemWasteAbnormal reduceResult = calculateMemoryWaste(detectorResult, mrAppInfo.getReduceList(), reduceMemoryMB, MRTaskType.REDUCE.getName()); - if (detectorResult.getAbnormal()) { - if (mapResult != null) { - data.add(mapResult); - } - if (reduceResult != null) { - data.add(reduceResult); - } - } - detectorResult.setData(data); - return detectorResult; - } - - - private MRMemWasteAbnormal calculateMemoryWaste(DetectorResult detectorResult, List lists, - long allocationMB, String taskType) { - MRMemWasteAbnormal memWasteAbnormal = new MRMemWasteAbnormal(); - memWasteAbnormal.setAbnormal(false); - memWasteAbnormal.setTaskType(taskType); - memWasteAbnormal.setMemory(allocationMB); - - List peakList = new ArrayList<>(); - long memComputeSeconds = 0L; - long memAvailableSeconds = 0L; - - for (MRTaskAttemptInfo task : lists) { - String group = CounterInfo.CounterGroupName.TASK_COUNTER.getCounterGroupName(); - Map metrics = task.getCounters().get(group); - long memMBValue = metrics.get(CounterInfo.CounterName.PHYSICAL_MEMORY_BYTES.getCounterName()) / 1024 / 1024; - long elapsedSecondsValue = task.getElapsedTime() / 1000; - memComputeSeconds += memMBValue * elapsedSecondsValue; - memAvailableSeconds += allocationMB * elapsedSecondsValue; - peakList.add(new MRTaskMemPeak(task.getTaskId(), (int) memMBValue)); - } - - if (memComputeSeconds == 0) { - return null; - } - if (memAvailableSeconds < memComputeSeconds) { - log.warn("appId:{}, memAvailableSeconds less than memComputeSeconds", param.getAppId()); - return null; - } - - double wastePercent = (((double) memAvailableSeconds - memComputeSeconds) / memAvailableSeconds) * 100; - boolean abnormal; - if (MRTaskType.MAP.getName().equals(taskType)) { - abnormal = wastePercent > config.getMapThreshold(); - } else { - abnormal = wastePercent > config.getReduceThreshold(); - } - if (abnormal) { - detectorResult.setAbnormal(true); - } - memWasteAbnormal.setAbnormal(abnormal); - memWasteAbnormal.setWastePercent(wastePercent); - memWasteAbnormal.setTaskMemPeakList(peakList); - return memWasteAbnormal; - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRSpeculativeTaskDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRSpeculativeTaskDetector.java deleted file mode 100644 index 5648a5946..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRSpeculativeTaskDetector.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.mr; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.mr.MRSpeculativeAbnormal; -import com.oppo.cloud.common.domain.mr.config.MRSpeculativeTaskConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.mr.SpeculationInfo; -import com.oppo.cloud.parser.service.job.detector.IDetector; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MRSpeculativeTaskDetector implements IDetector { - - private final DetectorParam param; - - private final MRSpeculativeTaskConfig config; - - public MRSpeculativeTaskDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getMrSpeculativeTaskConfig(); - } - - @Override - public DetectorResult detect() { - DetectorResult detectorResult = - new DetectorResult<>(AppCategoryEnum.MR_SPECULATIVE_TASK.getCategory(), false); - - SpeculationInfo speculationInfo = param.getMrAppInfo().getSpeculationInfo(); - if (speculationInfo.getTaskAttemptIds().size() < config.getThreshold()) { - return null; - } - MRSpeculativeAbnormal mrSpeculativeAbnormal = new MRSpeculativeAbnormal(); - mrSpeculativeAbnormal.setAbnormal(true); - mrSpeculativeAbnormal.setElapsedTime(speculationInfo.getElapsedTime()); - mrSpeculativeAbnormal.setTaskAttemptIds(speculationInfo.getTaskAttemptIds()); - detectorResult.setAbnormal(true); - detectorResult.setData(mrSpeculativeAbnormal); - - return detectorResult; - } - - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRTaskDurationDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRTaskDurationDetector.java deleted file mode 100644 index 218e86dff..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/mr/MRTaskDurationDetector.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.mr; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.constant.MRTaskType; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.GraphType; -import com.oppo.cloud.common.domain.eventlog.TaskDurationGraph; -import com.oppo.cloud.common.domain.mr.MRTaskDurationAbnormal; -import com.oppo.cloud.common.domain.mr.config.MRTaskDurationConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.mr.MRAppInfo; -import com.oppo.cloud.parser.domain.mr.MRTaskAttemptInfo; -import com.oppo.cloud.parser.service.job.detector.IDetector; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; - -import java.util.*; - -@Slf4j -public class MRTaskDurationDetector implements IDetector { - - private final DetectorParam param; - - private final MRTaskDurationConfig config; - - public MRTaskDurationDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getMrTaskDurationConfig(); - } - - @Override - public DetectorResult detect() { - DetectorResult> detectorResult = - new DetectorResult<>(AppCategoryEnum.MR_TASK_DURATION.getCategory(), false); - - List data = new ArrayList<>(); - MRAppInfo mrAppInfo = param.getMrAppInfo(); - MRTaskDurationAbnormal mapResult = judgeTaskDuration(mrAppInfo.getMapList(), MRTaskType.MAP.getName()); - MRTaskDurationAbnormal reduceResult = judgeTaskDuration(mrAppInfo.getReduceList(), MRTaskType.REDUCE.getName()); - if (mapResult.getAbnormal() || reduceResult.getAbnormal()) { - detectorResult.setAbnormal(true); - data.add(mapResult); - data.add(reduceResult); - } - detectorResult.setData(data); - return detectorResult; - } - - private MRTaskDurationAbnormal judgeTaskDuration(List lists, String taskType) { - MRTaskDurationAbnormal taskDurationAbnormal = new MRTaskDurationAbnormal(); - taskDurationAbnormal.setAbnormal(false); - taskDurationAbnormal.setTaskType(taskType); - - List graphList = new ArrayList<>(); - - double[] elapsedTimes = new double[lists.size()]; - - for (int i = 0; i < lists.size(); i++) { - MRTaskAttemptInfo task = lists.get(i); - graphList.add(new TaskDurationGraph( - (long) task.getTaskId(), - (double) task.getElapsedTime(), - GraphType.normal.toString())); - elapsedTimes[i] = task.getElapsedTime(); - } - - DescriptiveStatistics statistics = new DescriptiveStatistics(elapsedTimes); - double median = statistics.getPercentile(50); - double max = statistics.getMax(); - double ratio = max / median; - boolean abnormal; - if (MRTaskType.MAP.getName().equals(taskType)) { - abnormal = ratio > config.getMapThreshold() && max > config.getTaskDuration(); - } else { - abnormal = ratio > config.getReduceThreshold() && max > config.getTaskDuration(); - } - taskDurationAbnormal.setAbnormal(abnormal); - if (graphList.size() > 0) { - taskDurationAbnormal.setGraphList(handleGraph(graphList)); - } - return taskDurationAbnormal; - } - - private List handleGraph(List graphList) { - graphList.sort(Comparator.comparing(TaskDurationGraph::getDuration)); - Map statisticsMap = getStatisticsMap(graphList); - graphList.forEach(data -> { - TaskDurationGraph cache = statisticsMap.get(data.getTaskId()); - if (cache != null) { - data.setGraphType(cache.getGraphType()); - statisticsMap.remove(cache.getTaskId()); - } - }); - statisticsMap.forEach((k, v) -> graphList.add(v)); - return graphList; - } - - - private Map getStatisticsMap(List graphList) { - Map statisticsMap = new HashMap<>(); - int middleIndex = graphList.size() / 2; - if (graphList.size() % 2 == 0) { - - TaskDurationGraph pre = graphList.get(middleIndex - 1); - statisticsMap.put( - pre.getTaskId(), - new TaskDurationGraph( - pre.getTaskId(), - pre.getDuration(), - GraphType.median.toString() - ) - ); - - TaskDurationGraph post = graphList.get(middleIndex); - statisticsMap.put( - post.getTaskId(), - new TaskDurationGraph( - post.getTaskId(), - post.getDuration(), - GraphType.median.toString() - ) - ); - - } else { - TaskDurationGraph mediaData = graphList.get(middleIndex); - statisticsMap.put( - mediaData.getTaskId(), - new TaskDurationGraph( - mediaData.getTaskId(), - mediaData.getDuration(), - GraphType.median.toString() - ) - ); - } - TaskDurationGraph maxGraph = graphList.get(graphList.size() - 1); - maxGraph.setGraphType(GraphType.max.toString()); - statisticsMap.put(maxGraph.getTaskId(), maxGraph); - return statisticsMap; - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/CpuWasteDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/CpuWasteDetector.java deleted file mode 100644 index 505568772..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/CpuWasteDetector.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.spark; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.domain.eventlog.CpuWasteAbnormal; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.config.CpuWasteConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.job.ExecutorTimeSpan; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkApplication; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkJob; -import com.oppo.cloud.parser.service.job.detector.IDetector; -import lombok.extern.slf4j.Slf4j; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -@Slf4j -public class CpuWasteDetector implements IDetector { - - private final DetectorParam param; - - private final CpuWasteConfig config; - - public CpuWasteDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getCpuWasteConfig(); - } - - @Override - public DetectorResult detect() { - - DetectorResult detectorResult = - new DetectorResult<>(AppCategoryEnum.CPU_WASTE.getCategory(), false); - SparkApplication application = this.param.getReplayEventLogs().getApplication(); - - long appTotalTime = application.getAppEndTimestamp() - application.getAppStartTimestamp(); - if (appTotalTime < 0) { - log.warn("appTotalTime:{},{}", this.param.getAppId(), appTotalTime); - return null; - } - - long jobTime = estimatedTimeSpentInJobs(); - - long maxExecutors = getMaxConcurrent(); - long executorCores = Long.parseLong(application.getSparkExecutorCores()); - long totalCores = executorCores * maxExecutors; - - long appComputeMillisAvailable = totalCores * appTotalTime; - long inJobComputeMillisAvailable = totalCores * jobTime; - - if (inJobComputeMillisAvailable == 0) { - return null; - } - - long inJobComputeMillisUsed = getInJobComputeMillisUsed(); - - long driverTimeJobBased = appTotalTime - jobTime; - long driverComputeMillisWastedJobBased = driverTimeJobBased * totalCores; - - float driverWastedPercentOverAll = - ((float) driverComputeMillisWastedJobBased / appComputeMillisAvailable) * 100; - - float executorWastedPercentOverAll = - (((float) inJobComputeMillisAvailable - inJobComputeMillisUsed) / appComputeMillisAvailable) * 100; - - float driverThreshold = this.config.getDriverThreshold(); - float executorThreshold = this.config.getExecutorThreshold(); - CpuWasteAbnormal cpuWasteAbnormal = new CpuWasteAbnormal(); - cpuWasteAbnormal.setAppComputeMillisAvailable(appComputeMillisAvailable); - cpuWasteAbnormal.setInJobComputeMillisAvailable(inJobComputeMillisAvailable); - cpuWasteAbnormal.setInJobComputeMillisUsed(inJobComputeMillisUsed); - cpuWasteAbnormal.setDriverWastedPercentOverAll(driverWastedPercentOverAll); - cpuWasteAbnormal.setExecutorWastedPercentOverAll(executorWastedPercentOverAll); - - cpuWasteAbnormal.setMaxExecutors(maxExecutors); - cpuWasteAbnormal.setExecutorCores(executorCores); - - if (driverWastedPercentOverAll < 0 || executorWastedPercentOverAll < 0) { - log.error("CpuWasteDetectErr:{},{}", this.param.getAppId(), cpuWasteAbnormal); - return null; - } - if ((executorWastedPercentOverAll > executorThreshold || driverWastedPercentOverAll > driverThreshold) && - this.param.getAppDuration() > this.config.getDuration()) { - detectorResult.setAbnormal(true); - } - detectorResult.setData(cpuWasteAbnormal); - return detectorResult; - } - - private long estimatedTimeSpentInJobs() { - List lists = new ArrayList<>(); - for (Map.Entry jobs : this.param.getReplayEventLogs().getJobs().entrySet()) { - lists.add(jobs.getValue()); - } - if (lists.size() == 0) { - return 0; - } - lists.sort(Comparator.comparing(SparkJob::getSubmissionTime)); - // job累计时间 - long jobAcc = 0; - SparkJob cur = lists.get(0); - for (int i = 1; i < lists.size(); i++) { - SparkJob jobInfo = lists.get(i); - if (cur.getEndTime() <= jobInfo.getSubmissionTime()) { - jobAcc += cur.getEndTime() - cur.getSubmissionTime(); - } - if (jobInfo.getEndTime() > cur.getEndTime()) { - cur = jobInfo; - } - } - if (cur.getEndTime() == null) { - return jobAcc; - } - jobAcc += cur.getEndTime() - cur.getSubmissionTime(); - return jobAcc; - } - - private long getMaxConcurrent() { - List executorTimeSpans = new ArrayList<>(); - this.param.getReplayEventLogs().getExecutors().values().forEach(sparkExecutor -> { - long correctedEndTime; - if (sparkExecutor.getRemoveTimestamp() == 0) { - if (this.param.getReplayEventLogs().getApplication().getAppEndTimestamp() == 0) { - correctedEndTime = System.currentTimeMillis(); - } else { - correctedEndTime = this.param.getReplayEventLogs().getApplication().getAppEndTimestamp(); - } - } else { - correctedEndTime = sparkExecutor.getRemoveTimestamp(); - } - executorTimeSpans.add(new ExecutorTimeSpan(sparkExecutor.getStartTimestamp(), 1L)); - executorTimeSpans.add(new ExecutorTimeSpan(correctedEndTime, -1L)); - }); - executorTimeSpans.sort((o1, o2) -> { - if (o1.getStartTime().equals(o2.getStartTime())) { - return Integer.compare(o1.getEndTime().compareTo(o2.getEndTime()), 0); - } else { - return o1.getStartTime().compareTo(o2.getStartTime()); - } - }); - long count = 0L; - long maxConcurrent = 0L; - - for (ExecutorTimeSpan span : executorTimeSpans) { - count += span.getEndTime(); - maxConcurrent = Math.max(maxConcurrent, count); - } - - return Math.max(maxConcurrent, 1); - } - - private long getInJobComputeMillisUsed() { - long sum = 0L; - for (Map.Entry entry : this.param.getReplayEventLogs().getJobs().entrySet()) { - sum += entry.getValue().getExecutorRunTime(); - } - return sum; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/DataSkewDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/DataSkewDetector.java deleted file mode 100644 index 4d55fd477..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/DataSkewDetector.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.spark; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.domain.eventlog.DataSkewAbnormal; -import com.oppo.cloud.common.domain.eventlog.DataSkewGraph; -import com.oppo.cloud.common.domain.eventlog.GraphType; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.config.DataSkewConfig; -import com.oppo.cloud.common.domain.eventlog.config.MedianInterval; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkJob; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkStage; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkTask; -import com.oppo.cloud.parser.service.job.detector.IDetector; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; - -import java.util.*; - -/** - * Data Skew Detector - */ -@Slf4j -public class DataSkewDetector implements IDetector { - - private final DetectorParam param; - - private final DataSkewConfig config; - - public DataSkewDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getDataSkewConfig(); - } - - @Override - public DetectorResult detect() { - - DetectorResult> detectorResult = - new DetectorResult<>(AppCategoryEnum.DATA_SKEW.getCategory(), false); - - Map jobs = this.param.getReplayEventLogs().getJobs(); - List dataSkewTasks = new ArrayList<>(); - for (SparkJob job : jobs.values()) { - processingStages(job, dataSkewTasks, detectorResult); - } - - detectorResult.setData(dataSkewTasks); - - return detectorResult; - } - - public void processingStages(SparkJob job, List dataSkewTasks, DetectorResult detectorResult) { - - for (SparkStage stage : job.getStages()) { - DataSkewAbnormal taskDataSkew = new DataSkewAbnormal(); - taskDataSkew.setJobId(job.getJobId()); - taskDataSkew.setStageId(stage.getStageId()); - taskDataSkew.setAbnormal(false); - - if (stage.getSubmissionTimeMap().size() == 0 || stage.getCompleteTimeMap().size() == 0) { - continue; - } - // for stages with retries, the submissionTime and launchTime of different retries should be distinguished. - for (Map.Entry entry : stage.getSubmissionTimeMap().entrySet()) { - Integer attemptId = entry.getKey(); - Long submissionTime = entry.getValue(); - Long completeTime = stage.getCompleteTimeMap().get(attemptId); - if (submissionTime == null || completeTime == null) { - continue; - } - List tasks = stage.getTasksMap().get(attemptId); - if (tasks == null) { - continue; - } - List dataSkewGraphs = new ArrayList<>(); - Long maxShuffleReadBytes = 0L; - Long firstLaunchTime = 0L; - double[] totalRecordsList = new double[tasks.size()]; - for (int i = 0; i < tasks.size(); i++) { - SparkTask task = tasks.get(i); - if (firstLaunchTime == 0 || task.getLaunchTime() < firstLaunchTime) { - firstLaunchTime = task.getLaunchTime(); - } - if (task.getTotalShuffleReadBytes() > maxShuffleReadBytes) { - maxShuffleReadBytes = task.getTotalShuffleReadBytes(); - } - dataSkewGraphs.add(new DataSkewGraph(task.getTaskId(), task.getTotalShuffleReadBytes(), - task.getTotalRecordsRead(), GraphType.normal.toString())); - totalRecordsList[i] = task.getTotalRecordsRead(); - - } - if (dataSkewGraphs.size() == 0) { - continue; - } - - if (maxShuffleReadBytes == 0) { - continue; - } - float stageDuration = completeTime - firstLaunchTime; - - float percentage = (stageDuration / this.param.getAppDuration()) * 100; - - DescriptiveStatistics statistics = new DescriptiveStatistics(totalRecordsList); - double median = statistics.getPercentile(50); - double max = statistics.getMax(); - - taskDataSkew.setDuration(stageDuration); - taskDataSkew.setAttemptNumber(attemptId); - taskDataSkew.setMaxShuffleReadBytes(maxShuffleReadBytes); - taskDataSkew.setMaxShuffleReadRecords((long) max); - taskDataSkew.setMedianRecords((long) median); - taskDataSkew.setRatio(max / median); - - boolean dataSkew = judgeDataSkew(taskDataSkew, max, median, percentage); - taskDataSkew.setAbnormal(dataSkew); - if (dataSkew) { - detectorResult.setAbnormal(true); - dataSkewGraphs.sort(Comparator.comparing(DataSkewGraph::getTotalRecordsRead)); - // get statistics data - Map statisticsMap = getStatisticsMap(dataSkewGraphs); - - if (dataSkewGraphs.size() <= 30) { - taskDataSkew.setDataSkewGraphs(dataSkewGraphs); - } else { - taskDataSkew.setDataSkewGraphs( - dataSkewGraphs.subList(dataSkewGraphs.size() - 30, dataSkewGraphs.size())); - } - - taskDataSkew.getDataSkewGraphs().forEach(data -> { - DataSkewGraph cache = statisticsMap.get(data.getTaskId()); - if (cache != null) { - // type replacement - data.setGraphType(cache.getGraphType()); - statisticsMap.remove(cache.getTaskId()); - } - }); - statisticsMap.forEach((k, v) -> taskDataSkew.getDataSkewGraphs().add(v)); - - } - dataSkewTasks.add(taskDataSkew); - } - } - } - - /** - * Data skew judgment - */ - private boolean judgeDataSkew(DataSkewAbnormal taskDataSkew, double max, double median, float percentage) { - double threshold = 0; - if (median > 0) { - Float multiple = getThreshold(median); - if (multiple != 0) { - threshold = multiple * median; - } - } - taskDataSkew.setThreshold(threshold); - - if (threshold == 0) { - return false; - } - // Stage duration / total task duration ratio less than the threshold stage or app duration < configDuration - if (percentage < this.config.getStageDurationPercentage() - || this.param.getAppDuration() < this.config.getDuration()) { - return false; - } - return max > threshold; - } - - /** - * Get the threshold - */ - private Float getThreshold(double size) { - for (MedianInterval interval : this.config.getInterval()) { - if (interval.getEnd() == 0 && size > interval.getStart()) { - return interval.getThreshold(); - } - if (size > interval.getStart() && size <= interval.getEnd()) { - return interval.getThreshold(); - } - } - return 0F; - } - - /** - * Statistical value processing - */ - private Map getStatisticsMap(List dataSkewGraphs) { - Map statisticsMap = new HashMap<>(); - int middleIndex = dataSkewGraphs.size() / 2; - if (dataSkewGraphs.size() % 2 == 0) { - - DataSkewGraph pre = dataSkewGraphs.get(middleIndex - 1); - statisticsMap.put(pre.getTaskId(), new DataSkewGraph(pre.getTaskId(), pre.getTotalShuffleReadBytes(), - pre.getTotalRecordsRead(), GraphType.median.toString())); - - DataSkewGraph post = dataSkewGraphs.get(middleIndex); - statisticsMap.put(post.getTaskId(), new DataSkewGraph(post.getTaskId(), post.getTotalShuffleReadBytes(), - post.getTotalRecordsRead(), GraphType.median.toString())); - - } else { - DataSkewGraph mediaData = dataSkewGraphs.get(middleIndex); - statisticsMap.put(mediaData.getTaskId(), - new DataSkewGraph(mediaData.getTaskId(), mediaData.getTotalShuffleReadBytes(), - mediaData.getTotalRecordsRead(), GraphType.median.toString())); - } - DataSkewGraph maxGraph = dataSkewGraphs.get(dataSkewGraphs.size() - 1); - maxGraph.setGraphType(GraphType.max.toString()); - statisticsMap.put(maxGraph.getTaskId(), maxGraph); - return statisticsMap; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/GlobalSortDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/GlobalSortDetector.java deleted file mode 100644 index d92266e45..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/GlobalSortDetector.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.spark; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.GlobalSortAbnormal; -import com.oppo.cloud.common.domain.eventlog.config.GlobalSortConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkJob; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkStage; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkTask; -import com.oppo.cloud.parser.service.job.detector.IDetector; -import lombok.extern.slf4j.Slf4j; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Slf4j -public class GlobalSortDetector implements IDetector { - - private final DetectorParam param; - - private final GlobalSortConfig config; - - public GlobalSortDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getGlobalSortConfig(); - } - - @Override - public DetectorResult detect() { - DetectorResult> detectorResult = - new DetectorResult<>(AppCategoryEnum.GLOBAL_SORT.getCategory(), false); - - List globalSortAbnormalList = new ArrayList<>(); - for (Map.Entry job : this.param.getReplayEventLogs().getJobs().entrySet()) { - List stages = job.getValue().getStages(); - for (SparkStage stage : stages) { - for (Map.Entry entry : stage.getSubmissionTimeMap().entrySet()) { - Integer attemptId = entry.getKey(); - List taskList = stage.getTasksMap().get(attemptId); - if (taskList == null) { - continue; - } - if (taskList.size() != this.param.getConfig().getGlobalSortConfig().getTaskCount()) { - continue; - } - SparkTask sparkTask = taskList.get(0); - long duration = sparkTask.getFinishTime() - sparkTask.getLaunchTime(); - - if (sparkTask.getTotalRecordsRead() > this.config.getRecords() - && duration > this.config.getDuration()) { - detectorResult.setAbnormal(true); - globalSortAbnormalList.add(new GlobalSortAbnormal( - job.getKey(), stage.getStageId(), attemptId, sparkTask.getTaskId(), - sparkTask.getTotalRecordsRead(), duration, true)); - } - } - } - } - - if (detectorResult.getAbnormal()) { - detectorResult.setData(globalSortAbnormalList); - return detectorResult; - } - - return null; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/HdfsStuckDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/HdfsStuckDetector.java deleted file mode 100644 index 6ba748aab..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/HdfsStuckDetector.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.spark; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.domain.eventlog.*; -import com.oppo.cloud.common.domain.eventlog.config.HdfsStuckConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkJob; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkStage; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkTask; -import com.oppo.cloud.parser.service.job.detector.IDetector; -import lombok.extern.slf4j.Slf4j; - -import java.util.*; - -@Slf4j -public class HdfsStuckDetector implements IDetector { - - private final DetectorParam param; - - private final HdfsStuckConfig config; - - public HdfsStuckDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getHdfsStuckConfig(); - } - - @Override - public DetectorResult detect() { - DetectorResult> detectorResult = - new DetectorResult<>(AppCategoryEnum.HDFS_STUCK.getCategory(), false); - - List hdfsStuckAbnormalList = new ArrayList<>(); - for (Map.Entry job : this.param.getReplayEventLogs().getJobs().entrySet()) { - List stages = job.getValue().getStages(); - for (SparkStage stage : stages) { - for (Map.Entry entry : stage.getSubmissionTimeMap().entrySet()) { - Integer attemptId = entry.getKey(); - if (stage.getFailed().get(attemptId) != null) { - continue; - } - List tasks = stage.getTasksMap().get(attemptId); - List hdfsStuckGraphs = new ArrayList<>(); - for (SparkTask task : tasks) { - if (task.getBytesRead() == 0) { - continue; - } - double duration = (double) (task.getFinishTime() - task.getLaunchTime()); - // Task data read volume (MB) / time distribution (s) - double taskSize = task.getBytesRead() / 1024.0 / 1024.0; - - if (taskSize < this.config.getTaskSize() || duration < this.config.getDuration()) { - continue; - } - double percent = taskSize / (duration / 1000); - hdfsStuckGraphs.add(new HdfsStuckGraph(task.getTaskId(), percent, GraphType.normal.toString())); - } - if (hdfsStuckGraphs.size() == 0) { - continue; - } - - judge(hdfsStuckGraphs, hdfsStuckAbnormalList, detectorResult, job.getKey(), stage.getStageId(), - attemptId); - - } - } - } - - if (hdfsStuckAbnormalList.size() == 0) { - return null; - } - detectorResult.setData(hdfsStuckAbnormalList); - return detectorResult; - } - - // Threshold judgment - private void judge(List hdfsSlowGraphList, - List hdfsSlowAbnormalList, - DetectorResult detectorResult, - Integer jobId, - Integer stageId, - Integer attemptId) { - - hdfsSlowGraphList.sort(Comparator.comparing(HdfsStuckGraph::getPercent)); - - double median; - int middleIndex = hdfsSlowGraphList.size() / 2; - - Map statisticsMap = new HashMap<>(); - if (hdfsSlowGraphList.size() % 2 == 0) { - - HdfsStuckGraph pre = hdfsSlowGraphList.get(middleIndex - 1); - statisticsMap.put( - pre.getTaskId(), - new HdfsStuckGraph(pre.getTaskId(), pre.getPercent(), GraphType.median.toString())); - - HdfsStuckGraph post = hdfsSlowGraphList.get(middleIndex); - statisticsMap.put( - post.getTaskId(), - new HdfsStuckGraph(post.getTaskId(), post.getPercent(), GraphType.median.toString())); - - median = (pre.getPercent() + post.getPercent()) / 2; - } else { - HdfsStuckGraph mediaData = hdfsSlowGraphList.get(middleIndex); - statisticsMap.put( - mediaData.getTaskId(), - new HdfsStuckGraph(mediaData.getTaskId(), mediaData.getPercent(), GraphType.median.toString())); - median = mediaData.getPercent(); - } - - HdfsStuckGraph minGraph = hdfsSlowGraphList.get(0); - minGraph.setGraphType(GraphType.min.toString()); - double min = minGraph.getPercent(); - statisticsMap.put(minGraph.getTaskId(), minGraph); - - double ratio = 0.0; - if (min != 0) { - ratio = median / min; - } - double threshold = this.config.getThreshold(); - if (ratio > threshold) { - List graphs; - if (hdfsSlowGraphList.size() <= 30) { - graphs = hdfsSlowGraphList; - } else { - graphs = hdfsSlowGraphList.subList(hdfsSlowGraphList.size() - 30, - hdfsSlowGraphList.size()); - } - // Statistical type processing - graphs.forEach(data -> { - HdfsStuckGraph cache = statisticsMap.get(data.getTaskId()); - if (cache != null) { - // Type replacement - data.setGraphType(cache.getGraphType()); - statisticsMap.remove(cache.getTaskId()); - } - }); - - statisticsMap.forEach((k, v) -> graphs.add(v)); - - hdfsSlowAbnormalList.add( - new HdfsStuckAbnormal(jobId, stageId, attemptId, ratio, median, min, threshold, graphs, true)); - - detectorResult.setAbnormal(true); - - } else { - hdfsSlowAbnormalList.add( - new HdfsStuckAbnormal(jobId, stageId, attemptId, ratio, median, min, threshold, null, false)); - } - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/JobDurationDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/JobDurationDetector.java deleted file mode 100644 index c38f36b6f..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/JobDurationDetector.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.spark; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.JobDurationAbnormal; -import com.oppo.cloud.common.domain.eventlog.config.JobDurationConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.job.StageDetectionInfo; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkJob; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkStage; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkTask; -import com.oppo.cloud.parser.service.job.detector.IDetector; -import lombok.extern.slf4j.Slf4j; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -@Slf4j -public class JobDurationDetector implements IDetector { - - private final DetectorParam param; - - private final JobDurationConfig config; - - public JobDurationDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getJobDurationConfig(); - } - - @Override - public DetectorResult detect() { - DetectorResult> detectorResult = - new DetectorResult<>(AppCategoryEnum.JOB_DURATION.getCategory(), false); - List jobLists = new ArrayList<>(); - for (Map.Entry job : param.getReplayEventLogs().getJobs().entrySet()) { - long jobStartTime = job.getValue().getSubmissionTime(); - if (job.getValue().getEndTime() == null) { - continue; - } - long jobEndTime = job.getValue().getEndTime(); - long jobDuration = jobEndTime - jobStartTime; - - List stages = job.getValue().getStages(); - - List lists = new ArrayList<>(); - for (SparkStage stage : stages) { - - for (Map.Entry entry : stage.getSubmissionTimeMap().entrySet()) { - Integer attemptId = entry.getKey(); - List tasks = stage.getTasksMap().get(attemptId); - if (tasks == null) { - continue; - } - Long firstLaunchTime = 0L; - for (SparkTask task : tasks) { - if (firstLaunchTime == 0 || task.getLaunchTime() < firstLaunchTime) { - firstLaunchTime = task.getLaunchTime(); - } - } - Long submissionTime = entry.getValue(); - Long completeTime = stage.getCompleteTimeMap().get(attemptId); - lists.add(new StageDetectionInfo(stage.getStageId(), attemptId, firstLaunchTime, submissionTime, - completeTime)); - } - } - - if (lists.size() == 0) { - continue; - } - // Sorted by firstLaunchTime - lists.sort(Comparator.comparing(StageDetectionInfo::getFirstLaunchTime)); - - // Cumulative stage time - long stageAcc = 0; - StageDetectionInfo cur = lists.get(0); - long start = cur.getFirstLaunchTime(); - for (int i = 1; i < lists.size(); i++) { - StageDetectionInfo stageInfo = lists.get(i); - if (cur.getCompleteTime() <= stageInfo.getFirstLaunchTime()) { - stageAcc += cur.getCompleteTime() - start; - start = stageInfo.getFirstLaunchTime(); - } - if (stageInfo.getCompleteTime() > cur.getCompleteTime()) { - cur = stageInfo; - } - } - stageAcc += cur.getCompleteTime() - start; - if (stageAcc > jobDuration) { - log.warn("stageAcc more than jobDuration:{},{},{},{}", this.param.getAppId(), job.getKey(), jobDuration, - stageAcc); - stageAcc = jobDuration; - } - double threshold = this.config.getThreshold(); - long duration = this.config.getDuration(); - double ratio = ((double) (jobDuration - stageAcc) / jobDuration) * 100; - if (ratio > threshold && jobDuration > duration) { - // Abnormal duration - jobLists.add( - new JobDurationAbnormal(job.getKey(), jobDuration, stageAcc, ratio, threshold, duration, true)); - detectorResult.setAbnormal(true); - } else { - jobLists.add(new JobDurationAbnormal(job.getKey(), jobDuration, stageAcc, ratio, threshold, duration, - false)); - } - } - if (jobLists.size() == 0) { - return null; - } - detectorResult.setData(jobLists); - return detectorResult; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/LargeTableScanDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/LargeTableScanDetector.java deleted file mode 100644 index 943acad55..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/LargeTableScanDetector.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.spark; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.LargeTableScanAbnormal; -import com.oppo.cloud.common.domain.eventlog.config.LargeTableScanConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.spark.eventlog.AccumulableInfo; -import com.oppo.cloud.parser.domain.spark.eventlog.SQLMetricInfo; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkListenerSQLExecutionStart; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkPlanInfo; -import com.oppo.cloud.parser.service.job.detector.IDetector; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class LargeTableScanDetector implements IDetector { - - private final DetectorParam param; - - private final LargeTableScanConfig config; - - public LargeTableScanDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getLargeTableScanConfig(); - } - - @Override - public DetectorResult detect() { - DetectorResult detectorResult = - new DetectorResult<>(AppCategoryEnum.LARGE_TABLE_SCAN.getCategory(), false); - - LargeTableScanAbnormal largeTableScanAbnormal = new LargeTableScanAbnormal(); - largeTableScanAbnormal.setAbnormal(false); - - Map accumulableInfoMap = this.param.getReplayEventLogs().getAccumulableInfoMap(); - List sqlExecutionStarts = - this.param.getReplayEventLogs().getSqlExecutionStarts(); - List> tables = new ArrayList<>(); - - for (SparkListenerSQLExecutionStart sqlExecutionStart : sqlExecutionStarts) { - SparkPlanInfo node = sqlExecutionStart.getSparkPlanInfo(); - List queue = new ArrayList<>(); - queue.add(node); - while (queue.size() != 0) { - node = queue.remove(0); - if (node.getNodeName().startsWith("Scan")) { - if (node.getMetrics() != null) { - for (SQLMetricInfo info : node.getMetrics()) { - AccumulableInfo accumulableInfo = accumulableInfoMap.get(info.getAccumulatorId()); - if (accumulableInfo == null) { - continue; - } - - if (accumulableInfo.getName().equals("number of output rows")) { - Long value = Long.valueOf(accumulableInfo.getValue().toString()); - - Map m = new HashMap<>(); - m.put("table", getTableName(node.getNodeName())); - m.put("rows", value); - m.put("threshold", this.config.getScanTable()); - m.put("abnormal", "false"); - - if (value >= this.config.getScanTable() - && this.param.getAppDuration() > this.config.getDuration()) { - largeTableScanAbnormal.setAbnormal(true); - detectorResult.setAbnormal(true); - m.put("abnormal", "true"); - } - tables.add(m); - - } - } - } - } - queue.addAll(node.getChildren()); - } - } - largeTableScanAbnormal.setTables(tables); - detectorResult.setData(largeTableScanAbnormal); - return detectorResult; - } - - public String getTableName(String nodeName) { - if (nodeName == null || nodeName.isEmpty()) { - return ""; - } - String[] fields = nodeName.split(" "); - if (fields.length > 0) { - return fields[fields.length - 1]; - } - return ""; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/MemWasteDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/MemWasteDetector.java deleted file mode 100644 index 07e3327c1..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/MemWasteDetector.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.spark; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.MemWasteAbnormal; -import com.oppo.cloud.common.domain.eventlog.config.MemWasteConfig; -import com.oppo.cloud.common.domain.gc.ExecutorPeakMemory; -import com.oppo.cloud.common.domain.gc.GCReport; -import com.oppo.cloud.parser.domain.job.MemoryCalculateParam; -import lombok.extern.slf4j.Slf4j; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -@Slf4j -public class MemWasteDetector { - - private final MemWasteConfig config; - - public MemWasteDetector(MemWasteConfig config) { - this.config = config; - } - - public DetectorResult detect(List gcReports, MemoryCalculateParam memoryCalculateParam) { - DetectorResult detectorResult = - new DetectorResult<>(AppCategoryEnum.MEMORY_WASTE.getCategory(), false); - long appTotalTime = memoryCalculateParam.getAppTotalTime(); - MemWasteAbnormal memWastedAbnormal = new MemWasteAbnormal(); - memWastedAbnormal.setAbnormal(false); - memWastedAbnormal.setTotalTime(appTotalTime); - memWastedAbnormal.setDriverMemory(memoryCalculateParam.getDriverMemory()); - memWastedAbnormal.setExecutorMemory(memoryCalculateParam.getExecutorMemory()); - - Map executorRuntimeMap = memoryCalculateParam.getExecutorRuntimeMap(); - - long mb = 1024 * 1024; - long totalMemoryTime = memWastedAbnormal.getDriverMemory() / mb * appTotalTime; - - long totalMemoryTimeComputeMemoryTime = 0L; - for (GCReport gcReport : gcReports) { - if (gcReport.getExecutorId() == 0) { - totalMemoryTimeComputeMemoryTime += - gcReport.getMaxHeapUsedSize() / 1024 * memWastedAbnormal.getTotalTime(); - } else { - Long executorTime = executorRuntimeMap.get(String.valueOf(gcReport.getExecutorId())); - if (executorTime != null) { - totalMemoryTimeComputeMemoryTime += gcReport.getMaxHeapUsedSize() / 1024 * executorTime; - totalMemoryTime += memWastedAbnormal.getExecutorMemory() / mb * executorTime; - } else { - log.error("cannotGetExecutorTime:{},{}", gcReport.getApplicationId(), gcReport.getExecutorId()); - } - } - } - memWastedAbnormal.setTotalMemoryTime(totalMemoryTime); - memWastedAbnormal.setTotalMemoryComputeTime(totalMemoryTimeComputeMemoryTime); - float wastePercent = ((float) (totalMemoryTime - totalMemoryTimeComputeMemoryTime) / totalMemoryTime) * 100; - memWastedAbnormal.setWastePercent(wastePercent); - if (wastePercent > this.config.getThreshold() - && memoryCalculateParam.getAppTotalTime() > this.config.getDuration()) { - memWastedAbnormal.setAbnormal(true); - detectorResult.setAbnormal(true); - } - memWastedAbnormal.setThreshold(this.config.getThreshold()); - - gcReports.sort(Comparator.comparing(GCReport::getMaxHeapUsedSize)); - if (gcReports.size() > 10) { - List results = new ArrayList<>(); - GCReport driverGc = gcReports.stream().filter(gc -> gc.getExecutorId() == 0).findFirst().orElse(null); - List executorGcs = gcReports.subList(gcReports.size() - 9, gcReports.size()); - if (driverGc != null) { - results.add(driverGc); - } - results.addAll(executorGcs); - memWastedAbnormal.setGcReportList(results); - } else { - memWastedAbnormal.setGcReportList(gcReports); - } - - List executorPeakMemoryList = new ArrayList<>(); - for (GCReport gcReport : gcReports) { - executorPeakMemoryList.add(new ExecutorPeakMemory(gcReport.getExecutorId(), gcReport.getMaxHeapUsedSize(), - gcReport.getLogPath())); - } - - memWastedAbnormal.setExecutorPeakMemoryList(executorPeakMemoryList); - detectorResult.setData(memWastedAbnormal); - return detectorResult; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/OOMWarnDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/OOMWarnDetector.java deleted file mode 100644 index 5cacc31a1..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/OOMWarnDetector.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.spark; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.OOMAbnormal; -import com.oppo.cloud.common.domain.eventlog.OOMTableInfo; -import com.oppo.cloud.common.domain.eventlog.config.OOMWarnConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.spark.eventlog.*; -import com.oppo.cloud.parser.service.job.detector.IDetector; -import lombok.extern.slf4j.Slf4j; - -import java.util.*; - -@Slf4j -public class OOMWarnDetector implements IDetector { - - private final DetectorParam param; - - private final OOMWarnConfig config; - - public OOMWarnDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getOomWarnConfig(); - } - - @Override - public DetectorResult detect() { - Map accumulableInfoMap = this.param.getReplayEventLogs().getAccumulableInfoMap(); - Map driverUpdateMap = this.param.getReplayEventLogs().getDriverUpdateMap(); - List sqlExecutionStarts = - this.param.getReplayEventLogs().getSqlExecutionStarts(); - - long maxMemory = 0L; - long maxRows = 0L; - StringBuilder scanNodeTable = new StringBuilder(); - HashSet broadcastExchangeSet = new HashSet<>(); - List tables = new ArrayList<>(); - - for (SparkListenerSQLExecutionStart sqlExecutionStart : sqlExecutionStarts) { - if (sqlExecutionStart.getDescription().startsWith("collectAsList")) { - continue; - } - SparkPlanInfo node = sqlExecutionStart.getSparkPlanInfo(); - List queue = new ArrayList<>(); - queue.add(node); - long sumMemory = 0L; - long nodeMaxRows = 0L; - while (queue.size() != 0) { - node = queue.remove(0); - if (node.getNodeName().startsWith("BroadcastExchange")) { - DetectResult detectResult = new DetectResult(); - - long rows = 0L; - if (detectBroadcastOOM(node, detectResult)) { - AccumulableInfo accumulableInfo = accumulableInfoMap.getOrDefault(detectResult.getMaxResult(), - null); - if (accumulableInfo != null) { - rows = Long.parseLong(accumulableInfo.getValue().toString()); - nodeMaxRows = Math.max(rows, nodeMaxRows); - } - } - - if (node.getMetrics() != null) { - for (SQLMetricInfo info : node.getMetrics()) { - if (!info.getNodeName().startsWith("data size")) { - continue; - } - if (broadcastExchangeSet.contains(info.getAccumulatorId())) { - continue; - } - broadcastExchangeSet.add(info.getAccumulatorId()); - Long val = driverUpdateMap.get(info.getAccumulatorId()); - if (val == null) { - continue; - } - sumMemory += val; - String table = detectResult.getScanNodeTable(); - if (rows > 0) { - OOMTableInfo oomTableInfo = new OOMTableInfo(); - oomTableInfo.setTable(table); - oomTableInfo.setRows(rows); - oomTableInfo.setMemory(val); - tables.add(oomTableInfo); - } - scanNodeTable.append(table).append(","); - } - } - } - queue.addAll(node.getChildren()); - } - - maxMemory = Math.max(sumMemory, maxMemory); - maxRows = Math.max(nodeMaxRows, maxRows); - } - - String scanNodeTableStr = ""; - if (scanNodeTable.length() > 1) { - scanNodeTableStr = scanNodeTable.deleteCharAt(scanNodeTable.length() - 1).toString(); - } - - return judgeOOM(maxMemory, maxRows, scanNodeTableStr, tables); - } - - public DetectorResult judgeOOM(Long sumMemory, Long maxRows, String scanNodeTable, List tables) { - DetectorResult detectorResult = new DetectorResult<>(AppCategoryEnum.OOMWarn.getCategory(), false); - - OOMAbnormal abnormal = new OOMAbnormal(); - - abnormal.setTables(tables); - - SparkApplication application = this.param.getReplayEventLogs().getApplication(); - - if (sumMemory != null) { - float driverUsePercent = sumMemory.floatValue() / application.getDriverMemory() * 100; - - int driverThresholdType = isOverThreshold(driverUsePercent, maxRows); - if (application.getDriverMemory() != 0 && driverThresholdType < 2) { - Map actionData = getActionData(driverThresholdType); - String usePercent = String.format("%.2f", driverUsePercent); - Map varsMap = getVars("driver", actionData.get("action"), - application.getDriverMemory(), application.getExecutorMemory(), - sumMemory, scanNodeTable, usePercent, maxRows); - - abnormal.setVars(varsMap); - detectorResult.setAbnormal(true); - detectorResult.setData(abnormal); - return detectorResult; - } - float executorUsePercent = sumMemory.floatValue() / application.getExecutorMemory().floatValue() * 100; - int executorThresholdType = isOverThreshold(executorUsePercent, maxRows); - if (application.getExecutorMemory() != 0 && executorThresholdType < 2) { - Map actionData = getActionData(executorThresholdType); - - String usePercent = String.format("%.2f", executorUsePercent); - - Map varsMap = getVars("executor", actionData.get("action"), - application.getDriverMemory(), application.getExecutorMemory(), - sumMemory, scanNodeTable, usePercent, maxRows); - - detectorResult.setAbnormal(true); - abnormal.setVars(varsMap); - detectorResult.setData(abnormal); - return detectorResult; - } - } - detectorResult.setData(abnormal); - return detectorResult; - } - - /** - * Get action data - */ - public Map getActionData(int thresholdType) { - Map m = new HashMap<>(); - switch (thresholdType) { - case 0: - m.put("action", "broadcastOOM"); - m.put("threshold", String.valueOf(this.config.getBroadcastRows())); - break; - case 1: - m.put("action", "oom"); - m.put("threshold", String.valueOf(this.config.getOom())); - break; - default: - break; - } - return m; - } - - /** - * Get vars data - */ - public Map getVars(String execType, String action, Long driverMemory, Long executorMemory, - Long useMemory, String scanNodeTable, String usePercent, Long maxRows) { - Map m = new HashMap<>(); - m.put("execType", execType); - m.put("action", action); - m.put("driverMemory", String.valueOf(driverMemory)); - m.put("executorMemory", String.valueOf(executorMemory)); - m.put("useMemory", String.valueOf(useMemory)); - m.put("maxRows", String.valueOf(maxRows)); - m.put("usePercent", usePercent); - m.put("scanTable", scanNodeTable); - return m; - } - - /** - * Memory usage exceeds threshold, or broadcast filter data row count exceeds threshold - */ - public int isOverThreshold(float usePercent, Long maxRows) { - // Too many broadcast filter data rows - if (((maxRows >= this.config.getBroadcastRows()) && (usePercent >= this.config.getBroadcastRowsOom())) && - this.param.getAppDuration() > this.config.getDuration()) { - return 0; - } - // Memory exceeds threshold - if (usePercent >= this.config.getOom() && this.param.getAppDuration() > this.config.getDuration()) { - return 1; - } - return 2; - } - - public boolean detectBroadcastOOM(SparkPlanInfo planInfo, DetectResult maxDetectResult) { - if (planInfo.getNodeName().startsWith("Filter")) { - if (planInfo.getMetrics() != null) { - for (SQLMetricInfo metricInfo : planInfo.getMetrics()) { - if (metricInfo.getNodeName() == null) { - continue; - } - if (metricInfo.getNodeName().equals("number of output rows")) { - maxDetectResult.setMaxResult(metricInfo.getAccumulatorId()); - } - } - } - } else if (planInfo.getNodeName().startsWith("Scan")) { - maxDetectResult.setSparkPlanInfo(planInfo); - return true; - } - for (SparkPlanInfo children : planInfo.getChildren()) { - if (detectBroadcastOOM(children, maxDetectResult)) { - return true; - } - } - return false; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/SpeculativeTaskDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/SpeculativeTaskDetector.java deleted file mode 100644 index 3bd8936cf..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/SpeculativeTaskDetector.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.spark; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.SpeculativeTaskAbnormal; -import com.oppo.cloud.common.domain.eventlog.config.SpeculativeTaskConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkJob; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkStage; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkTask; -import com.oppo.cloud.parser.service.job.detector.IDetector; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class SpeculativeTaskDetector implements IDetector { - - private DetectorParam param; - - private SpeculativeTaskConfig config; - - public SpeculativeTaskDetector(DetectorParam detectorParam) { - this.param = detectorParam; - this.config = detectorParam.getConfig().getSpeculativeTaskConfig(); - } - - @Override - public DetectorResult detect() { - DetectorResult> detectorResult = - new DetectorResult<>(AppCategoryEnum.SPECULATIVE_TASK.getCategory(), false); - - List speculativeTaskAbnormalList = new ArrayList<>(); - - for (Map.Entry job : this.param.getReplayEventLogs().getJobs().entrySet()) { - List stages = job.getValue().getStages(); - for (SparkStage stage : stages) { - for (Map.Entry entry : stage.getSubmissionTimeMap().entrySet()) { - Integer attemptId = entry.getKey(); - List tasks = stage.getTasksMap().get(attemptId); - if (tasks == null) { - continue; - } - List taskIds = new ArrayList<>(); - for (SparkTask task : tasks) { - // Speculative task - if (task.getSpeculative() != null && task.getSpeculative()) { - taskIds.add(task.getTaskId()); - } - } - if (taskIds.size() == 0) { - continue; - } - - long threshold = this.config.getThreshold(); - if (taskIds.size() > threshold && this.param.getAppDuration() > this.config.getDuration()) { - detectorResult.setAbnormal(true); - speculativeTaskAbnormalList.add(new SpeculativeTaskAbnormal(job.getKey(), stage.getStageId(), - attemptId, taskIds.size(), taskIds, true, threshold)); - } else { - speculativeTaskAbnormalList.add(new SpeculativeTaskAbnormal(job.getKey(), stage.getStageId(), - attemptId, taskIds.size(), taskIds, false, threshold)); - } - } - } - } - - if (speculativeTaskAbnormalList.size() == 0) { - return null; - } - - detectorResult.setData(speculativeTaskAbnormalList); - return detectorResult; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/StageDurationDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/StageDurationDetector.java deleted file mode 100644 index b6ebe1c33..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/StageDurationDetector.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.spark; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.StageDurationAbnormal; -import com.oppo.cloud.common.domain.eventlog.config.StageDurationConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.job.TaskDetectionInfo; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkJob; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkStage; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkTask; -import com.oppo.cloud.parser.service.job.detector.IDetector; -import lombok.extern.slf4j.Slf4j; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -@Slf4j -public class StageDurationDetector implements IDetector { - - private final DetectorParam param; - - private final StageDurationConfig config; - - public StageDurationDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getStageDurationConfig(); - } - - @Override - public DetectorResult detect() { - DetectorResult> detectorResult = - new DetectorResult<>(AppCategoryEnum.STAGE_DURATION.getCategory(), false); - - List stageList = new ArrayList<>(); - - for (Map.Entry jobs : this.param.getReplayEventLogs().getJobs().entrySet()) { - List stages = jobs.getValue().getStages(); - for (SparkStage stage : stages) { - for (Map.Entry entry : stage.getSubmissionTimeMap().entrySet()) { - Integer attemptId = entry.getKey(); - List tasks = stage.getTasksMap().get(attemptId); - if (stage.getFailed().get(attemptId) != null) { - continue; - } - List taskDetectionInfoList = new ArrayList<>(); - Long firstLaunchTime = 0L; - for (SparkTask task : tasks) { - if (firstLaunchTime == 0 || task.getLaunchTime() < firstLaunchTime) { - firstLaunchTime = task.getLaunchTime(); - } - - taskDetectionInfoList.add(new TaskDetectionInfo(jobs.getKey(), stage.getStageId(), attemptId, - task.getTaskId(), task.getLaunchTime(), task.getFinishTime())); - } - - Long completeTime = stage.getCompleteTimeMap().get(attemptId); - // refer to https://github.com/apache/spark/pull/9051 - long stageDuration = completeTime - firstLaunchTime; - - taskDetectionInfoList.sort(Comparator.comparing(TaskDetectionInfo::getFirstLaunchTime)); - // Cumulative task time - TaskDetectionInfo curTask = taskDetectionInfoList.get(0); - long start = curTask.getFirstLaunchTime(); - long taskAcc = 0; - - for (int i = 1; i < taskDetectionInfoList.size(); i++) { - TaskDetectionInfo taskInfo = taskDetectionInfoList.get(i); - if (curTask.getFinishTime() <= taskInfo.getFirstLaunchTime()) { - // finishTime is 0, speculative may have occurred - if (curTask.getFinishTime() != 0) { - taskAcc += curTask.getFinishTime() - start; - } - start = taskInfo.getFirstLaunchTime(); - } - if (taskInfo.getFinishTime() > curTask.getFinishTime()) { - curTask = taskInfo; - } - } - - taskAcc += curTask.getFinishTime() - start; - // finishTime time is 0, tasks in stage are always in running state - if (taskAcc <= 0) { - log.warn("taskAcc less than zero:{},{},{},{}", param.getAppId(), - stage.getStageId(), stageDuration, taskAcc); - taskAcc = stageDuration; - } - if (taskAcc > stageDuration) { - taskAcc = stageDuration; - } - - double ratio = ((double) (stageDuration - taskAcc) / stageDuration) * 100; - double threshold = this.config.getThreshold(); - long duration = this.config.getDuration(); - if (ratio > threshold && stageDuration > duration) { - // Abnormal duration - stageList.add(new StageDurationAbnormal(jobs.getKey(), stage.getStageId(), attemptId, - stageDuration, taskAcc, ratio, threshold, duration, true)); - detectorResult.setAbnormal(true); - } else { - stageList.add(new StageDurationAbnormal(jobs.getKey(), stage.getStageId(), attemptId, - stageDuration, taskAcc, ratio, threshold, duration, false)); - } - - } - - } - } - if (stageList.size() == 0) { - return null; - } - detectorResult.setData(stageList); - return detectorResult; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/TaskDurationDetector.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/TaskDurationDetector.java deleted file mode 100644 index 3f9dd9b37..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/detector/spark/TaskDurationDetector.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.detector.spark; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.GraphType; -import com.oppo.cloud.common.domain.eventlog.TaskDurationAbnormal; -import com.oppo.cloud.common.domain.eventlog.TaskDurationGraph; -import com.oppo.cloud.common.domain.eventlog.config.TaskDurationConfig; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkJob; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkStage; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkTask; -import com.oppo.cloud.parser.service.job.detector.IDetector; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; - -import java.util.*; - -@Slf4j -public class TaskDurationDetector implements IDetector { - - private final DetectorParam param; - - private final TaskDurationConfig config; - - public TaskDurationDetector(DetectorParam param) { - this.param = param; - this.config = param.getConfig().getTaskDurationConfig(); - } - - @Override - public DetectorResult detect() { - DetectorResult> detectorResult = - new DetectorResult<>(AppCategoryEnum.TASK_DURATION.getCategory(), false); - - List taskDurationAbnormalList = new ArrayList<>(); - for (Map.Entry job : param.getReplayEventLogs().getJobs().entrySet()) { - List stages = job.getValue().getStages(); - for (SparkStage stage : stages) { - for (Map.Entry entry : stage.getSubmissionTimeMap().entrySet()) { - Integer attemptId = entry.getKey(); - if (stage.getFailed().get(attemptId) != null) { - continue; - } - List tasks = stage.getTasksMap().get(attemptId); - double[] taskDurationList = new double[tasks.size()]; - List taskDurationGraphList = new ArrayList<>(); - for (int i = 0; i < tasks.size(); i++) { - SparkTask task = tasks.get(i); - double duration = (double) (task.getFinishTime() - task.getLaunchTime()); - taskDurationList[i] = duration; - taskDurationGraphList.add(new TaskDurationGraph( - task.getTaskId(), - duration, - GraphType.normal.toString())); - } - Arrays.sort(taskDurationList); - taskDurationGraphList.sort(Comparator.comparing(TaskDurationGraph::getDuration)); - - DescriptiveStatistics statistics = new DescriptiveStatistics(taskDurationList); - double median = statistics.getPercentile(50); - double max = statistics.getMax(); - - double threshold = this.config.getThreshold(); - Long duration = this.config.getDuration(); - - double ratio = max / median; - - TaskDurationAbnormal taskDurationAbnormal = new TaskDurationAbnormal(job.getKey(), - stage.getStageId(), attemptId, median, max, ratio, null, false); - - // Threshold judgment - if (ratio > threshold && this.param.getAppDuration() > duration) { - Map statisticsMap = getStatisticsMap(taskDurationGraphList); - List graphs; - if (taskDurationGraphList.size() <= 30) { - graphs = taskDurationGraphList; - } else { - graphs = taskDurationGraphList.subList(taskDurationGraphList.size() - 30, - taskDurationGraphList.size()); - } - taskDurationAbnormal.setAbnormal(true); - graphs.forEach(data -> { - TaskDurationGraph cache = statisticsMap.get(data.getTaskId()); - { - if (cache != null) { - // Type replacement - data.setGraphType(cache.getGraphType()); - statisticsMap.remove(cache.getTaskId()); - } - } - }); - statisticsMap.forEach((k, v) -> graphs.add(v)); - taskDurationAbnormal.setGraphs(graphs); - detectorResult.setAbnormal(true); - } - taskDurationAbnormalList.add(taskDurationAbnormal); - } - } - } - if (taskDurationAbnormalList.size() == 0) { - return null; - } - detectorResult.setData(taskDurationAbnormalList); - return detectorResult; - } - - /** - * Statistical value processing - */ - private Map getStatisticsMap(List taskDurationGraphs) { - Map statisticsMap = new HashMap<>(); - int middleIndex = taskDurationGraphs.size() / 2; - if (taskDurationGraphs.size() % 2 == 0) { - - TaskDurationGraph pre = taskDurationGraphs.get(middleIndex - 1); - statisticsMap.put( - pre.getTaskId(), - new TaskDurationGraph( - pre.getTaskId(), - pre.getDuration(), - GraphType.median.toString())); - - TaskDurationGraph post = taskDurationGraphs.get(middleIndex); - statisticsMap.put( - post.getTaskId(), - new TaskDurationGraph( - post.getTaskId(), - post.getDuration(), - GraphType.median.toString())); - - } else { - TaskDurationGraph mediaData = taskDurationGraphs.get(middleIndex); - statisticsMap.put( - mediaData.getTaskId(), - new TaskDurationGraph( - mediaData.getTaskId(), - mediaData.getDuration(), - GraphType.median.toString())); - } - TaskDurationGraph maxGraph = taskDurationGraphs.get(taskDurationGraphs.size() - 1); - maxGraph.setGraphType(GraphType.max.toString()); - statisticsMap.put(maxGraph.getTaskId(), maxGraph); - return statisticsMap; - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/oneclick/IProgressListener.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/oneclick/IProgressListener.java deleted file mode 100644 index 5a836a67b..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/oneclick/IProgressListener.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.oneclick; - -import com.oppo.cloud.common.domain.oneclick.OneClickProgress; - -/** - * one click progress listener - */ -public interface IProgressListener { - - void update(OneClickProgress oneClickProgress); - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/oneclick/ParserListenerBus.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/oneclick/ParserListenerBus.java deleted file mode 100644 index 7d5a740f2..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/oneclick/ParserListenerBus.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.oneclick; - -import com.oppo.cloud.common.domain.oneclick.OneClickProgress; - -import java.util.ArrayList; -import java.util.List; - -public abstract class ParserListenerBus { - - private List listenerList = new ArrayList<>(); - - public void addListener(IProgressListener progressListener) { - listenerList.add(progressListener); - } - - public void update(OneClickProgress oneClickProgress) { - for (IProgressListener listener : listenerList) { - listener.update(oneClickProgress); - } - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/oneclick/ProgressListener.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/oneclick/ProgressListener.java deleted file mode 100644 index ab38aedc8..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/oneclick/ProgressListener.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.oneclick; - -import com.alibaba.fastjson2.JSONObject; -import com.oppo.cloud.common.domain.oneclick.OneClickProgress; -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.common.util.spring.SpringBeanUtil; -import lombok.extern.slf4j.Slf4j; - -/** - * one click progress listener - */ -@Slf4j -public class ProgressListener implements IProgressListener { - - private final RedisService redisService; - - public ProgressListener() { - - redisService = (RedisService) SpringBeanUtil.getBean(RedisService.class); - } - - @Override - public void update(OneClickProgress oneClickProgress) { - log.info("progressListener:{},{}", oneClickProgress.getLogType(), oneClickProgress); - String key = String.format("%s:%s", oneClickProgress.getAppId(), oneClickProgress.getLogType().getName()); - try { - redisService.set(key, JSONObject.toJSONString(oneClickProgress)); - } catch (Exception e) { - log.error("Exception:", e); - } - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/CommonTextParser.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/CommonTextParser.java deleted file mode 100644 index 64eab131a..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/CommonTextParser.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.parser; - -import com.oppo.cloud.common.constant.ProgressState; -import com.oppo.cloud.common.domain.job.LogPath; -import com.oppo.cloud.common.util.textparser.*; -import com.oppo.cloud.parser.domain.job.CommonResult; -import com.oppo.cloud.parser.domain.job.ParserParam; -import com.oppo.cloud.parser.domain.reader.ReaderObject; -import com.oppo.cloud.parser.service.reader.ILogReaderFactory; -import com.oppo.cloud.parser.service.reader.IReader; -import com.oppo.cloud.parser.service.reader.LogReaderFactory; -import com.oppo.cloud.parser.service.writer.ParserResultSink; -import lombok.extern.slf4j.Slf4j; - -import java.io.BufferedReader; -import java.util.*; - -@Slf4j -public abstract class CommonTextParser extends IParser { - - private ReaderObject readerObject; - private List actions; - - private ParserResultSink parserResultSink; - - public CommonTextParser(ParserParam param, - ILogReaderFactory logReaderFactory, - List actions, - ParserResultSink parserResultSink) { - super(param, logReaderFactory); - this.actions = actions; - this.parserResultSink = parserResultSink; - } - - @Override - public CommonResult run() { - CommonResult commonResult = new CommonResult(); - commonResult.setLogType(this.param.getLogType()); - List categories = new ArrayList<>(); - updateParserProgress(ProgressState.PROCESSING, 0, 0); - String logType = this.param.getLogType().getName(); - - for (LogPath logPath : this.param.getLogPaths()) { - List readerObjects; - try { - IReader reader = getReader(logPath); - readerObjects = reader.getReaderObjects(); - } catch (Exception e) { - log.error("Exception:", e); - continue; - } - updateParserProgress(ProgressState.PROCESSING, 0, readerObjects.size()); - for (ReaderObject readerObject : readerObjects) { - Map results; - try { - results = parse(readerObject); - } catch (Exception e) { - log.error("Exception:", e); - continue; - } finally { - readerObject.close(); - } - List list = getSink().saveParserActions( - logType, readerObject.getLogPath(), this.param, results); - categories.addAll(list); - } - } - - commonResult.setResult(categories); - updateParserProgress(ProgressState.SUCCEED, 0, 0); - return commonResult; - } - - public Map parse(ReaderObject readerObject) throws Exception { - this.readerObject = readerObject; - Map rootActions = getRootAction(); - for (Map.Entry action : rootActions.entrySet()) { - ParserManager.parseChildActions(action.getValue()); - } - - return rootActions; - } - - public Map getRootAction() throws Exception { - TextParser headTextParser = new TextParser(this.actions); - BufferedReader bufferedReader = this.readerObject.getBufferedReader(); - while (true) { - String line; - try { - line = bufferedReader.readLine(); - } catch (Exception e) { - log.error("Exception:", e); - break; - } - if (line == null) { - headTextParser.close(); - break; - } - headTextParser.parse(line); - } - return headTextParser.getResults(); - } - - public List getActions() { - return this.actions; - } - - public ParserResultSink getSink() { - return this.parserResultSink; - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/IParser.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/IParser.java deleted file mode 100644 index 1b6565942..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/IParser.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.parser; - -import com.oppo.cloud.common.constant.ProgressState; -import com.oppo.cloud.common.domain.job.LogPath; -import com.oppo.cloud.common.domain.oneclick.OneClickProgress; -import com.oppo.cloud.common.domain.oneclick.ProgressInfo; -import com.oppo.cloud.parser.domain.job.CommonResult; -import com.oppo.cloud.parser.domain.job.ParserParam; -import com.oppo.cloud.parser.service.job.oneclick.ParserListenerBus; -import com.oppo.cloud.parser.service.reader.ILogReaderFactory; -import com.oppo.cloud.parser.service.reader.IReader; - -public abstract class IParser extends ParserListenerBus { - - protected ParserParam param; - - protected ILogReaderFactory logReaderFactory; - - public IParser(ParserParam param, ILogReaderFactory logReaderFactory) { - this.param = param; - this.logReaderFactory = logReaderFactory; - } - - public CommonResult run() { - return null; - } - - public IReader getReader(LogPath logPath) throws Exception { - return logReaderFactory.create(logPath); - } - - public void updateParserProgress(ProgressState state, Integer progress, Integer count) { - if (!this.param.getLogRecord().getIsOneClick()) { - return; - } - OneClickProgress oneClickProgress = new OneClickProgress(); - oneClickProgress.setAppId(this.param.getApp().getAppId()); - oneClickProgress.setLogType(this.param.getLogType()); - ProgressInfo executorProgress = new ProgressInfo(); - executorProgress.setCount(count); - executorProgress.setProgress(progress); - executorProgress.setState(state); - oneClickProgress.setProgressInfo(executorProgress); - super.update(oneClickProgress); - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/IParserFactory.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/IParserFactory.java deleted file mode 100644 index 5f4c71269..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/IParserFactory.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.parser; - -import com.oppo.cloud.common.constant.LogType; -import com.oppo.cloud.common.domain.eventlog.config.DetectorConfig; -import com.oppo.cloud.common.util.textparser.ParserAction; -import com.oppo.cloud.parser.domain.job.ParserParam; -import com.oppo.cloud.parser.service.job.oneclick.IProgressListener; -import com.oppo.cloud.parser.service.reader.ILogReaderFactory; -import com.oppo.cloud.parser.service.writer.ParserResultSink; - -import java.util.List; -import java.util.concurrent.Executor; - -public interface IParserFactory { - - DetectorConfig getDetectorConf(); - - List getParserActions(LogType logType); - - Executor getTaskExecutor(); - - List getJvmList(); - - ParserResultSink getParserResultSink(); - - ILogReaderFactory createLogReaderFactory(); - - /** - * create parser - */ - default IParser create(ParserParam parserParam, IProgressListener listener) { - IParser parser = createParserInternal(parserParam); - if (parser != null && listener != null) { - parser.addListener(listener); - } - return parser; - } - - default IParser createParserInternal(ParserParam parserParam) { - LogType logType = parserParam.getLogType(); - switch (logType) { - - case SCHEDULER: - return new SchedulerLogParser(parserParam, - createLogReaderFactory(), getParserActions(logType), getParserResultSink()); - - case SPARK_EVENT: - return new SparkEventLogParser(parserParam, - createLogReaderFactory(), getDetectorConf()); - - case SPARK_EXECUTOR: - return new SparkExecutorLogParser(parserParam, - createLogReaderFactory(), getParserActions(logType), getParserResultSink(), - getTaskExecutor(), getJvmList()); - - case MAPREDUCE_JOB_HISTORY: - return new MapReduceJobHistoryParser(parserParam, - createLogReaderFactory(), getDetectorConf()); - - case MAPREDUCE_CONTAINER: - return new MapReduceContainerLogParser(parserParam, - createLogReaderFactory(), getParserActions(logType), getParserResultSink()); - - default: - return null; - } - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/MapReduceContainerLogParser.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/MapReduceContainerLogParser.java deleted file mode 100644 index cb53f36d8..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/MapReduceContainerLogParser.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.parser; - -import com.oppo.cloud.common.util.textparser.ParserAction; -import com.oppo.cloud.parser.domain.job.ParserParam; -import com.oppo.cloud.parser.service.reader.ILogReaderFactory; -import com.oppo.cloud.parser.service.writer.ParserResultSink; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; - -@Slf4j -public class MapReduceContainerLogParser extends CommonTextParser { - - public MapReduceContainerLogParser(ParserParam param, - ILogReaderFactory logReaderFactory, - List actions, - ParserResultSink parserResultSink) { - super(param, logReaderFactory, actions, parserResultSink); - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/MapReduceJobHistoryParser.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/MapReduceJobHistoryParser.java deleted file mode 100644 index a03a363ec..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/MapReduceJobHistoryParser.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.parser; - -import com.oppo.cloud.common.constant.ApplicationType; -import com.oppo.cloud.common.constant.ProgressState; -import com.oppo.cloud.common.domain.eventlog.DetectorStorage; -import com.oppo.cloud.common.domain.eventlog.config.DetectorConfig; -import com.oppo.cloud.common.domain.job.LogPath; -import com.oppo.cloud.common.domain.mr.config.MREnvironmentConfig; -import com.oppo.cloud.parser.domain.job.CommonResult; -import com.oppo.cloud.parser.domain.job.DetectorParam; -import com.oppo.cloud.parser.domain.job.ParserParam; -import com.oppo.cloud.parser.domain.mr.MRAppInfo; -import com.oppo.cloud.parser.domain.reader.ReaderObject; -import com.oppo.cloud.parser.service.job.detector.DetectorManager; -import com.oppo.cloud.parser.service.reader.ILogReaderFactory; -import com.oppo.cloud.parser.service.reader.IReader; -import com.oppo.cloud.parser.service.reader.LogReaderFactory; -import com.oppo.cloud.parser.utils.JobHistoryUtil; -import lombok.extern.slf4j.Slf4j; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Slf4j -public class MapReduceJobHistoryParser extends IParser { - - private DetectorConfig config; - - public MapReduceJobHistoryParser(ParserParam param, - ILogReaderFactory logReaderFactory, - DetectorConfig config) { - super(param, logReaderFactory); - this.config = config; - } - - @Override - public CommonResult run() { - updateParserProgress(ProgressState.PROCESSING, 0, this.param.getLogPaths().size()); - if (this.param.getLogPaths() == null) { - return null; - } - List readerObjects = null; - for (LogPath logPath : this.param.getLogPaths()) { - List results; - try { - IReader reader = getReader(logPath); - results = reader.getReaderObjects(); - } catch (Exception e) { - log.error("Exception: ", e); - continue; - } - if (results != null && results.size() > 0) { - readerObjects = results; - break; - } - } - if (readerObjects == null) { - updateParserProgress(ProgressState.FAILED, 0, 0); - return null; - } - return parse(readerObjects); - } - - private CommonResult parse(List readerObjects) { - MRAppInfo mrAppInfo; - try { - mrAppInfo = JobHistoryUtil.parseJobHistory(readerObjects); - } catch (Exception e) { - log.error("Exception:", e); - updateParserProgress(ProgressState.FAILED, 0, 0); - return null; - } finally { - readerObjects.forEach(ReaderObject::close); - } - return detect(mrAppInfo); - } - - private CommonResult detect(MRAppInfo mrAppInfo) { - - long appDuration = mrAppInfo.getElapsedTime(); - - DetectorParam detectorParam = new DetectorParam(this.param.getLogRecord().getJobAnalysis().getFlowName(), - this.param.getLogRecord().getJobAnalysis().getProjectName(), - this.param.getLogRecord().getJobAnalysis().getTaskName(), - this.param.getLogRecord().getJobAnalysis().getExecutionDate(), - this.param.getLogRecord().getJobAnalysis().getRetryTimes(), - this.param.getApp().getAppId(), ApplicationType.MAPREDUCE, appDuration, "", config, - this.param.getLogRecord().getIsOneClick()); - detectorParam.setMrAppInfo(mrAppInfo); - - DetectorManager detectorManager = new DetectorManager(); - // run all detector - DetectorStorage detectorStorage = detectorManager.run(detectorParam); - - detectorStorage.setEnv(getMREnvironmentConfig(mrAppInfo)); - - CommonResult result = new CommonResult<>(); - result.setLogType(this.param.getLogType()); - result.setResult(detectorStorage); - - updateParserProgress(ProgressState.SUCCEED, 0, 0); - return result; - } - - private Map getMREnvironmentConfig(MRAppInfo mrAppInfo) { - Map env = new HashMap<>(); - MREnvironmentConfig envConfig = config.getMrEnvironmentConfig(); - if (envConfig != null && envConfig.getKeys() != null) { - for (String key : envConfig.getKeys()) { - env.put(key, mrAppInfo.getConfMap().get(key)); - } - } - return env; - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/ParserFactory.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/ParserFactory.java deleted file mode 100644 index 6dc25cdb5..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/ParserFactory.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.parser; - -import com.oppo.cloud.common.constant.LogType; -import com.oppo.cloud.common.domain.eventlog.config.DetectorConfig; -import com.oppo.cloud.common.util.spring.SpringBeanUtil; -import com.oppo.cloud.common.util.textparser.ParserAction; -import com.oppo.cloud.parser.config.CustomConfig; -import com.oppo.cloud.parser.config.DiagnosisConfig; -import com.oppo.cloud.parser.config.ThreadPoolConfig; -import com.oppo.cloud.parser.service.reader.ILogReaderFactory; -import com.oppo.cloud.parser.service.reader.LogReaderFactory; -import com.oppo.cloud.parser.service.rules.JobRulesConfigService; -import com.oppo.cloud.parser.service.writer.OpenSearchWriter; -import com.oppo.cloud.parser.service.writer.ParserResultSink; - -import java.util.List; -import java.util.concurrent.Executor; - - -public class ParserFactory implements IParserFactory { - - @Override - public DetectorConfig getDetectorConf() { - return ((JobRulesConfigService) SpringBeanUtil.getBean( - JobRulesConfigService.class)).detectorConfig; - } - - @Override - public List getParserActions(LogType logType) { - return DiagnosisConfig.getInstance().getActions(logType.getName()); - } - - @Override - public Executor getTaskExecutor() { - return (Executor) SpringBeanUtil.getBean(ThreadPoolConfig.PARSER_THREAD_POOL); - } - - @Override - public List getJvmList() { - return (List) SpringBeanUtil.getBean(CustomConfig.GC_CONFIG); - } - - @Override - public ParserResultSink getParserResultSink() { - // TODO implement more kinds of writers and let them configurable. - ParserResultSink parserResultSink = new ParserResultSink(); - parserResultSink.register(OpenSearchWriter.getInstance()); - return parserResultSink; - } - - @Override - public ILogReaderFactory createLogReaderFactory() { - return new LogReaderFactory(); - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/SchedulerLogParser.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/SchedulerLogParser.java deleted file mode 100644 index 3e9fa7284..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/SchedulerLogParser.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.parser; - -import com.oppo.cloud.common.util.textparser.ParserAction; -import com.oppo.cloud.parser.domain.job.ParserParam; -import com.oppo.cloud.parser.service.reader.ILogReaderFactory; -import com.oppo.cloud.parser.service.writer.ParserResultSink; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; - -@Slf4j -public class SchedulerLogParser extends CommonTextParser { - - public SchedulerLogParser(ParserParam param, - ILogReaderFactory logReaderFactory, - List actions, - ParserResultSink parserResultSink) { - super(param, logReaderFactory, actions, parserResultSink); - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/SparkEventLogParser.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/SparkEventLogParser.java deleted file mode 100644 index 525046d85..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/SparkEventLogParser.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.parser; - -import com.oppo.cloud.common.constant.ApplicationType; -import com.oppo.cloud.common.constant.ProgressState; -import com.oppo.cloud.common.domain.eventlog.DetectorStorage; -import com.oppo.cloud.common.domain.eventlog.config.DetectorConfig; -import com.oppo.cloud.common.domain.eventlog.config.SparkEnvironmentConfig; -import com.oppo.cloud.common.domain.job.LogPath; -import com.oppo.cloud.parser.domain.job.*; -import com.oppo.cloud.parser.domain.reader.ReaderObject; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkApplication; -import com.oppo.cloud.parser.domain.spark.eventlog.SparkExecutor; -import com.oppo.cloud.parser.service.job.detector.DetectorManager; -import com.oppo.cloud.parser.service.reader.ILogReaderFactory; -import com.oppo.cloud.parser.service.reader.IReader; -import com.oppo.cloud.parser.service.reader.LogReaderFactory; -import com.oppo.cloud.parser.utils.ReplaySparkEventLogs; -import lombok.extern.slf4j.Slf4j; - -import java.io.FileNotFoundException; -import java.util.HashMap; -import java.util.Map; - -@Slf4j -public class SparkEventLogParser extends IParser { - - private DetectorConfig config; - - public SparkEventLogParser(ParserParam param, - ILogReaderFactory logReaderFactory, - DetectorConfig config) { - super(param, logReaderFactory); - this.config = config; - } - - @Override - public CommonResult run() { - updateParserProgress(ProgressState.PROCESSING, 0, this.param.getLogPaths().size()); - if (this.param.getLogPaths().size() > 0) { - LogPath logPath = this.param.getLogPaths().get(0); - ReaderObject readerObject; - try { - IReader reader = getReader(logPath); - readerObject = reader.getReaderObject(); - } catch (FileNotFoundException e) { - String path = logPath.getLogPath().substring(0, logPath.getLogPath().lastIndexOf("_")); - logPath.setLogPath(path); - try { - readerObject = getReader(logPath).getReaderObject(); - } catch (Exception ex) { - log.error("Exception:", e); - updateParserProgress(ProgressState.FAILED, 0, 0); - return null; - } - } catch (Exception e) { - log.error("Exception:", e); - updateParserProgress(ProgressState.FAILED, 0, 0); - return null; - } - return parse(readerObject); - } - return null; - } - - private CommonResult parse(ReaderObject readerObject) { - ReplaySparkEventLogs replayEventLogs = new ReplaySparkEventLogs(); - try { - replayEventLogs.replay(readerObject); - } catch (Exception e) { - log.error("Exception:", e); - updateParserProgress(ProgressState.FAILED, 0, 0); - return null; - } finally { - readerObject.close(); - } - return detect(replayEventLogs, readerObject.getLogPath()); - } - - private CommonResult detect(ReplaySparkEventLogs replayEventLogs, String logPath) { - Map env = getSparkEnvironmentConfig(replayEventLogs); - - Long appDuration = replayEventLogs.getApplication().getAppDuration(); - if (appDuration == null || appDuration < 0) { - appDuration = 0L; - } - - DetectorParam detectorParam = new DetectorParam(this.param.getLogRecord().getJobAnalysis().getFlowName(), - this.param.getLogRecord().getJobAnalysis().getProjectName(), - this.param.getLogRecord().getJobAnalysis().getTaskName(), - this.param.getLogRecord().getJobAnalysis().getExecutionDate(), - this.param.getLogRecord().getJobAnalysis().getRetryTimes(), - this.param.getApp().getAppId(), ApplicationType.SPARK, appDuration, logPath, config, - this.param.getLogRecord().getIsOneClick()); - detectorParam.setReplayEventLogs(replayEventLogs); - - DetectorManager detectorManager = new DetectorManager(); - // run all detector - DetectorStorage detectorStorage = detectorManager.run(detectorParam); - - detectorStorage.setEnv(env); - SparkEventLogParserResult sparkEventLogParserResult = new SparkEventLogParserResult(); - sparkEventLogParserResult.setDetectorStorage(detectorStorage); - sparkEventLogParserResult.setMemoryCalculateParam(getMemoryCalculateParam(replayEventLogs)); - - CommonResult result = new CommonResult<>(); - result.setLogType(this.param.getLogType()); - result.setResult(sparkEventLogParserResult); - - updateParserProgress(ProgressState.SUCCEED, 0, 0); - return result; - } - - private Map getSparkEnvironmentConfig(ReplaySparkEventLogs replayEventLogs) { - Map env = new HashMap<>(); - SparkEnvironmentConfig envConfig = config.getSparkEnvironmentConfig(); - if (envConfig != null) { - if (envConfig.getJvmInformation() != null) { - for (String key : envConfig.getJvmInformation()) { - env.put(key, replayEventLogs.getApplication().getJvmInformation().get(key)); - } - } - if (envConfig.getSparkProperties() != null) { - for (String key : envConfig.getSparkProperties()) { - env.put(key, replayEventLogs.getApplication().getSparkProperties().get(key)); - } - } - if (envConfig.getSystemProperties() != null) { - for (String key : envConfig.getSystemProperties()) { - env.put(key, replayEventLogs.getApplication().getSystemProperties().get(key)); - } - } - } - return env; - } - - - public MemoryCalculateParam getMemoryCalculateParam(ReplaySparkEventLogs replayEventLogs) { - SparkApplication application = replayEventLogs.getApplication(); - long appTotalTime = application.getAppEndTimestamp() - application.getAppStartTimestamp(); - MemoryCalculateParam memoryCalculateParam = new MemoryCalculateParam(); - memoryCalculateParam.setAppTotalTime(appTotalTime > 0 ? appTotalTime : 0); - memoryCalculateParam.setDriverMemory(application.getDriverMemory()); - memoryCalculateParam.setExecutorMemory(application.getExecutorMemory()); - - Map executorRuntimeMap = new HashMap<>(); - for (Map.Entry executor : replayEventLogs.getExecutors().entrySet()) { - SparkExecutor sparkExecutor = executor.getValue(); - long endTime = sparkExecutor.getRemoveTimestamp() > 0 ? sparkExecutor.getRemoveTimestamp() - : application.getAppEndTimestamp(); - - long startTime = sparkExecutor.getStartTimestamp() > 0 ? sparkExecutor.getStartTimestamp() - : application.getAppStartTimestamp(); - - long executorRuntime = endTime - startTime; - executorRuntimeMap.put(executor.getValue().getId(), executorRuntime); - } - memoryCalculateParam.setExecutorRuntimeMap(executorRuntimeMap); - return memoryCalculateParam; - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/SparkExecutorLogParser.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/SparkExecutorLogParser.java deleted file mode 100644 index c1efab5bc..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/parser/SparkExecutorLogParser.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.parser; - -import com.oppo.cloud.common.constant.LogType; -import com.oppo.cloud.common.constant.ProgressState; -import com.oppo.cloud.common.domain.job.LogPath; -import com.oppo.cloud.common.util.textparser.ParserAction; -import com.oppo.cloud.common.util.textparser.ParserManager; -import com.oppo.cloud.common.util.textparser.TextParser; -import com.oppo.cloud.parser.domain.job.CommonResult; -import com.oppo.cloud.parser.domain.job.ParserParam; -import com.oppo.cloud.parser.domain.job.SparkExecutorLogParserResult; -import com.oppo.cloud.parser.domain.reader.ReaderObject; -import com.oppo.cloud.parser.service.reader.ILogReaderFactory; -import com.oppo.cloud.parser.service.reader.IReader; -import com.oppo.cloud.parser.service.reader.LogReaderFactory; -import com.oppo.cloud.parser.service.writer.ParserResultSink; -import com.oppo.cloud.parser.utils.GCReportUtil; -import lombok.extern.slf4j.Slf4j; - -import java.io.*; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; -import java.util.concurrent.Future; - -@Slf4j -public class SparkExecutorLogParser extends CommonTextParser { - - private final Executor parserThreadPool; - - private final List jvmTypeList; - - public SparkExecutorLogParser(ParserParam param, - ILogReaderFactory logReaderFactory, - List actions, - ParserResultSink parserResultSink, - Executor threadPool, - List jvmTypeList) { - super(param, logReaderFactory, actions, parserResultSink); - this.parserThreadPool = threadPool; - this.jvmTypeList = jvmTypeList; - } - - @Override - public CommonResult run() { - updateParserProgress(ProgressState.PROCESSING, 0, 0); - CommonResult> commonResult = new CommonResult<>(); - List parserResults = new ArrayList<>(); - for (LogPath logPath : this.param.getLogPaths()) { - List readerObjects; - try { - IReader reader = getReader(logPath); - readerObjects = reader.getReaderObjects(); - } catch (Exception e) { - log.error("Exception:", e); - continue; - } - if (readerObjects.size() > 0) { - updateParserProgress(ProgressState.PROCESSING, 0, readerObjects.size()); - parserResults.addAll(handleReaderObjects(readerObjects)); - } - } - updateParserProgress(ProgressState.SUCCEED, 0, 0); - commonResult.setLogType(this.param.getLogType()); - commonResult.setResult(parserResults); - return commonResult; - - } - - private List handleReaderObjects(List readerObjects) { - List> futures = new ArrayList<>(); - for (ReaderObject readerObject : readerObjects) { - CompletableFuture future = - CompletableFuture.supplyAsync(() -> handleReaderObject(readerObject), parserThreadPool); - futures.add(future); - } - List results = new ArrayList<>(); - int i = 0; - for (Future result : futures) { - SparkExecutorLogParserResult sp = null; - try { - sp = result.get(); - } catch (Exception e) { - log.error("Exception:", e); - } - updateParserProgress(ProgressState.PROCESSING, i++, readerObjects.size()); - if (sp != null) { - results.add(sp); - } - } - return results; - } - - private SparkExecutorLogParserResult handleReaderObject(ReaderObject readerObject) { - String logType = getLogType(readerObject.getLogPath()); - SparkExecutorLogParserResult result = null; - try { - result = parseAction(logType, readerObject); - } catch (Exception e) { - log.error("Exception:", e); - } finally { - readerObject.close(); - } - if (result != null && result.getActionMap() != null) { - List categories = getSink().saveParserActions( - logType, readerObject.getLogPath(), this.param, result.getActionMap()); - result.setCategories(categories); - } - return result; - } - - private SparkExecutorLogParserResult parseAction(String logType, ReaderObject readerObject) throws Exception { - SparkExecutorLogParserResult result = parseRootAction(logType, readerObject); - for (Map.Entry action : result.getActionMap().entrySet()) { - ParserManager.parseChildActions(action.getValue()); - } - return result; - } - - private SparkExecutorLogParserResult parseRootAction(String logType, ReaderObject readerObject) throws Exception { - Map gcLogMap = new HashMap<>(); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - boolean isGCLog = false; - boolean isStderr = false; - - TextParser headTextParser = new TextParser(this.getActions()); - BufferedReader bufferedReader = readerObject.getBufferedReader(); - while (true) { - String line; - try { - line = bufferedReader.readLine(); - } catch (IOException e) { - log.error(e.getMessage()); - break; - } - if (line == null) { - headTextParser.close(); - break; - } - headTextParser.parse(line); - - // get gc log - if (line.contains("stderr")) { - isGCLog = false; - if (LogType.SPARK_DRIVER.getName().equals(logType) && byteArrayOutputStream.size() > 0) { - gcLogMap.put(0, byteArrayOutputStream.toByteArray()); - } - isStderr = true; - } - - if (jvmTypeList != null && !isGCLog) { - for (String jvm : jvmTypeList) { - if (line.contains(jvm)) { - isGCLog = true; - line = jvm + line.split(jvm)[1]; - break; - } - } - } - - if (isGCLog) { - line += "\n"; - byteArrayOutputStream.write(line.getBytes()); - } - if (isStderr && line.contains("Starting executor ID")) { - String id = line.split("ID")[1].split("on")[0].trim(); - if (byteArrayOutputStream.size() > 0) { - String gcLog = byteArrayOutputStream.toString(); - log.debug("gcLog:{}\n{}", readerObject.getLogPath(), gcLog); - gcLogMap.put(Integer.valueOf(id), byteArrayOutputStream.toByteArray()); - byteArrayOutputStream = new ByteArrayOutputStream(); - } - } - - } - - SparkExecutorLogParserResult result = new SparkExecutorLogParserResult(); - result.setActionMap(headTextParser.getResults()); - if (gcLogMap.size() > 0) { - result.setGcReports(GCReportUtil.generateGCReports(gcLogMap, readerObject.getLogPath())); - } - result.setLogPath(readerObject.getLogPath()); - - return result; - } - - - private String getLogType(String logPath) { - if (logPath.contains(this.param.getApp().getAmHost())) { - return LogType.SPARK_DRIVER.getName(); - } - return LogType.SPARK_EXECUTOR.getName(); - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/MapReduceTask.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/MapReduceTask.java deleted file mode 100644 index f865646f6..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/MapReduceTask.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.task; - -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.DetectorStorage; -import com.oppo.cloud.common.util.spring.SpringBeanUtil; -import com.oppo.cloud.parser.config.ThreadPoolConfig; -import com.oppo.cloud.parser.domain.job.CommonResult; -import com.oppo.cloud.parser.domain.job.TaskParam; -import com.oppo.cloud.parser.domain.job.TaskResult; -import com.oppo.cloud.parser.service.job.parser.IParser; -import com.oppo.cloud.parser.service.writer.OpenSearchWriter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; -import java.util.concurrent.Future; - -/** - * MapReduce task - */ -@Slf4j -public class MapReduceTask extends Task { - - private final TaskParam taskParam; - - private final Executor taskThreadPool; - - public MapReduceTask(TaskParam taskParam) { - super(taskParam); - this.taskParam = taskParam; - taskThreadPool = (ThreadPoolTaskExecutor) SpringBeanUtil.getBean(ThreadPoolConfig.TASK_THREAD_POOL); - } - - @Override - public TaskResult run() { - - List parsers = super.createTasks(); - if (parsers.size() == 0) { - return null; - } - - List> futures = super.createFutures(parsers, taskThreadPool); - - DetectorStorage detectorStorage = null; - List categories = new ArrayList<>(); - for (Future result : futures) { - CommonResult commonResult; - try { - commonResult = result.get(); - if (commonResult != null) { - switch (commonResult.getLogType()) { - case MAPREDUCE_JOB_HISTORY: - detectorStorage = (DetectorStorage) commonResult.getResult(); - break; - case MAPREDUCE_CONTAINER: - categories.addAll((List) commonResult.getResult()); - break; - default: - break; - } - } - } catch (Exception e) { - log.error("Exception: ", e); - } - } - - if (detectorStorage == null) { - return new TaskResult(this.taskParam.getApp().getAppId(), categories); - } - - if (detectorStorage.getAbnormal()) { - // get mr jobhistory categories - for (DetectorResult detectorResult : detectorStorage.getDataList()) { - if (detectorResult.getAbnormal()) { - categories.add(detectorResult.getAppCategory()); - } - } - // save all detector results - OpenSearchWriter.getInstance().saveDetectorStorage(detectorStorage); - } else { - // save mr conf - if (detectorStorage.getDataList() != null) { - detectorStorage.setDataList(null); - } - OpenSearchWriter.getInstance().saveDetectorStorage(detectorStorage); - } - - return new TaskResult(this.taskParam.getApp().getAppId(), categories); - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/SchedulerTask.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/SchedulerTask.java deleted file mode 100644 index 7a81ad413..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/SchedulerTask.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.task; - -import com.oppo.cloud.common.util.spring.SpringBeanUtil; -import com.oppo.cloud.parser.config.ThreadPoolConfig; -import com.oppo.cloud.parser.domain.job.CommonResult; -import com.oppo.cloud.parser.domain.job.TaskParam; -import com.oppo.cloud.parser.domain.job.TaskResult; -import com.oppo.cloud.parser.service.job.parser.IParser; -import lombok.extern.slf4j.Slf4j; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; -import java.util.concurrent.Future; - -/** - * Scheduler task - */ -@Slf4j -public class SchedulerTask extends Task { - - private Executor taskThreadPool; - - private final TaskParam taskParam; - - public SchedulerTask(TaskParam taskParam) { - super(taskParam); - this.taskParam = taskParam; - taskThreadPool = (ThreadPoolTaskExecutor) SpringBeanUtil.getBean(ThreadPoolConfig.TASK_THREAD_POOL); - } - - @Override - public TaskResult run() { - List parsers = super.createTasks(); - if (parsers.size() == 0) { - return null; - } - List> futures = super.createFutures(parsers, taskThreadPool); - - TaskResult taskResult = new TaskResult(); - taskResult.setAppId(this.taskParam.getApp().getAppId()); - - List categories = new ArrayList<>(); - for (Future result : futures) { - CommonResult commonResult; - try { - commonResult = result.get(); - if (commonResult != null) { - switch (commonResult.getLogType()) { - case SCHEDULER: - categories.addAll((List) commonResult.getResult()); - break; - default: - break; - } - } - } catch (Exception e) { - log.error("Exception:", e); - } - } - taskResult.setCategories(categories); - return taskResult; - - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/SparkTask.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/SparkTask.java deleted file mode 100644 index c06b63443..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/SparkTask.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.task; - -import com.oppo.cloud.common.domain.eventlog.DetectorResult; -import com.oppo.cloud.common.domain.eventlog.DetectorStorage; -import com.oppo.cloud.common.domain.eventlog.config.MemWasteConfig; -import com.oppo.cloud.common.domain.gc.GCReport; -import com.oppo.cloud.common.util.spring.SpringBeanUtil; -import com.oppo.cloud.parser.config.ThreadPoolConfig; -import com.oppo.cloud.parser.domain.job.*; -import com.oppo.cloud.parser.service.rules.JobRulesConfigService; -import com.oppo.cloud.parser.service.job.detector.spark.MemWasteDetector; -import com.oppo.cloud.parser.service.job.parser.IParser; -import com.oppo.cloud.parser.service.writer.OpenSearchWriter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; -import java.util.concurrent.Future; - -/** - * Spark task - */ -@Slf4j -public class SparkTask extends Task { - - private final TaskParam taskParam; - private final MemWasteConfig memWasteConfig; - private final Executor taskThreadPool; - - public SparkTask(TaskParam taskParam) { - super(taskParam); - this.taskParam = taskParam; - taskThreadPool = (ThreadPoolTaskExecutor) SpringBeanUtil.getBean(ThreadPoolConfig.TASK_THREAD_POOL); - JobRulesConfigService jobRulesConfigService = (JobRulesConfigService) SpringBeanUtil.getBean(JobRulesConfigService.class); - this.memWasteConfig = jobRulesConfigService.detectorConfig.getMemWasteConfig(); - } - - @Override - public TaskResult run() { - List parsers = super.createTasks(); - if (parsers.size() == 0) { - return null; - } - List> futures = super.createFutures(parsers, taskThreadPool); - - TaskResult taskResult = new TaskResult(); - taskResult.setAppId(this.taskParam.getApp().getAppId()); - - List gcReports = new ArrayList<>(); - List sparkExecutorLogParserResults = null; - SparkEventLogParserResult sparkEventLogParserResult = null; - - for (Future result : futures) { - CommonResult commonResult; - try { - commonResult = result.get(); - if (commonResult != null) { - switch (commonResult.getLogType()) { - case SPARK_DRIVER: - case SPARK_EXECUTOR: - sparkExecutorLogParserResults = - (List) commonResult.getResult(); - break; - case SPARK_EVENT: - sparkEventLogParserResult = (SparkEventLogParserResult) commonResult.getResult(); - break; - default: - break; - } - } - } catch (Exception e) { - log.error("Exception:{}", e); - } - } - - if (sparkEventLogParserResult == null) { - log.error("sparkEventLogParserResultNull:{}", taskParam.getApp()); - return null; - } - // get driver/executor categories - List executorCategories = new ArrayList<>(); - if (sparkExecutorLogParserResults != null) { - for (SparkExecutorLogParserResult result : sparkExecutorLogParserResults) { - if (result.getGcReports() != null) { - gcReports.addAll(result.getGcReports()); - } - if (result.getCategories() != null) { - executorCategories.addAll(result.getCategories()); - } - } - } - taskResult.setCategories(executorCategories); - - DetectorStorage detectorStorage = sparkEventLogParserResult.getDetectorStorage(); - if (detectorStorage == null) { - log.error("detectorStorageNull:{}", taskParam.getApp()); - return taskResult; - } - // calculate memory metrics - if (!this.memWasteConfig.getDisable() && gcReports.size() > 0 - && sparkEventLogParserResult.getMemoryCalculateParam() != null) { - MemWasteDetector memWasteDetector = new MemWasteDetector(this.memWasteConfig); - DetectorResult detectorResult = - memWasteDetector.detect(gcReports, sparkEventLogParserResult.getMemoryCalculateParam()); - detectorStorage.addDetectorResult(detectorResult); - if (detectorResult.getAbnormal()) { - detectorStorage.setAbnormal(true); - } - } - // get event log categories - List eventLogCategories = new ArrayList<>(); - if (this.taskParam.getLogRecord().getIsOneClick() || detectorStorage.getAbnormal()) { - for (DetectorResult detectorResult : detectorStorage.getDataList()) { - if (detectorResult.getAbnormal()) { - eventLogCategories.add(detectorResult.getAppCategory()); - } - } - // save all detector results - OpenSearchWriter.getInstance().saveDetectorStorage(detectorStorage); - } else { - // save event log env - detectorStorage.setDataList(null); - OpenSearchWriter.getInstance().saveDetectorStorage(detectorStorage); - } - eventLogCategories.addAll(executorCategories); - // set all spark categories - taskResult.setCategories(eventLogCategories); - - // save gc reports - if ((this.taskParam.getLogRecord().getIsOneClick() || eventLogCategories.size() > 0) && gcReports.size() > 0) { - gcReports.sort(Comparator.comparing(GCReport::getMaxHeapUsedSize)); - if (gcReports.size() > 11) { - List results = new ArrayList<>(); - GCReport driverGc = gcReports.stream().filter(gc -> gc.getExecutorId() == 0).findFirst().orElse(null); - List executorGcs = gcReports.subList(gcReports.size() - 10, gcReports.size()); - if (driverGc != null) { - results.add(driverGc); - } - results.addAll(executorGcs); - OpenSearchWriter.getInstance().saveGCReports(results, detectorStorage.getExecutionTime(), - detectorStorage.getApplicationId()); - } else { - OpenSearchWriter.getInstance().saveGCReports(gcReports, detectorStorage.getExecutionTime(), - detectorStorage.getApplicationId()); - } - } - - return taskResult; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/Task.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/Task.java deleted file mode 100644 index 8b7cfe1e2..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/Task.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.task; - -import com.oppo.cloud.common.domain.job.LogPath; -import com.oppo.cloud.parser.domain.job.CommonResult; -import com.oppo.cloud.parser.domain.job.ParserParam; -import com.oppo.cloud.parser.domain.job.TaskParam; -import com.oppo.cloud.parser.domain.job.TaskResult; -import com.oppo.cloud.parser.service.job.oneclick.ProgressListener; -import com.oppo.cloud.parser.service.job.parser.IParser; -import com.oppo.cloud.parser.service.job.parser.ParserFactory; -import lombok.extern.slf4j.Slf4j; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; - -@Slf4j -public abstract class Task { - - private final TaskParam taskParam; - - public Task(TaskParam taskParam) { - this.taskParam = taskParam; - } - - public List createTasks() { - List parsers = new ArrayList<>(); - Map> logPathMap = this.taskParam.getLogInfo().getLogPathMap(); - if (logPathMap != null && logPathMap.size() > 0) { - for (Map.Entry> map : logPathMap.entrySet()) { - ParserFactory parserFactory = new ParserFactory(); - IParser parser = parserFactory.create(new ParserParam(map.getKey(), this.taskParam.getLogRecord(), - this.taskParam.getApp(), map.getValue()), new ProgressListener()); - if (parser != null) { - parsers.add(parser); - } - } - } - return parsers; - } - - public List> createFutures(List parsers, Executor taskThreadPool) { - - List> futures = new ArrayList<>(); - - for (IParser parser : parsers) { - CompletableFuture future = CompletableFuture.supplyAsync(() -> { - CommonResult commonResult = null; - try { - commonResult = parser.run(); - } catch (Exception e) { - log.error("Exception:", e); - } - return commonResult; - }, taskThreadPool); - futures.add(future); - } - - return futures; - } - - public abstract TaskResult run(); -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/TaskFactory.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/TaskFactory.java deleted file mode 100644 index d5a37264d..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/TaskFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.task; - -import com.oppo.cloud.parser.domain.job.TaskParam; - - -public class TaskFactory { - - - public static Task create(TaskParam taskParam) { - switch (taskParam.getCategory()) { - case SCHEDULER: - return new SchedulerTask(taskParam); - case SPARK: - return new SparkTask(taskParam); - case MAPREDUCE: - return new MapReduceTask(taskParam); - case YARN: - return new YarnTask(taskParam); - default: - return null; - } - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/YarnTask.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/YarnTask.java deleted file mode 100644 index 0c2e04528..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/job/task/YarnTask.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.job.task; - -import com.oppo.cloud.common.constant.AppCategoryEnum; -import com.oppo.cloud.common.constant.LogType; -import com.oppo.cloud.common.domain.opensearch.TaskApp; -import com.oppo.cloud.common.util.textparser.ParserAction; -import com.oppo.cloud.common.util.textparser.ParserManager; -import com.oppo.cloud.parser.config.DiagnosisConfig; -import com.oppo.cloud.parser.domain.job.ParserParam; -import com.oppo.cloud.parser.domain.job.TaskParam; -import com.oppo.cloud.parser.domain.job.TaskResult; -import com.oppo.cloud.parser.service.writer.OpenSearchWriter; -import com.oppo.cloud.parser.service.writer.ParserResultSink; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Yarn diagnostics log - */ -@Slf4j -public class YarnTask extends Task { - - private final TaskParam taskParam; - - public YarnTask(TaskParam taskParam) { - super(taskParam); - this.taskParam = taskParam; - } - - @Override - public TaskResult run() { - List actions = DiagnosisConfig.getInstance().getActions(LogType.YARN.getName()); - String appId = this.taskParam.getApp().getAppId(); - if (StringUtils.isBlank(appId)) { - return null; - } - TaskApp taskApp = this.taskParam.getLogRecord().getTaskApp(appId); - if (taskApp == null) { - return null; - } - String diagnostics = taskApp.getDiagnostics(); - if (StringUtils.isBlank(diagnostics)) { - return null; - } - - Map results = ParserManager.parse(new String[]{diagnostics}, actions); - if (results == null || results.size() == 0) { - List list = new ArrayList<>(); - list.add(AppCategoryEnum.OTHER_EXCEPTION.getCategory()); - return new TaskResult(this.taskParam.getApp().getAppId(), list); - } - - ParserParam param = new ParserParam(LogType.YARN.getName(), this.taskParam.getLogRecord(), - this.taskParam.getApp(), null); - - ParserResultSink parserResultSink = new ParserResultSink(); - parserResultSink.register(OpenSearchWriter.getInstance()); - List list = parserResultSink.saveParserActions( - LogType.YARN.getName(), "", param, results); - - return new TaskResult(this.taskParam.getApp().getAppId(), new ArrayList<>(list)); - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/HDFSReader.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/HDFSReader.java deleted file mode 100644 index be71671cc..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/HDFSReader.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.reader; - -import com.oppo.cloud.common.domain.cluster.hadoop.NameNodeConf; -import com.oppo.cloud.common.domain.job.LogPath; -import com.oppo.cloud.common.util.spring.SpringBeanUtil; -import com.oppo.cloud.parser.config.HadoopConfig; -import com.oppo.cloud.parser.domain.reader.ReaderObject; -import com.oppo.cloud.parser.utils.HDFSUtil; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * HDFS logs reader - */ -public class HDFSReader implements IReader { - - /** - * Log Path - */ - private final LogPath logPath; - /** - * NameNode configuration - */ - private final NameNodeConf nameNode; - - public HDFSReader(LogPath logPath, Map nameNodeMap) throws Exception { - this.logPath = logPath; - nameNode = HDFSUtil.getNameNode(nameNodeMap, logPath.getLogPath()); - if (nameNode == null) { - throw new Exception("cant get hdfs nameNode " + logPath.getLogPath()); - } - } - - @Override - public List listFiles() throws Exception { - return HDFSUtil.listFiles(nameNode, logPath.getLogPath()); - } - - @Override - public List filesPattern() throws Exception { - return HDFSUtil.filesPattern(nameNode, logPath.getLogPath()); - } - - @Override - public ReaderObject getReaderObject() throws Exception { - return HDFSUtil.getReaderObject(nameNode, logPath.getLogPath()); - } - - @Override - public List getReaderObjects() throws Exception { - List list = new ArrayList<>(); - switch (logPath.getLogPathType()) { - case FILE: - list.add(HDFSUtil.getReaderObject(nameNode, logPath.getLogPath())); - break; - case DIRECTORY: - List files = listFiles(); - if (files.size() > 0) { - for (String path : files) { - list.add(HDFSUtil.getReaderObject(nameNode, path)); - } - } - break; - case PATTERN: - files = filesPattern(); - if (files.size() > 0) { - for (String path : files) { - list.add(HDFSUtil.getReaderObject(nameNode, path)); - } - } - break; - default: - return null; - } - return list; - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/ILogReaderFactory.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/ILogReaderFactory.java deleted file mode 100644 index d8789e469..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/ILogReaderFactory.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.reader; - -import com.oppo.cloud.common.domain.cluster.hadoop.NameNodeConf; -import com.oppo.cloud.common.domain.job.LogPath; - -import java.util.Map; - -/** - * Log reader factory interface - */ -public interface ILogReaderFactory { - - String HDFS = "hdfs"; - String STREAM = "stream"; - String S3 = "s3"; - - /** - * Create reader type - */ - default IReader create(LogPath logPath) throws Exception { - switch (logPath.getProtocol()) { - case HDFS: - return new HDFSReader(logPath, getNameNodeConf()); - case STREAM: - return new InputStreamReader(logPath); - default: - break; - } - return new HDFSReader(logPath, getNameNodeConf()); - } - - Map getNameNodeConf(); - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/IReader.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/IReader.java deleted file mode 100644 index 9236517df..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/IReader.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.reader; - -import com.oppo.cloud.parser.domain.reader.ReaderObject; - -import java.util.List; - -/** - * Log reader interface - */ -public interface IReader { - - List listFiles() throws Exception; - - List filesPattern() throws Exception; - - ReaderObject getReaderObject() throws Exception; - - List getReaderObjects() throws Exception; -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/InputStreamReader.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/InputStreamReader.java deleted file mode 100644 index 4372f5082..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/InputStreamReader.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oppo.cloud.parser.service.reader; - -import com.oppo.cloud.common.domain.job.LogPath; -import com.oppo.cloud.parser.domain.reader.ReaderObject; - -import java.util.List; - -public class InputStreamReader implements IReader { - private final LogPath logPath; - - public InputStreamReader(LogPath logPath) { - this.logPath = logPath; - } - - @Override - public List listFiles() throws Exception { - return null; - } - - @Override - public List filesPattern() throws Exception { - return null; - } - - @Override - public ReaderObject getReaderObject() throws Exception { - ReaderObject readerObject = new ReaderObject(); - readerObject.setLogPath(logPath.getLogPath()); - readerObject.setFs(null); - readerObject.setInputStream(logPath.getInputStream()); - return readerObject; - } - - @Override - public List getReaderObjects() throws Exception { - return null; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/LogReaderFactory.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/LogReaderFactory.java deleted file mode 100644 index 9421e29b4..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/reader/LogReaderFactory.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.reader; - -import com.oppo.cloud.common.domain.cluster.hadoop.NameNodeConf; -import com.oppo.cloud.common.util.spring.SpringBeanUtil; -import com.oppo.cloud.parser.config.HadoopConfig; - -import java.util.Map; - -/** - * Log reader factory - */ -public class LogReaderFactory implements ILogReaderFactory{ - - @Override - public Map getNameNodeConf() { - return (Map) SpringBeanUtil.getBean(HadoopConfig.NAME_NODE_MAP); - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/rules/JobRulesConfigService.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/rules/JobRulesConfigService.java deleted file mode 100644 index 6ebe146a2..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/rules/JobRulesConfigService.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.rules; - -import com.alibaba.fastjson2.JSON; -import com.oppo.cloud.common.domain.eventlog.config.DetectorConfig; -import com.oppo.cloud.common.util.textparser.ParserActionUtil; -import com.oppo.cloud.parser.config.CustomConfig; -import com.oppo.cloud.parser.config.DiagnosisConfig; -import com.oppo.cloud.parser.domain.rule.Rules; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import javax.annotation.PostConstruct; -import java.io.InputStream; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Slf4j -@Service -public class JobRulesConfigService { - @Autowired - private CustomConfig customConfig; - - public DetectorConfig detectorConfig; - - private Map rulesConfig; - - @PostConstruct - private void init() { - readConfig(); - DiagnosisConfig.getInstance().setDetectorConfig(this.detectorConfig); - DiagnosisConfig.getInstance().setRuleMap(this.rulesConfig); - } - - public void readConfig() { - Map rulesMap = new HashMap<>(); - InputStream input = JobRulesConfigService.class.getClassLoader().getResourceAsStream("rules.json"); - if (input == null) { - return; - } - List rules = JSON.parseArray(JSON.parseArray(input).toString(), Rules.class); - - for (Rules rule : rules) { - rulesMap.put(rule.getLogType(), JSON.toJSONString(rule.getActions())); - } - - this.detectorConfig = this.customConfig.eventLogDetectorConfig(); - this.rulesConfig = rulesMap; - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/scheduled/LogRecordProcessingTask.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/scheduled/LogRecordProcessingTask.java deleted file mode 100644 index f33a42acb..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/scheduled/LogRecordProcessingTask.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.scheduled; - -import com.alibaba.fastjson2.JSONObject; -import com.oppo.cloud.common.constant.Constant; -import com.oppo.cloud.common.domain.job.LogRecord; -import com.oppo.cloud.common.service.RedisService; -import com.oppo.cloud.parser.config.CustomConfig; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.Scheduled; - -import javax.annotation.Resource; -import java.util.Map; - -/** - * remove task record log in redis if the task is consumed too many times exceed the limit. - */ -@Slf4j -@Configuration -@ConditionalOnProperty(prefix = "custom.redisConsumer", name = "enable", havingValue = "true") -public class LogRecordProcessingTask { - - /** - * The delta time between now and the time of the task consumed first time - * when it was not finished to parse log, unit: hour. - */ - @Value("${custom.redisConsumer.scheduled.limitTime:2}") - private long limitTime; - - /** - * The threshold of frequency when failed to consume task record. - */ - @Value("${custom.redisConsumer.scheduled.limitCount:10}") - private long limitCount; - - @Resource - private CustomConfig customConfig; - - @Resource - private RedisService redisService; - - @Scheduled(cron = "${custom.redisConsumer.scheduled.cron}") - public void run() { - Long size = redisService.hLen(customConfig.getProcessingHash()); - if (size == 0) { - return; - } - Map map = redisService.hGetAll(customConfig.getProcessingHash()); - if (map == null || map.size() == 0) { - return; - } - log.info("processingJob size:{}", size); - map.forEach((key, val) -> { - LogRecord logRecord = JSONObject.parseObject((String) val, LogRecord.class); - long spendTime = System.currentTimeMillis() - logRecord.getCreateTime(); - long limitTimesMs = limitTime * Constant.HOUR_MS; - if (spendTime > limitTimesMs && logRecord.getConsumeCount() > limitCount) { - log.warn("delete logRecord: {}, id: {}", customConfig.getProcessingHash(), key); - try { - redisService.hDel(customConfig.getProcessingHash(), key); - } catch (Exception e) { - log.error(e.getMessage()); - } - } - }); - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/writer/IParserResultWriter.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/writer/IParserResultWriter.java deleted file mode 100644 index f876812ab..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/writer/IParserResultWriter.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.writer; - -import com.oppo.cloud.common.util.textparser.ParserAction; -import com.oppo.cloud.parser.domain.job.ParserParam; - -/** - * ParserResultWriter interface - */ -public interface IParserResultWriter { - - void write(String logType, String logPath, ParserParam param, ParserAction parserAction); - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/writer/LogWriter.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/writer/LogWriter.java deleted file mode 100644 index 226485cfc..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/writer/LogWriter.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.writer; - -import com.oppo.cloud.common.util.textparser.ParserAction; -import com.oppo.cloud.parser.domain.job.ParserParam; -import lombok.extern.slf4j.Slf4j; - -import java.util.ArrayList; -import java.util.List; - -/** - * a writer to log parser results, now just for testing - */ -@Slf4j -public class LogWriter implements IParserResultWriter { - - private List parserActionList; - - public LogWriter() { - this.parserActionList = new ArrayList<>(); - } - - @Override - public void write(String logType, String logPath, ParserParam param, ParserAction parserAction) { - log.info("Parsed results for logType {}," + - "logPath {}, param {}, parserAction {}", - logType, logPath, param, parserAction); - parserActionList.add(parserAction); - } - - public List getParserActionList() { - return parserActionList; - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/writer/OpenSearchWriter.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/writer/OpenSearchWriter.java deleted file mode 100644 index 6c9a64b0a..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/writer/OpenSearchWriter.java +++ /dev/null @@ -1,443 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.writer; - -import com.alibaba.fastjson2.JSON; -import com.oppo.cloud.common.domain.LogMessage; -import com.oppo.cloud.common.domain.opensearch.OpenSearchInfo; -import com.oppo.cloud.common.domain.opensearch.JobAnalysis; -import com.oppo.cloud.common.domain.opensearch.LogSummary; -import com.oppo.cloud.common.domain.opensearch.TaskApp; -import com.oppo.cloud.common.domain.eventlog.DetectorStorage; -import com.oppo.cloud.common.domain.gc.ExecutorPeakMemory; -import com.oppo.cloud.common.domain.gc.GCReport; -import com.oppo.cloud.common.domain.job.LogRecord; -import com.oppo.cloud.common.util.DateUtil; -import com.oppo.cloud.common.util.opensearch.BulkApi; -import com.oppo.cloud.common.util.opensearch.UpdateApi; -import com.oppo.cloud.common.util.spring.SpringBeanUtil; -import com.oppo.cloud.common.util.textparser.ParserAction; -import com.oppo.cloud.common.util.textparser.ParserActionUtil; -import com.oppo.cloud.common.util.textparser.ParserResult; -import com.oppo.cloud.parser.config.CustomConfig; -import com.oppo.cloud.parser.config.KafkaConfig; -import com.oppo.cloud.parser.config.OpenSearchConfig; -import com.oppo.cloud.parser.domain.job.ParserParam; -import com.oppo.cloud.parser.domain.job.TaskResult; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.opensearch.action.bulk.BulkItemResponse; -import org.opensearch.action.bulk.BulkResponse; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.update.UpdateResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.search.SearchHit; -import org.opensearch.search.builder.SearchSourceBuilder; - -import java.util.*; - -/** - * OpenSearch writer - */ -@Slf4j -public class OpenSearchWriter implements IParserResultWriter { - - public RestHighLevelClient client; - - public String logSummaryPrefix; - - public String detectorPrefix; - - public String gcPrefix; - - public String jobPrefix; - - public String taskAppPrefix; - - public KafkaConfig kafkaConfig; - - private OpenSearchWriter() { - client = (RestHighLevelClient) SpringBeanUtil.getBean(OpenSearchConfig.SEARCH_CLIENT); - CustomConfig yml = (CustomConfig) SpringBeanUtil.getBean(CustomConfig.class); - logSummaryPrefix = yml.getLogSummaryPrefix(); - detectorPrefix = yml.getDetectorPrefix(); - gcPrefix = yml.getGcPrefix(); - jobPrefix = yml.getJobPrefix(); - taskAppPrefix = yml.getTaskAppPrefix(); - kafkaConfig = (KafkaConfig) SpringBeanUtil.getBean(KafkaConfig.class); - } - - public static OpenSearchWriter getInstance() { - return OpenSearch.INSTANCE.getInstance(); - } - - /** - * Write matching results to OpenSearch - */ - @Override - public void write(String logType, String logPath, ParserParam param, ParserAction parserAction) { - if (parserAction.getParserResults() == null) { - return; - } - String index = logSummaryPrefix + DateUtil.formatToDay(param.getLogRecord().getJobAnalysis().getExecutionDate()); - - List logMessages = new ArrayList<>(); - List> docs = new ArrayList<>(); - for (ParserResult parserResult : parserAction.getParserResults()) { - Map logSummary; - try { - logSummary = getDoc(logType, parserResult, logPath, param, parserAction); - } catch (Exception e) { - log.error("logSummaryGetDoc:{},{}", logPath, e); - continue; - } - docs.add(logSummary); - if (parserAction.getAction().toUpperCase().contains("OTHER")) { - logMessages.add(JSON.toJSONString(new LogMessage((String) logSummary.get("docId"), index, logType, (String) logSummary.get("rawLog")))); - } - } - - if (docs.size() > 0) { - BulkResponse response; - try { - response = BulkApi.bulk(client, index, docs); - } catch (Exception e) { - log.error("writeLogSummaryErr:{},{}", logPath, e); - return; - } - BulkItemResponse[] responses = response.getItems(); - - for (BulkItemResponse r : responses) { - if (r.isFailed()) { - log.info("writeLogSummaryErr:{},{}", logPath, r.getFailure().getCause()); - } - } - log.info("writeLogSummaryCount:{},{},{},{}", logType, logPath, parserAction.getAction(), docs.size()); - } - - sendOtherErrorLogToGPT(logMessages); - - } - - /** - * Send other error log to GPT - */ - public void sendOtherErrorLogToGPT(List logMessages) { - if (logMessages != null && logMessages.size() > 0) { - log.info("logMessages:{}", logMessages); - try { - kafkaConfig.sendMessages(logMessages); - } catch (Exception e) { - log.info("sendMessagesErr:{}", logMessages); - } - } - } - - /** - * Get document content - */ - public Map getDoc(String logType, ParserResult parserResult, String logPath, ParserParam param, - ParserAction parserAction) throws Exception { - LogSummary logSummary = new LogSummary(); - logSummary.setApplicationId(param.getApp().getAppId()); - logSummary.setLogType(logType); - logSummary.setProjectName(param.getLogRecord().getJobAnalysis().getProjectName()); - logSummary.setFlowName(param.getLogRecord().getJobAnalysis().getFlowName()); - logSummary.setTaskName(param.getLogRecord().getJobAnalysis().getTaskName()); - logSummary.setExecutionDate(param.getLogRecord().getJobAnalysis().getExecutionDate()); - logSummary.setRetryTimes(param.getApp().getTryNumber()); - logSummary.setAction(parserAction.getAction()); - logSummary.setStep(parserAction.getStep()); - logSummary.setGroupNames(Arrays.asList(parserAction.getGroupNames())); - logSummary.setRawLog(String.join("\n", parserResult.getLines())); - logSummary.setLogPath(logPath); - logSummary.setGroupData(parserResult.getGroupData()); - if (parserResult.getGroupData() != null) { - long timestamp; - try { - String dateStr = parserResult.getGroupData().get("datetime"); - // fix datetime - if (dateStr.contains("/")) { - dateStr = "20" + dateStr; - dateStr = dateStr.replace("/", "-"); - } - timestamp = DateUtil.dateToTimeStamp(dateStr); - } catch (Exception e) { - log.error("get datetime err:{},{}", param.getApp().getAppId(), parserResult.getGroupData()); - timestamp = System.currentTimeMillis() / 1000; - } - logSummary.setLogTimestamp((int) timestamp); - } else { - logSummary.setLogTimestamp((int) (System.currentTimeMillis() / 1000)); - - } - return logSummary.genDoc(); - } - - public void saveDetectorStorage(DetectorStorage detectionStorage) { - String index = detectorPrefix + DateUtil.formatToDay(detectionStorage.getExecutionTime()); - UpdateApi api = new UpdateApi(); - try { - UpdateResponse resp = api.upsertJson(client, index, detectionStorage.getApplicationId(), JSON.toJSONString(detectionStorage)); - log.info("saveDetectorStorage:{},{}", detectionStorage.getApplicationId(), resp); - } catch (Exception e) { - log.info("saveDetectorStorageErr:{},{}", detectionStorage.getApplicationId(), e); - } - } - - /** - * Update job categories information - */ - public void updateJob(JobAnalysis jobAnalysis, Map categoryMap) throws Exception { - if (jobAnalysis == null || jobAnalysis.getTaskName() == null) { - return; - } - UpdateApi api = new UpdateApi(); - String index = jobPrefix + DateUtil.formatToDay(jobAnalysis.getExecutionDate()); - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.filter(QueryBuilders.termQuery("projectName.keyword", jobAnalysis.getProjectName())); - boolQueryBuilder.filter(QueryBuilders.termQuery("flowName.keyword", jobAnalysis.getFlowName())); - boolQueryBuilder.filter(QueryBuilders.termQuery("taskName.keyword", jobAnalysis.getTaskName())); - boolQueryBuilder.filter(QueryBuilders.termQuery("executionDate", - DateUtil.timestampToUTCDate(jobAnalysis.getExecutionDate().getTime()))); - - JobAnalysis jobAnalysisResult = search(JobAnalysis.class, index, boolQueryBuilder); - - String id; - List existedCategories = null; - if (jobAnalysisResult == null || jobAnalysisResult.getDocId() == null) { - id = UUID.randomUUID().toString(); - jobAnalysis.setCreateTime(new Date()); - jobAnalysis.setUpdateTime(new Date()); - } else { - id = jobAnalysisResult.getDocId(); - if (jobAnalysisResult.getCategories() != null && jobAnalysisResult.getCategories().size() > 0) { - existedCategories = jobAnalysisResult.getCategories(); - } - jobAnalysis.setDocId(id); - } - - if (existedCategories == null) { - existedCategories = new ArrayList<>(categoryMap.keySet()); - } else { - updateCategories(existedCategories, categoryMap); - } - jobAnalysis.setCategories(existedCategories); - String data = JSON.toJSONString(jobAnalysis.genDoc()); - log.debug("updateJob:{},{}", id, data); - api.upsertJson(client, index, id, data); - - } - - private void updateCategories(List list, Map categoryMap) { - list.forEach(categoryMap::remove); - if (categoryMap.size() > 0) { - categoryMap.forEach((k, v) -> list.add(k)); - } - } - - /** - * Update task-app categories information - */ - public void updateTaskApp(TaskApp taskApp, Map categoryMap) throws Exception { - String index = taskAppPrefix + DateUtil.formatToDay(taskApp.getExecutionDate()); - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.filter(QueryBuilders.termQuery("applicationId.keyword", taskApp.getApplicationId())); - - TaskApp taskAppResult = search(TaskApp.class, index, boolQueryBuilder); - - String id; - List existedCategories = null; - if (taskAppResult == null || taskAppResult.getDocId() == null) { - id = UUID.randomUUID().toString(); - taskApp.setCreateTime(new Date()); - - } else { - id = taskAppResult.getDocId(); - if (taskAppResult.getCategories() != null && taskAppResult.getCategories().size() > 0) { - existedCategories = taskAppResult.getCategories(); - } - } - - if (existedCategories == null) { - existedCategories = new ArrayList<>(categoryMap.keySet()); - } else { - updateCategories(existedCategories, categoryMap); - } - - if (existedCategories.size() > 0) { - taskApp.setDiagnoseResult("abnormal"); - } - taskApp.setUpdateTime(new Date()); - taskApp.setCategories(existedCategories); - UpdateApi api = new UpdateApi(); - String data = JSON.toJSONString(taskApp.genDoc()); - log.debug("updateTaskApp:{},{}", id, data); - api.upsertJson(client, index, id, data); - - } - - public void saveGCReports(List gcReports, Date executionTime, String appId) { - List gcReportDocs = new ArrayList<>(); - for (GCReport gcReport : gcReports) { - String data = JSON.toJSONString(gcReport); - gcReportDocs.add(data); - } - String index = gcPrefix + DateUtil.formatToDay(executionTime); - - BulkResponse response; - try { - response = BulkApi.bulkJson(client, index, gcReportDocs); - } catch (Exception e) { - log.error("saveGCReportsErr:{},{}", appId, e); - return; - } - BulkItemResponse[] responses = response.getItems(); - - for (BulkItemResponse r : responses) { - if (r.isFailed()) { - log.info("saveGCReportsErr:{},{}", appId, r.getFailure().getCause()); - } - } - - // save executor peak memory - List executorPeakList = new ArrayList<>(); - - for (GCReport gcReport : gcReports) { - executorPeakList.add(new ExecutorPeakMemory(gcReport.getExecutorId(), gcReport.getMaxHeapUsedSize(), - gcReport.getLogPath())); - } - Map m = new HashMap<>(); - m.put("applicationId", appId); - m.put("executorPeak", executorPeakList); - List gcReportPeakDocs = new ArrayList<>(); - - gcReportPeakDocs.add(JSON.toJSONString(m)); - - BulkResponse gcPeakMemResponse; - try { - gcPeakMemResponse = BulkApi.bulkJson(client, index, gcReportPeakDocs); - } catch (Exception e) { - log.error("saveGCReportsErr:{},{}", appId, e); - return; - } - BulkItemResponse[] gcPeakMemResponses = gcPeakMemResponse.getItems(); - - for (BulkItemResponse r : gcPeakMemResponses) { - if (r.isFailed()) { - log.info("saveGCReportsErr:{},{}", appId, r.getFailure().getCause()); - } - } - log.info("saveGCReports:{}", appId); - - } - - private T search(Class itemType, String index, BoolQueryBuilder boolQueryBuilder) throws Exception { - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices(index); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(boolQueryBuilder); - searchRequest.source(searchSourceBuilder); - log.info("es index:{}, search condition:{}", searchRequest.indices(), searchSourceBuilder.toString()); - - SearchResponse searchResponse = null; - try { - searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - } catch (Exception e) { - log.warn("Exception:{}", e.getMessage()); - } - - T t = null; - if (searchResponse != null) { - for (SearchHit hit : searchResponse.getHits()) { - t = JSON.parseObject(hit.getSourceAsString(), itemType); - t.setDocId(hit.getId()); - } - } - - return t; - } - - public void saveTaskResults(LogRecord logRecord, List taskResults) throws Exception { - - Map jobCategoryMap = new HashMap<>(); - Map> appCategoryMap = new HashMap<>(); - for (TaskResult result : taskResults) { - if (result.getCategories() != null) { - for (String category : result.getCategories()) { - jobCategoryMap.put(category, true); - } - if (StringUtils.isNotBlank(result.getAppId())) { - List categories = appCategoryMap.get(result.getAppId()); - if (categories == null) { - appCategoryMap.put(result.getAppId(), result.getCategories()); - } else { - categories.addAll(result.getCategories()); - } - } - } - } - // update job categories - if (jobCategoryMap.size() > 0) { - log.info("updateJob:{},{}", logRecord.getId(), jobCategoryMap); - try { - OpenSearchWriter.getInstance().updateJob(logRecord.getJobAnalysis(), jobCategoryMap); - } catch (Exception e) { - log.error("updateJobErr:", e); - } - } - - for (Map.Entry> item : appCategoryMap.entrySet()) { - TaskApp taskApp = logRecord.getTaskApp(item.getKey()); - if (taskApp == null) { - log.error("get {} taskApp null", item.getKey()); - continue; - } - List categories = item.getValue(); - Map appCategories = new HashMap<>(); - categories.forEach(data -> appCategories.put(data, true)); - // update task-app categories - log.info("updateTaskApp:{},{}", logRecord.getId(), appCategories); - if (!appCategories.isEmpty()) { - OpenSearchWriter.getInstance().updateTaskApp(taskApp, appCategories); - continue; - } - if (logRecord.getIsOneClick()) { - OpenSearchWriter.getInstance().updateTaskApp(taskApp, appCategories); - } - } - } - - private enum OpenSearch { - - INSTANCE; - - private final OpenSearchWriter singleton; - - OpenSearch() { - singleton = new OpenSearchWriter(); - } - - public OpenSearchWriter getInstance() { - return singleton; - } - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/service/writer/ParserResultSink.java b/task-parser/src/main/java/com/oppo/cloud/parser/service/writer/ParserResultSink.java deleted file mode 100644 index 78ba6e77c..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/service/writer/ParserResultSink.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.service.writer; - -import com.oppo.cloud.common.util.textparser.ParserAction; -import com.oppo.cloud.common.util.textparser.ParserActionUtil; -import com.oppo.cloud.parser.domain.job.ParserParam; -import lombok.extern.slf4j.Slf4j; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * A collection of writers, responsible for calling multiple writers - * to write parser results to different locations. - */ -@Slf4j -public class ParserResultSink { - - List writers; - public ParserResultSink() { - this.writers = new ArrayList<>(); - } - - public void register(IParserResultWriter writer) { - this.writers.add(writer); - } - - /** - * Save matching results - */ - public List saveParserActions(String logType, String logPath, ParserParam param, Map results) { - List categories = new ArrayList<>(); - results.forEach((k, v) -> { - List list = ParserActionUtil.getLeafAction(v, true); - if (list.size() == 0) { - log.error("getLeafAction:{},{}", k, v); - list.add(v); - } - for (ParserAction parserAction : list) { - categories.add(parserAction.getCategory()); - // write parsed data by all writers registered. - for (IParserResultWriter writer : writers) { - writer.write(logType, logPath, param, parserAction); - } - } - }); - return categories; - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/utils/GCReportUtil.java b/task-parser/src/main/java/com/oppo/cloud/parser/utils/GCReportUtil.java deleted file mode 100644 index 04c3428e4..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/utils/GCReportUtil.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.utils; - -import com.oppo.cloud.common.constant.LogType; -import com.oppo.cloud.common.domain.gc.GCReport; -import com.oppo.cloud.parser.utils.gc.GCLogParserManager; -import lombok.extern.slf4j.Slf4j; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * GC log parsing util - */ -@Slf4j -public class GCReportUtil { - - public static List generateGCReports(Map gcLogMap, - String logPath) throws Exception { - if (gcLogMap == null) { - return null; - } - List gcReports = new ArrayList<>(); - for (Map.Entry executorGc : gcLogMap.entrySet()) { - GCReport gcReport = GCLogParserManager.generateGCReport(executorGc.getValue(), logPath); - if (gcReport == null) { - continue; - } - gcReport.setLogPath(logPath); - gcReport.setExecutorId(executorGc.getKey()); - if (executorGc.getKey() == 0) { - gcReport.setLogType(LogType.SPARK_DRIVER.getName()); - } else { - gcReport.setLogType(LogType.SPARK_EXECUTOR.getName()); - } - gcReports.add(gcReport); - } - return gcReports; - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/utils/HDFSUtil.java b/task-parser/src/main/java/com/oppo/cloud/parser/utils/HDFSUtil.java deleted file mode 100644 index 5146ae2c0..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/utils/HDFSUtil.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.utils; - -import com.oppo.cloud.common.constant.Constant; -import com.oppo.cloud.common.domain.cluster.hadoop.NameNodeConf; -import com.oppo.cloud.parser.domain.reader.ReaderObject; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.security.UserGroupInformation; - -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.InputStreamReader; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.security.PrivilegedExceptionAction; -import java.util.*; - -/** - * HDFSUtil - */ -@Slf4j -public class HDFSUtil { - - /** - * get hdfs NameNode - */ - public static NameNodeConf getNameNode(Map nameNodeMap, String path) { - for (String key : nameNodeMap.keySet()) { - if (key != null && !key.isEmpty() && path.contains(key)) { - return nameNodeMap.get(key); - } - } - return null; - } - - private static FileSystem getFileSystem(NameNodeConf nameNodeConf) throws Exception { - Configuration conf = new Configuration(false); - conf.setBoolean("fs.hdfs.impl.disable.cache", true); - - if (nameNodeConf.getNamenodes().length == 1) { - String defaultFs = - String.format("%s%s:%s", Constant.HDFS_SCHEME, nameNodeConf.getNamenodesAddr()[0], nameNodeConf.getPort()); - conf.set("fs.defaultFS", defaultFs); - if (nameNodeConf.isEnableKerberos()) { - return getAuthenticationFileSystem(nameNodeConf, conf); - } - URI uri = new URI(defaultFs); - return FileSystem.get(uri, conf); - } - - conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); - - String nameservices = nameNodeConf.getNameservices(); - - conf.set("fs.defaultFS", Constant.HDFS_SCHEME + nameservices); - conf.set("dfs.nameservices", nameservices); - conf.set("dfs.client.failover.proxy.provider." + nameservices, - "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"); - - for (int i = 0; i < nameNodeConf.getNamenodes().length; i++) { - String r = nameNodeConf.getNamenodes()[i]; - conf.set("dfs.namenode.rpc-address." + nameNodeConf.getNameservices() + "." + r, - nameNodeConf.getNamenodesAddr()[i] + ":" + nameNodeConf.getPort()); - } - - String nameNodes = String.join(",", nameNodeConf.getNamenodes()); - conf.set("dfs.ha.namenodes." + nameNodeConf.getNameservices(), nameNodes); - URI uri = new URI(Constant.HDFS_SCHEME + nameservices); - if (StringUtils.isNotBlank(nameNodeConf.getUser())) { - System.setProperty("HADOOP_USER_NAME", nameNodeConf.getUser()); - } - if (StringUtils.isNotBlank(nameNodeConf.getPassword())) { - System.setProperty("HADOOP_USER_PASSWORD", nameNodeConf.getPassword()); - } - if (nameNodeConf.isEnableKerberos()) { - return getAuthenticationFileSystem(nameNodeConf, conf); - } - return FileSystem.get(uri, conf); - } - - private static FileSystem getAuthenticationFileSystem(NameNodeConf nameNodeConf, Configuration conf) throws Exception { - conf.set("hadoop.security.authorization", "true"); - conf.set("hadoop.security.authentication", "kerberos"); - System.setProperty("java.security.krb5.conf", nameNodeConf.getKrb5Conf()); - conf.set("dfs.namenode.kerberos.principal.pattern", nameNodeConf.getPrincipalPattern()); - UserGroupInformation.setConfiguration(conf); - UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI( - nameNodeConf.getLoginUser(), nameNodeConf.getKeytabPath()); - return ugi.doAs((PrivilegedExceptionAction) () -> FileSystem.get(conf)); - } - - public static String[] readLines(NameNodeConf nameNode, String logPath) throws Exception { - FSDataInputStream fsDataInputStream = null; - try { - FileSystem fs = HDFSUtil.getFileSystem(nameNode); - fsDataInputStream = fs.open(new Path(logPath)); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - // 64kb - byte[] buffer = new byte[65536]; - int byteRead; - - while ((byteRead = fsDataInputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, byteRead); - } - byte[] contents = outputStream.toByteArray(); - String s = new String(contents, StandardCharsets.UTF_8); - return s.split("\n"); - } catch (Exception e) { - throw new Exception(); - } finally { - if (Objects.nonNull(fsDataInputStream)) { - fsDataInputStream.close(); - } - } - } - - public static ReaderObject getReaderObject(NameNodeConf nameNode, String path) throws Exception { - FileSystem fs = HDFSUtil.getFileSystem(nameNode); - FSDataInputStream fsDataInputStream = fs.open(new Path(path)); - ReaderObject readerObject = new ReaderObject(); - readerObject.setLogPath(path); - readerObject.setInputStream(fsDataInputStream); - readerObject.setFs(fs); - return readerObject; - } - - - public static List listFiles(NameNodeConf nameNode, String path) throws Exception { - FileSystem fs = HDFSUtil.getFileSystem(nameNode); - RemoteIterator it = fs.listFiles(new Path(path), true); - List result = new ArrayList<>(); - while (it.hasNext()) { - LocatedFileStatus locatedFileStatus = it.next(); - result.add(locatedFileStatus.getPath().toString()); - } - fs.close(); - return result; - } - - - public static List filesPattern(NameNodeConf nameNodeConf, String filePath) throws Exception { - FileSystem fs = HDFSUtil.getFileSystem(nameNodeConf); - FileStatus[] fileStatuses = fs.globStatus(new Path(filePath)); - List result = new ArrayList<>(); - if (fileStatuses == null) { - return result; - } - for (FileStatus fileStatus : fileStatuses) { - if (fs.exists(fileStatus.getPath())) { - result.add(fileStatus.getPath().toString()); - } - } - fs.close(); - return result; - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/utils/JobHistoryUtil.java b/task-parser/src/main/java/com/oppo/cloud/parser/utils/JobHistoryUtil.java deleted file mode 100644 index 6761bfc9a..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/utils/JobHistoryUtil.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.utils; - -import com.oppo.cloud.parser.domain.mr.JobHistoryFileInfo; -import com.oppo.cloud.parser.domain.mr.MRAppInfo; -import com.oppo.cloud.parser.domain.reader.ReaderObject; -import lombok.extern.slf4j.Slf4j; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -@Slf4j -public class JobHistoryUtil { - - private static final String PROPERTY = "property"; - - private static final String NAME = "name"; - - private static final String VALUE = "value"; - - - /** - * Job History File extension. - */ - public static final String JOB_HISTORY_FILE_EXTENSION = ".jhist"; - - /** - * Suffix for configuration files. - */ - public static final String JOB_CONF_FILE_NAME_SUFFIX = "_conf.xml"; - - - public static boolean isJobHistoryFile(String file) { - return file.endsWith(JOB_HISTORY_FILE_EXTENSION); - } - - public static boolean isJobConfFile(String file) { - return file.endsWith(JOB_CONF_FILE_NAME_SUFFIX); - } - - public static JobHistoryFileInfo getJobHistoryFileInfo(List readerObjects) { - ReaderObject jobHistoryReader = null; - ReaderObject confReader = null; - for (ReaderObject readerObject : readerObjects) { - if (JobHistoryUtil.isJobHistoryFile(readerObject.getLogPath())) { - jobHistoryReader = readerObject; - } - if (JobHistoryUtil.isJobConfFile(readerObject.getLogPath())) { - confReader = readerObject; - } - } - return new JobHistoryFileInfo(jobHistoryReader, confReader); - } - - public static Map parseJobConf(ReaderObject confReader) throws IOException { - Map map = new HashMap<>(); - if (confReader == null || confReader.getBufferedReader() == null) { - return map; - } - try { - String line = confReader.getBufferedReader().lines().collect(Collectors.joining()); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - Document document = builder.parse(new ByteArrayInputStream(line.getBytes())); - Element root = document.getDocumentElement(); - NodeList nodeList = root.getElementsByTagName(PROPERTY); - for (int i = 0; i < nodeList.getLength(); i++) { - Element element = (Element) nodeList.item(i); - String name = element.getElementsByTagName(NAME).item(0).getTextContent(); - String value = element.getElementsByTagName(VALUE).item(0).getTextContent(); - map.put(name, value); - } - } catch (Exception e) { - log.error("Exception: ", e); - } - return map; - } - - - public static MRAppInfo parseJobHistory(List readerObjects) throws Exception { - JobHistoryFileInfo jobHistoryFileInfo = getJobHistoryFileInfo(readerObjects); - Map conMap = parseJobConf(jobHistoryFileInfo.getConfReader()); - InputStream inputStream = jobHistoryFileInfo.getJobHistoryReader().getInputStream(); - ReplayMREventLogs replayMREventLogs = new ReplayMREventLogs(inputStream); - replayMREventLogs.parse(); - MRAppInfo appData = replayMREventLogs.getMRAppInfo(); - appData.setConfMap(conMap); - return appData; - } - - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/utils/ReplayMREventLogs.java b/task-parser/src/main/java/com/oppo/cloud/parser/utils/ReplayMREventLogs.java deleted file mode 100644 index a662d056c..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/utils/ReplayMREventLogs.java +++ /dev/null @@ -1,485 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.utils; - -import com.alibaba.fastjson2.JSON; -import com.oppo.cloud.parser.domain.mr.MRAppInfo; -import com.oppo.cloud.parser.domain.mr.MRTaskAttemptInfo; -import com.oppo.cloud.parser.domain.mr.SpeculationInfo; -import com.oppo.cloud.parser.domain.mr.event.*; -import lombok.extern.slf4j.Slf4j; -import org.apache.avro.Schema; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.Decoder; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.specific.SpecificDatumReader; -import org.apache.hadoop.fs.FSDataInputStream; - -import java.io.InputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Slf4j -public class ReplayMREventLogs { - private MRAppInfo mrAppInfo; - - private final FSDataInputStream in; - - private static final String VERSION_JSON = "Avro-Json"; - private static final String VERSION_BINARY = "Avro-Binary"; - - private static final String SPECULATION = "Speculation"; - - public ReplayMREventLogs(InputStream in) { - this.in = new FSDataInputStream(in); - this.mrAppInfo = new MRAppInfo(); - } - - - public void parse() throws Exception { - - String version = this.in.readLine(); - String eventSchema = this.in.readLine(); - Schema schema = new Schema.Parser().parse(eventSchema); - DatumReader reader = new SpecificDatumReader(schema); - Decoder decoder; - - switch (version) { - case VERSION_BINARY: - decoder = DecoderFactory.get().binaryDecoder(this.in, null); - break; - case VERSION_JSON: - decoder = DecoderFactory.get().jsonDecoder(schema, this.in); - break; - default: - throw new Exception("incompatible event log version: " + version); - } - - while (true) { - String eventStr; - try { - eventStr = reader.read(null, decoder).toString(); - } catch (Exception e) { - break; - } - try { - parseEvent(eventStr); - } catch (Exception e) { - log.error("parseEventErr: ", e); - } - } - - correlate(); - - } - - public MRAppInfo getMRAppInfo() { - return this.mrAppInfo; - } - - private void parseEvent(String eventStr) throws Exception { - Event event = JSON.parseObject(eventStr, Event.class); - switch (event.getType()) { - case JOB_SUBMITTED: - JobSubmitted jobSubmitted = JSON.parseObject(event.getEvent(), JobSubmitted.class); - handleJobSubmitted(jobSubmitted); - break; - case JOB_STATUS_CHANGED: - break; - case JOB_INFO_CHANGED: - JobInfoChange jobInfoChange = JSON.parseObject(event.getEvent(), JobInfoChange.class); - handleJobInfoChange(jobInfoChange); - break; - case JOB_INITED: - JobInit jobInit = JSON.parseObject(event.getEvent(), JobInit.class); - handleJobInit(jobInit); - break; - case JOB_PRIORITY_CHANGED: - break; - case JOB_QUEUE_CHANGED: - JobQueueChange jobQueueChange = JSON.parseObject(event.getEvent(), JobQueueChange.class); - handleJobQueueChange(jobQueueChange); - break; - case JOB_FAILED: - case JOB_KILLED: - case JOB_ERROR: - JobUnsuccessfulCompletion jobUnsuccessfulCompletion = JSON.parseObject(event.getEvent(), JobUnsuccessfulCompletion.class); - handleJobFailedEvent(jobUnsuccessfulCompletion); - break; - case JOB_FINISHED: - JobFinished jobFinished = JSON.parseObject(event.getEvent(), JobFinished.class); - handleJobFinished(jobFinished); - break; - case TASK_STARTED: - TaskStarted taskStarted = JSON.parseObject(event.getEvent(), TaskStarted.class); - handleTaskStarted(taskStarted); - break; - case TASK_FAILED: - TaskFailed taskFailed = JSON.parseObject(event.getEvent(), TaskFailed.class); - handleTaskFailed(taskFailed); - break; - case TASK_UPDATED: - TaskUpdated taskUpdated = JSON.parseObject(event.getEvent(), TaskUpdated.class); - handleTaskUpdated(taskUpdated); - break; - case TASK_FINISHED: - TaskFinished taskFinished = JSON.parseObject(event.getEvent(), TaskFinished.class); - handleTaskFinished(taskFinished); - break; - case MAP_ATTEMPT_STARTED: - case CLEANUP_ATTEMPT_STARTED: - case REDUCE_ATTEMPT_STARTED: - case SETUP_ATTEMPT_STARTED: - TaskAttemptStarted taskAttemptStarted = JSON.parseObject(event.getEvent(), TaskAttemptStarted.class); - handleTaskAttemptStarted(taskAttemptStarted); - break; - case MAP_ATTEMPT_FAILED: - case CLEANUP_ATTEMPT_FAILED: - case REDUCE_ATTEMPT_FAILED: - case SETUP_ATTEMPT_FAILED: - case MAP_ATTEMPT_KILLED: - case CLEANUP_ATTEMPT_KILLED: - case REDUCE_ATTEMPT_KILLED: - case SETUP_ATTEMPT_KILLED: - TaskAttemptUnsuccessfulCompletion taskAttemptUnsuccessfulCompletionEvent = JSON.parseObject(event.getEvent(), - TaskAttemptUnsuccessfulCompletion.class); - - handleTaskAttemptFailed(taskAttemptUnsuccessfulCompletionEvent); - break; - case MAP_ATTEMPT_FINISHED: - MapAttemptFinished mapAttemptFinishedEvent = JSON.parseObject(event.getEvent(), MapAttemptFinished.class); - handleMapAttemptFinished(mapAttemptFinishedEvent); - break; - case REDUCE_ATTEMPT_FINISHED: - ReduceAttemptFinished reduceAttemptFinished = JSON.parseObject(event.getEvent(), ReduceAttemptFinished.class); - handleReduceAttemptFinished(reduceAttemptFinished); - break; - case SETUP_ATTEMPT_FINISHED: - case CLEANUP_ATTEMPT_FINISHED: - TaskAttemptFinished taskAttemptFinished = JSON.parseObject(event.getEvent(), TaskAttemptFinished.class); - handleTaskAttemptFinished(taskAttemptFinished); - break; - case AM_STARTED: - break; - default: - log.warn("unexpected event type: {}", event.getType()); - } - } - - private void handleJobSubmitted(JobSubmitted event) { - this.mrAppInfo.setJobId(event.getJobid()); - this.mrAppInfo.setJobName(event.getJobName()); - this.mrAppInfo.setUsername(event.getUserName()); - this.mrAppInfo.setSubmitTime(event.getSubmitTime()); - } - - private void handleJobInfoChange(JobInfoChange event) { - this.mrAppInfo.setSubmitTime(event.getSubmitTime()); - this.mrAppInfo.setLaunchTime(event.getLaunchTime()); - } - - private void handleJobInit(JobInit event) { - this.mrAppInfo.setLaunchTime(event.getLaunchTime()); - this.mrAppInfo.setTotalMaps(event.getTotalMaps()); - this.mrAppInfo.setTotalReduces(event.getTotalReduces()); - } - - private void handleJobQueueChange(JobQueueChange event) { - this.mrAppInfo.setJobQueueName(event.getJobQueueName()); - } - - private void handleJobFailedEvent(JobUnsuccessfulCompletion event) { - this.mrAppInfo.setFinishTime(event.getFinishTime()); - this.mrAppInfo.setSucceededMaps(event.getFinishedMaps()); - this.mrAppInfo.setSucceededReduces(event.getFinishedReduces()); - this.mrAppInfo.setFailedMaps(event.getFailedMaps()); - this.mrAppInfo.setFailedReduces(event.getFailedReduces()); - this.mrAppInfo.setKilledMaps(event.getKilledMaps()); - this.mrAppInfo.setKilledReduces(event.getKilledReduces()); - this.mrAppInfo.setJobStatus(event.getJobStatus()); - this.mrAppInfo.setErrorInfo(event.getDiagnostics()); - } - - private void handleJobFinished(JobFinished event) { - this.mrAppInfo.setFinishTime(event.getFinishTime()); - this.mrAppInfo.setSucceededMaps(event.getFinishedMaps()); - this.mrAppInfo.setSucceededReduces(event.getFinishedReduces()); - this.mrAppInfo.setFailedMaps(event.getFailedMaps()); - this.mrAppInfo.setFailedReduces(event.getFailedReduces()); - this.mrAppInfo.setKilledMaps(event.getKilledMaps()); - this.mrAppInfo.setKilledReduces(event.getKilledReduces()); - this.mrAppInfo.setTotalCounters(handleCounters(event.getTotalCounters())); - this.mrAppInfo.setJobStatus(JobStatus.SUCCEEDED.toString()); - } - - private void handleTaskStarted(TaskStarted event) { - TaskInfo taskInfo = new TaskInfo(); - taskInfo.setTaskId(event.getTaskid()); - taskInfo.setStartTime(event.getStartTime()); - taskInfo.setTaskType(event.getTaskType()); - taskInfo.setSplitLocations(event.getSplitLocations()); - Map tasksMap = this.mrAppInfo.getTasksMap(); - tasksMap.put(event.getTaskid(), taskInfo); - this.mrAppInfo.setTasksMap(tasksMap); - } - - private void handleTaskFailed(TaskFailed event) throws Exception { - TaskInfo taskInfo = this.mrAppInfo.getTasksMap().get(event.getTaskid()); - if (taskInfo == null) { - log.warn("TaskInfo is null:{}", event.getTaskid()); - return; - } - taskInfo.setStatus(JobStatus.FAILED.name()); - taskInfo.setFinishTime(event.getFinishTime()); - taskInfo.setError(event.getError()); - taskInfo.setFailedDueToAttemptId(event.getFailedDueToAttempt()); - taskInfo.setCounters(event.getCounters()); - } - - private void handleTaskUpdated(TaskUpdated event) throws Exception { - TaskInfo taskInfo = this.mrAppInfo.getTasksMap().get(event.getTaskid()); - if (taskInfo == null) { - log.warn("TaskInfo is null:{}", event.getTaskid()); - return; - } - taskInfo.setFinishTime(event.getFinishTime()); - } - - private void handleTaskFinished(TaskFinished event) throws Exception { - TaskInfo taskInfo = this.mrAppInfo.getTasksMap().get(event.getTaskid()); - if (taskInfo == null) { - log.warn("TaskInfo is null:{}", event.getTaskid()); - return; - } - taskInfo.setCounters(event.getCounters()); - taskInfo.setFinishTime(event.getFinishTime()); - taskInfo.setStatus(JobStatus.SUCCEEDED.toString()); - taskInfo.setSuccessfulAttemptId(event.getSuccessfulAttemptId()); - } - - private void handleTaskAttemptStarted(TaskAttemptStarted event) throws Exception { - String attemptId = event.getAttemptId(); - TaskInfo taskInfo = this.mrAppInfo.getTasksMap().get(event.getTaskid()); - - TaskAttemptInfo attemptInfo = new TaskAttemptInfo(); - attemptInfo.setStartTime(event.getStartTime()); - attemptInfo.setAttemptId(event.getAttemptId()); - attemptInfo.setHttpPort(event.getHttpPort()); - attemptInfo.setTrackerName(event.getTrackerName()); - attemptInfo.setTaskType(event.getTaskType()); - attemptInfo.setShufflePort(event.getShufflePort()); - attemptInfo.setContainerId(event.getContainerId()); - Map attemptsMap = new HashMap<>(); - attemptsMap.put(attemptId, attemptInfo); - taskInfo.setAttemptsMap(attemptsMap); - } - - private void handleTaskAttemptFailed(TaskAttemptUnsuccessfulCompletion event) throws Exception { - TaskInfo taskInfo = this.mrAppInfo.getTasksMap().get(event.getTaskid()); - if (taskInfo == null) { - log.warn("TaskInfo is null:{}", event.getTaskid()); - return; - } - if (taskInfo.getAttemptsMap() == null) { - log.warn("TaskAttemptMap is null:{},{}", this.mrAppInfo.getJobId(), taskInfo.getTaskId()); - return; - } - TaskAttemptInfo attemptInfo = - taskInfo.getAttemptsMap().get(event.getAttemptId()); - if (attemptInfo == null) { - log.warn("TaskAttemptInfo is null:{}", event.getAttemptId()); - return; - } - attemptInfo.setFinishTime(event.getFinishTime()); - attemptInfo.setError(event.getError()); - attemptInfo.setStatus(event.getStatus()); - attemptInfo.setHostname(event.getHostname()); - attemptInfo.setPort(event.getPort()); - attemptInfo.setRackname(event.getRackname()); - attemptInfo.setShuffleFinishTime(event.getFinishTime()); - attemptInfo.setSortFinishTime(event.getFinishTime()); - attemptInfo.setMapFinishTime(event.getFinishTime()); - attemptInfo.setCounters(event.getCounters()); - if (JobStatus.SUCCEEDED.toString().equals(taskInfo.getStatus())) { - if (attemptInfo.getAttemptId().equals(taskInfo.getSuccessfulAttemptId())) { - taskInfo.setCounters(null); - taskInfo.setFinishTime(-1); - taskInfo.setStatus(null); - taskInfo.setSuccessfulAttemptId(null); - } - } - } - - private void handleMapAttemptFinished(MapAttemptFinished event) throws Exception { - TaskInfo taskInfo = this.mrAppInfo.getTasksMap().get(event.getTaskid()); - if (taskInfo == null) { - log.warn("TaskInfo is null:{}", event.getTaskid()); - return; - } - TaskAttemptInfo attemptInfo = - taskInfo.getAttemptsMap().get(event.getAttemptId()); - if (attemptInfo == null) { - log.warn("TaskAttemptInfo is null:{}", event.getAttemptId()); - return; - } - attemptInfo.setFinishTime(event.getFinishTime()); - attemptInfo.setStatus(event.getTaskStatus()); - attemptInfo.setState(event.getState()); - attemptInfo.setMapFinishTime(event.getMapFinishTime()); - attemptInfo.setCounters(event.getCounters()); - attemptInfo.setHostname(event.getHostname()); - attemptInfo.setPort(event.getPort()); - attemptInfo.setRackname(event.getRackname()); - } - - private void handleReduceAttemptFinished(ReduceAttemptFinished event) throws Exception { - TaskInfo taskInfo = this.mrAppInfo.getTasksMap().get(event.getTaskid()); - if (taskInfo == null) { - log.warn("TaskInfo is null:{}", event.getTaskid()); - return; - } - TaskAttemptInfo attemptInfo = - taskInfo.getAttemptsMap().get(event.getAttemptId()); - if (attemptInfo == null) { - log.warn("TaskAttemptInfo is null:{}", event.getAttemptId()); - return; - } - attemptInfo.setFinishTime(event.getFinishTime()); - attemptInfo.setStatus(event.getTaskStatus()); - attemptInfo.setState(event.getState()); - attemptInfo.setShuffleFinishTime(event.getShuffleFinishTime()); - attemptInfo.setSortFinishTime(event.getSortFinishTime()); - attemptInfo.setCounters(event.getCounters()); - attemptInfo.setHostname(event.getHostname()); - attemptInfo.setPort(event.getPort()); - attemptInfo.setRackname(event.getRackname()); - } - - private void handleTaskAttemptFinished(TaskAttemptFinished event) throws Exception { - TaskInfo taskInfo = this.mrAppInfo.getTasksMap().get(event.getTaskid()); - if (taskInfo == null) { - log.warn("TaskInfo is null:{}", event.getTaskid()); - return; - } - TaskAttemptInfo attemptInfo = - taskInfo.getAttemptsMap().get(event.getAttemptId()); - if (attemptInfo == null) { - log.warn("TaskAttemptInfo is null:{}", event.getAttemptId()); - return; - } - attemptInfo.setFinishTime(event.getFinishTime()); - attemptInfo.setStatus(event.getTaskStatus()); - attemptInfo.setState(event.getState()); - attemptInfo.setCounters(event.getJhCounters()); - attemptInfo.setHostname(event.getHostname()); - } - - private void correlate() { - List mapList = new ArrayList<>(); - List reduceList = new ArrayList<>(); - SpeculationInfo speculationInfo = new SpeculationInfo(); - for (Map.Entry map : this.mrAppInfo.getTasksMap().entrySet()) { - TaskInfo taskInfo = map.getValue(); - if (taskInfo.getAttemptsMap() == null) { - continue; - } - if (TaskType.MAP.toString().equals(taskInfo.getTaskType())) { - setMRTaskAttemptInfoList(taskInfo, mapList, speculationInfo); - } else if (TaskType.REDUCE.toString().equals(taskInfo.getTaskType())) { - setMRTaskAttemptInfoList(taskInfo, reduceList, speculationInfo); - } - } - this.mrAppInfo.setMapList(mapList); - this.mrAppInfo.setReduceList(reduceList); - this.mrAppInfo.setSpeculationInfo(speculationInfo); - if (mrAppInfo.getFinishTime() == null) { - mrAppInfo.setFinishTime(System.currentTimeMillis()); - } - long elapsedTime = mrAppInfo.getFinishTime() - mrAppInfo.getSubmitTime(); - if (elapsedTime < 0) { - elapsedTime = 0L; - } - this.mrAppInfo.setElapsedTime(elapsedTime); - } - - private void setMRTaskAttemptInfoList(TaskInfo taskInfo, List list, SpeculationInfo speculationInfo) { - - for (Map.Entry taskAttemptInfoMap : taskInfo.getAttemptsMap().entrySet()) { - TaskAttemptInfo taskAttemptInfo = taskAttemptInfoMap.getValue(); - long finishTime = taskAttemptInfo.getFinishTime() == null ? 0 : taskAttemptInfo.getFinishTime(); - long startTime = taskAttemptInfo.getStartTime() == null ? 0 : taskAttemptInfo.getStartTime(); - long elapsedTime = finishTime - startTime; - - if (taskAttemptInfo.getError() != null && taskAttemptInfo.getError().contains(SPECULATION)) { - List elapsedTimeList = speculationInfo.getElapsedTime(); - elapsedTimeList.add(elapsedTime); - List taskAttemptIds = speculationInfo.getTaskAttemptIds(); - taskAttemptIds.add(taskAttemptInfo.getAttemptId()); - speculationInfo.setTaskAttemptIds(taskAttemptIds); - } - - if (!JobStatus.SUCCEEDED.toString().equals(taskAttemptInfo.getStatus())) { - continue; - } - - Map> mapCounters = handleCounters(taskInfo.getCounters()); - MRTaskAttemptInfo mrTaskAttemptInfo = new MRTaskAttemptInfo(); - mrTaskAttemptInfo.setTaskId(getTaskId(taskAttemptInfo.getAttemptId())); - mrTaskAttemptInfo.setAttemptId(taskAttemptInfo.getAttemptId()); - mrTaskAttemptInfo.setTaskStatus(taskAttemptInfo.getStatus()); - mrTaskAttemptInfo.setStartTime(startTime); - mrTaskAttemptInfo.setFinishTime(finishTime); - mrTaskAttemptInfo.setShuffleFishTime(taskAttemptInfo.getShuffleFinishTime() == null ? 0 : taskAttemptInfo.getShuffleFinishTime()); - mrTaskAttemptInfo.setSortFinishTime(taskAttemptInfo.getSortFinishTime() == null ? 0 : taskAttemptInfo.getSortFinishTime()); - mrTaskAttemptInfo.setElapsedTime(elapsedTime); - mrTaskAttemptInfo.setError(taskAttemptInfo.getError()); - mrTaskAttemptInfo.setCounters(mapCounters); - list.add(mrTaskAttemptInfo); - - } - - } - - private Map> handleCounters(JhCounters counters) { - Map> result = new HashMap<>(); - if (counters.getGroups() == null) { - return result; - } - for (JhCounterGroup group : counters.getGroups()) { - String groupName = group.getName(); - Map counterMap = result.computeIfAbsent(groupName, k -> new HashMap<>()); - if (group.getCounts() == null) { - continue; - } - for (JhCounter counter : group.getCounts()) { - counterMap.put(counter.getName(), counter.getValue()); - } - } - return result; - } - - private int getTaskId(String id) { - String[] sp = id.split("_"); - if (sp.length < 2) { - return 0; - } - return Integer.parseInt(sp[sp.length - 2]); - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/utils/ReplaySparkEventLogs.java b/task-parser/src/main/java/com/oppo/cloud/parser/utils/ReplaySparkEventLogs.java deleted file mode 100644 index 59dca9d94..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/utils/ReplaySparkEventLogs.java +++ /dev/null @@ -1,331 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.utils; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.oppo.cloud.parser.domain.reader.ReaderObject; -import com.oppo.cloud.parser.domain.spark.eventlog.*; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * spark event log parser - */ -@Slf4j -@Data -public class ReplaySparkEventLogs { - - private SparkApplication application; - private Map jobs; - private Map jobSQLExecIDMap; - private Map executors; - private Map tasks; - private List blockManagers; - private List failedStages; - private List driverAccumUpdates; - private List sqlExecutionStarts; - private Map accumulableInfoMap; - private Map driverUpdateMap; - private List rawSQLExecutions; - private ObjectMapper objectMapper; - private Map stageIDToJobID; - private long logSize; - - public ReplaySparkEventLogs() { - application = new SparkApplication(); - jobs = new HashMap<>(); - jobSQLExecIDMap = new HashMap<>(); - executors = new HashMap<>(); - tasks = new HashMap<>(); - blockManagers = new ArrayList<>(); - failedStages = new ArrayList<>(); - driverAccumUpdates = new ArrayList<>(); - sqlExecutionStarts = new ArrayList<>(); - accumulableInfoMap = new HashMap<>(); - driverUpdateMap = new HashMap<>(); - rawSQLExecutions = new ArrayList<>(); - stageIDToJobID = new HashMap<>(); - objectMapper = new ObjectMapper(); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - } - - public void replay(ReaderObject readerObject) throws Exception { - String compressCodec = getCompressCodec(readerObject.getLogPath()); - while (true) { - String line; - try { - line = readerObject.getBufferedReader(compressCodec).readLine(); - } catch (IOException e) { - log.error(e.getMessage()); - break; - } - if (line == null) { - break; - } - parseLine(line); - } - this.correlate(); - } - - public void replay(String[] lines) throws Exception { - for (String line : lines) { - parseLine(line); - } - this.correlate(); - } - - /** - * Parse line by line - */ - private void parseLine(String line) throws Exception { - SparkListenerEvent event; - try { - event = objectMapper.readValue(line, SparkListenerEvent.class); - } catch (JsonProcessingException e) { - log.error("parseSparkEventErr:{}", line); - return; - } - switch (event.getEvent()) { - case "SparkListenerApplicationStart": - SparkListenerApplicationStart sparkListenerApplicationStart = objectMapper.readValue(line, - SparkListenerApplicationStart.class); - Long appStartTimestamp = this.application.getAppStartTimestamp(); - if (appStartTimestamp != null && appStartTimestamp < sparkListenerApplicationStart.getTime()) { - this.application.setAppStartTimestamp(appStartTimestamp); - }else { - this.application.setAppStartTimestamp(sparkListenerApplicationStart.getTime()); - } - break; - case "SparkListenerApplicationEnd": - SparkListenerApplicationEnd sparkListenerApplicationEnd = objectMapper.readValue(line, - SparkListenerApplicationEnd.class); - Long appEndTimestamp = this.application.getAppEndTimestamp(); - if (appEndTimestamp != null && appEndTimestamp > sparkListenerApplicationEnd.getTime()){ - this.application.setAppEndTimestamp(appEndTimestamp); - }else { - this.application.setAppEndTimestamp(sparkListenerApplicationEnd.getTime()); - } - break; - case "SparkListenerBlockManagerAdded": - SparkListenerBlockManagerAdded sparkListenerBlockManagerAdded = objectMapper.readValue(line, - SparkListenerBlockManagerAdded.class); - SparkBlockManager blockManager = new SparkBlockManager(sparkListenerBlockManagerAdded); - this.blockManagers.add(blockManager); - break; - case "SparkListenerBlockManagerRemoved": - SparkListenerBlockManagerRemoved sparkListenerBlockManagerRemoved = objectMapper.readValue(line, - SparkListenerBlockManagerRemoved.class); - break; - case "org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates": - SparkListenerDriverAccumUpdates sparkListenerDriverAccumUpdates = objectMapper.readValue(line, - SparkListenerDriverAccumUpdates.class); - this.driverAccumUpdates.add(sparkListenerDriverAccumUpdates); - break; - case "SparkListenerEnvironmentUpdate": - SparkListenerEnvironmentUpdate sparkListenerEnvironmentUpdate = objectMapper.readValue(line, - SparkListenerEnvironmentUpdate.class); - this.application.setSparkApplication(sparkListenerEnvironmentUpdate); - break; - case "SparkListenerExecutorAdded": - SparkListenerExecutorAdded sparkListenerExecutorAdded = objectMapper.readValue(line, - SparkListenerExecutorAdded.class); - String id = sparkListenerExecutorAdded.getExecutorId(); - this.executors.put(id, new SparkExecutor(sparkListenerExecutorAdded)); - break; - case "SparkListenerExecutorRemoved": - SparkListenerExecutorRemoved sparkListenerExecutorRemoved = objectMapper.readValue(line, - SparkListenerExecutorRemoved.class); - String execId = sparkListenerExecutorRemoved.getExecutorId(); - this.executors.get(execId).remove(sparkListenerExecutorRemoved); - break; - case "SparkListenerJobEnd": - SparkListenerJobEnd sparkListenerJobEnd = objectMapper.readValue(line, - SparkListenerJobEnd.class); - this.jobs.get(sparkListenerJobEnd.getJobId()).complete(sparkListenerJobEnd); - break; - case "SparkListenerJobStart": - SparkListenerJobStart sparkListenerJobStart = objectMapper.readValue(line, - SparkListenerJobStart.class); - if (this.jobs.get(sparkListenerJobStart.getJobId()) != null) { - log.error("ERROR: Duplicate job ID:{}", sparkListenerJobStart.getJobId()); - break; - } - SparkJob job = new SparkJob(sparkListenerJobStart); - this.jobs.put(job.getJobId(), job); - // JobId sql.execution.id relationship mapping - String sqlExecutionID = sparkListenerJobStart.getProperties().getProperty("spark.sql.execution" + - ".id"); - if (!StringUtils.isEmpty(sqlExecutionID)) { - jobSQLExecIDMap.put(job.getJobId(), Long.valueOf(sqlExecutionID)); - } - // StateId jobId relationship mapping - if (sparkListenerJobStart.getStageInfos() != null) { - sparkListenerJobStart.getStageInfos().forEach(stageInfo -> { - stageIDToJobID.put(stageInfo.getStageId(), sparkListenerJobStart.getJobId()); - }); - } - - break; - case "SparkListenerLogStart": - SparkListenerLogStart sparkListenerLogStart = objectMapper.readValue(line, - SparkListenerLogStart.class); - // this.application.setSparkVersion(sparkListenerLogStart.getSparkVersion()); - break; - case "SparkListenerSQLAdaptiveExecutionUpdate": - SparkListenerSQLAdaptiveExecutionUpdate sparkListenerSQLAdaptiveExecutionUpdate = - objectMapper.readValue(line, SparkListenerSQLAdaptiveExecutionUpdate.class); - break; - case "org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd": - SparkListenerSQLExecutionEnd sparkListenerSQLExecutionEnd = objectMapper.readValue(line, - SparkListenerSQLExecutionEnd.class); - rawSQLExecutions.add(line); - break; - case "org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart": - SparkListenerSQLExecutionStart sparkListenerSQLExecutionStart = objectMapper.readValue(line, - SparkListenerSQLExecutionStart.class); - sqlExecutionStarts.add(sparkListenerSQLExecutionStart); - rawSQLExecutions.add(line); - break; - case "SparkListenerStageCompleted": - SparkListenerStageCompleted sparkListenerStageCompleted = objectMapper.readValue(line, - SparkListenerStageCompleted.class); - Integer stageId = sparkListenerStageCompleted.getStageInfo().getStageId(); - for (SparkJob sparkJob : this.jobs.values()) { - for (SparkStage stage : sparkJob.getStages()) { - if (stage.getStageId().equals(stageId)) { - stage.complete(sparkListenerStageCompleted); - } - } - } - StageInfo stageInfo = sparkListenerStageCompleted.getStageInfo(); - if (stageInfo != null && stageInfo.getFailureReason() != null - && !stageInfo.getFailureReason().isEmpty()) { - this.getFailedStages().add(stageInfo); - } - break; - case "SparkListenerStageSubmitted": - SparkListenerStageSubmitted sparkListenerStageSubmitted = objectMapper.readValue(line, - SparkListenerStageSubmitted.class); - break; - case "SparkListenerTaskEnd": - SparkListenerTaskEnd sparkListenerTaskEnd = objectMapper.readValue(line, - SparkListenerTaskEnd.class); - Long taskId = sparkListenerTaskEnd.getTaskInfo().getTaskId(); - this.tasks.get(taskId).finish(sparkListenerTaskEnd); - // Update job data - Integer jobID = stageIDToJobID.get(sparkListenerTaskEnd.getStageId()); - if (jobID != null && sparkListenerTaskEnd.getTaskMetrics() != null) { - SparkJob sparkJob = jobs.get(jobID); - if (sparkJob != null) { - Long sum = sparkJob.getExecutorRunTime() + - sparkListenerTaskEnd.getTaskMetrics().getExecutorRunTime(); - sparkJob.setExecutorRunTime(sum); - } - } - break; - case "SparkListenerTaskGettingResult": - SparkListenerTaskGettingResult sparkListenerTaskGettingResult = objectMapper.readValue(line, - SparkListenerTaskGettingResult.class); - break; - case "SparkListenerTaskStart": - SparkListenerTaskStart sparkListenerTaskStart = objectMapper.readValue(line, - SparkListenerTaskStart.class); - taskId = sparkListenerTaskStart.getTaskInfo().getTaskId(); - this.tasks.put(taskId, new SparkTask(sparkListenerTaskStart)); - break; - default: - break; - } - - } - - /** - * Related data processing - */ - private void correlate() { - for (SparkBlockManager blockManager : this.blockManagers) { - if (!"driver".equals(blockManager.getExecutorId())) { - SparkExecutor executor = this.executors.get(blockManager.getExecutorId()); - if (executor != null) { - executor.getBlockManagers().add(blockManager); - } - } - } - - for (SparkTask task : this.tasks.values()) { - this.executors.get(task.getExecutorId()).getTasks().add(task); - for (SparkJob job : this.jobs.values()) { - for (SparkStage stage : job.getStages()) { - // Retry stageId mapping - for (Map.Entry entry : stage.getSubmissionTimeMap().entrySet()) { - Integer attemptId = entry.getKey(); - if (stage.getStageId().equals(task.getStageId()) - && attemptId.equals(task.getStageAttemptId())) { - if (stage.getTasksMap().containsKey(attemptId)) { - stage.getTasksMap().get(attemptId).add(task); - } else { - List sparkTasks = new ArrayList<>(); - sparkTasks.add(task); - stage.getTasksMap().put(attemptId, sparkTasks); - } - } - } - } - } - } - updateAccum(); - } - - /** - * update AccumulableInfo driverUpdateMap - */ - private void updateAccum() { - for (SparkJob job : this.jobs.values()) { - for (SparkStage stage : job.getStages()) { - if (stage.getAccumulableInfos() != null) { - for (AccumulableInfo info : stage.getAccumulableInfos()) { - this.accumulableInfoMap.put(info.getId(), info); - } - } - } - } - for (SparkListenerDriverAccumUpdates updates : this.driverAccumUpdates) { - for (List update : updates.getAccumUpdates()) { - this.driverUpdateMap.put(update.get(0), update.get(1)); - } - } - } - - private String getCompressCodec(String fileName) { - int lastSlashIndex = fileName.lastIndexOf("/"); - int lastDotIndex = fileName.lastIndexOf("."); - if (lastDotIndex > lastSlashIndex) { - return fileName.substring(lastDotIndex + 1); - } else { - return "none"; - } - } -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/utils/UnitUtil.java b/task-parser/src/main/java/com/oppo/cloud/parser/utils/UnitUtil.java deleted file mode 100644 index 9020eb92a..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/utils/UnitUtil.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.utils; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Unit conversion util - **/ - -@Slf4j -public class UnitUtil { - - public static Pattern UnitRegex = Pattern.compile("(?[0-9]+)(?[a-zA-Z]+)"); - - /** - * Other units convert to Byte - */ - public static Long toBytes(String s) { - if (StringUtils.isEmpty(s)) { - return 0L; - } - if (StringUtils.isNumeric(s)) { - log.warn("isNumeric:{}", s); - return Long.parseLong(s) * 1024 * 1024; - } - Matcher matcher = UnitUtil.UnitRegex.matcher(s); - if (!matcher.matches()) { - log.error("Can not match unit, raw: {}", s); - return 0L; - } - long num = Long.parseLong(matcher.group("num")); - String unit = matcher.group("unit"); - - switch (unit) { - case "g": - case "G": - case "GB": - return num * 1024 * 1024 * 1024; - case "M": - case "m": - case "MiB": - return num * 1024 * 1024; - default: - log.error("Unknown unit: {}, raw: {}", unit, s); - return 0L; - } - } - - public static Long MBToByte(Long mb) { - return mb * 1024 * 1024; - } - - public static double toKBByUnit(String doubleSize, String unit) { - switch (unit) { - case "m": - case "M": - return Double.parseDouble(doubleSize) * 1024; - case "g": - case "G": - return Double.parseDouble(doubleSize) * 1024 * 1024; - default: - return Double.parseDouble(doubleSize); - } - } - -} diff --git a/task-parser/src/main/java/com/oppo/cloud/parser/utils/gc/CMSGCParser.java b/task-parser/src/main/java/com/oppo/cloud/parser/utils/gc/CMSGCParser.java deleted file mode 100644 index 47dfed3f4..000000000 --- a/task-parser/src/main/java/com/oppo/cloud/parser/utils/gc/CMSGCParser.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2023 OPPO. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oppo.cloud.parser.utils.gc; - -import com.oppo.cloud.common.domain.gc.GCReport; -import com.oppo.cloud.parser.utils.UnitUtil; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Support to parse jdk8 cms gc logs - */ -public class CMSGCParser implements GCLogParser { - - private static final String SIZE_CHANGE_TOKEN = "->"; - private static final String CMS_INITIAL_MARK_MATCHER = "CMS Initial Mark"; - private static final String ALLOCATION_FAILURE_MATCHER = "Allocation Failure"; - private static final String CMS_REMARK_MATCHER = "CMS Final Remark"; - private static final String FULL_GC_MATCHER = "Full GC"; - - /** - * 10208K(1048576K)] 596900K(1992320K) - */ - public static String MEMORY_SIZE_REGEX = "(?[0-9]*)(?[KMG])\\((?[0-9]*)(?[KMG])\\)\\] " - + "(?[0-9]*)(?[KMG])\\((?[0-9]*)(?[KMG])\\)"; - - /** - * 897412K->70278K(943744K), 0.0936951 secs] 904070K->80486K(1992321K) - */ - public static String MEMORY_SIZE_TIME_REGEX = "(?[0-9]*)(?[KMG])->(?[0-9]*)(?[KMG])\\((?[0-9]*)(?[KMG])\\), " - + "(?