Refine Configurable Ports in DocSum (#1978)

Signed-off-by: letonghan <[email protected]>
Co-authored-by: chen, suyue <[email protected]>

Author: letonghan

DocSum/docker_compose/intel/cpu/xeon/README.md

@@ -31,7 +31,9 @@ cd GenAIExamples/DocSum/docker_compose
 source intel/set_env.sh
 ```
 
-NOTE: by default vLLM does "warmup" at start, to optimize its performance for the specified model and the underlying platform, which can take long time. For development (and e.g. autoscaling) it can be skipped with `export VLLM_SKIP_WARMUP=true`.
+> NOTE: by default vLLM does "warmup" at start, to optimize its performance for the specified model and the underlying platform, which can take long time. For development (and e.g. autoscaling) it can be skipped with `export VLLM_SKIP_WARMUP=true`.
+
+> NOTE: If any port on your local machine is occupied (like `9000/8008/8888`, etc.), modify it in `set_env.sh`, then run `source set_env.sh` again.
 
 Checkout a released version, such as v1.3:
 

DocSum/docker_compose/intel/cpu/xeon/compose.yaml

@@ -6,7 +6,7 @@ services:
     image: ${REGISTRY:-opea}/vllm:${TAG:-latest}
     container_name: docsum-xeon-vllm-service
     ports:
-      - "8008:80"
+      - ${LLM_ENDPOINT_PORT:-8008}:80
     volumes:
       - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub"
     shm_size: 1g
@@ -52,7 +52,7 @@ services:
     image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
     container_name: docsum-xeon-whisper-server
     ports:
-      - "7066:7066"
+      - ${ASR_SERVICE_PORT:-7066}:7066
     ipc: host
     environment:
       no_proxy: ${no_proxy}
@@ -74,7 +74,9 @@ services:
       - http_proxy=${http_proxy}
       - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
       - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
+      - LLM_SERVICE_PORT=${LLM_PORT}
       - ASR_SERVICE_HOST_IP=${ASR_SERVICE_HOST_IP}
+      - ASR_SERVICE_PORT=${ASR_SERVICE_PORT}
     ipc: host
     restart: always
 

DocSum/docker_compose/intel/cpu/xeon/compose_tgi.yaml

@@ -51,7 +51,7 @@ services:
     image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
     container_name: docsum-xeon-whisper-server
     ports:
-      - "7066:7066"
+      - ${ASR_SERVICE_PORT:-7066}:7066
     ipc: host
     environment:
       no_proxy: ${no_proxy}
@@ -73,7 +73,9 @@ services:
       - http_proxy=${http_proxy}
       - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
       - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
+      - LLM_SERVICE_PORT=${LLM_PORT}
       - ASR_SERVICE_HOST_IP=${ASR_SERVICE_HOST_IP}
+      - ASR_SERVICE_PORT=${ASR_SERVICE_PORT}
     ipc: host
     restart: always
 

DocSum/docker_compose/intel/hpu/gaudi/README.md

@@ -33,7 +33,9 @@ cd GenAIExamples/DocSum/docker_compose
 source intel/set_env.sh
 ```
 
-NOTE: by default vLLM does "warmup" at start, to optimize its performance for the specified model and the underlying platform, which can take long time. For development (and e.g. autoscaling) it can be skipped with `export VLLM_SKIP_WARMUP=true`.
+> NOTE: by default vLLM does "warmup" at start, to optimize its performance for the specified model and the underlying platform, which can take long time. For development (and e.g. autoscaling) it can be skipped with `export VLLM_SKIP_WARMUP=true`.
+
+> NOTE: If any port on your local machine is occupied (like `9000/8008/8888`, etc.), modify it in `set_env.sh`, then run `source set_env.sh` again.
 
 Checkout a released version, such as v1.3:
 

DocSum/docker_compose/intel/hpu/gaudi/compose.yaml

@@ -6,7 +6,7 @@ services:
     image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest}
     container_name: docsum-gaudi-vllm-service
     ports:
-      - "8008:80"
+      - ${LLM_ENDPOINT_PORT:-8008}:80
     volumes:
       - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub"
     environment:
@@ -58,7 +58,7 @@ services:
     image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
     container_name: docsum-gaudi-whisper-server
     ports:
-      - "7066:7066"
+      - ${ASR_SERVICE_PORT:-7066}:7066
     ipc: host
     environment:
       no_proxy: ${no_proxy}
@@ -85,8 +85,9 @@ services:
       - http_proxy=${http_proxy}
       - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
       - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
+      - LLM_SERVICE_PORT=${LLM_PORT}
       - ASR_SERVICE_HOST_IP=${ASR_SERVICE_HOST_IP}
-
+      - ASR_SERVICE_PORT=${ASR_SERVICE_PORT}
     ipc: host
     restart: always
 

DocSum/docker_compose/intel/hpu/gaudi/compose_tgi.yaml

@@ -62,7 +62,7 @@ services:
     image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
     container_name: docsum-gaudi-whisper-server
     ports:
-      - "7066:7066"
+      - ${ASR_SERVICE_PORT:-7066}:7066
     ipc: host
     environment:
       no_proxy: ${no_proxy}
@@ -89,8 +89,9 @@ services:
       - http_proxy=${http_proxy}
       - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
       - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
+      - LLM_SERVICE_PORT=${LLM_PORT}
       - ASR_SERVICE_HOST_IP=${ASR_SERVICE_HOST_IP}
-
+      - ASR_SERVICE_PORT=${ASR_SERVICE_PORT}
     ipc: host
     restart: always
 

DocSum/docker_compose/intel/set_env.sh

@@ -24,6 +24,7 @@ export MAX_TOTAL_TOKENS=2048
 
 export LLM_PORT=9000
 export LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}"
+export ASR_SERVICE_PORT=7066
 export DocSum_COMPONENT_NAME="OpeaDocSumvLLM" # OpeaDocSumTgi
 export FRONTEND_SERVICE_PORT=5173
 export MEGA_SERVICE_HOST_IP=${host_ip}

DocSum/tests/test_compose_on_gaudi.sh

@@ -27,6 +27,12 @@ export MAX_SEQ_LEN_TO_CAPTURE=2048
 export MAX_INPUT_TOKENS=2048
 export MAX_TOTAL_TOKENS=4096
 
+# set service host and no_proxy
+export LLM_ENDPOINT="http://vllm-service:80"
+export LLM_SERVICE_HOST_IP="llm-docsum-vllm"
+export ASR_SERVICE_HOST_IP="whisper"
+export no_proxy=$no_proxy,$LLM_SERVICE_HOST_IP,$ASR_SERVICE_HOST_IP,"vllm-service"
+
 # Get the root folder of the current script
 ROOT_FOLDER=$(dirname "$(readlink -f "$0")")
 
@@ -54,7 +60,7 @@ function build_docker_images() {
 function start_services() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
     docker compose -f compose.yaml up -d > ${LOG_PATH}/start_services_with_compose.log
-    sleep 1m
+    sleep 2m
 }
 
 get_base64_str() {