[fix] Fix the kind environemnt and set gateway service to be NodePort

Signed-off-by: Kfir Toledo <[email protected]>

Author: kfirtoledo

DEVELOPMENT.md

@@ -191,7 +191,7 @@ export REGISTRY_SECRET=anna-pull-secret
 
 Set the `VLLM_MODE` environment variable based on which version of vLLM you want to deploy:
 
-* `vllm-sim`: Lightweight simulator for simple environments (defult).
+* `vllm-sim`: Lightweight simulator for simple environments (default).
 * `vllm`: Full vLLM model server, using GPU/CPU for inferencing
 * `vllm-p2p`: Full vLLM with LMCache P2P support for enable KV-Cache aware routing
 
@@ -224,19 +224,19 @@ kubectl port-forward service/inference-gateway 8080:80
 
 And making requests with `curl`:
 
-- vllm-sim
+**vllm-sim:**
 
-    ```bash
-    curl -s -w '\n' http://localhost:8080/v1/completions -H 'Content-Type: application/json' \
-      -d '{"model":"food-review","prompt":"hi","max_tokens":10,"temperature":0}' | jq
-    ```
+```bash
+curl -s -w '\n' http://localhost:8080/v1/completions -H 'Content-Type: application/json' \
+  -d '{"model":"food-review","prompt":"hi","max_tokens":10,"temperature":0}' | jq
+```
 
-- vllm or vllm-p2p
+**vllm or vllm-p2p:**
 
-  ```bash
-  curl -s -w '\n' http://localhost:8080/v1/completions -H 'Content-Type: application/json' \
-    -d '{"model":"meta-llama/Llama-3.1-8B-Instruct","prompt":"hi","max_tokens":10,"temperature":0}' | jq
-  ```
+```bash
+curl -s -w '\n' http://localhost:8080/v1/completions -H 'Content-Type: application/json' \
+  -d '{"model":"meta-llama/Llama-3.1-8B-Instruct","prompt":"hi","max_tokens":10,"temperature":0}' | jq
+```
 
 #### Environment Configurateion
 

deploy/components/inference-gateway/deployments.yaml

@@ -48,11 +48,3 @@ spec:
             service: inference-extension
           initialDelaySeconds: 5
           periodSeconds: 10
-        env:
-          - name: KVCACHE_INDEXER_REDIS_ADDR
-            value: ${REDIS_HOST}:${REDIS_PORT}
-          - name: HF_TOKEN
-            valueFrom:
-              secretKeyRef:
-                name: ${HF_SECRET_NAME}
-                key: ${HF_SECRET_KEY}

deploy/components/inference-gateway/kustomization.yaml

@@ -26,8 +26,6 @@ resources:
 - deployments.yaml
 - gateways.yaml
 - httproutes.yaml
-- secret.yaml
-
 
 images:
 - name: quay.io/vllm-d/gateway-api-inference-extension/epp

deploy/components/vllm/kustomization.yaml

@@ -33,4 +33,4 @@ images:
 configMapGenerator:
 - name: vllm-model-config
   literals:
-    - MODEL_NAME=${MODEL_NAME}
+    - MODEL_NAME=${MODEL_NAME}

deploy/environments/dev/kubernetes-kgateway/gateway-parameters.yaml

@@ -11,7 +11,7 @@ spec:
         runAsNonRoot: true
         runAsUser: "${PROXY_UID}"
     service:
-      type: LoadBalancer
+      type: ${GATEWAY_SERVICE_TYPE}
       extraLabels:
         gateway: custom
     podTemplate:

deploy/environments/dev/kubernetes-kgateway/kustomization.yaml

@@ -4,6 +4,7 @@ kind: Kustomization
 namespace: ${NAMESPACE}
 
 resources:
+- secret.yaml
 - ../../../components/inference-gateway/
 - gateway-parameters.yaml
 
@@ -14,4 +15,4 @@ images:
 
 patches:
 - path: patch-deployments.yaml
-- path: patch-gateways.yaml
+- path: patch-gateways.yaml

deploy/environments/dev/kubernetes-kgateway/patch-deployments.yaml

@@ -22,3 +22,11 @@ spec:
         - "9002"
         - -grpcHealthPort
         - "9003"
+        env:
+          - name: KVCACHE_INDEXER_REDIS_ADDR
+            value: ${REDIS_HOST}:${REDIS_PORT}
+          - name: HF_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: hf-token
+                key: ${HF_SECRET_KEY}

deploy/components/inference-gateway/secret.yaml renamed to deploy/environments/dev/kubernetes-kgateway/secret.yaml

@@ -5,7 +5,7 @@ resources:
 - ../../../../components/vllm/
 
 images:
-- name: quay.io/vllm-d/vllm-d-dev:0.0.2
+- name: quay.io/vllm-d/vllm-d-dev
   newName: ${VLLM_IMAGE}
   newTag: ${VLLM_TAG}
 

deploy/environments/dev/kubernetes-vllm/vllm/kustomization.yaml

@@ -25,6 +25,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # Set the host port to map to the Gateway's inbound port (30080)
 : "${GATEWAY_HOST_PORT:=30080}"
 
+# Set the inference pool name for the deployment
+export POOL_NAME="${POOL_NAME:-vllm-llama3-8b-instruct}"
+
+# Set the model name to deploy
+export MODEL_NAME="${MODEL_NAME:-meta-llama/Llama-3.1-8B-Instruct}"
 # ------------------------------------------------------------------------------
 # Setup & Requirement Checks
 # ------------------------------------------------------------------------------
@@ -113,7 +118,7 @@ kustomize build --enable-helm deploy/components/crds-kgateway |
 
 # Deploy the environment to the "default" namespace
 kustomize build --enable-helm deploy/environments/dev/kind-kgateway \
-	| sed "s/REPLACE_NAMESPACE/${PROJECT_NAMESPACE}/gI" \
+	| envsubst | sed "s/REPLACE_NAMESPACE/${PROJECT_NAMESPACE}/gI" \
 	| kubectl --context ${KUBE_CONTEXT} apply -f -
 
 # Wait for all control-plane pods to be ready