Terratorch version update & updates to ran inference on CPU

deploy_studio_k8s.sh

@@ -291,14 +291,35 @@ sed -i -e "s|<pgbouncer_port>|${pgbouncer_port}|g" workspace/${DEPLOYMENT_ENV}/v
 sed -i -e "s|<pgbouncer_user>|${pgbouncer_username}|g" workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml
 sed -i -e "s|<pgbouncer_pass>|${pgbouncer_password}|g" workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml
 
+# The line below removes GPUs from the pipeline components and Finetuning job, to leave GPUs activated, copy out this line
+
+# Call the function
+get_menu_selection \
+    "Select whether you have GPU available in your cluster: " \
+    gpu_configuration_type \
+    "$gpu_configuration_options"
+
 # The line below removes GPUs from the pipeline components, to leave GPUs activated, copy out this line
 NVIDIA_GPUS_AVAILABLE=$(kubectl describe node ${CLUSTER_NODE_NAME} | grep -c "nvidia.com")
-if [ "$NVIDIA_GPUS_AVAILABLE" -gt 0 ]; then
+
+if [[ "$gpu_configuration_type" == "GPU-Available" && "$NVIDIA_GPUS_AVAILABLE" -gt 0 ]]; then
+    # Get number of GPUs
     echo "Cluster Type: nvkind"
     python ./deployment-scripts/remove-pipeline-gpu.py --remove-affinity-only workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml
+
+    # Keep the Job GPU configuration as is. 
+    echo -e "\n Keeping GPU configuration for Finetuning job in values.yaml. You can update these later in workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml "
+    echo -e "and update the cluster later using: helm upgrade geospatial-studio ./geospatial-studio/ \n"
 else
     echo "Cluster Type: standard kind"
     python ./deployment-scripts/remove-pipeline-gpu.py workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml
+
+    # remove job GPU request
+    echo -e "\n Removing GPU configuration from values.yaml"
+    python ./deployment-scripts/update_jobs_gpu.py --filename workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml \
+    --gpu-limit 0 \
+    --gpu-request 0
+    echo -e "--------------------------- Removed GPUs in the Cluster ------------------- \n"
 fi
 
 echo "**********************************************************************"
@@ -312,6 +333,64 @@ if [[ "${NON_INTERACTIVE:-false}" != "true" ]]; then
     read ans
 fi
 
+echo "**********************************************************************"
+echo "**********************************************************************"
+echo "------  Configure Fine-Tuning Job Resources  -------------------------"
+echo "**********************************************************************"
+echo "**********************************************************************"
+
+
+# Ask user if they want to alter memory, CPU requests and limits for finetuning.
+if [[ "${NON_INTERACTIVE:-false}" != "true" ]]; then
+    printf "%s " "Do you want to alter memory, CPU requests and limits for finetuning? (y/n) "
+    read ans
+else
+    # Non-interactive mode: use CONFIGURE_RESOURCES environment variable (default to "n")
+    ans="${CONFIGURE_RESOURCES:-n}"
+    echo "Non-interactive mode: CONFIGURE_RESOURCES=$ans"
+fi
+
+# If yes, prompt user for memory limit, CPU limit, memory request and CPU request.
+if [ "$ans" = "y" ]; then
+    echo "Updating memory, CPU requests and limits for finetuning."
+    echo ""
+
+    # Prompt for CPU limit
+    printf "%s " "CPU limit in cores (default: 4): "
+    read cpu_limit
+    cpu_limit=${cpu_limit:-4}
+
+    # Prompt for CPU request
+    printf "%s " "CPU request in cores (default: 2): "
+    read cpu_request
+    cpu_request=${cpu_request:-2}
+
+    # Prompt for Memory limit
+    printf "%s " "Memory limit in GB (default: 10): "
+    read memory_limit
+    memory_limit=${memory_limit:-10}
+
+    # Prompt for Memory request
+    printf "%s " "Memory request in GB (default: 6): "
+    read memory_request
+    memory_request=${memory_request:-6}
+
+    echo -e "\n Applying configuration:"
+    echo "  CPU Limit: ${cpu_limit} cores, CPU Request: ${cpu_request} cores"
+    echo -e "  Memory Limit: ${memory_limit}GB, Memory Request: ${memory_request}GB \n"
+
+    # Call the update script with user-provided values
+    python3 ./deployment-scripts/update_jobs_gpu.py --filename workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml \
+        --cpu-limit "$cpu_limit" \
+        --cpu-request "$cpu_request" \
+        --memory-limit "$memory_limit" \
+        --memory-request "$memory_request"
+    echo -e "\n Updated finetuning resource configurations \n"
+else
+    echo -e "\n Not updating resource configurations."
+    echo "You can manually edit workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml"
+    echo -e "and update the cluster later using: helm upgrade geospatial-studio ./geospatial-studio/ \n"
+fi
 
 echo "----------------------------------------------------------------------"
 echo "----------------  Building Helm dependencies  ------------------------"

deploy_studio_lima.sh

@@ -278,6 +278,87 @@ if [[ "${NON_INTERACTIVE:-false}" != "true" ]]; then
     read ans
 fi
 
+echo "**********************************************************************"
+echo "**********************************************************************"
+echo "------  Configure Fine-Tuning Job Resources  -------------------------"
+echo "**********************************************************************"
+echo "**********************************************************************"
+
+# Ask user if they have GPUs in their cluster. If Yes, keep configuration as is. If No, remove GPU configuration via the values.yaml
+if [[ "${NON_INTERACTIVE:-false}" != "true" ]]; then
+    printf "%s " "Do you have GPUs in your cluster? (y/n): "
+    read ans
+else
+    # Non-interactive mode: use HAS_GPU environment variable (default to "n" for no GPU)
+    ans="${HAS_GPU:-n}"
+    echo "Non-interactive mode: HAS_GPU=$ans"
+fi
+
+if [ "$ans" = "y" ]; then
+    echo -e "\n Keeping GPU configuration in values.yaml. You can update these later in workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml "
+    echo -e "and update the cluster later using: helm upgrade geospatial-studio ./geospatial-studio/ \n"
+else
+    echo -e "\n Removing GPU configuration from values.yaml"
+    python ./deployment-scripts/update_jobs_gpu.py --filename workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml \
+      --gpu-limit 0 \
+      --gpu-request 0
+    echo -e "\n --------------------------- Removed GPUs in the Cluster ------------------- \n"
+fi
+
+
+# Ask user if they want to alter memory, CPU requests and limits for finetuning.
+if [[ "${NON_INTERACTIVE:-false}" != "true" ]]; then
+    printf "%s " "Do you want to alter memory, CPU requests and limits for finetuning? (y/n) "
+    read ans
+else
+    # Non-interactive mode: use CONFIGURE_RESOURCES environment variable (default to "n")
+    ans="${CONFIGURE_RESOURCES:-n}"
+    echo "Non-interactive mode: CONFIGURE_RESOURCES=$ans"
+fi
+
+# If yes, prompt user for memory limit, CPU limit, memory request and CPU request.
+if [ "$ans" = "y" ]; then
+    echo "Updating memory, CPU requests and limits for finetuning."
+    echo ""
+
+    # Prompt for CPU limit
+    printf "%s " "CPU limit in cores (default: 4): "
+    read cpu_limit
+    cpu_limit=${cpu_limit:-4}
+
+    # Prompt for CPU request
+    printf "%s " "CPU request in cores (default: 2): "
+    read cpu_request
+    cpu_request=${cpu_request:-2}
+
+    # Prompt for Memory limit
+    printf "%s " "Memory limit in GB (default: 10): "
+    read memory_limit
+    memory_limit=${memory_limit:-10}
+
+    # Prompt for Memory request
+    printf "%s " "Memory request in GB (default: 6): "
+    read memory_request
+    memory_request=${memory_request:-6}
+
+    echo -e "\n Applying configuration:"
+    echo "  CPU Limit: ${cpu_limit} cores, CPU Request: ${cpu_request} cores"
+    echo -e "  Memory Limit: ${memory_limit}GB, Memory Request: ${memory_request}GB \n"
+
+    # Call the update script with user-provided values
+    python3 ./deployment-scripts/update_jobs_gpu.py --filename workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml \
+        --cpu-limit "$cpu_limit" \
+        --cpu-request "$cpu_request" \
+        --memory-limit "$memory_limit" \
+        --memory-request "$memory_request"
+    echo -e " \n Updated finetuning resource configurations \n"
+else
+    echo -e "\n Not updating resource configurations"
+    echo  "You can manually edit workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml"
+    echo -e "and update the cluster later using: helm upgrade geospatial-studio ./geospatial-studio/ \n"
+fi
+
+
 
 echo "----------------------------------------------------------------------"
 echo "----------------  Building Helm dependencies  ------------------------"

deploy_studio_ocp.sh

@@ -271,6 +271,8 @@ EOF
     echo "--------------------------  COS_STORAGE_CLASS -------------------------------------"
     echo "------------------------  NON_COS_STORAGE_CLASS ---------------------------------"
     echo "***********************************************************************************"
+    echo "-- Check StorageClasses values in the cluster for COS storage and block storage ---"
+
 
     while true; do
         printf "%s " "Press enter to continue"
@@ -459,7 +461,7 @@ EOF
 
         # For CRC, we need volume permissions enabled, so don't use DO_NOT_SET_SCC
         # For other OpenShift environments, storage may be pre-configured
-        if [[ "$DEPLOYMENT_ENV" == "crc" ]] || [[ "$DEPLOYMENT_ENV" == "crc-local" ]]; then
+        if ([[ "$DEPLOYMENT_ENV" == "crc" ]] || [[ "$DEPLOYMENT_ENV" == "crc-local" ]]) && [[ "$OC_PROJECT" == "default" ]]; then
             ./deployment-scripts/install-postgres.sh UPDATE_STORAGE DISABLE_PV
         else
             ./deployment-scripts/install-postgres.sh UPDATE_STORAGE DISABLE_PV DO_NOT_SET_SCC
@@ -791,8 +793,19 @@ EOF
 
     if [[ "$gpu_configuration_type" == "GPU-Available" ]]; then
         python ./deployment-scripts/remove-pipeline-gpu.py --remove-affinity-only workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml
+
+        # Keep the Job GPU configuration as is. 
+        echo "Keeping GPU configuration for Finetuning job in values.yaml. You can update these later in workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml "
+        echo "and update the cluster later using: helm upgrade geospatial-studio ./geospatial-studio/"
     else
         python ./deployment-scripts/remove-pipeline-gpu.py workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml
+
+        # remove job GPU request
+        echo "Removing GPU configuration from values.yaml"
+        python ./deployment-scripts/update_jobs_gpu.py --filename workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml \
+        --gpu-limit 0 \
+        --gpu-request 0
+        echo "--------------------------- Removed GPUs in the Cluster -------------------"
     fi
 
 else
@@ -819,8 +832,70 @@ echo "-----------  Make any changes to deployment values yaml --------------"
 echo "**********************************************************************"
 echo "**********************************************************************"
 
-printf "%s " "Press enter to continue"
-read ans
+if [[ "${NON_INTERACTIVE:-false}" != "true" ]]; then
+    printf "%s " "Press enter to continue"
+    read ans
+fi
+
+echo "**********************************************************************"
+echo "**********************************************************************"
+echo "------  Configure Fine-Tuning Job Resources  -------------------------"
+echo "**********************************************************************"
+echo "**********************************************************************"
+
+
+# Ask user if they want to alter memory, CPU requests and limits for finetuning.
+configure_resources_options="No Yes"
+typeset configure_resources
+
+# Call the function
+get_menu_selection \
+    "Do you want to alter memory, CPU requests and limits for finetuning?" \
+    configure_resources \
+    "$configure_resources_options"
+
+# If yes, prompt user for memory limit, CPU limit, memory request and CPU request.
+if [ "$configure_resources" = "Yes" ]; then
+    echo "Updating memory, CPU requests and limits for finetuning."
+    echo ""
+
+    # Prompt for CPU limit
+    printf "%s " "CPU limit in cores (default: 4): "
+    read cpu_limit
+    cpu_limit=${cpu_limit:-4}
+
+    # Prompt for CPU request
+    printf "%s " "CPU request in cores (default: 2): "
+    read cpu_request
+    cpu_request=${cpu_request:-2}
+
+    # Prompt for Memory limit
+    printf "%s " "Memory limit in GB (default: 10): "
+    read memory_limit
+    memory_limit=${memory_limit:-10}
+
+    # Prompt for Memory request
+    printf "%s " "Memory request in GB (default: 6): "
+    read memory_request
+    memory_request=${memory_request:-6}
+
+    echo -e "\n Applying configuration:"
+    echo "  CPU Limit: ${cpu_limit} cores, CPU Request: ${cpu_request} cores"
+    echo -e "  Memory Limit: ${memory_limit}GB, Memory Request: ${memory_request}GB \n"
+
+    # Call the update script with user-provided values
+    python3 ./deployment-scripts/update_jobs_gpu.py --filename workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml \
+        --cpu-limit "$cpu_limit" \
+        --cpu-request "$cpu_request" \
+        --memory-limit "$memory_limit" \
+        --memory-request "$memory_request"
+    echo -e " \n Updated finetuning resource configurations \n"
+else
+    echo -e "\n Not updating resource configurations."
+    echo "You can manually edit workspace/${DEPLOYMENT_ENV}/values/geospatial-studio/values-deploy.yaml"
+    echo -e "and update the cluster later using: helm upgrade geospatial-studio ./geospatial-studio/ \n"
+fi
+
 
 echo "----------------------------------------------------------------------"
 echo "----------------  Building Helm dependencies  ------------------------"

geospatial-studio/values.yaml

@@ -206,6 +206,17 @@ gfm-studio-gateway:
       ## If multiple values are given, the pod can be scheduled on any node
       ## matching one of them.
       # NODE_GPU_SPEC: NVIDIA-A100-SXM4-80GB
+      # Use defaults in the defined jobs
+      # Resource requests (guaranteed minimum for fine-tuning jobs)
+      RESOURCE_LIMIT_GPU: 1         # Number of GPUs
+      RESOURCE_REQUEST_GPU: 1      # Number of GPUs
+
+      RESOURCE_LIMIT_CPU: 4   # CPU cores
+      RESOURCE_REQUEST_CPU: 2     
+
+      RESOURCE_LIMIT_Memory: 10     # Memory in GB
+      RESOURCE_REQUEST_Memory: 6   # Memory in GB
+
 
 # geofm-ui
 geofm-ui: