diff --git a/.github/workflows/dev-images.yaml b/.github/workflows/dev-images.yaml index 4f03d4917..07e8e5076 100644 --- a/.github/workflows/dev-images.yaml +++ b/.github/workflows/dev-images.yaml @@ -20,7 +20,7 @@ env: jobs: build-and-push-images: - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: component: @@ -46,9 +46,6 @@ jobs: go-version: ${{ env.GO_VERSION }} cache: true - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -79,7 +76,7 @@ jobs: with: context: . file: ${{ matrix.component.dockerfile }} - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ matrix.component.image }}:dev @@ -114,7 +111,7 @@ jobs: publish-dev-charts: needs: build-and-push-images - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 @@ -181,7 +178,7 @@ jobs: notify: needs: [build-and-push-images, publish-dev-charts] - runs-on: ubuntu-latest + runs-on: self-hosted if: always() steps: - name: Summary diff --git a/.github/workflows/openvex.yaml b/.github/workflows/openvex.yaml index 88c105adf..b3d3467cd 100644 --- a/.github/workflows/openvex.yaml +++ b/.github/workflows/openvex.yaml @@ -16,7 +16,7 @@ permissions: jobs: generate-vex: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 7fad26eef..45b986235 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -23,7 +23,7 @@ concurrency: jobs: # Build job build: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout uses: actions/checkout@v6 @@ -75,7 +75,7 @@ jobs: environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest + runs-on: self-hosted needs: build steps: - name: Deploy to GitHub Pages diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index f15254a7e..ad35064e3 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -10,7 +10,7 @@ permissions: jobs: label: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Auto-label by file changes uses: actions/labeler@v6 diff --git a/.github/workflows/pr-push-validation.yml b/.github/workflows/pr-push-validation.yml index 2ffff71ec..4519c0eb5 100644 --- a/.github/workflows/pr-push-validation.yml +++ b/.github/workflows/pr-push-validation.yml @@ -12,7 +12,7 @@ permissions: jobs: lint: name: Lint - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 @@ -45,7 +45,7 @@ jobs: test: name: Test - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: test-suite: [unit] # TODO: Add 'integration' back once fixed @@ -99,7 +99,7 @@ jobs: build: name: Build - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: component: [ome-manager, model-agent, multinode-prober, ome-agent] @@ -144,7 +144,7 @@ jobs: docker-build: name: Docker Build - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: image: [ome-image, model-agent-image, multinode-prober-image, ome-agent-image] @@ -189,7 +189,7 @@ jobs: security-scan: name: Security Scan - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 @@ -212,7 +212,7 @@ jobs: license-check: name: License Check - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 @@ -233,7 +233,7 @@ jobs: all-checks-passed: name: All CI Checks Passed needs: [lint, test, build, docker-build, security-scan, license-check] - runs-on: ubuntu-latest + runs-on: self-hosted if: always() steps: - name: Check all job statuses diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index f3e943fce..c1f887444 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -18,7 +18,7 @@ concurrency: jobs: pre-commit-checks: name: Pre-commit Checks - runs-on: ubuntu-latest + runs-on: self-hosted timeout-minutes: 10 steps: - name: Checkout code @@ -79,7 +79,7 @@ jobs: test-and-build: name: Test and Build - runs-on: ubuntu-latest + runs-on: self-hosted timeout-minutes: 20 # Remove dependency so tests always run independently steps: @@ -141,7 +141,7 @@ jobs: docker-validation: name: Docker Build Validation - runs-on: ubuntu-latest + runs-on: self-hosted timeout-minutes: 15 needs: pre-commit-checks strategy: @@ -193,7 +193,7 @@ jobs: # Optional multi-arch validation - only runs when 'test-multiarch' label is present docker-multiarch-validation: name: Docker Multi-Arch Build Validation - runs-on: ubuntu-latest + runs-on: self-hosted timeout-minutes: 30 needs: pre-commit-checks if: contains(github.event.pull_request.labels.*.name, 'test-multiarch') @@ -252,7 +252,7 @@ jobs: summary: name: PR Validation Summary - runs-on: ubuntu-latest + runs-on: self-hosted needs: [pre-commit-checks, test-and-build, docker-validation] if: always() steps: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d2851a069..410a343b8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -25,7 +25,7 @@ env: jobs: prepare: - runs-on: ubuntu-latest + runs-on: self-hosted outputs: version: ${{ steps.version.outputs.version }} tag: ${{ steps.version.outputs.tag }} @@ -45,7 +45,7 @@ jobs: build-images: needs: prepare - runs-on: ubuntu-latest + runs-on: self-hosted strategy: matrix: component: @@ -71,9 +71,6 @@ jobs: go-version: ${{ env.GO_VERSION }} cache: true - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -93,7 +90,7 @@ jobs: with: context: . file: ${{ matrix.component.dockerfile }} - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ matrix.component.image }}:${{ needs.prepare.outputs.tag }} @@ -127,7 +124,7 @@ jobs: publish-helm-charts: needs: [prepare, build-images] - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 @@ -195,7 +192,7 @@ jobs: generate-sboms: needs: [prepare, build-images] - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 @@ -224,7 +221,7 @@ jobs: create-release: needs: [prepare, build-images, publish-helm-charts, generate-sboms] - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 diff --git a/.github/workflows/sbom.yaml b/.github/workflows/sbom.yaml index 49667c4d0..b7e0f0c85 100644 --- a/.github/workflows/sbom.yaml +++ b/.github/workflows/sbom.yaml @@ -15,7 +15,7 @@ permissions: jobs: generate-sbom: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 diff --git a/.github/workflows/sync-dependabot.yaml b/.github/workflows/sync-dependabot.yaml index dccf58f60..4d6be8d19 100644 --- a/.github/workflows/sync-dependabot.yaml +++ b/.github/workflows/sync-dependabot.yaml @@ -18,7 +18,7 @@ permissions: jobs: sync: if: github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v6 diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index 70c3aad6c..02ee82364 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -14,7 +14,7 @@ permissions: jobs: sync: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout uses: actions/checkout@v6 diff --git a/.github/workflows/vsco-publish-helm-charts.yaml b/.github/workflows/vsco-publish-helm-charts.yaml new file mode 100644 index 000000000..0e4f814f4 --- /dev/null +++ b/.github/workflows/vsco-publish-helm-charts.yaml @@ -0,0 +1,101 @@ +# ============================================================================= +# VSCO FORK ONLY — REMOVE BEFORE OPENING A PR TO sgl-project/ome +# +# Packages ome-crd, ome-resources, ome-serving and pushes OCI charts to: +# oci://ghcr.io/vsco/charts +# +# Use a Helm semver for chart_version (e.g. 0.1.5-vsco7), not a leading "v" on chart_version. +# Point shared-infra at the same semver: var.ome_version = "0.1.5-vsco7" and +# repository = "oci://ghcr.io/vsco/charts" +# in terraform/modules/eks/backend-services/ome.tf +# +# Auth: repo secret VSCO_GHCR_TOKEN (PAT write:packages) if GITHUB_TOKEN cannot push. +# ============================================================================= + +name: VSCO (fork) publish OME Helm charts to ghcr.io/vsco + +on: + workflow_dispatch: + inputs: + chart_version: + description: 'Helm chart semver (e.g. 0.1.5-vsco7) — no leading v' + required: true + default: '0.1.5-vsco7' + image_tag: + description: 'OME image tag written into values.yaml (e.g. v0.1.5-vsco7)' + required: true + default: 'v0.1.5-vsco7' + +permissions: + contents: read + packages: write + +env: + REGISTRY: ghcr.io + CHART_ORG: vsco + +jobs: + publish-charts: + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Helm + uses: azure/setup-helm@v4 + with: + version: v3.16.0 + + - name: Install yq + run: | + curl -fsSL -X GET "https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_amd64" -o /usr/local/bin/yq + chmod +x /usr/local/bin/yq + + - name: Log in to ghcr.io (org vsco packages) + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.VSCO_GHCR_TOKEN || secrets.GITHUB_TOKEN }} + + - name: Stamp chart and image versions + run: | + set -euo pipefail + VERSION="${{ inputs.chart_version }}" + TAG="${{ inputs.image_tag }}" + for chart in charts/*/; do + yq eval -i ".version = \"${VERSION}\"" "${chart}Chart.yaml" + yq eval -i ".appVersion = \"${TAG}\"" "${chart}Chart.yaml" + done + chart="charts/ome-resources/" + if [[ -f "${chart}values.yaml" ]]; then + yq eval -i ".ome.version = \"${TAG}\"" "${chart}values.yaml" + yq eval -i ".ome.benchmarkJob.tag = \"${TAG}\"" "${chart}values.yaml" + yq eval -i ".ome.multinodeProber.tag = \"${TAG}\"" "${chart}values.yaml" + yq eval -i ".ome.omeAgent.tag = \"${TAG}\"" "${chart}values.yaml" + yq eval -i ".modelAgent.image.tag = \"${TAG}\"" "${chart}values.yaml" + fi + + - name: Package and push charts + env: + CHART_VERSION: ${{ inputs.chart_version }} + run: | + set -euo pipefail + mkdir -p .charts-out + for chart in charts/*/; do + helm package "${chart}" -d .charts-out + done + for tgz in .charts-out/*.tgz; do + helm push "${tgz}" "oci://${{ env.REGISTRY }}/${{ env.CHART_ORG }}/charts" + done + echo "Published charts at oci://${{ env.REGISTRY }}/${{ env.CHART_ORG }}/charts (chart version ${CHART_VERSION})" + + - name: Summary + run: | + echo "## VSCO OME Helm charts" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "OCI registry: \`oci://${{ env.REGISTRY }}/${{ env.CHART_ORG }}/charts\`" >> "$GITHUB_STEP_SUMMARY" + echo "Chart semver: \`${{ inputs.chart_version }}\`" >> "$GITHUB_STEP_SUMMARY" + echo "Image tag in ome-resources values: \`${{ inputs.image_tag }}\`" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Set \`var.ome_version\` to the chart semver and \`repository\` to this OCI URL in shared-infra \`ome.tf\`." >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/vsco-publish-ome-manager.yaml b/.github/workflows/vsco-publish-ome-manager.yaml new file mode 100644 index 000000000..5a5f29f4f --- /dev/null +++ b/.github/workflows/vsco-publish-ome-manager.yaml @@ -0,0 +1,97 @@ +## ============================================================================= +## VSCO FORK ONLY — REMOVE BEFORE OPENING A PR TO sgl-project/ome +## +## This workflow builds dockerfiles/manager.Dockerfile and pushes ONLY to: +## ghcr.io/vsco/ome-manager: +## (Not ghcr.io/moirai-internal — that is upstream Makefile / other workflows.) +## +## Delete this entire file when upstreaming your fork (or keep it only on a +## long-lived internal branch). Upstream does not need VSCO registry automation. +## +## Usage: +## Actions → "VSCO (fork): publish ome-manager to ghcr.io/vsco" → Run workflow +## Default tag matches local manual pushes: v0.1.5-vsco7. Override "image_tag" +## for a new release, or clear it to use git short SHA instead. +## +## Auth: default GITHUB_TOKEN usually works for packages:write in the vsco org. +## If pushes fail with 403, add a repo secret VSCO_GHCR_TOKEN (PAT with +## write:packages) and uncomment the password line below. +## ============================================================================= +# +#name: VSCO (fork) publish ome-manager to ghcr.io/vsco +# +#on: +# workflow_dispatch: +# inputs: +# image_tag: +# description: 'Tag for ghcr.io/vsco/ome-manager (empty = git short SHA)' +# required: false +# default: 'v0.1.5-vsco7' +# +#permissions: +# contents: read +# packages: write +# +#env: +# REGISTRY: ghcr.io +# IMAGE_ORG: vsco +# IMAGE_NAME: ome-manager +# GO_VERSION: '1.25' +# +#jobs: +# build-push-ome-manager: +# runs-on: self-hosted +# steps: +# - name: Checkout +# uses: actions/checkout@v6 +# +# - name: Setup Go +# uses: actions/setup-go@v6 +# with: +# go-version: ${{ env.GO_VERSION }} +# cache: true +# +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v3 +# +# - name: Log in to ghcr.io (org vsco packages) +# uses: docker/login-action@v4 +# with: +# registry: ${{ env.REGISTRY }} +# username: ${{ github.actor }} +# # Optional: set repo secret VSCO_GHCR_TOKEN (PAT, write:packages) if GITHUB_TOKEN cannot push +# password: ${{ secrets.VSCO_GHCR_TOKEN || secrets.GITHUB_TOKEN }} +# +# - name: Compute image tag +# id: tag +# run: | +# TAG="${{ inputs.image_tag }}" +# if [ -z "$TAG" ]; then +# TAG="sha-$(echo '${{ github.sha }}' | cut -c1-7)" +# fi +# echo "tag=${TAG}" >> "$GITHUB_OUTPUT" +# echo "Publishing to ghcr.io/vsco → ${REGISTRY}/${IMAGE_ORG}/${IMAGE_NAME}:${TAG}" +# +# - name: Build and push ome-manager (linux/amd64) +# uses: docker/build-push-action@v7 +# with: +# context: . +# file: dockerfiles/manager.Dockerfile +# platforms: linux/amd64 +# push: true +# tags: | +# ${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} +# cache-from: type=gha +# cache-to: type=gha,mode=max +# build-args: | +# VERSION=${{ steps.tag.outputs.tag }} +# GIT_TAG=${{ steps.tag.outputs.tag }} +# GIT_COMMIT=${{ github.sha }} +# +# - name: Summary +# run: | +# echo "## VSCO ome-manager image" >> "$GITHUB_STEP_SUMMARY" +# echo "" >> "$GITHUB_STEP_SUMMARY" +# echo "Pushed: \`${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}\`" >> "$GITHUB_STEP_SUMMARY" +# echo "" >> "$GITHUB_STEP_SUMMARY" +# echo "Remove \`.github/workflows/vsco-publish-ome-manager.yaml\` before submitting a PR to sgl-project/ome." >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/web-console.yml b/.github/workflows/web-console.yml index 8295f75b3..c14a39e02 100644 --- a/.github/workflows/web-console.yml +++ b/.github/workflows/web-console.yml @@ -24,7 +24,7 @@ jobs: # Frontend checks - lint, typecheck, build, test frontend: name: Frontend - runs-on: ubuntu-latest + runs-on: self-hosted timeout-minutes: 15 steps: - name: Checkout code @@ -64,7 +64,7 @@ jobs: # Backend checks - build and vet backend: name: Backend - runs-on: ubuntu-latest + runs-on: self-hosted timeout-minutes: 10 steps: - name: Checkout code @@ -97,7 +97,7 @@ jobs: # Summary job summary: name: Web Console CI Summary - runs-on: ubuntu-latest + runs-on: self-hosted needs: [frontend, backend] if: always() steps: diff --git a/Makefile b/Makefile index d423967f4..34d186890 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,12 @@ TAG ?= $(GIT_TAG) ARCH ?= linux/amd64 MANAGER_IMG ?= $(REGISTRY)/ome-manager:$(TAG) +# Optional final-stage base for manager/model-agent/ome-agent Dockerfiles (default oraclelinux:10-slim). +# Apple Silicon + --platform=linux/amd64 uses QEMU; OL10 glibc can fail with "CPU does not support x86-64-v3". +# For local amd64 builds on Mac, use: BASE_IMAGE=ubuntu:24.04 make ome-image +BASE_IMAGE ?= +BASE_IMAGE_DOCKER_ARGS = $(if $(BASE_IMAGE),--build-arg BASE_IMAGE=$(BASE_IMAGE),) + # Git version and commit information for build version_pkg = github.com/sgl-project/ome/pkg/version GIT_TAG ?= $(shell git describe --tags --dirty --always) @@ -332,6 +338,7 @@ run-ome-agent-replica: fmt vet ome-agent ## Run ome-agent binary from local host ome-image: fmt vet ## Build ome-manager image. @echo "🚀 Building ome-manager image..." $(DOCKER_BUILD_CMD) build --platform=$(ARCH) \ + $(BASE_IMAGE_DOCKER_ARGS) \ --build-arg VERSION=$(GIT_TAG) \ --build-arg GIT_TAG=$(GIT_TAG) \ --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ @@ -342,6 +349,7 @@ ome-image: fmt vet ## Build ome-manager image. model-agent-image: fmt vet ## Build model-agent image. @echo "🚀 Building model-agent image..." $(DOCKER_BUILD_CMD) build --platform=$(ARCH) \ + $(BASE_IMAGE_DOCKER_ARGS) \ --build-arg VERSION=$(GIT_TAG) \ --build-arg GIT_TAG=$(GIT_TAG) \ --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ @@ -362,6 +370,7 @@ multinode-prober-image: fmt vet ## Build multinode-prober image. ome-agent-image: fmt vet xet-build ## Build ome-agent image. @echo "🚀 Building ome-agent image..." $(DOCKER_BUILD_CMD) build --platform=$(ARCH) \ + $(BASE_IMAGE_DOCKER_ARGS) \ --build-arg VERSION=$(GIT_TAG) \ --build-arg GIT_TAG=$(GIT_TAG) \ --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ @@ -388,11 +397,13 @@ build-all-images: fmt vet ## 🚀 Build all images for current architecture build-all-images-multiarch: fmt vet docker-buildx-setup ## 🌍 Build all images for multiple architectures @echo "🌍 Building all OME images for linux/amd64,linux/arm64..." $(DOCKER_BUILD_CMD) buildx build --platform=linux/amd64,linux/arm64 \ + $(BASE_IMAGE_DOCKER_ARGS) \ --build-arg VERSION=$(GIT_TAG) \ --build-arg GIT_TAG=$(GIT_TAG) \ --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ . -f dockerfiles/manager.Dockerfile -t $(MANAGER_IMG) --push $(DOCKER_BUILD_CMD) buildx build --platform=linux/amd64,linux/arm64 \ + $(BASE_IMAGE_DOCKER_ARGS) \ --build-arg VERSION=$(GIT_TAG) \ --build-arg GIT_TAG=$(GIT_TAG) \ --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ @@ -403,6 +414,7 @@ build-all-images-multiarch: fmt vet docker-buildx-setup ## 🌍 Build all images --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ . -f dockerfiles/multinode-prober.Dockerfile -t $(REGISTRY)/multinode-prober:$(TAG) --push $(DOCKER_BUILD_CMD) buildx build --platform=linux/amd64,linux/arm64 \ + $(BASE_IMAGE_DOCKER_ARGS) \ --build-arg VERSION=$(GIT_TAG) \ --build-arg GIT_TAG=$(GIT_TAG) \ --build-arg GIT_COMMIT=$(shell git rev-parse HEAD) \ diff --git a/charts/ome-resources/README.md b/charts/ome-resources/README.md index 60989a56d..59959b78b 100644 --- a/charts/ome-resources/README.md +++ b/charts/ome-resources/README.md @@ -90,3 +90,21 @@ OME Resources and Controller | ome.omeAgent.tag | string | `"v0.1.2"` | | | ome.omeAgent.vaultId | string | `"ocid1.vault.oc1.ap-osaka-1.dummy.dummy-vault"` | | | ome.version | string | `"v0.1.2"` | | + +## Autoscaling (Karpenter / cluster autoscaler) + +Engine and decoder pods normally get a **required** `nodeSelector` of the form `models.ome.io/clusterbasemodel.=Ready` (or the namespace-scoped base-model equivalent). The model-agent DaemonSet applies that label only **after** weights are on disk. On elastic GPU pools, autoscalers such as Karpenter may refuse to provision a node when the pod requires a label that no template advertises yet, which can deadlock cold starts. + +**Opt-in:** set this annotation on the `InferenceService` to **omit** the model-ready `nodeSelector` (accelerator / runtime merged selectors still apply): + +```yaml +metadata: + annotations: + ome.io/skip-model-ready-node-selector: "true" +``` + +**Semantics:** pods may schedule on a GPU node before the model is present on the host; you rely on hostPath + model-agent download, container restarts, or similar until the model is available. GPU pool labels, taints, and tolerations must still align (for example `gpu=true` and `nvidia.com/gpu` tolerations). + +**BenchmarkJob:** if the job references an `InferenceService`, the same annotation on that `InferenceService` controls whether the benchmark pod gets the model-ready selector. + +Multi-node or PD-disaggregated behavior is unchanged; this only affects the optional model-ready scheduling constraint for cluster- or namespace-scoped base models. diff --git a/charts/ome-resources/templates/model-agent-daemonset/configmap.yaml b/charts/ome-resources/templates/model-agent-daemonset/configmap.yaml index 2d0026701..47ce0e4d5 100644 --- a/charts/ome-resources/templates/model-agent-daemonset/configmap.yaml +++ b/charts/ome-resources/templates/model-agent-daemonset/configmap.yaml @@ -4,32 +4,7 @@ metadata: name: model-agent-config-map namespace: {{ .Release.Namespace }} data: - # Instance type mappings for various cloud providers: - # - Oracle Cloud (OCI) shapes - # - AWS instance types - # - Azure instance types - # - Google Cloud instance types - # - CoreWeave instance types - # - Nebius instance types + # Instance type → GPU name mapping (AWS, OCI, Azure, GCP, CoreWeave, Nebius, …). + # Configure via .Values.modelAgent.instanceTypeMap (map of string → string, JSON-serialized). instance-type-map: |- - { - "BM.GPU.A10.4": "A10", - "BM.GPU.A100-v2.8": "A100-80G", - "BM.GPU4.8": "A100-40G", - "BM.GPU.B4.8": "A100-40G", - "BM.GPU.H100.8": "H100", - "BM.GPU.H100-NC.8": "H100", - "BM.GPU.H200.8": "H200", - "BM.GPU.H200-NC.8": "H200", - "BM.GPU.B200.8": "B200", - "p5.48xlarge": "H100", - "Standard_ND96isr_H100_v5": "H100", - "a3-highgpu-8g": "H100", - "gd-8xh100ib-i128": "H100", - "gd-8xh200ib-i128": "H200", - "gd-8xl40-i128": "L40", - "gpu-h100-sxm": "H100", - "gpu-h200-sxm": "H200", - "gpu-b200-sxm": "B200", - "gpu-l40s": "L40S" - } +{{- toJson .Values.modelAgent.instanceTypeMap | nindent 4 }} diff --git a/charts/ome-resources/templates/model-agent-daemonset/daemonset.yaml b/charts/ome-resources/templates/model-agent-daemonset/daemonset.yaml index 4bcadcef6..833fa933d 100644 --- a/charts/ome-resources/templates/model-agent-daemonset/daemonset.yaml +++ b/charts/ome-resources/templates/model-agent-daemonset/daemonset.yaml @@ -38,10 +38,16 @@ spec: {{- toYaml $imagePullSecrets | nindent 8 }} {{- end }} volumes: +{{- if .Values.modelAgent.persistence.enabled }} + - name: host-models + persistentVolumeClaim: + claimName: {{ .Values.modelAgent.persistence.claimName | quote }} +{{- else }} - name: host-models hostPath: path: {{ .Values.modelAgent.hostPath }} type: DirectoryOrCreate +{{- end }} {{- with .Values.modelAgent.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/ome-resources/templates/model-agent-daemonset/pvc.yaml b/charts/ome-resources/templates/model-agent-daemonset/pvc.yaml new file mode 100644 index 000000000..c85dfcce8 --- /dev/null +++ b/charts/ome-resources/templates/model-agent-daemonset/pvc.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.modelAgent.persistence.enabled .Values.modelAgent.persistence.create }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.modelAgent.persistence.claimName }} + namespace: {{ .Release.Namespace }} +spec: + accessModes: + {{- range .Values.modelAgent.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.modelAgent.persistence.storageClassName | quote }} + resources: + requests: + storage: {{ .Values.modelAgent.persistence.size | quote }} +{{- end }} diff --git a/charts/ome-resources/templates/ome-controller/configmap.yaml b/charts/ome-resources/templates/ome-controller/configmap.yaml index 5c9253deb..c8f53bc0e 100644 --- a/charts/ome-resources/templates/ome-controller/configmap.yaml +++ b/charts/ome-resources/templates/ome-controller/configmap.yaml @@ -78,6 +78,8 @@ data: "scalingThreshold": "{{ .Values.ome.kedaConfig.scalingThreshold | default "10" }}", "scalingOperator": "{{ .Values.ome.kedaConfig.scalingOperator | default "GreaterThanOrEqual" }}" } + modelStorage: |- +{{ (.Values.ome.modelStorage | default dict) | toJson | nindent 4 }} --- apiVersion: v1 kind: ConfigMap diff --git a/charts/ome-resources/values.yaml b/charts/ome-resources/values.yaml index cb4f96397..d43b71d00 100644 --- a/charts/ome-resources/values.yaml +++ b/charts/ome-resources/values.yaml @@ -1,14 +1,12 @@ -# Global settings that apply to all resources +# VSCO fork: Helm charts are published to oci://ghcr.io/vsco/charts (see .github/workflows/vsco-publish-helm-charts.yaml). +# shared-infra only sets var.ome_version (chart semver). Forked ome-manager uses a full image ref so other images can stay on global.hub. global: - # Image pull secrets for all containers in the chart - # Example: - # imagePullSecrets: - # - name: my-registry-secret - hub: "ghcr.io/moirai-internal" - imagePullSecrets: [] + hub: "ghcr.io/vsco" + imagePullSecrets: + - name: ghcr-creds ome: - version: &defaultVersion v0.1.4 + version: &defaultVersion v0.1.5-vsco7 metricsaggregator: enableMetricAggregation: "false" enablePrometheusScraping: "false" @@ -89,8 +87,20 @@ ome: customPromQuery: "" scalingThreshold: "10" scalingOperator: "GreaterThanOrEqual" + # JSON merged into inferenceservice-config modelStorage (pvcClaimName / pvcMountRoot). + # Per-InferenceService override: annotation ome.io/model-storage-pvc. + modelStorage: {} modelAgent: hostPath: /mnt/data/models + # EFS (ReadWriteMany) or other RWX PVC for shared model cache; aligns with ome.modelStorage. + persistence: + enabled: false + create: false + claimName: ome-models-efs + accessModes: + - ReadWriteMany + storageClassName: efs-sc-gp + size: 200Gi priorityClassName: system-node-critical serviceAccountName: ome-model-agent image: @@ -102,6 +112,54 @@ modelAgent: pullPolicy: Always tag: *defaultVersion + # Instance type → GPU label map for model-agent (JSON in ConfigMap key instance-type-map). + # Extend or replace per cluster; aligns with shared-infra AWS GPU pools when published from this fork. + instanceTypeMap: + "BM.GPU.A10.4": "A10" + "BM.GPU.A100-v2.8": "A100-80G" + "BM.GPU4.8": "A100-40G" + "BM.GPU.B4.8": "A100-40G" + "BM.GPU.H100.8": "H100" + "BM.GPU.H100-NC.8": "H100" + "BM.GPU.H200.8": "H200" + "BM.GPU.H200-NC.8": "H200" + "BM.GPU.B200.8": "B200" + "p5.48xlarge": "H100" + "Standard_ND96isr_H100_v5": "H100" + "a3-highgpu-8g": "H100" + "gd-8xh100ib-i128": "H100" + "gd-8xh200ib-i128": "H200" + "gd-8xl40-i128": "L40" + "gpu-h100-sxm": "H100" + "gpu-h200-sxm": "H200" + "gpu-b200-sxm": "B200" + "gpu-l40s": "L40S" + "g4dn.xlarge": "T4" + "g4dn.2xlarge": "T4" + "g4dn.4xlarge": "T4" + "g4dn.8xlarge": "T4" + "g4dn.12xlarge": "T4" + "g4dn.16xlarge": "T4" + "g4dn.metal": "T4" + "g6e.xlarge": "L40S" + "g6e.2xlarge": "L40S" + "g6e.4xlarge": "L40S" + "g6e.8xlarge": "L40S" + "g6e.12xlarge": "L40S" + "g6e.16xlarge": "L40S" + "g6e.24xlarge": "L40S" + "g6e.48xlarge": "L40S" + "g5.xlarge": "A10G" + "g5.2xlarge": "A10G" + "g5.4xlarge": "A10G" + "g5.8xlarge": "A10G" + "g5.12xlarge": "A10G" + "g5.16xlarge": "A10G" + "g5.24xlarge": "A10G" + "g5.48xlarge": "A10G" + "p4d.24xlarge": "A100-40G" + "p4de.24xlarge": "A100-80G" + # When enabled, the model agent will only run on nodes with GPU gpuNodesOnly: false diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 171fc8582..bc10d8eae 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -335,6 +335,20 @@ const ( InferenceServiceLabel = "ome.io/inferenceservice" ) +// SkipModelReadyNodeSelectorAnnotationKey, when set to "true" on an InferenceService, omits the +// models.ome.io/...=Ready nodeSelector from engine and decoder pods. Use this with elastic GPU +// pools (e.g. Karpenter) where nodes are provisioned before the model-agent applies the Ready label. +// +// ModelStoragePVCAnnotationKey names the PersistentVolumeClaim (in the InferenceService namespace) +// for base model files instead of hostPath; overrides ConfigMap modelStorage.pvcClaimName. +// DefaultModelPVCMountRoot must align with model-agent --models-root-dir and storage paths. +const ( + SkipModelReadyNodeSelectorAnnotationKey = "ome.io/skip-model-ready-node-selector" + // ModelStoragePVCAnnotationKey must stay in sync with OMEAPIGroupName + "/model-storage-pvc" (OMEAPIGroupName is a var). + ModelStoragePVCAnnotationKey = "ome.io/model-storage-pvc" + DefaultModelPVCMountRoot = "/mnt/data/models" +) + // InferenceService default/canary constants const ( InferenceServiceDefault = "default" @@ -420,6 +434,7 @@ var ( ModelInitInjectionKey, // ome.io/inject-model-init - triggers model init container injection via webhook FineTunedAdapterInjectionKey, // ome.io/inject-fine-tuned-adapter - triggers fine-tuned adapter injection via webhook ServingSidecarInjectionKey, // ome.io/inject-serving-sidecar - triggers serving sidecar injection via webhook + ModelStoragePVCAnnotationKey, // ome.io/model-storage-pvc - per-ISVC PVC claim for base model files } ) diff --git a/pkg/controller/v1beta1/benchmark/controller.go b/pkg/controller/v1beta1/benchmark/controller.go index 705a1c60f..6ccbb30ad 100644 --- a/pkg/controller/v1beta1/benchmark/controller.go +++ b/pkg/controller/v1beta1/benchmark/controller.go @@ -24,6 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/sgl-project/ome/pkg/apis/ome/v1beta1" + "github.com/sgl-project/ome/pkg/constants" "github.com/sgl-project/ome/pkg/controller/v1beta1/benchmark/reconcilers/job" benchmarkutils "github.com/sgl-project/ome/pkg/controller/v1beta1/benchmark/utils" "github.com/sgl-project/ome/pkg/controller/v1beta1/controllerconfig" @@ -247,11 +248,24 @@ func (r *BenchmarkJobReconciler) addNodeSelectorFromInferenceService(ctx context return err } - isvcutils.AddNodeSelectorForModelReadyNode(podSpec, baseModelMeta) - r.Log.Info("Added node selector for benchmark job", - "baseModel", baseModelMeta.Name, - "namespace", baseModelMeta.Namespace, - "benchmarkJob", benchmarkJob.Name) + // Model-ready node selector follows the InferenceService: when the skip annotation is absent or + // not "true", require models.ome.io/...=Ready so the benchmark runs on nodes where model-agent + // has already placed weights. When constants.SkipModelReadyNodeSelectorAnnotationKey is "true", + // omit that selector so the job can schedule like engine pods on elastic GPU nodes (e.g. + // Karpenter) before the Ready label exists. + if isvcutils.IsSkipModelReadyNodeSelector(inferenceService.Annotations) { + r.Log.Info("Skipping model-ready nodeSelector for benchmark job (InferenceService annotation)", + "annotation", constants.SkipModelReadyNodeSelectorAnnotationKey, + "inferenceService", inferenceService.Namespace+"/"+inferenceService.Name, + "benchmarkJob", benchmarkJob.Name, + "baseModel", baseModelMeta.Name) + } else { + isvcutils.AddNodeSelectorForModelReadyNode(podSpec, baseModelMeta) + r.Log.Info("Added model-ready nodeSelector for benchmark job", + "baseModel", baseModelMeta.Name, + "namespace", baseModelMeta.Namespace, + "benchmarkJob", benchmarkJob.Name) + } return nil } diff --git a/pkg/controller/v1beta1/controllerconfig/configmap.go b/pkg/controller/v1beta1/controllerconfig/configmap.go index 44a727a54..ffd36e201 100644 --- a/pkg/controller/v1beta1/controllerconfig/configmap.go +++ b/pkg/controller/v1beta1/controllerconfig/configmap.go @@ -14,10 +14,11 @@ import ( ) const ( - IngressConfigKeyName = "ingress" - DeployConfigName = "deploy" - MultiNodeProberName = "multinodeProber" - BenchmarkJobConfigName = "benchmarkjob" + IngressConfigKeyName = "ingress" + DeployConfigName = "deploy" + MultiNodeProberName = "multinodeProber" + ModelStorageConfigName = "modelStorage" + BenchmarkJobConfigName = "benchmarkjob" DefaultDomainTemplate = "{{ .Name }}.{{ .Namespace }}.{{ .IngressDomain }}" DefaultIngressDomain = "example.com" @@ -44,10 +45,21 @@ type PodConfig struct { MemoryLimit string `json:"memoryLimit"` } +// +kubebuilder:object:generate=false +type ModelStorageConfig struct { + // PVCClaimName, when set, switches engine/decoder base-model volumes from hostPath to this + // PersistentVolumeClaim in the InferenceService namespace (ReadWriteMany, e.g. EFS). + PVCClaimName string `json:"pvcClaimName,omitempty"` + // PVCMountRoot is the path inside the pod that is the root of the PVC (SubPath is computed + // relative to this and the model Storage.Path). Defaults to /mnt/data/models. + PVCMountRoot string `json:"pvcMountRoot,omitempty"` +} + // +kubebuilder:object:generate=false type InferenceServicesConfig struct { // MultiNodeProber contains all MultiNodeProber Configuration MultiNodeProber MultiNodeProberConfig `json:"multinodeProber"` + ModelStorage ModelStorageConfig `json:"modelStorage,omitempty"` } // +kubebuilder:object:generate=false @@ -96,6 +108,7 @@ func NewInferenceServicesConfig(clientset kubernetes.Interface) (*InferenceServi icfg := &InferenceServicesConfig{} for _, err := range []error{ getComponentConfig(MultiNodeProberName, configMap, &icfg.MultiNodeProber), + getComponentConfig(ModelStorageConfigName, configMap, &icfg.ModelStorage), } { if err != nil { return nil, err diff --git a/pkg/controller/v1beta1/controllerconfig/configmap_test.go b/pkg/controller/v1beta1/controllerconfig/configmap_test.go index 0558ee5a3..04c4fae89 100644 --- a/pkg/controller/v1beta1/controllerconfig/configmap_test.go +++ b/pkg/controller/v1beta1/controllerconfig/configmap_test.go @@ -69,11 +69,14 @@ func TestNewInferenceServicesConfig(t *testing.T) { "startupTimeoutSeconds": 30, "unavailableThresholdSeconds": 60 }`, + ModelStorageConfigName: `{"pvcClaimName":"ome-models-efs","pvcMountRoot":"/mnt/data/models"}`, }, expectedError: false, validateConfig: func(t *testing.T, cfg *InferenceServicesConfig) { assert.Equal(t, "test-image", cfg.MultiNodeProber.Image) assert.Equal(t, "100m", cfg.MultiNodeProber.CPURequest) + assert.Equal(t, "ome-models-efs", cfg.ModelStorage.PVCClaimName) + assert.Equal(t, "/mnt/data/models", cfg.ModelStorage.PVCMountRoot) }, }, { diff --git a/pkg/controller/v1beta1/inferenceservice/components/base.go b/pkg/controller/v1beta1/inferenceservice/components/base.go index efe00c40a..5c4f78166 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/base.go +++ b/pkg/controller/v1beta1/inferenceservice/components/base.go @@ -4,6 +4,7 @@ import ( "fmt" "path/filepath" "strconv" + "strings" "github.com/go-logr/logr" "github.com/pkg/errors" @@ -19,6 +20,7 @@ import ( "github.com/sgl-project/ome/pkg/controller/v1beta1/inferenceservice/status" isvcutils "github.com/sgl-project/ome/pkg/controller/v1beta1/inferenceservice/utils" "github.com/sgl-project/ome/pkg/utils" + "github.com/sgl-project/ome/pkg/utils/storage" ) // BaseComponentFields contains common fields for all components @@ -96,12 +98,20 @@ func UpdateVolumeMounts(b *BaseComponentFields, isvc *v1beta1.InferenceService, MountPath: *b.BaseModel.Storage.Path, ReadOnly: true, } + if isvcutils.ModelStoragePVCClaimName(isvc, b.InferenceServiceConfig) != "" { + mountRoot := isvcutils.ModelStoragePVCMountRoot(b.InferenceServiceConfig) + if sub := isvcutils.ModelVolumeMountSubPathForPVC(mountRoot, *b.BaseModel.Storage.Path); sub != "" { + vm.SubPath = sub + } + } isvcutils.AppendVolumeMount(container, &vm) } } - // Add fine-tuned serving volume mounts - if b.FineTunedServing { + // Add fine-tuned serving volume mounts for /opt/ml/model only when the emptyDir volume is + // required (OCI/S3 adapter init or model-init). PVC-backed FineTunedWeights skip inject + // annotations; UpdatePodSpecVolumes then omits model-empty-dir, so mounts must match. + if b.FineTunedServing && isvcutils.IsEmptyModelDirVolumeRequired(objectMeta.Annotations) { defaultModelVolumeMount := corev1.VolumeMount{ Name: constants.ModelEmptyDirVolumeName, MountPath: constants.ModelDefaultMountPath, @@ -128,6 +138,46 @@ func UpdateVolumeMounts(b *BaseComponentFields, isvc *v1beta1.InferenceService, } } +// UpdateInitContainerBaseModelVolumeMounts sets the PVC SubPath on init container volume mounts +// that target the base model volume and mount path. ServingRuntime templates usually specify +// only name + mountPath; UpdateVolumeMounts adds SubPath for the main container. Without the +// same SubPath here, the full claim is mounted at storagePath and paths like +// ${storagePath}/config.json (used by wait-for-base-model) miss the per-model subdirectory that +// prefetch and model-agent use on the shared PVC. +func UpdateInitContainerBaseModelVolumeMounts(b *BaseComponentFields, isvc *v1beta1.InferenceService, podSpec *corev1.PodSpec, objectMeta *metav1.ObjectMeta) { + if podSpec == nil || len(podSpec.InitContainers) == 0 { + return + } + if b.BaseModel == nil || b.BaseModel.Storage == nil || b.BaseModel.Storage.Path == nil || b.BaseModelMeta == nil { + return + } + if !isvcutils.IsOriginalModelVolumeMountNecessary(objectMeta.Annotations) { + return + } + if strings.TrimSpace(isvcutils.ModelStoragePVCClaimName(isvc, b.InferenceServiceConfig)) == "" { + return + } + mountRoot := isvcutils.ModelStoragePVCMountRoot(b.InferenceServiceConfig) + storagePath := filepath.Clean(*b.BaseModel.Storage.Path) + sub := isvcutils.ModelVolumeMountSubPathForPVC(mountRoot, storagePath) + if sub == "" { + return + } + volName := b.BaseModelMeta.Name + for i := range podSpec.InitContainers { + for j := range podSpec.InitContainers[i].VolumeMounts { + vm := &podSpec.InitContainers[i].VolumeMounts[j] + if vm.Name != volName { + continue + } + if filepath.Clean(vm.MountPath) != storagePath { + continue + } + vm.SubPath = sub + } + } +} + // UpdateEnvVariables updates environment variables for the container func UpdateEnvVariables(b *BaseComponentFields, isvc *v1beta1.InferenceService, container *corev1.Container, objectMeta *metav1.ObjectMeta) { if container == nil { @@ -204,8 +254,15 @@ func UpdatePodSpecNodeSelector(b *BaseComponentFields, isvc *v1beta1.InferenceSe return } - // Add preferred node affinity for model readiness using the shared utility function - isvcutils.AddNodeSelectorForModelReadyNode(podSpec, b.BaseModelMeta) + // Optional: omit models.ome.io/...=Ready so autoscalers can provision GPU nodes before the + // model-agent labels them (see SkipModelReadyNodeSelectorAnnotationKey). + if isvcutils.IsSkipModelReadyNodeSelector(isvc.Annotations) { + b.Log.Info("Skipping model-ready nodeSelector per InferenceService annotation", + "annotation", constants.SkipModelReadyNodeSelectorAnnotationKey, + "inferenceService", isvc.Name, "namespace", isvc.Namespace) + } else { + isvcutils.AddNodeSelectorForModelReadyNode(podSpec, b.BaseModelMeta) + } // Add node selector merged from AcceleratorClass if applicable // Only add mergedNodeSelector to engine and decoder component. @@ -219,7 +276,7 @@ func UpdatePodSpecNodeSelector(b *BaseComponentFields, isvc *v1beta1.InferenceSe } } - b.Log.Info("Added preferred node affinity for model scheduling", + b.Log.Info("Updated pod nodeSelector for model scheduling", "modelName", b.BaseModelMeta.Name, "namespace", b.BaseModelMeta.Namespace, "inferenceService", isvc.Name) @@ -229,15 +286,23 @@ func UpdatePodSpecNodeSelector(b *BaseComponentFields, isvc *v1beta1.InferenceSe func UpdatePodSpecVolumes(b *BaseComponentFields, isvc *v1beta1.InferenceService, podSpec *corev1.PodSpec, objectMeta *metav1.ObjectMeta) { // Add model volume if base model is specified if b.BaseModel != nil && b.BaseModel.Storage != nil && b.BaseModel.Storage.Path != nil && b.BaseModelMeta != nil { - modelVolume := corev1.Volume{ - Name: b.BaseModelMeta.Name, - VolumeSource: corev1.VolumeSource{ + pvcClaim := isvcutils.ModelStoragePVCClaimName(isvc, b.InferenceServiceConfig) + modelVolume := corev1.Volume{Name: b.BaseModelMeta.Name} + if pvcClaim != "" { + modelVolume.VolumeSource = corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcClaim, + ReadOnly: true, + }, + } + } else { + modelVolume.VolumeSource = corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: *b.BaseModel.Storage.Path, }, - }, + } } - podSpec.Volumes = append(podSpec.Volumes, modelVolume) + podSpec.Volumes = utils.AppendVolumeIfNotExists(podSpec.Volumes, modelVolume) } // Add empty model directory volume if required for fine-tuned serving @@ -371,16 +436,26 @@ func UpdateDecoderAffinity(b *BaseComponentFields, isvc *v1beta1.InferenceServic func ProcessBaseAnnotations(b *BaseComponentFields, isvc *v1beta1.InferenceService, annotations map[string]string) (map[string]string, error) { // Add fine-tuned weight annotations if applicable if b.FineTunedServing && len(b.FineTunedWeights) > 0 { - // Inject ft adapter for single/non-stacked fine-tuned weight downloading - annotations[constants.FineTunedAdapterInjectionKey] = b.FineTunedWeights[0].Name - - // Add fine-tuned weight ft strategy - fineTunedWeightFTStrategy, err := isvcutils.GetValueFromRawExtension(b.FineTunedWeights[0].Spec.HyperParameters, constants.StrategyConfigKey) + ftw := b.FineTunedWeights[0] + fineTunedWeightFTStrategy, err := isvcutils.GetValueFromRawExtension(ftw.Spec.HyperParameters, constants.StrategyConfigKey) if err != nil || fineTunedWeightFTStrategy == nil { - b.Log.Error(err, "Error getting hyper-parameter strategy from FineTunedWeight", "FineTunedWeight", b.FineTunedWeights[0].Name, "namespace", isvc.Namespace) + b.Log.Error(err, "Error getting hyper-parameter strategy from FineTunedWeight", "FineTunedWeight", ftw.Name, "namespace", isvc.Namespace) return nil, err } annotations[constants.FineTunedWeightFTStrategyKey] = fineTunedWeightFTStrategy.(string) + + uri := "" + if ftw.Spec.Storage != nil && ftw.Spec.Storage.StorageUri != nil { + uri = strings.TrimSpace(*ftw.Spec.Storage.StorageUri) + } + // OCI/S3 fine-tuned adapter init expects object storage URIs. PVC-backed weights (e.g. Git LFS + // materialized onto a shared volume) are already on disk; skip injection. + if strings.HasPrefix(uri, storage.PVCStoragePrefix) { + b.Log.Info("Skipping fine-tuned adapter injection for PVC-backed FineTunedWeight", + "FineTunedWeight", ftw.Name, "namespace", isvc.Namespace) + } else { + annotations[constants.FineTunedAdapterInjectionKey] = ftw.Name + } } if b.FineTunedServingWithMergedWeights { diff --git a/pkg/controller/v1beta1/inferenceservice/components/base_test.go b/pkg/controller/v1beta1/inferenceservice/components/base_test.go index 9de3eae38..75215dc49 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/base_test.go +++ b/pkg/controller/v1beta1/inferenceservice/components/base_test.go @@ -11,6 +11,7 @@ import ( "github.com/sgl-project/ome/pkg/apis/ome/v1beta1" "github.com/sgl-project/ome/pkg/constants" + "github.com/sgl-project/ome/pkg/controller/v1beta1/controllerconfig" ) func TestUpdatePodSpecNodeSelector(t *testing.T) { @@ -22,6 +23,7 @@ func TestUpdatePodSpecNodeSelector(t *testing.T) { baseModelMeta *metav1.ObjectMeta fineTunedServingWithMergedWeights bool existingNodeSelector map[string]string + isvcAnnotations map[string]string expectedLabelKey string expectNodeSelector bool }{ @@ -90,6 +92,39 @@ func TestUpdatePodSpecNodeSelector(t *testing.T) { baseModelMeta: nil, expectNodeSelector: false, }, + { + name: "Skip model-ready nodeSelector when InferenceService annotation is true", + baseModel: &v1beta1.BaseModelSpec{ + ModelFormat: v1beta1.ModelFormat{ + Name: "safetensors", + }, + }, + baseModelMeta: &metav1.ObjectMeta{ + Name: "my-model", + Namespace: "", + }, + isvcAnnotations: map[string]string{ + constants.SkipModelReadyNodeSelectorAnnotationKey: "true", + }, + expectNodeSelector: false, + }, + { + name: "Annotation false still adds model-ready nodeSelector", + baseModel: &v1beta1.BaseModelSpec{ + ModelFormat: v1beta1.ModelFormat{ + Name: "safetensors", + }, + }, + baseModelMeta: &metav1.ObjectMeta{ + Name: "my-model", + Namespace: "", + }, + isvcAnnotations: map[string]string{ + constants.SkipModelReadyNodeSelectorAnnotationKey: "false", + }, + expectedLabelKey: "models.ome.io/clusterbasemodel.my-model", + expectNodeSelector: true, + }, { name: "Long model names should be handled", baseModel: &v1beta1.BaseModelSpec{ @@ -128,13 +163,14 @@ func TestUpdatePodSpecNodeSelector(t *testing.T) { // Create inference service isvc := &v1beta1.InferenceService{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-isvc", - Namespace: "default", + Name: "test-isvc", + Namespace: "default", + Annotations: tt.isvcAnnotations, }, } // Call the function - UpdatePodSpecNodeSelector(b, isvc, podSpec, "") + UpdatePodSpecNodeSelector(b, isvc, podSpec, v1beta1.EngineComponent) // Verify the result if !tt.expectNodeSelector { @@ -218,3 +254,188 @@ func TestProcessBaseLabels(t *testing.T) { g.Expect(labels).To(gomega.HaveKeyWithValue(constants.BaseModelTypeLabelKey, string(constants.ServingBaseModel))) g.Expect(labels).To(gomega.HaveKeyWithValue(constants.BaseModelVendorLabelKey, "meta")) } + +func TestUpdatePodSpecVolumes_PVCAndHostPath(t *testing.T) { + g := gomega.NewGomegaWithT(t) + modelPath := "/mnt/data/models/my-model" + b := &BaseComponentFields{ + BaseModel: &v1beta1.BaseModelSpec{ + Storage: &v1beta1.StorageSpec{ + Path: &modelPath, + }, + }, + BaseModelMeta: &metav1.ObjectMeta{Name: "cluster-model-x"}, + Log: logr.Discard(), + } + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "default"}, + } + + pod := &v1.PodSpec{} + UpdatePodSpecVolumes(b, isvc, pod, &metav1.ObjectMeta{}) + g.Expect(pod.Volumes).To(gomega.HaveLen(1)) + g.Expect(pod.Volumes[0].HostPath).NotTo(gomega.BeNil()) + g.Expect(pod.Volumes[0].HostPath.Path).To(gomega.Equal(modelPath)) + + b.InferenceServiceConfig = &controllerconfig.InferenceServicesConfig{ + ModelStorage: controllerconfig.ModelStorageConfig{ + PVCClaimName: "ome-models-efs", + PVCMountRoot: "/mnt/data/models", + }, + } + pod2 := &v1.PodSpec{} + UpdatePodSpecVolumes(b, isvc, pod2, &metav1.ObjectMeta{}) + g.Expect(pod2.Volumes).To(gomega.HaveLen(1)) + g.Expect(pod2.Volumes[0].PersistentVolumeClaim).NotTo(gomega.BeNil()) + g.Expect(pod2.Volumes[0].PersistentVolumeClaim.ClaimName).To(gomega.Equal("ome-models-efs")) + g.Expect(pod2.Volumes[0].PersistentVolumeClaim.ReadOnly).To(gomega.BeTrue()) +} + +func TestUpdateVolumeMounts_PVCSubPath(t *testing.T) { + g := gomega.NewGomegaWithT(t) + modelPath := "/mnt/data/models/my-model" + b := &BaseComponentFields{ + BaseModel: &v1beta1.BaseModelSpec{ + Storage: &v1beta1.StorageSpec{ + Path: &modelPath, + }, + }, + BaseModelMeta: &metav1.ObjectMeta{Name: "cluster-model-x"}, + InferenceServiceConfig: &controllerconfig.InferenceServicesConfig{ + ModelStorage: controllerconfig.ModelStorageConfig{ + PVCClaimName: "ome-models-efs", + PVCMountRoot: "/mnt/data/models", + }, + }, + Log: logr.Discard(), + } + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "default"}, + } + container := &v1.Container{} + UpdateVolumeMounts(b, isvc, container, &metav1.ObjectMeta{Annotations: map[string]string{}}) + g.Expect(container.VolumeMounts).To(gomega.HaveLen(1)) + g.Expect(container.VolumeMounts[0].SubPath).To(gomega.Equal("my-model")) +} + +// Fine-tuned serving with PVC-backed weights skips inject annotations; emptyDir for /opt/ml/model +// must not be mounted unless adapter/model-init injection requires it (matches UpdatePodSpecVolumes). +func TestUpdateVolumeMounts_FTWithoutEmptyDirSkipsModelEmptyDirMount(t *testing.T) { + g := gomega.NewGomegaWithT(t) + modelPath := "/mnt/data/models/qwen3-vl-8b-instruct" + b := &BaseComponentFields{ + FineTunedServing: true, + BaseModel: &v1beta1.BaseModelSpec{ + Storage: &v1beta1.StorageSpec{Path: &modelPath}, + }, + BaseModelMeta: &metav1.ObjectMeta{Name: "qwen3-vl-8b-instruct"}, + InferenceServiceConfig: &controllerconfig.InferenceServicesConfig{ + ModelStorage: controllerconfig.ModelStorageConfig{ + PVCClaimName: "ome-models-efs", + PVCMountRoot: "/mnt/data/models", + }, + }, + Log: logr.Discard(), + } + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "default"}, + } + ann := map[string]string{ + constants.BaseModelName: "qwen3-vl-8b-instruct", + } + container := &v1.Container{} + UpdateVolumeMounts(b, isvc, container, &metav1.ObjectMeta{Annotations: ann}) + g.Expect(container.VolumeMounts).To(gomega.HaveLen(1)) + g.Expect(container.VolumeMounts[0].Name).To(gomega.Equal("qwen3-vl-8b-instruct")) + for _, vm := range container.VolumeMounts { + g.Expect(vm.Name).NotTo(gomega.Equal(constants.ModelEmptyDirVolumeName)) + } +} + +func TestUpdateVolumeMounts_FTWithInjectAddsModelEmptyDirMount(t *testing.T) { + g := gomega.NewGomegaWithT(t) + modelPath := "/mnt/data/models/my-model" + b := &BaseComponentFields{ + FineTunedServing: true, + BaseModel: &v1beta1.BaseModelSpec{ + Storage: &v1beta1.StorageSpec{Path: &modelPath}, + }, + BaseModelMeta: &metav1.ObjectMeta{Name: "cluster-model-x"}, + InferenceServiceConfig: &controllerconfig.InferenceServicesConfig{ + ModelStorage: controllerconfig.ModelStorageConfig{ + PVCClaimName: "ome-models-efs", + PVCMountRoot: "/mnt/data/models", + }, + }, + Log: logr.Discard(), + } + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "default"}, + } + ann := map[string]string{ + constants.FineTunedAdapterInjectionKey: "some-adapter", + } + container := &v1.Container{} + UpdateVolumeMounts(b, isvc, container, &metav1.ObjectMeta{Annotations: ann}) + names := make([]string, 0, len(container.VolumeMounts)) + for _, vm := range container.VolumeMounts { + names = append(names, vm.Name) + } + g.Expect(names).To(gomega.ContainElement(constants.ModelEmptyDirVolumeName)) +} + +func TestUpdateInitContainerBaseModelVolumeMounts_PVCSubPath(t *testing.T) { + g := gomega.NewGomegaWithT(t) + modelPath := "/mnt/data/models/my-model" + isvc := &v1beta1.InferenceService{ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "default"}} + b := &BaseComponentFields{ + BaseModel: &v1beta1.BaseModelSpec{ + Storage: &v1beta1.StorageSpec{Path: &modelPath}, + }, + BaseModelMeta: &metav1.ObjectMeta{Name: "my-model"}, + InferenceServiceConfig: &controllerconfig.InferenceServicesConfig{ + ModelStorage: controllerconfig.ModelStorageConfig{ + PVCClaimName: "ome-models-efs", + PVCMountRoot: "/mnt/data/models", + }, + }, + Log: logr.Discard(), + } + pod := &v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "wait-for-base-model", + VolumeMounts: []v1.VolumeMount{ + {Name: "my-model", MountPath: modelPath}, + }, + }, + }, + } + UpdateInitContainerBaseModelVolumeMounts(b, isvc, pod, &metav1.ObjectMeta{Annotations: map[string]string{}}) + g.Expect(pod.InitContainers[0].VolumeMounts[0].SubPath).To(gomega.Equal("my-model")) +} + +func TestUpdateInitContainerBaseModelVolumeMounts_NoPVCNoOp(t *testing.T) { + g := gomega.NewGomegaWithT(t) + modelPath := "/mnt/data/models/my-model" + isvc := &v1beta1.InferenceService{ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "default"}} + b := &BaseComponentFields{ + BaseModel: &v1beta1.BaseModelSpec{ + Storage: &v1beta1.StorageSpec{Path: &modelPath}, + }, + BaseModelMeta: &metav1.ObjectMeta{Name: "my-model"}, + Log: logr.Discard(), + } + pod := &v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "wait", + VolumeMounts: []v1.VolumeMount{ + {Name: "my-model", MountPath: modelPath}, + }, + }, + }, + } + UpdateInitContainerBaseModelVolumeMounts(b, isvc, pod, &metav1.ObjectMeta{Annotations: map[string]string{}}) + g.Expect(pod.InitContainers[0].VolumeMounts[0].SubPath).To(gomega.BeEmpty()) +} diff --git a/pkg/controller/v1beta1/inferenceservice/components/decoder.go b/pkg/controller/v1beta1/inferenceservice/components/decoder.go index 662baad31..00d219c54 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/decoder.go +++ b/pkg/controller/v1beta1/inferenceservice/components/decoder.go @@ -301,6 +301,7 @@ func (d *Decoder) reconcilePodSpec(isvc *v1beta1.InferenceService, objectMeta *m return nil, err } + UpdateInitContainerBaseModelVolumeMounts(&d.BaseComponentFields, isvc, podSpec, objectMeta) UpdatePodSpecVolumes(&d.BaseComponentFields, isvc, podSpec, objectMeta) UpdatePodSpecNodeSelector(&d.BaseComponentFields, isvc, podSpec, v1beta1.DecoderComponent) UpdateDecoderAffinity(&d.BaseComponentFields, isvc, podSpec) @@ -336,6 +337,7 @@ func (d *Decoder) reconcileWorkerPodSpec(isvc *v1beta1.InferenceService, objectM if err != nil { return nil, err } + UpdateInitContainerBaseModelVolumeMounts(&d.BaseComponentFields, isvc, workerPodSpec, objectMeta) UpdatePodSpecVolumes(&d.BaseComponentFields, isvc, workerPodSpec, objectMeta) UpdatePodSpecNodeSelector(&d.BaseComponentFields, isvc, workerPodSpec, v1beta1.DecoderComponent) UpdateDecoderAffinity(&d.BaseComponentFields, isvc, workerPodSpec) diff --git a/pkg/controller/v1beta1/inferenceservice/components/engine.go b/pkg/controller/v1beta1/inferenceservice/components/engine.go index 71539ce5a..f87fe1ca6 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/engine.go +++ b/pkg/controller/v1beta1/inferenceservice/components/engine.go @@ -304,6 +304,7 @@ func (e *Engine) reconcilePodSpec(isvc *v1beta1.InferenceService, objectMeta *me if err != nil { return nil, err } + UpdateInitContainerBaseModelVolumeMounts(&e.BaseComponentFields, isvc, podSpec, objectMeta) UpdatePodSpecVolumes(&e.BaseComponentFields, isvc, podSpec, objectMeta) UpdatePodSpecNodeSelector(&e.BaseComponentFields, isvc, podSpec, v1beta1.EngineComponent) UpdateEngineAffinity(&e.BaseComponentFields, isvc, podSpec) @@ -339,6 +340,7 @@ func (e *Engine) reconcileWorkerPodSpec(isvc *v1beta1.InferenceService, objectMe if err != nil { return nil, err } + UpdateInitContainerBaseModelVolumeMounts(&e.BaseComponentFields, isvc, workerPodSpec, objectMeta) UpdatePodSpecVolumes(&e.BaseComponentFields, isvc, workerPodSpec, objectMeta) UpdatePodSpecNodeSelector(&e.BaseComponentFields, isvc, workerPodSpec, v1beta1.EngineComponent) UpdateEngineAffinity(&e.BaseComponentFields, isvc, workerPodSpec) diff --git a/pkg/controller/v1beta1/inferenceservice/components/engine_test.go b/pkg/controller/v1beta1/inferenceservice/components/engine_test.go index 4fd0fe40b..751683b33 100644 --- a/pkg/controller/v1beta1/inferenceservice/components/engine_test.go +++ b/pkg/controller/v1beta1/inferenceservice/components/engine_test.go @@ -743,6 +743,7 @@ func TestEngineReconcileObjectMeta(t *testing.T) { fineTunedServing bool fineTunedWeights []*v1beta1.FineTunedWeight expectedAnnotations map[string]string + annotationAbsences []string expectedLabels map[string]string expectedName string }{ @@ -862,6 +863,62 @@ func TestEngineReconcileObjectMeta(t *testing.T) { }, expectedName: "ft-isvc-engine", }, + { + name: "Fine-tuned serving metadata with PVC-backed adapter", + isvc: &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ft-pvc-isvc", + Namespace: "default", + }, + }, + engineSpec: &v1beta1.EngineSpec{}, + baseModel: &v1beta1.BaseModelSpec{ + ModelFormat: v1beta1.ModelFormat{ + Name: "safetensors", + }, + ModelExtensionSpec: v1beta1.ModelExtensionSpec{ + Vendor: stringPtr("meta"), + }, + }, + baseModelMeta: &metav1.ObjectMeta{ + Name: "llama-base", + }, + fineTunedServing: true, + fineTunedWeights: []*v1beta1.FineTunedWeight{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ft-weight-pvc", + }, + Spec: v1beta1.FineTunedWeightSpec{ + HyperParameters: runtime.RawExtension{ + Raw: []byte(`{"strategy": "lora"}`), + }, + Storage: &v1beta1.StorageSpec{ + StorageUri: stringPtr("pvc://ome-models-efs/adapters/qwen3-vl-8b-lora"), + }, + }, + }, + }, + expectedAnnotations: map[string]string{ + constants.FineTunedWeightFTStrategyKey: "lora", + constants.BaseModelName: "llama-base", + constants.BaseModelFormat: "safetensors", + constants.BaseModelVendorAnnotationKey: "meta", + }, + annotationAbsences: []string{constants.FineTunedAdapterInjectionKey}, + expectedLabels: map[string]string{ + constants.InferenceServicePodLabelKey: "ft-pvc-isvc", + constants.OMEComponentLabel: "engine", + constants.FTServingLabelKey: "true", + constants.FineTunedWeightFTStrategyLabelKey: "lora", + constants.FTServingWithMergedWeightsLabelKey: "false", + constants.InferenceServiceBaseModelNameLabelKey: "llama-base", + constants.InferenceServiceBaseModelSizeLabelKey: "SMALL", + constants.BaseModelTypeLabelKey: "Serving", + constants.BaseModelVendorLabelKey: "meta", + }, + expectedName: "ft-pvc-isvc-engine", + }, } for _, tt := range tests { @@ -908,6 +965,9 @@ func TestEngineReconcileObjectMeta(t *testing.T) { for k, v := range tt.expectedAnnotations { g.Expect(objectMeta.Annotations).To(gomega.HaveKeyWithValue(k, v)) } + for _, k := range tt.annotationAbsences { + g.Expect(objectMeta.Annotations).NotTo(gomega.HaveKey(k)) + } // Validate labels for k, v := range tt.expectedLabels { diff --git a/pkg/controller/v1beta1/inferenceservice/controller.go b/pkg/controller/v1beta1/inferenceservice/controller.go index 7da7bc32c..a1d7970d8 100644 --- a/pkg/controller/v1beta1/inferenceservice/controller.go +++ b/pkg/controller/v1beta1/inferenceservice/controller.go @@ -24,6 +24,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" knapis "knative.dev/pkg/apis" duckv1 "knative.dev/pkg/apis/duck/v1" "knative.dev/pkg/network" @@ -556,24 +557,29 @@ func (r *InferenceServiceReconciler) handleServerlessPrerequisites(isvc *v1beta1 } func (r *InferenceServiceReconciler) updateStatus(desiredService *v1beta1.InferenceService, deploymentMode constants.DeploymentModeType) error { - existingService := &v1beta1.InferenceService{} - namespacedName := types.NamespacedName{Name: desiredService.Name, Namespace: desiredService.Namespace} - if err := r.Get(context.TODO(), namespacedName, existingService); err != nil { - return err - } - wasReady := inferenceServiceReadiness(existingService.Status) - if inferenceServiceStatusEqual(existingService.Status, desiredService.Status) { - // If we didn't change anything then don't call updateStatus. - // This is important because the copy we loaded from the informer's - // cache may be stale, and we don't want to overwrite a prior update - // to status with this stale state. - } else if err := r.Status().Update(context.TODO(), desiredService); err != nil { - r.Log.Error(err, "Failed to update InferenceService status", "InferenceService", desiredService.Name) - r.Recorder.Eventf(desiredService, v1.EventTypeWarning, "UpdateFailed", - "Failed to update status for InferenceService %q: %v", desiredService.Name, err) - return errors.Wrapf(err, "fails to update InferenceService status") - } else { - // If there was a difference and there was no error. + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + existingService := &v1beta1.InferenceService{} + namespacedName := types.NamespacedName{Name: desiredService.Name, Namespace: desiredService.Namespace} + if err := r.Get(context.TODO(), namespacedName, existingService); err != nil { + return err + } + wasReady := inferenceServiceReadiness(existingService.Status) + if inferenceServiceStatusEqual(existingService.Status, desiredService.Status) { + // If we didn't change anything then don't call updateStatus. + // This is important because the copy we loaded from the informer's + // cache may be stale, and we don't want to overwrite a prior update + // to status with this stale state. + return nil + } + // Apply only status onto the object we just read from the API. desiredService may carry + // stale spec/metadata from the reconcile queue while Helm/Flux updated the live object; + // Status().Update with mismatched spec+RV can still fail with 409 or other conflicts. + existingService.Status = *desiredService.Status.DeepCopy() + if err := r.Status().Update(context.TODO(), existingService); err != nil { + return err + } + desiredService.Status = existingService.Status + desiredService.ResourceVersion = existingService.ResourceVersion isReady := inferenceServiceReadiness(desiredService.Status) if wasReady && !isReady { // Moved to NotReady State r.Recorder.Eventf(desiredService, v1.EventTypeWarning, string(InferenceServiceNotReadyState), @@ -582,6 +588,13 @@ func (r *InferenceServiceReconciler) updateStatus(desiredService *v1beta1.Infere r.Recorder.Eventf(desiredService, v1.EventTypeNormal, string(InferenceServiceReadyState), fmt.Sprintf("InferenceService [%v] is Ready", desiredService.GetName())) } + return nil + }) + if err != nil { + r.Log.Error(err, "Failed to update InferenceService status", "InferenceService", desiredService.Name) + r.Recorder.Eventf(desiredService, v1.EventTypeWarning, "UpdateFailed", + "Failed to update status for InferenceService %q: %v", desiredService.Name, err) + return errors.Wrapf(err, "fails to update InferenceService status") } return nil } diff --git a/pkg/controller/v1beta1/inferenceservice/utils/model_storage.go b/pkg/controller/v1beta1/inferenceservice/utils/model_storage.go new file mode 100644 index 000000000..26822d5a4 --- /dev/null +++ b/pkg/controller/v1beta1/inferenceservice/utils/model_storage.go @@ -0,0 +1,58 @@ +package utils + +import ( + "path/filepath" + "strings" + + "github.com/sgl-project/ome/pkg/apis/ome/v1beta1" + "github.com/sgl-project/ome/pkg/constants" + "github.com/sgl-project/ome/pkg/controller/v1beta1/controllerconfig" +) + +// ModelStoragePVCClaimName returns the PVC claim name for base-model storage when using a +// shared filesystem (e.g. EFS ReadWriteMany). Order: InferenceService annotation +// (ome.io/model-storage-pvc), then cluster ConfigMap modelStorage.pvcClaimName, else empty +// (hostPath mode). +func ModelStoragePVCClaimName(isvc *v1beta1.InferenceService, cfg *controllerconfig.InferenceServicesConfig) string { + if isvc != nil { + if v := strings.TrimSpace(isvc.Annotations[constants.ModelStoragePVCAnnotationKey]); v != "" { + return v + } + } + if cfg != nil { + return strings.TrimSpace(cfg.ModelStorage.PVCClaimName) + } + return "" +} + +// ModelStoragePVCMountRoot returns the directory inside the pod that corresponds to the root of +// the shared PVC (used with SubPath so per-model paths align with hostPath layout). +func ModelStoragePVCMountRoot(cfg *controllerconfig.InferenceServicesConfig) string { + if cfg != nil && strings.TrimSpace(cfg.ModelStorage.PVCMountRoot) != "" { + return filepath.Clean(cfg.ModelStorage.PVCMountRoot) + } + return constants.DefaultModelPVCMountRoot +} + +// ModelVolumeMountSubPathForPVC returns the SubPath for a volumeMount when the model directory +// (storagePath) is under pvcMountRoot on the shared PVC. Empty means mount the whole claim at +// storagePath (single-directory layout). +func ModelVolumeMountSubPathForPVC(pvcMountRoot, storagePath string) string { + if pvcMountRoot == "" || storagePath == "" { + return "" + } + root := filepath.Clean(pvcMountRoot) + path := filepath.Clean(storagePath) + if root == path { + return "" + } + sep := string(filepath.Separator) + if path != root && !strings.HasPrefix(path, root+sep) { + return "" + } + rel, err := filepath.Rel(root, path) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") { + return "" + } + return rel +} diff --git a/pkg/controller/v1beta1/inferenceservice/utils/model_storage_test.go b/pkg/controller/v1beta1/inferenceservice/utils/model_storage_test.go new file mode 100644 index 000000000..4cfdf9ad0 --- /dev/null +++ b/pkg/controller/v1beta1/inferenceservice/utils/model_storage_test.go @@ -0,0 +1,52 @@ +package utils + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/sgl-project/ome/pkg/apis/ome/v1beta1" + "github.com/sgl-project/ome/pkg/constants" + "github.com/sgl-project/ome/pkg/controller/v1beta1/controllerconfig" +) + +func TestModelVolumeMountSubPathForPVC(t *testing.T) { + root := constants.DefaultModelPVCMountRoot + nested := filepath.Join(root, "a", "b") + tests := []struct { + name string + mountRoot string + storagePath string + want string + }{ + {"equal root", root, root, ""}, + {"subdir", root, filepath.Join(root, "qwen3-vl-8b-instruct"), "qwen3-vl-8b-instruct"}, + {"nested", root, nested, filepath.Join("a", "b")}, + {"outside tree", root, "/other/path", ""}, + {"empty root", "", filepath.Join(root, "x"), ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ModelVolumeMountSubPathForPVC(tt.mountRoot, tt.storagePath) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestModelStoragePVCClaimName(t *testing.T) { + cfg := &controllerconfig.InferenceServicesConfig{ + ModelStorage: controllerconfig.ModelStorageConfig{PVCClaimName: "from-config"}, + } + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{constants.ModelStoragePVCAnnotationKey: " from-annotation "}, + }, + } + assert.Equal(t, "from-annotation", ModelStoragePVCClaimName(isvc, cfg)) + + isvc2 := &v1beta1.InferenceService{ObjectMeta: metav1.ObjectMeta{}} + assert.Equal(t, "from-config", ModelStoragePVCClaimName(isvc2, cfg)) + assert.Equal(t, "", ModelStoragePVCClaimName(isvc2, nil)) +} diff --git a/pkg/controller/v1beta1/inferenceservice/utils/utils.go b/pkg/controller/v1beta1/inferenceservice/utils/utils.go index 31b877e36..f2c2d2ef2 100644 --- a/pkg/controller/v1beta1/inferenceservice/utils/utils.go +++ b/pkg/controller/v1beta1/inferenceservice/utils/utils.go @@ -94,6 +94,16 @@ func GetTargetServicePort(ctx context.Context, c client.Client, isvc *v1beta1.In return port, nil } +// IsSkipModelReadyNodeSelector reports whether the InferenceService annotation requests omitting +// the models.ome.io/...=Ready nodeSelector (for autoscaling / cold-start scheduling). +func IsSkipModelReadyNodeSelector(annotations map[string]string) bool { + if annotations == nil { + return false + } + v, ok := annotations[constants.SkipModelReadyNodeSelectorAnnotationKey] + return ok && v == "true" +} + // AddNodeSelectorForModelReadyNode adds a node selector to the pod spec // for scheduling pods on nodes where the base model is ready. // This is used by both InferenceService and BenchmarkJob controllers to ensure pods diff --git a/pkg/controller/v1beta1/inferenceservice/utils/utils_test.go b/pkg/controller/v1beta1/inferenceservice/utils/utils_test.go index 4fdd72246..15b5485a3 100644 --- a/pkg/controller/v1beta1/inferenceservice/utils/utils_test.go +++ b/pkg/controller/v1beta1/inferenceservice/utils/utils_test.go @@ -1870,6 +1870,25 @@ func TestGetTargetServicePort_ServiceNameResolution(t *testing.T) { } } +func TestIsSkipModelReadyNodeSelector(t *testing.T) { + tests := []struct { + name string + annotations map[string]string + want bool + }{ + {name: "nil annotations", annotations: nil, want: false}, + {name: "empty annotations", annotations: map[string]string{}, want: false}, + {name: "true", annotations: map[string]string{constants.SkipModelReadyNodeSelectorAnnotationKey: "true"}, want: true}, + {name: "false is not skip", annotations: map[string]string{constants.SkipModelReadyNodeSelectorAnnotationKey: "false"}, want: false}, + {name: "other value is not skip", annotations: map[string]string{constants.SkipModelReadyNodeSelectorAnnotationKey: "yes"}, want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, IsSkipModelReadyNodeSelector(tt.annotations)) + }) + } +} + func TestAddNodeSelectorForReadyModel(t *testing.T) { tests := []struct { name string diff --git a/pkg/webhook/admission/pod/fine_tuned_adapter_injector.go b/pkg/webhook/admission/pod/fine_tuned_adapter_injector.go index a5cb59fd9..c98d1fd97 100644 --- a/pkg/webhook/admission/pod/fine_tuned_adapter_injector.go +++ b/pkg/webhook/admission/pod/fine_tuned_adapter_injector.go @@ -3,6 +3,7 @@ package pod import ( "encoding/json" "fmt" + "strings" "github.com/go-playground/validator/v10" v1 "k8s.io/api/core/v1" @@ -18,6 +19,14 @@ const ( fineTunedAdapterConfigMapKeyName = "fineTunedAdapter" ) +// skipFineTunedAdapterInit returns true when the OCI fine-tuned-adapter init container must not run. +// That init image only understands oci:// object storage; PVC paths, s3:// (e.g. Mountpoint CSI on the pod), +// hf://, etc. are delivered without it. +func skipFineTunedAdapterInit(storageURI string) bool { + u := strings.TrimSpace(storageURI) + return u == "" || !strings.HasPrefix(u, storage.OCIStoragePrefix) +} + // FineTunedAdapterInjector represents configuration parameters for the Fine-Tuned Adapter. type FineTunedAdapterInjector struct { Image string `json:"image" validate:"required"` @@ -47,7 +56,15 @@ func newFineTunedAdapterInjector(configMap *v1.ConfigMap, client client.Client) // InjectFineTunedAdapter injects the fine-tuned weight initialization container into the pod if necessary. func (fa *FineTunedAdapterInjector) InjectFineTunedAdapter(pod *v1.Pod) error { if fineTunedWeightName, ok := pod.ObjectMeta.Annotations[constants.FineTunedAdapterInjectionKey]; ok && len(fineTunedWeightName) > 0 { - // set the fine-tuned weight name + // OCI adapter init only downloads oci:// objects. Skip before validate() for PVC (on-disk), + // s3:// (e.g. LoRA via Mountpoint CSI), hf://, etc., so clusters without fineTunedAdapter + // ConfigMap (Image/CompartmentId/…) do not deny pod admission. + ftw, err := isvcutils.GetFineTunedWeight(fa.client, fineTunedWeightName) + if err == nil && ftw.Spec.Storage != nil && ftw.Spec.Storage.StorageUri != nil { + if skipFineTunedAdapterInit(*ftw.Spec.Storage.StorageUri) { + return nil + } + } fa.fineTunedWeightName = fineTunedWeightName return fa.injectFineTunedAdapter(pod) } diff --git a/pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go b/pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go new file mode 100644 index 000000000..75d068429 --- /dev/null +++ b/pkg/webhook/admission/pod/fine_tuned_adapter_injector_test.go @@ -0,0 +1,30 @@ +package pod + +import ( + "testing" +) + +func TestSkipFineTunedAdapterInit(t *testing.T) { + t.Parallel() + cases := []struct { + name string + uri string + skip bool + }{ + {"empty", "", true}, + {"whitespace", " ", true}, + {"pvc", "pvc://harmony-ome-models/qwen3-vl-8b-instruct", true}, + {"s3", "s3://vsco-lora-adapters-dev/studio/harmony/qwen3-vl-8b-instruct/v0p2", true}, + {"hf", "hf://org/repo@main", true}, + {"oci", "oci://namespace/bucket@object", false}, + {"oci_trimmed", " oci://ns/bucket@obj ", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := skipFineTunedAdapterInit(tc.uri); got != tc.skip { + t.Fatalf("skipFineTunedAdapterInit(%q) = %v, want %v", tc.uri, got, tc.skip) + } + }) + } +} diff --git a/site/content/en/docs/reference/labels-and-annotations.md b/site/content/en/docs/reference/labels-and-annotations.md index 7aebd75b3..4639171a4 100644 --- a/site/content/en/docs/reference/labels-and-annotations.md +++ b/site/content/en/docs/reference/labels-and-annotations.md @@ -30,7 +30,7 @@ These annotations are used to configure InferenceService behavior: | Annotation | Description | |-------------------------------------------------|------------------------------------------------------| | `ome.io/inject-model-init` | Enables injection of model initialization containers | -| `ome.io/inject-fine-tuned-adapter` | Enables injection of fine-tuned adapter containers | +| `ome.io/inject-fine-tuned-adapter` | When set on engine pods, may trigger the OCI fine-tuned-adapter init container only if the referenced `FineTunedWeight` uses `oci://` storage; `pvc://`, `s3://`, `hf://`, and other non-OCI schemes are skipped (weights are already supplied elsewhere, e.g. CSI mounts). | | `ome.io/inject-serving-sidecar` | Enables injection of serving sidecar containers | | `ome.io/fine-tuned-weight-ft-strategy` | Specifies the fine-tuning strategy for weights | | `ome.io/base-model-name` | Specifies the base model name |