Bump AppWrapper version to 1.1.1

Author: dgrove-oss

Makefile

@@ -12,12 +12,12 @@ VERSION ?= v0.0.0-dev
 BUNDLE_VERSION ?= $(VERSION:v%=%)
 
 # APPWRAPPER_VERSION defines the default version of the AppWrapper controller
-APPWRAPPER_VERSION ?= v1.0.7
+APPWRAPPER_VERSION ?= v1.1.1
 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper
 APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION}
 
 # KUEUE_VERSION defines the default version of Kueue (used for testing)
-KUEUE_VERSION ?= v0.10.2
+KUEUE_VERSION ?= v0.11.0
 
 USE_RHOAI ?= true
 # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing)

config/crd/appwrapper/kustomization.yaml

@@ -1,4 +1,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- github.com/project-codeflare/appwrapper/config/crd?ref=v1.0.7
+- github.com/project-codeflare/appwrapper/config/crd?ref=v1.1.1

config/rbac/role.yaml

@@ -14,14 +14,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - ""
-  resources:
-  - namespaces
-  verbs:
-  - get
-  - list
-  - watch
 - apiGroups:
   - ""
   resources:
@@ -188,58 +180,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - kueue.x-k8s.io
-  resources:
-  - clusterqueues
-  verbs:
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - kueue.x-k8s.io
-  resources:
-  - resourceflavors
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - kueue.x-k8s.io
-  resources:
-  - workloadpriorityclasses
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - kueue.x-k8s.io
-  resources:
-  - workloads
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - kueue.x-k8s.io
-  resources:
-  - workloads/finalizers
-  verbs:
-  - update
-- apiGroups:
-  - kueue.x-k8s.io
-  resources:
-  - workloads/status
-  verbs:
-  - get
-  - patch
-  - update
 - apiGroups:
   - networking.k8s.io
   resources:
@@ -327,14 +267,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - scheduling.k8s.io
-  resources:
-  - priorityclasses
-  verbs:
-  - get
-  - list
-  - watch
 - apiGroups:
   - scheduling.sigs.k8s.io
   resources:

go.mod

@@ -4,76 +4,84 @@ go 1.23.0
 
 require (
 	github.com/go-logr/logr v1.4.2
-	github.com/onsi/ginkgo/v2 v2.22.0
-	github.com/onsi/gomega v1.36.1
+	github.com/onsi/ginkgo/v2 v2.23.0
+	github.com/onsi/gomega v1.36.2
 	github.com/open-policy-agent/cert-controller v0.12.0
 	github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0
 	github.com/openshift/api v0.0.0-20240904015708-69df64132c91
 	github.com/openshift/client-go v0.0.0-20240904130219-3795e907a202
-	github.com/project-codeflare/appwrapper v1.0.7
+	github.com/project-codeflare/appwrapper v1.1.1
 	github.com/project-codeflare/codeflare-common v0.0.0-20250321141415-67bb8bd932df
-	github.com/ray-project/kuberay/ray-operator v1.2.2
+	github.com/ray-project/kuberay/ray-operator v1.3.1
 	go.uber.org/zap v1.27.0
-	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
-	k8s.io/api v0.31.4
-	k8s.io/apiextensions-apiserver v0.31.2
-	k8s.io/apimachinery v0.31.4
+	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394
+	k8s.io/api v0.32.3
+	k8s.io/apiextensions-apiserver v0.32.3
+	k8s.io/apimachinery v0.32.3
 	k8s.io/client-go v11.0.0+incompatible
-	k8s.io/component-base v0.31.4
+	k8s.io/component-base v0.32.3
 	k8s.io/klog/v2 v2.130.1
-	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
-	sigs.k8s.io/controller-runtime v0.19.3
-	sigs.k8s.io/kueue v0.10.2
+	k8s.io/utils v0.0.0-20241210054802-24370beab758
+	sigs.k8s.io/controller-runtime v0.20.3
+	sigs.k8s.io/kueue v0.11.0
 	sigs.k8s.io/yaml v1.4.0
 )
 
 replace k8s.io/client-go => k8s.io/client-go v0.31.4
 
+replace k8s.io/api => k8s.io/api v0.31.4
+
+replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.31.2
+
+replace k8s.io/apimachinery v0.32.3 => k8s.io/apimachinery v0.31.4
+
+replace sigs.k8s.io/controller-runtime v0.20.3 => sigs.k8s.io/controller-runtime v0.19.3
+
+replace sigs.k8s.io/kueue v0.11.0 => sigs.k8s.io/kueue v0.10.1
+
 replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3
 
 replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0
 
 replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4
 
 // Workaround deprecation of cluster.Status.State in ray-operator v1.2.2
-replace github.com/ray-project/kuberay/ray-operator v1.2.2 => github.com/ray-project/kuberay/ray-operator v1.2.1
+replace github.com/ray-project/kuberay/ray-operator v1.3.1 => github.com/ray-project/kuberay/ray-operator v1.2.1
 
 require (
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/distribution/reference v0.5.0 // indirect
-	github.com/emicklei/go-restful/v3 v3.12.1 // indirect
-	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.12.2 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
-	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonpointer v0.21.1 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
-	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-openapi/swag v0.23.1 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/glog v1.2.4 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
-	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/gnostic-models v0.6.9 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
+	github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/jackc/pgx/v5 v5.5.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
-	github.com/kubeflow/mpi-operator v0.6.0 // indirect
-	github.com/kubeflow/training-operator v1.8.1 // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/kubeflow/training-operator v1.9.0 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/microcosm-cc/bluemonday v1.0.18 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -83,33 +91,32 @@ require (
 	github.com/openshift-online/ocm-sdk-go v0.1.411 // indirect
 	github.com/openshift/custom-resource-status v1.1.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.20.5 // indirect
+	github.com/prometheus/client_golang v1.21.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.57.0 // indirect
-	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/prometheus/common v0.63.0 // indirect
+	github.com/prometheus/procfs v0.16.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/net v0.33.0 // indirect
-	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
-	golang.org/x/term v0.27.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
-	golang.org/x/time v0.6.0 // indirect
-	golang.org/x/tools v0.28.0 // indirect
-	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/protobuf v1.35.1 // indirect
+	golang.org/x/net v0.37.0 // indirect
+	golang.org/x/oauth2 v0.28.0 // indirect
+	golang.org/x/sys v0.31.0 // indirect
+	golang.org/x/term v0.30.0 // indirect
+	golang.org/x/text v0.23.0 // indirect
+	golang.org/x/time v0.11.0 // indirect
+	golang.org/x/tools v0.31.0 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect
+	google.golang.org/protobuf v1.36.5 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiserver v0.31.4 // indirect
-	k8s.io/component-helpers v0.31.4 // indirect
-	k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f // indirect
-	sigs.k8s.io/jobset v0.7.1 // indirect
-	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.4.3 // indirect
+	k8s.io/kube-openapi v0.0.0-20250304201544-e5f78fe3ede9 // indirect
+	sigs.k8s.io/jobset v0.8.0 // indirect
+	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
 )

go.sum

@@ -63,7 +63,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
-	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
 	"sigs.k8s.io/yaml"
 
 	routev1 "github.com/openshift/api/route/v1"
@@ -83,7 +82,6 @@ var (
 )
 
 const (
-	workloadAPI   = "workloads.kueue.x-k8s.io"
 	rayclusterAPI = "rayclusters.ray.io"
 )
 
@@ -97,8 +95,6 @@ func init() {
 	utilruntime.Must(dsciv1.AddToScheme(scheme))
 	// AppWrapper
 	utilruntime.Must(awv1beta2.AddToScheme(scheme))
-	// Kueue
-	utilruntime.Must(kueue.AddToScheme(scheme))
 }
 
 // +kubebuilder:rbac:groups=config.openshift.io,resources=ingresses,verbs=get
@@ -277,38 +273,17 @@ func setupAppWrapperComponents(ctx context.Context, cancel context.CancelFunc, m
 		return nil
 	}
 
-	// AppWrapper webhook doesn't depend on WorkloadAPI availablity but does need certsReady
-	go setupAppWrapperWebhooks(mgr, cfg, certsReady)
-
-	if isAPIAvailable(ctx, mgr, workloadAPI) {
-		setupLog.Info("Workload API available, enabling AppWrappers")
-		go setupAppWrapperController(mgr, cfg, certsReady)
-		return awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config)
-	} else {
-		// If AppWrappers are enabled and the Workload API becomes available later, initiate an orderly
-		// restart of the codeflare operator to enable the workload indexer to be setup in the the new instance of the operator.
-		// It is not possible to add an indexer once the mgr has started so, a restart if the only avenue.
-		setupLog.Info("Workload API not available, setting up waiter for Workload API availability")
-		go waitForAPI(ctx, mgr, workloadAPI, func() {
-			setupLog.Info("Workload API now available, triggering controller restart")
-			cancel()
-		})
-		return nil
-	}
+	// AppWrapper webhook and controller need certsReady
+	go setupAppWrapperControllerAndWebhooks(mgr, cfg, certsReady)
+	return awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config)
 }
 
-func setupAppWrapperWebhooks(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) {
+func setupAppWrapperControllerAndWebhooks(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) {
 	setupLog.Info("Waiting for certificate generation to complete")
 	<-certsReady
-	setupLog.Info("Setting up AppWrapper webhooks")
-	exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks")
-}
-
-func setupAppWrapperController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) {
-	setupLog.Info("Waiting for certificate generation to complete")
-	<-certsReady
-	setupLog.Info("Setting up AppWrapper controller")
+	setupLog.Info("Setting up AppWrapper controller and webhooks")
 	exitOnError(awctrl.SetupControllers(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper controller")
+	exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks")
 }
 
 // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update

main.go

@@ -23,6 +23,9 @@ package controllers
 //+kubebuilder:rbac:groups=workload.codeflare.dev,resources=appwrappers/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=workload.codeflare.dev,resources=appwrappers/finalizers,verbs=update
 
+// permission for events
+//+kubebuilder:rbac:groups="",resources=events,verbs=create;watch;update;patch
+
 // permission to edit wrapped resources: pods, services, jobs, podgroups, pytorchjobs, rayclusters
 //+kubebuilder:rbac:groups="",resources=pods;services,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=apps,resources=deployments;statefulsets,verbs=get;list;watch;create;update;patch;delete
@@ -34,18 +37,5 @@ package controllers
 //+kubebuilder:rbac:groups=ray.io,resources=rayjobs,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=jobset.x-k8s.io,resources=jobsets,verbs=get;list;watch;create;update;patch;delete
 
-// permissions needed by Kueue's generic reconciller
-// +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=list;get;watch
-// +kubebuilder:rbac:groups="",resources=events,verbs=create;watch;update;patch
-// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads/status,verbs=get;update;patch
-// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads/finalizers,verbs=update
-// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=resourceflavors,verbs=get;list;watch
-// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloadpriorityclasses,verbs=get;list;watch
-
-// permissions needed by Kueue's base webhook
-// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch
-
-// permission to watch nodes and edit clusterqueues for Autopilot integration
+// permission to watch nodes for the Autopilot integration
 //+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch
-//+kubebuilder:rbac:groups=kueue.x-k8s.io,resources=clusterqueues,verbs=get;list;watch;update;patch