diff --git a/api/runtime/hooks/v1alpha1/lifecyclehooks_types.go b/api/runtime/hooks/v1alpha1/lifecyclehooks_types.go index bf0b8e15384b..61e979fd01f1 100644 --- a/api/runtime/hooks/v1alpha1/lifecyclehooks_types.go +++ b/api/runtime/hooks/v1alpha1/lifecyclehooks_types.go @@ -97,6 +97,22 @@ type BeforeClusterUpgradeRequest struct { // toKubernetesVersion is the target Kubernetes version of the upgrade. // +required ToKubernetesVersion string `json:"toKubernetesVersion"` + + // controlPlaneUpgrades is the list of version upgrade steps for the control plane. + // +optional + ControlPlaneUpgrades []UpgradeStepInfo `json:"controlPlaneUpgrades,omitempty"` + + // workersUpgrades is the list of version upgrade steps for the workers. + // +optional + WorkersUpgrades []UpgradeStepInfo `json:"workersUpgrades,omitempty"` +} + +// UpgradeStepInfo provide info about a single version upgrade step. +type UpgradeStepInfo struct { + // version is the Kubernetes version for this upgrade step. + // +required + // +kubebuilder:validation:MinLength=1 + Version string `json:"version,omitempty"` } var _ RetryResponseObject = &BeforeClusterUpgradeResponse{} @@ -114,6 +130,50 @@ type BeforeClusterUpgradeResponse struct { // before the updated version is propagated to the underlying objects. func BeforeClusterUpgrade(*BeforeClusterUpgradeRequest, *BeforeClusterUpgradeResponse) {} +// BeforeControlPlaneUpgradeRequest is the request of the BeforeControlPlane hook. +// +kubebuilder:object:root=true +type BeforeControlPlaneUpgradeRequest struct { + metav1.TypeMeta `json:",inline"` + + // CommonRequest contains fields common to all request types. + CommonRequest `json:",inline"` + + // cluster is the cluster object the lifecycle hook corresponds to. + // +required + Cluster clusterv1beta1.Cluster `json:"cluster"` + + // fromKubernetesVersion is the current Kubernetes version of the control plane for the next upgrade step. + // +required + FromKubernetesVersion string `json:"fromKubernetesVersion"` + + // toKubernetesVersion is the target Kubernetes version of the control plane for the next upgrade step. + // +required + ToKubernetesVersion string `json:"toKubernetesVersion"` + + // controlPlaneUpgrades is the list of the remaining version upgrade steps for the control plane, if any. + // +optional + ControlPlaneUpgrades []UpgradeStepInfo `json:"controlPlaneUpgrades,omitempty"` + + // workersUpgrades is the list of the remaining version upgrade steps for workers, if any. + // +optional + WorkersUpgrades []UpgradeStepInfo `json:"workersUpgrades,omitempty"` +} + +var _ RetryResponseObject = &BeforeControlPlaneUpgradeResponse{} + +// BeforeControlPlaneUpgradeResponse is the response of the BeforeControlPlaneUpgrade hook. +// +kubebuilder:object:root=true +type BeforeControlPlaneUpgradeResponse struct { + metav1.TypeMeta `json:",inline"` + + // CommonRetryResponse contains Status, Message and RetryAfterSeconds fields. + CommonRetryResponse `json:",inline"` +} + +// BeforeControlPlaneUpgrade is the hook that will be called before a new version is propagated to the control plane object. +func BeforeControlPlaneUpgrade(*BeforeControlPlaneUpgradeRequest, *BeforeControlPlaneUpgradeResponse) { +} + // AfterControlPlaneUpgradeRequest is the request of the AfterControlPlaneUpgrade hook. // +kubebuilder:object:root=true type AfterControlPlaneUpgradeRequest struct { @@ -126,9 +186,17 @@ type AfterControlPlaneUpgradeRequest struct { // +required Cluster clusterv1beta1.Cluster `json:"cluster"` - // kubernetesVersion is the Kubernetes version of the Control Plane after the upgrade. + // kubernetesVersion is the Kubernetes version of the control plane after an upgrade step. // +required KubernetesVersion string `json:"kubernetesVersion"` + + // controlPlaneUpgrades is the list of the remaining version upgrade steps for the control plane, if any. + // +optional + ControlPlaneUpgrades []UpgradeStepInfo `json:"controlPlaneUpgrades,omitempty"` + + // workersUpgrades is the list of the remaining version upgrade steps for workers, if any. + // +optional + WorkersUpgrades []UpgradeStepInfo `json:"workersUpgrades,omitempty"` } var _ RetryResponseObject = &AfterControlPlaneUpgradeResponse{} @@ -146,6 +214,90 @@ type AfterControlPlaneUpgradeResponse struct { // Kubernetes version and before the target version is propagated to the workload machines. func AfterControlPlaneUpgrade(*AfterControlPlaneUpgradeRequest, *AfterControlPlaneUpgradeResponse) {} +// BeforeWorkersUpgradeRequest is the request of the BeforeWorkersUpgrade hook. +// +kubebuilder:object:root=true +type BeforeWorkersUpgradeRequest struct { + metav1.TypeMeta `json:",inline"` + + // CommonRequest contains fields common to all request types. + CommonRequest `json:",inline"` + + // cluster is the cluster object the lifecycle hook corresponds to. + // +required + Cluster clusterv1beta1.Cluster `json:"cluster"` + + // fromKubernetesVersion is the current Kubernetes version of the workers for the next upgrade step. + // +required + FromKubernetesVersion string `json:"fromKubernetesVersion"` + + // toKubernetesVersion is the target Kubernetes version of the workers for the next upgrade step. + // +required + ToKubernetesVersion string `json:"toKubernetesVersion"` + + // controlPlaneUpgrades is the list of the remaining version upgrade steps for the control plane, if any. + // +optional + ControlPlaneUpgrades []UpgradeStepInfo `json:"controlPlaneUpgrades,omitempty"` + + // workersUpgrades is the list of the remaining version upgrade steps for workers, if any. + // +optional + WorkersUpgrades []UpgradeStepInfo `json:"workersUpgrades,omitempty"` +} + +var _ RetryResponseObject = &BeforeWorkersUpgradeResponse{} + +// BeforeWorkersUpgradeResponse is the response of the BeforeWorkersUpgrade hook. +// +kubebuilder:object:root=true +type BeforeWorkersUpgradeResponse struct { + metav1.TypeMeta `json:",inline"` + + // CommonRetryResponse contains Status, Message and RetryAfterSeconds fields. + CommonRetryResponse `json:",inline"` +} + +// BeforeWorkersUpgrade is the hook that will be called before a new version is propagated to workers. +func BeforeWorkersUpgrade(*BeforeWorkersUpgradeRequest, *BeforeWorkersUpgradeResponse) { +} + +// AfterWorkersUpgradeRequest is the request of the AfterWorkersUpgrade hook. +// +kubebuilder:object:root=true +type AfterWorkersUpgradeRequest struct { + metav1.TypeMeta `json:",inline"` + + // CommonRequest contains fields common to all request types. + CommonRequest `json:",inline"` + + // cluster is the cluster object the lifecycle hook corresponds to. + // +required + Cluster clusterv1beta1.Cluster `json:"cluster"` + + // kubernetesVersion is the Kubernetes version of the workers after an upgrade step. + // +required + KubernetesVersion string `json:"kubernetesVersion"` + + // controlPlaneUpgrades is the list of the remaining version upgrade steps for the control plane, if any. + // +optional + ControlPlaneUpgrades []UpgradeStepInfo `json:"controlPlaneUpgrades,omitempty"` + + // workersUpgrades is the list of the remaining version upgrade steps for workers, if any. + // +optional + WorkersUpgrades []UpgradeStepInfo `json:"workersUpgrades,omitempty"` +} + +var _ RetryResponseObject = &AfterWorkersUpgradeResponse{} + +// AfterWorkersUpgradeResponse is the response of the AfterWorkersUpgrade hook. +// +kubebuilder:object:root=true +type AfterWorkersUpgradeResponse struct { + metav1.TypeMeta `json:",inline"` + + // CommonRetryResponse contains Status, Message and RetryAfterSeconds fields. + CommonRetryResponse `json:",inline"` +} + +// AfterWorkersUpgrade is the hook called after the control plane is successfully upgraded to the target +// Kubernetes version and before the target version is propagated to the workload machines. +func AfterWorkersUpgrade(*AfterWorkersUpgradeRequest, *AfterWorkersUpgradeResponse) {} + // AfterClusterUpgradeRequest is the request of the AfterClusterUpgrade hook. // +kubebuilder:object:root=true type AfterClusterUpgradeRequest struct { @@ -243,18 +395,58 @@ func init() { "tasks before the new version is propagated to the control plane", }) + catalogBuilder.RegisterHook(BeforeControlPlaneUpgrade, &runtimecatalog.HookMeta{ + Tags: []string{"Lifecycle Hooks"}, + Summary: "Cluster API Runtime will call this hook before the control plane is upgraded", + Description: "This hook is called before a new version is propagated to the control plane object.\n" + + "\n" + + "Notes:\n" + + "- This hook will be called only for Clusters with a managed topology\n" + + "- When an upgrade is starting, BeforeControlPlaneUpgrade will be called after BeforeClusterUpgrade is completed\n" + + "- When an upgrade is in progress BeforeControlPlaneUpgrade will be called for each intermediate version that will be applied " + + "to the control plane (instead BeforeClusterUpgrade will be called only once at the beginning of the upgrade)" + + "- This is a blocking hook; Runtime Extension implementers can use this hook to execute " + + "tasks before the new version is propagated to the control plane", + }) + catalogBuilder.RegisterHook(AfterControlPlaneUpgrade, &runtimecatalog.HookMeta{ Tags: []string{"Lifecycle Hooks"}, Summary: "Cluster API Runtime will call this hook after the control plane is upgraded", Description: "Cluster API Runtime will call this hook after the a cluster's control plane has been upgraded to the version specified " + - "in spec.topology.version, and immediately before the new version is going to be propagated to the MachineDeployments. " + + "in spec.topology.version or to an intermediate version in the upgrade plan." + "A control plane upgrade is completed when all the machines in the control plane have been upgraded.\n" + "\n" + "Notes:\n" + "- This hook will be called only for Clusters with a managed topology\n" + "- The call's request contains the Cluster object and the Kubernetes version we upgraded to\n" + "- This is a blocking hook; Runtime Extension implementers can use this hook to execute " + - "tasks before the new version is propagated to the MachineDeployments", + "tasks before the new version is propagated to the MachineDeployments and Machine Pools", + }) + + catalogBuilder.RegisterHook(BeforeWorkersUpgrade, &runtimecatalog.HookMeta{ + Tags: []string{"Lifecycle Hooks"}, + Summary: "Cluster API Runtime will call this hook before the workers are upgraded", + Description: "This hook is called before a new version is propagated to workers.\n" + + "\n" + + "Notes:\n" + + "- This hook will be called only for Clusters with a managed topology\n" + + "- This hook will be called only if workers upgrade must be performed for an intermediate version of " + + "a chained upgrade or when upgrading to the target spec.topology.version.\n" + + "- This is a blocking hook; Runtime Extension implementers can use this hook to execute " + + "tasks before the new version is propagated to the MachineDeployments and Machine Pools", + }) + + catalogBuilder.RegisterHook(AfterWorkersUpgrade, &runtimecatalog.HookMeta{ + Tags: []string{"Lifecycle Hooks"}, + Summary: "Cluster API Runtime will call this hook after workers are upgraded", + Description: "This hook is called after all the workers have been upgraded to the version specified in spec.topology.version " + + "or to an intermediate version in the upgrade plan.\n" + + "\n" + + "Notes:\n" + + "- This hook will be called only for Clusters with a managed topology\n" + + "- The call's request contains the Cluster object, the current Kubernetes version and the Kubernetes version we are upgrading to\n" + + "- This is a blocking hook; Runtime Extension implementers can use this hook to execute " + + "tasks before the upgrade plan continues, or when already at the target spec.topology.version, before AfterClusterUpgrade is called.\n", }) catalogBuilder.RegisterHook(AfterClusterUpgrade, &runtimecatalog.HookMeta{ diff --git a/api/runtime/hooks/v1alpha1/zz_generated.deepcopy.go b/api/runtime/hooks/v1alpha1/zz_generated.deepcopy.go index 8ea3e72b3523..153829b842b0 100644 --- a/api/runtime/hooks/v1alpha1/zz_generated.deepcopy.go +++ b/api/runtime/hooks/v1alpha1/zz_generated.deepcopy.go @@ -133,6 +133,16 @@ func (in *AfterControlPlaneUpgradeRequest) DeepCopyInto(out *AfterControlPlaneUp out.TypeMeta = in.TypeMeta in.CommonRequest.DeepCopyInto(&out.CommonRequest) in.Cluster.DeepCopyInto(&out.Cluster) + if in.ControlPlaneUpgrades != nil { + in, out := &in.ControlPlaneUpgrades, &out.ControlPlaneUpgrades + *out = make([]UpgradeStepInfo, len(*in)) + copy(*out, *in) + } + if in.WorkersUpgrades != nil { + in, out := &in.WorkersUpgrades, &out.WorkersUpgrades + *out = make([]UpgradeStepInfo, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AfterControlPlaneUpgradeRequest. @@ -178,6 +188,67 @@ func (in *AfterControlPlaneUpgradeResponse) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AfterWorkersUpgradeRequest) DeepCopyInto(out *AfterWorkersUpgradeRequest) { + *out = *in + out.TypeMeta = in.TypeMeta + in.CommonRequest.DeepCopyInto(&out.CommonRequest) + in.Cluster.DeepCopyInto(&out.Cluster) + if in.ControlPlaneUpgrades != nil { + in, out := &in.ControlPlaneUpgrades, &out.ControlPlaneUpgrades + *out = make([]UpgradeStepInfo, len(*in)) + copy(*out, *in) + } + if in.WorkersUpgrades != nil { + in, out := &in.WorkersUpgrades, &out.WorkersUpgrades + *out = make([]UpgradeStepInfo, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AfterWorkersUpgradeRequest. +func (in *AfterWorkersUpgradeRequest) DeepCopy() *AfterWorkersUpgradeRequest { + if in == nil { + return nil + } + out := new(AfterWorkersUpgradeRequest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AfterWorkersUpgradeRequest) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AfterWorkersUpgradeResponse) DeepCopyInto(out *AfterWorkersUpgradeResponse) { + *out = *in + out.TypeMeta = in.TypeMeta + out.CommonRetryResponse = in.CommonRetryResponse +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AfterWorkersUpgradeResponse. +func (in *AfterWorkersUpgradeResponse) DeepCopy() *AfterWorkersUpgradeResponse { + if in == nil { + return nil + } + out := new(AfterWorkersUpgradeResponse) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AfterWorkersUpgradeResponse) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BeforeClusterCreateRequest) DeepCopyInto(out *BeforeClusterCreateRequest) { *out = *in @@ -286,6 +357,16 @@ func (in *BeforeClusterUpgradeRequest) DeepCopyInto(out *BeforeClusterUpgradeReq out.TypeMeta = in.TypeMeta in.CommonRequest.DeepCopyInto(&out.CommonRequest) in.Cluster.DeepCopyInto(&out.Cluster) + if in.ControlPlaneUpgrades != nil { + in, out := &in.ControlPlaneUpgrades, &out.ControlPlaneUpgrades + *out = make([]UpgradeStepInfo, len(*in)) + copy(*out, *in) + } + if in.WorkersUpgrades != nil { + in, out := &in.WorkersUpgrades, &out.WorkersUpgrades + *out = make([]UpgradeStepInfo, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BeforeClusterUpgradeRequest. @@ -331,6 +412,128 @@ func (in *BeforeClusterUpgradeResponse) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BeforeControlPlaneUpgradeRequest) DeepCopyInto(out *BeforeControlPlaneUpgradeRequest) { + *out = *in + out.TypeMeta = in.TypeMeta + in.CommonRequest.DeepCopyInto(&out.CommonRequest) + in.Cluster.DeepCopyInto(&out.Cluster) + if in.ControlPlaneUpgrades != nil { + in, out := &in.ControlPlaneUpgrades, &out.ControlPlaneUpgrades + *out = make([]UpgradeStepInfo, len(*in)) + copy(*out, *in) + } + if in.WorkersUpgrades != nil { + in, out := &in.WorkersUpgrades, &out.WorkersUpgrades + *out = make([]UpgradeStepInfo, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BeforeControlPlaneUpgradeRequest. +func (in *BeforeControlPlaneUpgradeRequest) DeepCopy() *BeforeControlPlaneUpgradeRequest { + if in == nil { + return nil + } + out := new(BeforeControlPlaneUpgradeRequest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BeforeControlPlaneUpgradeRequest) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BeforeControlPlaneUpgradeResponse) DeepCopyInto(out *BeforeControlPlaneUpgradeResponse) { + *out = *in + out.TypeMeta = in.TypeMeta + out.CommonRetryResponse = in.CommonRetryResponse +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BeforeControlPlaneUpgradeResponse. +func (in *BeforeControlPlaneUpgradeResponse) DeepCopy() *BeforeControlPlaneUpgradeResponse { + if in == nil { + return nil + } + out := new(BeforeControlPlaneUpgradeResponse) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BeforeControlPlaneUpgradeResponse) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BeforeWorkersUpgradeRequest) DeepCopyInto(out *BeforeWorkersUpgradeRequest) { + *out = *in + out.TypeMeta = in.TypeMeta + in.CommonRequest.DeepCopyInto(&out.CommonRequest) + in.Cluster.DeepCopyInto(&out.Cluster) + if in.ControlPlaneUpgrades != nil { + in, out := &in.ControlPlaneUpgrades, &out.ControlPlaneUpgrades + *out = make([]UpgradeStepInfo, len(*in)) + copy(*out, *in) + } + if in.WorkersUpgrades != nil { + in, out := &in.WorkersUpgrades, &out.WorkersUpgrades + *out = make([]UpgradeStepInfo, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BeforeWorkersUpgradeRequest. +func (in *BeforeWorkersUpgradeRequest) DeepCopy() *BeforeWorkersUpgradeRequest { + if in == nil { + return nil + } + out := new(BeforeWorkersUpgradeRequest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BeforeWorkersUpgradeRequest) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BeforeWorkersUpgradeResponse) DeepCopyInto(out *BeforeWorkersUpgradeResponse) { + *out = *in + out.TypeMeta = in.TypeMeta + out.CommonRetryResponse = in.CommonRetryResponse +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BeforeWorkersUpgradeResponse. +func (in *BeforeWorkersUpgradeResponse) DeepCopy() *BeforeWorkersUpgradeResponse { + if in == nil { + return nil + } + out := new(BeforeWorkersUpgradeResponse) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BeforeWorkersUpgradeResponse) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Builtins) DeepCopyInto(out *Builtins) { *out = *in @@ -1286,6 +1489,21 @@ func (in *UpgradeStep) DeepCopy() *UpgradeStep { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpgradeStepInfo) DeepCopyInto(out *UpgradeStepInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpgradeStepInfo. +func (in *UpgradeStepInfo) DeepCopy() *UpgradeStepInfo { + if in == nil { + return nil + } + out := new(UpgradeStepInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ValidateTopologyRequest) DeepCopyInto(out *ValidateTopologyRequest) { *out = *in diff --git a/api/runtime/hooks/v1alpha1/zz_generated.openapi.go b/api/runtime/hooks/v1alpha1/zz_generated.openapi.go index 5f75b60873aa..52a2a54e106f 100644 --- a/api/runtime/hooks/v1alpha1/zz_generated.openapi.go +++ b/api/runtime/hooks/v1alpha1/zz_generated.openapi.go @@ -34,12 +34,18 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.AfterControlPlaneInitializedResponse": schema_api_runtime_hooks_v1alpha1_AfterControlPlaneInitializedResponse(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.AfterControlPlaneUpgradeRequest": schema_api_runtime_hooks_v1alpha1_AfterControlPlaneUpgradeRequest(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.AfterControlPlaneUpgradeResponse": schema_api_runtime_hooks_v1alpha1_AfterControlPlaneUpgradeResponse(ref), + "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.AfterWorkersUpgradeRequest": schema_api_runtime_hooks_v1alpha1_AfterWorkersUpgradeRequest(ref), + "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.AfterWorkersUpgradeResponse": schema_api_runtime_hooks_v1alpha1_AfterWorkersUpgradeResponse(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.BeforeClusterCreateRequest": schema_api_runtime_hooks_v1alpha1_BeforeClusterCreateRequest(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.BeforeClusterCreateResponse": schema_api_runtime_hooks_v1alpha1_BeforeClusterCreateResponse(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.BeforeClusterDeleteRequest": schema_api_runtime_hooks_v1alpha1_BeforeClusterDeleteRequest(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.BeforeClusterDeleteResponse": schema_api_runtime_hooks_v1alpha1_BeforeClusterDeleteResponse(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.BeforeClusterUpgradeRequest": schema_api_runtime_hooks_v1alpha1_BeforeClusterUpgradeRequest(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.BeforeClusterUpgradeResponse": schema_api_runtime_hooks_v1alpha1_BeforeClusterUpgradeResponse(ref), + "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.BeforeControlPlaneUpgradeRequest": schema_api_runtime_hooks_v1alpha1_BeforeControlPlaneUpgradeRequest(ref), + "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.BeforeControlPlaneUpgradeResponse": schema_api_runtime_hooks_v1alpha1_BeforeControlPlaneUpgradeResponse(ref), + "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.BeforeWorkersUpgradeRequest": schema_api_runtime_hooks_v1alpha1_BeforeWorkersUpgradeRequest(ref), + "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.BeforeWorkersUpgradeResponse": schema_api_runtime_hooks_v1alpha1_BeforeWorkersUpgradeResponse(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.Builtins": schema_api_runtime_hooks_v1alpha1_Builtins(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.CanUpdateMachineRequest": schema_api_runtime_hooks_v1alpha1_CanUpdateMachineRequest(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.CanUpdateMachineRequestObjects": schema_api_runtime_hooks_v1alpha1_CanUpdateMachineRequestObjects(ref), @@ -80,6 +86,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpdateMachineRequestObjects": schema_api_runtime_hooks_v1alpha1_UpdateMachineRequestObjects(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpdateMachineResponse": schema_api_runtime_hooks_v1alpha1_UpdateMachineResponse(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStep": schema_api_runtime_hooks_v1alpha1_UpgradeStep(ref), + "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo": schema_api_runtime_hooks_v1alpha1_UpgradeStepInfo(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.ValidateTopologyRequest": schema_api_runtime_hooks_v1alpha1_ValidateTopologyRequest(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.ValidateTopologyRequestItem": schema_api_runtime_hooks_v1alpha1_ValidateTopologyRequestItem(ref), "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.ValidateTopologyResponse": schema_api_runtime_hooks_v1alpha1_ValidateTopologyResponse(ref), @@ -335,18 +342,46 @@ func schema_api_runtime_hooks_v1alpha1_AfterControlPlaneUpgradeRequest(ref commo }, "kubernetesVersion": { SchemaProps: spec.SchemaProps{ - Description: "kubernetesVersion is the Kubernetes version of the Control Plane after the upgrade.", + Description: "kubernetesVersion is the Kubernetes version of the control plane after an upgrade step.", Default: "", Type: []string{"string"}, Format: "", }, }, + "controlPlaneUpgrades": { + SchemaProps: spec.SchemaProps{ + Description: "controlPlaneUpgrades is the list of the remaining version upgrade steps for the control plane, if any.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"), + }, + }, + }, + }, + }, + "workersUpgrades": { + SchemaProps: spec.SchemaProps{ + Description: "workersUpgrades is the list of the remaining version upgrade steps for workers, if any.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"), + }, + }, + }, + }, + }, }, Required: []string{"cluster", "kubernetesVersion"}, }, }, Dependencies: []string{ - "sigs.k8s.io/cluster-api/api/core/v1beta1.Cluster"}, + "sigs.k8s.io/cluster-api/api/core/v1beta1.Cluster", "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"}, } } @@ -402,6 +437,147 @@ func schema_api_runtime_hooks_v1alpha1_AfterControlPlaneUpgradeResponse(ref comm } } +func schema_api_runtime_hooks_v1alpha1_AfterWorkersUpgradeRequest(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "AfterWorkersUpgradeRequest is the request of the AfterWorkersUpgrade hook.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "settings": { + SchemaProps: spec.SchemaProps{ + Description: "settings defines key value pairs to be passed to the call.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "cluster": { + SchemaProps: spec.SchemaProps{ + Description: "cluster is the cluster object the lifecycle hook corresponds to.", + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/cluster-api/api/core/v1beta1.Cluster"), + }, + }, + "kubernetesVersion": { + SchemaProps: spec.SchemaProps{ + Description: "kubernetesVersion is the Kubernetes version of the workers after an upgrade step.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "controlPlaneUpgrades": { + SchemaProps: spec.SchemaProps{ + Description: "controlPlaneUpgrades is the list of the remaining version upgrade steps for the control plane, if any.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"), + }, + }, + }, + }, + }, + "workersUpgrades": { + SchemaProps: spec.SchemaProps{ + Description: "workersUpgrades is the list of the remaining version upgrade steps for workers, if any.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"), + }, + }, + }, + }, + }, + }, + Required: []string{"cluster", "kubernetesVersion"}, + }, + }, + Dependencies: []string{ + "sigs.k8s.io/cluster-api/api/core/v1beta1.Cluster", "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"}, + } +} + +func schema_api_runtime_hooks_v1alpha1_AfterWorkersUpgradeResponse(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "AfterWorkersUpgradeResponse is the response of the AfterWorkersUpgrade hook.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Description: "status of the call. One of \"Success\" or \"Failure\".\n\nPossible enum values:\n - `\"Failure\"` represents a failure response.\n - `\"Success\"` represents a success response.", + Default: "", + Type: []string{"string"}, + Format: "", + Enum: []interface{}{"Failure", "Success"}, + }, + }, + "message": { + SchemaProps: spec.SchemaProps{ + Description: "message is a human-readable description of the status of the call.", + Type: []string{"string"}, + Format: "", + }, + }, + "retryAfterSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "retryAfterSeconds when set to a non-zero value signifies that the hook will be called again at a future time.", + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + Required: []string{"status", "retryAfterSeconds"}, + }, + }, + } +} + func schema_api_runtime_hooks_v1alpha1_BeforeClusterCreateRequest(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -672,12 +848,40 @@ func schema_api_runtime_hooks_v1alpha1_BeforeClusterUpgradeRequest(ref common.Re Format: "", }, }, + "controlPlaneUpgrades": { + SchemaProps: spec.SchemaProps{ + Description: "controlPlaneUpgrades is the list of version upgrade steps for the control plane.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"), + }, + }, + }, + }, + }, + "workersUpgrades": { + SchemaProps: spec.SchemaProps{ + Description: "workersUpgrades is the list of version upgrade steps for the workers.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"), + }, + }, + }, + }, + }, }, Required: []string{"cluster", "fromKubernetesVersion", "toKubernetesVersion"}, }, }, Dependencies: []string{ - "sigs.k8s.io/cluster-api/api/core/v1beta1.Cluster"}, + "sigs.k8s.io/cluster-api/api/core/v1beta1.Cluster", "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"}, } } @@ -733,6 +937,304 @@ func schema_api_runtime_hooks_v1alpha1_BeforeClusterUpgradeResponse(ref common.R } } +func schema_api_runtime_hooks_v1alpha1_BeforeControlPlaneUpgradeRequest(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "BeforeControlPlaneUpgradeRequest is the request of the BeforeControlPlane hook.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "settings": { + SchemaProps: spec.SchemaProps{ + Description: "settings defines key value pairs to be passed to the call.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "cluster": { + SchemaProps: spec.SchemaProps{ + Description: "cluster is the cluster object the lifecycle hook corresponds to.", + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/cluster-api/api/core/v1beta1.Cluster"), + }, + }, + "fromKubernetesVersion": { + SchemaProps: spec.SchemaProps{ + Description: "fromKubernetesVersion is the current Kubernetes version of the control plane for the next upgrade step.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "toKubernetesVersion": { + SchemaProps: spec.SchemaProps{ + Description: "toKubernetesVersion is the target Kubernetes version of the control plane for the next upgrade step.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "controlPlaneUpgrades": { + SchemaProps: spec.SchemaProps{ + Description: "controlPlaneUpgrades is the list of the remaining version upgrade steps for the control plane, if any.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"), + }, + }, + }, + }, + }, + "workersUpgrades": { + SchemaProps: spec.SchemaProps{ + Description: "workersUpgrades is the list of the remaining version upgrade steps for workers, if any.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"), + }, + }, + }, + }, + }, + }, + Required: []string{"cluster", "fromKubernetesVersion", "toKubernetesVersion"}, + }, + }, + Dependencies: []string{ + "sigs.k8s.io/cluster-api/api/core/v1beta1.Cluster", "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"}, + } +} + +func schema_api_runtime_hooks_v1alpha1_BeforeControlPlaneUpgradeResponse(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "BeforeControlPlaneUpgradeResponse is the response of the BeforeControlPlaneUpgrade hook.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Description: "status of the call. One of \"Success\" or \"Failure\".\n\nPossible enum values:\n - `\"Failure\"` represents a failure response.\n - `\"Success\"` represents a success response.", + Default: "", + Type: []string{"string"}, + Format: "", + Enum: []interface{}{"Failure", "Success"}, + }, + }, + "message": { + SchemaProps: spec.SchemaProps{ + Description: "message is a human-readable description of the status of the call.", + Type: []string{"string"}, + Format: "", + }, + }, + "retryAfterSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "retryAfterSeconds when set to a non-zero value signifies that the hook will be called again at a future time.", + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + Required: []string{"status", "retryAfterSeconds"}, + }, + }, + } +} + +func schema_api_runtime_hooks_v1alpha1_BeforeWorkersUpgradeRequest(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "BeforeWorkersUpgradeRequest is the request of the BeforeWorkersUpgrade hook.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "settings": { + SchemaProps: spec.SchemaProps{ + Description: "settings defines key value pairs to be passed to the call.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "cluster": { + SchemaProps: spec.SchemaProps{ + Description: "cluster is the cluster object the lifecycle hook corresponds to.", + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/cluster-api/api/core/v1beta1.Cluster"), + }, + }, + "fromKubernetesVersion": { + SchemaProps: spec.SchemaProps{ + Description: "fromKubernetesVersion is the current Kubernetes version of the workers for the next upgrade step.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "toKubernetesVersion": { + SchemaProps: spec.SchemaProps{ + Description: "toKubernetesVersion is the target Kubernetes version of the workers for the next upgrade step.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "controlPlaneUpgrades": { + SchemaProps: spec.SchemaProps{ + Description: "controlPlaneUpgrades is the list of the remaining version upgrade steps for the control plane, if any.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"), + }, + }, + }, + }, + }, + "workersUpgrades": { + SchemaProps: spec.SchemaProps{ + Description: "workersUpgrades is the list of the remaining version upgrade steps for workers, if any.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"), + }, + }, + }, + }, + }, + }, + Required: []string{"cluster", "fromKubernetesVersion", "toKubernetesVersion"}, + }, + }, + Dependencies: []string{ + "sigs.k8s.io/cluster-api/api/core/v1beta1.Cluster", "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1.UpgradeStepInfo"}, + } +} + +func schema_api_runtime_hooks_v1alpha1_BeforeWorkersUpgradeResponse(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "BeforeWorkersUpgradeResponse is the response of the BeforeWorkersUpgrade hook.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Description: "status of the call. One of \"Success\" or \"Failure\".\n\nPossible enum values:\n - `\"Failure\"` represents a failure response.\n - `\"Success\"` represents a success response.", + Default: "", + Type: []string{"string"}, + Format: "", + Enum: []interface{}{"Failure", "Success"}, + }, + }, + "message": { + SchemaProps: spec.SchemaProps{ + Description: "message is a human-readable description of the status of the call.", + Type: []string{"string"}, + Format: "", + }, + }, + "retryAfterSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "retryAfterSeconds when set to a non-zero value signifies that the hook will be called again at a future time.", + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + Required: []string{"status", "retryAfterSeconds"}, + }, + }, + } +} + func schema_api_runtime_hooks_v1alpha1_Builtins(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -2544,6 +3046,27 @@ func schema_api_runtime_hooks_v1alpha1_UpgradeStep(ref common.ReferenceCallback) } } +func schema_api_runtime_hooks_v1alpha1_UpgradeStepInfo(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "UpgradeStepInfo provide info about a single version upgrade step.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "version": { + SchemaProps: spec.SchemaProps{ + Description: "version is the Kubernetes version for this upgrade step.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"version"}, + }, + }, + } +} + func schema_api_runtime_hooks_v1alpha1_ValidateTopologyRequest(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/exp/topology/desiredstate/desired_state.go b/exp/topology/desiredstate/desired_state.go index afc5dcc34ebd..9e4a817f62a3 100644 --- a/exp/topology/desiredstate/desired_state.go +++ b/exp/topology/desiredstate/desired_state.go @@ -19,11 +19,8 @@ package desiredstate import ( "context" - "fmt" "maps" "reflect" - "slices" - "strings" "time" "github.com/pkg/errors" @@ -33,7 +30,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/klog/v2" "k8s.io/utils/ptr" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" @@ -41,7 +37,6 @@ import ( runtimehooksv1 "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1" "sigs.k8s.io/cluster-api/controllers/clustercache" "sigs.k8s.io/cluster-api/controllers/external" - runtimecatalog "sigs.k8s.io/cluster-api/exp/runtime/catalog" runtimeclient "sigs.k8s.io/cluster-api/exp/runtime/client" "sigs.k8s.io/cluster-api/exp/topology/scope" "sigs.k8s.io/cluster-api/feature" @@ -507,7 +502,6 @@ func (g *generator) computeControlPlane(ctx context.Context, s *scope.Scope, inf // The version is calculated using the state of the current machine deployments, the current control plane // and the version defined in the topology. func (g *generator) computeControlPlaneVersion(ctx context.Context, s *scope.Scope) (string, error) { - log := ctrl.LoggerFrom(ctx) topologyVersion := s.Blueprint.Topology.Version // If we are creating the control plane object (current control plane is nil), use version from topology. if s.Current.ControlPlane == nil || s.Current.ControlPlane.Object == nil { @@ -560,41 +554,12 @@ func (g *generator) computeControlPlaneVersion(ctx context.Context, s *scope.Sco // if the control plane is not upgrading, before making further considerations about if to pick up another version, // we should call the AfterControlPlaneUpgrade hook if not already done. if feature.Gates.Enabled(feature.RuntimeSDK) { - // Call the hook only if we are tracking the intent to do so. If it is not tracked it means we don't need to call the - // hook because we didn't go through an upgrade or we already called the hook after the upgrade. - if hooks.IsPending(runtimehooksv1.AfterControlPlaneUpgrade, s.Current.Cluster) { - v1beta1Cluster := &clusterv1beta1.Cluster{} - // DeepCopy cluster because ConvertFrom has side effects like adding the conversion annotation. - if err := v1beta1Cluster.ConvertFrom(s.Current.Cluster.DeepCopy()); err != nil { - return "", errors.Wrap(err, "error converting Cluster to v1beta1 Cluster") - } - - // Call all the registered extension for the hook. - hookRequest := &runtimehooksv1.AfterControlPlaneUpgradeRequest{ - Cluster: *cleanupCluster(v1beta1Cluster), - KubernetesVersion: *currentVersion, - } - hookResponse := &runtimehooksv1.AfterControlPlaneUpgradeResponse{} - if err := g.RuntimeClient.CallAllExtensions(ctx, runtimehooksv1.AfterControlPlaneUpgrade, s.Current.Cluster, hookRequest, hookResponse); err != nil { - return "", err - } - // Add the response to the tracker so we can later update condition or requeue when required. - s.HookResponseTracker.Add(runtimehooksv1.AfterControlPlaneUpgrade, hookResponse) - - // If the extension responds to hold off on starting Machine deployments upgrades, - // change the UpgradeTracker accordingly, otherwise the hook call is completed and we - // can remove this hook from the list of pending-hooks. - if hookResponse.RetryAfterSeconds != 0 { - v := topologyVersion - if len(s.UpgradeTracker.ControlPlane.UpgradePlan) > 0 { - v = s.UpgradeTracker.ControlPlane.UpgradePlan[0] - } - log.Info(fmt.Sprintf("Upgrade to version %q is blocked by %q hook", v, runtimecatalog.HookName(runtimehooksv1.AfterControlPlaneUpgrade))) - return *currentVersion, nil - } - if err := hooks.MarkAsDone(ctx, g.Client, s.Current.Cluster, runtimehooksv1.AfterControlPlaneUpgrade); err != nil { - return "", err - } + hookCompleted, err := g.callAfterControlPlaneUpgradeHook(ctx, s, currentVersion, topologyVersion) + if err != nil { + return "", err + } + if !hookCompleted { + return *currentVersion, nil } } @@ -632,62 +597,12 @@ func (g *generator) computeControlPlaneVersion(ctx context.Context, s *scope.Sco // are not upgrading/are not required to upgrade. // If not already done, call the BeforeClusterUpgrade hook before picking up the desired version. if feature.Gates.Enabled(feature.RuntimeSDK) { - // NOTE: the hook should be called only at the beginning of either a regular upgrade or a multistep upgrade sequence (it should not be called when in the middle of a multistep upgrade sequence); - // to detect if we are at the beginning of an upgrade, we check if the intent to call the AfterClusterUpgrade is not yet tracked. - if !hooks.IsPending(runtimehooksv1.AfterClusterUpgrade, s.Current.Cluster) { - var hookAnnotations []string - for key := range s.Current.Cluster.Annotations { - if strings.HasPrefix(key, clusterv1.BeforeClusterUpgradeHookAnnotationPrefix) { - hookAnnotations = append(hookAnnotations, key) - } - } - if len(hookAnnotations) > 0 { - slices.Sort(hookAnnotations) - message := fmt.Sprintf("annotations [%s] are set", strings.Join(hookAnnotations, ", ")) - if len(hookAnnotations) == 1 { - message = fmt.Sprintf("annotation [%s] is set", strings.Join(hookAnnotations, ", ")) - } - // Add the hook with a response to the tracker so we can later update the condition. - s.HookResponseTracker.Add(runtimehooksv1.BeforeClusterUpgrade, &runtimehooksv1.BeforeClusterUpgradeResponse{ - CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ - // RetryAfterSeconds needs to be set because having only hooks without RetryAfterSeconds - // would lead to not updating the condition. We can rely on getting an event when the - // annotation gets removed so we set twice of the default sync-period to not cause additional reconciles. - RetryAfterSeconds: 20 * 60, - CommonResponse: runtimehooksv1.CommonResponse{ - Message: message, - }, - }, - }) - - log.Info(fmt.Sprintf("Cluster upgrade to version %q is blocked by %q hook (via annotations)", topologyVersion, runtimecatalog.HookName(runtimehooksv1.BeforeClusterUpgrade)), "hooks", strings.Join(hookAnnotations, ",")) - return *currentVersion, nil - } - - // At this point the control plane and the machine deployments are stable and we are almost ready to pick - // up the topologyVersion. Call the BeforeClusterUpgrade hook before picking up the desired version. - v1beta1Cluster := &clusterv1beta1.Cluster{} - // DeepCopy cluster because ConvertFrom has side effects like adding the conversion annotation. - if err := v1beta1Cluster.ConvertFrom(s.Current.Cluster.DeepCopy()); err != nil { - return "", errors.Wrap(err, "error converting Cluster to v1beta1 Cluster") - } - - hookRequest := &runtimehooksv1.BeforeClusterUpgradeRequest{ - Cluster: *cleanupCluster(v1beta1Cluster), - FromKubernetesVersion: *currentVersion, - ToKubernetesVersion: topologyVersion, - } - hookResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{} - if err := g.RuntimeClient.CallAllExtensions(ctx, runtimehooksv1.BeforeClusterUpgrade, s.Current.Cluster, hookRequest, hookResponse); err != nil { - return "", err - } - // Add the response to the tracker so we can later update condition or requeue when required. - s.HookResponseTracker.Add(runtimehooksv1.BeforeClusterUpgrade, hookResponse) - if hookResponse.RetryAfterSeconds != 0 { - // Cannot pickup the new version right now. Need to try again later. - log.Info(fmt.Sprintf("Cluster upgrade to version %q is blocked by %q hook", topologyVersion, runtimecatalog.HookName(runtimehooksv1.BeforeClusterUpgrade))) - return *currentVersion, nil - } + hookCompleted, err := g.callBeforeClusterUpgradeHook(ctx, s, currentVersion, topologyVersion) + if err != nil { + return "", err + } + if !hookCompleted { + return *currentVersion, nil } } diff --git a/exp/topology/desiredstate/desired_state_test.go b/exp/topology/desiredstate/desired_state_test.go index 2639ee076338..1edecdfc6889 100644 --- a/exp/topology/desiredstate/desired_state_test.go +++ b/exp/topology/desiredstate/desired_state_test.go @@ -19,17 +19,14 @@ package desiredstate import ( "encoding/json" "fmt" - "maps" "strings" "testing" "time" "github.com/google/go-cmp/cmp" . "github.com/onsi/gomega" - "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -51,7 +48,6 @@ import ( "sigs.k8s.io/cluster-api/exp/topology/scope" "sigs.k8s.io/cluster-api/feature" "sigs.k8s.io/cluster-api/internal/contract" - "sigs.k8s.io/cluster-api/internal/hooks" fakeruntimeclient "sigs.k8s.io/cluster-api/internal/runtime/client/fake" "sigs.k8s.io/cluster-api/internal/topology/clustershim" topologynames "sigs.k8s.io/cluster-api/internal/topology/names" @@ -1061,498 +1057,31 @@ func TestComputeControlPlaneVersion(t *testing.T) { } clusterv1beta1.SetAPIVersionGetter(apiVersionGetter) - t.Run("Compute control plane version under various circumstances", func(t *testing.T) { - utilfeature.SetFeatureGateDuringTest(t, feature.Gates, feature.RuntimeSDK, true) - - nonBlockingBeforeClusterUpgradeResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{ - CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ - CommonResponse: runtimehooksv1.CommonResponse{ - Status: runtimehooksv1.ResponseStatusSuccess, - }, - }, - } - - blockingBeforeClusterUpgradeResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{ - CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ - CommonResponse: runtimehooksv1.CommonResponse{ - Status: runtimehooksv1.ResponseStatusSuccess, - }, - RetryAfterSeconds: int32(10), - }, - } - - failureBeforeClusterUpgradeResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{ - CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ - CommonResponse: runtimehooksv1.CommonResponse{ - Status: runtimehooksv1.ResponseStatusFailure, - }, - }, - } - - catalog := runtimecatalog.New() - _ = runtimehooksv1.AddToCatalog(catalog) - - beforeClusterUpgradeGVH, err := catalog.GroupVersionHook(runtimehooksv1.BeforeClusterUpgrade) - if err != nil { - panic("unable to compute GVH") - } - - tests := []struct { - name string - hookResponse *runtimehooksv1.BeforeClusterUpgradeResponse - topologyVersion string - clusterModifier func(c *clusterv1.Cluster) - controlPlaneObj *unstructured.Unstructured - controlPlaneUpgradePlan []string - machineDeploymentsUpgradePlan []string - machinePoolsUpgradePlan []string - upgradingMachineDeployments []string - upgradingMachinePools []string - expectedVersion string - expectedIsPendingUpgrade bool - expectedIsStartingUpgrade bool - expectedIsWaitingForWorkersUpgrade bool - wantErr bool - }{ - { - name: "should return cluster.spec.topology.version if creating a new control plane", - topologyVersion: "v1.2.3", - controlPlaneObj: nil, - expectedVersion: "v1.2.3", - expectedIsPendingUpgrade: false, - expectedIsStartingUpgrade: false, - }, - { - // Control plane is not upgrading implies that controlplane.spec.version is equal to controlplane.status.version. - // Control plane is not scaling implies that controlplane.spec.replicas is equal to controlplane.status.replicas, - // Controlplane.status.updatedReplicas and controlplane.status.readyReplicas. - name: "should return cluster.spec.topology.version if the control plane is not upgrading and not scaling", - hookResponse: nonBlockingBeforeClusterUpgradeResponse, - topologyVersion: "v1.2.3", - controlPlaneObj: builder.ControlPlane("test1", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build(), - controlPlaneUpgradePlan: []string{"v1.2.3"}, - expectedVersion: "v1.2.3", - expectedIsPendingUpgrade: false, - expectedIsStartingUpgrade: true, - }, - { - name: "should return cluster.spec.topology.version if the control plane is already at the target version", - topologyVersion: "v1.2.3", - controlPlaneObj: builder.ControlPlane("test1", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.3", - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.3", - }). - Build(), - controlPlaneUpgradePlan: nil, - expectedVersion: "v1.2.3", - expectedIsPendingUpgrade: false, - expectedIsStartingUpgrade: false, - }, - { - // Control plane is considered upgrading if controlplane.spec.version is not equal to controlplane.status.version. - name: "should return controlplane.spec.version if the control plane is upgrading", - topologyVersion: "v1.2.3", - controlPlaneObj: builder.ControlPlane("test1", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.1", - }). - Build(), - controlPlaneUpgradePlan: []string{"v1.2.3"}, - expectedVersion: "v1.2.2", - expectedIsPendingUpgrade: true, - expectedIsStartingUpgrade: false, - }, - { - name: "should return cluster.spec.topology.version if the control plane is scaling", - hookResponse: nonBlockingBeforeClusterUpgradeResponse, - topologyVersion: "v1.2.3", - controlPlaneObj: builder.ControlPlane("test1", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(1), - "status.updatedReplicas": int64(1), - "status.readyReplicas": int64(1), - "status.unavailableReplicas": int64(0), - }). - Build(), - controlPlaneUpgradePlan: []string{"v1.2.3"}, - expectedVersion: "v1.2.3", - expectedIsPendingUpgrade: false, - expectedIsStartingUpgrade: true, - }, - { - name: "should return controlplane.spec.version if control plane is not upgrading and not scaling and one of the MachineDeployments and one of the MachinePools is upgrading", - topologyVersion: "v1.2.3", - controlPlaneObj: builder.ControlPlane("test1", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build(), - controlPlaneUpgradePlan: []string{"v1.2.3"}, - upgradingMachineDeployments: []string{"md1"}, - upgradingMachinePools: []string{"mp1"}, - expectedVersion: "v1.2.2", - expectedIsPendingUpgrade: true, - expectedIsStartingUpgrade: false, - }, - { - name: "should return cluster.spec.topology.version if control plane is not upgrading and not scaling and none of the MachineDeployments and MachinePools are upgrading - hook returns non blocking response", - hookResponse: nonBlockingBeforeClusterUpgradeResponse, - topologyVersion: "v1.2.3", - controlPlaneObj: builder.ControlPlane("test1", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build(), - controlPlaneUpgradePlan: []string{"v1.2.3"}, - upgradingMachineDeployments: []string{}, - upgradingMachinePools: []string{}, - expectedVersion: "v1.2.3", - expectedIsPendingUpgrade: false, - expectedIsStartingUpgrade: true, - }, - { - name: "should return an intermediate version when upgrading by more than 1 minor and control plane should perform the first step of the upgrade sequence", - hookResponse: nonBlockingBeforeClusterUpgradeResponse, - topologyVersion: "v1.5.3", - controlPlaneObj: builder.ControlPlane("test1", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build(), - controlPlaneUpgradePlan: []string{"v1.3.2", "v1.4.2", "v1.5.3"}, - upgradingMachineDeployments: []string{}, - upgradingMachinePools: []string{}, - expectedVersion: "v1.3.2", // first step of the upgrade plan - expectedIsPendingUpgrade: false, // there are still upgrade in the queue, but we are starting one (so not pending) - expectedIsStartingUpgrade: true, - }, - { - name: "should return cluster.spec.topology.version when performing a multi step upgrade and control plane is at the second last minor in the upgrade sequence", - hookResponse: nonBlockingBeforeClusterUpgradeResponse, - topologyVersion: "v1.5.3", - controlPlaneObj: builder.ControlPlane("test1", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.4.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.4.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build(), - controlPlaneUpgradePlan: []string{"v1.5.3"}, - upgradingMachineDeployments: []string{}, - upgradingMachinePools: []string{}, - expectedVersion: "v1.5.3", // last step of the upgrade plan - expectedIsPendingUpgrade: false, - expectedIsStartingUpgrade: true, - }, - { - name: "should remain on the current version when upgrading by more than 1 minor and MachineDeployments have to upgrade", - hookResponse: nonBlockingBeforeClusterUpgradeResponse, - topologyVersion: "v1.5.3", - controlPlaneObj: builder.ControlPlane("test1", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build(), - controlPlaneUpgradePlan: []string{"v1.3.2", "v1.4.2", "v1.5.3"}, - machineDeploymentsUpgradePlan: []string{"v1.2.2"}, - upgradingMachineDeployments: []string{}, - upgradingMachinePools: []string{}, - expectedVersion: "v1.2.2", - expectedIsPendingUpgrade: true, - expectedIsWaitingForWorkersUpgrade: true, - expectedIsStartingUpgrade: false, - }, - { - name: "should remain on the current version when upgrading by more than 1 minor and MachinePools have to upgrade", - hookResponse: nonBlockingBeforeClusterUpgradeResponse, - topologyVersion: "v1.5.3", - controlPlaneObj: builder.ControlPlane("test1", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build(), - controlPlaneUpgradePlan: []string{"v1.3.2", "v1.4.2", "v1.5.3"}, - machinePoolsUpgradePlan: []string{"v1.2.2"}, - upgradingMachineDeployments: []string{}, - upgradingMachinePools: []string{}, - expectedVersion: "v1.2.2", - expectedIsPendingUpgrade: true, - expectedIsWaitingForWorkersUpgrade: true, - expectedIsStartingUpgrade: false, - }, - { - name: "should return the controlplane.spec.version if a BeforeClusterUpgradeHook returns a blocking response", - hookResponse: blockingBeforeClusterUpgradeResponse, - topologyVersion: "v1.2.3", - controlPlaneObj: builder.ControlPlane("test1", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build(), - controlPlaneUpgradePlan: []string{"v1.2.3"}, - expectedVersion: "v1.2.2", - expectedIsPendingUpgrade: true, - expectedIsStartingUpgrade: false, - }, - { - name: "should fail if the BeforeClusterUpgrade hooks returns a failure response", - hookResponse: failureBeforeClusterUpgradeResponse, - topologyVersion: "v1.2.3", - controlPlaneObj: builder.ControlPlane("test1", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build(), - controlPlaneUpgradePlan: []string{"v1.2.3"}, - wantErr: true, - }, - { - name: "should return the controlplane.spec.version if a BeforeClusterUpgradeHook annotation is set", - hookResponse: nonBlockingBeforeClusterUpgradeResponse, - topologyVersion: "v1.2.3", - controlPlaneObj: builder.ControlPlane("test1", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build(), - clusterModifier: func(c *clusterv1.Cluster) { - c.Annotations = map[string]string{ - clusterv1.BeforeClusterUpgradeHookAnnotationPrefix + "/test": "true", - } - }, - controlPlaneUpgradePlan: []string{"v1.2.3"}, - expectedVersion: "v1.2.2", - expectedIsPendingUpgrade: true, - expectedIsStartingUpgrade: false, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - s := &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{Topology: clusterv1.Topology{ - Version: tt.topologyVersion, - ControlPlane: clusterv1.ControlPlaneTopology{ - Replicas: ptr.To[int32](2), - }, - }}, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - // Add managedFields and annotations that should be cleaned up before the Cluster is sent to the RuntimeExtension. - ManagedFields: []metav1.ManagedFieldsEntry{ - { - APIVersion: builder.InfrastructureGroupVersion.String(), - Manager: "manager", - Operation: "Apply", - Time: ptr.To(metav1.Now()), - FieldsType: "FieldsV1", - }, - }, - Annotations: map[string]string{ - "fizz": "buzz", - corev1.LastAppliedConfigAnnotation: "should be cleaned up", - conversion.DataAnnotation: "should be cleaned up", - }, - }, - // Add some more fields to check that conversion implemented when calling RuntimeExtension are properly handled. - Spec: clusterv1.ClusterSpec{ - InfrastructureRef: clusterv1.ContractVersionedObjectReference{ - APIGroup: "refAPIGroup1", - Kind: "refKind1", - Name: "refName1", - }}, - }, - ControlPlane: &scope.ControlPlaneState{Object: tt.controlPlaneObj}, - }, - UpgradeTracker: scope.NewUpgradeTracker(), - HookResponseTracker: scope.NewHookResponseTracker(), - } - if tt.clusterModifier != nil { - tt.clusterModifier(s.Current.Cluster) - } - if len(tt.controlPlaneUpgradePlan) > 0 { - s.UpgradeTracker.ControlPlane.UpgradePlan = tt.controlPlaneUpgradePlan - } - if len(tt.machineDeploymentsUpgradePlan) > 0 { - s.UpgradeTracker.MachineDeployments.UpgradePlan = tt.machineDeploymentsUpgradePlan - } - if len(tt.machinePoolsUpgradePlan) > 0 { - s.UpgradeTracker.MachinePools.UpgradePlan = tt.machinePoolsUpgradePlan - } - if len(tt.upgradingMachineDeployments) > 0 { - s.UpgradeTracker.MachineDeployments.MarkUpgrading(tt.upgradingMachineDeployments...) - } - if len(tt.upgradingMachinePools) > 0 { - s.UpgradeTracker.MachinePools.MarkUpgrading(tt.upgradingMachinePools...) - } - - runtimeClient := fakeruntimeclient.NewRuntimeClientBuilder(). - WithCatalog(catalog). - WithCallAllExtensionResponses(map[runtimecatalog.GroupVersionHook]runtimehooksv1.ResponseObject{ - beforeClusterUpgradeGVH: tt.hookResponse, - }). - WithCallAllExtensionValidations(validateClusterParameter(s.Current.Cluster)). - Build() - - fakeClient := fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(s.Current.Cluster).Build() - - r := &generator{ - Client: fakeClient, - RuntimeClient: runtimeClient, - } - version, err := r.computeControlPlaneVersion(ctx, s) - if tt.wantErr { - g.Expect(err).To(HaveOccurred()) - return - } - g.Expect(err).ToNot(HaveOccurred()) - - g.Expect(version).To(Equal(tt.expectedVersion)) - g.Expect(s.UpgradeTracker.ControlPlane.IsPendingUpgrade).To(Equal(tt.expectedIsPendingUpgrade)) - g.Expect(s.UpgradeTracker.ControlPlane.IsStartingUpgrade).To(Equal(tt.expectedIsStartingUpgrade)) - g.Expect(s.UpgradeTracker.ControlPlane.IsWaitingForWorkersUpgrade).To(Equal(tt.expectedIsWaitingForWorkersUpgrade)) - }) - } - }) -} - -func TestComputeControlPlaneVersion_callAfterControlPlaneUpgrade(t *testing.T) { utilfeature.SetFeatureGateDuringTest(t, feature.Gates, feature.RuntimeSDK, true) catalog := runtimecatalog.New() _ = runtimehooksv1.AddToCatalog(catalog) - afterControlPlaneUpgradeGVH, err := catalog.GroupVersionHook(runtimehooksv1.AfterControlPlaneUpgrade) - if err != nil { - panic(err) - } beforeClusterUpgradeGVH, err := catalog.GroupVersionHook(runtimehooksv1.BeforeClusterUpgrade) if err != nil { panic("unable to compute GVH") } - beforeClusterUpgradeNonBlockingResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{ + nonBlockingBeforeClusterUpgradeResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{ CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ CommonResponse: runtimehooksv1.CommonResponse{ Status: runtimehooksv1.ResponseStatusSuccess, }, }, } - - blockingResponse := &runtimehooksv1.AfterControlPlaneUpgradeResponse{ + blockingBeforeClusterUpgradeResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{ CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ - RetryAfterSeconds: int32(10), - CommonResponse: runtimehooksv1.CommonResponse{ - Status: runtimehooksv1.ResponseStatusSuccess, - }, - }, - } - nonBlockingResponse := &runtimehooksv1.AfterControlPlaneUpgradeResponse{ - CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ - RetryAfterSeconds: int32(0), CommonResponse: runtimehooksv1.CommonResponse{ Status: runtimehooksv1.ResponseStatusSuccess, }, + RetryAfterSeconds: int32(10), }, } - failureResponse := &runtimehooksv1.AfterControlPlaneUpgradeResponse{ + failureBeforeClusterUpgradeResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{ CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ CommonResponse: runtimehooksv1.CommonResponse{ Status: runtimehooksv1.ResponseStatusFailure, @@ -1560,815 +1089,402 @@ func TestComputeControlPlaneVersion_callAfterControlPlaneUpgrade(t *testing.T) { }, } - topologyVersion := "v1.2.3" - lowerVersion := "v1.2.2" - - controlPlaneStable := builder.ControlPlane("test-ns", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": topologyVersion, - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": topologyVersion, - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - }). - Build() - - controlPlaneUpgrading := builder.ControlPlane("test-ns", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": topologyVersion, - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": lowerVersion, - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - }). - Build() - - controlPlaneProvisioning := builder.ControlPlane("test-ns", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": lowerVersion, - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "", - }). - Build() - - newUpgradeTrackerWithUpgradePlan := func(upgradePlan []string) *scope.UpgradeTracker { - ut := scope.NewUpgradeTracker() - ut.ControlPlane.UpgradePlan = upgradePlan - return ut - } - tests := []struct { - name string - s *scope.Scope - hookResponse *runtimehooksv1.AfterControlPlaneUpgradeResponse - wantIntentToCall bool - wantHookToBeCalled bool - wantHookToBlock bool - wantErr bool + name string + beforeClusterUpgradeResponse *runtimehooksv1.BeforeClusterUpgradeResponse + topologyVersion string + clusterModifier func(c *clusterv1.Cluster) + controlPlaneObj *unstructured.Unstructured + controlPlaneUpgradePlan []string + machineDeploymentsUpgradePlan []string + machinePoolsUpgradePlan []string + upgradingMachineDeployments []string + upgradingMachinePools []string + expectedVersion string + expectedIsPendingUpgrade bool + expectedIsStartingUpgrade bool + expectedIsWaitingForWorkersUpgrade bool + wantErr bool }{ { - name: "should not call hook if it is not marked", - s: &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{ - Topology: clusterv1.Topology{ - Version: topologyVersion, - ControlPlane: clusterv1.ControlPlaneTopology{}, - }, - }, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - }, - Spec: clusterv1.ClusterSpec{}, - }, - ControlPlane: &scope.ControlPlaneState{ - Object: controlPlaneStable, - }, - }, - UpgradeTracker: scope.NewUpgradeTracker(), // already at topology version, upgrade plan is empty. - HookResponseTracker: scope.NewHookResponseTracker(), - }, - wantIntentToCall: false, // preserve existing value (not set) - wantHookToBeCalled: false, - wantErr: false, + name: "should return cluster.spec.topology.version if creating a new control plane", + topologyVersion: "v1.2.3", + controlPlaneObj: nil, + expectedVersion: "v1.2.3", + expectedIsPendingUpgrade: false, + expectedIsStartingUpgrade: false, }, { - name: "should not call hook if the control plane is provisioning - there is intent to call hook", - s: &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{ - Topology: clusterv1.Topology{ - Version: topologyVersion, - ControlPlane: clusterv1.ControlPlaneTopology{}, - }, - }, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - Annotations: map[string]string{ - runtimev1.PendingHooksAnnotation: "AfterControlPlaneUpgrade", - }, - }, - Spec: clusterv1.ClusterSpec{}, - }, - ControlPlane: &scope.ControlPlaneState{ - Object: controlPlaneProvisioning, - }, - }, - UpgradeTracker: newUpgradeTrackerWithUpgradePlan([]string{topologyVersion}), - HookResponseTracker: scope.NewHookResponseTracker(), - }, - wantIntentToCall: true, // preserve existing value (set) - wantHookToBeCalled: false, - wantErr: false, + name: "should return cluster.spec.topology.version if the control plane is already at the target version", + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.3", + }). + Build(), + controlPlaneUpgradePlan: nil, + expectedVersion: "v1.2.3", + expectedIsPendingUpgrade: false, + expectedIsStartingUpgrade: false, }, { - name: "should not call hook if the control plane is upgrading - there is intent to call hook", - s: &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{ - Topology: clusterv1.Topology{ - Version: topologyVersion, - ControlPlane: clusterv1.ControlPlaneTopology{}, - }, - }, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - Annotations: map[string]string{ - runtimev1.PendingHooksAnnotation: "AfterControlPlaneUpgrade", - }, - }, - Spec: clusterv1.ClusterSpec{}, - }, - ControlPlane: &scope.ControlPlaneState{ - Object: controlPlaneUpgrading, - }, - }, - UpgradeTracker: scope.NewUpgradeTracker(), // already at topology version, upgrade plan is empty. - HookResponseTracker: scope.NewHookResponseTracker(), - }, - wantIntentToCall: true, // preserve existing value (set) - wantHookToBeCalled: false, - wantErr: false, + // Control plane is considered upgrading if controlplane.spec.version is not equal to controlplane.status.version. + name: "should return controlplane.spec.version if the control plane is upgrading", + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.1", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.2.3"}, + expectedVersion: "v1.2.2", + expectedIsPendingUpgrade: true, + expectedIsStartingUpgrade: false, }, { - name: "should call hook if the control plane is at desired version - non blocking response should remove hook from pending hooks list and allow MD upgrades", - s: &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{ - Topology: clusterv1.Topology{ - Version: topologyVersion, - ControlPlane: clusterv1.ControlPlaneTopology{}, - }, - }, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - Annotations: map[string]string{ - runtimev1.PendingHooksAnnotation: "AfterControlPlaneUpgrade", - }, - }, - Spec: clusterv1.ClusterSpec{}, - }, - ControlPlane: &scope.ControlPlaneState{ - Object: controlPlaneStable, - }, - }, - UpgradeTracker: scope.NewUpgradeTracker(), // already at topology version, upgrade plan is empty. - HookResponseTracker: scope.NewHookResponseTracker(), - }, - hookResponse: nonBlockingResponse, - wantIntentToCall: false, // remove the intent to call the hook (hook called, we are at target state) - wantHookToBeCalled: true, - wantHookToBlock: false, - wantErr: false, + name: "should return controlplane.spec.version if control plane is not upgrading and not scaling and one of the MachineDeployments and one of the MachinePools is upgrading", + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.2.3"}, + upgradingMachineDeployments: []string{"md1"}, + upgradingMachinePools: []string{"mp1"}, + expectedVersion: "v1.2.2", + expectedIsPendingUpgrade: true, + expectedIsStartingUpgrade: false, }, { - name: "should call hook if the control plane is at desired version - blocking response should leave the hook in pending hooks list and block MD upgrades", - s: &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{ - Topology: clusterv1.Topology{ - Version: topologyVersion, - ControlPlane: clusterv1.ControlPlaneTopology{}, - }, - }, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - Annotations: map[string]string{ - runtimev1.PendingHooksAnnotation: "AfterControlPlaneUpgrade", - }, - }, - Spec: clusterv1.ClusterSpec{}, - }, - ControlPlane: &scope.ControlPlaneState{ - Object: controlPlaneStable, - }, - }, - UpgradeTracker: scope.NewUpgradeTracker(), // already at topology version, upgrade plan is empty. - HookResponseTracker: scope.NewHookResponseTracker(), - }, - hookResponse: blockingResponse, - wantIntentToCall: true, // preserve existing value (set) - wantHookToBeCalled: true, - wantHookToBlock: true, - wantErr: false, + name: "should return cluster.spec.topology.version if control plane is not upgrading and not scaling and none of the MachineDeployments and MachinePools are upgrading - BeforeClusterUpgrade hook returns non blocking response", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.2.3"}, + upgradingMachineDeployments: []string{}, + upgradingMachinePools: []string{}, + expectedVersion: "v1.2.3", + expectedIsPendingUpgrade: false, + expectedIsStartingUpgrade: true, }, { - name: "should call hook if the control plane is at desired version - failure response should leave the hook in pending hooks list", - s: &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{ - Topology: clusterv1.Topology{ - Version: topologyVersion, - ControlPlane: clusterv1.ControlPlaneTopology{}, - }, - }, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - Annotations: map[string]string{ - runtimev1.PendingHooksAnnotation: "AfterControlPlaneUpgrade", - }, - }, - Spec: clusterv1.ClusterSpec{}, - }, - ControlPlane: &scope.ControlPlaneState{ - Object: controlPlaneStable, - }, - }, - UpgradeTracker: scope.NewUpgradeTracker(), // already at topology version, upgrade plan is empty. - HookResponseTracker: scope.NewHookResponseTracker(), + name: "should return cluster.spec.topology.version if the control plane is not upgrading or scaling and none of the MachineDeployments and MachinePools are upgrading - BeforeClusterUpgrade, BeforeControlPlaneUpgrade, BeforeWorkersUpgrade and AfterWorkersUpgrade hooks returns non blocking response", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(1), + "status.updatedReplicas": int64(1), + "status.readyReplicas": int64(1), + "status.unavailableReplicas": int64(0), + }). + Build(), + clusterModifier: func(c *clusterv1.Cluster) { + c.Annotations = map[string]string{ + runtimev1.PendingHooksAnnotation: "AfterWorkersUpgrade", + } }, - hookResponse: failureResponse, - wantIntentToCall: true, // preserve existing value (set) - wantHookToBeCalled: true, - wantErr: true, + controlPlaneUpgradePlan: []string{"v1.2.3"}, + expectedVersion: "v1.2.3", + expectedIsPendingUpgrade: false, + expectedIsStartingUpgrade: true, }, { - name: "should call hook if the control plane is at the first step of a multistep upgrade - intent to call for next minor should be tracked", - s: &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{ - Topology: clusterv1.Topology{ - Version: "v1.5.3", - ControlPlane: clusterv1.ControlPlaneTopology{}, - }, - }, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - Annotations: map[string]string{ - runtimev1.PendingHooksAnnotation: "AfterControlPlaneUpgrade", - }, - }, - Spec: clusterv1.ClusterSpec{}, - }, - ControlPlane: &scope.ControlPlaneState{ - Object: builder.ControlPlane("test-ns", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.3.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.3.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - }). - Build(), - }, - }, - UpgradeTracker: newUpgradeTrackerWithUpgradePlan([]string{"v1.4.2", "v1.5.3"}), - HookResponseTracker: scope.NewHookResponseTracker(), - }, - hookResponse: nonBlockingResponse, - wantIntentToCall: true, // new intent to call the hook for the next minor - wantHookToBeCalled: true, // the hook has been called for the current minor - wantHookToBlock: false, - - wantErr: false, + name: "should return an intermediate version when upgrading by more than 1 minor and control plane should perform the first step of the upgrade sequence", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + topologyVersion: "v1.5.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.3.2", "v1.4.2", "v1.5.3"}, + upgradingMachineDeployments: []string{}, + upgradingMachinePools: []string{}, + expectedVersion: "v1.3.2", // first step of the upgrade plan + expectedIsPendingUpgrade: false, // there are still upgrade in the queue, but we are starting one (so not pending) + expectedIsStartingUpgrade: true, }, { - name: "should call hook if the control plane is at the last step of a multistep upgrade", - s: &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{ - Topology: clusterv1.Topology{ - Version: "v1.5.3", - ControlPlane: clusterv1.ControlPlaneTopology{}, + name: "should return cluster.spec.topology.version when performing a multi step upgrade and control plane is at the second last minor in the upgrade sequence", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + topologyVersion: "v1.5.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.4.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.4.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.5.3"}, + upgradingMachineDeployments: []string{}, + upgradingMachinePools: []string{}, + expectedVersion: "v1.5.3", // last step of the upgrade plan + expectedIsPendingUpgrade: false, + expectedIsStartingUpgrade: true, + }, + { + name: "should remain on the current version when upgrading by more than 1 minor and MachineDeployments have to upgrade", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + topologyVersion: "v1.5.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.3.2", "v1.4.2", "v1.5.3"}, + machineDeploymentsUpgradePlan: []string{"v1.2.2"}, + upgradingMachineDeployments: []string{}, + upgradingMachinePools: []string{}, + expectedVersion: "v1.2.2", + expectedIsPendingUpgrade: true, + expectedIsWaitingForWorkersUpgrade: true, + expectedIsStartingUpgrade: false, + }, + { + name: "should remain on the current version when upgrading by more than 1 minor and MachinePools have to upgrade", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + topologyVersion: "v1.5.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.3.2", "v1.4.2", "v1.5.3"}, + machinePoolsUpgradePlan: []string{"v1.2.2"}, + upgradingMachineDeployments: []string{}, + upgradingMachinePools: []string{}, + expectedVersion: "v1.2.2", + expectedIsPendingUpgrade: true, + expectedIsWaitingForWorkersUpgrade: true, + expectedIsStartingUpgrade: false, + }, + { + name: "should return the controlplane.spec.version if a BeforeClusterUpgradeHook returns a blocking response", + beforeClusterUpgradeResponse: blockingBeforeClusterUpgradeResponse, + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.2.3"}, + expectedVersion: "v1.2.2", + expectedIsPendingUpgrade: true, + expectedIsStartingUpgrade: false, + }, + { + name: "should fail if the BeforeClusterUpgrade hooks returns a failure response", + beforeClusterUpgradeResponse: failureBeforeClusterUpgradeResponse, + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.2.3"}, + wantErr: true, + }, + { + name: "should return the controlplane.spec.version if a BeforeClusterUpgradeHook annotation is set", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + clusterModifier: func(c *clusterv1.Cluster) { + c.Annotations = map[string]string{ + clusterv1.BeforeClusterUpgradeHookAnnotationPrefix + "/test": "true", + } + }, + controlPlaneUpgradePlan: []string{"v1.2.3"}, + expectedVersion: "v1.2.2", + expectedIsPendingUpgrade: true, + expectedIsStartingUpgrade: false, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + s := &scope.Scope{ + Blueprint: &scope.ClusterBlueprint{Topology: clusterv1.Topology{ + Version: tt.topologyVersion, + ControlPlane: clusterv1.ControlPlaneTopology{ + Replicas: ptr.To[int32](2), }, - }, + }}, Current: &scope.ClusterState{ Cluster: &clusterv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "test-cluster", Namespace: "test-ns", + // Add managedFields and annotations that should be cleaned up before the Cluster is sent to the RuntimeExtension. + ManagedFields: []metav1.ManagedFieldsEntry{ + { + APIVersion: builder.InfrastructureGroupVersion.String(), + Manager: "manager", + Operation: "Apply", + Time: ptr.To(metav1.Now()), + FieldsType: "FieldsV1", + }, + }, Annotations: map[string]string{ - runtimev1.PendingHooksAnnotation: "AfterControlPlaneUpgrade", + "fizz": "buzz", + corev1.LastAppliedConfigAnnotation: "should be cleaned up", + conversion.DataAnnotation: "should be cleaned up", }, }, - Spec: clusterv1.ClusterSpec{}, - }, - ControlPlane: &scope.ControlPlaneState{ - Object: builder.ControlPlane("test-ns", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.5.3", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.5.3", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - }). - Build(), + // Add some more fields to check that conversion implemented when calling RuntimeExtension are properly handled. + Spec: clusterv1.ClusterSpec{ + InfrastructureRef: clusterv1.ContractVersionedObjectReference{ + APIGroup: "refAPIGroup1", + Kind: "refKind1", + Name: "refName1", + }}, }, + ControlPlane: &scope.ControlPlaneState{Object: tt.controlPlaneObj}, }, - UpgradeTracker: scope.NewUpgradeTracker(), // already at topology version, upgrade plan is empty. + UpgradeTracker: scope.NewUpgradeTracker(), HookResponseTracker: scope.NewHookResponseTracker(), - }, - hookResponse: nonBlockingResponse, - wantIntentToCall: false, // remove the intent to call the hook (hook called, we are at target state) - wantHookToBeCalled: true, // the hook has been called for the current minor - wantHookToBlock: false, - - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - // Add managedFields and annotations that should be cleaned up before the Cluster is sent to the RuntimeExtension. - tt.s.Current.Cluster.SetManagedFields([]metav1.ManagedFieldsEntry{ - { - APIVersion: builder.InfrastructureGroupVersion.String(), - Manager: "manager", - Operation: "Apply", - Time: ptr.To(metav1.Now()), - FieldsType: "FieldsV1", - }, - }) - if tt.s.Current.Cluster.Annotations == nil { - tt.s.Current.Cluster.Annotations = map[string]string{} } - tt.s.Current.Cluster.Annotations[corev1.LastAppliedConfigAnnotation] = "should be cleaned up" - tt.s.Current.Cluster.Annotations[conversion.DataAnnotation] = "should be cleaned up" + if tt.clusterModifier != nil { + tt.clusterModifier(s.Current.Cluster) + } + if len(tt.controlPlaneUpgradePlan) > 0 { + s.UpgradeTracker.ControlPlane.UpgradePlan = tt.controlPlaneUpgradePlan + } + if len(tt.machineDeploymentsUpgradePlan) > 0 { + s.UpgradeTracker.MachineDeployments.UpgradePlan = tt.machineDeploymentsUpgradePlan + } + if len(tt.machinePoolsUpgradePlan) > 0 { + s.UpgradeTracker.MachinePools.UpgradePlan = tt.machinePoolsUpgradePlan + } + if len(tt.upgradingMachineDeployments) > 0 { + s.UpgradeTracker.MachineDeployments.MarkUpgrading(tt.upgradingMachineDeployments...) + } + if len(tt.upgradingMachinePools) > 0 { + s.UpgradeTracker.MachinePools.MarkUpgrading(tt.upgradingMachinePools...) + } - fakeRuntimeClient := fakeruntimeclient.NewRuntimeClientBuilder(). - WithCallAllExtensionResponses(map[runtimecatalog.GroupVersionHook]runtimehooksv1.ResponseObject{ - afterControlPlaneUpgradeGVH: tt.hookResponse, - beforeClusterUpgradeGVH: beforeClusterUpgradeNonBlockingResponse, - }).WithCallAllExtensionValidations(validateClusterParameter(tt.s.Current.Cluster)). + runtimeClient := fakeruntimeclient.NewRuntimeClientBuilder(). WithCatalog(catalog). + WithCallAllExtensionResponses(map[runtimecatalog.GroupVersionHook]runtimehooksv1.ResponseObject{ + beforeClusterUpgradeGVH: tt.beforeClusterUpgradeResponse, + }). + WithCallAllExtensionValidations(validateClusterParameter(s.Current.Cluster)). Build() - fakeClient := fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(tt.s.Current.Cluster).Build() + fakeClient := fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(s.Current.Cluster).Build() r := &generator{ Client: fakeClient, - RuntimeClient: fakeRuntimeClient, + RuntimeClient: runtimeClient, } - - _, err := r.computeControlPlaneVersion(ctx, tt.s) + version, err := r.computeControlPlaneVersion(ctx, s) if tt.wantErr { g.Expect(err).To(HaveOccurred()) - } else { - g.Expect(err).ToNot(HaveOccurred()) - } - - if tt.wantHookToBeCalled { - g.Expect(fakeRuntimeClient.CallAllCount(runtimehooksv1.AfterControlPlaneUpgrade)).To(Equal(1), "Expected hook to be called once") - } else { - g.Expect(fakeRuntimeClient.CallAllCount(runtimehooksv1.AfterControlPlaneUpgrade)).To(Equal(0), "Did not expect hook to be called") + return } + g.Expect(err).ToNot(HaveOccurred()) - g.Expect(hooks.IsPending(runtimehooksv1.AfterControlPlaneUpgrade, tt.s.Current.Cluster)).To(Equal(tt.wantIntentToCall)) - - if tt.wantHookToBeCalled && !tt.wantErr { - g.Expect(tt.s.HookResponseTracker.IsBlocking(runtimehooksv1.AfterControlPlaneUpgrade)).To(Equal(tt.wantHookToBlock)) - } + g.Expect(version).To(Equal(tt.expectedVersion)) + g.Expect(s.UpgradeTracker.ControlPlane.IsPendingUpgrade).To(Equal(tt.expectedIsPendingUpgrade)) + g.Expect(s.UpgradeTracker.ControlPlane.IsStartingUpgrade).To(Equal(tt.expectedIsStartingUpgrade)) + g.Expect(s.UpgradeTracker.ControlPlane.IsWaitingForWorkersUpgrade).To(Equal(tt.expectedIsWaitingForWorkersUpgrade)) }) } } -func TestComputeControlPlaneVersion_callBeforeClusterUpgrade_trackIntentOfCallingAfterClusterUpgrade(t *testing.T) { - utilfeature.SetFeatureGateDuringTest(t, feature.Gates, feature.RuntimeSDK, true) - - catalog := runtimecatalog.New() - _ = runtimehooksv1.AddToCatalog(catalog) - beforeClusterUpgradeGVH, err := catalog.GroupVersionHook(runtimehooksv1.BeforeClusterUpgrade) - if err != nil { - panic("unable to compute GVH") - } - blockingResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{ - CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ - RetryAfterSeconds: int32(10), - CommonResponse: runtimehooksv1.CommonResponse{ - Status: runtimehooksv1.ResponseStatusSuccess, - }, - }, - } - nonBlockingResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{ - CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ - RetryAfterSeconds: int32(0), - CommonResponse: runtimehooksv1.CommonResponse{ - Status: runtimehooksv1.ResponseStatusSuccess, - }, - }, - } - failureResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{ - CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ - CommonResponse: runtimehooksv1.CommonResponse{ - Status: runtimehooksv1.ResponseStatusFailure, - }, - }, - } - - t.Run("Call BeforeClusterUpgrade hook when doing simple upgrades", func(t *testing.T) { - controlPlaneStable := builder.ControlPlane("test-ns", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build() - - s := &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{Topology: clusterv1.Topology{ - Version: "v1.2.3", - ControlPlane: clusterv1.ControlPlaneTopology{ - Replicas: ptr.To[int32](2), - }, - }}, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - }, - }, - ControlPlane: &scope.ControlPlaneState{Object: controlPlaneStable}, - }, - UpgradeTracker: scope.NewUpgradeTracker(), - HookResponseTracker: scope.NewHookResponseTracker(), - } - s.UpgradeTracker.ControlPlane.UpgradePlan = []string{"v1.2.3"} - - fakeClient := fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(s.Current.Cluster).Build() - - runtimeClient := fakeruntimeclient.NewRuntimeClientBuilder(). - WithCatalog(catalog). - WithCallAllExtensionResponses(map[runtimecatalog.GroupVersionHook]runtimehooksv1.ResponseObject{ - beforeClusterUpgradeGVH: nonBlockingResponse, - }). - Build() - - r := &generator{ - Client: fakeClient, - RuntimeClient: runtimeClient, - } - - desiredVersion, err := r.computeControlPlaneVersion(ctx, s) - g := NewWithT(t) - g.Expect(err).ToNot(HaveOccurred()) - - // Before Cluster upgrade hook must have been be called. - g.Expect(runtimeClient.CallAllCount(runtimehooksv1.BeforeClusterUpgrade)).To(Equal(1)) - - // When successfully picking up the new version the intent to call AfterControlPlaneUpgrade and AfterClusterUpgrade hooks should be registered. - g.Expect(desiredVersion).To(Equal("v1.2.3")) - g.Expect(hooks.IsPending(runtimehooksv1.AfterControlPlaneUpgrade, s.Current.Cluster)).To(BeTrue()) - g.Expect(hooks.IsPending(runtimehooksv1.AfterClusterUpgrade, s.Current.Cluster)).To(BeTrue()) - }) - - t.Run("Call BeforeClusterUpgrade hook when doing simple upgrades - failure response should block picking up a new version", func(t *testing.T) { - controlPlaneStable := builder.ControlPlane("test-ns", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build() - - s := &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{Topology: clusterv1.Topology{ - Version: "v1.2.3", - ControlPlane: clusterv1.ControlPlaneTopology{ - Replicas: ptr.To[int32](2), - }, - }}, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - }, - }, - ControlPlane: &scope.ControlPlaneState{Object: controlPlaneStable}, - }, - UpgradeTracker: scope.NewUpgradeTracker(), - HookResponseTracker: scope.NewHookResponseTracker(), - } - s.UpgradeTracker.ControlPlane.UpgradePlan = []string{"v1.2.3"} - - fakeClient := fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(s.Current.Cluster).Build() - - runtimeClient := fakeruntimeclient.NewRuntimeClientBuilder(). - WithCatalog(catalog). - WithCallAllExtensionResponses(map[runtimecatalog.GroupVersionHook]runtimehooksv1.ResponseObject{ - beforeClusterUpgradeGVH: failureResponse, - }). - Build() - - r := &generator{ - Client: fakeClient, - RuntimeClient: runtimeClient, - } - - desiredVersion, err := r.computeControlPlaneVersion(ctx, s) - g := NewWithT(t) - g.Expect(desiredVersion).To(Equal("")) - g.Expect(err).To(HaveOccurred()) - - // Before Cluster upgrade hook must have been be called. - g.Expect(runtimeClient.CallAllCount(runtimehooksv1.BeforeClusterUpgrade)).To(Equal(1)) - - // After a failure, intent to call AfterControlPlaneUpgrade and AfterClusterUpgrade hooks should not be registered. - g.Expect(hooks.IsPending(runtimehooksv1.AfterControlPlaneUpgrade, s.Current.Cluster)).To(BeFalse()) - g.Expect(hooks.IsPending(runtimehooksv1.AfterClusterUpgrade, s.Current.Cluster)).To(BeFalse()) - }) - - t.Run("Call BeforeClusterUpgrade hook when doing simple upgrades - blocking response should block picking up a new version", func(t *testing.T) { - controlPlaneStable := builder.ControlPlane("test-ns", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build() - - s := &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{Topology: clusterv1.Topology{ - Version: "v1.2.3", - ControlPlane: clusterv1.ControlPlaneTopology{ - Replicas: ptr.To[int32](2), - }, - }}, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - }, - }, - ControlPlane: &scope.ControlPlaneState{Object: controlPlaneStable}, - }, - UpgradeTracker: scope.NewUpgradeTracker(), - HookResponseTracker: scope.NewHookResponseTracker(), - } - s.UpgradeTracker.ControlPlane.UpgradePlan = []string{"v1.2.3"} - - fakeClient := fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(s.Current.Cluster).Build() - - runtimeClient := fakeruntimeclient.NewRuntimeClientBuilder(). - WithCatalog(catalog). - WithCallAllExtensionResponses(map[runtimecatalog.GroupVersionHook]runtimehooksv1.ResponseObject{ - beforeClusterUpgradeGVH: blockingResponse, - }). - Build() - - r := &generator{ - Client: fakeClient, - RuntimeClient: runtimeClient, - } - - desiredVersion, err := r.computeControlPlaneVersion(ctx, s) - g := NewWithT(t) - g.Expect(err).ToNot(HaveOccurred()) - - // Before Cluster upgrade hook must have been be called. - g.Expect(runtimeClient.CallAllCount(runtimehooksv1.BeforeClusterUpgrade)).To(Equal(1)) - - // After a blocking response, current version should not be picked up, intent to call AfterControlPlaneUpgrade and AfterClusterUpgrade hooks should not be registered. - g.Expect(desiredVersion).To(Equal("v1.2.2")) - g.Expect(hooks.IsPending(runtimehooksv1.AfterControlPlaneUpgrade, s.Current.Cluster)).To(BeFalse()) - g.Expect(hooks.IsPending(runtimehooksv1.AfterClusterUpgrade, s.Current.Cluster)).To(BeFalse()) - }) - - t.Run("Call BeforeClusterUpgrade hook when doing the first step of a multistep cluster upgrade", func(t *testing.T) { - controlPlaneStable := builder.ControlPlane("test-ns", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.2.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.2.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build() - - s := &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{Topology: clusterv1.Topology{ - Version: "v1.5.3", // more than one minor after current - ControlPlane: clusterv1.ControlPlaneTopology{ - Replicas: ptr.To[int32](2), - }, - }}, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - }, - }, - ControlPlane: &scope.ControlPlaneState{Object: controlPlaneStable}, - }, - UpgradeTracker: scope.NewUpgradeTracker(), - HookResponseTracker: scope.NewHookResponseTracker(), - } - s.UpgradeTracker.ControlPlane.UpgradePlan = []string{"v1.3.2", "v1.4.2", "v1.5.3"} - - fakeClient := fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(s.Current.Cluster).Build() - - runtimeClient := fakeruntimeclient.NewRuntimeClientBuilder(). - WithCatalog(catalog). - WithCallAllExtensionResponses(map[runtimecatalog.GroupVersionHook]runtimehooksv1.ResponseObject{ - beforeClusterUpgradeGVH: nonBlockingResponse, - }). - Build() - - r := &generator{ - Client: fakeClient, - RuntimeClient: runtimeClient, - } - - desiredVersion, err := r.computeControlPlaneVersion(ctx, s) - g := NewWithT(t) - g.Expect(err).ToNot(HaveOccurred()) - - // Before Cluster upgrade hook must have been be called. - g.Expect(runtimeClient.CallAllCount(runtimehooksv1.BeforeClusterUpgrade)).To(Equal(1)) - - // When successfully picking up the new version the intent to call AfterControlPlaneUpgrade and AfterClusterUpgrade hooks should be registered. - g.Expect(desiredVersion).To(Equal("v1.3.2")) - g.Expect(hooks.IsPending(runtimehooksv1.AfterControlPlaneUpgrade, s.Current.Cluster)).To(BeTrue()) - g.Expect(hooks.IsPending(runtimehooksv1.AfterClusterUpgrade, s.Current.Cluster)).To(BeTrue()) - }) - - t.Run("Don't call BeforeClusterUpgrade hook after the first step of a multistep upgrade", func(t *testing.T) { - controlPlaneStable := builder.ControlPlane("test-ns", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.3.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.3.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build() - - s := &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{Topology: clusterv1.Topology{ - Version: "v1.5.3", // more than one minor after current - ControlPlane: clusterv1.ControlPlaneTopology{ - Replicas: ptr.To[int32](2), - }, - }}, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - Annotations: map[string]string{ - runtimev1.PendingHooksAnnotation: runtimecatalog.HookName(runtimehooksv1.AfterClusterUpgrade), // This signal that the upgrade is already in progress. - }, - }, - }, - ControlPlane: &scope.ControlPlaneState{Object: controlPlaneStable}, - }, - UpgradeTracker: scope.NewUpgradeTracker(), - HookResponseTracker: scope.NewHookResponseTracker(), - } - s.UpgradeTracker.ControlPlane.UpgradePlan = []string{"v1.4.2", "v1.5.3"} - - fakeClient := fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(s.Current.Cluster).Build() - - runtimeClient := fakeruntimeclient.NewRuntimeClientBuilder(). - WithCatalog(catalog). - WithCallAllExtensionResponses(map[runtimecatalog.GroupVersionHook]runtimehooksv1.ResponseObject{ - beforeClusterUpgradeGVH: nonBlockingResponse, - }). - Build() - - r := &generator{ - Client: fakeClient, - RuntimeClient: runtimeClient, - } - - desiredVersion, err := r.computeControlPlaneVersion(ctx, s) - g := NewWithT(t) - g.Expect(err).ToNot(HaveOccurred()) - - // Before Cluster upgrade hook must not have been be called. - g.Expect(runtimeClient.CallAllCount(runtimehooksv1.BeforeClusterUpgrade)).To(Equal(0)) - - // When successfully picking up the new version the intent to call AfterControlPlaneUpgrade and AfterClusterUpgrade hooks should be registered. - g.Expect(desiredVersion).To(Equal("v1.4.2")) - g.Expect(hooks.IsPending(runtimehooksv1.AfterControlPlaneUpgrade, s.Current.Cluster)).To(BeTrue()) - g.Expect(hooks.IsPending(runtimehooksv1.AfterClusterUpgrade, s.Current.Cluster)).To(BeTrue()) - }) - - t.Run("Don't call BeforeClusterUpgrade hook when at the last step of a multi cluster upgrade", func(t *testing.T) { - controlPlaneStable := builder.ControlPlane("test-ns", "cp1"). - WithSpecFields(map[string]interface{}{ - "spec.version": "v1.4.2", - "spec.replicas": int64(2), - }). - WithStatusFields(map[string]interface{}{ - "status.version": "v1.4.2", - "status.replicas": int64(2), - "status.updatedReplicas": int64(2), - "status.readyReplicas": int64(2), - "status.unavailableReplicas": int64(0), - }). - Build() - - s := &scope.Scope{ - Blueprint: &scope.ClusterBlueprint{Topology: clusterv1.Topology{ - Version: "v1.5.3", // one minor after current - ControlPlane: clusterv1.ControlPlaneTopology{ - Replicas: ptr.To[int32](2), - }, - }}, - Current: &scope.ClusterState{ - Cluster: &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "test-ns", - Annotations: map[string]string{ - runtimev1.PendingHooksAnnotation: runtimecatalog.HookName(runtimehooksv1.AfterClusterUpgrade), // This signal that the upgrade is already in progress. - }, - }, - }, - ControlPlane: &scope.ControlPlaneState{Object: controlPlaneStable}, - }, - UpgradeTracker: scope.NewUpgradeTracker(), - HookResponseTracker: scope.NewHookResponseTracker(), - } - s.UpgradeTracker.ControlPlane.UpgradePlan = []string{"v1.5.3"} - - fakeClient := fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(s.Current.Cluster).Build() - - runtimeClient := fakeruntimeclient.NewRuntimeClientBuilder(). - WithCatalog(catalog). - WithCallAllExtensionResponses(map[runtimecatalog.GroupVersionHook]runtimehooksv1.ResponseObject{ - beforeClusterUpgradeGVH: nonBlockingResponse, - }). - Build() - - r := &generator{ - Client: fakeClient, - RuntimeClient: runtimeClient, - } - - desiredVersion, err := r.computeControlPlaneVersion(ctx, s) - g := NewWithT(t) - g.Expect(err).ToNot(HaveOccurred()) - - // Before Cluster upgrade hook must not have been be called. - g.Expect(runtimeClient.CallAllCount(runtimehooksv1.BeforeClusterUpgrade)).To(Equal(0)) - - // When successfully picking up the new version the intent to call AfterControlPlaneUpgrade and AfterClusterUpgrade hooks should be registered. - g.Expect(desiredVersion).To(Equal("v1.5.3")) - g.Expect(hooks.IsPending(runtimehooksv1.AfterControlPlaneUpgrade, s.Current.Cluster)).To(BeTrue()) - g.Expect(hooks.IsPending(runtimehooksv1.AfterClusterUpgrade, s.Current.Cluster)).To(BeTrue()) - }) -} - func TestComputeCluster(t *testing.T) { g := NewWithT(t) @@ -4546,54 +3662,3 @@ func TestGenerate(t *testing.T) { g.Expect(s.UpgradeTracker.MachinePools.UpgradingNames()).To(ConsistOf(mp.Name)) }) } - -func validateClusterParameter(originalCluster *clusterv1.Cluster) func(req runtimehooksv1.RequestObject) error { - // return a func that allows to check if expected transformations are applied to the Cluster parameter which is - // included in the payload for lifecycle hooks calls. - return func(req runtimehooksv1.RequestObject) error { - var cluster clusterv1beta1.Cluster - switch req := req.(type) { - case *runtimehooksv1.BeforeClusterUpgradeRequest: - cluster = req.Cluster - case *runtimehooksv1.AfterControlPlaneUpgradeRequest: - cluster = req.Cluster - default: - return fmt.Errorf("unhandled request type %T", req) - } - - // check if managed fields and well know annotations have been removed from the Cluster parameter included in the payload lifecycle hooks calls. - if cluster.GetManagedFields() != nil { - return errors.New("managedFields should have been cleaned up") - } - if _, ok := cluster.Annotations[corev1.LastAppliedConfigAnnotation]; ok { - return errors.New("last-applied-configuration annotation should have been cleaned up") - } - if _, ok := cluster.Annotations[conversion.DataAnnotation]; ok { - return errors.New("conversion annotation should have been cleaned up") - } - - // check the Cluster parameter included in the payload lifecycle hooks calls has been properly converted from v1beta2 to v1beta1. - // Note: to perform this check we convert the parameter back to v1beta2 and compare with the original cluster +/- expected transformations. - v1beta2Cluster := &clusterv1.Cluster{} - if err := cluster.ConvertTo(v1beta2Cluster); err != nil { - return err - } - - originalClusterCopy := originalCluster.DeepCopy() - originalClusterCopy.SetManagedFields(nil) - if originalClusterCopy.Annotations != nil { - annotations := maps.Clone(cluster.Annotations) - delete(annotations, corev1.LastAppliedConfigAnnotation) - delete(annotations, conversion.DataAnnotation) - originalClusterCopy.Annotations = annotations - } - - // drop conditions, it is not possible to round trip without the data annotation. - originalClusterCopy.Status.Conditions = nil - - if !apiequality.Semantic.DeepEqual(originalClusterCopy, v1beta2Cluster) { - return errors.Errorf("call to extension is not passing the expected cluster object: %s", cmp.Diff(originalClusterCopy, v1beta2Cluster)) - } - return nil - } -} diff --git a/exp/topology/desiredstate/lifecycle_hooks.go b/exp/topology/desiredstate/lifecycle_hooks.go new file mode 100644 index 000000000000..4916ea00d8be --- /dev/null +++ b/exp/topology/desiredstate/lifecycle_hooks.go @@ -0,0 +1,157 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package desiredstate + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/pkg/errors" + ctrl "sigs.k8s.io/controller-runtime" + + clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + runtimehooksv1 "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1" + runtimecatalog "sigs.k8s.io/cluster-api/exp/runtime/catalog" + "sigs.k8s.io/cluster-api/exp/topology/scope" + "sigs.k8s.io/cluster-api/internal/hooks" +) + +func (g *generator) callBeforeClusterUpgradeHook(ctx context.Context, s *scope.Scope, currentVersion *string, topologyVersion string) (bool, error) { + log := ctrl.LoggerFrom(ctx) + + // NOTE: the hook should be called only at the beginning of either a regular upgrade or a multistep upgrade sequence (it should not be called when in the middle of a multistep upgrade sequence); + // to detect if we are at the beginning of an upgrade, we check if the intent to call the AfterClusterUpgrade is not yet tracked. + if !hooks.IsPending(runtimehooksv1.AfterClusterUpgrade, s.Current.Cluster) { + var hookAnnotations []string + for key := range s.Current.Cluster.Annotations { + if strings.HasPrefix(key, clusterv1.BeforeClusterUpgradeHookAnnotationPrefix) { + hookAnnotations = append(hookAnnotations, key) + } + } + if len(hookAnnotations) > 0 { + slices.Sort(hookAnnotations) + message := fmt.Sprintf("annotations [%s] are set", strings.Join(hookAnnotations, ", ")) + if len(hookAnnotations) == 1 { + message = fmt.Sprintf("annotation [%s] is set", strings.Join(hookAnnotations, ", ")) + } + // Add the hook with a response to the tracker so we can later update the condition. + s.HookResponseTracker.Add(runtimehooksv1.BeforeClusterUpgrade, &runtimehooksv1.BeforeClusterUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + // RetryAfterSeconds needs to be set because having only hooks without RetryAfterSeconds + // would lead to not updating the condition. We can rely on getting an event when the + // annotation gets removed so we set twice of the default sync-period to not cause additional reconciles. + RetryAfterSeconds: 20 * 60, + CommonResponse: runtimehooksv1.CommonResponse{ + Message: message, + }, + }, + }) + + log.Info(fmt.Sprintf("Cluster upgrade to version %q is blocked by %q hook (via annotations)", topologyVersion, runtimecatalog.HookName(runtimehooksv1.BeforeClusterUpgrade)), "hooks", strings.Join(hookAnnotations, ",")) + return false, nil + } + + v1beta1Cluster := &clusterv1beta1.Cluster{} + // DeepCopy cluster because ConvertFrom has side effects like adding the conversion annotation. + if err := v1beta1Cluster.ConvertFrom(s.Current.Cluster.DeepCopy()); err != nil { + return false, errors.Wrap(err, "error converting Cluster to v1beta1 Cluster") + } + + hookRequest := &runtimehooksv1.BeforeClusterUpgradeRequest{ + Cluster: *cleanupCluster(v1beta1Cluster), + FromKubernetesVersion: *currentVersion, + ToKubernetesVersion: topologyVersion, + ControlPlaneUpgrades: toUpgradeStep(s.UpgradeTracker.ControlPlane.UpgradePlan), + WorkersUpgrades: toUpgradeStep(s.UpgradeTracker.MachineDeployments.UpgradePlan, s.UpgradeTracker.MachinePools.UpgradePlan), + } + hookResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{} + if err := g.RuntimeClient.CallAllExtensions(ctx, runtimehooksv1.BeforeClusterUpgrade, s.Current.Cluster, hookRequest, hookResponse); err != nil { + return false, err + } + // Add the response to the tracker so we can later update condition or requeue when required. + s.HookResponseTracker.Add(runtimehooksv1.BeforeClusterUpgrade, hookResponse) + if hookResponse.RetryAfterSeconds != 0 { + // Cannot pickup the new version right now. Need to try again later. + log.Info(fmt.Sprintf("Cluster upgrade to version %q is blocked by %q hook", topologyVersion, runtimecatalog.HookName(runtimehooksv1.BeforeClusterUpgrade))) + return false, nil + } + } + return true, nil +} + +func (g *generator) callAfterControlPlaneUpgradeHook(ctx context.Context, s *scope.Scope, currentVersion *string, topologyVersion string) (bool, error) { + log := ctrl.LoggerFrom(ctx) + + // Call the hook only if we are tracking the intent to do so. If it is not tracked it means we don't need to call the + // hook because we didn't go through an upgrade or we already called the hook after the upgrade. + if hooks.IsPending(runtimehooksv1.AfterControlPlaneUpgrade, s.Current.Cluster) { + v1beta1Cluster := &clusterv1beta1.Cluster{} + // DeepCopy cluster because ConvertFrom has side effects like adding the conversion annotation. + if err := v1beta1Cluster.ConvertFrom(s.Current.Cluster.DeepCopy()); err != nil { + return false, errors.Wrap(err, "error converting Cluster to v1beta1 Cluster") + } + + // Call all the registered extension for the hook. + hookRequest := &runtimehooksv1.AfterControlPlaneUpgradeRequest{ + Cluster: *cleanupCluster(v1beta1Cluster), + KubernetesVersion: *currentVersion, + ControlPlaneUpgrades: toUpgradeStep(s.UpgradeTracker.ControlPlane.UpgradePlan), + WorkersUpgrades: toUpgradeStep(s.UpgradeTracker.MachineDeployments.UpgradePlan, s.UpgradeTracker.MachinePools.UpgradePlan), + } + hookResponse := &runtimehooksv1.AfterControlPlaneUpgradeResponse{} + if err := g.RuntimeClient.CallAllExtensions(ctx, runtimehooksv1.AfterControlPlaneUpgrade, s.Current.Cluster, hookRequest, hookResponse); err != nil { + return false, err + } + // Add the response to the tracker so we can later update condition or requeue when required. + s.HookResponseTracker.Add(runtimehooksv1.AfterControlPlaneUpgrade, hookResponse) + + // If the extension responds to hold off on starting Machine deployments upgrades, + // change the UpgradeTracker accordingly, otherwise the hook call is completed and we + // can remove this hook from the list of pending-hooks. + if hookResponse.RetryAfterSeconds != 0 { + v := topologyVersion + if len(s.UpgradeTracker.ControlPlane.UpgradePlan) > 0 { + v = s.UpgradeTracker.ControlPlane.UpgradePlan[0] + } + log.Info(fmt.Sprintf("Upgrade to version %q is blocked by %q hook", v, runtimecatalog.HookName(runtimehooksv1.AfterControlPlaneUpgrade))) + return false, nil + } + if err := hooks.MarkAsDone(ctx, g.Client, s.Current.Cluster, runtimehooksv1.AfterControlPlaneUpgrade); err != nil { + return false, err + } + } + return true, nil +} + +// toUpgradeStep converts a list of version to a list of upgrade steps. +// Note. when called for workers, the function will receive in input two plans one for the MachineDeployments if any, the other for MachinePools if any. +// Considering that both plans, if defined, have to be equal, the function picks the first one not empty. +func toUpgradeStep(plans ...[]string) []runtimehooksv1.UpgradeStepInfo { + var steps []runtimehooksv1.UpgradeStepInfo + for _, plan := range plans { + if len(plan) != 0 { + for _, step := range plan { + steps = append(steps, runtimehooksv1.UpgradeStepInfo{Version: step}) + } + break + } + } + return steps +} diff --git a/exp/topology/desiredstate/lifecycle_hooks_test.go b/exp/topology/desiredstate/lifecycle_hooks_test.go new file mode 100644 index 000000000000..e8290a6473ec --- /dev/null +++ b/exp/topology/desiredstate/lifecycle_hooks_test.go @@ -0,0 +1,766 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package desiredstate + +import ( + "maps" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/component-base/featuregate/testing" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + runtimehooksv1 "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1" + runtimev1 "sigs.k8s.io/cluster-api/api/runtime/v1beta2" + runtimecatalog "sigs.k8s.io/cluster-api/exp/runtime/catalog" + "sigs.k8s.io/cluster-api/exp/topology/scope" + "sigs.k8s.io/cluster-api/feature" + fakeruntimeclient "sigs.k8s.io/cluster-api/internal/runtime/client/fake" + "sigs.k8s.io/cluster-api/util/conversion" + "sigs.k8s.io/cluster-api/util/test/builder" +) + +func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { + var testGVKs = []schema.GroupVersionKind{ + { + Group: "refAPIGroup1", + Kind: "refKind1", + Version: "v1beta4", + }, + } + + apiVersionGetter := func(gk schema.GroupKind) (string, error) { + for _, gvk := range testGVKs { + if gvk.GroupKind() == gk { + return schema.GroupVersion{ + Group: gk.Group, + Version: gvk.Version, + }.String(), nil + } + } + return "", errors.Errorf("unknown GroupVersionKind: %v", gk) + } + clusterv1beta1.SetAPIVersionGetter(apiVersionGetter) + + utilfeature.SetFeatureGateDuringTest(t, feature.Gates, feature.RuntimeSDK, true) + + catalog := runtimecatalog.New() + _ = runtimehooksv1.AddToCatalog(catalog) + + beforeClusterUpgradeGVH, err := catalog.GroupVersionHook(runtimehooksv1.BeforeClusterUpgrade) + if err != nil { + panic("unable to compute GVH") + } + blockingBeforeClusterUpgradeResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + RetryAfterSeconds: int32(10), + }, + } + nonBlockingBeforeClusterUpgradeResponse := &runtimehooksv1.BeforeClusterUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + }, + } + + afterControlPlaneUpgradeGVH, err := catalog.GroupVersionHook(runtimehooksv1.AfterControlPlaneUpgrade) + if err != nil { + panic("unable to compute GVH") + } + + blockingAfterControlPlaneUpgradeResponse := &runtimehooksv1.AfterControlPlaneUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + RetryAfterSeconds: int32(10), + }, + } + nonBlockingAfterControlPlaneUpgradeResponse := &runtimehooksv1.AfterControlPlaneUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + }, + } + + tests := []struct { + name string + topologyVersion string + pendingHookAnnotation string + controlPlaneObj *unstructured.Unstructured + controlPlaneUpgradePlan []string + machineDeploymentsUpgradePlan []string + machinePoolsUpgradePlan []string + upgradingMachineDeployments []string + upgradingMachinePools []string + wantBeforeClusterUpgradeRequest *runtimehooksv1.BeforeClusterUpgradeRequest + beforeClusterUpgradeResponse *runtimehooksv1.BeforeClusterUpgradeResponse + wantAfterControlPlaneUpgradeRequest *runtimehooksv1.AfterControlPlaneUpgradeRequest + afterControlPlaneUpgradeResponse *runtimehooksv1.AfterControlPlaneUpgradeResponse + wantVersion string + wantIsPendingUpgrade bool + wantIsStartingUpgrade bool + wantIsWaitingForWorkersUpgrade bool + wantPendingHookAnnotation string + }{ + // Upgrade cluster with CP, MD, MP (upgrade by one minor) + + { + name: "no hook called before starting the upgrade", + topologyVersion: "v1.2.2", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + wantVersion: "v1.2.2", + }, + { + name: "when an upgrade starts: call the BeforeClusterUpgrade hook, blocking answer", + topologyVersion: "v1.2.3", // changed from previous step + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.2.3"}, + machineDeploymentsUpgradePlan: []string{"v1.2.3"}, + machinePoolsUpgradePlan: []string{"v1.2.3"}, + wantBeforeClusterUpgradeRequest: &runtimehooksv1.BeforeClusterUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.2.3", + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.2.3"}), + WorkersUpgrades: toUpgradeStep([]string{"v1.2.3"}), + }, + beforeClusterUpgradeResponse: blockingBeforeClusterUpgradeResponse, + wantVersion: "v1.2.2", + wantIsPendingUpgrade: true, + }, + { + name: "when an upgrade starts: pick up a new version when BeforeClusterUpgrade hook unblocks", + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.2.3"}, + machineDeploymentsUpgradePlan: []string{"v1.2.3"}, + machinePoolsUpgradePlan: []string{"v1.2.3"}, + wantBeforeClusterUpgradeRequest: &runtimehooksv1.BeforeClusterUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.2.3", + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.2.3"}), + WorkersUpgrades: toUpgradeStep([]string{"v1.2.3"}), + }, + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + wantVersion: "v1.2.3", // changed from previous step + wantIsStartingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", // changed from previous step + }, + { + name: "when control plane is upgrading: do not call hooks", + topologyVersion: "v1.2.3", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.2.3"}, + machineDeploymentsUpgradePlan: []string{"v1.2.3"}, + machinePoolsUpgradePlan: []string{"v1.2.3"}, + wantVersion: "v1.2.3", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + }, + { + name: "after control plane is upgraded: call the AfterControlPlaneUpgrade hook, blocking answer", + topologyVersion: "v1.2.3", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.3", // changed from previous step + }). + Build(), + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{"v1.2.3"}, + machinePoolsUpgradePlan: []string{"v1.2.3"}, + wantAfterControlPlaneUpgradeRequest: &runtimehooksv1.AfterControlPlaneUpgradeRequest{ + KubernetesVersion: "v1.2.3", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{"v1.2.3"}), + }, + afterControlPlaneUpgradeResponse: blockingAfterControlPlaneUpgradeResponse, + wantVersion: "v1.2.3", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + }, + { + name: "after control plane is upgraded: AfterControlPlaneUpgrade hook unblocks", + topologyVersion: "v1.2.3", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.3", + }). + Build(), + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{"v1.2.3"}, + machinePoolsUpgradePlan: []string{"v1.2.3"}, + wantAfterControlPlaneUpgradeRequest: &runtimehooksv1.AfterControlPlaneUpgradeRequest{ + KubernetesVersion: "v1.2.3", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{"v1.2.3"}), + }, + afterControlPlaneUpgradeResponse: nonBlockingAfterControlPlaneUpgradeResponse, + wantVersion: "v1.2.3", + wantPendingHookAnnotation: "AfterClusterUpgrade", // changed from previous step + }, + { + name: "when machine deployment are upgrading: do not call hooks", + topologyVersion: "v1.2.3", + pendingHookAnnotation: "AfterClusterUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.3", + }). + Build(), + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{"v1.2.3"}, + machinePoolsUpgradePlan: []string{"v1.2.3"}, + wantVersion: "v1.2.3", + wantPendingHookAnnotation: "AfterClusterUpgrade", + }, + { + name: "when machine pools are upgrading: do not call hooks", + topologyVersion: "v1.2.3", + pendingHookAnnotation: "AfterClusterUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.3", + }). + Build(), + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{}, // changed from previous step + machinePoolsUpgradePlan: []string{"v1.2.3"}, + wantVersion: "v1.2.3", + wantPendingHookAnnotation: "AfterClusterUpgrade", + }, + // Note: After MP upgrade completes, the AfterClusterUpgrade is called from reconcile_state.go + + // Upgrade cluster with CP, MD (upgrade by two minors, workers skip the first one) + + { + name: "no hook called before starting the upgrade", + topologyVersion: "v1.2.2", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + wantVersion: "v1.2.2", + }, + { + name: "when an upgrade to the first minor starts: call the BeforeClusterUpgrade hook, blocking answer", + topologyVersion: "v1.4.4", // changed from previous step + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.3.3", "v1.4.4"}, + machineDeploymentsUpgradePlan: []string{"v1.4.4"}, + machinePoolsUpgradePlan: []string{}, + wantBeforeClusterUpgradeRequest: &runtimehooksv1.BeforeClusterUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.4.4", + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.3.3", "v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), + }, + beforeClusterUpgradeResponse: blockingBeforeClusterUpgradeResponse, + wantVersion: "v1.2.2", + wantIsPendingUpgrade: true, + }, + { + name: "when an upgrade to the first minor starts: BeforeClusterUpgrade hook unblocks, pick up the new version", + topologyVersion: "v1.4.4", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.3.3", "v1.4.4"}, + machineDeploymentsUpgradePlan: []string{"v1.4.4"}, + machinePoolsUpgradePlan: []string{}, + wantBeforeClusterUpgradeRequest: &runtimehooksv1.BeforeClusterUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.4.4", + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.3.3", "v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), + }, + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + wantVersion: "v1.3.3", // changed from previous step + wantIsStartingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", // changed from previous step + }, + { + name: "when control plane is upgrading to the first minor: do not call hooks", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.3.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.4.4"}, // changed from previous step + machineDeploymentsUpgradePlan: []string{"v1.4.4"}, + machinePoolsUpgradePlan: []string{}, + wantVersion: "v1.3.3", + wantIsPendingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + }, + { + name: "after control plane is upgraded to the first minor: call the AfterControlPlaneUpgrade hook, blocking answer", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.3.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.3.3", // changed from previous step + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.4.4"}, + machineDeploymentsUpgradePlan: []string{"v1.4.4"}, + machinePoolsUpgradePlan: []string{}, + wantAfterControlPlaneUpgradeRequest: &runtimehooksv1.AfterControlPlaneUpgradeRequest{ + KubernetesVersion: "v1.3.3", + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), + }, + afterControlPlaneUpgradeResponse: blockingAfterControlPlaneUpgradeResponse, + wantVersion: "v1.3.3", + wantIsPendingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + }, + { + name: "when an upgrade to the second minor starts: pick up a new version when AfterControlPlaneUpgrade hook unblocks", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.3.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.3.3", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.4.4"}, + machineDeploymentsUpgradePlan: []string{"v1.4.4"}, + machinePoolsUpgradePlan: []string{}, + wantAfterControlPlaneUpgradeRequest: &runtimehooksv1.AfterControlPlaneUpgradeRequest{ + KubernetesVersion: "v1.3.3", + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), + }, + afterControlPlaneUpgradeResponse: nonBlockingAfterControlPlaneUpgradeResponse, + wantVersion: "v1.4.4", // changed from previous step + wantIsStartingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", // changed from previous step + }, + { + name: "when control plane is upgrading to the second minor: do not call hooks", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.4.4", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.3.3", + }). + Build(), + controlPlaneUpgradePlan: []string{}, // changed from previous step + machineDeploymentsUpgradePlan: []string{"v1.4.4"}, + machinePoolsUpgradePlan: []string{}, + wantVersion: "v1.4.4", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + }, + { + name: "after control plane is upgraded to the second minor: call the AfterControlPlaneUpgrade hook, blocking answer", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.4.4", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.4.4", // changed from previous step + }). + Build(), + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{"v1.4.4"}, + machinePoolsUpgradePlan: []string{}, + wantAfterControlPlaneUpgradeRequest: &runtimehooksv1.AfterControlPlaneUpgradeRequest{ + KubernetesVersion: "v1.4.4", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), + }, + afterControlPlaneUpgradeResponse: blockingAfterControlPlaneUpgradeResponse, + wantVersion: "v1.4.4", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + }, + { + name: "when machine deployment are upgrading to the second minor: do not call hooks", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.4.4", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.4.4", + }). + Build(), + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{"v1.4.4"}, + machinePoolsUpgradePlan: []string{}, + wantVersion: "v1.4.4", + wantPendingHookAnnotation: "AfterClusterUpgrade", + }, + // Note: After MD upgrade completes, the AfterClusterUpgrade is called from reconcile_state.go + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + s := &scope.Scope{ + Blueprint: &scope.ClusterBlueprint{Topology: clusterv1.Topology{ + Version: tt.topologyVersion, + ControlPlane: clusterv1.ControlPlaneTopology{ + Replicas: ptr.To[int32](2), + }, + }}, + Current: &scope.ClusterState{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-ns", + // Add managedFields and annotations that should be cleaned up before the Cluster is sent to the RuntimeExtension. + ManagedFields: []metav1.ManagedFieldsEntry{ + { + APIVersion: builder.InfrastructureGroupVersion.String(), + Manager: "manager", + Operation: "Apply", + Time: ptr.To(metav1.Now()), + FieldsType: "FieldsV1", + }, + }, + Annotations: map[string]string{ + "fizz": "buzz", + corev1.LastAppliedConfigAnnotation: "should be cleaned up", + conversion.DataAnnotation: "should be cleaned up", + }, + }, + // Add some more fields to check that conversion implemented when calling RuntimeExtension are properly handled. + Spec: clusterv1.ClusterSpec{ + InfrastructureRef: clusterv1.ContractVersionedObjectReference{ + APIGroup: "refAPIGroup1", + Kind: "refKind1", + Name: "refName1", + }}, + }, + ControlPlane: &scope.ControlPlaneState{Object: tt.controlPlaneObj}, + }, + UpgradeTracker: scope.NewUpgradeTracker(), + HookResponseTracker: scope.NewHookResponseTracker(), + } + if tt.pendingHookAnnotation != "" { + if s.Current.Cluster.Annotations == nil { + s.Current.Cluster.Annotations = map[string]string{} + } + s.Current.Cluster.Annotations[runtimev1.PendingHooksAnnotation] = tt.pendingHookAnnotation + } + if len(tt.controlPlaneUpgradePlan) > 0 { + s.UpgradeTracker.ControlPlane.UpgradePlan = tt.controlPlaneUpgradePlan + } + if len(tt.machineDeploymentsUpgradePlan) > 0 { + s.UpgradeTracker.MachineDeployments.UpgradePlan = tt.machineDeploymentsUpgradePlan + } + if len(tt.machinePoolsUpgradePlan) > 0 { + s.UpgradeTracker.MachinePools.UpgradePlan = tt.machinePoolsUpgradePlan + } + if len(tt.upgradingMachineDeployments) > 0 { + s.UpgradeTracker.MachineDeployments.MarkUpgrading(tt.upgradingMachineDeployments...) + } + if len(tt.upgradingMachinePools) > 0 { + s.UpgradeTracker.MachinePools.MarkUpgrading(tt.upgradingMachinePools...) + } + + hooksCalled := sets.Set[string]{} + validateHookCall := func(request runtimehooksv1.RequestObject) error { + switch request := request.(type) { + case *runtimehooksv1.BeforeClusterUpgradeRequest: + hooksCalled.Insert("BeforeClusterUpgrade") + if err := validateHookRequest(request, tt.wantBeforeClusterUpgradeRequest); err != nil { + return err + } + case *runtimehooksv1.AfterControlPlaneUpgradeRequest: + hooksCalled.Insert("AfterControlPlaneUpgrade") + if err := validateHookRequest(request, tt.wantAfterControlPlaneUpgradeRequest); err != nil { + return err + } + default: + return errors.Errorf("unhandled request type %T", request) + } + return validateClusterParameter(s.Current.Cluster)(request) + } + + runtimeClient := fakeruntimeclient.NewRuntimeClientBuilder(). + WithCatalog(catalog). + WithCallAllExtensionResponses(map[runtimecatalog.GroupVersionHook]runtimehooksv1.ResponseObject{ + beforeClusterUpgradeGVH: tt.beforeClusterUpgradeResponse, + afterControlPlaneUpgradeGVH: tt.afterControlPlaneUpgradeResponse, + }). + WithCallAllExtensionValidations(validateHookCall). + Build() + + fakeClient := fake.NewClientBuilder().WithScheme(fakeScheme).WithObjects(s.Current.Cluster).Build() + + r := &generator{ + Client: fakeClient, + RuntimeClient: runtimeClient, + } + version, err := r.computeControlPlaneVersion(ctx, s) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(version).To(Equal(tt.wantVersion), "unexpected version") + g.Expect(s.UpgradeTracker.ControlPlane.IsPendingUpgrade).To(Equal(tt.wantIsPendingUpgrade), "unexpected IsPendingUpgrade") + g.Expect(s.UpgradeTracker.ControlPlane.IsStartingUpgrade).To(Equal(tt.wantIsStartingUpgrade), "unexpected IsStartingUpgrade") + g.Expect(s.UpgradeTracker.ControlPlane.IsWaitingForWorkersUpgrade).To(Equal(tt.wantIsWaitingForWorkersUpgrade), "unexpected IsWaitingForWorkersUpgrade") + + // check call received + g.Expect(hooksCalled.Has("BeforeClusterUpgrade")).To(Equal(tt.wantBeforeClusterUpgradeRequest != nil), "Unexpected call/missing call to BeforeClusterUpgrade") + g.Expect(hooksCalled.Has("AfterControlPlaneUpgrade")).To(Equal(tt.wantAfterControlPlaneUpgradeRequest != nil), "Unexpected call/missing call to AfterControlPlaneUpgrade") + + // check intent to call hooks + if tt.wantPendingHookAnnotation != "" { + g.Expect(s.Current.Cluster.Annotations).To(HaveKeyWithValue(runtimev1.PendingHooksAnnotation, tt.wantPendingHookAnnotation), "Unexpected PendingHookAnnotation") + } else { + g.Expect(s.Current.Cluster.Annotations).ToNot(HaveKey(runtimev1.PendingHooksAnnotation), "Unexpected PendingHookAnnotation") + } + }) + } +} + +func validateHookRequest(request runtimehooksv1.RequestObject, wantRequest runtimehooksv1.RequestObject) error { + if request, ok := request.(*runtimehooksv1.BeforeClusterUpgradeRequest); ok { + if wantRequest, ok := wantRequest.(*runtimehooksv1.BeforeClusterUpgradeRequest); ok && wantRequest != nil { + if wantRequest.FromKubernetesVersion != request.FromKubernetesVersion { + return errors.Errorf("unexpected BeforeClusterUpgradeRequest.FromKubernetesVersion version %s, want %s", request.FromKubernetesVersion, wantRequest.FromKubernetesVersion) + } + if wantRequest.ToKubernetesVersion != request.ToKubernetesVersion { + return errors.Errorf("unexpected BeforeClusterUpgradeRequest.ToKubernetes version %s, want %s", request.ToKubernetesVersion, wantRequest.ToKubernetesVersion) + } + if !reflect.DeepEqual(wantRequest.ControlPlaneUpgrades, request.ControlPlaneUpgrades) { + return errors.Errorf("unexpected BeforeClusterUpgradeRequest.ControlPlaneUpgrades %s, want %s", request.ControlPlaneUpgrades, wantRequest.ControlPlaneUpgrades) + } + if !reflect.DeepEqual(wantRequest.WorkersUpgrades, request.WorkersUpgrades) { + return errors.Errorf("unexpected BeforeClusterUpgradeRequest.WorkersUpgrades %s, want %s", request.WorkersUpgrades, wantRequest.WorkersUpgrades) + } + } else { + return errors.Errorf("got an unexpected request of type %T", request) + } + } + if request, ok := request.(*runtimehooksv1.BeforeControlPlaneUpgradeRequest); ok { + if wantRequest, ok := wantRequest.(*runtimehooksv1.BeforeControlPlaneUpgradeRequest); ok && wantRequest != nil { + if wantRequest.FromKubernetesVersion != request.FromKubernetesVersion { + return errors.Errorf("unexpected BeforeControlPlaneUpgradeRequest.FromKubernetesVersion version %s, want %s", request.FromKubernetesVersion, wantRequest.FromKubernetesVersion) + } + if wantRequest.ToKubernetesVersion != request.ToKubernetesVersion { + return errors.Errorf("unexpected BeforeControlPlaneUpgradeRequest.ToKubernetes version %s, want %s", request.ToKubernetesVersion, wantRequest.ToKubernetesVersion) + } + if !reflect.DeepEqual(wantRequest.ControlPlaneUpgrades, request.ControlPlaneUpgrades) { + return errors.Errorf("unexpected BeforeControlPlaneUpgradeRequest.ControlPlaneUpgrades %s, want %s", request.ControlPlaneUpgrades, wantRequest.ControlPlaneUpgrades) + } + if !reflect.DeepEqual(wantRequest.WorkersUpgrades, request.WorkersUpgrades) { + return errors.Errorf("unexpected BeforeControlPlaneUpgradeRequest.WorkersUpgrades %s, want %s", request.WorkersUpgrades, wantRequest.WorkersUpgrades) + } + } else { + return errors.Errorf("got an unexpected request of type %T", request) + } + } + if request, ok := request.(*runtimehooksv1.BeforeWorkersUpgradeRequest); ok { + if wantRequest, ok := wantRequest.(*runtimehooksv1.BeforeWorkersUpgradeRequest); ok && wantRequest != nil { + if wantRequest.FromKubernetesVersion != request.FromKubernetesVersion { + return errors.Errorf("unexpected BeforeWorkersUpgradeRequest.FromKubernetesVersion version %s, want %s", request.FromKubernetesVersion, wantRequest.FromKubernetesVersion) + } + if wantRequest.ToKubernetesVersion != request.ToKubernetesVersion { + return errors.Errorf("unexpected BeforeWorkersUpgradeRequest.ToKubernetes version %s, want %s", request.ToKubernetesVersion, wantRequest.ToKubernetesVersion) + } + if !reflect.DeepEqual(wantRequest.ControlPlaneUpgrades, request.ControlPlaneUpgrades) { + return errors.Errorf("unexpected BeforeWorkersUpgradeRequest.ControlPlaneUpgrades %s, want %s", request.ControlPlaneUpgrades, wantRequest.ControlPlaneUpgrades) + } + if !reflect.DeepEqual(wantRequest.WorkersUpgrades, request.WorkersUpgrades) { + return errors.Errorf("unexpected BeforeWorkersUpgradeRequest.WorkersUpgrades %s, want %s", request.WorkersUpgrades, wantRequest.WorkersUpgrades) + } + } else { + return errors.Errorf("got an unexpected request of type %T", request) + } + } + if request, ok := request.(*runtimehooksv1.AfterControlPlaneUpgradeRequest); ok { + if wantRequest, ok := wantRequest.(*runtimehooksv1.AfterControlPlaneUpgradeRequest); ok && wantRequest != nil { + if wantRequest.KubernetesVersion != request.KubernetesVersion { + return errors.Errorf("unexpected AfterControlPlaneUpgradeRequest.Kubernetes version %s, want %s", request.KubernetesVersion, wantRequest.KubernetesVersion) + } + if !reflect.DeepEqual(wantRequest.ControlPlaneUpgrades, request.ControlPlaneUpgrades) { + return errors.Errorf("unexpected AfterControlPlaneUpgradeRequest.ControlPlaneUpgrades %s, want %s", request.ControlPlaneUpgrades, wantRequest.ControlPlaneUpgrades) + } + if !reflect.DeepEqual(wantRequest.WorkersUpgrades, request.WorkersUpgrades) { + return errors.Errorf("unexpected AfterControlPlaneUpgradeRequest.WorkersUpgrades %s, want %s", request.WorkersUpgrades, wantRequest.WorkersUpgrades) + } + } else { + return errors.Errorf("got an unexpected request of type %T", request) + } + } + if request, ok := request.(*runtimehooksv1.AfterWorkersUpgradeRequest); ok { + if wantRequest, ok := wantRequest.(*runtimehooksv1.AfterWorkersUpgradeRequest); ok && wantRequest != nil { + if wantRequest.KubernetesVersion != request.KubernetesVersion { + return errors.Errorf("unexpected AfterWorkersUpgradeRequest.Kubernetes version %s, want %s", request.KubernetesVersion, wantRequest.KubernetesVersion) + } + if !reflect.DeepEqual(wantRequest.ControlPlaneUpgrades, request.ControlPlaneUpgrades) { + return errors.Errorf("unexpected AfterWorkersUpgradeRequest.ControlPlaneUpgrades %s, want %s", request.ControlPlaneUpgrades, wantRequest.ControlPlaneUpgrades) + } + if !reflect.DeepEqual(wantRequest.WorkersUpgrades, request.WorkersUpgrades) { + return errors.Errorf("unexpected AfterWorkersUpgradeRequest.WorkersUpgrades %s, want %s", request.WorkersUpgrades, wantRequest.WorkersUpgrades) + } + } else { + return errors.Errorf("got an unexpected request of type %T", request) + } + } + return nil +} + +func validateClusterParameter(originalCluster *clusterv1.Cluster) func(req runtimehooksv1.RequestObject) error { + // return a func that allows to check if expected transformations are applied to the Cluster parameter which is + // included in the payload for lifecycle hooks calls. + return func(req runtimehooksv1.RequestObject) error { + var cluster clusterv1beta1.Cluster + switch req := req.(type) { + case *runtimehooksv1.BeforeClusterUpgradeRequest: + cluster = req.Cluster + case *runtimehooksv1.BeforeControlPlaneUpgradeRequest: + cluster = req.Cluster + case *runtimehooksv1.AfterControlPlaneUpgradeRequest: + cluster = req.Cluster + case *runtimehooksv1.BeforeWorkersUpgradeRequest: + cluster = req.Cluster + case *runtimehooksv1.AfterWorkersUpgradeRequest: + cluster = req.Cluster + default: + return errors.Errorf("unhandled request type %T", req) + } + + // check if managed fields and well know annotations have been removed from the Cluster parameter included in the payload lifecycle hooks calls. + if cluster.GetManagedFields() != nil { + return errors.New("managedFields should have been cleaned up") + } + if _, ok := cluster.Annotations[corev1.LastAppliedConfigAnnotation]; ok { + return errors.New("last-applied-configuration annotation should have been cleaned up") + } + if _, ok := cluster.Annotations[conversion.DataAnnotation]; ok { + return errors.New("conversion annotation should have been cleaned up") + } + + // check the Cluster parameter included in the payload lifecycle hooks calls has been properly converted from v1beta2 to v1beta1. + // Note: to perform this check we convert the parameter back to v1beta2 and compare with the original cluster +/- expected transformations. + v1beta2Cluster := &clusterv1.Cluster{} + if err := cluster.ConvertTo(v1beta2Cluster); err != nil { + return err + } + + originalClusterCopy := originalCluster.DeepCopy() + originalClusterCopy.SetManagedFields(nil) + if originalClusterCopy.Annotations != nil { + annotations := maps.Clone(cluster.Annotations) + delete(annotations, corev1.LastAppliedConfigAnnotation) + delete(annotations, conversion.DataAnnotation) + originalClusterCopy.Annotations = annotations + } + + // drop conditions, it is not possible to round trip without the data annotation. + originalClusterCopy.Status.Conditions = nil + + if !apiequality.Semantic.DeepEqual(originalClusterCopy, v1beta2Cluster) { + return errors.Errorf("call to extension is not passing the expected cluster object: %s", cmp.Diff(originalClusterCopy, v1beta2Cluster)) + } + return nil + } +}