HypervisorMaintenance: Trigger eviction based on maintenance flag

When maintenance is set, it will trigger an eviction via the respective
CRD and reflect the result back to the hypervisor status.

Author: fwiesel

api/v1/hypervisor_types.go

@@ -35,6 +35,7 @@ const (
 	ConditionReasonReadyReady = "ready"
 	// or not
 	ConditionReasonReadyMaintenance = "maintenance"
+	ConditionReasonReadyEvicted     = "evicted"
 )
 
 // HypervisorSpec defines the desired state of Hypervisor

internal/controller/hypervisor_maintenance_controller.go

@@ -24,11 +24,13 @@ import (
 	"context"
 	"fmt"
 
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	logger "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/gophercloud/gophercloud/v2"
@@ -39,7 +41,7 @@ import (
 )
 
 const (
-	HypervisorMaintenanceControllerName = "HypervisorMaintenanceController"
+	HypervisorMaintenanceControllerName = "HypervisorMaintenance"
 )
 
 type HypervisorMaintenanceController struct {
@@ -50,6 +52,7 @@ type HypervisorMaintenanceController struct {
 
 // +kubebuilder:rbac:groups=kvm.cloud.sap,resources=hypervisors,verbs=get;list;watch
 // +kubebuilder:rbac:groups=kvm.cloud.sap,resources=hypervisors/status,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=kvm.cloud.sap,resources=evictions,verbs=get;list;watch;create;update;patch;delete
 
 func (hec *HypervisorMaintenanceController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	hv := &kvmv1.Hypervisor{}
@@ -69,15 +72,20 @@ func (hec *HypervisorMaintenanceController) Reconcile(ctx context.Context, req c
 	}
 
 	log := logger.FromContext(ctx).
-		WithName("HypervisorService")
+		WithName(HypervisorMaintenanceControllerName)
 	ctx = logger.IntoContext(ctx, log)
 
 	changed, err := hec.reconcileComputeService(ctx, hv)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
 
-	if changed {
+	changed1, err := hec.reconcileEviction(ctx, hv)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if changed || changed1 {
 		return ctrl.Result{}, hec.Status().Update(ctx, hv)
 	} else {
 		return ctrl.Result{}, nil
@@ -115,6 +123,9 @@ func (hec *HypervisorMaintenanceController) reconcileComputeService(ctx context.
 			return false, fmt.Errorf("failed to enable hypervisor due to %w", err)
 		}
 	case "manual", "auto", "ha": // Disable the compute service
+		// Also in case of HA, as it doesn't hurt to disable it twice, and this
+		// allows us to enable the service again, when the maintenance field is
+		// cleared in the case above.
 		if !meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
 			Type:    kvmv1.ConditionTypeHypervisorDisabled,
 			Status:  metav1.ConditionTrue,
@@ -147,6 +158,106 @@ func (hec *HypervisorMaintenanceController) reconcileComputeService(ctx context.
 	return true, nil
 }
 
+func (hec *HypervisorMaintenanceController) reconcileEviction(ctx context.Context, hv *kvmv1.Hypervisor) (bool, error) {
+	eviction := &kvmv1.Eviction{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: hv.Name,
+		},
+	}
+
+	switch hv.Spec.Maintenance {
+	case "":
+		// Avoid deleting the eviction over and over.
+		if hv.Status.Evicted || meta.RemoveStatusCondition(&hv.Status.Conditions, kvmv1.ConditionTypeEvicting) {
+			err := k8sclient.IgnoreNotFound(hec.Delete(ctx, eviction))
+			hv.Status.Evicted = false
+			return true, err
+		}
+		return false, nil
+	case "manual", "auto": // In case of "ha", the host gets emptied from the HA service
+		if cond := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeEvicting); cond != nil {
+			if cond.Reason == kvmv1.ConditionReasonSucceeded {
+				// We are done here, no need to look at the eviction any more
+				return false, nil
+			}
+		}
+		status, err := hec.ensureEviction(ctx, eviction, hv)
+		if err != nil {
+			return false, err
+		}
+		var reason, message string
+		changed := false
+		if status == metav1.ConditionFalse {
+			message = "Evicted"
+			reason = kvmv1.ConditionReasonSucceeded
+			if !hv.Status.Evicted {
+				changed = true
+				hv.Status.Evicted = true
+			}
+		} else {
+			message = "Evicting"
+			reason = kvmv1.ConditionReasonRunning
+			if hv.Status.Evicted {
+				changed = true
+				hv.Status.Evicted = false
+			}
+		}
+
+		if meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
+			Type:    kvmv1.ConditionTypeEvicting,
+			Status:  status,
+			Reason:  reason,
+			Message: message,
+		}) {
+			changed = true
+		}
+
+		if meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
+			Type:    kvmv1.ConditionTypeReady,
+			Status:  metav1.ConditionFalse,
+			Reason:  kvmv1.ConditionReasonReadyEvicted,
+			Message: "Hypervisor is disabled and evicted",
+		}) {
+			changed = true
+		}
+		logger.FromContext(ctx).Info("reconcile", "changed", changed, "conditions", hv.Status.Conditions)
+		return changed, nil
+	}
+
+	return false, nil
+}
+
+func (hec *HypervisorMaintenanceController) ensureEviction(ctx context.Context, eviction *kvmv1.Eviction, hypervisor *kvmv1.Hypervisor) (metav1.ConditionStatus, error) {
+	log := logger.FromContext(ctx)
+	if err := hec.Get(ctx, k8sclient.ObjectKeyFromObject(eviction), eviction); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return metav1.ConditionUnknown, fmt.Errorf("failed to get eviction due to %w", err)
+		}
+		if err := controllerutil.SetControllerReference(hypervisor, eviction, hec.Scheme); err != nil {
+			return metav1.ConditionUnknown, err
+		}
+		log.Info("Creating new eviction", "name", eviction.Name)
+		eviction.Spec = kvmv1.EvictionSpec{
+			Hypervisor: hypervisor.Name,
+			Reason:     "openstack-hypervisor-operator maintenance",
+		}
+
+		// This also transports the label-selector, if set
+		transportLabels(&eviction.ObjectMeta, hypervisor)
+
+		if err = hec.Create(ctx, eviction); err != nil {
+			return metav1.ConditionUnknown, fmt.Errorf("failed to create eviction due to %w", err)
+		}
+	}
+
+	// check if we are still evicting (defaulting to yes)
+	if meta.IsStatusConditionFalse(eviction.Status.Conditions, kvmv1.ConditionTypeEvicting) {
+		return metav1.ConditionFalse, nil
+	} else {
+		return metav1.ConditionTrue, nil
+	}
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (hec *HypervisorMaintenanceController) SetupWithManager(mgr ctrl.Manager) error {
 	ctx := context.Background()
@@ -161,5 +272,6 @@ func (hec *HypervisorMaintenanceController) SetupWithManager(mgr ctrl.Manager) e
 	return ctrl.NewControllerManagedBy(mgr).
 		Named(HypervisorMaintenanceControllerName).
 		For(&kvmv1.Hypervisor{}).
+		Owns(&kvmv1.Eviction{}). // trigger Reconcile whenever an Own-ed eviction is created/updated/deleted
 		Complete(hec)
 }

internal/controller/hypervisor_maintenance_controller_test.go

@@ -30,6 +30,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
+	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	kvmv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
 )
@@ -72,7 +74,7 @@ var _ = Describe("HypervisorMaintenanceController", func() {
 
 			w.WriteHeader(http.StatusOK)
 			_, err = fmt.Fprint(w, ServiceEnabledResponse)
-			Expect(err).NotTo(HaveOccurred())
+			Expect(err).To(Succeed())
 		})
 	}
 
@@ -113,6 +115,16 @@ var _ = Describe("HypervisorMaintenanceController", func() {
 		Expect(err).NotTo(HaveOccurred())
 	})
 
+	AfterEach(func(ctx context.Context) {
+		eviction := &kvmv1.Eviction{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      hypervisorName.Name,
+				Namespace: hypervisorName.Namespace,
+			},
+		}
+		Expect(k8sclient.IgnoreNotFound(k8sClient.Delete(ctx, eviction))).To(Succeed())
+	})
+
 	// Tests
 	Context("Onboarded Hypervisor", func() {
 		BeforeEach(func() {
@@ -127,7 +139,6 @@ var _ = Describe("HypervisorMaintenanceController", func() {
 					Message: "random text",
 				},
 			)
-
 			Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed())
 		})
 
@@ -169,9 +180,6 @@ var _ = Describe("HypervisorMaintenanceController", func() {
 					Expect(k8sClient.Update(ctx, hypervisor)).To(Succeed())
 					expectedBody := fmt.Sprintf(`{"disabled_reason": "Hypervisor CRD: spec.maintenance=%v", "status": "disabled"}`, mode)
 					mockServiceUpdate(expectedBody)
-					req := ctrl.Request{NamespacedName: hypervisorName}
-					_, err := controller.Reconcile(ctx, req)
-					Expect(err).NotTo(HaveOccurred())
 				})
 
 				It("should set the ConditionTypeHypervisorDisabled to true", func() {
@@ -192,5 +200,153 @@ var _ = Describe("HypervisorMaintenanceController", func() {
 			}) // Spec.Maintenance="<mode>"
 		}
 
+		Describe("Eviction reconciliation", func() {
+			Context("Spec.Maintenance=\"\"", func() {
+				BeforeEach(func() {
+					hypervisor := &kvmv1.Hypervisor{}
+					Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed())
+					hypervisor.Spec.Maintenance = ""
+					Expect(k8sClient.Update(ctx, hypervisor)).To(Succeed())
+					Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed())
+					meta.SetStatusCondition(&hypervisor.Status.Conditions,
+						metav1.Condition{
+							Type:    kvmv1.ConditionTypeEvicting,
+							Reason:  "dontcare",
+							Status:  metav1.ConditionUnknown,
+							Message: "dontcare",
+						})
+					Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed())
+					expectedBody := `{"status": "enabled"}`
+					mockServiceUpdate(expectedBody)
+
+					eviction := &kvmv1.Eviction{
+						ObjectMeta: metav1.ObjectMeta{Name: hypervisorName.Name},
+						Spec: kvmv1.EvictionSpec{
+							Hypervisor: hypervisorName.Name,
+							Reason:     "test",
+						},
+					}
+					Expect(controllerutil.SetControllerReference(hypervisor, eviction, controller.Scheme)).To(Succeed())
+					Expect(k8sClient.Create(ctx, eviction)).To(Succeed())
+				})
+
+				It("should delete the created eviction", func() {
+					eviction := &kvmv1.Eviction{}
+					err := k8sClient.Get(ctx, hypervisorName, eviction)
+					By(fmt.Sprintf("%+v", *eviction))
+					Expect(err).To(HaveOccurred())
+					Expect(k8sclient.IgnoreNotFound(err)).To(Succeed())
+				})
+			}) // Spec.Maintenance=""
+
+			Context("Spec.Maintenance=\"ha\"", func() {
+				BeforeEach(func() {
+					hypervisor := &kvmv1.Hypervisor{}
+					Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed())
+					hypervisor.Spec.Maintenance = "ha"
+					Expect(k8sClient.Update(ctx, hypervisor)).To(Succeed())
+					expectedBody := `{"disabled_reason": "Hypervisor CRD: spec.maintenance=ha", "status": "disabled"}`
+					mockServiceUpdate(expectedBody)
+				})
+				It("should not create an eviction resource", func() {
+					eviction := &kvmv1.Eviction{}
+					err := k8sClient.Get(ctx, hypervisorName, eviction)
+					Expect(err).To(HaveOccurred())
+					Expect(k8sclient.IgnoreNotFound(err)).To(Succeed())
+				})
+			}) // Spec.Maintenance="ha"
+
+			for _, mode := range []string{"auto", "manual"} {
+				Context(fmt.Sprintf("Spec.Maintenance=\"%v\"", mode), func() {
+					BeforeEach(func() {
+						hypervisor := &kvmv1.Hypervisor{}
+						Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed())
+						hypervisor.Spec.Maintenance = mode
+						Expect(k8sClient.Update(ctx, hypervisor)).To(Succeed())
+						expectedBody := fmt.Sprintf(`{"disabled_reason": "Hypervisor CRD: spec.maintenance=%v", "status": "disabled"}`, mode)
+						mockServiceUpdate(expectedBody)
+					})
+
+					When("there is no eviction yet", func() {
+						It("should create an eviction resource named as the hypervisor", func() {
+							eviction := &kvmv1.Eviction{}
+							Expect(k8sClient.Get(ctx, hypervisorName, eviction)).To(Succeed())
+						})
+
+						It("should create an evicting condition", func() {
+							hypervisor := &kvmv1.Hypervisor{}
+							Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed())
+							Expect(hypervisor.Status.Conditions).To(ContainElement(
+								SatisfyAll(
+									HaveField("Type", kvmv1.ConditionTypeEvicting),
+									HaveField("Status", metav1.ConditionTrue),
+									HaveField("Reason", kvmv1.ConditionReasonRunning),
+								),
+							))
+						})
+
+						It("should reflect it in the hypervisor evicted status", func() {
+							hypervisor := &kvmv1.Hypervisor{}
+							Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed())
+							Expect(hypervisor.Status.Evicted).To(BeFalse())
+						})
+					})
+
+					When("there is a finished eviction", func() {
+						BeforeEach(func() {
+							eviction := &kvmv1.Eviction{
+								ObjectMeta: metav1.ObjectMeta{Name: hypervisorName.Name},
+								Spec: kvmv1.EvictionSpec{
+									Hypervisor: hypervisorName.Name,
+									Reason:     "test",
+								},
+							}
+							hypervisor := &kvmv1.Hypervisor{}
+							Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed())
+							Expect(controllerutil.SetControllerReference(hypervisor, eviction, controller.Scheme)).To(Succeed())
+							Expect(k8sClient.Create(ctx, eviction)).To(Succeed())
+
+							Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed())
+							meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{
+								Type:    kvmv1.ConditionTypeEvicting,
+								Status:  metav1.ConditionFalse,
+								Message: "whatever",
+								Reason:  kvmv1.ConditionReasonSucceeded,
+							})
+							Expect(k8sClient.Status().Update(ctx, eviction)).To(Succeed())
+						})
+
+						It("should reflect it in the hypervisor evicting condition", func() {
+							hypervisor := &kvmv1.Hypervisor{}
+							Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed())
+							Expect(hypervisor.Status.Conditions).To(ContainElement(
+								SatisfyAll(
+									HaveField("Type", kvmv1.ConditionTypeEvicting),
+									HaveField("Status", metav1.ConditionFalse),
+									HaveField("Reason", kvmv1.ConditionReasonSucceeded),
+								),
+							))
+						})
+
+						It("should reflect it in the hypervisor evicted status", func() {
+							hypervisor := &kvmv1.Hypervisor{}
+							Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed())
+							Expect(hypervisor.Status.Evicted).To(BeTrue())
+						})
+
+						It("should set the ConditionTypeReady to false and reason to evicted", func() {
+							updated := &kvmv1.Hypervisor{}
+							Expect(k8sClient.Get(ctx, hypervisorName, updated)).To(Succeed())
+							Expect(updated.Status.Conditions).To(ContainElement(
+								SatisfyAll(
+									HaveField("Type", kvmv1.ConditionTypeReady),
+									HaveField("Status", metav1.ConditionFalse),
+									HaveField("Reason", kvmv1.ConditionReasonReadyEvicted),
+								)))
+						})
+					})
+				}) // Spec.Maintenance="<mode>"
+			}
+		})
 	}) // Context Onboarded Hypervisor
 })