HypervisorMaintenance: Trigger eviction based on maintenance flag

When maintenance is set, it will trigger an eviction via the respective
CRD and reflect the result back to the hypervisor status.

Author: fwiesel

api/v1/hypervisor_types.go

@@ -35,6 +35,7 @@ const (
 	ConditionReasonReadyReady = "ready"
 	// or not
 	ConditionReasonReadyMaintenance = "maintenance"
+	ConditionReasonReadyEvicted     = "evicted"
 )
 
 // HypervisorSpec defines the desired state of Hypervisor

charts/openstack-hypervisor-operator/templates/manager-rbac.yaml

@@ -48,15 +48,14 @@ rules:
   - kvm.cloud.sap
   resources:
   - evictions
-  - hypervisors
-  - hypervisors/status
   verbs:
   - create
   - delete
   - get
   - list
   - patch
   - update
+  - verbs=get
   - watch
 - apiGroups:
   - kvm.cloud.sap
@@ -72,6 +71,19 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - kvm.cloud.sap
+  resources:
+  - hypervisors
+  - hypervisors/status
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - policy
   resources:

config/rbac/role.yaml

@@ -47,15 +47,14 @@ rules:
   - kvm.cloud.sap
   resources:
   - evictions
-  - hypervisors
-  - hypervisors/status
   verbs:
   - create
   - delete
   - get
   - list
   - patch
   - update
+  - verbs=get
   - watch
 - apiGroups:
   - kvm.cloud.sap
@@ -71,6 +70,19 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - kvm.cloud.sap
+  resources:
+  - hypervisors
+  - hypervisors/status
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - policy
   resources:

internal/controller/hypervisor_maintenance_controller.go

@@ -24,11 +24,13 @@ import (
 	"context"
 	"fmt"
 
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	logger "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/gophercloud/gophercloud/v2"
@@ -39,7 +41,7 @@ import (
 )
 
 const (
-	HypervisorMaintenanceControllerName = "HypervisorMaintenanceController"
+	HypervisorMaintenanceControllerName = "HypervisorMaintenance"
 )
 
 type HypervisorMaintenanceController struct {
@@ -50,6 +52,7 @@ type HypervisorMaintenanceController struct {
 
 // +kubebuilder:rbac:groups=kvm.cloud.sap,resources=hypervisors,verbs=get;list;watch
 // +kubebuilder:rbac:groups=kvm.cloud.sap,resources=hypervisors/status,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=kvm.cloud.sap,resources=evictions,verbs=verbs=get;list;watch;create;update;patch;delete
 
 func (hec *HypervisorMaintenanceController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	hv := &kvmv1.Hypervisor{}
@@ -69,15 +72,20 @@ func (hec *HypervisorMaintenanceController) Reconcile(ctx context.Context, req c
 	}
 
 	log := logger.FromContext(ctx).
-		WithName("HypervisorService")
+		WithName(HypervisorMaintenanceControllerName)
 	ctx = logger.IntoContext(ctx, log)
 
 	changed, err := hec.reconcileComputeService(ctx, hv)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
 
-	if changed {
+	changed1, err := hec.reconcileEviction(ctx, hv)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if changed || changed1 {
 		return ctrl.Result{}, hec.Status().Update(ctx, hv)
 	} else {
 		return ctrl.Result{}, nil
@@ -115,6 +123,9 @@ func (hec *HypervisorMaintenanceController) reconcileComputeService(ctx context.
 			return false, fmt.Errorf("failed to enable hypervisor due to %w", err)
 		}
 	case "manual", "auto", "ha": // Disable the compute service
+		// Also in case of HA, as it doesn't hurt to disable it twice, and this
+		// allows us to enable the service again, when the maintenance field is
+		// cleared in the case above.
 		if !meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
 			Type:    kvmv1.ConditionTypeHypervisorDisabled,
 			Status:  metav1.ConditionTrue,
@@ -147,6 +158,106 @@ func (hec *HypervisorMaintenanceController) reconcileComputeService(ctx context.
 	return true, nil
 }
 
+func (hec *HypervisorMaintenanceController) reconcileEviction(ctx context.Context, hv *kvmv1.Hypervisor) (bool, error) {
+	eviction := &kvmv1.Eviction{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: hv.Name,
+		},
+	}
+
+	switch hv.Spec.Maintenance {
+	case "":
+		// Avoid deleting the eviction over and over.
+		if hv.Status.Evicted || meta.RemoveStatusCondition(&hv.Status.Conditions, kvmv1.ConditionTypeEvicting) {
+			err := k8sclient.IgnoreNotFound(hec.Delete(ctx, eviction))
+			hv.Status.Evicted = false
+			return true, err
+		}
+		return false, nil
+	case "manual", "auto": // In case of "ha", the host gets emptied from the HA service
+		if cond := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeEvicting); cond != nil {
+			if cond.Reason == kvmv1.ConditionReasonSucceeded {
+				// We are done here, no need to look at the eviction any more
+				return false, nil
+			}
+		}
+		status, err := hec.ensureEviction(ctx, eviction, hv)
+		if err != nil {
+			return false, err
+		}
+		var reason, message string
+		changed := false
+		if status == metav1.ConditionFalse {
+			message = "Evicted"
+			reason = kvmv1.ConditionReasonSucceeded
+			if !hv.Status.Evicted {
+				changed = true
+				hv.Status.Evicted = true
+			}
+		} else {
+			message = "Evicting"
+			reason = kvmv1.ConditionReasonRunning
+			if hv.Status.Evicted {
+				changed = true
+				hv.Status.Evicted = false
+			}
+		}
+
+		if meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
+			Type:    kvmv1.ConditionTypeEvicting,
+			Status:  status,
+			Reason:  reason,
+			Message: message,
+		}) {
+			changed = true
+		}
+
+		if meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
+			Type:    kvmv1.ConditionTypeReady,
+			Status:  metav1.ConditionFalse,
+			Reason:  kvmv1.ConditionReasonReadyEvicted,
+			Message: "Hypervisor is disabled and evicted",
+		}) {
+			changed = true
+		}
+		logger.FromContext(ctx).Info("reconcile", "changed", changed, "conditions", hv.Status.Conditions)
+		return changed, nil
+	}
+
+	return false, nil
+}
+
+func (hec *HypervisorMaintenanceController) ensureEviction(ctx context.Context, eviction *kvmv1.Eviction, hypervisor *kvmv1.Hypervisor) (metav1.ConditionStatus, error) {
+	log := logger.FromContext(ctx)
+	if err := hec.Get(ctx, k8sclient.ObjectKeyFromObject(eviction), eviction); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return metav1.ConditionUnknown, fmt.Errorf("failed to get eviction due to %w", err)
+		}
+		if err := controllerutil.SetControllerReference(hypervisor, eviction, hec.Scheme); err != nil {
+			return metav1.ConditionUnknown, err
+		}
+		log.Info("Creating new eviction", "name", eviction.Name)
+		eviction.Spec = kvmv1.EvictionSpec{
+			Hypervisor: hypervisor.Name,
+			Reason:     "openstack-hypervisor-operator maintenance",
+		}
+
+		// This also transports the label-selector, if set
+		transportLabels(&eviction.ObjectMeta, hypervisor)
+
+		if err = hec.Create(ctx, eviction); err != nil {
+			return metav1.ConditionUnknown, fmt.Errorf("failed to create eviction due to %w", err)
+		}
+	}
+
+	// check if we are still evicting (defaulting to yes)
+	if meta.IsStatusConditionFalse(eviction.Status.Conditions, kvmv1.ConditionTypeEvicting) {
+		return metav1.ConditionFalse, nil
+	} else {
+		return metav1.ConditionTrue, nil
+	}
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (hec *HypervisorMaintenanceController) SetupWithManager(mgr ctrl.Manager) error {
 	ctx := context.Background()
@@ -161,5 +272,6 @@ func (hec *HypervisorMaintenanceController) SetupWithManager(mgr ctrl.Manager) e
 	return ctrl.NewControllerManagedBy(mgr).
 		Named(HypervisorMaintenanceControllerName).
 		For(&kvmv1.Hypervisor{}).
+		Owns(&kvmv1.Eviction{}). // trigger Reconcile whenever an Own-ed eviction is created/updated/deleted
 		Complete(hec)
 }