Create new disk (#32)

* Create new disk if reahc max capacity prototype * Ready to test with AWS (#33) * First prototype of AWS (#34) * Fix pre push Git hook (#35) * Aws (#36) * Simplify git hook * Simplify git hook * Fix metrics label to allow multi diskconfig (#37) * Fix metrics label render (#38) * Aws (#39) * Rename sample pod container * AWS EBS works with etx4 * AWS EBS works as well (#40) * Fix golint error (#41) * e2e test for new disk and restart (#42) * Fix install dependency order in e2e tests * Use Qualtity instead of string in CRD * Remove or fix TODOs * Give name to pod if missing * Increase uniqueness of random pod name generation * Make pod name generation by owner more human friendly * Fix sample diskconfig * Generate UID for pod if missing * Move service creation to a separate controller * Fix metrics connection between pod and service * Set namespace of service assert * Fix service discovery in e2e tests * Fix diskconfig per pod detection * Lable service with discoblocks to find in monitor * Simplify AWS EBS CSI driver lookup * Remove obsolete diskconfig detection by label * Delete pod of jobs on success * Optimize Pod name generation * Container order not guaranteed
ondat · Sep 22, 2022 · 7b72c8d · 7b72c8d
1 parent 241521b
commit 7b72c8d
Show file tree

Hide file tree

Showing 49 changed files with 1,861 additions and 446 deletions.
diff --git a/.golangci.yaml b/.golangci.yaml
@@ -14,8 +14,6 @@ linters-settings:
     disabled-checks:
       - unnamedResult
       - hugeParam
-  govet:
-    check-shadowing: true
   nolintlint:
     require-explanation: true
     require-specific: true

diff --git a/.husky/hooks/pre-push b/.husky/hooks/pre-push
@@ -4,7 +4,7 @@ if [[ "$SKIP_GIT_PUSH_HOOK" ]]; then exit 0; fi
 
 set -e
 
-if [[ -n "$(git status --short)" ]]; then
+if git status --short | grep -qv "??"; then
     git stash
     function unstash() {
         git reset --hard

diff --git a/api/v1/diskconfig_types.go b/api/v1/diskconfig_types.go
@@ -18,6 +18,7 @@ package v1
 
 import (
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -35,11 +36,11 @@ type DiskConfigSpec struct {
 
 	// Capacity represents the desired capacity of the underlying volume.
 	//+kubebuilder:default:="1Gi"
-	//+kubebuilder:validation:Pattern:="^(\\d+)(m|Mi|g|Gi|t|Ti|p|Pi)$"
 	//+kubebuilder:validation:Optional
-	Capacity string `json:"capacity,omitempty" yaml:"capacity,omitempty"`
+	Capacity resource.Quantity `json:"capacity,omitempty" yaml:"capacity,omitempty"`
 
 	// MountPointPattern is the mount point of the disk. %d is optional and represents disk number in order. Will be automatically appended for second drive if missing.
+	// Reserved characters: ><|:&.+*!?^$()[]{}, only 1 %d allowed.
 	//+kubebuilder:default:="/media/discoblocks/<name>-%d"
 	//+kubebuilder:validation:Pattern:="^/(.*)"
 	//+kubebuilder:validation:Optional
@@ -52,7 +53,7 @@ type DiskConfigSpec struct {
 	AccessModes []corev1.PersistentVolumeAccessMode `json:"accessModes,omitempty" yaml:"accessModes,omitempty"`
 
 	// AvailabilityMode defines the desired number of instances.
-	//+kubebuilder:default:="Multiple"
+	//+kubebuilder:default:="ReadWriteOnce"
 	//+kubebuilder:validation:Optional
 	AvailabilityMode AvailabilityMode `json:"availabilityMode,omitempty" yaml:"availabilityMode,omitempty"`
 
@@ -78,18 +79,27 @@ type Policy struct {
 	UpscaleTriggerPercentage uint8 `json:"upscaleTriggerPercentage,omitempty" yaml:"upscaleTriggerPercentage,omitempty"`
 
 	// MaximumCapacityOfDisks defines maximum capacity of a disk.
-	//+kubebuilder:validation:Pattern:="^(\\d+)(m|Mi|g|Gi|t|Ti|p|Pi)$"
-	//+kubebuilder:default:="100Gi"
+	//+kubebuilder:default:="1000Gi"
 	//+kubebuilder:validation:Optional
-	MaximumCapacityOfDisk string `json:"maximumCapacityOfDisk,omitempty" yaml:"maximumCapacityOfDisk,omitempty"`
+	MaximumCapacityOfDisk resource.Quantity `json:"maximumCapacityOfDisk,omitempty" yaml:"maximumCapacityOfDisk,omitempty"`
 
 	// MaximumCapacityOfDisks defines maximum number of a disks.
-	//+kubebuilder:default:=10
+	//+kubebuilder:default:=1
 	//+kubebuilder:validation:Minimum:=1
-	//+kubebuilder:validation:Maximum:=1000
+	//+kubebuilder:validation:Maximum:=150
 	//+kubebuilder:validation:Optional
 	MaximumNumberOfDisks uint8 `json:"maximumNumberOfDisks,omitempty" yaml:"maximumNumberOfDisks,omitempty"`
 
+	// ExtendCapacity represents the capacity to extend with.
+	//+kubebuilder:default:="1Gi"
+	//+kubebuilder:validation:Optional
+	ExtendCapacity resource.Quantity `json:"extendCapacity,omitempty" yaml:"extendCapacity,omitempty"`
+
+	// CoolDown defines temporary pause of scaling.
+	//+kubebuilder:default:="5m"
+	//+kubebuilder:validation:Optional
+	CoolDown metav1.Duration `json:"coolDown,omitempty" yaml:"coolDown,omitempty"`
+
 	// Pause disables autoscaling of disks.
 	//+kubebuilder:default:=false
 	//+kubebuilder:validation:Optional
@@ -120,12 +130,12 @@ const (
 	Deleting Phase = "Deleting"
 )
 
-// +kubebuilder:validation:Enum=Singleton;Multiple
+// +kubebuilder:validation:Enum=ReadWriteSame;ReadWriteOnce
 type AvailabilityMode string
 
 const (
-	Singleton AvailabilityMode = "Singleton"
-	Multiple  AvailabilityMode = "Multiple"
+	ReadWriteSame AvailabilityMode = "ReadWriteSame"
+	ReadWriteOnce AvailabilityMode = "ReadWriteOnce"
 )
 
 //+kubebuilder:object:root=true

diff --git a/api/v1/diskconfig_webhook.go b/api/v1/diskconfig_webhook.go
@@ -20,13 +20,14 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
+	"regexp"
+	"strings"
 	"time"
 
 	"github.com/ondat/discoblocks/pkg/drivers"
 	"golang.org/x/net/context"
 	storagev1 "k8s.io/api/storage/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -35,7 +36,9 @@ import (
 )
 
 // log is for logging in this package
-var diskConfigLog = logf.Log.WithName("DiskConfigWebhook")
+var diskConfigLog = logf.Log.WithName("v1.DiskConfigWebhook")
+
+var reservedCharacters = regexp.MustCompile(`[>|<|||:|&|.|\+|\*|!|\?|\^|\$|\(|\)|\[|\]|\{|\}]`)
 
 // SetupWebhookWithManager sets up the webhook with the Manager.
 func (r *DiskConfig) SetupWebhookWithManager(mgr ctrl.Manager) error {
@@ -68,44 +71,30 @@ func (r *DiskConfig) validate(old runtime.Object) error {
 	logger.Info("Validate update...")
 	defer logger.Info("Validated")
 
-	// TODO remove once we generate detault
 	if r.Spec.StorageClassName == "" {
 		logger.Info("StorageClass name is invalid")
 		return errors.New("invalid StorageClass name")
 	}
 
-	if _, err := resource.ParseQuantity(r.Spec.Policy.MaximumCapacityOfDisk); err != nil {
-		logger.Info("Max capacity is invalid")
-		return errors.New("invalid max capacity")
-	}
-
-	newCapacity, err := resource.ParseQuantity(r.Spec.Capacity)
-	if err != nil {
-		logger.Info("Capacity is invalid")
-		return errors.New("invalid new capacity")
-	}
-
-	maxCapacity, err := resource.ParseQuantity(r.Spec.Policy.MaximumCapacityOfDisk)
-	if err != nil {
-		logger.Info("Max capacity is invalid")
-		return errors.New("invalid max capacity")
-	}
-
-	if maxCapacity.CmpInt64(0) != 0 && maxCapacity.Cmp(newCapacity) == -1 {
+	if r.Spec.Policy.MaximumCapacityOfDisk.CmpInt64(0) != 0 && r.Spec.Policy.MaximumCapacityOfDisk.Cmp(r.Spec.Capacity) == -1 {
 		logger.Info("Capacity is more then max")
 		return errors.New("invalid new capacity, more then max")
 	}
 
+	if err := validateMountPattern(r.Spec.MountPointPattern); err != nil {
+		logger.Info("Invalid mount pattern", "error", err.Error())
+		return err
+	}
+
 	if old != nil {
 		oldDC, ok := old.(*DiskConfig)
 		if !ok {
-			err = errors.New("invalid old object")
+			err := errors.New("invalid old object")
 			logger.Error(err, "this should not happen")
 			return err
 		}
 
 		if !reflect.DeepEqual(oldDC.Spec.AccessModes, r.Spec.AccessModes) {
-			// TODO count PVCs by label, if 0 mode is ok to change
 			logger.Info("AccessModes is immutable")
 			return errors.New("access modes is immutable field")
 		}
@@ -116,20 +105,11 @@ func (r *DiskConfig) validate(old runtime.Object) error {
 		}
 
 		if oldDC.Spec.MountPointPattern != r.Spec.MountPointPattern {
-			// TODO count PVCs by label, if 0 mode is ok to change
 			logger.Info("Mount pattern of StorageClass is immutable")
 			return errors.New("mount point pattern is immutable field")
 		}
 
-		var oldCapacity resource.Quantity
-		oldCapacity, err = resource.ParseQuantity(oldDC.Spec.Capacity)
-		if err != nil {
-			err = errors.New("invalid old capacity")
-			logger.Error(err, "this should not happen")
-			return err
-		}
-
-		if oldCapacity.CmpInt64(0) != 0 && oldCapacity.Cmp(newCapacity) == 1 {
+		if oldDC.Spec.Capacity.CmpInt64(0) != 0 && oldDC.Spec.Capacity.Cmp(r.Spec.Capacity) == 1 {
 			logger.Info("Shrinking disk is not supported")
 			return errors.New("shrinking disk is not supported")
 		}
@@ -146,7 +126,7 @@ func (r *DiskConfig) validate(old runtime.Object) error {
 	logger.Info("Fetch StorageClass...")
 
 	sc := storagev1.StorageClass{}
-	if err = diskConfigWebhookDependencies.client.Get(ctx, types.NamespacedName{Name: r.Spec.StorageClassName}, &sc); err != nil {
+	if err := diskConfigWebhookDependencies.client.Get(ctx, types.NamespacedName{Name: r.Spec.StorageClassName}, &sc); err != nil {
 		if apierrors.IsNotFound(err) {
 			logger.Info("StorageClass not found")
 		} else {
@@ -169,7 +149,7 @@ func (r *DiskConfig) validate(old runtime.Object) error {
 
 	valid, err := driver.IsStorageClassValid(&sc)
 	if err != nil {
-		logger.Error(err, "Failed to call driver")
+		logger.Error(err, "Failed to call driver", "method", "IsStorageClassValid")
 		return fmt.Errorf("failed to call driver: %w", err)
 	} else if !valid {
 		logger.Info("Invalid StorageClass", "error", err.Error())
@@ -185,3 +165,15 @@ func (r *DiskConfig) ValidateDelete() error {
 
 	return nil
 }
+
+func validateMountPattern(pattern string) error {
+	if strings.Count(pattern, "%d") > 1 {
+		return errors.New("invalid mount pattern, only one %d allowed")
+	}
+
+	if reservedCharacters.MatchString(pattern) {
+		return errors.New("invalid mount pattern, contains reserved characters")
+	}
+
+	return nil
+}
diff --git a/api/v1/webhook_suite_test.go b/api/v1/webhook_suite_test.go
@@ -59,7 +59,7 @@ func TestAPIs(t *testing.T) {
 var _ = BeforeSuite(func() {
 	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
 
-	ctx, cancel = context.WithCancel(context.TODO())
+	ctx, cancel = context.WithCancel(context.Background())
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{

diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/discoblocks.ondat.io_diskconfigs.yaml b/config/crd/bases/discoblocks.ondat.io_diskconfigs.yaml
@@ -44,23 +44,27 @@ spec:
                   type: string
                 type: array
               availabilityMode:
-                default: Multiple
+                default: ReadWriteOnce
                 description: AvailabilityMode defines the desired number of instances.
                 enum:
-                - Singleton
-                - Multiple
+                - ReadWriteSame
+                - ReadWriteOnce
                 type: string
               capacity:
+                anyOf:
+                - type: integer
+                - type: string
                 default: 1Gi
                 description: Capacity represents the desired capacity of the underlying
                   volume.
-                pattern: ^(\d+)(m|Mi|g|Gi|t|Ti|p|Pi)$
-                type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
               mountPointPattern:
                 default: /media/discoblocks/<name>-%d
-                description: MountPointPattern is the mount point of the disk. %d
+                description: 'MountPointPattern is the mount point of the disk. %d
                   is optional and represents disk number in order. Will be automatically
-                  appended for second drive if missing.
+                  appended for second drive if missing. Reserved characters: ><|:&.+*!?^$()[]{},
+                  only 1 %d allowed.'
                 pattern: ^/(.*)
                 type: string
               nodeSelector:
@@ -118,17 +122,33 @@ spec:
               policy:
                 description: Policy contains the disk scale policies.
                 properties:
+                  coolDown:
+                    default: 5m
+                    description: CoolDown defines temporary pause of scaling.
+                    type: string
+                  extendCapacity:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    default: 1Gi
+                    description: ExtendCapacity represents the capacity to extend
+                      with.
+                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                    x-kubernetes-int-or-string: true
                   maximumCapacityOfDisk:
-                    default: 100Gi
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    default: 1000Gi
                     description: MaximumCapacityOfDisks defines maximum capacity of
                       a disk.
-                    pattern: ^(\d+)(m|Mi|g|Gi|t|Ti|p|Pi)$
-                    type: string
+                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                    x-kubernetes-int-or-string: true
                   maximumNumberOfDisks:
-                    default: 10
+                    default: 1
                     description: MaximumCapacityOfDisks defines maximum number of
                       a disks.
-                    maximum: 1000
+                    maximum: 150
                     minimum: 1
                     type: integer
                   pause:

diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml
@@ -79,7 +79,7 @@ patches:
         path: "/spec/template/spec/containers/0/env/0"
         value:
           name: SUPPORTED_CSI_DRIVERS
-          value: "ebs.csi.aws.com"
+          value: "ebs.csi.aws.com,csi.storageos.com"
     target:
       kind: Deployment
       namespace: system

diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -60,7 +60,7 @@ spec:
         resources:
           limits:
             cpu: 500m
-            memory: 256Mi
+            memory: 512Mi
           requests:
             cpu: 10m
             memory: 64Mi