feat: nodeclaim.spec.terminationGracePeriod, allow disruption of node…

…s with do-not-disrupt pods and blocking PDBs

Signed-off-by: wmgroot <[email protected]>

kwok/charts/crds/karpenter.sh_nodeclaims.yaml

@@ -248,6 +248,26 @@ spec:
                       - key
                     type: object
                   type: array
+                terminationGracePeriod:
+                  description: |-
+                    TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated.
+
+
+                    Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation.
+
+
+                    This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period.
+                    When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached.
+
+
+                    Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod.
+                    If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout,
+                    that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds.
+
+
+                    The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks.
+                    If left undefined, the controller will wait indefinitely for pods to be drained.
+                  type: string
               required:
                 - nodeClassRef
                 - requirements
@@ -692,6 +712,26 @@ spec:
                       - key
                     type: object
                   type: array
+                terminationGracePeriod:
+                  description: |-
+                    TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated.
+
+
+                    Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation.
+
+
+                    This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period.
+                    When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached.
+
+
+                    Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod.
+                    If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout,
+                    that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds.
+
+
+                    The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks.
+                    If left undefined, the controller will wait indefinitely for pods to be drained.
+                  type: string
               required:
                 - nodeClassRef
                 - requirements

kwok/charts/crds/karpenter.sh_nodepools.yaml

@@ -388,6 +388,26 @@ spec:
                               - key
                             type: object
                           type: array
+                        terminationGracePeriod:
+                          description: |-
+                            TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated.
+
+
+                            Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation.
+
+
+                            This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period.
+                            When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached.
+
+
+                            Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod.
+                            If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout,
+                            that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds.
+
+
+                            The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks.
+                            If left undefined, the controller will wait indefinitely for pods to be drained.
+                          type: string
                       required:
                         - nodeClassRef
                         - requirements
@@ -970,6 +990,26 @@ spec:
                               - key
                             type: object
                           type: array
+                        terminationGracePeriod:
+                          description: |-
+                            TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated.
+
+
+                            Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation.
+
+
+                            This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period.
+                            When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached.
+
+
+                            Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod.
+                            If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout,
+                            that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds.
+
+
+                            The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks.
+                            If left undefined, the controller will wait indefinitely for pods to be drained.
+                          type: string
                       required:
                         - nodeClassRef
                         - requirements

pkg/apis/crds/karpenter.sh_nodeclaims.yaml

@@ -246,6 +246,26 @@ spec:
                       - key
                     type: object
                   type: array
+                terminationGracePeriod:
+                  description: |-
+                    TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated.
+
+
+                    Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation.
+
+
+                    This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period.
+                    When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached.
+
+
+                    Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod.
+                    If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout,
+                    that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds.
+
+
+                    The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks.
+                    If left undefined, the controller will wait indefinitely for pods to be drained.
+                  type: string
               required:
                 - nodeClassRef
                 - requirements
@@ -688,6 +708,26 @@ spec:
                       - key
                     type: object
                   type: array
+                terminationGracePeriod:
+                  description: |-
+                    TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated.
+
+
+                    Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation.
+
+
+                    This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period.
+                    When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached.
+
+
+                    Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod.
+                    If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout,
+                    that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds.
+
+
+                    The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks.
+                    If left undefined, the controller will wait indefinitely for pods to be drained.
+                  type: string
               required:
                 - nodeClassRef
                 - requirements

pkg/apis/crds/karpenter.sh_nodepools.yaml

@@ -386,6 +386,26 @@ spec:
                               - key
                             type: object
                           type: array
+                        terminationGracePeriod:
+                          description: |-
+                            TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated.
+
+
+                            Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation.
+
+
+                            This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period.
+                            When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached.
+
+
+                            Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod.
+                            If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout,
+                            that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds.
+
+
+                            The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks.
+                            If left undefined, the controller will wait indefinitely for pods to be drained.
+                          type: string
                       required:
                         - nodeClassRef
                         - requirements
@@ -966,6 +986,26 @@ spec:
                               - key
                             type: object
                           type: array
+                        terminationGracePeriod:
+                          description: |-
+                            TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated.
+
+
+                            Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation.
+
+
+                            This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period.
+                            When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached.
+
+
+                            Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod.
+                            If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout,
+                            that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds.
+
+
+                            The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks.
+                            If left undefined, the controller will wait indefinitely for pods to be drained.
+                          type: string
                       required:
                         - nodeClassRef
                         - requirements

pkg/apis/v1/labels.go

@@ -44,12 +44,13 @@ const (
 
 // Karpenter specific annotations
 const (
-	DoNotDisruptAnnotationKey          = apis.Group + "/do-not-disrupt"
-	ProviderCompatabilityAnnotationKey = apis.CompatabilityGroup + "/provider"
-	ManagedByAnnotationKey             = apis.Group + "/managed-by"
-	NodePoolHashAnnotationKey          = apis.Group + "/nodepool-hash"
-	NodePoolHashVersionAnnotationKey   = apis.Group + "/nodepool-hash-version"
-	KubeletCompatabilityAnnotationKey  = apis.CompatabilityGroup + "/v1beta1-kubelet-conversion"
+	DoNotDisruptAnnotationKey             = apis.Group + "/do-not-disrupt"
+	ProviderCompatabilityAnnotationKey    = apis.CompatabilityGroup + "/provider"
+	ManagedByAnnotationKey                = apis.Group + "/managed-by"
+	NodePoolHashAnnotationKey             = apis.Group + "/nodepool-hash"
+	NodePoolHashVersionAnnotationKey      = apis.Group + "/nodepool-hash-version"
+	KubeletCompatabilityAnnotationKey     = apis.CompatabilityGroup + "/v1beta1-kubelet-conversion"
+	NodeTerminationTimestampAnnotationKey = apis.Group + "/node-termination-timestamp"
 )
 
 // Karpenter specific finalizers

pkg/apis/v1/nodeclaim.go

@@ -46,6 +46,21 @@ type NodeClaimSpec struct {
 	// NodeClassRef is a reference to an object that defines provider specific configuration
 	// +required
 	NodeClassRef *NodeClassReference `json:"nodeClassRef"`
+	// TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated.
+	//
+	// Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation.
+	//
+	// This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period.
+	// When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached.
+	//
+	// Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod.
+	// If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout,
+	// that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds.
+	//
+	// The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks.
+	// If left undefined, the controller will wait indefinitely for pods to be drained.
+	// +optional
+	TerminationGracePeriod *metav1.Duration `json:"terminationGracePeriod,omitempty"`
 }
 
 // A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values

pkg/apis/v1beta1/labels.go

@@ -44,11 +44,12 @@ const (
 
 // Karpenter specific annotations
 const (
-	DoNotDisruptAnnotationKey          = apis.Group + "/do-not-disrupt"
-	ProviderCompatabilityAnnotationKey = apis.CompatabilityGroup + "/provider"
-	ManagedByAnnotationKey             = apis.Group + "/managed-by"
-	NodePoolHashAnnotationKey          = apis.Group + "/nodepool-hash"
-	NodePoolHashVersionAnnotationKey   = apis.Group + "/nodepool-hash-version"
+	DoNotDisruptAnnotationKey             = apis.Group + "/do-not-disrupt"
+	ProviderCompatabilityAnnotationKey    = apis.CompatabilityGroup + "/provider"
+	ManagedByAnnotationKey                = apis.Group + "/managed-by"
+	NodePoolHashAnnotationKey             = apis.Group + "/nodepool-hash"
+	NodePoolHashVersionAnnotationKey      = apis.Group + "/nodepool-hash-version"
+	NodeTerminationTimestampAnnotationKey = apis.Group + "/node-termination-timestamp"
 )
 
 // Karpenter specific finalizers

pkg/apis/v1beta1/nodeclaim.go

@@ -53,6 +53,21 @@ type NodeClaimSpec struct {
 	// NodeClassRef is a reference to an object that defines provider specific configuration
 	// +required
 	NodeClassRef *NodeClassReference `json:"nodeClassRef"`
+	// TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated.
+	//
+	// Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation.
+	//
+	// This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period.
+	// When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached.
+	//
+	// Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod.
+	// If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout,
+	// that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds.
+	//
+	// The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks.
+	// If left undefined, the controller will wait indefinitely for pods to be drained.
+	// +optional
+	TerminationGracePeriod *metav1.Duration `json:"terminationGracePeriod,omitempty"`
 }
 
 // A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values