Skip to content

Commit 2ba1566

Browse files
Ambient Code Botclaude
andcommitted
fix(control-plane): fallback to list+delete when deletecollection is forbidden
The MPP tenant SA lacks deletecollection permission on pods, secrets, serviceaccounts, and services in runner namespaces. Session cleanup silently fails and orphaned pods accumulate consuming ~4Gi each. All Delete*ByLabel methods now try deletecollection first, and on 403 Forbidden fall back to list matching resources then delete individually. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
1 parent bccc76c commit 2ba1566

File tree

1 file changed

+34
-6
lines changed
  • components/ambient-control-plane/internal/kubeclient

1 file changed

+34
-6
lines changed

components/ambient-control-plane/internal/kubeclient/kubeclient.go

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"strings"
88

99
"github.com/rs/zerolog"
10+
k8serrors "k8s.io/apimachinery/pkg/api/errors"
1011
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1112
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
1213
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -201,7 +202,7 @@ func (kc *KubeClient) DeletePod(ctx context.Context, namespace, name string, opt
201202
}
202203

203204
func (kc *KubeClient) DeletePodsByLabel(ctx context.Context, namespace, labelSelector string) error {
204-
return kc.dynamic.Resource(PodGVR).Namespace(namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: labelSelector})
205+
return kc.deleteCollectionWithFallback(ctx, PodGVR, namespace, labelSelector)
205206
}
206207

207208
// Service operations
@@ -214,7 +215,7 @@ func (kc *KubeClient) CreateService(ctx context.Context, obj *unstructured.Unstr
214215
}
215216

216217
func (kc *KubeClient) DeleteServicesByLabel(ctx context.Context, namespace, labelSelector string) error {
217-
return kc.dynamic.Resource(ServiceGVR).Namespace(namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: labelSelector})
218+
return kc.deleteCollectionWithFallback(ctx, ServiceGVR, namespace, labelSelector)
218219
}
219220

220221
// Secret operations
@@ -231,7 +232,7 @@ func (kc *KubeClient) UpdateSecret(ctx context.Context, obj *unstructured.Unstru
231232
}
232233

233234
func (kc *KubeClient) DeleteSecretsByLabel(ctx context.Context, namespace, labelSelector string) error {
234-
return kc.dynamic.Resource(SecretGVR).Namespace(namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: labelSelector})
235+
return kc.deleteCollectionWithFallback(ctx, SecretGVR, namespace, labelSelector)
235236
}
236237

237238
// ServiceAccount operations
@@ -244,7 +245,7 @@ func (kc *KubeClient) CreateServiceAccount(ctx context.Context, obj *unstructure
244245
}
245246

246247
func (kc *KubeClient) DeleteServiceAccountsByLabel(ctx context.Context, namespace, labelSelector string) error {
247-
return kc.dynamic.Resource(ServiceAccountGVR).Namespace(namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: labelSelector})
248+
return kc.deleteCollectionWithFallback(ctx, ServiceAccountGVR, namespace, labelSelector)
248249
}
249250

250251
// Role operations
@@ -257,11 +258,38 @@ func (kc *KubeClient) CreateRole(ctx context.Context, obj *unstructured.Unstruct
257258
}
258259

259260
func (kc *KubeClient) DeleteRolesByLabel(ctx context.Context, namespace, labelSelector string) error {
260-
return kc.dynamic.Resource(RoleGVR).Namespace(namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: labelSelector})
261+
return kc.deleteCollectionWithFallback(ctx, RoleGVR, namespace, labelSelector)
261262
}
262263

263264
func (kc *KubeClient) DeleteRoleBindingsByLabel(ctx context.Context, namespace, labelSelector string) error {
264-
return kc.dynamic.Resource(RoleBindingGVR).Namespace(namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: labelSelector})
265+
return kc.deleteCollectionWithFallback(ctx, RoleBindingGVR, namespace, labelSelector)
266+
}
267+
268+
func (kc *KubeClient) deleteCollectionWithFallback(ctx context.Context, gvr schema.GroupVersionResource, namespace, labelSelector string) error {
269+
err := kc.dynamic.Resource(gvr).Namespace(namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: labelSelector})
270+
if err == nil {
271+
return nil
272+
}
273+
if !k8serrors.IsForbidden(err) {
274+
return err
275+
}
276+
277+
kc.logger.Warn().Str("resource", gvr.Resource).Str("namespace", namespace).Msg("deletecollection forbidden, falling back to list+delete")
278+
279+
list, listErr := kc.dynamic.Resource(gvr).Namespace(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
280+
if listErr != nil {
281+
return fmt.Errorf("fallback list %s: %w", gvr.Resource, listErr)
282+
}
283+
284+
var lastErr error
285+
for i := range list.Items {
286+
name := list.Items[i].GetName()
287+
if delErr := kc.dynamic.Resource(gvr).Namespace(namespace).Delete(ctx, name, metav1.DeleteOptions{}); delErr != nil && !k8serrors.IsNotFound(delErr) {
288+
kc.logger.Warn().Err(delErr).Str("resource", gvr.Resource).Str("name", name).Msg("fallback delete failed")
289+
lastErr = delErr
290+
}
291+
}
292+
return lastErr
265293
}
266294

267295
func (kc *KubeClient) GetNetworkPolicy(ctx context.Context, namespace, name string) (*unstructured.Unstructured, error) {

0 commit comments

Comments
 (0)