Skip to content

Commit

Permalink
fix: configure Exec command to use ephemeral containers with less res…
Browse files Browse the repository at this point in the history 
…ources
  • Loading branch information
@ravilock
ravilock committed Feb 18, 2025
1 parent 7eba6d0 commit 3ede9b4
Show file tree
Hide file tree
Showing 2 changed files with 315 additions and 9 deletions.
70 changes: 65 additions & 5 deletions internal/pkg/rpaas/k8s.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,10 @@ const (
nginxContainerName = "nginx"
)

var _ RpaasManager = &k8sRpaasManager{}
var nameSuffixFunc = utilrand.String
var (
_ RpaasManager = &k8sRpaasManager{}
nameSuffixFunc = utilrand.String
)

var podAllowedReasonsToFail = map[string]bool{
"shutdown": true,
Expand Down Expand Up @@ -311,10 +313,14 @@ func getContainerStatusByName(pod *corev1.Pod, containerName string) *corev1.Con
}

func (m *k8sRpaasManager) Exec(ctx context.Context, instanceName string, args ExecArgs) error {
instance, err := m.checkPodOnInstance(ctx, instanceName, &args.CommonTerminalArgs)
instance, execContainerName, status, err := m.execOnPodWithContainerStatus(ctx, &args, instanceName)
if err != nil {
return err
}
if status.State.Terminated != nil {
req := m.kcs.CoreV1().Pods(instance.Namespace).GetLogs(args.Pod, &corev1.PodLogOptions{Container: execContainerName})
return logs.DefaultConsumeRequest(req, args.Stdout)
}

req := m.kcs.
CoreV1().
Expand All @@ -325,7 +331,7 @@ func (m *k8sRpaasManager) Exec(ctx context.Context, instanceName string, args Ex
Namespace(instance.Namespace).
SubResource("exec").
VersionedParams(&corev1.PodExecOptions{
Container: args.Container,
Container: execContainerName,
Command: args.Command,
Stdin: args.Stdin != nil,
Stdout: true,
Expand All @@ -341,6 +347,60 @@ func (m *k8sRpaasManager) Exec(ctx context.Context, instanceName string, args Ex
return executorStream(args.CommonTerminalArgs, executor, ctx)
}

func (m *k8sRpaasManager) execOnPodWithContainerStatus(ctx context.Context, args *ExecArgs, instanceName string) (*v1alpha1.RpaasInstance, string, *corev1.ContainerStatus, error) {
image := config.Get().DebugImage
if image == "" {
return nil, "", nil, ValidationError{Msg: "No debug image configured"}
}
instance, err := m.checkPodOnInstance(ctx, instanceName, &args.CommonTerminalArgs)
if err != nil {
return nil, "", nil, err
}
execContainerName, err := m.generateExecContainer(ctx, args, image, instance)
if err != nil {
return nil, "", nil, err
}
pod, err := m.waitForContainer(ctx, instance.Namespace, args.Pod, execContainerName)
if err != nil {
return nil, "", nil, err
}
status := getContainerStatusByName(pod, execContainerName)
if status == nil {
return nil, "", nil, fmt.Errorf("error getting container status of container name %q: %+v", execContainerName, err)
}
return instance, execContainerName, status, nil
}

func (m *k8sRpaasManager) generateExecContainer(ctx context.Context, args *ExecArgs, image string, instance *v1alpha1.RpaasInstance) (string, error) {
instancePod := corev1.Pod{}
err := m.cli.Get(ctx, types.NamespacedName{Name: args.Pod, Namespace: instance.Namespace}, &instancePod)
if err != nil {
return "", err
}
execContainerName := fmt.Sprintf("exec-%s", nameSuffixFunc(5))
execContainer := &corev1.EphemeralContainer{
EphemeralContainerCommon: corev1.EphemeralContainerCommon{
Name: execContainerName,
Command: args.Command,
Image: image,
ImagePullPolicy: corev1.PullIfNotPresent,
Stdin: args.Stdin != nil,
TTY: args.TTY,
VolumeMounts: []corev1.VolumeMount{
{
Name: "nginx-config",
MountPath: "/etc/nginx",
ReadOnly: true,
},
},
}, TargetContainerName: args.Container,
}
instancePodWithExec := instancePod.DeepCopy()
instancePodWithExec.Spec.EphemeralContainers = append(instancePod.Spec.EphemeralContainers, *execContainer)
err = m.patchEphemeralContainers(ctx, instancePodWithExec, instancePod)
return execContainerName, err
}

func executorStream(args CommonTerminalArgs, executor remotecommand.Executor, ctx context.Context) error {
var tsq remotecommand.TerminalSizeQueue
if args.TerminalWidth != uint16(0) && args.TerminalHeight != uint16(0) {
Expand Down Expand Up @@ -682,7 +742,7 @@ func (m *k8sRpaasManager) GetCertificates(ctx context.Context, instanceName stri
nginx = &nginxv1alpha1.Nginx{}
}

var certMap = make(map[string]clientTypes.CertificateInfo)
certMap := make(map[string]clientTypes.CertificateInfo)
allTLS := append([]nginxv1alpha1.NginxTLS{}, instance.Spec.TLS...)
allTLS = append(allTLS, nginx.Spec.TLS...)

Expand Down
254 changes: 250 additions & 4 deletions internal/pkg/rpaas/k8s_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4364,7 +4364,8 @@ func Test_k8sRpaasManager_GetInstanceInfo(t *testing.T) {
Name: "my-instance-service",
Namespace: "rpaasv2",
Annotations: map[string]string{
"external-dns.alpha.kubernetes.io/hostname": "my-instance.zone1.tld"},
"external-dns.alpha.kubernetes.io/hostname": "my-instance.zone1.tld",
},
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeLoadBalancer,
Expand Down Expand Up @@ -5440,7 +5441,7 @@ func Test_k8sRpaasManager_Debug(t *testing.T) {
require.NoError(t, err)
expectedEphemerals := []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{
Name: "debugger-1",
Image: "tsuru/debug-image",
Image: "tsuru/netshoot",
ImagePullPolicy: corev1.PullIfNotPresent,
Stdin: true,
VolumeMounts: []corev1.VolumeMount{{Name: "nginx-config", MountPath: "/etc/nginx", ReadOnly: true}},
Expand All @@ -5467,7 +5468,7 @@ func Test_k8sRpaasManager_Debug(t *testing.T) {
require.NoError(t, err)
expectedEphemerals := []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{
Name: "debugger-2",
Image: "tsuru/debug-image",
Image: "tsuru/netshoot",
ImagePullPolicy: corev1.PullIfNotPresent,
Stdin: true,
VolumeMounts: []corev1.VolumeMount{{Name: "nginx-config", MountPath: "/etc/nginx", ReadOnly: true}},
Expand All @@ -5480,7 +5481,7 @@ func Test_k8sRpaasManager_Debug(t *testing.T) {
for _, tt := range testCases {
cfg := config.Get()
defer func() { config.Set(cfg) }()
config.Set(config.RpaasConfig{DebugImage: "tsuru/debug-image"})
config.Set(config.RpaasConfig{DebugImage: "tsuru/netshoot"})
t.Run(tt.name, func(t *testing.T) {
var wg sync.WaitGroup
manager := &k8sRpaasManager{cli: fake.NewClientBuilder().WithScheme(newScheme()).WithRuntimeObjects(resources...).Build()}
Expand Down Expand Up @@ -5511,5 +5512,250 @@ func Test_k8sRpaasManager_Debug(t *testing.T) {
}
})
}
}

func Test_k8sRpaasManager_Exec(t *testing.T) {
defer func(old func(int) string) { nameSuffixFunc = old }(nameSuffixFunc)
var suffixCounter int
nameSuffixFunc = func(int) string {
suffixCounter++
return fmt.Sprint(suffixCounter)
}

instance1 := newEmptyRpaasInstance()
instance1.ObjectMeta.Name = "instance1"
instance2 := newEmptyRpaasInstance()
instance2.ObjectMeta.Name = "instance2"
instance3 := newEmptyRpaasInstance()
instance3.ObjectMeta.Name = "instance3"
instance4 := newEmptyRpaasInstance()
instance4.ObjectMeta.Name = "instance4"
instance5 := newEmptyRpaasInstance()
instance5.ObjectMeta.Name = "instance5"

nginx1 := &nginxv1alpha1.Nginx{
ObjectMeta: instance1.ObjectMeta,
Status: nginxv1alpha1.NginxStatus{
PodSelector: "nginx.tsuru.io/app=nginx,nginx.tsuru.io/resource-name=instance1",
},
}
nginx2 := &nginxv1alpha1.Nginx{
ObjectMeta: instance2.ObjectMeta,
Status: nginxv1alpha1.NginxStatus{
PodSelector: "nginx.tsuru.io/app=nginx,nginx.tsuru.io/resource-name=instance2",
},
}
nginx3 := &nginxv1alpha1.Nginx{
ObjectMeta: instance3.ObjectMeta,
Status: nginxv1alpha1.NginxStatus{
PodSelector: "nginx.tsuru.io/app=nginx,nginx.tsuru.io/resource-name=instance3",
},
}
nginx4 := &nginxv1alpha1.Nginx{
ObjectMeta: instance5.ObjectMeta,
Status: nginxv1alpha1.NginxStatus{
PodSelector: "nginx.tsuru.io/app=nginx,nginx.tsuru.io/resource-name=instance5",
},
}

pod1 := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
Namespace: instance2.Namespace,
Labels: map[string]string{
"nginx.tsuru.io/app": "nginx",
"nginx.tsuru.io/resource-name": "instance2",
},
UID: types.UID("pod1-uid"),
},
Status: corev1.PodStatus{
PodIP: "10.0.0.1",
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{
{
Name: "nginx",
Ready: true,
},
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{Name: "nginx"},
},
},
}
pod2 := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod2",
Namespace: instance2.Namespace,
Labels: map[string]string{
"nginx.tsuru.io/app": "nginx",
"nginx.tsuru.io/resource-name": "instance2",
},
UID: types.UID("pod2-uid"),
},
Status: corev1.PodStatus{
PodIP: "10.0.0.2",
Phase: corev1.PodPending,
ContainerStatuses: []corev1.ContainerStatus{
{
Name: "nginx",
Ready: false,
},
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{Name: "nginx"},
},
},
}
pod4 := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod4",
Namespace: instance5.Namespace,
Labels: map[string]string{
"nginx.tsuru.io/app": "nginx",
"nginx.tsuru.io/resource-name": "instance5",
},
UID: types.UID("pod4-uid"),
},
Status: corev1.PodStatus{
PodIP: "10.0.0.9",
ContainerStatuses: []corev1.ContainerStatus{
{
Ready: false,
},
},
},
}

resources := []runtime.Object{instance1, instance2, instance3, instance4, instance5, nginx1, nginx2, nginx3, nginx4, pod1, pod2, pod4}
staticTimeNow := metav1.Now()

testCases := []struct {
name string
instance string
args ExecArgs
assertion func(*testing.T, error, *k8sRpaasManager, *v1alpha1.RpaasInstance, string, *corev1.ContainerStatus)
pods func() []corev1.Pod
}{
{
name: "no debug image configured",
instance: "instance1",
pods: func() []corev1.Pod { return []corev1.Pod{} },
assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) {
assert.Error(t, err)
assert.EqualError(t, err, "No debug image configured")
},
},
{
name: "no pod running for debug",
instance: "instance1",
pods: func() []corev1.Pod { return []corev1.Pod{} },
assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) {
assert.Error(t, err)
assert.EqualError(t, err, "no pod running found in instance instance1")
},
},
{
name: "debug on invalid pod",
instance: "instance1",
args: ExecArgs{CommonTerminalArgs: CommonTerminalArgs{Pod: "pod1"}},
pods: func() []corev1.Pod { return []corev1.Pod{} },
assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) {
assert.Error(t, err)
assert.EqualError(t, err, "no such pod pod1 in instance instance1")
},
},
{
name: "run debug on pod1 with default image",
instance: "instance2",
args: ExecArgs{CommonTerminalArgs: CommonTerminalArgs{Pod: "pod1", Stdin: &bytes.Buffer{}, Stdout: io.Discard, Stderr: io.Discard}},
pods: func() []corev1.Pod {
pod1Debug := pod1.DeepCopy()
pod1Debug.Spec.EphemeralContainers = append(pod1Debug.Spec.EphemeralContainers, corev1.EphemeralContainer{EphemeralContainerCommon: corev1.EphemeralContainerCommon{Name: "exec-1"}, TargetContainerName: "nginx"})
pod1Debug.Status.ContainerStatuses = append(pod1Debug.Status.EphemeralContainerStatuses, corev1.ContainerStatus{Name: "exec-1", Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: staticTimeNow}}})
return []corev1.Pod{*pod1Debug}
},
assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) {
assert.NoError(t, err)
assert.Equal(t, "exec-1", debugContainerName)
assert.Equal(t, corev1.ContainerStatus{Name: "exec-1", Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: staticTimeNow}}}, *debugContainerStatus)
instancePod := corev1.Pod{}
err = m.cli.Get(context.Background(), types.NamespacedName{Name: "pod1", Namespace: instance2.Namespace}, &instancePod)
require.NoError(t, err)
expectedEphemerals := []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{
Name: "exec-1",
Image: "tsuru/netshoot",
ImagePullPolicy: corev1.PullIfNotPresent,
Stdin: true,
VolumeMounts: []corev1.VolumeMount{{Name: "nginx-config", MountPath: "/etc/nginx", ReadOnly: true}},
}, TargetContainerName: "nginx"}}
assert.Equal(t, expectedEphemerals, instancePod.Spec.EphemeralContainers)
},
},
{
name: "run debug on random pod with running status",
instance: "instance2",
args: ExecArgs{CommonTerminalArgs: CommonTerminalArgs{Stdin: &bytes.Buffer{}, Stdout: io.Discard, Stderr: io.Discard}},
pods: func() []corev1.Pod {
pod1Debug := pod1.DeepCopy()
pod1Debug.Spec.EphemeralContainers = append(pod1Debug.Spec.EphemeralContainers, corev1.EphemeralContainer{EphemeralContainerCommon: corev1.EphemeralContainerCommon{Name: "exec-2"}, TargetContainerName: "nginx"})
pod1Debug.Status.ContainerStatuses = append(pod1Debug.Status.EphemeralContainerStatuses, corev1.ContainerStatus{Name: "exec-2", Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: staticTimeNow}}})
return []corev1.Pod{*pod1Debug}
},
assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) {
assert.NoError(t, err)
assert.Equal(t, "exec-2", debugContainerName)
assert.Equal(t, corev1.ContainerStatus{Name: "exec-2", Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: staticTimeNow}}}, *debugContainerStatus)
instancePod := corev1.Pod{}
err = m.cli.Get(context.Background(), types.NamespacedName{Name: "pod1", Namespace: instance2.Namespace}, &instancePod)
require.NoError(t, err)
expectedEphemerals := []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{
Name: "exec-2",
Image: "tsuru/netshoot",
ImagePullPolicy: corev1.PullIfNotPresent,
Stdin: true,
VolumeMounts: []corev1.VolumeMount{{Name: "nginx-config", MountPath: "/etc/nginx", ReadOnly: true}},
}, TargetContainerName: "nginx"}}
assert.Equal(t, expectedEphemerals, instancePod.Spec.EphemeralContainers)
},
},
}

for _, tt := range testCases {
cfg := config.Get()
defer func() { config.Set(cfg) }()
config.Set(config.RpaasConfig{DebugImage: "tsuru/netshoot"})
t.Run(tt.name, func(t *testing.T) {
var wg sync.WaitGroup
manager := &k8sRpaasManager{cli: fake.NewClientBuilder().WithScheme(newScheme()).WithRuntimeObjects(resources...).Build()}
watcher := watch.NewFake()
kcs := k8sclient.NewSimpleClientset()
kcs.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(watcher, nil))
manager.kcs = kcs
if tt.name == "no debug image configured" {
config.Set(config.RpaasConfig{})
}
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
wg.Add(1)
go func() {
defer wg.Done()
defer watcher.Stop()
for _, pod := range tt.pods() {
watcher.Modify(&pod)
time.Sleep(1000 * time.Millisecond)
}
}()
rpaasInstance, debugContainerName, debugContainerStatus, err := manager.execOnPodWithContainerStatus(ctx, &tt.args, tt.instance)
wg.Wait()
if err != nil {
tt.assertion(t, err, manager, nil, "", nil)
} else {
tt.assertion(t, err, manager, rpaasInstance, debugContainerName, debugContainerStatus)
}
})
}
}

0 comments on commit 3ede9b4

Please sign in to comment.