diff --git a/internal/pkg/rpaas/k8s.go b/internal/pkg/rpaas/k8s.go index 4ce163c6..b51ba90a 100644 --- a/internal/pkg/rpaas/k8s.go +++ b/internal/pkg/rpaas/k8s.go @@ -80,8 +80,10 @@ const ( nginxContainerName = "nginx" ) -var _ RpaasManager = &k8sRpaasManager{} -var nameSuffixFunc = utilrand.String +var ( + _ RpaasManager = &k8sRpaasManager{} + nameSuffixFunc = utilrand.String +) var podAllowedReasonsToFail = map[string]bool{ "shutdown": true, @@ -311,10 +313,14 @@ func getContainerStatusByName(pod *corev1.Pod, containerName string) *corev1.Con } func (m *k8sRpaasManager) Exec(ctx context.Context, instanceName string, args ExecArgs) error { - instance, err := m.checkPodOnInstance(ctx, instanceName, &args.CommonTerminalArgs) + instance, execContainerName, status, err := m.execOnPodWithContainerStatus(ctx, &args, instanceName) if err != nil { return err } + if status.State.Terminated != nil { + req := m.kcs.CoreV1().Pods(instance.Namespace).GetLogs(args.Pod, &corev1.PodLogOptions{Container: execContainerName}) + return logs.DefaultConsumeRequest(req, args.Stdout) + } req := m.kcs. CoreV1(). @@ -325,7 +331,7 @@ func (m *k8sRpaasManager) Exec(ctx context.Context, instanceName string, args Ex Namespace(instance.Namespace). SubResource("exec"). VersionedParams(&corev1.PodExecOptions{ - Container: args.Container, + Container: execContainerName, Command: args.Command, Stdin: args.Stdin != nil, Stdout: true, @@ -341,6 +347,60 @@ func (m *k8sRpaasManager) Exec(ctx context.Context, instanceName string, args Ex return executorStream(args.CommonTerminalArgs, executor, ctx) } +func (m *k8sRpaasManager) execOnPodWithContainerStatus(ctx context.Context, args *ExecArgs, instanceName string) (*v1alpha1.RpaasInstance, string, *corev1.ContainerStatus, error) { + image := config.Get().DebugImage + if image == "" { + return nil, "", nil, ValidationError{Msg: "No debug image configured"} + } + instance, err := m.checkPodOnInstance(ctx, instanceName, &args.CommonTerminalArgs) + if err != nil { + return nil, "", nil, err + } + execContainerName, err := m.generateExecContainer(ctx, args, image, instance) + if err != nil { + return nil, "", nil, err + } + pod, err := m.waitForContainer(ctx, instance.Namespace, args.Pod, execContainerName) + if err != nil { + return nil, "", nil, err + } + status := getContainerStatusByName(pod, execContainerName) + if status == nil { + return nil, "", nil, fmt.Errorf("error getting container status of container name %q: %+v", execContainerName, err) + } + return instance, execContainerName, status, nil +} + +func (m *k8sRpaasManager) generateExecContainer(ctx context.Context, args *ExecArgs, image string, instance *v1alpha1.RpaasInstance) (string, error) { + instancePod := corev1.Pod{} + err := m.cli.Get(ctx, types.NamespacedName{Name: args.Pod, Namespace: instance.Namespace}, &instancePod) + if err != nil { + return "", err + } + execContainerName := fmt.Sprintf("exec-%s", nameSuffixFunc(5)) + execContainer := &corev1.EphemeralContainer{ + EphemeralContainerCommon: corev1.EphemeralContainerCommon{ + Name: execContainerName, + Command: args.Command, + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Stdin: args.Stdin != nil, + TTY: args.TTY, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "nginx-config", + MountPath: "/etc/nginx", + ReadOnly: true, + }, + }, + }, TargetContainerName: args.Container, + } + instancePodWithExec := instancePod.DeepCopy() + instancePodWithExec.Spec.EphemeralContainers = append(instancePod.Spec.EphemeralContainers, *execContainer) + err = m.patchEphemeralContainers(ctx, instancePodWithExec, instancePod) + return execContainerName, err +} + func executorStream(args CommonTerminalArgs, executor remotecommand.Executor, ctx context.Context) error { var tsq remotecommand.TerminalSizeQueue if args.TerminalWidth != uint16(0) && args.TerminalHeight != uint16(0) { @@ -682,7 +742,7 @@ func (m *k8sRpaasManager) GetCertificates(ctx context.Context, instanceName stri nginx = &nginxv1alpha1.Nginx{} } - var certMap = make(map[string]clientTypes.CertificateInfo) + certMap := make(map[string]clientTypes.CertificateInfo) allTLS := append([]nginxv1alpha1.NginxTLS{}, instance.Spec.TLS...) allTLS = append(allTLS, nginx.Spec.TLS...) diff --git a/internal/pkg/rpaas/k8s_test.go b/internal/pkg/rpaas/k8s_test.go index 1e4b40dc..e9ed56ac 100644 --- a/internal/pkg/rpaas/k8s_test.go +++ b/internal/pkg/rpaas/k8s_test.go @@ -4364,7 +4364,8 @@ func Test_k8sRpaasManager_GetInstanceInfo(t *testing.T) { Name: "my-instance-service", Namespace: "rpaasv2", Annotations: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "my-instance.zone1.tld"}, + "external-dns.alpha.kubernetes.io/hostname": "my-instance.zone1.tld", + }, }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeLoadBalancer, @@ -5440,7 +5441,7 @@ func Test_k8sRpaasManager_Debug(t *testing.T) { require.NoError(t, err) expectedEphemerals := []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{ Name: "debugger-1", - Image: "tsuru/debug-image", + Image: "tsuru/netshoot", ImagePullPolicy: corev1.PullIfNotPresent, Stdin: true, VolumeMounts: []corev1.VolumeMount{{Name: "nginx-config", MountPath: "/etc/nginx", ReadOnly: true}}, @@ -5467,7 +5468,7 @@ func Test_k8sRpaasManager_Debug(t *testing.T) { require.NoError(t, err) expectedEphemerals := []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{ Name: "debugger-2", - Image: "tsuru/debug-image", + Image: "tsuru/netshoot", ImagePullPolicy: corev1.PullIfNotPresent, Stdin: true, VolumeMounts: []corev1.VolumeMount{{Name: "nginx-config", MountPath: "/etc/nginx", ReadOnly: true}}, @@ -5480,7 +5481,7 @@ func Test_k8sRpaasManager_Debug(t *testing.T) { for _, tt := range testCases { cfg := config.Get() defer func() { config.Set(cfg) }() - config.Set(config.RpaasConfig{DebugImage: "tsuru/debug-image"}) + config.Set(config.RpaasConfig{DebugImage: "tsuru/netshoot"}) t.Run(tt.name, func(t *testing.T) { var wg sync.WaitGroup manager := &k8sRpaasManager{cli: fake.NewClientBuilder().WithScheme(newScheme()).WithRuntimeObjects(resources...).Build()} @@ -5511,5 +5512,250 @@ func Test_k8sRpaasManager_Debug(t *testing.T) { } }) } +} + +func Test_k8sRpaasManager_Exec(t *testing.T) { + defer func(old func(int) string) { nameSuffixFunc = old }(nameSuffixFunc) + var suffixCounter int + nameSuffixFunc = func(int) string { + suffixCounter++ + return fmt.Sprint(suffixCounter) + } + + instance1 := newEmptyRpaasInstance() + instance1.ObjectMeta.Name = "instance1" + instance2 := newEmptyRpaasInstance() + instance2.ObjectMeta.Name = "instance2" + instance3 := newEmptyRpaasInstance() + instance3.ObjectMeta.Name = "instance3" + instance4 := newEmptyRpaasInstance() + instance4.ObjectMeta.Name = "instance4" + instance5 := newEmptyRpaasInstance() + instance5.ObjectMeta.Name = "instance5" + + nginx1 := &nginxv1alpha1.Nginx{ + ObjectMeta: instance1.ObjectMeta, + Status: nginxv1alpha1.NginxStatus{ + PodSelector: "nginx.tsuru.io/app=nginx,nginx.tsuru.io/resource-name=instance1", + }, + } + nginx2 := &nginxv1alpha1.Nginx{ + ObjectMeta: instance2.ObjectMeta, + Status: nginxv1alpha1.NginxStatus{ + PodSelector: "nginx.tsuru.io/app=nginx,nginx.tsuru.io/resource-name=instance2", + }, + } + nginx3 := &nginxv1alpha1.Nginx{ + ObjectMeta: instance3.ObjectMeta, + Status: nginxv1alpha1.NginxStatus{ + PodSelector: "nginx.tsuru.io/app=nginx,nginx.tsuru.io/resource-name=instance3", + }, + } + nginx4 := &nginxv1alpha1.Nginx{ + ObjectMeta: instance5.ObjectMeta, + Status: nginxv1alpha1.NginxStatus{ + PodSelector: "nginx.tsuru.io/app=nginx,nginx.tsuru.io/resource-name=instance5", + }, + } + + pod1 := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: instance2.Namespace, + Labels: map[string]string{ + "nginx.tsuru.io/app": "nginx", + "nginx.tsuru.io/resource-name": "instance2", + }, + UID: types.UID("pod1-uid"), + }, + Status: corev1.PodStatus{ + PodIP: "10.0.0.1", + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "nginx", + Ready: true, + }, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "nginx"}, + }, + }, + } + pod2 := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: instance2.Namespace, + Labels: map[string]string{ + "nginx.tsuru.io/app": "nginx", + "nginx.tsuru.io/resource-name": "instance2", + }, + UID: types.UID("pod2-uid"), + }, + Status: corev1.PodStatus{ + PodIP: "10.0.0.2", + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "nginx", + Ready: false, + }, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "nginx"}, + }, + }, + } + pod4 := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod4", + Namespace: instance5.Namespace, + Labels: map[string]string{ + "nginx.tsuru.io/app": "nginx", + "nginx.tsuru.io/resource-name": "instance5", + }, + UID: types.UID("pod4-uid"), + }, + Status: corev1.PodStatus{ + PodIP: "10.0.0.9", + ContainerStatuses: []corev1.ContainerStatus{ + { + Ready: false, + }, + }, + }, + } + resources := []runtime.Object{instance1, instance2, instance3, instance4, instance5, nginx1, nginx2, nginx3, nginx4, pod1, pod2, pod4} + staticTimeNow := metav1.Now() + + testCases := []struct { + name string + instance string + args ExecArgs + assertion func(*testing.T, error, *k8sRpaasManager, *v1alpha1.RpaasInstance, string, *corev1.ContainerStatus) + pods func() []corev1.Pod + }{ + { + name: "no debug image configured", + instance: "instance1", + pods: func() []corev1.Pod { return []corev1.Pod{} }, + assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) { + assert.Error(t, err) + assert.EqualError(t, err, "No debug image configured") + }, + }, + { + name: "no pod running for debug", + instance: "instance1", + pods: func() []corev1.Pod { return []corev1.Pod{} }, + assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) { + assert.Error(t, err) + assert.EqualError(t, err, "no pod running found in instance instance1") + }, + }, + { + name: "debug on invalid pod", + instance: "instance1", + args: ExecArgs{CommonTerminalArgs: CommonTerminalArgs{Pod: "pod1"}}, + pods: func() []corev1.Pod { return []corev1.Pod{} }, + assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) { + assert.Error(t, err) + assert.EqualError(t, err, "no such pod pod1 in instance instance1") + }, + }, + { + name: "run debug on pod1 with default image", + instance: "instance2", + args: ExecArgs{CommonTerminalArgs: CommonTerminalArgs{Pod: "pod1", Stdin: &bytes.Buffer{}, Stdout: io.Discard, Stderr: io.Discard}}, + pods: func() []corev1.Pod { + pod1Debug := pod1.DeepCopy() + pod1Debug.Spec.EphemeralContainers = append(pod1Debug.Spec.EphemeralContainers, corev1.EphemeralContainer{EphemeralContainerCommon: corev1.EphemeralContainerCommon{Name: "exec-1"}, TargetContainerName: "nginx"}) + pod1Debug.Status.ContainerStatuses = append(pod1Debug.Status.EphemeralContainerStatuses, corev1.ContainerStatus{Name: "exec-1", Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: staticTimeNow}}}) + return []corev1.Pod{*pod1Debug} + }, + assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) { + assert.NoError(t, err) + assert.Equal(t, "exec-1", debugContainerName) + assert.Equal(t, corev1.ContainerStatus{Name: "exec-1", Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: staticTimeNow}}}, *debugContainerStatus) + instancePod := corev1.Pod{} + err = m.cli.Get(context.Background(), types.NamespacedName{Name: "pod1", Namespace: instance2.Namespace}, &instancePod) + require.NoError(t, err) + expectedEphemerals := []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{ + Name: "exec-1", + Image: "tsuru/netshoot", + ImagePullPolicy: corev1.PullIfNotPresent, + Stdin: true, + VolumeMounts: []corev1.VolumeMount{{Name: "nginx-config", MountPath: "/etc/nginx", ReadOnly: true}}, + }, TargetContainerName: "nginx"}} + assert.Equal(t, expectedEphemerals, instancePod.Spec.EphemeralContainers) + }, + }, + { + name: "run debug on random pod with running status", + instance: "instance2", + args: ExecArgs{CommonTerminalArgs: CommonTerminalArgs{Stdin: &bytes.Buffer{}, Stdout: io.Discard, Stderr: io.Discard}}, + pods: func() []corev1.Pod { + pod1Debug := pod1.DeepCopy() + pod1Debug.Spec.EphemeralContainers = append(pod1Debug.Spec.EphemeralContainers, corev1.EphemeralContainer{EphemeralContainerCommon: corev1.EphemeralContainerCommon{Name: "exec-2"}, TargetContainerName: "nginx"}) + pod1Debug.Status.ContainerStatuses = append(pod1Debug.Status.EphemeralContainerStatuses, corev1.ContainerStatus{Name: "exec-2", Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: staticTimeNow}}}) + return []corev1.Pod{*pod1Debug} + }, + assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) { + assert.NoError(t, err) + assert.Equal(t, "exec-2", debugContainerName) + assert.Equal(t, corev1.ContainerStatus{Name: "exec-2", Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: staticTimeNow}}}, *debugContainerStatus) + instancePod := corev1.Pod{} + err = m.cli.Get(context.Background(), types.NamespacedName{Name: "pod1", Namespace: instance2.Namespace}, &instancePod) + require.NoError(t, err) + expectedEphemerals := []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{ + Name: "exec-2", + Image: "tsuru/netshoot", + ImagePullPolicy: corev1.PullIfNotPresent, + Stdin: true, + VolumeMounts: []corev1.VolumeMount{{Name: "nginx-config", MountPath: "/etc/nginx", ReadOnly: true}}, + }, TargetContainerName: "nginx"}} + assert.Equal(t, expectedEphemerals, instancePod.Spec.EphemeralContainers) + }, + }, + } + + for _, tt := range testCases { + cfg := config.Get() + defer func() { config.Set(cfg) }() + config.Set(config.RpaasConfig{DebugImage: "tsuru/netshoot"}) + t.Run(tt.name, func(t *testing.T) { + var wg sync.WaitGroup + manager := &k8sRpaasManager{cli: fake.NewClientBuilder().WithScheme(newScheme()).WithRuntimeObjects(resources...).Build()} + watcher := watch.NewFake() + kcs := k8sclient.NewSimpleClientset() + kcs.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(watcher, nil)) + manager.kcs = kcs + if tt.name == "no debug image configured" { + config.Set(config.RpaasConfig{}) + } + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + wg.Add(1) + go func() { + defer wg.Done() + defer watcher.Stop() + for _, pod := range tt.pods() { + watcher.Modify(&pod) + time.Sleep(1000 * time.Millisecond) + } + }() + rpaasInstance, debugContainerName, debugContainerStatus, err := manager.execOnPodWithContainerStatus(ctx, &tt.args, tt.instance) + wg.Wait() + if err != nil { + tt.assertion(t, err, manager, nil, "", nil) + } else { + tt.assertion(t, err, manager, rpaasInstance, debugContainerName, debugContainerStatus) + } + }) + } }