minikube: instrument tests, to allow debugging failures

New run_minikube() helper, modeled after run_podman(). Echoes each command being run and its output. On failure, runs minikube logs. Addresses (does not close) #21931 which is hitting us hard in CI. Probably quay flakes, but it's impossible to tell without logs. Also: bug fix: one "run podman" fixed to run_podman Signed-off-by: Ed Santiago <[email protected]>
containers · Mar 19, 2024 · aa5c487 · aa5c487
1 parent 8a643c2
commit aa5c487
Show file tree

Hide file tree

Showing 2 changed files with 91 additions and 80 deletions.
diff --git a/test/minikube/001-kube.bats b/test/minikube/001-kube.bats
@@ -9,11 +9,9 @@ load helpers.bash
 # BEGIN tests
 
 @test "minikube - check cluster is up" {
-    run minikube kubectl get nodes
-    assert "$status" -eq 0 "get status of nodes"
+    run_minikube kubectl get nodes
     assert "$output" =~ "Ready"
-    run minikube kubectl get pods
-    assert "$status" -eq 0 "get pods in the default namespace"
+    run_minikube kubectl get pods
     assert "$output" == "No resources found in default namespace."
 }
 
@@ -25,14 +23,11 @@ load helpers.bash
 
     # deploy to the minikube cluster
     project="ctr-ns"
-    run minikube kubectl create namespace $project
-    assert "$status" -eq 0 "create new namespace $project"
-    run minikube kubectl -- apply -f $fname
-    assert "$status" -eq 0 "deploy $fname to the cluster"
+    run_minikube kubectl create namespace $project
+    run_minikube kubectl -- apply -f $fname
     assert "$output" == "pod/$cname-pod created"
     wait_for_pods_to_start
-    run minikube kubectl delete namespace $project
-    assert $status -eq 0 "delete namespace $project"
+    run_minikube kubectl delete namespace $project
 }
 
 @test "minikube - deploy generated pod yaml to minikube" {
@@ -48,14 +43,11 @@ load helpers.bash
 
     # deploy to the minikube cluster
     project="pod-ns"
-    run minikube kubectl create namespace $project
-    assert "$status" -eq 0 "create new namespace $project"
-    run minikube kubectl -- apply -f $fname
-    assert "$status" -eq 0 "deploy $fname to the cluster"
+    run_minikube kubectl create namespace $project
+    run_minikube kubectl -- apply -f $fname
     assert "$output" == "pod/$pname created"
     wait_for_pods_to_start
-    run minikube kubectl delete namespace $project
-    assert $status -eq 0 "delete namespace $project"
+    run_minikube kubectl delete namespace $project
 }
 
 @test "minikube - apply podman ctr to cluster" {
@@ -64,35 +56,29 @@ load helpers.bash
 
     # deploy to minikube cluster with kube apply
     project="ctr-apply"
-    run minikube kubectl create namespace $project
-    assert "$status" -eq 0 "create new namespace $project"
+    run_minikube kubectl create namespace $project
     run_podman kube apply --kubeconfig $KUBECONFIG --ns $project $cname
     assert "$output" =~ "Successfully deployed workloads to cluster!"
-    run minikube kubectl -- get pods --namespace $project
-    assert "$status" -eq 0 "kube apply $cname to the cluster"
+    run_minikube kubectl -- get pods --namespace $project
     assert "$output" =~ "$cname-pod"
     wait_for_pods_to_start
-    run minikube kubectl delete namespace $project
-    assert $status -eq 0 "delete namespace $project"
+    run_minikube kubectl delete namespace $project
 }
 
 @test "minikube - apply podman pod to cluster" {
     pname="test-pod-apply"
     run_podman pod create --name $pname
-    run podman container create --pod $pname $IMAGE top
+    run_podman container create --pod $pname $IMAGE top
 
     # deploy to minikube cluster with kube apply
     project="pod-apply"
-    run minikube kubectl create namespace $project
-    assert "$status" -eq 0 "create new namespace $project"
+    run_minikube kubectl create namespace $project
     run_podman kube apply --kubeconfig $KUBECONFIG --ns $project $pname
     assert "$output" =~ "Successfully deployed workloads to cluster!"
-    run minikube kubectl -- get pods --namespace $project
-    assert "$status" -eq 0 "kube apply $pname to the cluster"
+    run_minikube kubectl -- get pods --namespace $project
     assert "$output" =~ "$pname"
     wait_for_pods_to_start
-    run minikube kubectl delete namespace $project
-    assert $status -eq 0 "delete namespace $project"
+    run_minikube kubectl delete namespace $project
 }
 
 @test "minikube - deploy generated kube yaml with podman kube apply to cluster" {
@@ -108,16 +94,13 @@ load helpers.bash
 
     # deploy to minikube cluster with kube apply
     project="yaml-apply"
-    run minikube kubectl create namespace $project
-    assert "$status" -eq 0 "create new namespace $project"
+    run_minikube kubectl create namespace $project
     run_podman kube apply --kubeconfig $KUBECONFIG --ns $project -f $fname
     assert "$output" =~ "Successfully deployed workloads to cluster!"
-    run minikube kubectl -- get pods --namespace $project
-    assert "$status" -eq 0 "kube apply $pname to the cluster"
+    run_minikube kubectl -- get pods --namespace $project
     assert "$output" =~ "$pname"
     wait_for_pods_to_start
-    run minikube kubectl delete namespace $project
-    assert $status -eq 0 "delete namespace $project"
+    run_minikube kubectl delete namespace $project
 }
 
 @test "minikube - apply podman ctr with volume to cluster" {
@@ -127,19 +110,15 @@ load helpers.bash
 
     # deploy to minikube cluster with kube apply
     project="ctr-vol-apply"
-    run minikube kubectl create namespace $project
-    assert "$status" -eq 0 "create new namespace $project"
+    run_minikube kubectl create namespace $project
     run_podman kube apply --kubeconfig $KUBECONFIG --ns $project $cname $vname
     assert "$output" =~ "Successfully deployed workloads to cluster!"
-    run minikube kubectl -- get pods --namespace $project
-    assert "$status" -eq 0 "kube apply $cname to the cluster"
+    run_minikube kubectl -- get pods --namespace $project
     assert "$output" =~ "$cname-pod"
-    run minikube kubectl -- get pvc --namespace $project
-    assert "$status" -eq 0 "kube apply $vname to the cluster"
+    run_minikube kubectl -- get pvc --namespace $project
     assert "$output" =~ "$vname"
     wait_for_pods_to_start
-    run minikube kubectl delete namespace $project
-    assert $status -eq 0 "delete namespace $project"
+    run_minikube kubectl delete namespace $project
 }
 
 @test "minikube - apply podman ctr with service to cluster" {
@@ -148,19 +127,15 @@ load helpers.bash
 
     # deploy to minikube cluster with kube apply
     project="ctr-svc-apply"
-    run minikube kubectl create namespace $project
-    assert "$status" -eq 0 "create new namespace $project"
+    run_minikube kubectl create namespace $project
     run_podman kube apply --kubeconfig $KUBECONFIG -s --ns $project $cname
     assert "$output" =~ "Successfully deployed workloads to cluster!"
-    run minikube kubectl -- get pods --namespace $project
-    assert "$status" -eq 0 "kube apply $cname to the cluster"
+    run_minikube kubectl -- get pods --namespace $project
     assert "$output" =~ "$cname-pod"
-    run minikube kubectl -- get svc --namespace $project
-    assert "$status" -eq 0 "kube apply service to the cluster"
+    run_minikube kubectl -- get svc --namespace $project
     assert "$output" =~ "$cname-pod"
     wait_for_pods_to_start
-    run minikube kubectl delete namespace $project
-    assert $status -eq 0 "delete namespace $project"
+    run_minikube kubectl delete namespace $project
 }
 
 @test "minikube - deploy generated container yaml to minikube --type=deployment" {
@@ -171,14 +146,11 @@ load helpers.bash
 
     # deploy to the minikube cluster
     project="dep-ctr-ns"
-    run minikube kubectl create namespace $project
-    assert "$status" -eq 0 "create new namespace $project"
-    run minikube kubectl -- apply -f $fname
-    assert "$status" -eq 0 "deploy $fname to the cluster"
+    run_minikube kubectl create namespace $project
+    run_minikube kubectl -- apply -f $fname
     assert "$output" == "deployment.apps/$cname-pod-deployment created"
     wait_for_pods_to_start
-    run minikube kubectl delete namespace $project
-    assert $status -eq 0 "delete namespace $project"
+    run_minikube kubectl delete namespace $project
 }
 
 @test "minikube - deploy generated pod yaml to minikube --type=deployment" {
@@ -194,14 +166,11 @@ load helpers.bash
 
     # deploy to the minikube cluster
     project="dep-pod-ns"
-    run minikube kubectl create namespace $project
-    assert "$status" -eq 0 "create new namespace $project"
-    run minikube kubectl -- apply -f $fname
-    assert "$status" -eq 0 "deploy $fname to the cluster"
+    run_minikube kubectl create namespace $project
+    run_minikube kubectl -- apply -f $fname
     assert "$output" == "deployment.apps/$pname-deployment created"
     wait_for_pods_to_start
-    run minikube kubectl delete namespace $project
-    assert $status -eq 0 "delete namespace $project"
+    run_minikube kubectl delete namespace $project
 }
 
 @test "minikube - deploy generated container yaml to minikube --type=daemonset" {
@@ -212,14 +181,11 @@ load helpers.bash
 
     # deploy to the minikube cluster
     project="dep-ctr-ns"
-    run minikube kubectl create namespace $project
-    assert "$status" -eq 0 "create new namespace $project"
-    run minikube kubectl -- apply -f $fname
-    assert "$status" -eq 0 "deploy $fname to the cluster"
+    run_minikube kubectl create namespace $project
+    run_minikube kubectl -- apply -f $fname
     assert "$output" == "daemonset.apps/$cname-pod-daemonset created"
     wait_for_pods_to_start
-    run minikube kubectl delete namespace $project
-    assert $status -eq 0 "delete namespace $project"
+    run_minikube kubectl delete namespace $project
 }
 
 @test "minikube - deploy generated pod yaml to minikube --type=daemonset" {
@@ -235,12 +201,9 @@ load helpers.bash
 
     # deploy to the minikube cluster
     project="dep-pod-ns"
-    run minikube kubectl create namespace $project
-    assert "$status" -eq 0 "create new namespace $project"
-    run minikube kubectl -- apply -f $fname
-    assert "$status" -eq 0 "deploy $fname to the cluster"
+    run_minikube kubectl create namespace $project
+    run_minikube kubectl -- apply -f $fname
     assert "$output" == "daemonset.apps/$pname-daemonset created"
     wait_for_pods_to_start
-    run minikube kubectl delete namespace $project
-    assert $status -eq 0 "delete namespace $project"
+    run_minikube kubectl delete namespace $project
 }
diff --git a/test/minikube/helpers.bash b/test/minikube/helpers.bash
@@ -4,10 +4,55 @@ load ../system/helpers.bash
 
 KUBECONFIG="$HOME/.kube/config"
 
+##################
+#  run_minikube  #  Local helper, with instrumentation for debugging failures
+##################
+function run_minikube() {
+    # Number as first argument = expected exit code; default 0
+    local expected_rc=0
+    case "$1" in
+        [0-9])           expected_rc=$1; shift;;
+        [1-9][0-9])      expected_rc=$1; shift;;
+        [12][0-9][0-9])  expected_rc=$1; shift;;
+        '?')             expected_rc=  ; shift;;  # ignore exit code
+    esac
+
+    # stdout is only emitted upon error; this printf is to help in debugging
+    printf "\n%s %s %s %s\n" "$(timestamp)" "\$" "minikube" "$*"
+    run minikube "$@"
+    # without "quotes", multiple lines are glommed together into one
+    if [[ -n "$output" ]]; then
+        echo "$(timestamp) $output"
+    fi
+    if [[ "$status" -ne 0 ]]; then
+        echo -n "$(timestamp) [ rc=$status ";
+        if [[ -n "$expected_rc" ]]; then
+            if [[ "$status" -eq "$expected_rc" ]]; then
+                echo -n "(expected) ";
+            else
+                echo -n "(** EXPECTED $expected_rc **) ";
+            fi
+        fi
+        echo "]"
+    fi
+
+    if [[ -n "$expected_rc" ]]; then
+        if [[ "$status" -ne "$expected_rc" ]]; then
+            # Further debugging
+            echo "\$ minikube logs"
+            run minikube logs
+            echo "$output"
+
+            die "exit code is $status; expected $expected_rc"
+        fi
+    fi
+}
+
+
 function setup(){
     # only set up the minikube cluster before the first test
     if [[ "$BATS_TEST_NUMBER" -eq 1 ]]; then
-        minikube start
+        run_minikube start
         wait_for_default_sa
     fi
     basic_setup
@@ -17,8 +62,12 @@ function teardown(){
     # only delete the minikube cluster if we are done with the last test
     # the $DEBUG_MINIKUBE env can be set to preserve the cluster to debug if needed
     if [[ "$BATS_TEST_NUMBER" -eq ${#BATS_TEST_NAMES[@]} ]] && [[ "$DEBUG_MINIKUBE" == "" ]]; then
-        minikube delete
+        run_minikube delete
     fi
+
+    # Prevents nasty red warnings in log
+    run_podman rmi --ignore $(pause_image)
+
     basic_teardown
 }
 
@@ -29,8 +78,7 @@ function wait_for_default_sa(){
     # if the default service account hasn't been created yet, there is something else wrong
     while [[ $count -lt 30 ]] && [[ $sa_ready == false ]]
     do
-        run minikube kubectl get sa
-        assert "$status" -eq 0
+        run_minikube kubectl get sa
         if [[ "$output" != "No resources found in default namespace." ]]; then
             sa_ready=true
         fi
@@ -49,7 +97,7 @@ function wait_for_pods_to_start(){
     # if the pod hasn't started running after 30 seconds, there is something else wrong
     while [[ $count -lt 30 ]] && [[ $running == false ]]
     do
-        run minikube kubectl get pods
+        run_minikube kubectl get pods
         assert "$status" -eq 0
         if [[ "$output" =~ "Running" ]]; then
             running=true