Skip to content

Commit 9371121

Browse files
medyaghmedyagh
authored
Merge pull request #21972 from medyagh/test_none
ci: add none driver tests to prow
2 parents fafc2bb + 5150699 commit 9371121

10 files changed

+168
-25
lines changed

‎hack/prow/kvm.json‎ renamed to ‎hack/prow/bosksos-nested.json‎

File renamed without changes.

‎hack/prow/integration_docker_docker_linux_x86-64.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,9 @@ DRIVER="docker"
2323
CONTAINER_RUNTIME="docker"
2424
EXTRA_START_ARGS=""
2525
EXTRA_TEST_ARGS=""
26-
JOB_NAME="Docker_Linux"
26+
JOB_NAME="Docker_docker_Linux_X86-64"
2727

28+
# marking all directories ('*') as trusted, since .git belongs to root, not minikube user
2829
git config --global --add safe.directory '*'
2930
COMMIT=$(git rev-parse HEAD)
3031
MINIKUBE_LOCATION=$COMMIT

‎hack/prow/integration_kvm_docker_linux_x86-64.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ CONTAINER_RUNTIME="docker"
2424
# in prow, if you want libvirtd to be run, you have to start a privileged container as root
2525
EXTRA_START_ARGS=""
2626
EXTRA_TEST_ARGS=""
27-
JOB_NAME="KVM_Linux"
27+
JOB_NAME="KVM_Docker_Linux_X86-64"
28+
# marking all directories ('*') as trusted, since .git belongs to root, not minikube user
2829
git config --global --add safe.directory '*'
2930
COMMIT=$(git rev-parse HEAD)
3031
MINIKUBE_LOCATION=$COMMIT

‎hack/prow/integration_kvm_docker_linux_x86-64_pre.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ CONTAINER_RUNTIME="docker"
2424
# in prow, if you want libvirtd to be run, you have to start a privileged container as root
2525
EXTRA_START_ARGS=""
2626
EXTRA_TEST_ARGS=""
27-
JOB_NAME="KVM_Linux"
27+
JOB_NAME="KVM_Docker_Linux_X86-64"
2828

2929

3030
# install docker if not present

‎hack/prow/integration_none_docker_linux_x86-64.sh‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#!/bin/bash
2+
3+
# Copyright 2025 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -e
18+
set -x
19+
20+
OS="linux"
21+
ARCH="amd64"
22+
DRIVER="none"
23+
CONTAINER_RUNTIME="docker"
24+
EXTRA_START_ARGS=""
25+
EXTRA_TEST_ARGS=""
26+
JOB_NAME="None_Docker_Linux_X86-64"
27+
# marking all directories ('*') as trusted, since .git belongs to root, not minikube user
28+
git config --global --add safe.directory '*'
29+
COMMIT=$(git rev-parse HEAD)
30+
MINIKUBE_LOCATION=$COMMIT
31+
32+
33+
# conntrack is required for Kubernetes 1.18 and higher for none driver
34+
if ! conntrack --version &>/dev/null; then
35+
echo "WARNING: conntrack is not installed. will try to install."
36+
sudo apt-get update -qq
37+
sudo apt-get -qq -y install conntrack
38+
fi
39+
40+
# socat is required for kubectl port forward which is used in some tests such as validateHelmTillerAddon
41+
if ! which socat &>/dev/null; then
42+
echo "WARNING: socat is not installed. will try to install."
43+
sudo apt-get update -qq
44+
sudo apt-get -qq -y install socat
45+
fi
46+
47+
# cri-dockerd is required for Kubernetes v1.24+ with none driver
48+
CRI_DOCKERD_VERSION="0.4.1"
49+
if [[ $(cri-dockerd --version 2>&1) != *"$CRI_DOCKERD_VERSION"* ]]; then
50+
echo "WARNING: expected version of cri-dockerd is not installed. will try to install."
51+
sudo systemctl stop cri-docker.socket || true
52+
sudo systemctl stop cri-docker.service || true
53+
CRI_DOCKERD_COMMIT="55d6e1a1d6f2ee58949e13a0c66afe7d779ac942"
54+
CRI_DOCKERD_BASE_URL="https://storage.googleapis.com/kicbase-artifacts/cri-dockerd/${CRI_DOCKERD_COMMIT}"
55+
sudo curl -L "${CRI_DOCKERD_BASE_URL}/amd64/cri-dockerd" -o /usr/bin/cri-dockerd
56+
sudo curl -L "${CRI_DOCKERD_BASE_URL}/cri-docker.socket" -o /usr/lib/systemd/system/cri-docker.socket
57+
sudo curl -L "${CRI_DOCKERD_BASE_URL}/cri-docker.service" -o /usr/lib/systemd/system/cri-docker.service
58+
sudo chmod +x /usr/bin/cri-dockerd
59+
sudo systemctl daemon-reload
60+
sudo systemctl enable cri-docker.service
61+
sudo systemctl enable --now cri-docker.socket
62+
fi
63+
64+
# crictl is required for Kubernetes v1.24+ with none driver
65+
CRICTL_VERSION="v1.34.0"
66+
if [[ $(crictl --version) != *"$CRICTL_VERSION"* ]]; then
67+
echo "WARNING: expected version of crictl is not installed. will try to install."
68+
curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz --output crictl-${CRICTL_VERSION}-linux-amd64.tar.gz
69+
sudo tar zxvf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin
70+
fi
71+
72+
# cni-plugins is required for Kubernetes v1.24+ with none driver
73+
./hack/jenkins/installers/check_install_cni_plugins.sh
74+
75+
76+
# by default, prow jobs run in root, so we must switch to a non-root user to run docker driver
77+
source ./hack/prow/common.sh

‎hack/prow/minitest/main.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ var deployers = map[string]func(string) deployer.MiniTestDeployer{
3434
"docker": deployer.NewMiniTestDockerDeployerFromConfigFile,
3535
}
3636
var testers = map[string]tester.MiniTestTester{
37-
"kvm-docker-linux-amd64-integration": &tester.KVMDockerLinuxAmd64IntegrationTester{},
37+
"kvm-docker-linux-amd64-integration": &tester.KVMDockerLinuxAmd64IntegrationTester{},
38+
"none-docker-linux-amd64-integration": &tester.NoneDockerLinuxAmd64IntegrationTester{},
3839
}
3940

4041
func main() {

‎hack/prow/minitest/tester/none_docker_linux_amd64_integration.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package tester
18+
19+
import (
20+
"fmt"
21+
"os"
22+
23+
"k8s.io/klog/v2"
24+
)
25+
26+
var _ MiniTestTester = &KVMDockerLinuxAmd64IntegrationTester{}
27+
28+
// this runs the integration tests with none driver and a docker container runtime.
29+
type NoneDockerLinuxAmd64IntegrationTester struct {
30+
}
31+
32+
// Run implements MiniTestTester.
33+
func (k *NoneDockerLinuxAmd64IntegrationTester) Run(runner MiniTestRunner) error {
34+
35+
if up, err := runner.IsUp(); err != nil || !up {
36+
klog.Errorf("tester: deployed environment is not up: %v", err)
37+
}
38+
if err := runner.SyncToRemote(".", "~/minikube", []string{".cache"}); err != nil {
39+
klog.Errorf("failed to sync file in docker deployer: %v", err)
40+
}
41+
pr := os.Getenv("PULL_NUMBER")
42+
43+
var testErr error
44+
// install docker and libvirtd first then run the test in a new shell
45+
if err := runner.Execute("cd minikube && ./hack/prow/integration_kvm_docker_linux_x86-64_pre.sh"); err != nil {
46+
klog.Errorf("failed to install docker in env: %v", err)
47+
return err
48+
}
49+
if testErr = runner.Execute(fmt.Sprintf("cd minikube && PULL_NUMBER=\"%s\" ./hack/prow/integration_none_docker_linux_x86-64.sh", pr)); testErr != nil {
50+
klog.Errorf("failed to execute command in env: %v", testErr)
51+
// don't return here, we still want to collect the test reports
52+
}
53+
54+
// prow requires result file to be copied to $ARTIFACTS. All other files will not be persisted.
55+
if err := copyFileToArtifact(runner); err != nil {
56+
return err
57+
}
58+
return testErr
59+
60+
}

‎hack/prow/prow.mk‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,24 @@
11
.PHONY: integration-prow-kvm-docker-linux-x86-64
2-
3-
42
integration-prow-docker-docker-linux-x86-64:
5-
# build first
6-
# container-runtime=docker driver=docker on linux/amd64
73
./hack/prow/minikube_cross_build.sh $(GO_VERSION) linux amd64
84
./hack/prow/util/integration_prow_wrapper.sh ./hack/prow/integration_docker_docker_linux_x86-64.sh
95

10-
integration-prow-kvm-docker-linux-x86-64:
11-
# build first
12-
# container-runtime=docker driver=kvm on linux/amd64
6+
.PHONY: integration-prow-none-docker-linux-x86-64
7+
integration-prow-none-docker-linux-x86-64: setup-prow-gcp-ssh-keys build-mini-test
8+
./hack/prow/minikube_cross_build.sh $(GO_VERSION) linux amd64
9+
./out/minitest --deployer boskos --tester none-docker-linux-amd64-integration --config hack/prow/bosksos-nested.json
10+
11+
.PHONY: integration-prow-kvm-docker-linux-x86-64
12+
integration-prow-kvm-docker-linux-x86-64: setup-prow-gcp-ssh-keys build-mini-test
1313
./hack/prow/minikube_cross_build.sh $(GO_VERSION) linux amd64
14-
# set up ssh keys for gcloud cli. These env vars are set by test/infra
14+
./out/minitest --deployer boskos --tester kvm-docker-linux-amd64-integration --config hack/prow/bosksos-nested.json
15+
16+
.PHONY: build-mini-test
17+
build-mini-test: # build minitest binary
18+
GOTOOLCHAIN=auto go build -C ./hack/prow/minitest -o $(PWD)/out/minitest .
19+
20+
.PHONY: setup-prow-gcp-ssh-keys
21+
setup-prow-gcp-ssh-keys: # set up ssh keys for gcloud cli. These env vars are set by test/infra
1522
mkdir -p -m 0700 ~/.ssh
1623
cp -f "${GCE_SSH_PRIVATE_KEY_FILE}" ~/.ssh/google_compute_engine
17-
cp -f "${GCE_SSH_PUBLIC_KEY_FILE}" ~/.ssh/google_compute_engine.pub
18-
GOTOOLCHAIN=auto go build -C ./hack/prow/minitest -o $(PWD)/out/minitest .
19-
./out/minitest --deployer boskos --tester kvm-docker-linux-amd64-integration --config hack/prow/kvm.json
24+
cp -f "${GCE_SSH_PUBLIC_KEY_FILE}" ~/.ssh/google_compute_engine.pub

‎test/integration/preload_test.go‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,12 @@ package integration
2020

2121
import (
2222
"context"
23-
"fmt"
2423
"os/exec"
2524
"strings"
2625
"testing"
2726
)
2827

29-
// TestPreload verifies the preload tarballs get pulled in properly by minikube
28+
// TestPreload verifies that disabling the initial preload, pulling a specific image, and restarting the cluster preserves the image across restarts.
3029
func TestPreload(t *testing.T) {
3130
if NoneDriver() {
3231
t.Skipf("skipping %s - incompatible with none driver", t.Name())
@@ -38,7 +37,6 @@ func TestPreload(t *testing.T) {
3837

3938
startArgs := []string{"start", "-p", profile, "--memory=3072", "--alsologtostderr", "--wait=true", "--preload=false"}
4039
startArgs = append(startArgs, StartArgs()...)
41-
startArgs = append(startArgs, fmt.Sprintf("--kubernetes-version=%s", legacyVersion()))
4240

4341
rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...))
4442
if err != nil {
@@ -59,8 +57,8 @@ func TestPreload(t *testing.T) {
5957
t.Fatalf("%s failed: %v", rr.Command(), err)
6058
}
6159

62-
// re-start the cluster and check if image is preserved
63-
startArgs = []string{"start", "-p", profile, "--memory=3072", "--alsologtostderr", "-v=1", "--wait=true"}
60+
// re-start the cluster and check if image is preserved with enabled preload
61+
startArgs = []string{"start", "-p", profile, "--preload=true", "--alsologtostderr", "-v=1", "--wait=true"}
6462
startArgs = append(startArgs, StartArgs()...)
6563
rr, err = Run(t, exec.CommandContext(ctx, Target(), startArgs...))
6664
if err != nil {

‎test/integration/version_upgrade_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,11 @@ func installRelease(version string) (f *os.File, err error) {
6262
return tf, nil
6363
}
6464

65-
func legacyVersion() string {
65+
func legacyMinikubeVersion() string {
6666
// Should be a version from the last 6 months
6767
// note: Test*BinaryUpgrade require minikube v1.22+ to satisfy newer containerd config structure
6868
// note: TestMissingContainerUpgrade requires minikube v1.26.0+ where we copy over initial containerd config in kicbase via deploy/kicbase/Dockerfile
69-
version := "v1.32.0" // Dec 11, 2024
69+
version := "v1.35.0" // Jan 15, 2025
7070
return version
7171
}
7272

@@ -88,7 +88,7 @@ func TestRunningBinaryUpgrade(t *testing.T) {
8888

8989
defer CleanupWithLogs(t, profile, cancel)
9090

91-
desiredLegacyVersion := legacyVersion()
91+
desiredLegacyVersion := legacyMinikubeVersion()
9292
tf, err := installRelease(desiredLegacyVersion)
9393
if err != nil {
9494
t.Fatalf("%s release installation failed: %v", desiredLegacyVersion, err)
@@ -146,7 +146,7 @@ func TestStoppedBinaryUpgrade(t *testing.T) {
146146

147147
defer CleanupWithLogs(t, profile, cancel)
148148

149-
desiredLegacyVersion := legacyVersion()
149+
desiredLegacyVersion := legacyMinikubeVersion()
150150
var tf *os.File
151151
t.Run("Setup", func(t *testing.T) {
152152
var err error
@@ -295,7 +295,7 @@ func TestMissingContainerUpgrade(t *testing.T) {
295295

296296
defer CleanupWithLogs(t, profile, cancel)
297297

298-
legacyVersion := legacyVersion()
298+
legacyVersion := legacyMinikubeVersion()
299299

300300
tf, err := installRelease(legacyVersion)
301301
if err != nil {

0 commit comments

Comments
 (0)