From abe36bcb28a164639cbaa0620d7e44665618bb71 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 14 Jan 2020 17:49:49 -0800 Subject: [PATCH 1/2] implement kind test mvp --- cmd/kind/app/main.go | 6 +- pkg/cmd/kind/root.go | 2 + pkg/cmd/kind/test/conformance/conformance.go | 41 ++ pkg/cmd/kind/test/internal/testcmd/testcmd.go | 388 ++++++++++++++++++ pkg/cmd/kind/test/presubmit/presubmit.go | 41 ++ pkg/cmd/kind/test/test.go | 42 ++ pkg/exec/helpers.go | 10 + pkg/internal/cli/logger.go | 5 + 8 files changed, 530 insertions(+), 5 deletions(-) create mode 100644 pkg/cmd/kind/test/conformance/conformance.go create mode 100644 pkg/cmd/kind/test/internal/testcmd/testcmd.go create mode 100644 pkg/cmd/kind/test/presubmit/presubmit.go create mode 100644 pkg/cmd/kind/test/test.go diff --git a/cmd/kind/app/main.go b/cmd/kind/app/main.go index d68d9c0a57..1616ac7b04 100644 --- a/cmd/kind/app/main.go +++ b/cmd/kind/app/main.go @@ -79,11 +79,7 @@ func checkQuiet(args []string) bool { // logError logs the error and the root stacktrace if there is one func logError(logger log.Logger, err error) { - if cmd.ColorEnabled(logger) { - logger.Errorf("\x1b[31mERROR\x1b[0m: %v", err) - } else { - logger.Errorf("ERROR: %v", err) - } + logger.Errorf("%v", err) // If debugging is enabled (non-zero verbosity), display more info if logger.V(1).Enabled() { // Display Output if the error was running a command ... diff --git a/pkg/cmd/kind/root.go b/pkg/cmd/kind/root.go index 459e907400..ba4a0c1867 100644 --- a/pkg/cmd/kind/root.go +++ b/pkg/cmd/kind/root.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/kind/pkg/cmd/kind/export" "sigs.k8s.io/kind/pkg/cmd/kind/get" "sigs.k8s.io/kind/pkg/cmd/kind/load" + "sigs.k8s.io/kind/pkg/cmd/kind/test" "sigs.k8s.io/kind/pkg/cmd/kind/version" "sigs.k8s.io/kind/pkg/log" ) @@ -86,6 +87,7 @@ func NewCommand(logger log.Logger, streams cmd.IOStreams) *cobra.Command { cmd.AddCommand(get.NewCommand(logger, streams)) cmd.AddCommand(version.NewCommand(logger, streams)) cmd.AddCommand(load.NewCommand(logger, streams)) + cmd.AddCommand(test.NewCommand(logger, streams)) return cmd } diff --git a/pkg/cmd/kind/test/conformance/conformance.go b/pkg/cmd/kind/test/conformance/conformance.go new file mode 100644 index 0000000000..eeaac1775f --- /dev/null +++ b/pkg/cmd/kind/test/conformance/conformance.go @@ -0,0 +1,41 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package conformance implements the `test conformance` command +package conformance + +import ( + "github.com/spf13/cobra" + + "sigs.k8s.io/kind/pkg/cmd" + "sigs.k8s.io/kind/pkg/log" + + "sigs.k8s.io/kind/pkg/cmd/kind/test/internal/testcmd" +) + +// NewCommand returns a new cobra.Command for Kubernetes conformance testing +func NewCommand(logger log.Logger, streams cmd.IOStreams) *cobra.Command { + return testcmd.New(logger, streams, + testcmd.Details{ + Name: "conformance", + UsageShort: "Builds a node image and runs the Kubernetes conformance tests", + UsageLong: "Builds a node image and runs the Kubernetes conformance tests", + // test options for this subcommand + Focus: `\[Conformance\]`, + Skip: ``, + Parallel: false, + }) +} diff --git a/pkg/cmd/kind/test/internal/testcmd/testcmd.go b/pkg/cmd/kind/test/internal/testcmd/testcmd.go new file mode 100644 index 0000000000..00d48ca783 --- /dev/null +++ b/pkg/cmd/kind/test/internal/testcmd/testcmd.go @@ -0,0 +1,388 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package testcmd implements generic test command logic +package testcmd + +import ( + "fmt" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/spf13/cobra" + + "sigs.k8s.io/kind/pkg/apis/config/v1alpha4" + "sigs.k8s.io/kind/pkg/build/node" + "sigs.k8s.io/kind/pkg/cluster" + "sigs.k8s.io/kind/pkg/cluster/nodeutils" + "sigs.k8s.io/kind/pkg/cmd" + "sigs.k8s.io/kind/pkg/cmd/kind/version" + "sigs.k8s.io/kind/pkg/errors" + "sigs.k8s.io/kind/pkg/exec" + "sigs.k8s.io/kind/pkg/fs" + "sigs.k8s.io/kind/pkg/log" +) + +// flagpole holds our flags +type flagpole struct { + Name string + ImageName string + Artifacts string + IPFamily string +} + +func (f *flagpole) kubeconfigPath() string { + return filepath.Join(f.Artifacts, "kind", "kubeconfig") +} + +func (f *flagpole) Process() error { + // Default Artifacts + if f.Artifacts == "" { + f.Artifacts = os.Getenv("ARTIFACTS") + if f.Artifacts == "" { + tmpdir, err := fs.TempDir("", "kind-test") + if err != nil { + return errors.Wrap(err, "failed to create tempdir for artifacts") + } + f.Artifacts = tmpdir + } + } + // Validate + if f.IPFamily != string(v1alpha4.IPv4Family) && f.IPFamily != string(v1alpha4.IPv6Family) { + return fmt.Errorf( + "invalid --ipfamily %q; valid are: %q, %q", + f.IPFamily, v1alpha4.IPv4Family, v1alpha4.IPv6Family, + ) + } + return nil +} + +// Details represents injected details of a particular test command +type Details struct { + Name string + UsageLong string + UsageShort string + Focus string + Skip string + Parallel bool +} + +// New returns a new cobra.Command for Kubernetes testing with ginkgo-e2e.sh +func New(logger log.Logger, streams cmd.IOStreams, details Details) *cobra.Command { + flags := &flagpole{} + cmd := &cobra.Command{ + Args: cobra.NoArgs, + Use: details.Name, + Short: details.UsageShort, + Long: details.UsageLong, + RunE: func(cmd *cobra.Command, args []string) error { + // flag defaulting & validation + if err := flags.Process(); err != nil { + return err + } + + // handle test command overrides + if focus, set := os.LookupEnv("KIND_TEST_FOCUS_OVERRIDE"); set { + details.Focus = focus + logger.Warnf("KIND_TEST_FOCUS_OVERRIDE overrode --ginkgo.focus to be: %s", focus) + } + if skip, set := os.LookupEnv("KIND_TEST_SKIP_OVERRIDE"); set { + details.Skip = skip + logger.Warnf("KIND_TEST_SKIP_OVERRIDE overrode --ginkgo.skip to be: %s", skip) + } + + // actually run the command + return runGinkgo(logger, flags, details) + }, + } + cmd.Flags().StringVar(&flags.IPFamily, "ip-family", string(v1alpha4.IPv4Family), "test cluster IP family") + cmd.Flags().StringVar(&flags.Name, "name", details.Name, "test cluster context name") + cmd.Flags().StringVar(&flags.Artifacts, "artifacts-dir", "", "the directory in which to output test results") + cmd.Flags().StringVar(&flags.ImageName, "image", "kindest/node:kubernetes-presubmit", "name:tag of the built kind node image") + return cmd +} + +// TODO: junit output for steps a la kubetest +func runGinkgo(logger log.Logger, flags *flagpole, details Details) (err error) { + // first debug the version we're using + logger.V(0).Infof("Testing with %s", version.DisplayVersion()) + + // then build + if err := buildWithGinkgoAndTests(logger, flags); err != nil { + return err + } + + // setup shared provider for doing cluster things + provider := cluster.NewProvider( + cluster.ProviderWithLogger(logger), + ) + + // Check if the cluster already exists + if err := checkAlreadyExists(provider, flags.Name); err != nil { + return err + } + + // then before we do anything more, arrange for cleanup to happen + cleanup := func(calledFromSignalHandler bool) { + err2 := provider.CollectLogs(flags.Name, filepath.Join(flags.Artifacts, "logs")) + err3 := provider.Delete(flags.Name, flags.kubeconfigPath()) + // don't touch err if we're being called from another goroutine + // on program early exit, or if err is already non-nil + if calledFromSignalHandler || err != nil { + return + } else if err2 != nil { + err = err2 + } else { + err = err3 + } + } + var cleanupOnce sync.Once + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGTERM, os.Interrupt) + go func() { + <-signals + cleanupOnce.Do(func() { cleanup(true) }) + os.Exit(1) + }() + defer func() { + cleanupOnce.Do(func() { cleanup(false) }) + }() + + // test (including cluster bringup) + return testGinkgo(logger, provider, flags, details) +} + +func buildWithGinkgoAndTests(logger log.Logger, flags *flagpole) error { + // primarily for CI we allow overriding the build mode + buildType := os.Getenv("KIND_BUILD_TYPE") + switch buildType { + case "bazel", "docker": + default: + buildType = "docker" + } + + // possibly enable bazel build caching before building kubernetes + // TODO: remove this if we move to RBE ..? + if os.Getenv("BAZEL_REMOTE_CACHE_ENABLED") == "true" { + _ = exec.Command("create_bazel_cache_rcs.sh").Run() + } + + // first build the node image + ctx, err := node.NewBuildContext( + node.WithMode(buildType), + node.WithImage(flags.ImageName), + node.WithLogger(logger), + ) + if err != nil { + return errors.Wrap(err, "error creating build context") + } + if err := ctx.Build(); err != nil { + return errors.Wrap(err, "error building node image") + } + + // build test binaries + switch buildType { + case "bazel": + if err := exec.InheritOutput( + exec.Command("bazel", "build", "//test/e2e:e2e.test", "//vendor/github.com/onsi/ginkgo/ginkgo"), + ).Run(); err != nil { + return err + } + // TODO: cleanup or remove this + // we are getting kubectl into the PATH + p, err := exec.Output(exec.Command("sh", "-c", `echo "$(dirname "$(find "${PWD}/bazel-bin/" -name kubectl -type f)"):${PATH}"`).SetEnv(os.Environ()...)) + if err != nil { + return err + } + os.Setenv("PATH", strings.TrimSuffix(p, "\n")) + + case "docker": + if err := exec.InheritOutput( + exec.Command( + "build/run.sh", + "make", "all", `WHAT=cmd/kubectl test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo`, + ).SetEnv( + "KUBE_VERBOSE=0", + ), + ).Run(); err != nil { + return err + } + default: + return errors.Errorf("build mode %q is not implemented yet!", buildType) + } + + // In Kubernetes's CI: attempt to release some memory after building + if len(os.Getenv("KUBETEST_IN_DOCKER")) > 0 { + _ = exec.Command("sync").Run() + _ = exec.Command("/proc/sys/vm/drop_caches").SetStdin(strings.NewReader("1")).Run() + } + + return nil +} + +func checkAlreadyExists(provider *cluster.Provider, name string) error { + n, err := provider.ListNodes(name) + if err != nil { + return err + } + if len(n) != 0 { + return fmt.Errorf("node(s) already exist for a cluster with the name %q", name) + } + return nil +} + +func testGinkgo(logger log.Logger, provider *cluster.Provider, flags *flagpole, details Details) error { + // construct the cluster config + config := &v1alpha4.Cluster{ + Networking: v1alpha4.Networking{ + IPFamily: v1alpha4.ClusterIPFamily(flags.IPFamily), + }, + Nodes: []v1alpha4.Node{ + { + Role: v1alpha4.ControlPlaneRole, + Image: flags.ImageName, + }, + }, + } + // add e2e workers + numNodesForE2E := 2 + for i := 0; i < numNodesForE2E; i++ { + config.Nodes = append(config.Nodes, v1alpha4.Node{ + Role: v1alpha4.WorkerRole, + Image: flags.ImageName, + }) + } + + // create the cluster + logger.V(0).Infof("Creating cluster %q ...\n", flags.Name) + if err := provider.Create( + flags.Name, + cluster.CreateWithV1Alpha4Config(config), + cluster.CreateWithRetain(true), + cluster.CreateWithWaitForReady(time.Minute), + cluster.CreateWithKubeconfigPath(flags.kubeconfigPath()), + cluster.CreateWithDisplayUsage(false), + cluster.CreateWithDisplaySalutation(false), + ); err != nil { + if errs := errors.Errors(err); errs != nil { + for _, problem := range errs { + logger.Errorf("%v", problem) + } + return errors.New("aborting due to invalid configuration") + } + return errors.Wrap(err, "failed to create cluster") + } + + // In Kubernetes CI we don't have real IPv6 connectivity so we need to + // employ some work arounds when testing IPv6 clusters + // We do this all environments for consistency. + if config.Networking.IPFamily == v1alpha4.IPv6Family { + if err := enableIPv6Workarounds(provider, flags.Name); err != nil { + return err + } + } + + // run tests + skip := details.Skip + if details.Parallel { + skip = strings.Join([]string{`\[Serial\]`, skip}, "|") + } + // pass through host env and override a few specific ones + env := append([]string{}, os.Environ()...) + env = append(env, + // setting this env prevents ginkgo e2e from trying to run provider setup + "KUBERNETES_CONFORMANCE_TEST=y", + // setting these is required to make RuntimeClass tests work ... :/ + "KUBE_CONTAINER_RUNTIME=remote", + "KUBE_CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock", + "KUBE_CONTAINER_RUNTIME_NAME=containerd", + // ensure the test uses our kubeconfig + "KUBECONFIG="+flags.kubeconfigPath(), + "GINKGO_PARALLEL="+boolToYorN(details.Parallel), + ) + cmd := exec.Command( + "./hack/ginkgo-e2e.sh", + // don't use the horrid e2e.test """provider""" concept + "--provider=skeleton", + // need to inform tests of worker count + "--num-nodes", strconv.Itoa(numNodesForE2E), + // write test results into artifacts + "--report-dir", flags.Artifacts, + // we do our own log dump + "--disable-log-dump=true", + // plumb ginkgo options + "--ginkgo.focus", details.Focus, + "--ginkgo.skip", skip, + ).SetEnv(env...) + return exec.InheritOutput(cmd).Run() +} + +func boolToYorN(y bool) string { + if y { + return "y" + } + return "n" +} + +func enableIPv6Workarounds(provider *cluster.Provider, name string) error { + // get a control plane node to use run kubectl from + nodes, err := provider.ListInternalNodes(name) + if err != nil { + return errors.Wrap(err, "failed to get nodes for coreDNS IPv6 CI fixes") + } + nodes, err = nodeutils.ControlPlaneNodes(nodes) + if err != nil { + return errors.Wrap(err, "failed to get control-plane node for coreDNS IPv6 CI fixes") + } + if len(nodes) < 1 { + return errors.New("failed to locate a kind node for coreDNS IPv6 CI fixes") + } + node := nodes[0] + + // "fix" coreDNS configmap + // TODO: refactor this to not be a shell script. + /* + IPv6 clusters need some CoreDNS changes in order to work in k8s CI: + 1. k8s CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured + to work in an offline environment: + https://github.com/coredns/coredns/issues/2494#issuecomment-457215452 + + 2. k8s CI adds following domains to resolv.conf search field: + c.k8s-prow-builds.internal google.internal. + CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL + otherwise pods stops trying to resolve the domain. + */ + const hacks = `export KUBECONFIG=/etc/kubernetes/admin.conf; \ +kubectl get -oyaml -n=kube-system configmap/coredns | \ +sed \ + -e 's/^.*kubernetes cluster\.local/& internal/' \ + -e '/^.*upstream$/d' \ + -e '/^.*fallthrough.*$/d' \ + -e '/^.*forward . \/etc\/resolv.conf$/d' \ + -e '/^.*loop$/d' | \ +kubectl apply -f -` + if err := node.Command("sh", "-c", hacks).Run(); err != nil { + return errors.Wrap(err, "failed to patch coreDNS configmap for IPv6 CI fixes") + } + + return nil +} diff --git a/pkg/cmd/kind/test/presubmit/presubmit.go b/pkg/cmd/kind/test/presubmit/presubmit.go new file mode 100644 index 0000000000..4ef6cc1db9 --- /dev/null +++ b/pkg/cmd/kind/test/presubmit/presubmit.go @@ -0,0 +1,41 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package presubmit implements the `test presubmit` command +package presubmit + +import ( + "github.com/spf13/cobra" + + "sigs.k8s.io/kind/pkg/cmd" + "sigs.k8s.io/kind/pkg/log" + + "sigs.k8s.io/kind/pkg/cmd/kind/test/internal/testcmd" +) + +// NewCommand returns a new cobra.Command for Kubernetes presubmit testing +func NewCommand(logger log.Logger, streams cmd.IOStreams) *cobra.Command { + return testcmd.New(logger, streams, + testcmd.Details{ + Name: "presubmit", + UsageShort: "Builds a node image and runs the Kubernetes presubmit tests", + UsageLong: "Builds a node image and runs the Kubernetes presubmit tests", + // test options for this subcommand + Focus: `.`, + Skip: `\[Slow\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|PodSecurityPolicy|LoadBalancer|load.balancer|In-tree.Volumes.\[Driver:.nfs\]|PersistentVolumes.NFS|Network.should.set.TCP.CLOSE_WAIT.timeout|Simple.pod.should.support.exec.through.an.HTTP.proxy|subPath.should.support.existing|ReplicationController.light.Should.scale.from.1.pod.to.2.pods|should.provide.basic.identity|\[NodeFeature:PodReadinessGate\]`, + Parallel: true, + }) +} diff --git a/pkg/cmd/kind/test/test.go b/pkg/cmd/kind/test/test.go new file mode 100644 index 0000000000..a4404e8c05 --- /dev/null +++ b/pkg/cmd/kind/test/test.go @@ -0,0 +1,42 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package test implements the `test` command +package test + +import ( + "github.com/spf13/cobra" + + "sigs.k8s.io/kind/pkg/cmd" + "sigs.k8s.io/kind/pkg/cmd/kind/test/conformance" + "sigs.k8s.io/kind/pkg/cmd/kind/test/presubmit" + "sigs.k8s.io/kind/pkg/log" +) + +// NewCommand returns a new cobra.Command for test +func NewCommand(logger log.Logger, streams cmd.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Args: cobra.NoArgs, + // TODO(bentheelder): more detailed usage + Use: "test", + Short: "Tests one of [presubmit, conformance]", + Long: "Tests one of [presubmit, conformance]", + } + // add subcommands + cmd.AddCommand(presubmit.NewCommand(logger, streams)) + cmd.AddCommand(conformance.NewCommand(logger, streams)) + return cmd +} diff --git a/pkg/exec/helpers.go b/pkg/exec/helpers.go index 0a5a736c98..dfed25b83f 100644 --- a/pkg/exec/helpers.go +++ b/pkg/exec/helpers.go @@ -72,6 +72,16 @@ func CombinedOutputLines(cmd Cmd) (lines []string, err error) { return lines, err } +// Output is similar to os/exec's cmd.Output(), +// but over our Cmd interface, and instead of returning the byte buffer of +// stdout, it returns a string +func Output(cmd Cmd) (output string, err error) { + var buff bytes.Buffer + cmd.SetStdout(&buff) + err = cmd.Run() + return buff.String(), err +} + // OutputLines is like os/exec's cmd.Output(), // but over our Cmd interface, and instead of returning the byte buffer of // stdout, it scans these for lines and returns a slice of output lines diff --git a/pkg/internal/cli/logger.go b/pkg/internal/cli/logger.go index 73eff7a1d7..4d45fe571b 100644 --- a/pkg/internal/cli/logger.go +++ b/pkg/internal/cli/logger.go @@ -169,6 +169,11 @@ func (l *Logger) Error(message string) { // Errorf is part of the log.Logger interface func (l *Logger) Errorf(format string, args ...interface{}) { + if l.ColorEnabled() { + format = "\x1b[31mERROR\x1b[0m: " + format + } else { + format = "ERROR: " + format + } l.printf(format, args...) } From e44548136bebaf91c6517a5633f0811dcdef4f15 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 17 Jan 2020 17:01:09 -0800 Subject: [PATCH 2/2] port e2e-k8s.sh to `kind test presubmit` --- hack/ci/e2e-k8s.sh | 176 ++++----------------------------------------- 1 file changed, 12 insertions(+), 164 deletions(-) diff --git a/hack/ci/e2e-k8s.sh b/hack/ci/e2e-k8s.sh index b083cfaf4a..a2d37aa412 100755 --- a/hack/ci/e2e-k8s.sh +++ b/hack/ci/e2e-k8s.sh @@ -22,167 +22,15 @@ set -o errexit -o nounset -o xtrace # Settings: # SKIP: ginkgo skip regex # FOCUS: ginkgo focus regex -# BUILD_TYPE: bazel or make -# - -# our exit handler (trap) -cleanup() { - # KIND_CREATE_ATTEMPTED is true once we: kind create - if [ "${KIND_CREATE_ATTEMPTED:-}" = true ]; then - kind "export" logs "${ARTIFACTS}/logs" || true - kind delete cluster || true - fi - rm -f _output/bin/e2e.test || true - # remove our tempdir, this needs to be last, or it will prevent kind delete - [ -n "${TMP_DIR:-}" ] && rm -rf "${TMP_DIR:?}" -} - -# build kubernetes / node image, e2e binaries, with bazel -build_with_bazel() { - # possibly enable bazel build caching before building kubernetes - if [ "${BAZEL_REMOTE_CACHE_ENABLED:-false}" = "true" ]; then - create_bazel_cache_rcs.sh || true - fi - - # build the node image w/ kubernetes - kind build node-image --type=bazel - # make sure we have e2e requirements - bazel build //cmd/kubectl //test/e2e:e2e.test //vendor/github.com/onsi/ginkgo/ginkgo - - # ensure the e2e script will find our binaries ... - # https://github.com/kubernetes/kubernetes/issues/68306 - # TODO: remove this, it was fixed in 1.13+ - mkdir -p '_output/bin/' - cp 'bazel-bin/test/e2e/e2e.test' '_output/bin/' - PATH="$(dirname "$(find "${PWD}/bazel-bin/" -name kubectl -type f)"):${PATH}" - export PATH -} - -# build kubernetes / node image, e2e binaries -build() { - # build the node image w/ kubernetes - kind build node-image - # make sure we have e2e requirements - make all WHAT='cmd/kubectl test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo' -} - -# up a cluster with kind -create_cluster() { - # create the config file - cat < "${ARTIFACTS}/kind-config.yaml" -# config for 1 control plane node and 2 workers (necessary for conformance) -kind: Cluster -apiVersion: kind.sigs.k8s.io/v1alpha3 -networking: - ipFamily: ${IP_FAMILY:-ipv4} -nodes: -- role: control-plane -- role: worker -- role: worker -EOF - # NOTE: must match the number of workers above - NUM_NODES=2 - # actually create the cluster - # TODO(BenTheElder): settle on verbosity for this script - KIND_CREATE_ATTEMPTED=true - kind create cluster \ - --image=kindest/node:latest \ - --retain \ - --wait=1m \ - -v=3 \ - "--config=${ARTIFACTS}/kind-config.yaml" -} - -# run e2es with ginkgo-e2e.sh -run_tests() { - # IPv6 clusters need some CoreDNS changes in order to work in k8s CI: - # 1. k8s CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured - # to work in an offline environment: - # https://github.com/coredns/coredns/issues/2494#issuecomment-457215452 - # 2. k8s CI adds following domains to resolv.conf search field: - # c.k8s-prow-builds.internal google.internal. - # CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL - # otherwise pods stops trying to resolve the domain. - if [ "${IP_FAMILY:-ipv4}" = "ipv6" ]; then - # Get the current config - original_coredns=$(kubectl get -oyaml -n=kube-system configmap/coredns) - echo "Original CoreDNS config:" - echo "${original_coredns}" - # Patch it - fixed_coredns=$( - printf '%s' "${original_coredns}" | sed \ - -e 's/^.*kubernetes cluster\.local/& internal/' \ - -e '/^.*upstream$/d' \ - -e '/^.*fallthrough.*$/d' \ - -e '/^.*forward . \/etc\/resolv.conf$/d' \ - -e '/^.*loop$/d' \ - ) - echo "Patched CoreDNS config:" - echo "${fixed_coredns}" - printf '%s' "${fixed_coredns}" | kubectl apply -f - - fi - - # ginkgo regexes - SKIP="${SKIP:-}" - FOCUS="${FOCUS:-"\\[Conformance\\]"}" - # if we set PARALLEL=true, skip serial tests set --ginkgo-parallel - if [ "${PARALLEL:-false}" = "true" ]; then - export GINKGO_PARALLEL=y - if [ -z "${SKIP}" ]; then - SKIP="\\[Serial\\]" - else - SKIP="\\[Serial\\]|${SKIP}" - fi - fi - - # setting this env prevents ginkgo e2e from trying to run provider setup - export KUBERNETES_CONFORMANCE_TEST='y' - # setting these is required to make RuntimeClass tests work ... :/ - export KUBE_CONTAINER_RUNTIME=remote - export KUBE_CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock - export KUBE_CONTAINER_RUNTIME_NAME=containerd - ./hack/ginkgo-e2e.sh \ - '--provider=skeleton' "--num-nodes=${NUM_NODES}" \ - "--ginkgo.focus=${FOCUS}" "--ginkgo.skip=${SKIP}" \ - "--report-dir=${ARTIFACTS}" '--disable-log-dump=true' -} - -main() { - # create temp dir and setup cleanup - TMP_DIR=$(mktemp -d) - trap cleanup EXIT - - # ensure artifacts (results) directory exists when not in CI - export ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}" - mkdir -p "${ARTIFACTS}" - - # export the KUBECONFIG to a unique path for testing - KUBECONFIG="${HOME}/.kube/kind-test-config" - export KUBECONFIG - echo "exported KUBECONFIG=${KUBECONFIG}" - - # debug kind version - kind version - - # default to bazel - # TODO(bentheelder): remove this line once we've updated CI to explicitly choose - BUILD_TYPE="${BUILD_TYPE:-bazel}" - - # build kubernetes - if [ "${BUILD_TYPE:-}" = "bazel" ]; then - build_with_bazel - else - build - fi - - # in CI attempt to release some memory after building - if [ -n "${KUBETEST_IN_DOCKER:-}" ]; then - sync || true - echo 1 > /proc/sys/vm/drop_caches || true - fi - - # create the cluster and run tests - create_cluster && run_tests -} - -main +# BUILD_TYPE: bazel or docker +# +export KIND_BUILD_TYPE="${BUILD_TYPE:-bazel}" +# when neither of tehse are set we need the old default +# TODO: eliminate this script and upadte the prowjobs do do these explicitly +if [ -z "${FOCUS:-}${SKIP:-}" ]; then + kind test conformance --ip-family="${IP_FAMILY:-ipv4}" +else + export KIND_TEST_FOCUS_OVERRIDE="${FOCUS:-}" + export KIND_TEST_SKIP_OVERRIDE="${SKIP:-}" + kind test presubmit --ip-family="${IP_FAMILY:-ipv4}" +fi