test: refactor E2E examples tests and minor cleanup

This rewrites `hack/test-examples.sh` in Go to make those tests easier to debug and facilitate testing more complex scenarios, like the VAP added in argoproj#14045 The new tests do strict validation, which caught a bug in `examples/webhdfs-input-output-artifacts.yaml`: ``` $ kubectl apply -f examples/webhdfs-input-output-artifacts.yaml Error from server (BadRequest): error when creating "examples/webhdfs-input-output-artifacts.yaml": Workflow in version "v1alpha1" cannot be handled as a Workflow: strict decoding error: unknown field "spec.templates[0].outputs.artifacts[0].overwrite" ``` $ make test-examples <SNIP> examples_test.go:28: Error parsing ../../examples/webhdfs-input-output-artifacts.yaml: strict decoding error: unknown field "spec.templates[0].outputs.artifacts[0].overwrite" === FAIL: ExamplesSuite/TestExampleWorkflows FAIL github.com/argoproj/argo-workflows/v3/test/e2e 1.580s FAIL make: *** [Makefile:609: test-examples] Error 1 ``` Signed-off-by: Mason Malone <[email protected]>
MasonM · Jan 17, 2025 · d6406a2 · d6406a2
1 parent 8304dc7
commit d6406a2
Show file tree

Hide file tree

Showing 8 changed files with 78 additions and 50 deletions.
diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml
@@ -61,7 +61,6 @@ jobs:
               - sdks/**
               # example test suite
               - examples/**
-              - hack/test-examples.sh
             codegen:
               - *common
               # generated files

diff --git a/Makefile b/Makefile
@@ -38,7 +38,7 @@ E2E_WAIT_TIMEOUT      ?= 90s # timeout for wait conditions
 E2E_PARALLEL          ?= 20
 E2E_SUITE_TIMEOUT     ?= 15m
 GOTEST                ?= go test -v -p 20
-ALL_BUILD_TAGS        ?= api,cli,cron,executor,examples,corefunctional,functional,plugins
+ALL_BUILD_TAGS        ?= api,cli,cron,executor,examples,corefunctional,functional,plugins,examples
 BENCHMARK_COUNT       ?= 6
 
 # should we build the static files?
@@ -608,10 +608,6 @@ test-cli: ./dist/argo
 test-%:
 	E2E_WAIT_TIMEOUT=$(E2E_WAIT_TIMEOUT) go test -failfast -v -timeout $(E2E_SUITE_TIMEOUT) -count 1 --tags $* -parallel $(E2E_PARALLEL) ./test/e2e
 
-.PHONY: test-examples
-test-examples:
-	./hack/test-examples.sh
-
 .PHONY: test-%-sdk
 test-%-sdk:
 	make --directory sdks/$* install test -B

diff --git a/examples/webhdfs-input-output-artifacts.yaml b/examples/webhdfs-input-output-artifacts.yaml
@@ -49,7 +49,6 @@ spec:
       artifacts:
       - name: my-art2
         path: /my-artifact
-        overwrite: true
         http:
           # below is an example on how to use authentication via certificates
           # clientCert.clientCertSecret: points to a kubernetes secret named cert-sec with a data entry of "certificate.pem"

diff --git a/hack/test-examples.sh b/hack/test-examples.sh
diff --git a/test/e2e/cron_test.go b/test/e2e/cron_test.go
@@ -454,8 +454,7 @@ spec:
 }
 
 func (s *CronSuite) TestMalformedCronWorkflow() {
-	s.Given().
-		Exec("kubectl", []string{"apply", "-f", "testdata/malformed/malformed-cronworkflow.yaml"}, fixtures.ErrorOutput("unknown field \"spec.workflowSpec.arguments.parameters.someParam\""))
+	s.Given().KubectlApply("testdata/malformed/malformed-cronworkflow.yaml", fixtures.ErrorOutput(".spec.workflowSpec.arguments.parameters: expected list"))
 }
 
 func TestCronSuite(t *testing.T) {

diff --git a/test/e2e/examples_test.go b/test/e2e/examples_test.go
@@ -0,0 +1,47 @@
+//go:build examples
+
+package e2e
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+
+	"github.com/argoproj/argo-workflows/v3/test/e2e/fixtures"
+	fileutil "github.com/argoproj/argo-workflows/v3/util/file"
+	"github.com/argoproj/argo-workflows/v3/workflow/common"
+)
+
+type ExamplesSuite struct {
+	fixtures.E2ESuite
+}
+
+func (s *ExamplesSuite) BeforeTest(suiteName, testName string) {
+	s.E2ESuite.BeforeTest(suiteName, testName)
+	s.Given().KubectlApply("../../examples/configmaps/simple-parameters-configmap.yaml", fixtures.NoError)
+}
+
+func (s *ExamplesSuite) TestExampleWorkflows() {
+	err := fileutil.WalkManifests("../../examples", func(path string, data []byte) error {
+		wfs, err := common.SplitWorkflowYAMLFile(data, true)
+		if err != nil {
+			s.T().Fatalf("Error parsing %s: %v", path, err)
+		}
+		for _, wf := range wfs {
+			if _, ok := wf.GetLabels()["workflows.argoproj.io/test"]; ok {
+				s.T().Logf("Found example workflow at %s with test label\n", path)
+				s.Given().
+					Workflow(&wf).
+					When().
+					SubmitWorkflow().
+					WaitForWorkflow(fixtures.ToBeSucceeded)
+			}
+		}
+		return nil
+	})
+	s.CheckError(err)
+}
+
+func TestExamplesSuite(t *testing.T) {
+	suite.Run(t, new(ExamplesSuite))
+}
diff --git a/test/e2e/fixtures/given.go b/test/e2e/fixtures/given.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"testing"
 
@@ -41,10 +42,19 @@ type Given struct {
 //
 // 1. A file name if it starts with "@"
 // 2. Raw YAML.
-func (g *Given) Workflow(text string) *Given {
+// 3. An already parsed wfv1.Workflow object
+func (g *Given) Workflow(wf interface{}) *Given {
 	g.t.Helper()
-	g.wf = &wfv1.Workflow{}
-	g.readResource(text, g.wf)
+	switch v := wf.(type) {
+	case *wfv1.Workflow:
+		g.wf = v
+		g.checkLabels(v)
+	case string:
+		g.wf = &wfv1.Workflow{}
+		g.readResource(v, g.wf)
+	default:
+		g.t.Fatalf("Unsupported input: %s", reflect.TypeOf(wf).String())
+	}
 	g.checkImages(g.wf)
 	return g
 }
@@ -209,6 +219,11 @@ func (g *Given) Exec(name string, args []string, block func(t *testing.T, output
 	return g
 }
 
+func (g *Given) KubectlApply(file string, block func(t *testing.T, output string, err error)) *Given {
+	g.t.Helper()
+	return g.Exec("kubectl", append([]string{"-n", Namespace, "apply", "--server-side", "-f"}, file), block)
+}
+
 func (g *Given) RunCli(args []string, block func(t *testing.T, output string, err error)) *Given {
 	return g.Exec("../../dist/argo", append([]string{"-n", Namespace}, args...), block)
 }

diff --git a/test/e2e/malformed_resources_test.go b/test/e2e/malformed_resources_test.go
@@ -18,14 +18,14 @@ type MalformedResourcesSuite struct {
 }
 
 func (s *MalformedResourcesSuite) TestMalformedWorkflow() {
-	s.Given().Exec("kubectl", []string{"apply", "-f", "testdata/malformed/malformed-workflow.yaml"}, fixtures.ErrorOutput("unknown field \"spec.arguments.parameters.someParam\""))
+	s.Given().KubectlApply("testdata/malformed/malformed-workflow.yaml", fixtures.ErrorOutput(".spec.arguments.parameters: expected list"))
 }
 
 func (s *MalformedResourcesSuite) TestMalformedWorkflowTemplate() {
 	s.Given().
-		Exec("kubectl", []string{"apply", "-f", "testdata/malformed/malformed-workflowtemplate.yaml"}, fixtures.ErrorOutput("unknown field \"spec.arguments.parameters.someParam\"")).
-		Exec("kubectl", []string{"apply", "-f", "testdata/wellformed/wellformed-workflowtemplate.yaml"}, fixtures.NoError).
-		Exec("kubectl", []string{"apply", "-f", "testdata/wellformed/wellformed-workflow-with-workflow-template-ref.yaml"}, fixtures.NoError).
+		KubectlApply("testdata/malformed/malformed-workflowtemplate.yaml", fixtures.ErrorOutput(".spec.arguments.parameters: expected list")).
+		KubectlApply("testdata/wellformed/wellformed-workflowtemplate.yaml", fixtures.NoError).
+		KubectlApply("testdata/wellformed/wellformed-workflow-with-workflow-template-ref.yaml", fixtures.NoError).
 		When().
 		WaitForWorkflow().
 		Then().
@@ -37,8 +37,8 @@ func (s *MalformedResourcesSuite) TestMalformedWorkflowTemplate() {
 
 func (s *MalformedResourcesSuite) TestMalformedWorkflowTemplateRef() {
 	s.Given().
-		Exec("kubectl", []string{"apply", "-f", "testdata/malformed/malformed-workflowtemplate.yaml"}, fixtures.ErrorOutput("unknown field \"spec.arguments.parameters.someParam\"")).
-		Exec("kubectl", []string{"apply", "-f", "testdata/wellformed/wellformed-workflow-with-malformed-workflow-template-ref.yaml"}, fixtures.NoError).
+		KubectlApply("testdata/malformed/malformed-workflowtemplate.yaml", fixtures.ErrorOutput(".spec.arguments.parameters: expected list")).
+		KubectlApply("testdata/wellformed/wellformed-workflow-with-malformed-workflow-template-ref.yaml", fixtures.NoError).
 		When().
 		WaitForWorkflow().
 		Then().
@@ -51,9 +51,9 @@ func (s *MalformedResourcesSuite) TestMalformedWorkflowTemplateRef() {
 
 func (s *MalformedResourcesSuite) TestMalformedClusterWorkflowTemplate() {
 	s.Given().
-		Exec("kubectl", []string{"apply", "-f", "testdata/malformed/malformed-clusterworkflowtemplate.yaml"}, fixtures.ErrorOutput("unknown field \"spec.arguments.parameters.someParam\"")).
-		Exec("kubectl", []string{"apply", "-f", "testdata/wellformed/wellformed-clusterworkflowtemplate.yaml"}, fixtures.NoError).
-		Exec("kubectl", []string{"apply", "-f", "testdata/wellformed/wellformed-workflow-with-cluster-workflow-template-ref.yaml"}, fixtures.NoError).
+		KubectlApply("testdata/malformed/malformed-clusterworkflowtemplate.yaml", fixtures.ErrorOutput(".spec.arguments.parameters: expected list")).
+		KubectlApply("testdata/wellformed/wellformed-clusterworkflowtemplate.yaml", fixtures.NoError).
+		KubectlApply("testdata/wellformed/wellformed-workflow-with-cluster-workflow-template-ref.yaml", fixtures.NoError).
 		When().
 		WaitForWorkflow().
 		Then().
@@ -65,8 +65,8 @@ func (s *MalformedResourcesSuite) TestMalformedClusterWorkflowTemplate() {
 
 func (s *MalformedResourcesSuite) TestMalformedClusterWorkflowTemplateRef() {
 	s.Given().
-		Exec("kubectl", []string{"apply", "-f", "testdata/malformed/malformed-clusterworkflowtemplate.yaml"}, fixtures.ErrorOutput("unknown field \"spec.arguments.parameters.someParam\"")).
-		Exec("kubectl", []string{"apply", "-f", "testdata/wellformed/wellformed-workflow-with-malformed-cluster-workflow-template-ref.yaml"}, fixtures.NoError).
+		KubectlApply("testdata/malformed/malformed-clusterworkflowtemplate.yaml", fixtures.ErrorOutput(".spec.arguments.parameters: expected list")).
+		KubectlApply("testdata/wellformed/wellformed-workflow-with-malformed-cluster-workflow-template-ref.yaml", fixtures.NoError).
 		When().
 		WaitForWorkflow().
 		Then().