feat: signal stability of tests

Signed-off-by: Mattia Lavacca <[email protected]>

conformance/apis/v1/conformancereport.go

@@ -46,6 +46,10 @@ type ConformanceReport struct {
 	// ProfileReports is a list of the individual reports for each conformance
 	// profile that was enabled for a test run.
 	ProfileReports []ProfileReport `json:"profiles"`
+
+	// SucceededTrialTests is a list of the names of the trial tests that
+	// have been successfully run.
+	SucceededTrialTests []string `json:"succeededTrialTests,omitempty"`
 }
 
 // Implementation provides metadata information on the downstream

conformance/conformance.go

@@ -96,6 +96,7 @@ func DefaultOptions(t *testing.T) suite.ConformanceOptions {
 		SkipTests:                  skipTests,
 		SupportedFeatures:          supportedFeatures,
 		TimeoutConfig:              conformanceconfig.DefaultTimeoutConfig(),
+		SkipTrialTests:             *flags.SkipTrialTests,
 	}
 }
 

conformance/utils/flags/flags.go

@@ -48,4 +48,5 @@ var (
 	AllowCRDsMismatch          = flag.Bool("allow-crds-mismatch", false, "Flag to allow the suite not to fail in case there is a mismatch between CRDs versions and channels.")
 	ConformanceProfiles        = flag.String("conformance-profiles", "", "Comma-separated list of the conformance profiles to run")
 	ReportOutput               = flag.String("report-output", "", "The file where to write the conformance report")
+	SkipTrialTests             = flag.Bool("skip-trial-tests", false, "Whether to skip trial tests")
 )

conformance/utils/suite/conformance.go

@@ -36,6 +36,7 @@ type ConformanceTest struct {
 	Slow        bool
 	Parallel    bool
 	Test        func(*testing.T, *ConformanceTestSuite)
+	Trial       bool
 }
 
 // Run runs an individual tests, applying and cleaning up the required manifests

conformance/utils/suite/reports.go

@@ -42,6 +42,7 @@ var (
 	testFailed       resultType = "FAILED"
 	testSkipped      resultType = "SKIPPED"
 	testNotSupported resultType = "NOT_SUPPORTED"
+	testTrialSkipped resultType = "TRIAL_SKIPPED"
 )
 
 type profileReportsMap map[ConformanceProfileName]confv1.ProfileReport

conformance/utils/suite/suite.go

@@ -18,7 +18,6 @@ package suite
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io/fs"
 	"slices"
@@ -44,7 +43,6 @@ import (
 	"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
 	"sigs.k8s.io/gateway-api/conformance/utils/roundtripper"
 	"sigs.k8s.io/gateway-api/conformance/utils/tlog"
-	"sigs.k8s.io/gateway-api/pkg/consts"
 	"sigs.k8s.io/gateway-api/pkg/features"
 )
 
@@ -71,6 +69,7 @@ type ConformanceTestSuite struct {
 	SupportedFeatures        sets.Set[features.FeatureName]
 	TimeoutConfig            config.TimeoutConfig
 	SkipTests                sets.Set[string]
+	SkipTrialTests           bool
 	RunTest                  string
 	ManifestFS               []fs.FS
 	UsableNetworkAddresses   []v1beta1.GatewayAddress
@@ -144,6 +143,8 @@ type ConformanceOptions struct {
 	// SkipTests contains all the tests not to be run and can be used to opt out
 	// of specific tests
 	SkipTests []string
+	// SkipTrialTests indicates whether or not to skip trial tests.
+	SkipTrialTests bool
 	// RunTest is a single test to run, mostly for development/debugging convenience.
 	RunTest string
 
@@ -248,6 +249,7 @@ func NewConformanceTestSuite(options ConformanceOptions) (*ConformanceTestSuite,
 		TimeoutConfig:               options.TimeoutConfig,
 		SkipTests:                   sets.New(options.SkipTests...),
 		RunTest:                     options.RunTest,
+		SkipTrialTests:              options.SkipTrialTests,
 		ManifestFS:                  options.ManifestFS,
 		UsableNetworkAddresses:      options.UsableNetworkAddresses,
 		UnusableNetworkAddresses:    options.UnusableNetworkAddresses,
@@ -431,6 +433,9 @@ func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) er
 		if suite.SkipTests.Has(test.ShortName) {
 			res = testSkipped
 		}
+		if suite.SkipTrialTests && test.Trial {
+			res = testTrialSkipped
+		}
 		if !suite.SupportedFeatures.HasAll(test.Features...) {
 			res = testNotSupported
 		}
@@ -486,8 +491,15 @@ func (suite *ConformanceTestSuite) Report() (*confv1.ConformanceReport, error) {
 	}
 	sort.Strings(testNames)
 	profileReports := newReports()
+	succeededTrialTestSet := sets.Set[string]{}
 	for _, tN := range testNames {
 		testResult := suite.results[tN]
+		if testResult.result == testTrialSkipped {
+			continue
+		}
+		if testResult.result == testSucceeded && testResult.test.Trial {
+			succeededTrialTestSet.Insert(tN)
+		}
 		conformanceProfiles := getConformanceProfilesForTest(testResult.test, suite.conformanceProfiles).UnsortedList()
 		sort.Slice(conformanceProfiles, func(i, j int) bool {
 			return conformanceProfiles[i].Name < conformanceProfiles[j].Name
@@ -496,6 +508,10 @@ func (suite *ConformanceTestSuite) Report() (*confv1.ConformanceReport, error) {
 			profileReports.addTestResults(*profile, testResult)
 		}
 	}
+	var succeededTrialTests []string
+	if len(succeededTrialTestSet) > 0 {
+		succeededTrialTests = succeededTrialTestSet.UnsortedList()
+	}
 
 	profileReports.compileResults(suite.extendedSupportedFeatures, suite.extendedUnsupportedFeatures)
 
@@ -504,68 +520,12 @@ func (suite *ConformanceTestSuite) Report() (*confv1.ConformanceReport, error) {
 			APIVersion: confv1.GroupVersion.String(),
 			Kind:       "ConformanceReport",
 		},
-		Date:              time.Now().Format(time.RFC3339),
-		Mode:              suite.mode,
-		Implementation:    suite.implementation,
-		GatewayAPIVersion: suite.apiVersion,
-		GatewayAPIChannel: suite.apiChannel,
-		ProfileReports:    profileReports.list(),
+		Date:                time.Now().Format(time.RFC3339),
+		Mode:                suite.mode,
+		Implementation:      suite.implementation,
+		GatewayAPIVersion:   suite.apiVersion,
+		GatewayAPIChannel:   suite.apiChannel,
+		ProfileReports:      profileReports.list(),
+		SucceededTrialTests: succeededTrialTests,
 	}, nil
 }
-
-// ParseImplementation parses implementation-specific flag arguments and
-// creates a *confv1a1.Implementation.
-func ParseImplementation(org, project, url, version, contact string) confv1.Implementation {
-	return confv1.Implementation{
-		Organization: org,
-		Project:      project,
-		URL:          url,
-		Version:      version,
-		Contact:      strings.Split(contact, ","),
-	}
-}
-
-// ParseConformanceProfiles parses flag arguments and converts the string to
-// sets.Set[ConformanceProfileName].
-func ParseConformanceProfiles(p string) sets.Set[ConformanceProfileName] {
-	res := sets.Set[ConformanceProfileName]{}
-	if p == "" {
-		return res
-	}
-
-	for _, value := range strings.Split(p, ",") {
-		res.Insert(ConformanceProfileName(value))
-	}
-	return res
-}
-
-// getAPIVersionAndChannel iterates over all the crds installed in the cluster and check the version and channel annotations.
-// In case the annotations are not found or there are crds with different versions or channels, an error is returned.
-func getAPIVersionAndChannel(crds []apiextensionsv1.CustomResourceDefinition) (version string, channel string, err error) {
-	for _, crd := range crds {
-		v, okv := crd.Annotations[consts.BundleVersionAnnotation]
-		c, okc := crd.Annotations[consts.ChannelAnnotation]
-		if !okv && !okc {
-			continue
-		}
-		if !okv || !okc {
-			return "", "", errors.New("detected CRDs with partial version and channel annotations")
-		}
-		if version != "" && v != version {
-			return "", "", errors.New("multiple gateway API CRDs versions detected")
-		}
-		if channel != "" && c != channel {
-			return "", "", errors.New("multiple gateway API CRDs channels detected")
-		}
-		version = v
-		channel = c
-	}
-	if version == "" || channel == "" {
-		return "", "", errors.New("no Gateway API CRDs with the proper annotations found in the cluster")
-	}
-	if version != consts.BundleVersion {
-		return "", "", errors.New("the installed CRDs version is different from the suite version")
-	}
-
-	return version, channel, nil
-}