Signal test stability

conformance/apis/v1/conformancereport.go

@@ -46,6 +46,10 @@ type ConformanceReport struct {
 	// ProfileReports is a list of the individual reports for each conformance
 	// profile that was enabled for a test run.
 	ProfileReports []ProfileReport `json:"profiles"`
+
+	// SucceededProvisionalTests is a list of the names of the provisional tests that
+	// have been successfully run.
+	SucceededProvisionalTests []string `json:"succeededProvisionalTests,omitempty"`
 }
 
 // Implementation provides metadata information on the downstream

conformance/conformance.go

@@ -96,6 +96,7 @@ func DefaultOptions(t *testing.T) suite.ConformanceOptions {
 		SkipTests:                  skipTests,
 		SupportedFeatures:          supportedFeatures,
 		TimeoutConfig:              conformanceconfig.DefaultTimeoutConfig(),
+		SkipProvisionalTests:       *flags.SkipProvisionalTests,
 	}
 }
 

conformance/tests/udproute-simple.go

@@ -42,6 +42,7 @@ var UDPRouteTest = suite.ConformanceTest{
 		features.SupportUDPRoute,
 		features.SupportGateway,
 	},
+	Provisional: true,
 	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
 		t.Run("Simple UDP request matching UDPRoute should reach coredns backend", func(t *testing.T) {
 			namespace := "gateway-conformance-infra"

conformance/utils/flags/flags.go

@@ -48,4 +48,5 @@ var (
 	AllowCRDsMismatch          = flag.Bool("allow-crds-mismatch", false, "Flag to allow the suite not to fail in case there is a mismatch between CRDs versions and channels.")
 	ConformanceProfiles        = flag.String("conformance-profiles", "", "Comma-separated list of the conformance profiles to run")
 	ReportOutput               = flag.String("report-output", "", "The file where to write the conformance report")
+	SkipProvisionalTests       = flag.Bool("skip-provisional-tests", false, "Whether to skip provisional tests")
 )

conformance/utils/suite/conformance.go

@@ -36,6 +36,7 @@ type ConformanceTest struct {
 	Slow        bool
 	Parallel    bool
 	Test        func(*testing.T, *ConformanceTestSuite)
+	Provisional bool
 }
 
 // Run runs an individual tests, applying and cleaning up the required manifests

conformance/utils/suite/reports.go

@@ -18,7 +18,6 @@ package suite
 
 import (
 	"fmt"
-	"sort"
 
 	"k8s.io/apimachinery/pkg/util/sets"
 
@@ -38,10 +37,11 @@ type testResult struct {
 type resultType string
 
 var (
-	testSucceeded    resultType = "SUCCEEDED"
-	testFailed       resultType = "FAILED"
-	testSkipped      resultType = "SKIPPED"
-	testNotSupported resultType = "NOT_SUPPORTED"
+	testSucceeded          resultType = "SUCCEEDED"
+	testFailed             resultType = "FAILED"
+	testSkipped            resultType = "SKIPPED"
+	testNotSupported       resultType = "NOT_SUPPORTED"
+	testProvisionalSkipped resultType = "PROVISIONAL_SKIPPED"
 )
 
 type profileReportsMap map[ConformanceProfileName]confv1.ProfileReport
@@ -136,10 +136,7 @@ func (p profileReportsMap) compileResults(supportedFeaturesMap map[ConformancePr
 		supportedFeatures := supportedFeaturesMap[ConformanceProfileName(report.Name)]
 		if report.Extended != nil {
 			if supportedFeatures != nil {
-				supportedFeatures := supportedFeatures.UnsortedList()
-				sort.Slice(supportedFeatures, func(i, j int) bool {
-					return supportedFeatures[i] < supportedFeatures[j]
-				})
+				supportedFeatures := sets.List(supportedFeatures)
 				for _, f := range supportedFeatures {
 					report.Extended.SupportedFeatures = append(report.Extended.SupportedFeatures, string(f))
 				}
@@ -149,10 +146,7 @@ func (p profileReportsMap) compileResults(supportedFeaturesMap map[ConformancePr
 		unsupportedFeatures := unsupportedFeaturesMap[ConformanceProfileName(report.Name)]
 		if report.Extended != nil {
 			if unsupportedFeatures != nil {
-				unsupportedFeatures := unsupportedFeatures.UnsortedList()
-				sort.Slice(unsupportedFeatures, func(i, j int) bool {
-					return unsupportedFeatures[i] < unsupportedFeatures[j]
-				})
+				unsupportedFeatures := sets.List(unsupportedFeatures)
 				for _, f := range unsupportedFeatures {
 					report.Extended.UnsupportedFeatures = append(report.Extended.UnsupportedFeatures, string(f))
 				}

conformance/utils/suite/suite.go

@@ -71,6 +71,7 @@ type ConformanceTestSuite struct {
 	SupportedFeatures        sets.Set[features.FeatureName]
 	TimeoutConfig            config.TimeoutConfig
 	SkipTests                sets.Set[string]
+	SkipProvisionalTests     bool
 	RunTest                  string
 	ManifestFS               []fs.FS
 	UsableNetworkAddresses   []v1beta1.GatewayAddress
@@ -144,6 +145,8 @@ type ConformanceOptions struct {
 	// SkipTests contains all the tests not to be run and can be used to opt out
 	// of specific tests
 	SkipTests []string
+	// SkipProvisionalTests indicates whether or not to skip provisional tests.
+	SkipProvisionalTests bool
 	// RunTest is a single test to run, mostly for development/debugging convenience.
 	RunTest string
 
@@ -248,6 +251,7 @@ func NewConformanceTestSuite(options ConformanceOptions) (*ConformanceTestSuite,
 		TimeoutConfig:               options.TimeoutConfig,
 		SkipTests:                   sets.New(options.SkipTests...),
 		RunTest:                     options.RunTest,
+		SkipProvisionalTests:        options.SkipProvisionalTests,
 		ManifestFS:                  options.ManifestFS,
 		UsableNetworkAddresses:      options.UsableNetworkAddresses,
 		UnusableNetworkAddresses:    options.UnusableNetworkAddresses,
@@ -431,6 +435,9 @@ func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) er
 		if suite.SkipTests.Has(test.ShortName) {
 			res = testSkipped
 		}
+		if suite.SkipProvisionalTests && test.Provisional {
+			res = testProvisionalSkipped
+		}
 		if !suite.SupportedFeatures.HasAll(test.Features...) {
 			res = testNotSupported
 		}
@@ -486,16 +493,27 @@ func (suite *ConformanceTestSuite) Report() (*confv1.ConformanceReport, error) {
 	}
 	sort.Strings(testNames)
 	profileReports := newReports()
+	succeededProvisionalTestSet := sets.Set[string]{}
 	for _, tN := range testNames {
-		testResult := suite.results[tN]
-		conformanceProfiles := getConformanceProfilesForTest(testResult.test, suite.conformanceProfiles).UnsortedList()
+		tr := suite.results[tN]
+		if tr.result == testProvisionalSkipped {
+			continue
+		}
+		if tr.result == testSucceeded && tr.test.Provisional {
+			succeededProvisionalTestSet.Insert(tN)
+		}
+		conformanceProfiles := getConformanceProfilesForTest(tr.test, suite.conformanceProfiles).UnsortedList()
 		sort.Slice(conformanceProfiles, func(i, j int) bool {
 			return conformanceProfiles[i].Name < conformanceProfiles[j].Name
 		})
 		for _, profile := range conformanceProfiles {
-			profileReports.addTestResults(*profile, testResult)
+			profileReports.addTestResults(*profile, tr)
 		}
 	}
+	var succeededProvisionalTests []string
+	if len(succeededProvisionalTestSet) > 0 {
+		succeededProvisionalTests = sets.List(succeededProvisionalTestSet)
+	}
 
 	profileReports.compileResults(suite.extendedSupportedFeatures, suite.extendedUnsupportedFeatures)
 
@@ -504,12 +522,13 @@ func (suite *ConformanceTestSuite) Report() (*confv1.ConformanceReport, error) {
 			APIVersion: confv1.GroupVersion.String(),
 			Kind:       "ConformanceReport",
 		},
-		Date:              time.Now().Format(time.RFC3339),
-		Mode:              suite.mode,
-		Implementation:    suite.implementation,
-		GatewayAPIVersion: suite.apiVersion,
-		GatewayAPIChannel: suite.apiChannel,
-		ProfileReports:    profileReports.list(),
+		Date:                      time.Now().Format(time.RFC3339),
+		Mode:                      suite.mode,
+		Implementation:            suite.implementation,
+		GatewayAPIVersion:         suite.apiVersion,
+		GatewayAPIChannel:         suite.apiChannel,
+		ProfileReports:            profileReports.list(),
+		SucceededProvisionalTests: succeededProvisionalTests,
 	}, nil
 }