Separate kyverno-json tests

.chainsaw-config-kyverno-json.yaml

@@ -0,0 +1,21 @@
+apiVersion: chainsaw.kyverno.io/v1alpha2
+kind: Configuration
+metadata:
+  creationTimestamp: null
+  name: configuration
+spec:
+  timeouts:
+    apply: 1m30s
+    assert: 1m30s
+    cleanup: 2m30s
+    delete: 1m30s
+    error: 1m30s
+    exec: 1m30s
+  execution:
+    parallel: 1
+    forceTerminationGracePeriod: 5s
+  cleanup:
+    delayBeforeCleanup: 3s
+  discovery:
+    fullName: true
+    includeTestRegex: 'dockerfile-*'

.chainsaw-config-kyverno.yaml

@@ -0,0 +1,21 @@
+apiVersion: chainsaw.kyverno.io/v1alpha2
+kind: Configuration
+metadata:
+  creationTimestamp: null
+  name: configuration
+spec:
+  timeouts:
+    apply: 1m30s
+    assert: 1m30s
+    cleanup: 2m30s
+    delete: 1m30s
+    error: 1m30s
+    exec: 1m30s
+  execution:
+    parallel: 1
+    forceTerminationGracePeriod: 5s
+  cleanup:
+    delayBeforeCleanup: 3s
+  discovery:
+    fullName: true
+    excludeTestRegex: 'dockerfile-*'

.github/workflows/chainsaw-e2e.yaml

@@ -9,11 +9,11 @@ on:
       - main
 
 jobs:
-  run-e2etest-1:
+  run-e2etest-n4k-1-10:
     runs-on: ubuntu-latest
 
     strategy:
-      fail-fast: false    
+      fail-fast: false
       matrix:
         k8s-version: [v1.29.2, v1.28.0, v1.27.3, v1.26.3, v1.25.8, v1.24.12]
         n4k-chart-version: [3.0.29]
@@ -22,11 +22,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Install kyverno-json
-        uses: kyverno/action-install-kyverno-json@4ac28215fd015ff6aa5525afc67f119f9f758957 # v0.0.1
-        with:
-          release: main
-
       - name: Prepare environment
         run: |
           K8S_VERSION=${{ matrix.k8s-version }} make kind-create-cluster
@@ -39,32 +34,27 @@ jobs:
         run: make wait-for-kyverno
 
       - name: Install chainsaw
-        uses: kyverno/action-install-chainsaw@v0.1.6
+        uses: kyverno/action-install-chainsaw@v0.2.4
 
       - name: Verify Chainsaw Installation
         run: chainsaw version
 
       - name: Test with Chainsaw
-        run: make test-chainsaw
+        run: CHAINSAW_CONFIG=".chainsaw-config-kyverno.yaml" make test-chainsaw
 
-  run-e2etest-2:
+  run-e2etest-n4k-1-11:
     runs-on: ubuntu-latest
 
     strategy:
-      fail-fast: false    
+      fail-fast: false
       matrix:
         k8s-version: [v1.29.2, v1.28.0, v1.27.3, v1.26.3, v1.25.8]
-        n4k-chart-version: [3.1.14]
+        n4k-chart-version: [3.1.17]
 
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Install kyverno-json
-        uses: kyverno/action-install-kyverno-json@4ac28215fd015ff6aa5525afc67f119f9f758957 # v0.0.1
-        with:
-          release: main
-
       - name: Prepare environment
         run: |
           K8S_VERSION=${{ matrix.k8s-version }} make kind-create-cluster
@@ -77,10 +67,31 @@ jobs:
         run: make wait-for-kyverno
 
       - name: Install chainsaw
-        uses: kyverno/[email protected]
+        uses: kyverno/[email protected]
+
+      - name: Verify Chainsaw Installation
+        run: chainsaw version
+
+      - name: Test with Chainsaw
+        run: CHAINSAW_CONFIG=".chainsaw-config-kyverno.yaml" make test-chainsaw
+
+  run-e2etest-kyverno-json:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install kyverno-json
+        uses: kyverno/action-install-kyverno-json@4ac28215fd015ff6aa5525afc67f119f9f758957 # v0.0.1
+        with:
+          release: main
+
+      - name: Install chainsaw
+        uses: kyverno/[email protected]
 
       - name: Verify Chainsaw Installation
         run: chainsaw version
 
       - name: Test with Chainsaw
-        run: make test-chainsaw
+        run: CHAINSAW_CONFIG=".chainsaw-config-kyverno-json.yaml" NO_CLUSTER="--no-cluster" make test-chainsaw

Makefile

@@ -4,6 +4,8 @@ K8S_VERSION          ?= $(shell kubectl version --short | grep -i server | cut -
 KIND_IMAGE           ?= kindest/node:$(K8S_VERSION)
 KIND_NAME            ?= kind
 USE_CONFIG           ?= standard
+CHAINSAW_CONFIG 	 ?= .chainsaw-config-kyverno.yaml
+NO_CLUSTER 			 ?= ""
 
 TOOLS_DIR                          := $(PWD)/.tools
 KIND                               := $(TOOLS_DIR)/kind
@@ -35,7 +37,7 @@ clean-tools:
 .PHONY: test-chainsaw
 test-chainsaw:  
 	@echo Running chainsaw tests... >&2
-	@chainsaw test --config .chainsaw-config.yaml
+	@chainsaw test --config $(CHAINSAW_CONFIG) $(NO_CLUSTER)
 
 ## Create kind cluster
 .PHONY: kind-create-cluster

multitenancy-benchmarks/require-namespace-quota/.kyverno-test/kyverno-test.yaml

@@ -1,16 +1,16 @@
-apiVersion: cli.kyverno.io/v1alpha1
-kind: Test
-metadata:
-  name: kyverno-test.yaml
-policies:
-- ../require-namespace-quota.yaml
-resources:
-- resource.yaml
-results:
-- kind: Namespace
-  policy: require-namespace-quota
-  resources:
-  - testing-ns
-  result: pass
-  rule: resourcequotas
-variables: values.yaml
+# apiVersion: cli.kyverno.io/v1alpha1
+# kind: Test
+# metadata:
+#   name: kyverno-test.yaml
+# policies:
+# - ../require-namespace-quota.yaml
+# resources:
+# - resource.yaml
+# results:
+# - kind: Namespace
+#   policy: require-namespace-quota
+#   resources:
+#   - testing-ns
+#   result: pass
+#   rule: resourcequotas
+# variables: values.yaml

multitenancy-benchmarks/require-quota-for-all-objects/.kyverno-test/kyverno-test.yaml

@@ -1,16 +1,16 @@
-apiVersion: cli.kyverno.io/v1alpha1
-kind: Test
-metadata:
-  name: kyverno-test.yaml
-policies:
-- ../require-quota-for-all-objects.yaml
-resources:
-- resource.yaml
-results:
-- kind: Namespace
-  policy: require-quota-for-all-objects
-  resources:
-  - ns-resource-quota
-  result: pass
-  rule: resourcequotas
-variables: values.yaml
+# apiVersion: cli.kyverno.io/v1alpha1
+# kind: Test
+# metadata:
+#   name: kyverno-test.yaml
+# policies:
+# - ../require-quota-for-all-objects.yaml
+# resources:
+# - resource.yaml
+# results:
+# - kind: Namespace
+#   policy: require-quota-for-all-objects
+#   resources:
+#   - ns-resource-quota
+#   result: pass
+#   rule: resourcequotas
+# variables: values.yaml

pod-security/restricted/require-run-as-nonroot/e2e/good-resource.yaml

@@ -184,8 +184,8 @@ spec:
       containers:
       - name: container01
         image: nginx
-        securityContext:
-          runAsNonRoot: true
+      securityContext:
+        runAsNonRoot: true
 ---
 apiVersion: apps/v1
 kind: Deployment