diff --git a/experiment/compatibility-versions/compatibility-versions-feature-gate-test.sh b/experiment/compatibility-versions/compatibility-versions-feature-gate-test.sh index 800edad525b4e..28fbb9c1902e3 100755 --- a/experiment/compatibility-versions/compatibility-versions-feature-gate-test.sh +++ b/experiment/compatibility-versions/compatibility-versions-feature-gate-test.sh @@ -17,27 +17,21 @@ # must be run with a kubernetes checkout in $PWD (IE from the checkout) # Usage: compatibility-versions-feature-gate-test.sh -set -o errexit -o nounset -o xtrace +set -o errexit -o nounset -o pipefail +set -o xtrace # Settings: # GA_ONLY: true - limit to GA APIs/features as much as possible - # false - (default) APIs and features left at defaults # FEATURE_GATES: - # JSON or YAML encoding of a string/bool map: {"FeatureGateA": true, "FeatureGateB": false} - # Enables or disables feature gates in the entire cluster. - # Cannot be used when GA_ONLY=true. # RUNTIME_CONFIG: - # JSON or YAML encoding of a string/string (!) map: {"apia.example.com/v1alpha1": "true", "apib.example.com/v1beta1": "false"} - # Enables API groups in the apiserver via --runtime-config. - # Cannot be used when GA_ONLY=true. # cleanup logic for cleanup on exit @@ -102,7 +96,7 @@ create_cluster() { controllerManager_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\"" apiServer_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\"" kubelet_extra_args=" \"v\": \"${KIND_CLUSTER_LOG_LEVEL}\"" - + if [ -n "$CLUSTER_LOG_FORMAT" ]; then check_structured_log_support "CLUSTER_LOG_FORMAT" scheduler_extra_args="${scheduler_extra_args} @@ -215,214 +209,6 @@ fetch_metrics() { kubectl get --raw /metrics > "${output_file}" } -validate_feature_gates() { - local emulated_version="$1" # e.g. "1.32" - local metrics_file="$2" # path to /metrics - local feature_list="$3" # versioned_feature_list.yaml - local unversioned_feature_list="$4" # unversioned_feature_list.yaml - local results_file="$5" - - echo "Validating features for ${emulated_version}..." - rm -f "${results_file}" - touch "${results_file}" - - # Parse /metrics -> actual_features[featureName] = 0 or 1 - declare -A actual_features - declare -A actual_stages - - while IFS= read -r line; do - # Example line: - # kubernetes_feature_enabled{name="DisableKubeletCloudCredentialProviders",stage="Alpha"} 1 - - # Capture name in group [1], stage in [2], andnumeric value (0 or 1) in [3]. - # NOTE: The capture group for stage="([^"]*)" matches any stage text (including empty). - if [[ "$line" =~ ^kubernetes_feature_enabled\{name=\"([^\"]+)\",stage=\"([^\"]*)\"}.*\ ([0-9]+)$ ]]; then - feature_name="${BASH_REMATCH[1]}" - feature_stage="${BASH_REMATCH[2]}" - feature_value="${BASH_REMATCH[3]}" - - # Store these in two separate maps - actual_features["$feature_name"]="$feature_value" - actual_stages["$feature_name"]="$feature_stage" - fi - done < <(grep '^kubernetes_feature_enabled' "${metrics_file}") - - # Build the "expected" sets from versioned_feature_list.yaml - # => expected_stage[featureName], expected_lock[featureName], expected_value[featureName] - declare -A expected_stage - declare -A expected_lock - declare -A expected_value - - feature_stream="$( - yq e -o=json '.' "${feature_list}" \ - | jq -c '.[]' - )" - - while IFS= read -r feature_entry; do - feature_name=$(echo "${feature_entry}" | jq -r '.name') - specs_json=$(echo "${feature_entry}" | jq -c '.versionedSpecs') - - # Numeric parse for .version vs emulated_version - target_spec="$( - echo "${specs_json}" \ - | jq -r --arg ver "${emulated_version}" ' - [ .[] - | select( - ( .version | sub("^v"; "") | tonumber ) - <= - ($ver | sub("^v"; "") | tonumber) - ) - ] - | last - ' - )" - - # If no matching spec, skip - if [[ -z "$target_spec" || "$target_spec" == "null" ]]; then - continue - fi - - # Read fields - raw_stage=$(echo "$target_spec" | jq -r '.preRelease') - lockToDefault=$(echo "$target_spec" | jq -r '.lockToDefault') - defaultVal=$(echo "$target_spec" | jq -r '.default') - - # Convert defaultVal (true/false) -> 1/0 - local want="0" - if [[ "$defaultVal" == "true" ]]; then - want="1" - fi - - expected_stage["$feature_name"]="${raw_stage^^}" - expected_lock["$feature_name"]="$lockToDefault" - expected_value["$feature_name"]="$want" - done < <(echo "$feature_stream") - - # Build the "expected_unversioned" sets from unversioned_feature_list.yaml - # => expected_unversioned_stage[featureName], expected_unversioned_lock[featureName], expected_unversioned_value[featureName] - declare -A expected_unversioned_stage - declare -A expected_unversioned_lock - declare -A expected_unversioned_value - - unversioned_feature_stream="$( - yq e -o=json '.' "${unversioned_feature_list}" \ - | jq -c '.[]' - )" - - while IFS= read -r unversioned_feature_entry; do - unversioned_feature_name=$(echo "${unversioned_feature_entry}" | jq -r '.name') - unversioned_specs_json=$(echo "${unversioned_feature_entry}" | jq -c '.versionedSpecs') # Although named versionedSpecs in YAML, it's unversioned list. - - # Unversioned should always use the first spec (assuming only one exists or first one is the default) - target_unversioned_spec="$( - echo "${unversioned_specs_json}" \ - | jq -r '.[0]' # Get the first spec - )" - - # If no spec, skip (should not happen in valid file) - if [[ -z "$target_unversioned_spec" || "$target_unversioned_spec" == "null" ]]; then - continue - fi - - # Read fields - these are default values for unversioned features - raw_unversioned_stage=$(echo "$target_unversioned_spec" | jq -r '.preRelease') # Can use this or default to GA - unversioned_lockToDefault=$(echo "$target_unversioned_spec" | jq -r '.lockToDefault') - unversioned_defaultVal=$(echo "$target_unversioned_spec" | jq -r '.default') - - # Convert defaultVal (true/false) -> 1/0 - local unversioned_want="0" - if [[ "$unversioned_defaultVal" == "true" ]]; then - unversioned_want="1" - fi - - expected_unversioned_stage["$unversioned_feature_name"]="$raw_unversioned_stage" - expected_unversioned_lock["$unversioned_feature_name"]="$unversioned_lockToDefault" - expected_unversioned_value["$unversioned_feature_name"]="$unversioned_want" - done < <(echo "$unversioned_feature_stream") - - - # For each "expected" feature (versioned): - # - If missing from /metrics => fail unless stage==ALPHA or lock==true - # - If present & stage!=ALPHA => compare numeric value - for feature_name in "${!expected_stage[@]}"; do - local stage="${expected_stage[$feature_name]}" - local locked="${expected_lock[$feature_name]}" - local want="${expected_value[$feature_name]}" - - local got="${actual_features[$feature_name]:-}" # empty if missing - - # If present, but stage==ALPHA => no checks are done - if [[ "$stage" == "ALPHA" ]]; then - continue - fi - - if [[ -z "$got" ]]; then - # Missing from metrics - if [[ "$locked" == "true" ]]; then - continue - fi - echo "FAIL: expected feature gate '$feature_name' not found in metrics (stage=${stage}, lockToDefault=${locked})" \ - >> "${results_file}" - continue - fi - - # If present, stage!=ALPHA => compare true/false enabled value - if [[ "$got" != "$want" ]]; then - echo "FAIL: feature '$feature_name' expected value $want, got $got" \ - >> "${results_file}" - fi - done - - # For each "expected_unversioned" feature: - # - If missing from /metrics => fail unless stage==ALPHA or lock==true - # - If present => compare numeric value - for unversioned_feature_name in "${!expected_unversioned_stage[@]}"; do - local unversioned_stage="${expected_unversioned_stage[$unversioned_feature_name]}" - local unversioned_locked="${expected_unversioned_lock[$unversioned_feature_name]}" - local unversioned_want="${expected_unversioned_value[$unversioned_feature_name]}" - - local got="${actual_features[$unversioned_feature_name]:-}" # empty if missing - - # If present, but stage==ALPHA => no checks are done - if [[ "$stage" == "ALPHA" ]]; then - continue - fi - - if [[ -z "$got" ]]; then - # Missing from metrics - if [[ "$unversioned_locked" == "true" ]]; then - continue - fi - echo "FAIL: expected unversioned feature gate '$unversioned_feature_name' not found in metrics (lockToDefault=${unversioned_locked})" \ - >> "${results_file}" - continue - fi - - # If present, compare true/false enabled value - if [[ "$got" != "$unversioned_want" ]]; then - echo "FAIL: unversioned feature '$unversioned_feature_name' expected value $unversioned_want, got $got" \ - >> "${results_file}" - fi - done - - # For each actual feature in /metrics not in the "expected" maps (versioned OR unversioned), - # - if it's "1", we fail as "unexpected feature". because new gates not found in previous - # expected gates can only be introduced if they are off by default (0) but not on by default (1) - # NOTE: if the new feature is a client-go feature then we do not fail but continue - for feature_name in "${!actual_features[@]}"; do - if [[ -z "${expected_stage[$feature_name]:-}" ]] && [[ -z "${expected_unversioned_stage[$feature_name]:-}" ]]; then - local got="${actual_features[$feature_name]}" - if [[ "$got" == "1" ]]; then - # Check to see if gate is found in client-go and if so, continue - if grep -q "$feature_name" staging/src/k8s.io/client-go/features/known_features.go; then - continue - fi - echo "FAIL: unexpected feature '$feature_name' found in /metrics, got=1" \ - >> "${results_file}" - fi - fi - done -} main() { TMP_DIR=$(mktemp -d) @@ -431,9 +217,9 @@ main() { export EMULATED_VERSION=$(get_latest_release_version) export PREV_VERSIONED_FEATURE_LIST=${PREV_VERSIONED_FEATURE_LIST:-"release-${EMULATED_VERSION}/test/featuregates_linter/test_data/versioned_feature_list.yaml"} - export UNVERSIONED_FEATURE_LIST=${UNVERSIONED_FEATURE_LIST:-"release-${EMULATED_VERSION}/test/featuregates_linter/test_data/unversioned_feature_list.yaml"} # Add this line + export UNVERSIONED_FEATURE_LIST=${UNVERSIONED_FEATURE_LIST:-"release-${EMULATED_VERSION}/test/featuregates_linter/test_data/unversioned_feature_list.yaml"} -# Create and validate previous cluster + # Create and validate previous cluster git clone --filter=blob:none --single-branch --branch "release-${EMULATED_VERSION}" https://github.com/kubernetes/kubernetes.git "release-${EMULATED_VERSION}" # Build current version @@ -442,11 +228,13 @@ main() { # Create and validate latest cluster KUBECONFIG="${HOME}/.kube/kind-test-config-latest" export KUBECONFIG - create_cluster "${EMULATED_VERSION}" "${ARTIFACTS}/latest-config.yaml" + create_cluster LATEST_METRICS="${ARTIFACTS}/latest_metrics.txt" fetch_metrics "${LATEST_METRICS}" LATEST_RESULTS="${ARTIFACTS}/latest_results.txt" - validate_feature_gates "${EMULATED_VERSION}" "${LATEST_METRICS}" "${PREV_VERSIONED_FEATURE_LIST}" "${UNVERSIONED_FEATURE_LIST}" "${LATEST_RESULTS}" # Pass the new variable + + VALIDATE_SCRIPT="${PWD}/../test-infra/experiment/compatibility-versions/validate-compatibility-versions-feature-gates.sh" + "${VALIDATE_SCRIPT}" "${EMULATED_VERSION}" "${LATEST_METRICS}" "${PREV_VERSIONED_FEATURE_LIST}" "${UNVERSIONED_FEATURE_LIST}" "${LATEST_RESULTS}" # Report results echo "=== Latest Cluster (${EMULATED_VERSION}) Validation ===" @@ -468,4 +256,4 @@ get_latest_release_version() { cut -d- -f2 } -main +main \ No newline at end of file diff --git a/experiment/compatibility-versions/validate-compatibility-versions-feature-gates.sh b/experiment/compatibility-versions/validate-compatibility-versions-feature-gates.sh new file mode 100755 index 0000000000000..a27ead8bcc2ec --- /dev/null +++ b/experiment/compatibility-versions/validate-compatibility-versions-feature-gates.sh @@ -0,0 +1,245 @@ +#!/bin/bash +# Copyright 2025 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script validates feature gates from a Kubernetes cluster's /metrics endpoint +# against expected values defined in versioned and unversioned feature lists. +# +# Usage: validate_feature_gates.sh + +set -o errexit -o nounset -o pipefail + +validate_feature_gates() { + local emulated_version="$1" # e.g. "1.32" + local metrics_file="$2" # path to /metrics + local feature_list="$3" # versioned_feature_list.yaml + local unversioned_feature_list="$4" # unversioned_feature_list.yaml + local results_file="$5" + + echo "Validating features for ${emulated_version}..." + rm -f "${results_file}" + touch "${results_file}" + + # Parse /metrics -> actual_features[featureName] = 0 or 1 + declare -A actual_features + declare -A actual_stages + + while IFS= read -r line; do + # Example line: + # kubernetes_feature_enabled{name="DisableKubeletCloudCredentialProviders",stage="Alpha"} 1 + + # Capture name in group [1], stage in [2], and numeric value (0 or 1) in [3]. + # NOTE: The capture group for stage="([^"]*)" matches any stage text (including empty). + if [[ "$line" =~ ^kubernetes_feature_enabled\{name=\"([^\"]+)\",stage=\"([^\"]*)\"}.*\ ([0-9]+)$ ]]; then + feature_name="${BASH_REMATCH[1]}" + feature_stage="${BASH_REMATCH[2]}" + feature_value="${BASH_REMATCH[3]}" + + # Store these in two separate maps + actual_features["$feature_name"]="$feature_value" + actual_stages["$feature_name"]="$feature_stage" + fi + done < <(grep '^kubernetes_feature_enabled' "${metrics_file}") + + # Build the "expected" sets from versioned_feature_list.yaml + # => expected_stage[featureName], expected_lock[featureName], expected_value[featureName] + declare -A expected_stage + declare -A expected_lock + declare -A expected_value + + feature_stream="$( + yq e -o=json '.' "${feature_list}" \ + | jq -c '.[]' + )" + + while IFS= read -r feature_entry; do + feature_name=$(echo "${feature_entry}" | jq -r '.name') + specs_json=$(echo "${feature_entry}" | jq -c '.versionedSpecs') + + # Numeric parse for .version vs emulated_version + target_spec="$( + echo "${specs_json}" \ + | jq -r --arg ver "${emulated_version}" ' + [ .[] + | select( + ( .version | sub("^v"; "") | tonumber ) + <= + ($ver | sub("^v"; "") | tonumber) + ) + ] + | last + ' + )" + + # If no matching spec, skip + if [[ -z "$target_spec" || "$target_spec" == "null" ]]; then + continue + fi + + # Read fields + raw_stage=$(echo "$target_spec" | jq -r '.preRelease') + lockToDefault=$(echo "$target_spec" | jq -r '.lockToDefault') + defaultVal=$(echo "$target_spec" | jq -r '.default') + + # Convert defaultVal (true/false) -> 1/0 + local want="0" + if [[ "$defaultVal" == "true" ]]; then + want="1" + fi + + expected_stage["$feature_name"]="${raw_stage^^}" + expected_lock["$feature_name"]="$lockToDefault" + expected_value["$feature_name"]="$want" + done < <(echo "$feature_stream") + + # Build the "expected_unversioned" sets from unversioned_feature_list.yaml + # => expected_unversioned_stage[featureName], expected_unversioned_lock[featureName], expected_unversioned_value[featureName] + declare -A expected_unversioned_stage + declare -A expected_unversioned_lock + declare -A expected_unversioned_value + + unversioned_feature_stream="$( + yq e -o=json '.' "${unversioned_feature_list}" \ + | jq -c '.[]' + )" + + while IFS= read -r unversioned_feature_entry; do + unversioned_feature_name=$(echo "${unversioned_feature_entry}" | jq -r '.name') + unversioned_specs_json=$(echo "${unversioned_feature_entry}" | jq -c '.versionedSpecs') # Although named versionedSpecs in YAML, it's unversioned list. + + # Unversioned should always use the first spec (assuming only one exists or first one is the default) + target_unversioned_spec="$( + echo "${unversioned_specs_json}" \ + | jq -r '.[0]' # Get the first spec + )" + + # If no spec, skip (should not happen in valid file) + if [[ -z "$target_unversioned_spec" || "$target_unversioned_spec" == "null" ]]; then + continue + fi + + # Read fields - these are default values for unversioned features + raw_unversioned_stage=$(echo "$target_unversioned_spec" | jq -r '.preRelease') # Can use this or default to GA + unversioned_lockToDefault=$(echo "$target_unversioned_spec" | jq -r '.lockToDefault') + unversioned_defaultVal=$(echo "$target_unversioned_spec" | jq -r '.default') + + # Convert defaultVal (true/false) -> 1/0 + local unversioned_want="0" + if [[ "$unversioned_defaultVal" == "true" ]]; then + unversioned_want="1" + fi + + expected_unversioned_stage["$unversioned_feature_name"]="$raw_unversioned_stage" + expected_unversioned_lock["$unversioned_feature_name"]="$unversioned_lockToDefault" + expected_unversioned_value["$unversioned_feature_name"]="$unversioned_want" + done < <(echo "$unversioned_feature_stream") + + + # For each "expected" feature (versioned): + # - If missing from /metrics => fail unless stage==ALPHA or lock==true + # - If present & stage!=ALPHA => compare numeric value + for feature_name in "${!expected_stage[@]}"; do + local stage="${expected_stage[$feature_name]}" + local locked="${expected_lock[$feature_name]}" + local want="${expected_value[$feature_name]}" + + local got="${actual_features[$feature_name]:-}" # empty if missing + + # If present, but stage==ALPHA => no checks are done + if [[ "$stage" == "ALPHA" ]]; then + continue + fi + + if [[ -z "$got" ]]; then + # Missing from metrics + if [[ "$locked" == "true" ]]; then + continue + fi + echo "FAIL: expected feature gate '$feature_name' not found in metrics (stage=${stage}, lockToDefault=${locked})" \ + >> "${results_file}" + continue + fi + + # If present, stage!=ALPHA => compare true/false enabled value + if [[ "$got" != "$want" ]]; then + echo "FAIL: feature '$feature_name' expected value $want, got $got" \ + >> "${results_file}" + fi + done + + # For each "expected_unversioned" feature: + # - If missing from /metrics => fail unless stage==ALPHA or lock==true + # - If present => compare numeric value + for unversioned_feature_name in "${!expected_unversioned_stage[@]}"; do + local unversioned_stage="${expected_unversioned_stage[$unversioned_feature_name]}" + local unversioned_locked="${expected_unversioned_lock[$unversioned_feature_name]}" + local unversioned_want="${expected_unversioned_value[$unversioned_feature_name]}" + + local got="${actual_features[$unversioned_feature_name]:-}" # empty if missing + + # If present, but stage==ALPHA => no checks are done + if [[ "$unversioned_stage" == "ALPHA" ]]; then + continue + fi + + if [[ -z "$got" ]]; then + # Missing from metrics + if [[ "$unversioned_locked" == "true" ]]; then + continue + fi + echo "FAIL: expected unversioned feature gate '$unversioned_feature_name' not found in metrics (lockToDefault=${unversioned_locked})" \ + >> "${results_file}" + continue + fi + + # If present, compare true/false enabled value + if [[ "$got" != "$unversioned_want" ]]; then + echo "FAIL: unversioned feature '$unversioned_feature_name' expected value $unversioned_want, got $got" \ + >> "${results_file}" + fi + done + + # For each actual feature in /metrics not in the "expected" maps (versioned OR unversioned), + # - if it's "1", we fail as "unexpected feature". because new gates not found in previous + # expected gates can only be introduced if they are off by default (0) but not on by default (1) + # NOTE: if the new feature is a client-go feature then we do not fail but continue + for feature_name in "${!actual_features[@]}"; do + if [[ -z "${expected_stage[$feature_name]:-}" ]] && [[ -z "${expected_unversioned_stage[$feature_name]:-}" ]]; then + local got="${actual_features[$feature_name]}" + if [[ "$got" == "1" ]]; then + # Check to see if gate is found in client-go and if so, continue + if grep -q "$feature_name" staging/src/k8s.io/client-go/features/known_features.go; then + continue + fi + echo "FAIL: unexpected feature '$feature_name' found in /metrics, got=1" \ + >> "${results_file}" + fi + fi + done +} + +# --- Main execution when script is called directly --- +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + # Check arg count + if [[ $# -ne 5 ]]; then + echo "Usage: ${0} " + exit 1 + fi + validate_feature_gates "$1" "$2" "$3" "$4" "$5" + + if grep -q "FAIL" "$5"; then + echo "Validation failures detected" + exit 1 + fi +fi \ No newline at end of file