diff --git a/ci-operator/config/openshift/microshift/openshift-microshift-main.yaml b/ci-operator/config/openshift/microshift/openshift-microshift-main.yaml index 33a87fe5b78f2..d838d3ec2b355 100644 --- a/ci-operator/config/openshift/microshift/openshift-microshift-main.yaml +++ b/ci-operator/config/openshift/microshift/openshift-microshift-main.yaml @@ -434,6 +434,31 @@ tests: MICROSHIFT_OS: rhel-9.6 SCENARIO_TYPE: bootc-c2cc workflow: openshift-microshift-e2e-metal-tests +- always_run: false + as: perf-node-density + capabilities: + - intranet + disable_rehearsal: true + optional: true + restrict_network_access: false + steps: + cluster_profile: metal-perfscale-selfsched + env: + BUILD_FROM_PR: "true" + MICROSHIFT_BRANCH: main + MODEL: R640 + NUM_SERVERS: "1" + PROMETHEUS_LOGGING: "true" + QPS: "10" + WORKLOAD: node-density + post: + - ref: openshift-qe-installer-bm-self-sched-terminate + pre: + - ref: openshift-qe-installer-bm-self-sched + - ref: openshift-qe-installer-bm-ping + test: + - ref: openshift-qe-microshift-deploy + - ref: openshift-qe-microshift-kube-burner zz_generated_metadata: branch: main org: openshift diff --git a/ci-operator/config/openshift/microshift/openshift-microshift-release-4.22.yaml b/ci-operator/config/openshift/microshift/openshift-microshift-release-4.22.yaml index c712225b69a12..b846b751baf7b 100644 --- a/ci-operator/config/openshift/microshift/openshift-microshift-release-4.22.yaml +++ b/ci-operator/config/openshift/microshift/openshift-microshift-release-4.22.yaml @@ -376,6 +376,31 @@ tests: MICROSHIFT_OS: rhel-9.8 SCENARIO_TYPE: bootc-c2cc workflow: openshift-microshift-e2e-metal-tests +- always_run: false + as: perf-node-density + capabilities: + - intranet + disable_rehearsal: true + optional: true + restrict_network_access: false + steps: + cluster_profile: metal-perfscale-selfsched + env: + BUILD_FROM_PR: "true" + MICROSHIFT_BRANCH: release-4.22 + MODEL: R640 + NUM_SERVERS: "1" + PROMETHEUS_LOGGING: "true" + QPS: "10" + WORKLOAD: node-density + post: + - ref: openshift-qe-installer-bm-self-sched-terminate + pre: + - ref: openshift-qe-installer-bm-self-sched + - ref: openshift-qe-installer-bm-ping + test: + - ref: openshift-qe-microshift-deploy + - ref: openshift-qe-microshift-kube-burner zz_generated_metadata: branch: release-4.22 org: openshift diff --git a/ci-operator/jobs/openshift/microshift/openshift-microshift-main-presubmits.yaml b/ci-operator/jobs/openshift/microshift/openshift-microshift-main-presubmits.yaml index 943a943bc7f91..3091d375e60bb 100644 --- a/ci-operator/jobs/openshift/microshift/openshift-microshift-main-presubmits.yaml +++ b/ci-operator/jobs/openshift/microshift/openshift-microshift-main-presubmits.yaml @@ -2412,6 +2412,90 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )ocp-full-conformance-serial-rhel-eus,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build05 + context: ci/prow/perf-node-density + decorate: true + decoration_config: + skip_cloning: true + labels: + capability/intranet: intranet + ci-operator.openshift.io/cloud: metal-perfscale-selfsched + ci-operator.openshift.io/cloud-cluster-profile: metal-perfscale-selfsched + ci.openshift.io/generator: prowgen + job-release: "5.0" + name: pull-ci-openshift-microshift-main-perf-node-density + optional: true + rerun_command: /test perf-node-density + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=perf-node-density + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )perf-node-density,?($|\s.*) - agent: kubernetes always_run: true branches: diff --git a/ci-operator/jobs/openshift/microshift/openshift-microshift-release-4.22-presubmits.yaml b/ci-operator/jobs/openshift/microshift/openshift-microshift-release-4.22-presubmits.yaml index 512a0689f53c6..adcceb1223145 100644 --- a/ci-operator/jobs/openshift/microshift/openshift-microshift-release-4.22-presubmits.yaml +++ b/ci-operator/jobs/openshift/microshift/openshift-microshift-release-4.22-presubmits.yaml @@ -2244,6 +2244,90 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )ocp-full-conformance-serial-rhel-eus,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^release-4\.22$ + - ^release-4\.22- + cluster: build05 + context: ci/prow/perf-node-density + decorate: true + decoration_config: + skip_cloning: true + labels: + capability/intranet: intranet + ci-operator.openshift.io/cloud: metal-perfscale-selfsched + ci-operator.openshift.io/cloud-cluster-profile: metal-perfscale-selfsched + ci.openshift.io/generator: prowgen + job-release: "4.22" + name: pull-ci-openshift-microshift-release-4.22-perf-node-density + optional: true + rerun_command: /test perf-node-density + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=perf-node-density + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )perf-node-density,?($|\s.*) - agent: kubernetes always_run: true branches: diff --git a/ci-operator/step-registry/openshift-qe/microshift-deploy/openshift-qe-microshift-deploy-commands.sh b/ci-operator/step-registry/openshift-qe/microshift-deploy/openshift-qe-microshift-deploy-commands.sh index 7f861ceeac5e9..81f654ec1b1b4 100644 --- a/ci-operator/step-registry/openshift-qe/microshift-deploy/openshift-qe-microshift-deploy-commands.sh +++ b/ci-operator/step-registry/openshift-qe/microshift-deploy/openshift-qe-microshift-deploy-commands.sh @@ -4,11 +4,40 @@ set -o nounset set -o pipefail set -x -SSH_ARGS="-i ${CLUSTER_PROFILE_DIR}/jh_priv_ssh_key -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null" -bastion=$(cat ${CLUSTER_PROFILE_DIR}/address) +BUILD_FROM_PR=${BUILD_FROM_PR:-false} MICROSHIFT_PR=${MICROSHIFT_PR:-} REPO_NAME=${REPO_NAME:-} PULL_NUMBER=${PULL_NUMBER:-} +PULL_PULL_SHA=${PULL_PULL_SHA:-} +microshift_git_refspec="" + +if [[ "${BUILD_FROM_PR}" != "true" && "${BUILD_FROM_PR}" != "false" ]]; then + echo "ERROR: BUILD_FROM_PR must be either 'true' or 'false'" + exit 1 +fi + +if [[ "${BUILD_FROM_PR}" == "true" ]]; then + if [[ "${REPO_NAME}" != "microshift" ]]; then + echo "ERROR: BUILD_FROM_PR requires REPO_NAME=microshift" + exit 1 + fi + if [[ ! "${PULL_NUMBER}" =~ ^[0-9]+$ ]]; then + echo "ERROR: BUILD_FROM_PR requires a numeric PULL_NUMBER" + exit 1 + fi + if [[ ! "${PULL_PULL_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "ERROR: BUILD_FROM_PR requires PULL_PULL_SHA to be a full 40-character Git SHA" + exit 1 + fi + if [[ -n "${MICROSHIFT_PR}" ]]; then + echo "ERROR: BUILD_FROM_PR and MICROSHIFT_PR cannot be used together" + exit 1 + fi + microshift_git_refspec="+refs/pull/${PULL_NUMBER}/head:refs/remotes/origin/pr-${PULL_NUMBER}" +fi + +SSH_ARGS="-i ${CLUSTER_PROFILE_DIR}/jh_priv_ssh_key -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null" +bastion=$(cat ${CLUSTER_PROFILE_DIR}/address) LAB=$(cat ${CLUSTER_PROFILE_DIR}/lab) export LAB if [[ -f "${CLUSTER_PROFILE_DIR}/lab_cloud" ]]; then @@ -34,6 +63,7 @@ if [[ -z "${NODES}" ]]; then exit 1 fi echo "Nodes to deploy MicroShift on: $NODES" +first_node=$(printf '%s\n' "${NODES}" | head -n1) # Copy SSH keys from bastion to provisioned nodes echo "Copying SSH keys to provisioned nodes..." @@ -101,10 +131,20 @@ ssh ${SSH_ARGS} root@${bastion} " set -o pipefail git clone https://github.com/openshift/microshift.git --depth=1 --branch=${MICROSHIFT_BRANCH:-main} ${microshift_repo} cd ${microshift_repo} - # MICROSHIFT_PR or PULL_NUMBER can't be set at the same time + # Preserve the legacy PR inputs unless exact presubmit source was requested. if [[ -n '${MICROSHIFT_PR}' ]]; then git pull origin pull/${MICROSHIFT_PR}/head:${MICROSHIFT_PR} --rebase git switch ${MICROSHIFT_PR} + elif [[ '${BUILD_FROM_PR}' == 'true' ]]; then + if ! git fetch origin '${microshift_git_refspec}'; then + echo 'ERROR: Could not fetch MicroShift PR ${PULL_NUMBER}; its head may have moved' + exit 1 + fi + if ! git rev-parse 'refs/remotes/origin/pr-${PULL_NUMBER}' | grep -qx '${PULL_PULL_SHA}'; then + echo 'ERROR: MicroShift PR ${PULL_NUMBER} head moved after this job was created; expected ${PULL_PULL_SHA}' + exit 1 + fi + git checkout --detach '${PULL_PULL_SHA}' elif [[ -n '${PULL_NUMBER}' ]] && [[ '${REPO_NAME}' == 'microshift' ]]; then git pull origin pull/${PULL_NUMBER}/head:${PULL_NUMBER} --rebase git switch ${PULL_NUMBER} @@ -123,6 +163,28 @@ ssh ${SSH_ARGS} root@${bastion} " pip3 install kubernetes " +bastion_checkout_sha="" +pr_microshift_version="" +if [[ "${BUILD_FROM_PR}" == "true" ]]; then + bastion_checkout_sha=$(ssh ${SSH_ARGS} root@${bastion} "git -C '${microshift_repo}' rev-parse HEAD") + microshift_arch=$(ssh ${SSH_ARGS} root@${bastion} "ssh root@${first_node} uname -m") + if [[ ! "${microshift_arch}" =~ ^[a-zA-Z0-9_]+$ ]]; then + echo "ERROR: Unsupported target architecture value '${microshift_arch}'" + exit 1 + fi + version_file="${microshift_repo}/Makefile.version.${microshift_arch}.var" + if ! ssh ${SSH_ARGS} root@${bastion} "test -f '${version_file}'"; then + echo "ERROR: Version file Makefile.version.${microshift_arch}.var not found in the PR checkout" + exit 1 + fi + pr_microshift_version=$(ssh ${SSH_ARGS} root@${bastion} "awk '\$1 == \"OCP_VERSION\" { split(\$NF, version, \".\"); print version[1] \".\" version[2]; exit }' '${version_file}'") + if [[ ! "${pr_microshift_version}" =~ ^[0-9]+\.[0-9]+$ ]]; then + echo "ERROR: Could not derive a major.minor MicroShift version from ${version_file}" + exit 1 + fi + echo "Building MicroShift ${pr_microshift_version} from PR ${PULL_NUMBER} at ${PULL_PULL_SHA}" +fi + # Discover storage layout on target nodes echo "Discovering storage layout on target nodes..." for node in $NODES; do @@ -150,16 +212,28 @@ ssh ${SSH_ARGS} root@${bastion} " set -o pipefail cd ${microshift_repo}/ansible + microshift_version_arg='${MICROSHIFT_VERSION}' + source_build_args=() + if [[ '${BUILD_FROM_PR}' == 'true' ]]; then + microshift_version_arg='${pr_microshift_version}' + source_build_args=( + -e 'build_microshift=true' + -e 'microshift_git_revision=${PULL_PULL_SHA}' + -e 'microshift_git_refspec=${microshift_git_refspec}' + ) + fi + # Run the deployment playbook if [[ -f '${ANSIBLE_PLAYBOOK}' ]]; then ansible-playbook -i ${ANSIBLE_INVENTORY} ${ANSIBLE_PLAYBOOK} \ - -e "microshift_version=${MICROSHIFT_VERSION}" \ + -e \"microshift_version=\${microshift_version_arg}\" \ -e "setup_microshift_host=${SETUP_MICROSHIFT_HOST}" \ -e "install_microshift=${INSTALL_MICROSHIFT}" \ -e "manage_repos=${MANAGE_REPOS}" \ -e "prometheus_logging=${PROMETHEUS_LOGGING}" \ -e "vg_name=${VG_NAME}" \ -e "lvm_disk=${LVM_DISK}" \ + \"\${source_build_args[@]}\" \ -v | tee /tmp/ansible-microshift-deploy-$(date +%s).log else echo 'ERROR: Ansible playbook ${ANSIBLE_PLAYBOOK} not found' @@ -191,11 +265,47 @@ scp -q ${SSH_ARGS} root@${bastion}:/root/$LAB/$LAB_CLOUD/microshift/kubeconfig $ } # Publish handoff files for workload steps -first_node=$(echo "${NODES}" | head -n1) echo "${first_node}" > "${SHARED_DIR}/microshift_node" if [[ "${PROMETHEUS_LOGGING}" == "true" ]]; then # install-logging runs on the [logging] host (localhost = the bastion) echo "http://${bastion}:9091" > "${SHARED_DIR}/prometheus_url" fi +if [[ "${BUILD_FROM_PR}" == "true" ]]; then + node_verification="" + for attempt in 1 2 3; do + if node_verification=$(ssh ${SSH_ARGS} root@${bastion} " + ssh root@${first_node} ' + node_checkout_sha=\$(git -C /root/microshift rev-parse HEAD 2>/dev/null || printf unavailable) + printf \"node_checkout_sha=%s\\n\" \"\${node_checkout_sha}\" + printf \"rpm_query:\\n\" + rpm -q microshift 2>&1 || true + printf \"microshift_version:\\n\" + microshift version 2>&1 || true + ' + "); then + break + fi + echo "WARNING: Could not collect PR source verification from ${first_node} (attempt ${attempt}/3)" + done + if [[ -z "${node_verification}" ]]; then + node_verification=$'node_checkout_sha=unavailable\nrpm_query:\nunavailable\nmicroshift_version:\nunavailable' + fi + node_checkout_sha=${node_verification%%$'\n'*} + node_checkout_sha=${node_checkout_sha#node_checkout_sha=} + node_checkout_sha=${node_checkout_sha:-unavailable} + + mkdir -p "${ARTIFACT_DIR}" + { + printf 'expected_sha=%s\n' "${PULL_PULL_SHA}" + printf 'bastion_checkout_sha=%s\n' "${bastion_checkout_sha}" + printf '%s\n' "${node_verification}" + } > "${ARTIFACT_DIR}/pr-source-verification.txt" + + if [[ "${node_checkout_sha}" != "${PULL_PULL_SHA}" ]]; then + echo "ERROR: Node checkout ${node_checkout_sha} does not match expected PR SHA ${PULL_PULL_SHA}" + exit 1 + fi +fi + echo "MicroShift deployment completed successfully" diff --git a/ci-operator/step-registry/openshift-qe/microshift-deploy/openshift-qe-microshift-deploy-ref.yaml b/ci-operator/step-registry/openshift-qe/microshift-deploy/openshift-qe-microshift-deploy-ref.yaml index 7bd754337690f..303605bffd97c 100644 --- a/ci-operator/step-registry/openshift-qe/microshift-deploy/openshift-qe-microshift-deploy-ref.yaml +++ b/ci-operator/step-registry/openshift-qe/microshift-deploy/openshift-qe-microshift-deploy-ref.yaml @@ -22,6 +22,12 @@ ref: default: "" documentation: |- If defined, use a specific MicroShift PR. When defined, MICROSHIFT_BRANCH gets ignored. + - name: BUILD_FROM_PR + default: "false" + documentation: |- + Build and install MicroShift from the pull request under test instead + of released packages. Requires a MicroShift presubmit context + (REPO_NAME=microshift) with PULL_NUMBER and PULL_PULL_SHA set. - name: ANSIBLE_PLAYBOOK default: "setup-node.yml" documentation: |-