Merge remote-tracking branch 'origin/stackhpc/2024.1' into feature/jd-grafana-home-dash

Author: dougszumski

.automation

@@ -3,7 +3,14 @@
 
 # See: https://github.com/stackhpc/docker-rally/blob/master/bin/rally-verify-wrapper.sh for a full list of tempest parameters that can be overriden.
 # You can override tempest parameters like so:
-export TEMPEST_CONCURRENCY=2
+
+# The Tempest concurrency determines how many tests can be running at once.
+# Higher values run tests faster but risk running out of resources and failing tests
+# On production systems, Tempest concurrency can usually be set to a high number e.g. 16-64. It is often limited by the number of available floating IPs.
+# On virtualised test environments, compute and networking speeds often limit the concurrency to 1-16 before tests begin to fail due to timeouts.
+export TEMPEST_CONCURRENCY=16
+
+
 # Specify single test whilst experimenting
 #export TEMPEST_PATTERN="${TEMPEST_PATTERN:-tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name}"
 
@@ -21,9 +28,7 @@ if [ ! -z ${KAYOBE_ENVIRONMENT:+x} ]; then
   fi
 
   if [[ "$KAYOBE_ENVIRONMENT" =~ "ci-multinode" ]]; then
-    # SMSLab is currently running with 1G switches. This causes tests using volumes and images to fail if
-    # the concurrency is set too high.
-    export TEMPEST_CONCURRENCY=1
+    export TEMPEST_CONCURRENCY=4
     # Uncomment this to perform a full tempest test
     # export KAYOBE_AUTOMATION_TEMPEST_LOADLIST=tempest-full
     # export KAYOBE_AUTOMATION_TEMPEST_SKIPLIST=ci-multinode-tempest-full

.automation.conf/config.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -euE
+set -o pipefail
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+KAYOBE_AUTOMATION_DIR="$(realpath "${PARENT}/../../.automation")"
+
+function main {
+	if [ "${PULP_DO_CONTAINER_SYNC:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml' -e stackhpc_pulp_images_kolla_filter="${PULP_KOLLA_FILTER:-}"
+	fi
+	if [ "${PULP_DO_CONTAINER_PUBLISH:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml' -e stackhpc_pulp_images_kolla_filter="${PULP_KOLLA_FILTER:-}"
+	fi
+	if [ "${PULP_DO_REPO_SYNC:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml'
+	fi
+	if [ "${PULP_DO_REPO_PUBLISH:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml'
+	fi
+	if [ "${PULP_DO_REPO_PROMOTE:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-promote-production.yml'
+	fi
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+	main
+fi

.automation.conf/run-books/pulp-sync-content.sh

@@ -10,7 +10,7 @@ v3_endpoint_type = publicURL
 [compute]
 min_compute_nodes = 2
 min_microversion = 2.1
-max_microversion = 2.95
+max_microversion = 2.96
 
 [service-clients]
 http_timeout = 600
@@ -26,10 +26,10 @@ console_output = true
 storage_protocol = ceph
 build_timeout = 600
 min_microversion = 3.0
-max_microversion = 3.70
+max_microversion = 3.71
 
 [image]
 build_timeout = 600
 
 [dashboard]
-dashboard_url = http://192.168.39.2
+dashboard_url = https://192.168.39.2

.automation.conf/tempest/tempest-ci-multinode.overrides.conf

@@ -1,5 +1,5 @@
 # This file is a list of path filters for the PR workflow in .github/workflows/stackhpc-pull-request.yml.
-aio:
+aio: &aio
   - '.automation'
   - '.automation.conf/config.sh'
   - '.automation.conf/tempest/load-lists/default'
@@ -20,6 +20,11 @@ aio:
   - 'kayobe-env'
   - 'requirements.txt'
   - 'terraform/aio/**'
-check-tags:
+check-tags: &check-tags
   - '.github/workflows/stackhpc-check-tags.yml'
   - 'etc/kayobe/kolla-image-tags.yml'
+  - 'etc/kayobe/pulp.yml'
+  - 'tools/kolla-images.py'
+build-kayobe-image:
+  - *aio
+  - *check-tags

.github/path-filters.yml

@@ -0,0 +1,81 @@
+# Generate inputs for the reusable multinode.yml workflow.
+# The test scenario is randomly selected.
+# The inputs are printed to stdout in GitHub step output key=value format.
+
+from dataclasses import dataclass
+import random
+import typing as t
+
+
+@dataclass
+class OSRelease:
+    distribution: str
+    release: str
+    ssh_username: str
+
+
+@dataclass
+class OpenStackRelease:
+    version: str
+    previous_version: str
+    os_releases: t.List[OSRelease]
+
+
+@dataclass
+class Scenario:
+    openstack_release: OpenStackRelease
+    os_release: OSRelease
+    neutron_plugin: str
+    upgrade: bool
+
+
+ROCKY_9 = OSRelease("rocky", "9", "cloud-user")
+UBUNTU_JAMMY = OSRelease("ubuntu", "jammy", "ubuntu")
+# NOTE(upgrade): Add supported releases here.
+OPENSTACK_RELEASES = [
+    OpenStackRelease("2024.1", "2023.1", [ROCKY_9, UBUNTU_JAMMY]),
+    OpenStackRelease("2023.1", "zed", [ROCKY_9, UBUNTU_JAMMY]),
+]
+NEUTRON_PLUGINS = ["ovs", "ovn"]
+
+
+def main() -> None:
+    scenario = random_scenario()
+    inputs = generate_inputs(scenario)
+    for name, value in inputs.items():
+        write_output(name, value)
+
+
+def random_scenario() -> Scenario:
+    openstack_release = random.choice(OPENSTACK_RELEASES)
+    os_release = random.choice(openstack_release.os_releases)
+    neutron_plugin = random.choice(NEUTRON_PLUGINS)
+    upgrade = random.random() > 0.6
+    return Scenario(openstack_release, os_release, neutron_plugin, upgrade)
+
+
+def generate_inputs(scenario: Scenario) -> t.Dict[str, str]:
+    branch = get_branch(scenario.openstack_release.version)
+    previous_branch = get_branch(scenario.openstack_release.previous_version)
+    inputs = {
+        "os_distribution": scenario.os_release.distribution,
+        "os_release": scenario.os_release.release,
+        "ssh_username": scenario.os_release.ssh_username,
+        "neutron_plugin": scenario.neutron_plugin,
+        "upgrade": str(scenario.upgrade).lower(),
+        "stackhpc_kayobe_config_version": branch,
+        "stackhpc_kayobe_config_previous_version": previous_branch,
+    }
+    return inputs
+
+
+def get_branch(version: str) -> str:
+    return f"stackhpc/{version}"
+
+
+def write_output(name: str, value: str) -> None:
+    print(f"{name}={value}")
+
+
+if __name__ == "__main__":
+    main()