diff --git a/docs/run-the-playbooks-for-hcp-kubevirt-disconnected.md b/docs/run-the-playbooks-for-hcp-kubevirt-disconnected.md new file mode 100644 index 000000000..0671b62b6 --- /dev/null +++ b/docs/run-the-playbooks-for-hcp-kubevirt-disconnected.md @@ -0,0 +1,312 @@ +# Disconnected HCP KubeVirt Installation + +This guide covers the end-to-end steps to install a Hosted Control Plane (HCP) on KubeVirt in a disconnected (air-gapped) environment using the `hcpvirt.yaml` master playbook. + +--- + +## Prerequisites + +- An OpenShift management cluster is up and reachable from the bastion. +- A mirror registry (e.g. Quay) is running and accessible from the bastion and the management cluster. +- The bastion host is reachable via SSH from the Ansible controller. +- `oc` CLI is installed on the bastion. +- `oc-mirror` v2 plugin is installed on the mirror host (bastion). + +--- + +## Configuration Files + +### `inventories/default/group_vars/disconnected.yaml` + +Set `enabled: true` and fill in all registry fields: + +```yaml +disconnected: + enabled: true + registry: + url: ':' # e.g. sno-bastion.sno.com:8443 + ip: '' + pull_secret: '' # auth for the mirror registry only + mirror_pull_secret: '' # auth for all source registries + mirror + ca_trusted: false + ca_cert: | + -----BEGIN CERTIFICATE----- + + -----END CERTIFICATE----- + mirroring: + host: + name: + ip: + user: root + cluster_resources_dir: /root/ocpinst_disconnected/working-dir/cluster-resources + oc_mirror: + image_set: + apiVersion: mirror.openshift.io/v2alpha1 + mirror: + platform: + channels: + - name: stable-4.21 + minVersion: 4.21.0 + maxVersion: 4.21.0 + operators: + - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.21 + packages: + - name: multicluster-engine + - name: metallb-operator + - name: kubevirt-hyperconverged +``` + +> **Note:** `cluster_resources_dir` is where `oc-mirror v2` writes the cluster manifests +> (`idms-*.yaml`, `itms-*.yaml`, `cs-*.yaml`) after mirroring. Default: +> `/root/ocpinst_disconnected/working-dir/cluster-resources` + +--- + +### `inventories/default/group_vars/hcp-kubevirt.yaml` + +Key fields for disconnected installation: + +```yaml +hcp: + oc_url: https://api.:6443 + mgmt_cluster_bastion: + ip: + ansible_key_name: ansible-ocpz # SSH key generated on the bastion if absent + + mce: + version: 2.11 + mce_namespace: multicluster-engine + catalogsource_image: "" # Leave empty — disconnected CatalogSource is used + + metallb: + version: stable + ip_pool: + - # e.g. 172.23.232.232-172.23.232.234 + catalogsource_image: "" # Leave empty — disconnected CatalogSource is used + + ocpvirt: + version: stable + + storage: + type: hpp # hpp or odf + + hpp: + name: hostpath-provisioner + storageclass_name: hostpath-csi + storage_size: 50Gi + storage_path: /var/hpvolumes + + control_plane: + high_availability: false + clusters_namespace: + hosted_cluster_name: + basedomain: + ocp_release_image: 4.21.0-multi # Must match the version mirrored + arch: s390x + pull_secret: '' + + data_plane: + compute_count: 1 + cores: 4 + memory: 16Gi + root_volume_size: 60 +``` + +> **Important:** `catalogsource_image` fields for all operators must be left empty (`""`). +> In disconnected mode the CatalogSource applied from `oc-mirror` output is used automatically. + +--- + +### `inventories/default/group_vars/secrets.yaml` + +```yaml +vault_ocp_user: +vault_ocp_password: +vault_bastion_root_pass: +``` + +--- + +## Playbook Flow + +Run the master playbook: + +```bash +cd Ansible-OpenShift-Provisioning +ansible-playbook playbooks/hcpvirt.yaml +``` + +The master playbook executes the following sub-playbooks in order: + +``` +hcpvirt.yaml +├── 1. setup_inventory_hcp_kubevirt.yaml (always) +├── 2. disconnected_mirror_hcp_artifacts.yaml (disconnected.enabled == true) +├── 3. disconnected_hcp_kubevirt_setup.yaml (disconnected.enabled == true) +├── 4. hcp_kubevirt_prereqs_setup.yaml (always) +└── 5. create_hcpvirt.yaml (always) +``` + +--- + +## What Each Playbook Does + +### 1. `setup_inventory_hcp_kubevirt.yaml` +Sets up the Ansible inventory and SSH key configuration for the bastion. + +--- + +### 2. `disconnected_mirror_hcp_artifacts.yaml` +Runs `oc-mirror v2` on the mirror host to mirror all required images to the local registry. + +After mirroring completes, the following cluster manifests are written to +`/root/ocpinst_disconnected/working-dir/cluster-resources/` on the bastion: + +| File | Purpose | +|---|---| +| `idms-oc-mirror.yaml` | ImageDigestMirrorSet for release images | +| `itms-oc-mirror.yaml` | ImageTagMirrorSet for operator images | +| `cs-redhat-operator-index-v4-21.yaml` | CatalogSource for disconnected operators | +| `cc-redhat-operator-index-v4-21.yaml` | CatalogSource config | + +--- + +### 3. `disconnected_hcp_kubevirt_setup.yaml` +Applies the disconnected configuration to the management cluster. Executes the +`disconnected_apply_hcp_manifests` role which performs the following steps in order: + +**Step 1 — Registry CA ConfigMap** +Creates the `registry-config` ConfigMap in `openshift-config` with the mirror +registry CA certificate. The ConfigMap key uses `..` in place of `:` for the +port separator (e.g. `sno-bastion.sno.com..8443`): + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: registry-config + namespace: openshift-config +data: + sno-bastion.sno.com..8443: | + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- +``` + +**Step 2 — Patch image.config** +Patches the cluster image config to trust the mirror registry CA: + +```bash +oc patch image.config.openshift.io/cluster --type=merge \ + -p '{"spec":{"additionalTrustedCA":{"name":"registry-config"}}}' +``` + +**Step 3 — Wait for MCP and ClusterOperators** +- Pauses 2 minutes for the MCO to begin reconciling. +- Waits for `master` and `worker` MachineConfigPools to reach `Updated=True`. +- Polls until no ClusterOperator has `Degraded=True`. + +**Step 4 — Apply mirror manifests** +Applies all manifests from the `cluster-resources` directory in this order: + +1. `idms-*.yaml` — ImageDigestMirrorSets from oc-mirror +2. `idms-capi.yaml` — Additional IDMS for CAPI/importer pod images: + ```yaml + # mirrors quay.io/openshift-release-dev/ocp-v4.0-art-dev + # → //ocp-v4.0-art-dev + ``` +3. `itms-*.yaml` — ImageTagMirrorSets from oc-mirror +4. Patches `OperatorHub` to disable all default sources +5. `cs-*.yaml` — CatalogSources from oc-mirror +6. Waits for all CatalogSources to reach `READY` state + +--- + +### 4. `hcp_kubevirt_prereqs_setup.yaml` +Installs the required operators on the management cluster. + +Before installing operators, a pre-task reads the CatalogSource name from the +`cluster-resources` directory and sets it as the source for all Subscriptions: + +``` +/root/ocpinst_disconnected/working-dir/cluster-resources/cs-*.yaml + → disconnected_catalogsource_name = cs-redhat-operator-index-v4-21 +``` + +Operators installed (each Subscription points to the disconnected CatalogSource): + +| Operator | Namespace | +|---|---| +| MultiClusterEngine (MCE) | `multicluster-engine` | +| MetalLB | `metallb-system` | +| OpenShift Virtualization | `openshift-cnv` | +| HostPath Provisioner (HPP) | cluster-scoped *(when `storage.type: hpp`)* | +| LSO + ODF | `openshift-local-storage` / `openshift-storage` *(when `storage.type: odf`)* | + +--- + +### 5. `create_hcpvirt.yaml` +Creates the Hosted Control Plane cluster. Key steps in the `create_hcp_kubevirt` role: + +**ICSP file** rendered from the disconnected registry mirrors: +```yaml +- mirrors: + - /openshift/release + source: quay.io/openshift-release-dev/ocp-v4.0-art-dev +- mirrors: + - /openshift/release-images + source: quay.io/openshift-release-dev/ocp-release +``` + +**`hcp create cluster kubevirt` command** with disconnected flags: +```bash +hcp create cluster kubevirt \ + --name= \ + --namespace= \ + --arch=s390x \ + --pull-secret=/root/ansible_workdir/auth_file \ + --base-domain= \ + --image-content-sources /root/ansible_workdir/icsp.yaml \ + --release-image=/openshift/release-images: \ + --additional-trust-bundle /root/ansible_workdir/mirror-registry-ca.crt \ + --olm-disable-default-sources \ + --control-plane-availability-policy SingleReplica \ + --infra-availability-policy SingleReplica \ + --node-pool-replicas= \ + --memory= \ + --cores= \ + --root-volume-size= +``` + +--- + +## Running Individual Playbooks + +You can also run each playbook independently if needed: + +```bash +# Step 1 – Mirror images to local registry +ansible-playbook playbooks/disconnected_mirror_hcp_artifacts.yaml + +# Step 2 – Apply mirror manifests to management cluster +ansible-playbook playbooks/disconnected_hcp_kubevirt_setup.yaml + +# Step 3 – Install prerequisite operators +ansible-playbook playbooks/hcp_kubevirt_prereqs_setup.yaml + +# Step 4 – Create the HCP cluster +ansible-playbook playbooks/create_hcpvirt.yaml +``` + +--- + +## Troubleshooting + +| Symptom | Likely Cause | Fix | +|---|---|---| +| `cluster-resources directory not found` | Mirroring step not completed | Run `disconnected_mirror_hcp_artifacts.yaml` first | +| `ConfigMap invalid: data key contains ':'` | Registry URL used as-is | Ensure `disconnected.registry.url` does not have extra colons; the role replaces `:` with `..` automatically | +| CatalogSource not READY | Registry unreachable from cluster nodes | Verify `disconnected.registry.ca_cert` is correct and nodes can reach the mirror registry | +| Operator CSV stuck in `Pending` | Wrong CatalogSource name in Subscription | Check the `cs-*.yaml` filename in `cluster-resources`; the name is auto-discovered | +| `hcp` binary not found | MCE pod layout differs between versions | The role tries `hcp-linux-s390x.tar.gz` first, then falls back to the `linux/s390x/` directory | +| `cat ~/.ssh/ansible-ocpz.pub: No such file` | Key not yet generated | The role auto-generates the key with `ssh-keygen` if absent | diff --git a/docs/run-the-playbooks-for-hcp-kubevirt.md b/docs/run-the-playbooks-for-hcp-kubevirt.md index 7b5fa3680..30f4856d8 100644 --- a/docs/run-the-playbooks-for-hcp-kubevirt.md +++ b/docs/run-the-playbooks-for-hcp-kubevirt.md @@ -166,7 +166,21 @@ The `mce.catalogsource_image` parameter in `hcp-kubevirt.yaml` is optional: - **If empty or not provided**: The MCE operator will be installed from the default Red Hat Operators catalog -### FIPS Validation +### Additional Flags (`control_plane.additional_flags`) + +The `additional_flags` parameter in `hcp-kubevirt.yaml` passes extra flags directly to the `hcp create cluster kubevirt` command. Multiple flags can be combined in a single string. + +#### HPP storage class when ODF is also installed + +By default, the `hcp` CLI selects the first available default StorageClass. If both ODF and HPP are present on the management cluster, you must explicitly tell HCP to use the HPP StorageClass for node root volumes — otherwise ODF's StorageClass will be picked: + +```yaml +control_plane: + additional_flags: "--root-volume-storage-class hostpath-csi" +``` + +#### FIPS Validation + For FIPS-validated deployments: - The management cluster must be installed with FIPS mode enabled - In the `hcp-kubevirt.yaml` file, set the `additional_flags` parameter to enable FIPS for the hosted cluster: @@ -175,3 +189,10 @@ For FIPS-validated deployments: additional_flags: "--fips" ``` - The hosted cluster will be created with FIPS mode enabled + +#### Combining multiple flags + +```yaml +control_plane: + additional_flags: "--fips --root-volume-storage-class hostpath-csi" +``` diff --git a/inventories/default/group_vars/disconnected.yaml b/inventories/default/group_vars/disconnected.yaml index a94da6dfd..bb657ae08 100644 --- a/inventories/default/group_vars/disconnected.yaml +++ b/inventories/default/group_vars/disconnected.yaml @@ -16,7 +16,7 @@ disconnected: mirroring: # this is the host that can access the internet as well as the registry host: - name: hosname + name: ip: x.x.x.x user: mirroruser # with become access pass: mirrorpassword @@ -28,8 +28,10 @@ disconnected: oc_mirror_tgz: 'oc-mirror.tar.gz' # this will download oc binary to the mirror host for use on the mirror host for mirroring client_download: - ocp_download_url: "https://mirror.openshift.com/pub/openshift-v4/multi/clients/ocp/4.13.1/amd64/" + ocp_download_url: "https://mirror.openshift.com/pub/openshift-v4/s390x/clients/ocp/4.22.0/" ocp_client_tgz: 'openshift-client-linux.tar.gz' + oc_mirror_url: "https://mirror.openshift.com/pub/openshift-v4/s390x/clients/ocp/stable/" + oc_mirror_tgz: 'oc-mirror.tar.gz' legacy: platform: false ocp_quay_release_image_tag: '4.13.1-s390x' @@ -47,7 +49,7 @@ disconnected: list: - regexp: what replace: with - release_image_tag: '4.13.1-multi' + release_image_tag: '4.21.0-multi' release_image_registry: 'brew.registry.redhat.io' image_set: # this field is a standard image set from oc-mirror documentation. @@ -68,15 +70,16 @@ disconnected: # Note: cannot mirror tags like multi-s390x; list must contain pure # architecture names (e.g. s390x) or the multiarch token 'multi'. channels: - - name: stable-4.13 + - name: stable-4.21 full: false - minVersion: 4.13.1 - maxVersion: 4.13.1 + minVersion: 4.21.0 + maxVersion: 4.21.0 + kubeVirtContainer: true operators: - - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.13 + - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.21 full: false packages: - - name: serverless-operator + - name: multicluster-engine channels: - name: stable # minVersion: '2.4.1-0' diff --git a/playbooks/create_hcpvirt.yaml b/playbooks/create_hcpvirt.yaml index 287817962..5e768f9f7 100644 --- a/playbooks/create_hcpvirt.yaml +++ b/playbooks/create_hcpvirt.yaml @@ -3,8 +3,8 @@ - name: Add Bastion Host to Inventory hosts: localhost vars_files: - - "{{ inventory_dir }}/group_vars/secrets.yaml" - - "{{ inventory_dir }}/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" tasks: - name: Add bastion dynamically ansible.builtin.add_host: @@ -13,14 +13,14 @@ ansible_user: root ansible_ssh_private_key_file: "{{ lookup('env', 'HOME') + '/.ssh/' + (hcp.ansible_key_name | default('ansible-ocpz', true)) }}" ansible_connection: ssh - inventory_dir: "{{ inventory_dir }}" + inventory_dir: "{{ playbook_dir }}/../inventories/default" # Login to OCP Management Cluster (from bastion) - name: Login to OCP Management Cluster hosts: mgmt_bastion vars_files: - - "{{ inventory_dir }}/group_vars/secrets.yaml" - - "{{ inventory_dir }}/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" vars: vault_api_server: "{{ hcp.oc_url }}" vault_user_name: "{{ vault_ocp_user }}" @@ -32,8 +32,8 @@ - name: Pre HCP Creation - Enable Wildcard DNS Routes hosts: mgmt_bastion vars_files: - - "{{ inventory_dir }}/group_vars/secrets.yaml" - - "{{ inventory_dir }}/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" become: true gather_facts: false tasks: @@ -54,10 +54,15 @@ - name: Create Hosted Control Plane Cluster hosts: mgmt_bastion vars_files: - - "{{ inventory_dir }}/group_vars/secrets.yaml" - - "{{ inventory_dir }}/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" become: true gather_facts: false + pre_tasks: + - name: Load disconnected vars if file exists + ansible.builtin.include_vars: + file: "{{ playbook_dir }}/../inventories/default/group_vars/disconnected.yaml" + failed_when: false roles: - create_hcp_kubevirt @@ -65,8 +70,8 @@ - name: Post HCP Creation - Extract kubeconfig hosts: mgmt_bastion vars_files: - - "{{ inventory_dir }}/group_vars/secrets.yaml" - - "{{ inventory_dir }}/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" become: true gather_facts: false roles: @@ -75,8 +80,8 @@ - name: Fetching the HTTP and HTTPS NodePort hosts: mgmt_bastion vars_files: - - "{{ inventory_dir }}/group_vars/secrets.yaml" - - "{{ inventory_dir }}/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" become: true gather_facts: false tasks: @@ -123,11 +128,9 @@ hosts: mgmt_bastion become: true gather_facts: false - vars_files: - - "{{ inventory_dir }}/group_vars/secrets.yaml" - - "{{ inventory_dir }}/group_vars/hcp-kubevirt.yaml" - + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" roles: - hcpvirt_loadbalancer_dns @@ -136,7 +139,7 @@ become: true gather_facts: false vars_files: - - "{{ inventory_dir }}/group_vars/secrets.yaml" - - "{{ inventory_dir }}/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" roles: - wait_for_hc_complete diff --git a/playbooks/disconnected_hcp_kubevirt_setup.yaml b/playbooks/disconnected_hcp_kubevirt_setup.yaml new file mode 100644 index 000000000..c4ca96413 --- /dev/null +++ b/playbooks/disconnected_hcp_kubevirt_setup.yaml @@ -0,0 +1,46 @@ +--- +# Playbook for setting up HCP KubeVirt in disconnected environment +# This playbook applies IDMS, ITMS, CatalogSources and installs operators + +# Dynamically Add Bastion Host +- name: Add Bastion Host to Inventory + hosts: localhost + vars_files: + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/disconnected.yaml" + tasks: + - name: Add bastion dynamically + ansible.builtin.add_host: + name: "{{ hcp.mgmt_cluster_bastion.ip }}" + groups: mgmt_bastion + ansible_user: root + ansible_ssh_private_key_file: "{{ lookup('env', 'HOME') + '/.ssh/' + (hcp.ansible_key_name | default('ansible-ocpz', true)) }}" + ansible_connection: ssh + inventory_dir: "{{ playbook_dir }}/../inventories/default" + +# Login to OCP Management Cluster (from bastion) +- name: Login to OCP Management Cluster + hosts: mgmt_bastion + vars_files: + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/disconnected.yaml" + vars: + vault_api_server: "{{ hcp.oc_url }}" + vault_user_name: "{{ vault_ocp_user }}" + vault_password: "{{ vault_ocp_password }}" + roles: + - login_to_mgmt_cluster + +# Apply Disconnected Manifests (IDMS, ITMS, CatalogSource) +- name: Apply Disconnected Manifests for HCP + hosts: mgmt_bastion + vars_files: + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/disconnected.yaml" + become: true + gather_facts: false + roles: + - disconnected_apply_hcp_manifests diff --git a/playbooks/disconnected_mirror_hcp_artifacts.yaml b/playbooks/disconnected_mirror_hcp_artifacts.yaml new file mode 100644 index 000000000..c19c7016e --- /dev/null +++ b/playbooks/disconnected_mirror_hcp_artifacts.yaml @@ -0,0 +1,34 @@ +--- +- name: Add Mirror Host to Inventory + hosts: localhost + vars_files: + - "{{ playbook_dir }}/../inventories/default/group_vars/disconnected.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" + tasks: + - name: Add mirror host dynamically + ansible.builtin.add_host: + name: "{{ disconnected.mirroring.host.ip }}" + groups: mirror_host + ansible_user: "{{ disconnected.mirroring.host.user }}" + ansible_ssh_pass: "{{ disconnected.mirroring.host.pass }}" + ansible_connection: ssh + inventory_dir: "{{ playbook_dir }}/../inventories/default" + when: + - disconnected.mirroring.host.name is defined + - disconnected.mirroring.host.ip is defined + +- name: Mirror ocp artifacts + hosts: mirror_host + gather_facts: true + vars_files: + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/disconnected.yaml" + tasks: + - name: mirror ocp artifacts when disconnected install + ansible.builtin.include_role: + name: '{{ item }}' + loop: + - disconnected_check_vars + - disconnected_mirror_images + when: disconnected.enabled diff --git a/playbooks/hcp_kubevirt_prereqs_setup.yaml b/playbooks/hcp_kubevirt_prereqs_setup.yaml index 985bd0628..ce612b5ad 100644 --- a/playbooks/hcp_kubevirt_prereqs_setup.yaml +++ b/playbooks/hcp_kubevirt_prereqs_setup.yaml @@ -3,8 +3,8 @@ - name: Add Bastion Host to Inventory hosts: localhost vars_files: - - "{{ inventory_dir }}/group_vars/secrets.yaml" - - "{{ inventory_dir }}/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" tasks: - name: Add bastion dynamically ansible.builtin.add_host: @@ -13,14 +13,14 @@ ansible_user: root ansible_ssh_private_key_file: "{{ lookup('env', 'HOME') + '/.ssh/' + (hcp.ansible_key_name | default('ansible-ocpz', true)) }}" ansible_connection: ssh - inventory_dir: "{{ inventory_dir }}" + inventory_dir: "{{ playbook_dir }}/../inventories/default" # Login to OCP Management Cluster (from bastion) - name: Login to OCP Management Cluster hosts: mgmt_bastion vars_files: - - "{{ inventory_dir }}/group_vars/secrets.yaml" - - "{{ inventory_dir }}/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" vars: vault_api_server: "{{ hcp.oc_url }}" vault_user_name: "{{ vault_ocp_user }}" @@ -32,8 +32,8 @@ - name: Check OCP Node Resource Usage hosts: mgmt_bastion vars_files: - - "{{ inventory_dir }}/group_vars/secrets.yaml" - - "{{ inventory_dir }}/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" vars: resource_threshold: 75 debug_mode: true @@ -44,11 +44,16 @@ - name: Install Operators on the management cluster hosts: mgmt_bastion vars_files: - - "{{ inventory_dir }}/group_vars/secrets.yaml" - - "{{ inventory_dir }}/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" become: true gather_facts: false pre_tasks: + - name: Load disconnected vars if file exists + ansible.builtin.include_vars: + file: "{{ playbook_dir }}/../inventories/default/group_vars/disconnected.yaml" + failed_when: false + - name: Ensure ansible work directory exists ansible.builtin.file: path: /root/ansible_workdir @@ -61,6 +66,39 @@ state: directory mode: '0700' + - name: Find CatalogSource file in oc-mirror cluster-resources directory + ansible.builtin.find: + paths: "{{ disconnected.mirroring.cluster_resources_dir | default('/root/ocpinst_disconnected/working-dir/cluster-resources', true) }}" + patterns: "cs-*.yaml" + recurse: false + register: cs_file_find + when: disconnected.enabled | default(false) + + - name: Read CatalogSource name from oc-mirror output + ansible.builtin.shell: | + set -o pipefail + grep '^ name:' {{ cs_file_find.files[0].path }} | awk '{print $2}' + register: cs_name_raw + changed_when: false + when: + - disconnected.enabled | default(false) + - cs_file_find.matched > 0 + + - name: Set disconnected CatalogSource name fact + ansible.builtin.set_fact: + disconnected_catalogsource_name: "{{ cs_name_raw.stdout | trim }}" + when: + - disconnected.enabled | default(false) + - cs_name_raw.stdout is defined + - cs_name_raw.stdout | trim | length > 0 + + - name: Display resolved CatalogSource name + ansible.builtin.debug: + msg: "Using CatalogSource '{{ disconnected_catalogsource_name }}' for disconnected operator installs" + when: + - disconnected.enabled | default(false) + - disconnected_catalogsource_name is defined + roles: - install_mce_operator_kubevirt - install_metallb_operator diff --git a/playbooks/hcpvirt.yaml b/playbooks/hcpvirt.yaml index 06004d999..68292d21c 100644 --- a/playbooks/hcpvirt.yaml +++ b/playbooks/hcpvirt.yaml @@ -2,6 +2,14 @@ - name: Setup inventory and SSH keys import_playbook: setup_inventory_hcp_kubevirt.yaml +- name: Installing disconneted setup + import_playbook: disconnected_mirror_hcp_artifacts.yaml + when: disconnected.enabled + +- name: Applying manifests + import_playbook: disconnected_hcp_kubevirt_setup.yaml + when: disconnected.enabled + - name: Run HCP KubeVirt prerequisites setup import_playbook: hcp_kubevirt_prereqs_setup.yaml diff --git a/playbooks/setup_inventory_hcp_kubevirt.yaml b/playbooks/setup_inventory_hcp_kubevirt.yaml index c546d7372..a068b3d5c 100644 --- a/playbooks/setup_inventory_hcp_kubevirt.yaml +++ b/playbooks/setup_inventory_hcp_kubevirt.yaml @@ -3,7 +3,7 @@ - name: Setup inventory and passwordless SSH hosts: localhost vars_files: - - "{{ inventory_dir }}/group_vars/secrets.yaml" - - "{{ inventory_dir }}/group_vars/hcp-kubevirt.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/secrets.yaml" + - "{{ playbook_dir }}/../inventories/default/group_vars/hcp-kubevirt.yaml" roles: - create_inventory_setup_hcp_kubevirt diff --git a/roles/create_hcp_kubevirt/tasks/main.yaml b/roles/create_hcp_kubevirt/tasks/main.yaml index d7913d1aa..0b3ea3ed8 100644 --- a/roles/create_hcp_kubevirt/tasks/main.yaml +++ b/roles/create_hcp_kubevirt/tasks/main.yaml @@ -3,9 +3,19 @@ ansible.builtin.set_fact: hosted_control_plane_namespace: "{{ hcp.control_plane.clusters_namespace }}-{{ hcp.control_plane.hosted_cluster_name }}" +- name: Check if SSH key already exists + ansible.builtin.stat: + path: "~/.ssh/{{ hcp.ansible_key_name }}" + register: ssh_key_stat + +- name: Generate SSH key if not present + ansible.builtin.command: + cmd: ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/{{ hcp.ansible_key_name }} + when: not ssh_key_stat.stat.exists + - name: Get ssh key ansible.builtin.command: - cmd: cat ~/.ssh/{{ hcp.ansible_key_name }}.pub + cmd: cat /root/.ssh/{{ hcp.ansible_key_name }}.pub register: ssh_output - name: Load ssh_key into a variable @@ -13,17 +23,29 @@ ssh_key: "{{ ssh_output.stdout_lines[0] }}" - name: Get pod name for hcp-cli-download - kubernetes.core.k8s_info: - api_version: v1 - kind: Pod - namespace: "{{ hcp.mce.mce_namespace }}" - label_selectors: - - app= hcp-cli-download + ansible.builtin.shell: + cmd: > + oc get pod -n {{ hcp.mce.mce_namespace }} + -l app=hcp-cli-download + --no-headers -o custom-columns=NAME:.metadata.name register: hcp_pod_name + changed_when: false + +- name: Try copying hcp-linux-s390x.tar.gz directly from pod (newer MCE layout) + ansible.builtin.command: + cmd: > + oc cp {{ hcp.mce.mce_namespace }}/{{ hcp_pod_name.stdout | trim }}:/opt/app-root/src/hcp-linux-s390x.tar.gz + /root/ansible_workdir/hcp.tar.gz + register: hcp_cp_direct + failed_when: false + changed_when: hcp_cp_direct.rc == 0 -- name: Get hcp.tar.gz file from pod +- name: Fall back to copying from s390x directory (older MCE layout) ansible.builtin.command: - cmd: "oc cp multicluster-engine/{{ hcp_pod_name.resources[0].metadata.name }}:/opt/app-root/src/linux/s390x/ /root/ansible_workdir" + cmd: > + oc cp {{ hcp.mce.mce_namespace }}/{{ hcp_pod_name.stdout | trim }}:/opt/app-root/src/linux/s390x/ + /root/ansible_workdir + when: hcp_cp_direct.rc != 0 - name: Extract binary from hcp.tar.gz ansible.builtin.unarchive: @@ -43,6 +65,13 @@ dest: /root/ansible_workdir/auth_file mode: '0600' +- name: Write mirror registry CA certificate for HCP trust bundle + ansible.builtin.copy: + content: "{{ disconnected.registry.ca_cert }}" + dest: /root/ansible_workdir/mirror-registry-ca.crt + mode: '0600' + when: disconnected is defined and disconnected.enabled | default(false) + - name: Create a Hosted Cluster ansible.builtin.command: cmd: > @@ -59,12 +88,18 @@ {% endif %} --infra-availability-policy "SingleReplica" --image-content-sources /root/ansible_workdir/icsp.yaml + {% if disconnected is defined and disconnected.enabled | default(false) %} + --release-image={{ disconnected.registry.url }}/openshift/release-images:{{ hcp.control_plane.ocp_release_image }} + --additional-trust-bundle /root/ansible_workdir/mirror-registry-ca.crt + --olm-disable-default-sources + {% else %} {% set release_image = lookup('env', 'HCP_RELEASE_IMAGE') %} {% if release_image is defined and release_image != '' %} --release-image={{ release_image }} {% else %} --release-image=quay.io/openshift-release-dev/ocp-release:{{ hcp.control_plane.ocp_release_image }} {% endif %} + {% endif %} {% if hcp.control_plane.additional_flags is defined and hcp.control_plane.additional_flags != '' %} {{ hcp.control_plane.additional_flags }} {% endif %} diff --git a/roles/create_hcp_kubevirt/templates/icsp.yaml.j2 b/roles/create_hcp_kubevirt/templates/icsp.yaml.j2 index 447e8292b..ed51a6185 100644 --- a/roles/create_hcp_kubevirt/templates/icsp.yaml.j2 +++ b/roles/create_hcp_kubevirt/templates/icsp.yaml.j2 @@ -1,3 +1,11 @@ +{% if disconnected is defined and disconnected.enabled | default(false) %} +- mirrors: + - {{ disconnected.registry.url }}/openshift/release + source: quay.io/openshift-release-dev/ocp-v4.0-art-dev +- mirrors: + - {{ disconnected.registry.url }}/openshift/release-images + source: quay.io/openshift-release-dev/ocp-release +{% else %} - source: registry.redhat.io/rhacm2 mirrors: - quay.io:443/acm-d @@ -9,3 +17,4 @@ - source: registry.access.redhat.com/openshift4/ose-oauth-proxy mirrors: - registry.redhat.io/openshift4/ose-oauth-proxy +{% endif %} diff --git a/roles/delete_hcp_kubevirt/tasks/main.yaml b/roles/delete_hcp_kubevirt/tasks/main.yaml index 574210dd4..f0ced2ba3 100644 --- a/roles/delete_hcp_kubevirt/tasks/main.yaml +++ b/roles/delete_hcp_kubevirt/tasks/main.yaml @@ -6,5 +6,5 @@ - name: Destroy Hosted Control Plane ansible.builtin.command: - cmd: hcp destroy cluster agent --name {{ hcp.control_plane.hosted_cluster_name }} --namespace {{ hcp.control_plane.clusters_namespace }} + cmd: hcp destroy cluster kubevirt --name {{ hcp.control_plane.hosted_cluster_name }} --namespace {{ hcp.control_plane.clusters_namespace }} when: hcp.control_plane.hosted_cluster_name in hc_check.stdout diff --git a/roles/disconnected_apply_hcp_manifests/defaults/main.yaml b/roles/disconnected_apply_hcp_manifests/defaults/main.yaml new file mode 100644 index 000000000..11be319a5 --- /dev/null +++ b/roles/disconnected_apply_hcp_manifests/defaults/main.yaml @@ -0,0 +1,7 @@ +--- +# Default variables for disconnected_apply_hcp_manifests role +disconnected: + mirroring: + oc_mirror_results_dir: "/root/.oc-mirror-results" + # oc-mirror v2 plugin writes cluster-apply manifests here + cluster_resources_dir: "/root/ocpinst_disconnected/working-dir/cluster-resources" diff --git a/roles/disconnected_apply_hcp_manifests/tasks/main.yaml b/roles/disconnected_apply_hcp_manifests/tasks/main.yaml new file mode 100644 index 000000000..61fc1bdf7 --- /dev/null +++ b/roles/disconnected_apply_hcp_manifests/tasks/main.yaml @@ -0,0 +1,251 @@ +--- +- name: Apply disconnected manifests for HCP + tags: disconnected_apply_hcp_manifests + when: disconnected.enabled + block: + # ------------------------------------------------------------------------- + # Step 0 – Locate the oc-mirror cluster-resources directory. + # oc-mirror v2 writes manifests directly on the bastion at: + # /root/ocpinst_disconnected/working-dir/cluster-resources + # No sync is needed — the bastion IS the mirror host. + # ------------------------------------------------------------------------- + - name: Ensure ansible work directory exists + ansible.builtin.file: + path: /root/ansible_workdir + state: directory + mode: '0755' + + - name: Resolve cluster-resources directory path + ansible.builtin.set_fact: + oc_mirror_cluster_resources_dir: >- + {{ disconnected.mirroring.cluster_resources_dir + | default('/root/ocpinst_disconnected/working-dir/cluster-resources', true) }} + + - name: Verify cluster-resources directory exists + ansible.builtin.stat: + path: "{{ oc_mirror_cluster_resources_dir }}" + register: cluster_resources_stat + + - name: Fail with a clear message if cluster-resources directory is missing + ansible.builtin.fail: + msg: > + oc-mirror cluster-resources directory not found at + '{{ oc_mirror_cluster_resources_dir }}'. + Ensure the disconnected mirroring step completed successfully + (disconnected_mirror_hcp_artifacts.yaml) before running this playbook. + when: not cluster_resources_stat.stat.exists or not cluster_resources_stat.stat.isdir + + # ------------------------------------------------------------------------- + # Step 1 – Create registry-config ConfigMap + # The ConfigMap key MUST be the registry hostname:port so that the + # cluster CA-bundle lookup works correctly (double-dot notation for + # host:port separators is handled by OpenShift automatically). + # ------------------------------------------------------------------------- + - name: Build registry ConfigMap key (replace colon with double-dot) + ansible.builtin.set_fact: + registry_cm_key: "{{ disconnected.registry.url | replace(':', '..') }}" + registry_ca_cert_indented: "{{ disconnected.registry.ca_cert | trim | indent(4, first=True) }}" + when: disconnected.registry.ca_cert is defined + + - name: Write registry CA certificate ConfigMap to file + ansible.builtin.copy: + dest: /root/ansible_workdir/registry-config-cm.yaml + mode: '0644' + content: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: registry-config + namespace: openshift-config + data: + {{ registry_cm_key }}: | + {{ registry_ca_cert_indented }} + when: disconnected.registry.ca_cert is defined + + - name: Apply registry-config ConfigMap to cluster + ansible.builtin.command: + cmd: oc apply -f /root/ansible_workdir/registry-config-cm.yaml + register: registry_configmap_apply + changed_when: "'created' in registry_configmap_apply.stdout or 'configured' in registry_configmap_apply.stdout" + when: disconnected.registry.ca_cert is defined + + - name: Display registry-config ConfigMap apply status + ansible.builtin.debug: + msg: "registry-config ConfigMap: {{ registry_configmap_apply.stdout | default('skipped') }}" + when: disconnected.registry.ca_cert is defined + + # ------------------------------------------------------------------------- + # Step 2 – Patch image.config.openshift.io/cluster with additionalTrustedCA + # ------------------------------------------------------------------------- + - name: Patch image.config.openshift.io/cluster with additionalTrustedCA + ansible.builtin.command: > + oc patch image.config.openshift.io/cluster --type=merge + -p '{"spec":{"additionalTrustedCA":{"name":"registry-config"}}}' + register: image_config_patch + changed_when: "'patched' in image_config_patch.stdout" + when: disconnected.registry.ca_cert is defined + + - name: Display image.config patch status + ansible.builtin.debug: + msg: "image.config.openshift.io/cluster patch: {{ image_config_patch.stdout | default('skipped') }}" + when: disconnected.registry.ca_cert is defined + + # ------------------------------------------------------------------------- + # Step 3 – Wait 2-3 minutes, then verify MCP is updated and no CO is + # degraded before proceeding to apply mirror manifests. + # ------------------------------------------------------------------------- + - name: Initial pause to allow MCO to start reconciling the new CA + ansible.builtin.pause: + seconds: 120 + prompt: "Waiting 2 minutes for MCO to pick up additionalTrustedCA change..." + when: disconnected.registry.ca_cert is defined + + - name: Wait for master MachineConfigPool to finish updating + ansible.builtin.command: + cmd: oc wait machineconfigpool/master --for=condition=Updated=True --timeout=600s + register: mcp_master_ca_wait + retries: 5 + delay: 30 + until: mcp_master_ca_wait.rc == 0 + changed_when: false + when: disconnected.registry.ca_cert is defined + + - name: Wait for worker MachineConfigPool to finish updating + ansible.builtin.command: + cmd: oc wait machineconfigpool/worker --for=condition=Updated=True --timeout=600s + register: mcp_worker_ca_wait + retries: 5 + delay: 30 + until: mcp_worker_ca_wait.rc == 0 + changed_when: false + when: disconnected.registry.ca_cert is defined + + - name: Verify no ClusterOperator is degraded + ansible.builtin.shell: | + set -o pipefail + oc get clusteroperators -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{range .status.conditions[?(@.type=="Degraded")]}{.status}{end}{"\n"}{end}' \ + | grep -c '|True' || true + register: co_degraded_count + retries: 18 + delay: 10 + until: co_degraded_count.stdout | int == 0 + changed_when: false + when: disconnected.registry.ca_cert is defined + + - name: Display cluster operator health status + ansible.builtin.debug: + msg: "All ClusterOperators healthy – proceeding to apply mirror manifests" + when: disconnected.registry.ca_cert is defined + + # ------------------------------------------------------------------------- + # Step 4 – Apply IDMS, ITMS, and CatalogSource files + # ------------------------------------------------------------------------- + - name: Find IDMS files in oc-mirror cluster-resources + ansible.builtin.find: + paths: "{{ oc_mirror_cluster_resources_dir }}" + patterns: + - "imageDigestMirrorSet*.yaml" + - "idms-*.yaml" + - "idms*.yaml" + recurse: true + register: idms_files + + - name: Find ITMS files in oc-mirror cluster-resources + ansible.builtin.find: + paths: "{{ oc_mirror_cluster_resources_dir }}" + patterns: + - "imageTagMirrorSet*.yaml" + - "itms-*.yaml" + - "itms*.yaml" + recurse: true + register: itms_files + + - name: Find CatalogSource files in oc-mirror cluster-resources + ansible.builtin.find: + paths: "{{ oc_mirror_cluster_resources_dir }}" + patterns: + - "catalogSource*.yaml" + - "cs-*.yaml" + - "catalog-source*.yaml" + recurse: true + register: catalogsource_files + + - name: Apply IDMS manifests to management cluster + ansible.builtin.command: oc apply -f {{ item.path }} + loop: "{{ idms_files.files }}" + when: idms_files.matched > 0 + register: idms_apply + changed_when: "'created' in idms_apply.stdout or 'configured' in idms_apply.stdout" + + - name: Display IDMS apply results + ansible.builtin.debug: + msg: "Applied {{ idms_files.matched }} IDMS file(s)" + when: idms_files.matched > 0 + + - name: Render CAPI/importer IDMS for disconnected registry + ansible.builtin.copy: + dest: /root/ansible_workdir/idms-capi.yaml + mode: '0644' + content: | + apiVersion: config.openshift.io/v1 + kind: ImageDigestMirrorSet + metadata: + name: idms-capi + spec: + imageDigestMirrors: + - mirrors: + - {{ disconnected.registry.url }}/openshift-release-dev/ocp-v4.0-art-dev + source: quay.io/openshift-release-dev/ocp-v4.0-art-dev + + - name: Apply CAPI/importer IDMS to management cluster + ansible.builtin.command: oc apply -f /root/ansible_workdir/idms-capi.yaml + register: idms_capi_apply + changed_when: "'created' in idms_capi_apply.stdout or 'configured' in idms_capi_apply.stdout" + + - name: Apply ITMS manifests to management cluster + ansible.builtin.command: oc apply -f {{ item.path }} + loop: "{{ itms_files.files }}" + when: itms_files.matched > 0 + register: itms_apply + changed_when: "'created' in itms_apply.stdout or 'configured' in itms_apply.stdout" + + - name: Display ITMS apply results + ansible.builtin.debug: + msg: "Applied {{ itms_files.matched }} ITMS file(s)" + when: itms_files.matched > 0 + + - name: Disable default OperatorHub sources + ansible.builtin.command: > + oc patch OperatorHub cluster --type json + -p '[{"op": "add", "path": "/spec/disableAllDefaultSources", "value": true}]' + register: operatorhub_patch + changed_when: "'patched' in operatorhub_patch.stdout" + + - name: Display OperatorHub patch status + ansible.builtin.debug: + msg: "Default OperatorHub sources disabled for disconnected environment" + + - name: Apply CatalogSource manifests to management cluster + ansible.builtin.command: oc apply -f {{ item.path }} + loop: "{{ catalogsource_files.files }}" + when: catalogsource_files.matched > 0 + register: catalogsource_apply + changed_when: "'created' in catalogsource_apply.stdout or 'configured' in catalogsource_apply.stdout" + + - name: Display CatalogSource apply results + ansible.builtin.debug: + msg: "Applied {{ catalogsource_files.matched }} CatalogSource file(s)" + when: catalogsource_files.matched > 0 + + - name: Wait for CatalogSources to reach READY state + ansible.builtin.shell: | + set -o pipefail + oc get catalogsource -n openshift-marketplace \ + -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.status.connectionState.lastObservedState}{"\n"}{end}' \ + | awk -F'|' '$2 != "READY" {count++} END {print count+0}' + register: catalogsource_status + retries: 30 + delay: 10 + until: catalogsource_status.stdout | int == 0 + changed_when: false + when: catalogsource_files.matched > 0 diff --git a/roles/disconnected_mirror_images/tasks/main.yaml b/roles/disconnected_mirror_images/tasks/main.yaml index bedf5fbbf..ae7754561 100644 --- a/roles/disconnected_mirror_images/tasks/main.yaml +++ b/roles/disconnected_mirror_images/tasks/main.yaml @@ -10,6 +10,11 @@ owner: root group: root mode: "644" + - name: Set ansible_user if not defined (for localhost) + tags: mirror_artifacts + ansible.builtin.set_fact: + ansible_user: "{{ disconnected.mirroring.host.user }}" + when: ansible_user is not defined - name: Get user home directory tags: mirror_artifacts ansible.builtin.shell: | @@ -39,11 +44,7 @@ remote_src: true loop: - "{{ disconnected.mirroring.client_download.ocp_download_url }}{{ disconnected.mirroring.client_download.ocp_client_tgz }}" - - >- - {{ env.file_server.protocol ~ '://' ~ - (env.file_server.user ~ ':' ~ env.file_server.pass ~ '@' if env.file_server.protocol == 'ftp' else '') ~ - env.file_server.ip ~ (':' ~ env.file_server.port if env.file_server.port | default('') | length > 0 else '') ~ - '/' ~ disconnected.mirroring.file_server.clients_dir ~ '/' ~ disconnected.mirroring.file_server.oc_mirror_tgz }} + - "{{ disconnected.mirroring.client_download.ocp_download_url }}{{ disconnected.mirroring.client_download.oc_mirror_tgz }}" - name: Copy kubectl, oc, and oc-mirror binaries to /usr/local/sbin tags: mirror_artifacts become: true @@ -227,17 +228,39 @@ - name: print output of oc image mirror ansible.builtin.debug: var: cmd_oc_image_mirror + - name: Find oc-mirror result files on remote host (v1 plugin) + tags: mirror_artifacts + ansible.builtin.find: + paths: "{{ user_home.stdout }}/ocpinst_disconnected/oc-mirror-workspace/{{ oc_mirror_results_dir_name.stdout }}" + recurse: true + file_type: file + register: oc_mirror_v1_files + when: disconnected.mirroring.oc_mirror.image_set.apiVersion == "mirror.openshift.io/v1alpha2" - name: Fetch the files that need to be applied on the cluster to the controller for oc-mirror v1 plugin tags: mirror_artifacts - ansible.posix.synchronize: - mode: pull - src: "{{ user_home.stdout }}/ocpinst_disconnected/oc-mirror-workspace/{{ oc_mirror_results_dir_name.stdout }}/*" - dest: "../.oc-mirror-results" + ansible.builtin.fetch: + src: "{{ item.path }}" + dest: >- + {{ playbook_dir }}/../.oc-mirror-results/{{ + item.path | relpath(user_home.stdout + + '/ocpinst_disconnected/oc-mirror-workspace/' + + oc_mirror_results_dir_name.stdout) }} + flat: true + loop: "{{ oc_mirror_v1_files.files }}" when: disconnected.mirroring.oc_mirror.image_set.apiVersion == "mirror.openshift.io/v1alpha2" + - name: Find oc-mirror result files on remote host (v2 plugin) + tags: mirror_artifacts + ansible.builtin.find: + paths: "{{ user_home.stdout }}/ocpinst_disconnected/working-dir/cluster-resources" + recurse: true + file_type: file + register: oc_mirror_v2_files + when: disconnected.mirroring.oc_mirror.image_set.apiVersion == "mirror.openshift.io/v2alpha1" - name: Fetch the files that need to be applied on the cluster to the controller for oc-mirror v2 plugin tags: mirror_artifacts - ansible.posix.synchronize: - mode: pull - src: "{{ user_home.stdout }}/ocpinst_disconnected/working-dir/cluster-resources/*" - dest: "../.oc-mirror-results" + ansible.builtin.fetch: + src: "{{ item.path }}" + dest: "{{ playbook_dir }}/../.oc-mirror-results/{{ item.path | relpath(user_home.stdout + '/ocpinst_disconnected/working-dir/cluster-resources') }}" + flat: true + loop: "{{ oc_mirror_v2_files.files }}" when: disconnected.mirroring.oc_mirror.image_set.apiVersion == "mirror.openshift.io/v2alpha1" diff --git a/roles/disconnected_mirror_images/templates/resolv.conf.j2 b/roles/disconnected_mirror_images/templates/resolv.conf.j2 index e640d6703..6b0e65288 100644 --- a/roles/disconnected_mirror_images/templates/resolv.conf.j2 +++ b/roles/disconnected_mirror_images/templates/resolv.conf.j2 @@ -1,3 +1,3 @@ -search {{ env.cluster.networking.base_domain }} -nameserver {{ env.bastion.networking.nameserver1 }} -{{ ('nameserver ' + env.bastion.networking.nameserver2) if env.bastion.networking.nameserver2 is defined else '' }} +search {{ hcp.control_plane.basedomain | default(env.cluster.networking.base_domain) }} +nameserver {{ hcp.mgmt_cluster_nameserver | default(env.bastion.networking.nameserver1) }} +{{ ('nameserver ' + env.bastion.networking.nameserver2) if (env is defined and env.bastion.networking.nameserver2 is defined) else '' }} diff --git a/roles/install_hpp/tasks/main.yaml b/roles/install_hpp/tasks/main.yaml index 6ff1f9cfb..5271b2095 100644 --- a/roles/install_hpp/tasks/main.yaml +++ b/roles/install_hpp/tasks/main.yaml @@ -32,9 +32,8 @@ mode: '0644' - name: Apply HostPathProvisioner CR - kubernetes.core.k8s: - state: present - src: /root/ansible_workdir/{{ hcp.hpp.name }}-cr.yaml + ansible.builtin.command: + cmd: oc apply -f /root/ansible_workdir/{{ hcp.hpp.name }}-cr.yaml - name: Render StorageClass ansible.builtin.template: @@ -43,6 +42,5 @@ mode: '0644' - name: Apply StorageClass - kubernetes.core.k8s: - state: present - src: /root/ansible_workdir/{{ hcp.hpp.storageclass_name }}-sc.yaml + ansible.builtin.command: + cmd: oc apply -f /root/ansible_workdir/{{ hcp.hpp.storageclass_name }}-sc.yaml diff --git a/roles/install_lso_operator/tasks/main.yaml b/roles/install_lso_operator/tasks/main.yaml index 02d214c14..117c1fd59 100644 --- a/roles/install_lso_operator/tasks/main.yaml +++ b/roles/install_lso_operator/tasks/main.yaml @@ -24,11 +24,11 @@ mode: '0755' - name: Ensure LSO namespace exists - kubernetes.core.k8s: - api_version: v1 - kind: Namespace - name: "{{ hcp.lso.namespace | default('openshift-local-storage', true) }}" - state: present + ansible.builtin.command: + cmd: oc create namespace {{ hcp.lso.namespace | default('openshift-local-storage', true) }} + register: lso_ns_create + changed_when: "'created' in lso_ns_create.stdout" + failed_when: lso_ns_create.rc != 0 and 'already exists' not in lso_ns_create.stderr - name: Create OperatorGroup.yaml ansible.builtin.template: diff --git a/roles/install_lso_operator/templates/Subscription.yaml.j2 b/roles/install_lso_operator/templates/Subscription.yaml.j2 index 6d0fed595..f6aa11273 100644 --- a/roles/install_lso_operator/templates/Subscription.yaml.j2 +++ b/roles/install_lso_operator/templates/Subscription.yaml.j2 @@ -5,7 +5,9 @@ metadata: namespace: "{{ hcp.lso.namespace | default('openshift-local-storage', true) }}" spec: sourceNamespace: openshift-marketplace -{% if hcp.lso.catalogsource_image is defined and hcp.lso.catalogsource_image | string | length > 0 %} +{% if disconnected is defined and disconnected.enabled | default(false) %} + source: "{{ disconnected_catalogsource_name }}" +{% elif hcp.lso.catalogsource_image is defined and hcp.lso.catalogsource_image | string | length > 0 %} source: custom-operators {% else %} source: redhat-operators diff --git a/roles/install_mce_operator_kubevirt/tasks/main.yaml b/roles/install_mce_operator_kubevirt/tasks/main.yaml index 568ffc7d9..a9fb7a06e 100644 --- a/roles/install_mce_operator_kubevirt/tasks/main.yaml +++ b/roles/install_mce_operator_kubevirt/tasks/main.yaml @@ -47,21 +47,12 @@ changed_when: false when: hcp.mce.catalogsource_image is defined and hcp.mce.catalogsource_image | length > 0 - - name: Check if multicluster-engine Namespace exists - kubernetes.core.k8s: - api_version: v1 - kind: Namespace - name: "{{ hcp.mce.mce_namespace | default('multicluster-engine', true) }}" - register: namespace_check - ignore_errors: true - - name: Create multicluster-engine Namespace - kubernetes.core.k8s: - api_version: v1 - kind: Namespace - name: "{{ hcp.mce.mce_namespace | default('multicluster-engine', true) }}" - state: present - when: namespace_check.resources is not defined or namespace_check.resources | length == 0 + ansible.builtin.command: + cmd: oc create namespace {{ hcp.mce.mce_namespace | default('multicluster-engine', true) }} + register: mce_ns_create + changed_when: "'created' in mce_ns_create.stdout" + failed_when: mce_ns_create.rc != 0 and 'already exists' not in mce_ns_create.stderr - name: Create OperatorGroup.yaml ansible.builtin.template: @@ -116,9 +107,3 @@ until: mce_status.stdout == "Available" retries: 40 delay: 30 - - - name: Enable hypershift-preview component in MCE - ansible.builtin.command: > - oc patch mce {{ hcp.mce.instance_name | default('engine', true) }} - -p '{"spec":{"overrides":{"components":[{"name":"hypershift-preview","enabled":true}]}}}' - --type merge diff --git a/roles/install_mce_operator_kubevirt/templates/Subscription.yaml.j2 b/roles/install_mce_operator_kubevirt/templates/Subscription.yaml.j2 index ee7491e56..5d8450b65 100644 --- a/roles/install_mce_operator_kubevirt/templates/Subscription.yaml.j2 +++ b/roles/install_mce_operator_kubevirt/templates/Subscription.yaml.j2 @@ -5,7 +5,9 @@ metadata: namespace: "{{ hcp.mce.mce_namespace | default('multicluster-engine', true) }}" spec: sourceNamespace: openshift-marketplace -{% if hcp.mce.catalogsource_image is defined and hcp.mce.catalogsource_image | string | length > 0 %} +{% if disconnected is defined and disconnected.enabled | default(false) %} + source: "{{ disconnected_catalogsource_name }}" +{% elif hcp.mce.catalogsource_image is defined and hcp.mce.catalogsource_image | string | length > 0 %} source: custom-redhat-operators {% else %} source: redhat-operators diff --git a/roles/install_metallb_operator/tasks/main.yaml b/roles/install_metallb_operator/tasks/main.yaml index 5e4c739bb..a562a4d1f 100644 --- a/roles/install_metallb_operator/tasks/main.yaml +++ b/roles/install_metallb_operator/tasks/main.yaml @@ -81,11 +81,11 @@ when: hcp.metallb.catalogsource_image is defined and hcp.metallb.catalogsource_image | length > 0 - name: Ensure MetalLB namespace exists - kubernetes.core.k8s: - api_version: v1 - kind: Namespace - name: "{{ hcp.metallb.namespace | default('metallb-system', true) }}" - state: present + ansible.builtin.command: + cmd: oc create namespace {{ hcp.metallb.namespace | default('metallb-system', true) }} + register: metallb_ns_create + changed_when: "'created' in metallb_ns_create.stdout" + failed_when: metallb_ns_create.rc != 0 and 'already exists' not in metallb_ns_create.stderr - name: Create OperatorGroup.yaml ansible.builtin.template: diff --git a/roles/install_metallb_operator/templates/Subscription.yaml.j2 b/roles/install_metallb_operator/templates/Subscription.yaml.j2 index 2f80c21f8..3845d642a 100644 --- a/roles/install_metallb_operator/templates/Subscription.yaml.j2 +++ b/roles/install_metallb_operator/templates/Subscription.yaml.j2 @@ -5,7 +5,9 @@ metadata: namespace: "{{ hcp.metallb.namespace | default('metallb-system', true) }}" spec: sourceNamespace: "{{ hcp.metallb.source_namespace | default('openshift-marketplace', true) }}" -{% if hcp.metallb.catalogsource_image is defined and hcp.metallb.catalogsource_image | string | length > 0 %} +{% if disconnected is defined and disconnected.enabled | default(false) %} + source: "{{ disconnected_catalogsource_name }}" +{% elif hcp.metallb.catalogsource_image is defined and hcp.metallb.catalogsource_image | string | length > 0 %} source: custom-operators {% else %} source: redhat-operators diff --git a/roles/install_ocpvirt_operator/tasks/main.yaml b/roles/install_ocpvirt_operator/tasks/main.yaml index 1cc915a2e..7015547dd 100644 --- a/roles/install_ocpvirt_operator/tasks/main.yaml +++ b/roles/install_ocpvirt_operator/tasks/main.yaml @@ -28,11 +28,11 @@ mode: '0755' - name: Ensure OpenShift Virtualization namespace exists - kubernetes.core.k8s: - api_version: v1 - kind: Namespace - name: "{{ hcp.ocpvirt.namespace | default('openshift-cnv', true) }}" - state: present + ansible.builtin.command: + cmd: oc create namespace {{ hcp.ocpvirt.namespace | default('openshift-cnv', true) }} + register: ocpvirt_ns_create + changed_when: "'created' in ocpvirt_ns_create.stdout" + failed_when: ocpvirt_ns_create.rc != 0 and 'already exists' not in ocpvirt_ns_create.stderr - name: Create OperatorGroup.yaml ansible.builtin.template: diff --git a/roles/install_ocpvirt_operator/templates/Subscription.yaml.j2 b/roles/install_ocpvirt_operator/templates/Subscription.yaml.j2 index 9c9ead0c7..62f4f9777 100644 --- a/roles/install_ocpvirt_operator/templates/Subscription.yaml.j2 +++ b/roles/install_ocpvirt_operator/templates/Subscription.yaml.j2 @@ -5,7 +5,11 @@ metadata: namespace: "{{ hcp.ocpvirt.namespace | default('openshift-cnv', true) }}" spec: sourceNamespace: openshift-marketplace +{% if disconnected is defined and disconnected.enabled | default(false) %} + source: "{{ disconnected_catalogsource_name }}" +{% else %} source: redhat-operators +{% endif %} {% if hcp.ocpvirt.version | default('', true) | length > 0 %} channel: "{{ hcp.ocpvirt.version }}" {% endif %} diff --git a/roles/install_odf_operator/tasks/main.yaml b/roles/install_odf_operator/tasks/main.yaml index cb76fa399..c58dd5410 100644 --- a/roles/install_odf_operator/tasks/main.yaml +++ b/roles/install_odf_operator/tasks/main.yaml @@ -26,11 +26,35 @@ mode: '0755' - name: Ensure ODF namespace exists - kubernetes.core.k8s: - api_version: v1 - kind: Namespace - name: "{{ (hcp.odf | default({})).namespace | default('openshift-storage', true) }}" - state: present + ansible.builtin.command: + cmd: oc create namespace {{ (hcp.odf | default({})).namespace | default('openshift-storage', true) }} + register: odf_ns_create + changed_when: "'created' in odf_ns_create.stdout" + failed_when: odf_ns_create.rc != 0 and 'already exists' not in odf_ns_create.stderr + + - name: Create CatalogSource.yaml from template + ansible.builtin.template: + src: CatalogSource.yaml.j2 + dest: /root/ansible_workdir/odf-CatalogSource.yaml + mode: '0644' + when: (hcp.odf | default({})).catalogsource_image is defined and (hcp.odf | default({})).catalogsource_image | string | length > 0 + + - name: Deploy CatalogSource + ansible.builtin.command: oc apply -f /root/ansible_workdir/odf-CatalogSource.yaml + when: (hcp.odf | default({})).catalogsource_image is defined and (hcp.odf | default({})).catalogsource_image | string | length > 0 + + - name: Wait for ODF CatalogSource to be READY + ansible.builtin.command: + cmd: > + oc get catalogsource custom-odf-operators + -n openshift-marketplace + -o jsonpath='{.status.connectionState.lastObservedState}' + register: odf_catalogsource_status + retries: 30 + delay: 10 + until: odf_catalogsource_status.stdout == "READY" + changed_when: false + when: (hcp.odf | default({})).catalogsource_image is defined and (hcp.odf | default({})).catalogsource_image | string | length > 0 - name: Create OperatorGroup.yaml ansible.builtin.template: diff --git a/roles/install_odf_operator/templates/CatalogSource.yaml.j2 b/roles/install_odf_operator/templates/CatalogSource.yaml.j2 new file mode 100644 index 000000000..42acb5dee --- /dev/null +++ b/roles/install_odf_operator/templates/CatalogSource.yaml.j2 @@ -0,0 +1,10 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: custom-odf-operators + namespace: openshift-marketplace +spec: + sourceType: grpc + publisher: redhat + displayName: Custom ODF Operators + image: {{ (hcp.odf | default({})).catalogsource_image }} diff --git a/roles/install_odf_operator/templates/Subscription.yaml.j2 b/roles/install_odf_operator/templates/Subscription.yaml.j2 index d0e27ab2a..2faf8d569 100644 --- a/roles/install_odf_operator/templates/Subscription.yaml.j2 +++ b/roles/install_odf_operator/templates/Subscription.yaml.j2 @@ -5,7 +5,13 @@ metadata: namespace: "{{ (hcp.odf | default({})).namespace | default('openshift-storage', true) }}" spec: sourceNamespace: openshift-marketplace +{% if disconnected is defined and disconnected.enabled | default(false) %} + source: "{{ disconnected_catalogsource_name }}" +{% elif (hcp.odf | default({})).catalogsource_image is defined and (hcp.odf | default({})).catalogsource_image | string | length > 0 %} + source: custom-odf-operators +{% else %} source: redhat-operators +{% endif %} {% if (hcp.odf | default({})).version | default('', true) | length > 0 %} channel: "{{ (hcp.odf | default({})).version }}" {% endif %} diff --git a/roles/login_to_mgmt_cluster/tasks/main.yaml b/roles/login_to_mgmt_cluster/tasks/main.yaml index 97f2bf7e1..eb317155a 100644 --- a/roles/login_to_mgmt_cluster/tasks/main.yaml +++ b/roles/login_to_mgmt_cluster/tasks/main.yaml @@ -21,6 +21,7 @@ oc login {{ vault_api_server }} -u {{ vault_user_name }} -p {{ vault_password }} + --insecure-skip-tls-verify=true no_log: false register: login_result changed_when: false diff --git a/roles/monitor_hcp_preq_installation/tasks/main.yaml b/roles/monitor_hcp_preq_installation/tasks/main.yaml index d6d0d700a..27e5f1ac8 100644 --- a/roles/monitor_hcp_preq_installation/tasks/main.yaml +++ b/roles/monitor_hcp_preq_installation/tasks/main.yaml @@ -32,7 +32,7 @@ ansible.builtin.shell: | set -o pipefail total=$(oc get po -n {{ metallb_ns }} --no-headers 2>/dev/null | wc -l) - not_running=$(oc get po -n {{ metallb_ns }} --no-headers 2>/dev/null | awk '{print $3}' | grep -v '^Running$' | wc -l) + not_running=$(oc get po -n {{ metallb_ns }} --no-headers 2>/dev/null | awk '{print $3}' | grep -vE '^(Running|Completed|Succeeded)$' | wc -l) echo "${total} ${not_running}" register: metallb_pods_status until: > @@ -270,7 +270,7 @@ ansible.builtin.shell: | set -o pipefail total=$(oc get po -n {{ ocpvirt_ns }} --no-headers 2>/dev/null | wc -l) - not_running=$(oc get po -n {{ ocpvirt_ns }} --no-headers 2>/dev/null | awk '{print $3}' | grep -v '^Running$' | wc -l) + not_running=$(oc get po -n {{ ocpvirt_ns }} --no-headers 2>/dev/null | awk '{print $3}' | grep -vE '^(Running|Completed|Succeeded)$' | wc -l) echo "${total} ${not_running}" register: ocpvirt_pods_status until: >