fix: Agent-Based Installer robustness fixes (workdir, DNS, ISO boot, ssh_agent)

inventories/default/group_vars/all.yaml.template

@@ -239,6 +239,10 @@ abi:
   ocp_installer_base_url: 'https://mirror.openshift.com/pub/openshift-v4/'
   architecture: <multi|s390x>  
   boot_method: <pxe|iso>
+  # (Optional) Pass --wait=-1 to virt-install for ISO boot. Defaults to true
+  # (upstream behavior). Set to false on hosts where the async virt-install
+  # job being reaped at timeout can power the agent VMs off mid-install.
+  iso_virt_install_wait: true
 
 # Openshift Settings  
 # Settings like architecture, hyperthreading, network cidr, etc.

playbooks/create_abi_cluster.yaml

@@ -47,5 +47,9 @@
 - name: Setup SSH agent
   tags: ssh_to_nodes
   hosts: bastion
+  become: false
+  vars_files:
+    - "{{ inventory_dir }}/group_vars/all.yaml"
+    - "{{ inventory_dir }}/group_vars/secrets.yaml"
   roles:
     - ssh_agent

roles/abi_install_complete/tasks/main.yaml

@@ -1,8 +1,12 @@
 ---
+- name: Resolve OCP install work directory
+  ansible.builtin.include_tasks: ../../common/tasks/resolve_ocp_install_workdir.yaml
+
 - name: wait-for install-complete
+  when: not install_config_vars.fips
   block:
     - name: Start openshift-installer with 'wait-for install-complete' (async task)
-      ansible.builtin.command: openshift-install agent wait-for install-complete --dir=/root/"{{ abi.ansible_workdir }}"
+      ansible.builtin.command: openshift-install agent wait-for install-complete --dir=/root/"{{ ocp_install_workdir }}"
       # Installer will wait up to ~70 min
       async: 4260
       poll: 0
@@ -16,12 +20,12 @@
       # Set wait time to 71 min, because it depends highly on system performance and network speed
       retries: 141
       delay: 30
-  when: not install_config_vars.fips
 
 - name: FIPS wait-for install-complete
+  when: install_config_vars.fips
   block:
     - name: Start openshift-installer with 'wait-for install-complete' (async task)
-      ansible.builtin.command: openshift-install-fips agent wait-for install-complete --dir=/root/"{{ abi.ansible_workdir }}"
+      ansible.builtin.command: openshift-install-fips agent wait-for install-complete --dir=/root/"{{ ocp_install_workdir }}"
       # Installer will wait up to ~71 min
       async: 4260
       poll: 0
@@ -35,41 +39,40 @@
       # Set wait time to 70 min, because it depends highly on system performance and network speed
       retries: 140
       delay: 30
-  when: install_config_vars.fips
 
 - name: Create ~/.kube directory if it does not exist
   tags: kubeconfig
-  file:
+  ansible.builtin.file:
     path: ~/.kube
     state: directory
     mode: '0755'
 
 - name: Copy kubeconfig to ~/.kube/config
   tags: kubeconfig
-  copy:
-    src: /root/{{ abi.ansible_workdir }}/auth/kubeconfig
+  ansible.builtin.copy:
+    src: /root/{{ ocp_install_workdir }}/auth/kubeconfig
     dest: ~/.kube/config
     owner: "{{ ansible_user_id }}"
-    group: "{{ ansible_user_gid }}" 
+    group: "{{ ansible_user_gid }}"
     mode: '0600'
-    remote_src: yes
+    remote_src: true
 
 - name: Check if kubeconfig was copied
   tags: kubeconfig
-  stat:
+  ansible.builtin.stat:
     path: ~/.kube/config
   register: kubeconfig_status
 
 - name: Display kubeconfig message
   tags: kubeconfig
-  debug:
+  ansible.builtin.debug:
     msg: "Kubeconfig is set at ~/.kube/config. No need to export KUBECONFIG unless you want to use a different one."
   when: kubeconfig_status.stat.exists
 
 - name: Display cluster details post-install
-  debug:
+  ansible.builtin.debug:
     msg:
       - Installation completed!
-      - Access the OpenShift web-console here --> https://console-openshift-console.apps.{{env.cluster.networking.metadata_name}}.{{env.cluster.networking.base_domain}}
-      - kubeadmin password of cluster is stored in ~/{{ abi.ansible_workdir }}/auth/kubeadmin-password on bastion.
-      - Login command --> oc login https://api.{{env.cluster.networking.metadata_name}}.{{env.cluster.networking.base_domain}}:6443 -u kubeadmin -p $( cat ~/{{ abi.ansible_workdir }}/auth/kubeadmin-password )
+      - Web console: https://console-openshift-console.apps.{{ env.cluster.networking.metadata_name }}.{{ env.cluster.networking.base_domain }}
+      - kubeadmin password of cluster is stored in ~/{{ ocp_install_workdir }}/auth/kubeadmin-password on bastion.
+      - Login command --> oc login https://api.{{env.cluster.networking.metadata_name}}.{{env.cluster.networking.base_domain}}:6443 -u kubeadmin -p $( cat ~/{{ ocp_install_workdir }}/auth/kubeadmin-password )

roles/boot_abi_agents/tasks/main.yml

@@ -1,5 +1,5 @@
 ---
 - name: Add Route To Bastion From KVM Host.
   ansible.builtin.command: "ip route add {{ env.bastion.networking.ip }} via {{ env.bastion.networking.gateway }}"
   ignore_errors: true
 
@@ -21,7 +21,7 @@
   when: abi.boot_method | lower == "iso"
 
 - name: Create CoreOS Control Agent Nodes On The KVM host using PXE boot.
   ansible.builtin.shell: |
     # Delete Agent node VM, if already exists
     virsh destroy {{ env.cluster.nodes.control.vm_name[i] }} || true
     virsh undefine {{ env.cluster.nodes.control.vm_name[i] }} --remove-all-storage --nvram || true
@@ -34,12 +34,12 @@
     {{ env.cluster.nodes.control.vcpu_model_option }} \
     --vcpus {{ env.cluster.nodes.control.vcpu }} \
     --network network={{ env.vnet_name }}{{ (',mac=' + env.cluster.nodes.control.mac[i]) }} \
     --location "http://{{ env.bastion.networking.ip }}:8080/,kernel=agent.{{ ansible_architecture }}-kernel.img,initrd=agent.{{ ansible_architecture }}-initrd.img" \
     --extra-args "coreos.live.rootfs_url=http://{{ env.bastion.networking.ip }}:8080/agent.{{ ansible_architecture }}-rootfs.img" \
     {% if (env.cluster.nodes.control.mac[i] is defined and env.use_dhcp) %}
     --extra-args "ip=dhcp" \
     {% else %}
     --extra-args "ip={{ env.cluster.nodes.control.ip[i] }}::{{ env.cluster.networking.gateway }}:{{ env.cluster.networking.subnetmask }}:{{ env.cluster.nodes.control.hostname[i] }}:{{ env.cluster.networking.interface }}:none" \
     {% endif %}
     --extra-args "rd.neednet=1 nameserver={{ env.cluster.networking.nameserver1 }}" \
     --extra-args "random.trust_cpu=on rd.luks.options=discard ignition.firstboot ignition.platform.id=metal" \
@@ -79,8 +79,7 @@
     --disk size={{ env.cluster.nodes.control.disk_size }} \
     --network network={{ env.vnet_name }}{{ (',mac=' + env.cluster.nodes.control.mac[i]) }} \
     --graphics none \
-    --noautoconsole \
-    --wait=-1 \
+    --noautoconsole {{ '--wait=-1' if (abi.iso_virt_install_wait | default(true)) else '' }}
   async: 3600
   poll: 0
   with_sequence: start=0 end={{ (env.cluster.nodes.control.hostname | length) - 1 }} stride=1
@@ -91,7 +90,7 @@
   when: abi.boot_method | lower == "iso"
 
 - name: Create CoreOS Compute Agent Nodes On The KVM host using PXE boot.
   ansible.builtin.shell: |
     # Delete Agent node VM, if already exists
     virsh destroy {{ env.cluster.nodes.compute.vm_name[i] }} || true
     virsh undefine {{ env.cluster.nodes.compute.vm_name[i] }} --remove-all-storage --nvram || true
@@ -104,12 +103,12 @@
     {{ env.cluster.nodes.compute.vcpu_model_option }} \
     --vcpus {{ env.cluster.nodes.compute.vcpu }} \
     --network network={{ env.vnet_name }}{{ (',mac=' + env.cluster.nodes.compute.mac[i]) }} \
     --location "http://{{ env.bastion.networking.ip }}:8080/,kernel=agent.{{ ansible_architecture }}-kernel.img,initrd=agent.{{ ansible_architecture }}-initrd.img" \
     --extra-args "coreos.live.rootfs_url=http://{{ env.bastion.networking.ip }}:8080/agent.{{ ansible_architecture }}-rootfs.img" \
     {% if (env.cluster.nodes.compute.mac[i] is defined and env.use_dhcp) %}
     --extra-args "ip=dhcp" \
     {% else %}
     --extra-args "ip={{ env.cluster.nodes.compute.ip[i] }}::{{ env.cluster.networking.gateway }}:{{ env.cluster.networking.subnetmask }}:{{ env.cluster.nodes.compute.hostname[i] }}:{{ env.cluster.networking.interface }}:none" \
     {% endif %}
     --extra-args "rd.neednet=1 nameserver={{ env.cluster.networking.nameserver1 }}" \
     --extra-args "random.trust_cpu=on rd.luks.options=discard ignition.firstboot ignition.platform.id=metal" \
@@ -149,8 +148,7 @@
     --disk size={{ env.cluster.nodes.compute.disk_size }} \
     --network network={{ env.vnet_name }}{{ (',mac=' + env.cluster.nodes.compute.mac[i]) }} \
     --graphics none \
-    --noautoconsole \
-    --wait=-1 \
+    --noautoconsole {{ '--wait=-1' if (abi.iso_virt_install_wait | default(true)) else '' }}
   async: 3600
   poll: 0
   with_sequence: start=0 end={{ (env.cluster.nodes.compute.hostname | length) - 1 }} stride=1
@@ -159,3 +157,22 @@
     index_var: i
     pause: 10
   when: env.cluster.nodes.compute is defined and abi.boot_method | lower == "iso"
+
+- name: Wait for all agent VM definitions to exist after ISO virt-install
+  ansible.builtin.shell: |
+    set -e
+    for vm in {{ env.cluster.nodes.control.vm_name | join(' ') }} {{ env.cluster.nodes.compute.vm_name | default([]) | join(' ') }}; do
+      virsh dominfo "$vm" >/dev/null 2>&1
+    done
+  register: agent_vms_defined
+  until: agent_vms_defined.rc == 0
+  retries: 36
+  delay: 10
+  when: abi.boot_method | lower == "iso"
+
+- name: Ensure ABI agent VMs are running after ISO boot
+  community.libvirt.virt:
+    name: "{{ item }}"
+    state: running
+  loop: "{{ env.cluster.nodes.control.vm_name + (env.cluster.nodes.compute.vm_name | default([])) }}"
+  when: abi.boot_method | lower == "iso"

roles/common/tasks/resolve_ocp_install_workdir.yaml

@@ -0,0 +1,12 @@
+---
+- name: Resolve OCP install work directory
+  ansible.builtin.set_fact:
+    ocp_install_workdir: "{{ abi.ansible_workdir }}"
+  when: ocp_install_workdir is not defined
+
+- name: Require OCP install work directory
+  ansible.builtin.assert:
+    that:
+      - ocp_install_workdir is defined
+      - ocp_install_workdir | length > 0
+    fail_msg: "Define ocp_install_workdir or abi.ansible_workdir in group_vars/all.yaml."

roles/create_agent/tasks/main.yml

@@ -1,53 +1,56 @@
 ---
 
+- name: Resolve OCP install work directory
+  ansible.builtin.include_tasks: ../../common/tasks/resolve_ocp_install_workdir.yaml
+
 - name: Create Agent Using PXE Boot (fips = false)
   ansible.builtin.command: openshift-install agent create pxe-files --log-level=debug
   args:
-    chdir: ~/{{ abi.ansible_workdir }}
+    chdir: ~/{{ ocp_install_workdir }}
   when: abi.boot_method | lower == "pxe" and not install_config_vars.fips
 
 - name: Create Agent Using ISO Image (fips = false)
   ansible.builtin.command: openshift-install agent create image --log-level=debug
   args:
-    chdir: ~/{{ abi.ansible_workdir }}
+    chdir: ~/{{ ocp_install_workdir }}
   when: abi.boot_method | lower == "iso" and not install_config_vars.fips
 
 - name: Create Agent Using PXE Boot (fips = true)
   ansible.builtin.command: openshift-install-fips agent create pxe-files --log-level=debug
   args:
-    chdir: ~/{{ abi.ansible_workdir }}
+    chdir: ~/{{ ocp_install_workdir }}
   when: abi.boot_method | lower == "pxe" and install_config_vars.fips
 
 - name: Create Agent Using ISO Image (fips = true)
   ansible.builtin.command: openshift-install-fips agent create image --log-level=debug
   args:
-    chdir: ~/{{ abi.ansible_workdir }}
+    chdir: ~/{{ ocp_install_workdir }}
   when: abi.boot_method | lower == "iso" and install_config_vars.fips
 
 - name: Check is vmlinuz exists in boot-artifacts
   stat:
-    path: "~/{{ abi.ansible_workdir }}/boot-artifacts/agent.{{ ansible_architecture }}-vmlinuz"
+    path: "~/{{ ocp_install_workdir }}/boot-artifacts/agent.{{ ansible_architecture }}-vmlinuz"
   register: vmlinuz
 
 - name: Copy and rename vmlinuz as kernel.img for PXE
   ansible.builtin.copy:
-    src: "~/{{ abi.ansible_workdir }}/boot-artifacts/agent.{{ ansible_architecture }}-vmlinuz"
+    src: "~/{{ ocp_install_workdir }}/boot-artifacts/agent.{{ ansible_architecture }}-vmlinuz"
     dest: /var/www/html/agent.{{ ansible_architecture }}-kernel.img
     mode: '644'
     remote_src: true
   when: vmlinuz.stat.exists
 
 - name: Copy initrd.img, kernel.img, and rootfs.img for PXE
   ansible.builtin.copy:
-    src: "~/{{ abi.ansible_workdir }}/boot-artifacts/"
+    src: "~/{{ ocp_install_workdir }}/boot-artifacts/"
     dest: /var/www/html/
     mode: '645'
     remote_src: true
   when: abi.boot_method | lower == "pxe"
 
 - name: Copy ISO image to the server
   ansible.builtin.copy:
-    src: "~/{{ abi.ansible_workdir }}/"
+    src: "~/{{ ocp_install_workdir }}/"
     dest: /var/www/html/
     mode: '645'
     remote_src: true

roles/dns/templates/dns-named.conf.j2

@@ -18,7 +18,11 @@ options {
 	secroots-file	"/var/named/data/named.secroots";
 	recursing-file	"/var/named/data/named.recursing";
 	allow-query     { any; };
-	forwarders { {{ env.bastion.networking.nameserver2}}; {{ env.cluster.networking.forwarder }}; };
+{% if env.bastion.networking.nameserver2 is defined %}
+	forwarders { {{ env.bastion.networking.nameserver2 }}; {{ env.cluster.networking.forwarder }}; };
+{% else %}
+	forwarders { {{ env.cluster.networking.forwarder }}; };
+{% endif %}
 
 	/*
 	 - If you are building an AUTHORITATIVE DNS server, do NOT enable recursion.

roles/prepare_configs/tasks/main.yaml

@@ -1,12 +1,15 @@
 ---
+- name: Resolve OCP install work directory
+  ansible.builtin.include_tasks: ../../common/tasks/resolve_ocp_install_workdir.yaml
+
 - name: Delete OCP Work Directory For Idempotency.
   file:
-    path: ~/ansible_workdir
+    path: "~/{{ ocp_install_workdir }}"
     state: absent
 
 - name: Create Work Directory
   file:
-    path: ~/ansible_workdir
+    path: "~/{{ ocp_install_workdir }}"
     state: directory
 
 - name: Prepare Agent Config
@@ -18,8 +21,8 @@
     mode: "0755"
     backup: yes
   loop:
-    - ~/ansible_workdir/agent-config.yaml
-    - ~/ansible_workdir/agent-config-backup.yaml
+    - "~/{{ ocp_install_workdir }}/agent-config.yaml"
+    - "~/{{ ocp_install_workdir }}/agent-config-backup.yaml"
 
 - name: Check if SSH key exists
   stat:
@@ -45,5 +48,5 @@
     mode: "0755"
     backup: yes
   loop:
-    - ~/ansible_workdir/install-config.yaml
-    - ~/ansible_workdir/install-config-backup.yaml
+    - "~/{{ ocp_install_workdir }}/install-config.yaml"
+    - "~/{{ ocp_install_workdir }}/install-config-backup.yaml"

roles/setup_params/tasks/main.yaml

@@ -1,4 +1,7 @@
 ---
+- name: Resolve OCP install work directory
+  ansible.builtin.include_tasks: ../../common/tasks/resolve_ocp_install_workdir.yaml
+
 - name: Ensure the PXE boot directory exists
   ansible.builtin.file:
     path: /var/lib/libvirt/images/pxeboot
@@ -10,7 +13,7 @@
 - name: Generate param files
   ansible.builtin.template:
     src: param-file.param.j2
-    dest: ~/{{ abi.ansible_workdir }}/{{ env.cluster.networking.metadata_name }}-{{ item }}.param
+    dest: ~/{{ ocp_install_workdir }}/{{ env.cluster.networking.metadata_name }}-{{ item }}.param
     mode: '644'
   with_sequence: start=0 end={{ (zvm.nodes | length) - 1 }} stride=1
   loop_control:
@@ -19,21 +22,21 @@
 
 - name: Copy rootfs.img
   ansible.builtin.copy:
-    src: "~/{{ abi.ansible_workdir }}/boot-artifacts/agent.{{ ansible_architecture }}-rootfs.img"
+    src: "~/{{ ocp_install_workdir }}/boot-artifacts/agent.{{ ansible_architecture }}-rootfs.img"
     dest: /var/www/html/rootfs.img
     mode: '745'
     remote_src: true
 
 - name: Copy initrd.img
   ansible.builtin.copy:
-    src: "~/{{ abi.ansible_workdir }}/boot-artifacts/agent.{{ ansible_architecture }}-initrd.img"
+    src: "~/{{ ocp_install_workdir }}/boot-artifacts/agent.{{ ansible_architecture }}-initrd.img"
     dest: /var/lib/libvirt/images/pxeboot/initrd.img
     mode: '745'
     remote_src: true
 
 - name: Copy kernel.img
   ansible.builtin.copy:
-    src: "~/{{ abi.ansible_workdir }}/boot-artifacts/agent.{{ ansible_architecture }}-kernel.img"
+    src: "~/{{ ocp_install_workdir }}/boot-artifacts/agent.{{ ansible_architecture }}-kernel.img"
     dest: /var/lib/libvirt/images/pxeboot/kernel.img
     mode: '745'
     remote_src: true