Changes for operator+ipa

Refactoring:
rhel_provisioning and ipa server setup are moved from rhosp to common level

partial-jira-bug: CEM-26529
Signed-off-by: Gleb Galkin <[email protected]>
Change-Id: Ic7df960c3d6b1bad56deba9bf6410861771701b9

Author: gleb108

common/common.sh

@@ -23,6 +23,13 @@ export PHYS_INT=`ip route get 1 | grep -o 'dev.*' | awk '{print($2)}'`
 export NODE_IP=`ip addr show dev $PHYS_INT | grep 'inet ' | awk '{print $2}' | head -n 1 | cut -d '/' -f 1`
 export NODE_CIDR=`ip r | grep -E "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[0-9]+ dev $PHYS_INT " | awk '{print $1}'`
 export SSH_USER=${SSH_USER:-${IMAGE_SSH_USER:-$(whoami)}}
+
+if [[ "$DISTRO" == "rhel" ]]; then
+  declare -A _default_rhel_version=( ['7.9']='rhel7.9' ['8.2']='rhel8.2' ['8.4']='rhel8.4' )
+  export RHEL_VERSION=${_default_rhel_version[$DISTRO_VERSION_ID]}
+  export RHEL_MAJOR_VERSION=$(echo $RHEL_VERSION | cut -d '.' -f1)
+fi
+
 # defaults
 
 # run build tf
@@ -43,6 +50,7 @@ export CONTROLLER_NODES="$(echo $CONTROLLER_NODES | tr ',' ' ')"
 export CONTROL_NODES="$(echo $CONTROL_NODES | tr ',' ' ')"
 AGENT_NODES="${AGENT_NODES:-$NODE_IP}"
 export AGENT_NODES="$(echo $AGENT_NODES | tr ',' ' ')"
+export IPA_NODES="$(echo $IPA_NODES | tr ',' ' ')"
 
 export TF_LOG_DIR=${TF_LOG_DIR:-${TF_CONFIG_DIR}/logs}
 export SSL_ENABLE=${SSL_ENABLE:-false}

common/deploy_kubespray.sh

@@ -48,6 +48,9 @@ LOOKUP_NODE_HOSTNAMES=${LOOKUP_NODE_HOSTNAMES:-true}
 CRYPTOGRAPHY_ALLOW_OPENSSL_102=true
 
 # set locale to prevent errors from pip install and similar
+if [[ "$DISTRO" == "rhel" ]]; then
+    sudo dnf install -y glibc-langpack-en
+fi
 sudo localectl set-locale LANG=en_US.UTF-8
 . /etc/locale.conf
 export LC_ALL=en_US.UTF-8
@@ -252,13 +255,18 @@ if [[ -z "$CONTAINER_RUNTIME" || "$CONTAINER_RUNTIME" == 'docker' ]]; then
 fi
 
 extra_vars=""
-[[ "$ENABLE_RHEL_REGISTRATION" == 'true' ]] || extra_vars="$extra_vars -e {\"rhel_enable_repos\":False}"
+[[ "$ENABLE_RHEL_REGISTRATION" == 'true' ]] || echo '{"rhel_enable_repos": False}' >/tmp/extravars.json
 [[ -z "$REGISTRY_PROXY" ]] || extra_vars="$extra_vars -e docker_image_repo=$REGISTRY_PROXY"
 [[ -z "$REGISTRY_PROXY" ]] || extra_vars="$extra_vars -e docker_image_repo=$REGISTRY_PROXY"
 [[ -z $K8S_POD_SUBNET ]] || extra_vars="$extra_vars -e kube_pods_subnet=$K8S_POD_SUBNET"
 [[ -z $K8S_SERVICE_SUBNET ]] || extra_vars="$extra_vars -e kube_service_addresses=$K8S_SERVICE_SUBNET"
 [[ -z $K8S_VERSION ]] || extra_vars="$extra_vars -e kube_version=$K8S_VERSION"
-ansible-playbook -i inventory/mycluster/hosts.yml --become --become-user=root cluster.yml $extra_vars "$@"
+
+echo i"INFO: Cleanup /etc/hosts before kubespay"
+cp $my_dir/deploy_kubespray_cleanup_hosts.yaml .
+ansible-playbook -i inventory/mycluster/hosts.yml deploy_kubespray_cleanup_hosts.yaml
+echo "INFO: Running kubespray playbook"
+ansible-playbook -i inventory/mycluster/hosts.yml --become --become-user=root cluster.yml --extra-vars "@/tmp/extravars.json" $extra_vars "$@"
 
 mkdir -p ~/.kube
 sudo cp /root/.kube/config ~/.kube/config

common/deploy_kubespray_cleanup_hosts.yaml

@@ -0,0 +1,19 @@
+---
+#Kubespray manages /etc/hosts and this playbook clean ansible inventory ip from /etc/hosts
+#It should be run with kubespray inventory file
+- name: Clear /etc/hosts from ansible inventory hosts
+  hosts: all
+  become: yes
+  tasks:
+
+    - set_fact:
+        cluster_hosts: "{{ cluster_hosts|default([]) + [ hostvars[item]['ansible_host'] ] }}"
+      loop: "{{ groups['all'] }}"
+      run_once: true
+
+    - name: Delete ip addresses from /etc/hosts
+      lineinfile: dest=/etc/hosts
+        state=absent
+        regexp='^{{ item }}'
+      with_items: "{{ cluster_hosts }}"
+

common/functions.sh

@@ -219,7 +219,7 @@ function check_tf_services() {
     # get contrail-status from $machine node
     # TODO: set timeout 15 sec (-t 15) - there is bug in agent - it does internally
     #       2 dns queries with 5 sec timeout that always fails in 10 sec,
-    #       so tool always fails with default 10 sec timeout. 
+    #       so tool always fails with default 10 sec timeout.
     local contrail_status=$(ssh $SSH_OPTIONS $addr "sudo contrail-status -t 15" 2>/dev/null)
     # keep first part of contrail-status with rows and columns
     # of pods and services in /tmp/_tmp_contrail_status file
@@ -258,7 +258,7 @@ function check_tf_active() {
     fi
     # TODO: set timeout 15 sec (-t 15) - there is bug in agent - it does internally
     #       2 dns queries with 5 sec timeout that always fails in 10 sec,
-    #       so tool always fails with default 10 sec timeout. 
+    #       so tool always fails with default 10 sec timeout.
     for line in $(ssh $SSH_OPTIONS $addr "sudo contrail-status -t 15" 2>/dev/null | egrep ": " | grep -v "WARNING" | awk '{print $2}'); do
       if [ "$line" != "active" ] && [ "$line" != "backup" ] ; then
         return 1
@@ -318,3 +318,66 @@ function get_vrouter_gateway() {
     local gw=$(ip route get "$cidr" | grep -o 'via .*' |  awk '{print($2)}' |head -n1)
     [ -z "$gw" ] || echo $gw
 }
+
+#Find full hostname by ip based on /etc/hosts
+function get_hostname_by_ip() {
+    local ip_addr=$1
+    [ -n "$ip_addr" ] || return
+    local str
+    str=$(getent hosts $ip_addr | awk '{print $2}')
+    echo $str
+}
+
+#Convert list of IP addresses into list of full hostnames
+function convert_ips_to_hostnames() {
+    local line=$1
+    local converted_line=''
+    local host_name
+    local ip_addr
+    for ip_addr in $(echo $line | tr ',' ' '); do
+        host_name=$(get_hostname_by_ip $ip_addr)
+        converted_line+="$host_name,"
+    done
+    echo $converted_line
+}
+
+function ensure_fqdn() {
+    local domain=${1}
+    if [ -z "$domain" ] ; then
+        echo "ERROR: domain must be set"
+        exit 1
+    fi
+    local cur_fqdn="$(hostname -f)"
+    local exp_fqdn="$(hostname -s).${domain}"
+    echo "INFO: cur_fqdn=$cur_fqdn exp_fqdn=$exp_fqdn"
+    if [[ "$cur_fqdn" != "$exp_fqdn" ]] ; then
+        echo "INFO: cur fqdn doesnt match to expected: $cur_fqdn != $exp_fqdn"
+        sudo hostnamectl set-hostname $exp_fqdn
+        echo "INFO: Changing /etc/resolv.conf on $(hostname -f)"
+        echo "#generated by tf-devstack" > /tmp/resolv.conf
+        echo "search $domain" >> /tmp/resolv.conf
+        echo "nameserver 8.8.8.8" >> /tmp/resolv.conf
+        sudo cp /tmp/resolv.conf /etc/resolv.conf
+    fi
+    echo "INFO: fqdn: $(hostname -f) host domain: $(hostname -d)"
+}
+
+function ensure_nameserver() {
+    local nameserver_ip=${1:-'8.8.8.8'}
+    if ! grep -q "nameserver $nameserver_ip" /etc/resolv.conf; then
+        echo "INFO: change_nameserver: nameserver $nameserver_ip is set in /etc/resolv.conf"
+        sudo sed -i '/nameserver/d'  /etc/resolv.conf
+        echo "nameserver $nameserver_ip" | sudo tee -a /etc/resolv.conf
+    fi
+}
+
+function ensure_record_in_etc_hosts() {
+    local ip_addr=$1
+    local fqdn=$2
+    local shortname=$(echo $fqdn | cut -d '.' -f1)
+    if ! grep -qE "$ip_addr\s+$fqdn" /etc/hosts; then
+        echo "INFO: Adding new record \"$ip_addr $fqdn\" to /etc/hosts"
+        sudo sed -i "/$ip_addr/d"  /etc/hosts
+        echo "$ip_addr $fqdn $shortname" | sudo tee -a /etc/hosts
+    fi
+}

rhosp/providers/common/rhel7_provisioning.sh renamed to common/rhel7_provisioning.sh

@@ -31,11 +31,15 @@ sudo dnf distro-sync -y
 
 sudo dnf update -y
 
-packages="chrony wget yum-utils vim iproute jq curl bind-utils network-scripts net-tools tmux createrepo bind-utils sshpass python36 podman"
+packages="chrony wget yum-utils vim iproute jq curl bind-utils network-scripts net-tools tmux createrepo sshpass python36 podman"
 [[ "$ENABLE_TLS" != 'ipa' ]] || packages+=" ipa-client python3-novajoin openssl-perl ca-certificates"
 
 sudo dnf install -y --allowerasing $packages
 
 sudo systemctl start chronyd
 
 sudo alternatives --set python /usr/bin/python3
+
+#service network is not enabled by default in RHEL8.4
+sudo systemctl enable network || true
+sudo systemctl start network || true

rhosp/providers/common/rhel8_provisioning.sh renamed to common/rhel8_provisioning.sh

@@ -4,13 +4,16 @@ my_file="$(readlink -e "$0")"
 my_dir="$(dirname $my_file)"
 
 cd
-source rhosp-environment.sh
-source $my_dir/../../../common/common.sh
-source $my_dir/../../../common/functions.sh
+
+#For rhosp compatibility
+if [[ -n "$domain" ]]; then
+    export DOMAIN=$domain
+fi
+
 source $my_dir/common.sh
 source $my_dir/functions.sh
 
-ensure_fqdn ${domain}
+ensure_fqdn ${DOMAIN}
 
 attach_opts='--auto'
 if [[ -n "$RHEL_POOL_ID" ]] ; then

rhosp/providers/common/rhel_provisioning.sh renamed to common/rhel_provisioning.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/bin/bash -ex
 
 IPA_IP=$1
 IPA_PRINCIPAL=$2
@@ -7,9 +7,20 @@ HOST_IP=$4
 CA_DIR=/etc/contrail/ssl/ca-certs
 CERTS_DIR=/etc/contrail/ssl/certs
 
+export DISTRO=$(cat /etc/*release | egrep '^ID=' | awk -F= '{print $2}' | tr -d \")
+export DISTRO_VERSION_ID=$(cat /etc/*release | egrep '^VERSION_ID=' | awk -F= '{print $2}' | tr -d \")
+
 sudo mkdir -p $CA_DIR
 sudo mkdir -p $CERTS_DIR
 
+#Managing SeLinux policies for directories
+sudo semanage fcontext -a -t cert_t "$CERTS_DIR(/.*)?"
+sudo restorecon -rv "$CERTS_DIR"
+sudo semanage fcontext -a -t cert_t "$CA_DIR(/.*)?"
+sudo restorecon -rv "$CA_DIR"
+ls -laZ $CA_DIR $CERTS_DIR
+
+sudo yum install -y bind-utils
 sudo sed -i.bak "s/\(nameserver\) .*/\1 $IPA_IP/" /etc/resolv.conf
 
 IPA_FQDN=$(nslookup "$IPA_IP" | grep name | awk '{print $4}' | rev | cut -c 2- | rev)
@@ -21,7 +32,16 @@ if [[ "$(hostname -f)" != "$HOST_FQDN" ]] ; then
     sudo hostnamectl set-hostname "$HOST_FQDN"
 fi
 
-sudo yum install ipa-client -y
+if [[ "$DISTRO" == "rhel" && "$DISTRO_VERSION_ID" =~ ^8\. ]]; then
+    sudo yum module install -y idm
+    #sudo yum module enable idm:DL1
+    sudo yum distro-sync -y
+    #sudo yum module install -y idm:DL1/client
+    #sudo ipa-client-install --enable-dns-updates --mkhomedir
+else
+    sudo yum install ipa-client -y
+fi
+
 sudo ipa-client-install --verbose -U --server "$IPA_FQDN" -p "$IPA_PRINCIPAL" -w "$IPA_PASSWORD" --domain "$IPA_DOMAIN" --hostname "$HOST_FQDN"
 
 sudo cp /etc/ipa/ca.crt $CA_DIR/ca-bundle.crt
@@ -34,18 +54,47 @@ sudo ipa dnsrecord-find --name="$Hostname" "$IPA_DOMAIN" || sudo ipa dnsrecord-a
 
 HOST_PRINCIPAL=contrail/"$HOST_FQDN"@"${IPA_DOMAIN^^}"
 
-if ! ipa service-find "$HOST_PRINCIPAL" ; then
+if ! sudo ipa service-find "$HOST_PRINCIPAL" ; then
     sudo ipa service-add "$HOST_PRINCIPAL" || true
     sudo ipa service-add-host --hosts "$HOST_FQDN" "$HOST_PRINCIPAL" || true
 fi
+
+res=1
 if [ ! -e $CERTS_DIR/client-"$HOST_IP".crt ] ; then
     sudo ipa-getcert request -f $CERTS_DIR/client-"$HOST_IP".crt -k $CERTS_DIR/client-key-"$HOST_IP".pem -D "$HOST_FQDN" -K contrail/"$HOST_FQDN"@"${IPA_DOMAIN^^}"
-    while [ ! -e $CERTS_DIR/client-"$HOST_IP".crt ] ; do sleep 1; done
+    for i in {1..30} ; do
+        echo "INFO: waiting for $CERTS_DIR/client-"$HOST_IP".crt to appear. Try $i from 30"
+        if [ -e $CERTS_DIR/client-"$HOST_IP".crt ] ; then
+            echo "INFO: $CERTS_DIR/client-"$HOST_IP".crt found"
+            res=0
+            break
+        fi
+        sleep 2
+    done
 fi
 
+if [[ $res == 1 ]]; then
+    echo "INFO: sudo ipa-getcert request $CERTS_DIR/client-"$HOST_IP".crt failed. Exit"
+    exit 1
+fi
+
+res=1
 if [ ! -e $CERTS_DIR/server-"$HOST_IP".crt ] ; then
     sudo ipa-getcert request -f $CERTS_DIR/server-"$HOST_IP".crt -k $CERTS_DIR/server-key-"$HOST_IP".pem -D "$HOST_FQDN" -K contrail/"$HOST_FQDN"@"${IPA_DOMAIN^^}"
-    while [ ! -e $CERTS_DIR/server-"$HOST_IP".crt ] ; do sleep 1; done
+    for i in {1..30} ; do
+        echo "INFO: waiting for $CERTS_DIR/server-"$HOST_IP".crt to appear. Try $i from 30"
+        if [ -e $CERTS_DIR/server-"$HOST_IP".crt ] ; then
+            echo "INFO: $CERTS_DIR/server-"$HOST_IP".crt found"
+            res=0
+            break
+        fi
+        sleep 2
+    done
+fi
+
+if [[ $res == 1 ]]; then
+    echo "INFO: sudo ipa-getcert request failed $CERTS_DIR/server-"$HOST_IP".crt. Exit"
+    exit 1
 fi
 
 sudo chmod -R a+rX $CA_DIR

contrib/ipa/enroll.sh

@@ -25,12 +25,14 @@ source /etc/os-release
 
 fqdn=$(hostname -f)
 domain=$(hostname -d)
+host_name=$(hostname -s)
 export Hostname=${Hostname:-"${fqdn}"}
 export DirectoryManagerPassword=${DirectoryManagerPassword:-"$AdminPassword"}
 export FreeIPAExtraArgs=${FreeIPAExtraArgs:-""}
 export CLOUD_DOMAIN_NAME=${CLOUD_DOMAIN_NAME:-"${domain}"}
-export FreeIPAIPSubnet=${FreeIPAIPSubnet:-'24'}
-export IPA_IFACE=${IPA_IFACE-'eth1'}
+export FreeIPAIPSubnet=$(ip addr show | grep -o "inet ${FreeIPAIP}/[0-9]* "| cut -d '/' -f2)
+export IPA_IFACE=$(ip addr show | grep "inet ${FreeIPAIP}" | awk '{print $(NF)}')
+export DEFAULT_IFACE=$(ip route list | grep default | cut -d ' ' -f5)
 export IPA_DNS1=${IPA_DNS1-'8.8.8.8'}
 export IPA_DNS2=${IPA_DNS2-'8.8.4.4'}
 
@@ -47,8 +49,10 @@ IPADDR=$FreeIPAIP
 PREFIX=$FreeIPAIPSubnet
 EOM
     modprobe ipv6 || true
-    ifdown $IPA_IFACE || true
-    ifup $IPA_IFACE || true
+    if [[ "$DEFAULT_IFACE" != "$IPA_IFACE" ]]; then
+        ifdown $IPA_IFACE || true
+        ifup $IPA_IFACE || true
+    fi
 fi
 
 if [[ -n "$IPA_DNS1" || -n "$IPA_DNS2" ]] ; then
@@ -155,6 +159,9 @@ function install_ipa_server() {
     return $res
 }
 
+#Adding ipaserver fqdn to the /etc/hosts
+sed -i /etc/hosts -e "s/^${FreeIPAIP}.*$/${FreeIPAIP} ${fqdn} ${host_name}/"
+
 for i in {1..5} ; do
     if install_ipa_server ; then
         break

rhosp/ipa/freeipa_setup_root.sh renamed to contrib/ipa/freeipa_setup_root.sh

@@ -153,3 +153,14 @@ function monitor_csr() {
     sleep 5
   done
 }
+
+function wait_vhost0_up() {
+    local node
+    for node in $(echo ${CONTROLLER_NODES} ${AGENT_NODES} | tr ',' ' ') ; do
+        scp $SSH_OPTIONS ${fmy_dir}/functions.sh ${node}:/tmp/functions.sh
+        if ! ssh $SSH_OPTIONS ${node} "export PATH=\$PATH:/usr/sbin ; source /tmp/functions.sh ; wait_nic_up vhost0" ; then
+            return 1
+        fi
+    done
+}
+