(PE-40379) CI to test Standard migration

.github/workflows/test-migration.yaml

@@ -0,0 +1,143 @@
+---
+name: Test Migration
+on:
+  pull_request:
+    paths:
+      - .github/workflows/**/*
+      - spec/**/*
+      - lib/**/*
+      - tasks/**/*
+      - functions/**/*
+      - types/**/*
+      - plans/**/*
+      - hiera/**/*
+      - manifests/**/*
+      - templates/**/*
+      - files/**/*
+      - metadata.json
+      - Rakefile
+      - Gemfile
+      - provision.yaml
+      - .rspec
+      - .rubocop.yml
+      - .puppet-lint.rc
+      - .fixtures.yml
+    branches: [main]
+  workflow_dispatch: {}
+jobs:
+  test-migration:
+    name: PE ${{ matrix.version }} ${{ matrix.architecture }} on ${{ matrix.image }}
+      PE upgrade? ${{ matrix.new_pe_version }}
+    runs-on: ubuntu-latest
+    env:
+      BOLT_GEM: true
+      BOLT_DISABLE_ANALYTICS: true
+      LANG: en_US.UTF-8
+    strategy:
+      fail-fast: false
+      matrix:
+        architecture:
+          - standard
+          - standard-with-dr
+            # - large
+            # - extra-large
+            # - large-with-dr
+            # - extra-large-with-dr
+        version: [2021.7.9, 2023.8.1, 2025.0.0]
+        image: [almalinux-cloud/almalinux-8]
+        include:
+          - architecture: standard
+            version: 2023.8.0
+            image: almalinux-cloud/almalinux-8
+            new_pe_version: 2025.0.0
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Activate Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '2.7'
+          bundler-cache: true
+      - name: Print bundle environment
+        if: ${{ github.repository_owner == 'puppetlabs' }}
+        run: |
+          echo ::group::info:bundler
+            bundle env
+          echo ::endgroup::
+      - name: Provision test cluster
+        timeout-minutes: 15
+        run: |
+          echo ::group::prepare
+            mkdir -p $HOME/.ssh
+            echo 'Host *'                      >  $HOME/.ssh/config
+            echo '    ServerAliveInterval 150' >> $HOME/.ssh/config
+            echo '    ServerAliveCountMax 2'   >> $HOME/.ssh/config
+            bundle exec rake spec_prep
+          echo ::endgroup::
+          echo ::group::provision
+            bundle exec bolt plan run peadm_spec::provision_test_cluster \
+              --modulepath spec/fixtures/modules \
+              provider=provision_service \
+              image=${{ matrix.image }} \
+              architecture=${{ matrix.architecture }}-migration \
+              --log-level trace
+          echo ::endgroup::
+          echo ::group::info:request
+            cat request.json || true; echo
+          echo ::endgroup::
+          echo ::group::info:inventory
+            sed -e 's/password: .*/password: "[redacted]"/' < spec/fixtures/litmus_inventory.yaml || true
+          echo ::endgroup::
+          echo ::group::certnames
+            bundle exec bolt plan run peadm_spec::add_inventory_hostnames \
+            --inventory spec/fixtures/litmus_inventory.yaml \
+            --modulepath spec/fixtures/modules \
+            --no-host-key-check \
+            inventory_file=spec/fixtures/litmus_inventory.yaml
+          echo ::endgroup::            
+      - name: Output contents of litmus_inventory.yaml
+        run: |
+          cat spec/fixtures/litmus_inventory.yaml
+      - name: Install PE on test cluster
+        timeout-minutes: 120
+        run: |
+          bundle exec bolt plan run peadm_spec::install_test_cluster \
+            --inventoryfile spec/fixtures/litmus_inventory.yaml \
+            --modulepath spec/fixtures/modules \
+            architecture=${{ matrix.architecture }} \
+            version=${{ matrix.version }} \
+            console_password=${{ secrets.CONSOLE_PASSWORD }} \
+            code_manager_auto_configure=true
+      - name: Perform migration
+        run: |
+          echo ::group::prepare
+            mkdir -p $HOME/.ssh
+            echo 'Host *'                      >  $HOME/.ssh/config
+            echo '    ServerAliveInterval 150' >> $HOME/.ssh/config
+            echo '    ServerAliveCountMax 2'   >> $HOME/.ssh/config
+            bundle exec rake spec_prep
+          echo ::endgroup::
+          bundle exec bolt plan run peadm_spec::test_migration \
+            --inventoryfile spec/fixtures/litmus_inventory.yaml \
+            --modulepath spec/fixtures/modules \
+            primary_host=$(yq -r '.groups[].targets[] | select(.vars.role == "primary") | .name // ""' spec/fixtures/litmus_inventory.yaml) \
+            new_primary_host=$(yq -r '.groups[].targets[] | select(.vars.role == "new-primary") | .name // ""' spec/fixtures/litmus_inventory.yaml) \
+            new_replica_host=$(yq -r '.groups[].targets[] | select(.vars.role == "new-replica") | .name // ""' spec/fixtures/litmus_inventory.yaml) \
+            new_primary_postgresql_host=$(yq -r '.groups[].targets[] | select(.vars.role == "new-primary-pdb-postgresql") | .name // ""' spec/fixtures/litmus_inventory.yaml) \
+            new_replica_postgresql_host=$(yq -r '.groups[].targets[] | select(.vars.role == "new-replica-pdb-postgresql") | .name // ""' spec/fixtures/litmus_inventory.yaml) \
+            upgrade_version=${{ matrix.new_pe_version }} \
+          --no-host-key-check
+      - name: Tear down PE ${{ matrix.architecture }} test cluster
+        if: ${{ always() }}
+        continue-on-error: true
+        run: |-
+          if [ -f spec/fixtures/litmus_inventory.yaml ]; then
+            echo ::group::tear_down
+              bundle exec rake 'litmus:tear_down'
+            echo ::endgroup::
+            echo ::group::info:request
+              cat request.json || true; echo
+            echo ::endgroup::
+          fi

spec/acceptance/peadm_spec/plans/provision_test_cluster.pp

@@ -54,6 +54,24 @@
         ['primary', 'primary-pdb-postgresql', 'compiler',
         'replica', 'replica-pdb-postgresql', 'compiler', 'spare-replica']
       }
+      'standard-migration': {
+        ['primary', 'new-primary']
+      }
+      'standard-with-dr-migration': {
+        ['primary', 'replica', 'new-primary', 'new-replica']
+      }
+      'large-migration': {
+        ['primary', 'compiler', 'new-primary']
+      }
+      'large-with-dr-migration': {
+        ['primary', 'compiler', 'replica', 'compiler', 'new-primary', 'new-replica']
+      }
+      'extra-large-migration': {
+        ['primary', 'primary-pdb-postgresql', 'compiler', 'new-primary', 'new-primary-pdb-postgresql']
+      }
+      'extra-large-with-dr-migration': {
+        ['primary', 'primary-pdb-postgresql', 'compiler', 'replica', 'replica-pdb-postgresql', 'compiler', 'new-primary', 'new-replica', 'new-primary-pdb-postgresql', 'new-replica-pdb-postgresql']
+      }
       default: {
         fail_plan("Unknown architecture: ${architecture}")
       }

spec/acceptance/peadm_spec/plans/test_migration.pp

@@ -0,0 +1,83 @@
+plan peadm_spec::test_migration(
+  String $primary_host,
+  String $new_primary_host,
+  Optional[String] $new_replica_host = undef,
+  Optional[String] $new_primary_postgresql_host = undef,
+  Optional[String] $new_replica_postgresql_host = undef,
+  Optional[String] $upgrade_version = undef,
+) {
+  # Convert String values to targets if they are not blank
+  $primary_target = $primary_host ? { '' => undef, default => peadm::get_targets($primary_host, 1) }
+  $new_primary_target = $new_primary_host ? { '' => undef, default => peadm::get_targets($new_primary_host, 1) }
+  $new_replica_target = $new_replica_host ? { '' => undef, default => peadm::get_targets($new_replica_host, 1) }
+  $new_primary_postgresql_target = $new_primary_postgresql_host ? { '' => undef, default => peadm::get_targets($new_primary_postgresql_host, 1) }
+  $new_replica_postgresql_target = $new_replica_postgresql_host ? { '' => undef, default => peadm::get_targets($new_replica_postgresql_host, 1) }
+
+  # run infra status on the primary
+  out::message("Running peadm::status on primary host ${primary_target}")
+  $primary_status = run_plan('peadm::status', $primary_target, { 'format' => 'json' })
+  out::message($primary_status)
+
+  if empty($primary_status['failed']) {
+    out::message('Cluster is healthy, continuing')
+  } else {
+    fail_plan('Cluster is not healthy, aborting')
+  }
+
+  # perform the migration
+  run_plan('peadm::migrate',
+    old_primary_host => $primary_target,
+    new_primary_host => $new_primary_target,
+    upgrade_version => $upgrade_version,
+    replica_host => $new_replica_target,
+  )
+
+  # run infra status on the new primary
+  peadm::wait_until_service_ready('all', $new_primary_target)
+  out::message("Running peadm::status on new primary host ${new_primary_target}")
+  $new_primary_status = run_plan('peadm::status', $new_primary_target, { 'format' => 'json' })
+  if empty($new_primary_status['failed']) {
+    out::message('Migrated cluster is healthy, continuing')
+  } else {
+    out::message('Migrated cluster is not healthy, verify status of services')
+  }
+
+  # get the config from new_primary_target and verify config looks as expected
+  $peadm_config = run_task('peadm::get_peadm_config', $new_primary_target).first.value
+  out::message("peadm_config:${peadm_config}.")
+  # if new_replica_target is supplied then check that is in the expected place in the config
+  if $new_replica_target {
+    if $peadm_config['params']['replica_host'] == $new_replica_target.peadm::certname() {
+      out::message("New replica host ${new_replica_target.peadm::certname()} set up correctly")
+    } else {
+      fail_plan("New replica host ${new_replica_target.peadm::certname()} was not set up correctly")
+    }
+  }
+
+  # if new_primary_postgresql_target is supplied then check that is in the expected place in the config
+  if $new_primary_postgresql_target {
+    if $peadm_config['params']['primary_postgresql_host'] == $new_primary_postgresql_target.peadm::certname() {
+      out::message("New primary postgres host ${new_primary_postgresql_target.peadm::certname()} set up correctly")
+    } else {
+      fail_plan("New primary postgres host ${new_primary_postgresql_target.peadm::certname()} was not set up correctly")
+    }
+  }
+
+  # if new_replica_postgresql_target is supplied then check that is in the expected place in the config
+  if $new_replica_postgresql_target {
+    if $peadm_config['params']['replica_postgresql_host'] == $new_replica_postgresql_target.peadm::certname() {
+      out::message("New primary postgres host ${new_replica_postgresql_target.peadm::certname()} set up correctly")
+    } else {
+      fail_plan("New primary postgres host ${new_replica_postgresql_target.peadm::certname()} was not set up correctly")
+    }
+  }
+
+  # if a new PE version was specified then check it has been upgraded
+  if $upgrade_version and $upgrade_version != '' and !empty($upgrade_version) {
+    if $peadm_config['pe_version'] == $upgrade_version {
+      out::message("Upgraded to new PE version ${upgrade_version} correctly")
+    } else {
+      fail_plan("Failed to upgrade to new PE version ${upgrade_version} correctly")
+    }
+  }
+}