openstack-cloudlinux-8.yml

name: "Test Run: CloudLinux-8"

on:
  workflow_call:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-openstack-cloudlinux-8
  cancel-in-progress: true

env:
  # secrets
  TF_VAR_application_credential_id: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
  TF_VAR_application_credential_secret: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
  TF_VAR_ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}
  TF_VAR_ssh_public_key: ${{ secrets.SSH_PUBLIC_KEY }}

  # vars
  TF_VAR_os_auth_url: ${{ vars.OS_AUTH_URL }}
  TF_VAR_os_auth_type: ${{ vars.OS_AUTH_TYPE }}
  TF_VAR_os_identity_api_version: ${{ vars.OS_IDENTITY_API_VERSION }}
  TF_VAR_os_interface: ${{ vars.OS_INTERFACE }}
  TF_VAR_os_region_name: ${{ vars.OS_REGION_NAME }}

  # github
  TF_VAR_github_repository: ${{ github.repository }}
  TF_VAR_github_run_id: ${{ github.run_id }}
  tf_working_directory: "${{ github.workspace }}/.github/workflows/openstack"

jobs:
  terraform_openstack_create:
    runs-on:
      group: org/cpanel-arc-elevate
    container:
      image: ghcr.io/webpros-cpanel/misc-ci-containers:ci-elevate-cpanel
    defaults:
      run:
        working-directory: "${{ github.workspace }}/.github/workflows/openstack"
    outputs:
      VM_IP: ${{ steps.get_vm_ip.outputs.VM_IP }}
      REPODIR: ${{ steps.expose_repodir.outputs.REPODIR }}
    steps:
      - uses: actions/checkout@v6

      - name: Install Packages Needed by GitHub Network Check
        run: |
          apk add --no-cache curl

      - name: Terraform Init & Apply
        run: |
          for i in {1..36}; do curl -I https://objects.githubusercontent.com && break || echo "Attempt $i failed. Retrying in 5s..."; sleep 5; done
          MAX_ATTEMPTS=5
          SLEEP_SECONDS=5
          for attempt in $(seq 1 $MAX_ATTEMPTS); do
            echo "Attempt $attempt of $MAX_ATTEMPTS: terraform init"
            if terraform init; then
              echo "✅ terraform init succeeded on attempt $attempt"
              break
            else
              echo "❌ terraform init failed on attempt $attempt"
              if [ "$attempt" -lt "$MAX_ATTEMPTS" ]; then
                echo "→ sleeping ${SLEEP_SECONDS}s before retry"
                sleep $SLEEP_SECONDS
              else
                echo "🚨 terraform init failed after $MAX_ATTEMPTS attempts"
                exit 1
              fi
            fi
          done
          terraform apply -var="image_name=11.134.0.* on CloudLinux 8" -var="cpanel_release_version=134" -no-color -auto-approve
          terraform output -json > ${{ github.run_id }}-tf.out.json
          mv -v .terraform.lock.hcl terraform.lock.hcl
      - name: Upload Terraform Artifacts
        uses: actions/upload-artifact@v7
        with:
          name: ${{ github.run_id }}-cloudlinux8-terraform.tfstate
          path: ${{ env.tf_working_directory }}/terraform*
          retention-days: 1 # override 90-day default
      - name: Get VM IP from Terraform Output
        id: get_vm_ip
        run: |
          echo "VM_IP=$(jq -r '.address.value' ${{ github.run_id }}-tf.out.json)" >> "$GITHUB_OUTPUT"
      - name: Expose REPODIR as a Variable
        id: expose_repodir
        run: |
          REPODIR=$(echo ${{ github.repository }} | cut -d / -f2)
          echo "REPODIR=$REPODIR" >> "$GITHUB_OUTPUT"

  clone_elevate_repo:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: terraform_openstack_create
    env:
      VM_IP: ${{ needs.terraform_openstack_create.outputs.VM_IP }}
      REPODIR: ${{ needs.terraform_openstack_create.outputs.REPODIR }}
    outputs:
      VM_IP: ${{ env.VM_IP }}
      REPODIR: ${{ env.REPODIR }}
    steps:
      - name: Checkout Repo
        uses: appleboy/ssh-action@v1.1.0
        with:
          host: ${{ env.VM_IP }}
          username: "root"
          key: ${{ secrets.SSH_PRIVATE_KEY }}
          port: "22"
          script: |
            cd /opt
            BRANCH_NAME=${{ github.ref_name }}
            if [ "${{ github.event_name }}" == "pull_request" ]; then
              BRANCH_NAME=${{ github.head_ref }}
            fi

            echo "## [INFO]: Checking out branch $BRANCH_NAME"
            echo "## [INFO]: Repository: ${{ github.repository }}"

            ssh-keyscan github.com >> ~/.ssh/known_hosts
            mkdir -p ~/.ssh
            echo "${{ secrets.ELEVATE_CHECKOUT_KEY }}" > ~/.ssh/elevate_checkout_key
            chmod 0600 ~/.ssh/elevate_checkout_key

            cat <<EOL > ~/.ssh/config
            Host github.com
              HostName github.com
              User root
              IdentityFile ~/.ssh/elevate_checkout_key
              IdentitiesOnly yes
              HostKeyAlgorithms +ssh-rsa
            EOL

            echo "## [DEBUG]: git clone --depth=1 --branch $BRANCH_NAME git@github.com:${{ github.repository }}.git"
            git clone --depth=1 --branch $BRANCH_NAME git@github.com:${{ github.repository }}.git

            cd /opt/$(basename "${{ github.repository }}")
            git status

  setup_integration_checks:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: clone_elevate_repo
    env:
      CLOUDLINUX_LICENSE_KEY: ${{ secrets.CLOUDLINUX_LICENSE_KEY }}
      VM_IP: ${{ needs.clone_elevate_repo.outputs.VM_IP }}
      REPODIR: ${{ needs.clone_elevate_repo.outputs.REPODIR }}
    outputs:
      VM_IP: ${{ env.VM_IP }}
      REPODIR: ${{ env.REPODIR }}
    steps:
      - name: Install Packages Needed by Scripts.
        run: |
          sudo apt-get update
          sudo apt install -y netcat-openbsd

      - name: Setup for Integration Checks Prior to Running Elevate
        uses: appleboy/ssh-action@v1.1.0
        with:
          host: ${{ env.VM_IP }}
          username: "root"
          key: ${{ secrets.SSH_PRIVATE_KEY }}
          port: "22"
          timeout: 45m
          command_timeout: 45m
          envs: REPODIR
          script: |
            chmod -v +x /opt/${{ env.REPODIR}}/t/integration/setup
            /opt/${{ env.REPODIR }}/t/integration/setup

      - name: Update License And Setup Server For ELevate
        uses: appleboy/ssh-action@v1.1.0
        with:
          host: ${{ env.VM_IP }}
          username: "root"
          key: ${{ secrets.SSH_PRIVATE_KEY }}
          port: "22"
          timeout: 90m
          command_timeout: 90m
          envs: REPODIR,CLOUDLINUX_LICENSE_KEY
          script: |
            echo "## [INFO]: Removing /etc/yum/pluginconf.d/rhnplugin.conf" && rm -fv /etc/yum/pluginconf.d/rhnplugin.conf
            echo "## [INFO]: /usr/sbin/rhnreg_ks --activationkey=${CLOUDLINUX_LICENSE_KEY:0:2}****" --force && /usr/sbin/rhnreg_ks --activationkey=$CLOUDLINUX_LICENSE_KEY --force
            echo "## [INFO]: Now running /usr/bin/cldetect --update-license" && /usr/bin/cldetect --update-license
            echo "## [INFO]: Now running /usr/bin/cldetect --check-license" && /usr/bin/cldetect --check-license
            until /usr/local/cpanel/cpkeyclt; do echo "Retrying /usr/local/cpanel/cpkeyct until it passes..."; sleep 5; done
            echo "[cloudlinux-base]" > /etc/yum.repos.d/cloudlinux-base.repo
            echo "name=CloudLinux-\$releasever - Base" >> /etc/yum.repos.d/cloudlinux-base.repo
            echo "baseurl=https://repo.cloudlinux.com/cloudlinux/\$releasever/BaseOS/\$basearch/os/" >> /etc/yum.repos.d/cloudlinux-base.repo
            echo "enabled=0" >> /etc/yum.repos.d/cloudlinux-base.repo
            echo "gpgcheck=1" >> /etc/yum.repos.d/cloudlinux-base.repo
            echo "gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CloudLinux" >> /etc/yum.repos.d/cloudlinux-base.repo
            echo "" >> /etc/yum.repos.d/cloudlinux-base.repo
            rm -f /etc/yum.repos.d/almalinux*.repo
            /usr/bin/yum -y reinstall cloudlinux-release
            sed -i 's/\[powertools\]/[cloudlinux-PowerTools]/' /etc/yum.repos.d/almalinux-powertools.repo
            /usr/bin/yum erase -y awscli git-lfs kmod-lve-2.0-43
            rpm -evh kmod-lve-2.0-43.el8.x86_64 || true
            echo "yum remove -y `yum list installed | awk '$3~/cloudlinux-PowerTools/{printf "%s ", $1}'`" | sh
            echo "yum reinstall -y `yum list installed | awk '$3~/cloudlinux-BaseOS/{printf "%s ", $1}'` --disablerepo=cloudlinux-BaseOS --enablerepo=cloudlinux-base" | sh
            mv -v /etc/yum.repos.d/clmirror.repo /root/
            yum-config-manager --enable cloudlinux-base
            /usr/bin/yum clean all
            /usr/bin/yum makecache
            /usr/bin/yum -y update
            mv -v /etc/yum.repos.d/cloudlinux-compat.repo /root/
            REPODIR=$(echo ${{ github.repository }} | cut -d / -f2)
            cp -pv /opt/${REPODIR}/elevate-cpanel /scripts/elevate-cpanel
            cp -pv /opt/${REPODIR}/.github/workflows/openstack/status_marker /scripts/status_marker
            cp -pv /opt/${REPODIR}/.github/workflows/openstack/reboot_watch /scripts/reboot_watch
            chmod -v +x /scripts/elevate-cpanel
            shutdown -r +0 -t 15 "Restarting system after updates before we Elevate." &
            exit 0

  wait_for_stage_1_reboot:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: setup_integration_checks
    env:
      VM_IP: ${{ needs.setup_integration_checks.outputs.VM_IP}}
    outputs:
      VM_IP: ${{ env.VM_IP }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Install Packages Needed by Scripts.
        run: |
          sudo apt-get update
          sudo apt install -y netcat-openbsd

      - name: Wait For VM to Come Back From Initial Reboot
        working-directory: ".github/workflows/openstack/"
        run: |
          ./ssh_retry ${{ env.VM_IP }}

  start_elevate:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: wait_for_stage_1_reboot
    env:
      VM_IP: ${{ needs.wait_for_stage_1_reboot.outputs.VM_IP}}
    outputs:
      VM_IP: ${{ env.VM_IP }}
    steps:
      - name: Start Elevate
        uses: appleboy/ssh-action@v1.1.0
        with:
          host: ${{ env.VM_IP }}
          username: "root"
          key: ${{ secrets.SSH_PRIVATE_KEY }}
          port: "22"
          timeout: 60m
          command_timeout: 35m
          debug: true
          script: |
            /scripts/status_marker 1
            nohup /scripts/elevate-cpanel --is-ran-from-ci --non-interactive --skip-cpanel-version-check --start &
            exit 0

  wait_for_stage_2_reboot:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: start_elevate
    env:
      VM_IP: ${{ needs.start_elevate.outputs.VM_IP}}
    outputs:
      VM_IP: ${{ env.VM_IP }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Install Packages Needed by Scripts.
        run: |
          sudo apt-get update
          sudo apt install -y netcat-openbsd

      - name: Wait For VM to Come Back From Stage 2 Reboot
        working-directory: ".github/workflows/openstack/"
        run: |
          ./ssh_retry ${{ env.VM_IP }}

  watch_for_stage_3_reboot:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: wait_for_stage_2_reboot
    env:
      VM_IP: ${{ needs.wait_for_stage_2_reboot.outputs.VM_IP}}
    outputs:
      VM_IP: ${{ env.VM_IP }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Monitor Elevate for Stage 3 Reboot
        uses: appleboy/ssh-action@v1.1.0
        with:
          host: ${{ env.VM_IP }}
          username: "root"
          key: ${{ secrets.SSH_PRIVATE_KEY }}
          port: "22"
          timeout: 60m
          command_timeout: 30m
          debug: true
          script: |
            /scripts/status_marker 2
            tail -n40 -F /var/log/elevate-cpanel.log &
            sleep .5
            REBOOT_STRING="Rebooting into stage 3 of 5" RETVAL=1 /scripts/reboot_watch

  wait_for_stage_3_reboot:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: watch_for_stage_3_reboot
    env:
      VM_IP: ${{ needs.watch_for_stage_3_reboot.outputs.VM_IP}}
    outputs:
      VM_IP: ${{ env.VM_IP }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Install Packages Needed by Scripts.
        run: |
          sudo apt-get update
          sudo apt install -y netcat-openbsd

      - name: Wait For VM to Come Back From Stage 3 Reboot
        working-directory: ".github/workflows/openstack/"
        run: |
          ./ssh_retry ${{ env.VM_IP }}

  watch_for_stage_4_reboot:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: wait_for_stage_3_reboot
    env:
      VM_IP: ${{ needs.wait_for_stage_3_reboot.outputs.VM_IP}}
    outputs:
      VM_IP: ${{ env.VM_IP }}
    steps:
      - name: Monitor Elevate for Stage 4 Reboot
        uses: appleboy/ssh-action@v1.1.0
        with:
          host: ${{ env.VM_IP }}
          username: "root"
          key: ${{ secrets.SSH_PRIVATE_KEY }}
          port: "22"
          timeout: 60m
          command_timeout: 60m
          script: |
            /scripts/status_marker 3
            tail -n40 -F /var/log/elevate-cpanel.log &
            sleep .5
            REBOOT_STRING="Rebooting into stage 4 of 5" RETVAL=1 /scripts/reboot_watch

  wait_for_stage_4_reboot:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: watch_for_stage_4_reboot
    env:
      VM_IP: ${{ needs.watch_for_stage_4_reboot.outputs.VM_IP}}
    outputs:
      VM_IP: ${{ env.VM_IP }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Install Packages Needed by Scripts.
        run: |
          sudo apt-get update
          sudo apt install -y netcat-openbsd

      - name: Wait For VM to Come Back From Stage 4 Reboot
        working-directory: ".github/workflows/openstack/"
        run: |
          ./ssh_retry ${{ env.VM_IP  }}

  watch_for_stage_5_reboot:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: wait_for_stage_4_reboot
    env:
      VM_IP: ${{ needs.wait_for_stage_4_reboot.outputs.VM_IP}}
    outputs:
      VM_IP: ${{ env.VM_IP }}
    steps:
      - name: Monitor Elevate for Stage 5 Reboot
        uses: appleboy/ssh-action@v1.1.0
        with:
          host: ${{ env.VM_IP }}
          username: "root"
          key: ${{ secrets.SSH_PRIVATE_KEY }}
          port: "22"
          timeout: 45m
          command_timeout: 35m
          script: |
            /scripts/status_marker 4
            tail -n40 -F /var/log/elevate-cpanel.log &
            sleep .5
            REBOOT_STRING="Rebooting into stage 5 of 5" RETVAL=1 /scripts/reboot_watch

  wait_for_stage_5_reboot:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: watch_for_stage_5_reboot
    env:
      VM_IP: ${{ needs.watch_for_stage_5_reboot.outputs.VM_IP}}
    outputs:
      VM_IP: ${{ env.VM_IP }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Install Packages Needed by Scripts.
        run: |
          sudo apt-get update
          sudo apt install -y netcat-openbsd

      - name: Wait For VM to Come Back From Stage 5 Reboot
        working-directory: ".github/workflows/openstack/"
        run: |
          ./ssh_retry ${{ env.VM_IP }}

  watch_for_final_reboot:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: wait_for_stage_5_reboot
    env:
      VM_IP: ${{ needs.wait_for_stage_5_reboot.outputs.VM_IP}}
    outputs:
      VM_IP: ${{ env.VM_IP }}
    steps:
      - name: Monitor Elevate for Final Reboot
        uses: appleboy/ssh-action@v1.1.0
        with:
          host: ${{ env.VM_IP }}
          username: "root"
          key: ${{ secrets.SSH_PRIVATE_KEY }}
          port: "22"
          timeout: 45m
          command_timeout: 35m
          script: |
            /scripts/status_marker 5
            tail -n40 -F /var/log/elevate-cpanel.log &
            sleep .5
            REBOOT_STRING="Doing final reboot" RETVAL=1 /scripts/reboot_watch

  wait_for_final_reboot:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: watch_for_final_reboot
    env:
      VM_IP: ${{ needs.watch_for_final_reboot.outputs.VM_IP}}
    outputs:
      VM_IP: ${{ env.VM_IP }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Install Packages Needed by Scripts.
        run: |
          sudo apt-get update
          sudo apt install -y netcat-openbsd

      - name: Wait For VM to Come Back From Final Reboot
        working-directory: ".github/workflows/openstack/"
        run: |
          ./ssh_retry ${{ env.VM_IP }}

  verify_upgraded_os:
    runs-on:
      group: org/cpanel-arc-elevate
    needs: wait_for_final_reboot
    env:
      VM_IP: ${{ needs.wait_for_final_reboot.outputs.VM_IP}}
    outputs:
      VM_IP: ${{ env.VM_IP }}
    steps:
      - name: Verify End Result Integration Tests
        uses: appleboy/ssh-action@v1.1.0
        with:
          host: ${{ env.VM_IP }}
          username: "root"
          key: ${{ secrets.SSH_PRIVATE_KEY }}
          port: "22"
          timeout: 10m
          command_timeout: 10m
          script: |
            REPODIR=$(echo ${{ github.repository }} | cut -d / -f2)
            /usr/local/cpanel/3rdparty/bin/prove -lvm /opt/${REPODIR}/t/integration/cloudlinux8-to-cloudlinux9/
            /usr/local/cpanel/3rdparty/bin/prove -lvm /opt/${REPODIR}/t/integration/

  terraform_openstack_destroy:
    runs-on:
      group: org/cpanel-arc-elevate
    container:
      image: ghcr.io/webpros-cpanel/misc-ci-containers:ci-elevate-cpanel
    needs: verify_upgraded_os
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Download Terraform State
        uses: actions/download-artifact@v7
        with:
          name: ${{ github.run_id }}-cloudlinux8-terraform.tfstate
          path: ${{ env.tf_working_directory }}
      - name: Stage Files & Show Artifact Paths
        run: |
          pwd
          find -type d
          cd ".github/workflows/openstack"
          mv -v terraform.lock.hcl .terraform.lock.hcl
      - name: Initialize Terraform & Destroy
        run: |
          cd ".github/workflows/openstack"
          rm -Rfv .terraform || true
          for i in {1..36}; do curl -I https://objects.githubusercontent.com && break || echo "Attempt $i failed. Retrying in 5s..."; sleep 5; done
          MAX_ATTEMPTS=5
          SLEEP_SECONDS=5
          for attempt in $(seq 1 $MAX_ATTEMPTS); do
            echo "Attempt $attempt of $MAX_ATTEMPTS: terraform init"
            if terraform init; then
              echo "✅ terraform init succeeded on attempt $attempt"
              break
            else
              echo "❌ terraform init failed on attempt $attempt"
              if [ "$attempt" -lt "$MAX_ATTEMPTS" ]; then
                echo "→ sleeping ${SLEEP_SECONDS}s before retry"
                sleep $SLEEP_SECONDS
              else
                echo "🚨 terraform init failed after $MAX_ATTEMPTS attempts"
                exit 1
              fi
            fi
          done
          terraform destroy -var="image_name=11.134.0.* on CloudLinux 8" -var="cpanel_release_version=134" -no-color -auto-approve