Add prepare_client_pod stage for warm client-pod bootstrap

When cifmw recreates shiftstackclient and jobs omit prepare (Pipeline B
run-tests), the empty pod lacks oc, clouds.shiftstack, /etc/hosts, and
resources.yml. Add a lightweight stage that restores only that subset
from the PVC / install-config without re-running full prepare.

Address review feedback:
- Assert a clouds.yaml source exists before creating ~/.config/openstack
- Bootstrap when oc or kubectl is missing
- Resolve API/apps from FloatingIP with VIP/VIPs fallback for proxy IC
- Run prepare_client_pod before create_installer_group
- Reject original-config fallback unless clouds[user_cloud] is present

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: tusharjadhav3302

collection/stages/roles/prepare_client_pod/defaults/main.yml

@@ -0,0 +1,24 @@
+---
+# Lightweight client bootstrap when full "prepare" is skipped (e.g. Pipeline B
+# run-tests after a warm guest install). Mirrors the subset of prepare that is
+# lost when the shiftstackclient pod is recreated: oc, clouds.shiftstack,
+# /etc/hosts, and resources.yml.
+#
+# Do NOT recreate OpenStack projects, FIPs, or installer hosts here.
+prepare_client_pod_bootstrap_oc_dir: "{{ home_dir }}/bootstrap-oc"
+prepare_client_pod_bootstrap_oc_url: >-
+  {{ openshift_mirror_url | default('https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp') }}/stable/openshift-client-linux.tar.gz
+prepare_client_pod_install_config: "{{ user_cloud_installation_dir }}/install-config.yaml"
+prepare_client_pod_metadata: "{{ ocp_installation_dir }}/metadata.json"
+# Same host rows as prepare/defaults etc_hosts_entries (api_ip / apps_ip at runtime).
+prepare_client_pod_etc_hosts_entries:
+  - {regex: 'api\..*\.{{ ocp_base_domain }}', row: '{{ api_ip }} api.{{ ocp_cluster_name }}.{{ ocp_base_domain }}'}
+  - {regex: 'oauth-openshift.apps\..*\.{{ ocp_base_domain }}', row: '{{ apps_ip }} oauth-openshift.apps.{{ ocp_cluster_name }}.{{ ocp_base_domain }}'}
+  - {regex: 'console-openshift-console.apps\..*\.{{ ocp_base_domain }}', row: '{{ apps_ip }} console-openshift-console.apps.{{ ocp_cluster_name }}.{{ ocp_base_domain }}'}
+  - {regex: 'downloads-openshift-console.apps\..*\.{{ ocp_base_domain }}', row: '{{ apps_ip }} downloads-openshift-console.apps.{{ ocp_cluster_name }}.{{ ocp_base_domain }}'}
+  - {regex: 'canary-openshift-ingress-canary.apps\..*\.{{ ocp_base_domain }}', row: '{{ apps_ip }} canary-openshift-ingress-canary.apps.{{ ocp_cluster_name }}.{{ ocp_base_domain }}'}
+  - {regex: 'alertmanager-main-openshift-monitoring.apps\..*\.{{ ocp_base_domain }}', row: '{{ apps_ip }} alertmanager-main-openshift-monitoring.apps.{{ ocp_cluster_name }}.{{ ocp_base_domain }}'}
+  - {regex: 'grafana-openshift-monitoring.apps\..*\.{{ ocp_base_domain }}', row: '{{ apps_ip }} grafana-openshift-monitoring.apps.{{ ocp_cluster_name }}.{{ ocp_base_domain }}'}
+  - {regex: 'prometheus-k8s-openshift-monitoring.apps\..*\.{{ ocp_base_domain }}', row: '{{ apps_ip }} prometheus-k8s-openshift-monitoring.apps.{{ ocp_cluster_name }}.{{ ocp_base_domain }}'}
+  - {regex: 'prometheus-k8s-federate-openshift-monitoring.apps\..*\.{{ ocp_base_domain }}', row: '{{ apps_ip }} prometheus-k8s-federate-openshift-monitoring.apps.{{ ocp_cluster_name }}.{{ ocp_base_domain }}'}
+  - {regex: 'thanos-querier-openshift-monitoring.apps\..*\.{{ ocp_base_domain }}', row: '{{ apps_ip }} thanos-querier-openshift-monitoring.apps.{{ ocp_cluster_name }}.{{ ocp_base_domain }}'}

collection/stages/roles/prepare_client_pod/meta/main.yml

@@ -0,0 +1,3 @@
+---
+collections:
+  - shiftstack.tools

collection/stages/roles/prepare_client_pod/tasks/main.yml

@@ -0,0 +1,287 @@
+# Prepare the recreated shiftstackclient / installer inventory host for warm
+# testing when the full prepare stage was skipped.
+
+- name: Check for PVC OpenStack clouds.yaml
+  ansible.builtin.stat:
+    path: "{{ osp_config_dir }}/clouds.yaml"
+  register: prepare_client_pod_pvc_clouds
+
+- name: Check for original-config OpenStack clouds.yaml
+  ansible.builtin.stat:
+    path: "{{ home_dir }}/.original-config/openstack/clouds.yaml"
+  register: prepare_client_pod_orig_clouds
+
+- name: Fail when no OpenStack clouds.yaml source is available
+  ansible.builtin.assert:
+    that:
+      - >-
+        (prepare_client_pod_pvc_clouds.stat.exists | default(false))
+        or (prepare_client_pod_orig_clouds.stat.exists | default(false))
+    fail_msg: >-
+      prepare_client_pod requires clouds.yaml at {{ osp_config_dir }}/clouds.yaml
+      or {{ home_dir }}/.original-config/openstack/clouds.yaml to restore
+      OpenStack client config for warm testing.
+
+- name: Ensure OpenStack client config directory exists
+  ansible.builtin.file:
+    path: "{{ home_dir }}/.config/openstack"
+    state: directory
+    mode: u=rwx,g=rx,o=rx
+
+- name: Restore OpenStack clouds config from PVC osp_config_dir
+  when: prepare_client_pod_pvc_clouds.stat.exists | default(false)
+  ansible.builtin.copy:
+    src: "{{ osp_config_dir }}/"
+    dest: "{{ home_dir }}/.config/openstack/"
+    remote_src: true
+    mode: preserve
+
+- name: Load original-config clouds.yaml when PVC copy is absent
+  when:
+    - not (prepare_client_pod_pvc_clouds.stat.exists | default(false))
+    - prepare_client_pod_orig_clouds.stat.exists | default(false)
+  ansible.builtin.slurp:
+    src: "{{ home_dir }}/.original-config/openstack/clouds.yaml"
+  register: prepare_client_pod_orig_clouds_slurp
+
+- name: Fail when original-config clouds.yaml lacks user_cloud
+  when:
+    - not (prepare_client_pod_pvc_clouds.stat.exists | default(false))
+    - prepare_client_pod_orig_clouds.stat.exists | default(false)
+  vars:
+    _orig_clouds: >-
+      {{
+        prepare_client_pod_orig_clouds_slurp.content
+        | b64decode
+        | from_yaml
+      }}
+  ansible.builtin.assert:
+    that:
+      - user_cloud in (_orig_clouds.clouds | default({}))
+    fail_msg: >-
+      prepare_client_pod fallback
+      {{ home_dir }}/.original-config/openstack/clouds.yaml does not define
+      clouds.{{ user_cloud }}. The prepare stage writes that cloud into
+      {{ osp_config_dir }}/clouds.yaml; restore the PVC copy or re-run prepare.
+
+- name: Fall back to original-config OpenStack clouds when PVC copy is absent
+  when:
+    - not (prepare_client_pod_pvc_clouds.stat.exists | default(false))
+    - prepare_client_pod_orig_clouds.stat.exists | default(false)
+  ansible.builtin.copy:
+    src: "{{ home_dir }}/.original-config/openstack/"
+    dest: "{{ home_dir }}/.config/openstack/"
+    remote_src: true
+    mode: preserve
+
+- name: Load restored clouds.yaml
+  ansible.builtin.slurp:
+    src: "{{ home_dir }}/.config/openstack/clouds.yaml"
+  register: prepare_client_pod_restored_clouds_slurp
+
+- name: Fail when restored clouds.yaml lacks user_cloud
+  vars:
+    _restored_clouds: >-
+      {{
+        prepare_client_pod_restored_clouds_slurp.content
+        | b64decode
+        | from_yaml
+      }}
+  ansible.builtin.assert:
+    that:
+      - user_cloud in (_restored_clouds.clouds | default({}))
+    fail_msg: >-
+      prepare_client_pod restored
+      {{ home_dir }}/.config/openstack/clouds.yaml but clouds.{{ user_cloud }}
+      is missing. Warm testing requires the project cloud written by prepare.
+
+- name: Check for secure.yaml on destination and original-config
+  ansible.builtin.stat:
+    path: "{{ item }}"
+  loop:
+    - "{{ home_dir }}/.config/openstack/secure.yaml"
+    - "{{ home_dir }}/.original-config/openstack/secure.yaml"
+  register: prepare_client_pod_secure_stats
+
+- name: Ensure secure.yaml is present when only available on original-config
+  when:
+    - not (prepare_client_pod_secure_stats.results[0].stat.exists | default(false))
+    - prepare_client_pod_secure_stats.results[1].stat.exists | default(false)
+  ansible.builtin.copy:
+    src: "{{ home_dir }}/.original-config/openstack/secure.yaml"
+    dest: "{{ home_dir }}/.config/openstack/secure.yaml"
+    remote_src: true
+    mode: preserve
+
+- name: Check whether oc is already available
+  ansible.builtin.command:
+    cmd: which oc
+  register: prepare_client_pod_oc_which
+  changed_when: false
+  failed_when: false
+
+- name: Check whether kubectl is already available
+  ansible.builtin.command:
+    cmd: which kubectl
+  register: prepare_client_pod_kubectl_which
+  changed_when: false
+  failed_when: false
+
+- name: Bootstrap oc and kubectl from the OpenShift mirror
+  when: >-
+    prepare_client_pod_oc_which.rc != 0
+    or prepare_client_pod_kubectl_which.rc != 0
+  block:
+    - name: Create bootstrap oc directory
+      ansible.builtin.file:
+        path: "{{ prepare_client_pod_bootstrap_oc_dir }}"
+        state: directory
+        mode: u=rwx,g=rw,o=r
+
+    - name: Download and extract stable oc client
+      ansible.builtin.unarchive:
+        src: "{{ prepare_client_pod_bootstrap_oc_url }}"
+        dest: "{{ prepare_client_pod_bootstrap_oc_dir }}"
+        remote_src: true
+        include:
+          - oc
+          - kubectl
+      register: prepare_client_pod_oc_download
+      until: prepare_client_pod_oc_download is not failed
+      retries: 3
+      delay: 10
+
+    - name: Symlink oc into /usr/local/bin
+      ansible.builtin.file:
+        src: "{{ prepare_client_pod_bootstrap_oc_dir }}/oc"
+        dest: /usr/local/bin/oc
+        state: link
+      become: true
+
+    - name: Symlink kubectl into /usr/bin
+      ansible.builtin.file:
+        src: "{{ prepare_client_pod_bootstrap_oc_dir }}/kubectl"
+        dest: /usr/bin/kubectl
+        state: link
+      become: true
+
+- name: Check install-config.yaml on the PVC
+  ansible.builtin.stat:
+    path: "{{ prepare_client_pod_install_config }}"
+  register: prepare_client_pod_ic_stat
+
+- name: Fail when install-config.yaml is missing
+  ansible.builtin.assert:
+    that:
+      - prepare_client_pod_ic_stat.stat.exists | default(false)
+    fail_msg: >-
+      prepare_client_pod requires {{ prepare_client_pod_install_config }} on the
+      PVC to restore guest API/apps addresses into /etc/hosts and resources.yml.
+
+- name: Load install-config.yaml
+  ansible.builtin.slurp:
+    src: "{{ prepare_client_pod_install_config }}"
+  register: prepare_client_pod_install_config_slurp
+
+- name: Extract OpenStack platform keys from install-config
+  vars:
+    _ic: "{{ prepare_client_pod_install_config_slurp.content | b64decode | from_yaml }}"
+  ansible.builtin.set_fact:
+    prepare_client_pod_osp: "{{ _ic.platform.openstack | default({}) }}"
+    prepare_client_pod_base_domain: "{{ _ic.baseDomain }}"
+    prepare_client_pod_cluster_name: "{{ ocp_cluster_name }}"
+
+# Prefer FloatingIP (standard IPI); fall back to VIP/VIPs used by proxy installs.
+- name: Resolve API accessible address (FloatingIP or VIP)
+  vars:
+    _fip: "{{ prepare_client_pod_osp.apiFloatingIP | default('') }}"
+    _vip: "{{ prepare_client_pod_osp.apiVIP | default(prepare_client_pod_osp.apiVIPs | default('')) }}"
+  ansible.builtin.set_fact:
+    api_accessible_ip: >-
+      {{
+        _fip
+        if (_fip | string | length > 0)
+        else (
+          _vip
+          if (_vip is string and (_vip | length > 0))
+          else ((_vip | list | first) | default(''))
+        )
+      }}
+
+- name: Resolve apps/ingress accessible address (FloatingIP or VIP)
+  vars:
+    _fip: "{{ prepare_client_pod_osp.ingressFloatingIP | default('') }}"
+    _vip: "{{ prepare_client_pod_osp.ingressVIP | default(prepare_client_pod_osp.ingressVIPs | default('')) }}"
+  ansible.builtin.set_fact:
+    apps_accessible_ip: >-
+      {{
+        _fip
+        if (_fip | string | length > 0)
+        else (
+          _vip
+          if (_vip is string and (_vip | length > 0))
+          else ((_vip | list | first) | default(''))
+        )
+      }}
+
+- name: Fail when API/apps addresses cannot be resolved from install-config
+  ansible.builtin.assert:
+    that:
+      - api_accessible_ip | string | length > 0
+      - apps_accessible_ip | string | length > 0
+    fail_msg: >-
+      prepare_client_pod could not resolve guest API/apps addresses from
+      {{ prepare_client_pod_install_config }}. Expected
+      platform.openstack.apiFloatingIP/ingressFloatingIP (standard IPI) or
+      apiVIP/ingressVIP (or apiVIPs/ingressVIPs) for proxy deployments.
+
+- name: Expose addresses for /etc/hosts template vars
+  ansible.builtin.set_fact:
+    api_ip: "{{ api_accessible_ip }}"
+    apps_ip: "{{ apps_accessible_ip }}"
+
+- name: Check for metadata.json
+  ansible.builtin.stat:
+    path: "{{ prepare_client_pod_metadata }}"
+  register: prepare_client_pod_meta_stat
+
+- name: Override cluster name from metadata.json when present
+  when: prepare_client_pod_meta_stat.stat.exists | default(false)
+  block:
+    - name: Load metadata.json
+      ansible.builtin.slurp:
+        src: "{{ prepare_client_pod_metadata }}"
+      register: prepare_client_pod_metadata_slurp
+
+    - name: Set cluster name from metadata
+      vars:
+        _meta: "{{ prepare_client_pod_metadata_slurp.content | b64decode | from_json }}"
+      ansible.builtin.set_fact:
+        prepare_client_pod_cluster_name: >-
+          {{ _meta.clusterName | default(ocp_cluster_name) }}
+
+- name: Check whether resources.yml already exists
+  ansible.builtin.stat:
+    path: "{{ resources_file }}"
+  register: prepare_client_pod_resources_stat
+
+- name: Write resources.yml when absent
+  when: not (prepare_client_pod_resources_stat.stat.exists | default(false))
+  ansible.builtin.include_role:
+    name: shiftstack.tools.tools_register_resources_file
+  vars:
+    input:
+      api_accessible_ip: "{{ api_accessible_ip }}"
+      apps_accessible_ip: "{{ apps_accessible_ip }}"
+
+- name: Restore guest API and apps entries in /etc/hosts
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/hosts
+    regexp: "{{ item.regex }}"
+    line: "{{ item.row }}"
+    unsafe_writes: true
+  vars:
+    ocp_cluster_name: "{{ prepare_client_pod_cluster_name }}"
+    ocp_base_domain: "{{ prepare_client_pod_base_domain }}"
+  loop: "{{ prepare_client_pod_etc_hosts_entries }}"

playbooks/ocp_testing.yaml

@@ -41,6 +41,12 @@
         mode: u=rwx,g=rwx,o=rwx
       when: "'cleanup' in stages or 'prepare' in stages"
 
+# Must run before create_installer_group so OpenStack credentials exist when
+# server_info looks up the installer VM on a recreated client pod.
+- name: Prepare the shiftstackclient pod when full prepare was skipped
+  ansible.builtin.import_playbook: plays/prepare_client_pod.yaml
+  when: "'prepare_client_pod' in stages"
+
 - name: Prepare the OpenShift environment
   hosts: localhost
   gather_facts: no

playbooks/plays/prepare_client_pod.yaml

@@ -0,0 +1,10 @@
+---
+- name: Prepare the shiftstackclient pod for warm testing
+  hosts: localhost
+  gather_facts: true
+  vars_files:
+    - "../../configs/global.yml"
+  tasks:
+    - name: Bootstrap oc, clouds, hosts, and resources.yml on the client pod
+      ansible.builtin.include_role:
+        name: shiftstack.stages.prepare_client_pod