From ae07ba4d2f90891bd3832aefb31a1a83c017be2c Mon Sep 17 00:00:00 2001 From: Neha Yadav Date: Tue, 23 Jun 2026 18:00:41 +0530 Subject: [PATCH 1/4] Fix signal handling to ensure metadata.json is copied on timeout --- .../ipi-install-powervs-install-commands.sh | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-commands.sh b/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-commands.sh index 0f18f7e173d52..093ea0d13bcdb 100755 --- a/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-commands.sh +++ b/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-commands.sh @@ -704,8 +704,10 @@ function dump_resources() { fi } -trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM -trap 'prepare_next_steps' EXIT TERM +# Combined trap to handle both child process cleanup and prepare_next_steps +# This allows the trap to execute when SIGTERM arrives by using background processes +# instead of blocking pipelines +trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill -TERM ${CHILDREN} 2>/dev/null; fi; wait; prepare_next_steps' EXIT TERM if [[ -z "$OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE" ]]; then echo "OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE is an empty string, exiting" @@ -935,8 +937,11 @@ export TF_LOG=debug echo "8<--------8<--------8<--------8<-------- BEGIN: create cluster 8<--------8<--------8<--------8<--------" echo "DATE=$(date --utc '+%Y-%m-%dT%H:%M:%S%:z')" -openshift-install --dir="${dir}" create cluster 2>&1 | grep --line-buffered -v 'password\|X-Auth-Token\|UserData:' -ret=${PIPESTATUS[0]} +# Run openshift-install in background with process substitution to allow trap to execute on SIGTERM +openshift-install --dir="${dir}" create cluster 2>&1 > >(grep --line-buffered -v 'password\|X-Auth-Token\|UserData:') & +INSTALL_PID=$! +wait $INSTALL_PID +ret=$? echo "ret=${ret}" if [ ${ret} -gt 0 ]; then SKIP_WAIT_FOR=false @@ -964,8 +969,11 @@ echo "SKIP_WAIT_FOR=${SKIP_WAIT_FOR}" if ! ${SKIP_WAIT_FOR}; then echo "8<--------8<--------8<--------8<-------- BEGIN: wait-for install-complete 8<--------8<--------8<--------8<--------" echo "DATE=$(date --utc '+%Y-%m-%dT%H:%M:%S%:z')" - openshift-install wait-for install-complete --dir="${dir}" | grep --line-buffered -v 'password\|X-Auth-Token\|UserData:' - ret=${PIPESTATUS[0]} + # Run openshift-install in background with process substitution to allow trap to execute on SIGTERM + openshift-install wait-for install-complete --dir="${dir}" 2>&1 > >(grep --line-buffered -v 'password\|X-Auth-Token\|UserData:') & + INSTALL_PID=$! + wait $INSTALL_PID + ret=$? echo "ret=${ret}" echo "8<--------8<--------8<--------8<-------- END: wait-for install-complete 8<--------8<--------8<--------8<--------" fi From 6196dfd463c901db6dd4689094639f9236a81408 Mon Sep 17 00:00:00 2001 From: Neha Yadav Date: Tue, 23 Jun 2026 18:16:12 +0530 Subject: [PATCH 2/4] Fix trap double-execution and secret filtering issues --- .../ipi-install-powervs-install-commands.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-commands.sh b/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-commands.sh index 093ea0d13bcdb..6a8b48f1ced8c 100755 --- a/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-commands.sh +++ b/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-commands.sh @@ -704,10 +704,11 @@ function dump_resources() { fi } -# Combined trap to handle both child process cleanup and prepare_next_steps -# This allows the trap to execute when SIGTERM arrives by using background processes -# instead of blocking pipelines -trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill -TERM ${CHILDREN} 2>/dev/null; fi; wait; prepare_next_steps' EXIT TERM +# Separate traps for EXIT and TERM to handle cleanup properly +# TERM: Kill children, run cleanup, then exit with proper signal status +# EXIT: Just run cleanup (for normal termination) +trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill -TERM ${CHILDREN} 2>/dev/null; fi; wait; trap - EXIT; prepare_next_steps; exit 143' TERM +trap 'prepare_next_steps' EXIT if [[ -z "$OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE" ]]; then echo "OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE is an empty string, exiting" @@ -938,7 +939,8 @@ export TF_LOG=debug echo "8<--------8<--------8<--------8<-------- BEGIN: create cluster 8<--------8<--------8<--------8<--------" echo "DATE=$(date --utc '+%Y-%m-%dT%H:%M:%S%:z')" # Run openshift-install in background with process substitution to allow trap to execute on SIGTERM -openshift-install --dir="${dir}" create cluster 2>&1 > >(grep --line-buffered -v 'password\|X-Auth-Token\|UserData:') & +# Note: Redirect order matters - stdout first, then stderr to stdout, so both go through grep +openshift-install --dir="${dir}" create cluster > >(grep --line-buffered -v 'password\|X-Auth-Token\|UserData:') 2>&1 & INSTALL_PID=$! wait $INSTALL_PID ret=$? @@ -970,7 +972,8 @@ if ! ${SKIP_WAIT_FOR}; then echo "8<--------8<--------8<--------8<-------- BEGIN: wait-for install-complete 8<--------8<--------8<--------8<--------" echo "DATE=$(date --utc '+%Y-%m-%dT%H:%M:%S%:z')" # Run openshift-install in background with process substitution to allow trap to execute on SIGTERM - openshift-install wait-for install-complete --dir="${dir}" 2>&1 > >(grep --line-buffered -v 'password\|X-Auth-Token\|UserData:') & + # Note: Redirect order matters - stdout first, then stderr to stdout, so both go through grep + openshift-install wait-for install-complete --dir="${dir}" > >(grep --line-buffered -v 'password\|X-Auth-Token\|UserData:') 2>&1 & INSTALL_PID=$! wait $INSTALL_PID ret=$? From 2bb8be2fbbf2f7c8abf80e07d2cce4522a04e83f Mon Sep 17 00:00:00 2001 From: Neha Yadav Date: Wed, 24 Jun 2026 13:56:37 +0530 Subject: [PATCH 3/4] TEMPORARY: Add 30m timeout and 15m grace period for TERM trap testing --- .../powervs/install/ipi-install-powervs-install-ref.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-ref.yaml b/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-ref.yaml index 7024546ece84b..5b539fe81a156 100644 --- a/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-ref.yaml +++ b/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-ref.yaml @@ -1,7 +1,8 @@ ref: as: ipi-install-powervs-install from: upi-installer - grace_period: 2h + timeout: 30m0s # Temporary: reduced from default for TERM trap testing + grace_period: 15m0s # Temporary: reduced from 2h for TERM trap testing commands: ipi-install-powervs-install-commands.sh cli: latest resources: From c33121f3fbbc8e09bb11c65842cd841b6bb4001e Mon Sep 17 00:00:00 2001 From: Neha Yadav Date: Thu, 25 Jun 2026 11:37:00 +0530 Subject: [PATCH 4/4] Revert "TEMPORARY: Add 30m timeout and 15m grace period for TERM trap testing" This reverts commit 2bb8be2fbbf2f7c8abf80e07d2cce4522a04e83f. --- .../powervs/install/ipi-install-powervs-install-ref.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-ref.yaml b/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-ref.yaml index 5b539fe81a156..7024546ece84b 100644 --- a/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-ref.yaml +++ b/ci-operator/step-registry/ipi/install/powervs/install/ipi-install-powervs-install-ref.yaml @@ -1,8 +1,7 @@ ref: as: ipi-install-powervs-install from: upi-installer - timeout: 30m0s # Temporary: reduced from default for TERM trap testing - grace_period: 15m0s # Temporary: reduced from 2h for TERM trap testing + grace_period: 2h commands: ipi-install-powervs-install-commands.sh cli: latest resources: