From 1bcc45a0d6fb7fe7296f2e7b584072240e10efd0 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 9 Aug 2026 17:02:45 -0400 Subject: [PATCH 1/2] Avoid NVHPC 25.5 fort2 ICE by expanding f_compute_multidim_cfl_terms NVHPC 25.5's fort2 segfaults (SIGSEGV) during the -Minline pass of our two-pass IPO when it cross-file inlines a routine whose body contains a call to f_compute_multidim_cfl_terms. It manifests as: nvfortran-Fatal-.../25.5/compilers/bin/tools/fort2 TERMINATED by signal 11 gmake[3]: *** [.../simulation.dir/fypp/simulation/m_data_output.fpp.f90.o] The crash is at the call site, not in the callee: it reproduces with the helper's body emptied and with its array dummy replaced by a scalar, and it survives except:f_compute_multidim_cfl_terms (excluding the callee leaves the offending call inside the inlined caller). Excluding the caller -- s_compute_stability_from_dt -- is what avoids it. Because the helper was private with exactly two call sites, expanding it in place removes the pattern entirely. The resulting if (p > 0) / else if (n > 0) / else structure matches the viscous and capillary blocks already present in both routines. Not architecture specific: originally reported on GH200/aarch64, and reproduced here on x86_64 with the stock NVHPC 25.5 tarball. NVHPC 25.11 compiles the unmodified source cleanly, so this is a 25.5 codegen bug; this change simply avoids the construct that triggers it. Verified: - NVHPC 25.5, GPU+MPI, case-optimized: fails before, builds after - NVHPC 25.11, same config: builds before and after (no regression) - cfl_adap_dt golden-file tests: 7/7 pass --- src/simulation/m_sim_helpers.fpp | 62 ++++++++++++++++---------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/src/simulation/m_sim_helpers.fpp b/src/simulation/m_sim_helpers.fpp index d0a59c4de..e50b4eed2 100644 --- a/src/simulation/m_sim_helpers.fpp +++ b/src/simulation/m_sim_helpers.fpp @@ -41,34 +41,6 @@ contains end function f_compute_filtered_dtheta - !> Computes inviscid CFL terms for multi-dimensional cases (2D/3D only) - function f_compute_multidim_cfl_terms(vel, c, j, k, l) result(cfl_terms) - - $:GPU_ROUTINE(parallelism='[seq]') - real(wp), dimension(num_vels), intent(in) :: vel - real(wp), intent(in) :: c - integer, intent(in) :: j, k, l - real(wp) :: cfl_terms - real(wp) :: fltr_dtheta - - fltr_dtheta = f_compute_filtered_dtheta(k, l) - - if (p > 0) then - ! 3D - #:if not MFC_CASE_OPTIMIZATION or num_dims > 2 - if (grid_geometry == 3) then - cfl_terms = min(dx(j)/(abs(vel(1)) + c), dy(k)/(abs(vel(2)) + c), fltr_dtheta/(abs(vel(3)) + c)) - else - cfl_terms = min(dx(j)/(abs(vel(1)) + c), dy(k)/(abs(vel(2)) + c), dz(l)/(abs(vel(3)) + c)) - end if - #:endif - else - ! 2D - cfl_terms = min(dx(j)/(abs(vel(1)) + c), dy(k)/(abs(vel(2)) + c)) - end if - - end function f_compute_multidim_cfl_terms - !> Computes enthalpy subroutine s_compute_enthalpy(q_prim_vf, pres, rho, gamma, pi_inf, Re, H, alpha, vel, vel_sum, qv, j, k, l) @@ -145,8 +117,21 @@ contains real(wp) :: fltr_dtheta ! Inviscid CFL calculation - if (p > 0 .or. n > 0) then - icfl = dt/f_compute_multidim_cfl_terms(vel, c, j, k, l) + ! The multi-dimensional CFL terms are written out here rather than + ! obtained from a shared helper procedure: NVHPC 25.5's fort2 segfaults + ! when a routine containing a call to that helper is cross-file inlined + ! by -Minline (the IPO setup in cmake/MFCTargets.cmake). + if (p > 0) then + #:if not MFC_CASE_OPTIMIZATION or num_dims > 2 + if (grid_geometry == 3) then + fltr_dtheta = f_compute_filtered_dtheta(k, l) + icfl = dt/min(dx(j)/(abs(vel(1)) + c), dy(k)/(abs(vel(2)) + c), fltr_dtheta/(abs(vel(3)) + c)) + else + icfl = dt/min(dx(j)/(abs(vel(1)) + c), dy(k)/(abs(vel(2)) + c), dz(l)/(abs(vel(3)) + c)) + end if + #:endif + else if (n > 0) then + icfl = dt/min(dx(j)/(abs(vel(1)) + c), dy(k)/(abs(vel(2)) + c)) else icfl = (dt/dx(j))*(abs(vel(1)) + c) end if @@ -206,8 +191,21 @@ contains real(wp) :: fltr_dtheta ! Inviscid CFL calculation - if (p > 0 .or. n > 0) then - max_dt = cfl_target*f_compute_multidim_cfl_terms(vel, c, j, k, l) + ! The multi-dimensional CFL terms are written out here rather than + ! obtained from a shared helper procedure: NVHPC 25.5's fort2 segfaults + ! when a routine containing a call to that helper is cross-file inlined + ! by -Minline (the IPO setup in cmake/MFCTargets.cmake). + if (p > 0) then + #:if not MFC_CASE_OPTIMIZATION or num_dims > 2 + if (grid_geometry == 3) then + fltr_dtheta = f_compute_filtered_dtheta(k, l) + max_dt = cfl_target*min(dx(j)/(abs(vel(1)) + c), dy(k)/(abs(vel(2)) + c), fltr_dtheta/(abs(vel(3)) + c)) + else + max_dt = cfl_target*min(dx(j)/(abs(vel(1)) + c), dy(k)/(abs(vel(2)) + c), dz(l)/(abs(vel(3)) + c)) + end if + #:endif + else if (n > 0) then + max_dt = cfl_target*min(dx(j)/(abs(vel(1)) + c), dy(k)/(abs(vel(2)) + c)) else max_dt = cfl_target*(dx(j)/(abs(vel(1)) + c)) end if From 04108333f2dbb721d5e5b7f3aca06d85a915abd5 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 9 Aug 2026 19:16:44 -0400 Subject: [PATCH 2/2] CI: add a case-optimized GPU build to the NVHPC version matrix The NVHPC 25.5 fort2 ICE fixed in the previous commit was invisible to CI, and would have stayed invisible. Two axes never intersect: - The NVHPC version matrix (23.11 -> 26.3) builds gpu targets without --case-optimization. An unmodified master builds cleanly on 25.5 in that configuration, which is why those jobs are green today. - The case-optimization jobs run only on self-hosted clusters: Phoenix, pinned to nvhpc/24.5, and Frontier, which is CCE/AMD. So "case-optimized x NVHPC 25.5" was never exercised. Case optimization hard-codes case parameters into the generated sources, which sharply increases cross-file inlining pressure in the -Mextract/-Minline pass -- precisely where NVHPC fort2 has repeatedly hit ICEs (see the -Mnoinline list in cmake/MFCTargets.cmake). This adds one extra build to the existing gpu matrix jobs. It is build-only and a single case, because the failure is at compile time and nothing needs to run. - 3D specifically: m_sim_helpers.fpp guards its 3D block with "#:if not MFC_CASE_OPTIMIZATION or num_dims > 2", so a case-optimized 2D build elides the code that crashed. - OpenACC only: the two-pass IPO is disabled for OpenMP offload (MFCTargets.cmake), so gpu-omp cannot hit this class of bug. Verified with a local NVHPC 25.5 install, running this exact command: - unmodified master: fort2 TERMINATED by signal 11 on m_data_output - with the previous commit: builds cleanly --- .github/workflows/test.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 00ece4de3..87163fe37 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -306,6 +306,30 @@ jobs: /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu mp ' + # Case-optimized build guard. + # + # --case-optimization hard-codes case parameters into the generated + # sources, which sharply increases cross-file inlining pressure in the + # -Mextract/-Minline IPO pass (cmake/MFCTargets.cmake). That pass is + # where NVHPC fort2 has historically hit internal compiler errors, and + # the jobs above cannot see them: they never pass --case-optimization, + # and the self-hosted case-opt jobs run a single pinned NVHPC. NVHPC + # 25.5 shipped an ICE that was invisible to CI for exactly that reason. + # + # One 3D case is enough: the failure is at compile time, so nothing is + # run. 3D specifically, because m_sim_helpers.fpp guards its 3D block + # with "#:if not MFC_CASE_OPTIMIZATION or num_dims > 2" -- a + # case-optimized 2D build elides that code entirely. OpenACC only, + # since the two-pass IPO is disabled for OpenMP offload. + - name: Build (NVHPC GPU, case-optimized) + if: matrix.nvhpc && matrix.target == 'gpu' + run: | + docker exec nvhpc bash -c ' + source /etc/nvhpc-env.sh + /bin/bash mfc.sh build -v -j 2 --gpu acc -t simulation \ + -i examples/3D_sphbubcollapse/case.py --case-optimization + ' + - name: Test (NVHPC) if: matrix.nvhpc && matrix.target == 'cpu' run: |