[FLINK-40224][ci] Move common scripts out of tools/azure-pipelines

.github/actions/job_init/action.yml

@@ -35,7 +35,7 @@ inputs:
     required: false
     default: ""
   free_disk_space:
-    description: "Runs tools/azure-pipelines/free_disk_space.sh to remove unneeded pre-installed software from the runner (only works for jobs that run directly on the host, not in a container)."
+    description: "Runs tools/ci/free_disk_space.sh to remove unneeded pre-installed software from the runner (only works for jobs that run directly on the host, not in a container)."
     required: false
     default: "false"
 outputs:
@@ -62,7 +62,7 @@ runs:
     - name: "Free up disk space"
       if: ${{ inputs.free_disk_space == 'true' }}
       shell: bash
-      run: ./tools/azure-pipelines/free_disk_space.sh
+      run: ./tools/ci/free_disk_space.sh
 
     - name: "Delete unused binaries"
       if: ${{ inputs.free_disk_space != 'true' }}

.github/workflows/template.flink-ci.yml

@@ -64,7 +64,7 @@ env:
 
   MAVEN_REPO_FOLDER: /root/.m2/repository
   MAVEN_ARGS: -Dmaven.repo.local=/root/.m2/repository
-  # required by tools/azure-pipelines/cache_docker_images.sh
+  # required by tools/ci/cache_docker_images.sh
   DOCKER_IMAGES_CACHE_FOLDER: /root/.docker-cache
 
 jobs:
@@ -113,7 +113,7 @@ jobs:
       - name: "Collect build artifacts"
         working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
         run: |
-          ./tools/azure-pipelines/create_build_artifact.sh
+          ./tools/ci/create_build_artifact.sh
 
       - name: "Upload artifacts to make them available in downstream jobs"
         uses: actions/upload-artifact@v7
@@ -172,7 +172,7 @@ jobs:
       - name: "Unpack build artifact"
         working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
         run: |
-          ./tools/azure-pipelines/unpack_build_artifact.sh
+          ./tools/ci/unpack_build_artifact.sh
 
       - name: "Test"
         working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
@@ -261,7 +261,7 @@ jobs:
       - name: "Unpack build artifact"
         working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
         run: |
-          ./tools/azure-pipelines/unpack_build_artifact.sh
+          ./tools/ci/unpack_build_artifact.sh
 
       - name: "Try loading Docker images from Cache"
         id: docker-cache
@@ -274,7 +274,7 @@ jobs:
       - name: "Load Docker images if not present in cache, yet"
         if: ${{ !cancelled() && steps.docker-cache.outputs.cache-hit != 'true' }}
         working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
-        run: ./tools/azure-pipelines/cache_docker_images.sh load
+        run: ./tools/ci/cache_docker_images.sh load
 
       - name: "Create non-root user for test execution"
         run: |
@@ -304,7 +304,7 @@ jobs:
           DOCKER_HOST: unix:///var/run/docker.sock
         timeout-minutes: ${{ fromJSON(env.GHA_JOB_TIMEOUT) }}
         run: |
-          runuser -u flink -- bash -c '${{ inputs.environment }} PROFILE="$PROFILE -Pgithub-actions" ./tools/azure-pipelines/uploading_watchdog.sh \
+          runuser -u flink -- bash -c '${{ inputs.environment }} PROFILE="$PROFILE -Pgithub-actions" ./tools/ci/uploading_watchdog.sh \
               ./tools/ci/test_controller.sh ${{ matrix.module }}'
 
       - name: "Post-build Disk Info"
@@ -332,7 +332,7 @@ jobs:
       - name: "Save Docker images to cache"
         working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
         if: ${{ !cancelled() && (failure() || steps.docker-cache.outputs.cache-hit != 'true') }}
-        run: ./tools/azure-pipelines/cache_docker_images.sh save
+        run: ./tools/ci/cache_docker_images.sh save
 
       - name: "Save Maven package cache"
         if: ${{ !cancelled() && steps.job-init.outputs.maven-cache-hit != 'true' }}
@@ -412,7 +412,7 @@ jobs:
           path: ${{ env.FLINK_ARTIFACT_DIR }}
 
       - name: "Unpack build artifact"
-        run: ./tools/azure-pipelines/unpack_build_artifact.sh
+        run: ./tools/ci/unpack_build_artifact.sh
 
       # the cache task does not create directories on a cache miss, and can later fail when trying to tar the directory if the test haven't created it
       # this may for example happen if a given directory is only used by a subset of tests, which are run in a different 'group'
@@ -444,7 +444,7 @@ jobs:
 
       - name: "Load Docker images if not present in Cache, yet"
         if: ${{ !cancelled() && steps.docker-cache.outputs.cache-hit != 'true' }}
-        run: ./tools/azure-pipelines/cache_docker_images.sh load
+        run: ./tools/ci/cache_docker_images.sh load
 
       - name: "Build Flink"
         uses: "./.github/actions/run_mvn"
@@ -460,7 +460,7 @@ jobs:
           IT_CASE_S3_SECRET_KEY: ${{ secrets.s3_secret_key }}
         timeout-minutes: ${{ fromJSON(env.GHA_JOB_TIMEOUT) }}
         run: |
-          ${{ inputs.environment }} FLINK_DIR=`pwd`/build-target PROFILE="$PROFILE -Pgithub-actions" ./tools/azure-pipelines/uploading_watchdog.sh \
+          ${{ inputs.environment }} FLINK_DIR=`pwd`/build-target PROFILE="$PROFILE -Pgithub-actions" ./tools/ci/uploading_watchdog.sh \
             flink-end-to-end-tests/run-nightly-tests.sh ${{ matrix.group }}
 
       - name: "Upload Logs"
@@ -472,7 +472,7 @@ jobs:
 
       - name: "Save Docker images to Cache"
         if: ${{ !cancelled() && (failure() || steps.docker-cache.outputs.cache-hit != 'true') }}
-        run: ./tools/azure-pipelines/cache_docker_images.sh save
+        run: ./tools/ci/cache_docker_images.sh save
 
       - name: "Save Maven package cache"
         if: ${{ !cancelled() && steps.job-init.outputs.maven-cache-hit != 'true' }}

tools/azure-pipelines/e2e-template.yml

@@ -45,7 +45,7 @@ jobs:
         fi
       displayName: Check if Docs only PR
     # free up disk space before running anything caching related. Caching has proven to fail in the past, due to lacking disk space.
-    - script: ./tools/azure-pipelines/free_disk_space.sh
+    - script: ./tools/ci/free_disk_space.sh
       target: host
       displayName: Free up disk space
     - task: UsePythonVersion@0
@@ -101,7 +101,7 @@ jobs:
       continueOnError: true
       condition: not(eq(variables['SKIP'], '1'))
       displayName: Cache docker images
-    - script: ./tools/azure-pipelines/cache_docker_images.sh load
+    - script: ./tools/ci/cache_docker_images.sh load
       displayName: Restore docker images
       condition: and(not(canceled()), eq(variables.DOCKER_IMAGES_CACHE_HIT, 'true'))
       continueOnError: true
@@ -131,7 +131,7 @@ jobs:
     - script: ${{parameters.environment}} PROFILE="$PROFILE -Dfast -Pskip-webui-build" ./tools/ci/compile_ci.sh
       displayName: Build Flink
       condition: not(eq(variables['SKIP'], '1'))
-    - script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target ./tools/azure-pipelines/uploading_watchdog.sh flink-end-to-end-tests/run-nightly-tests.sh ${{parameters.group}}
+    - script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target ./tools/ci/uploading_watchdog.sh flink-end-to-end-tests/run-nightly-tests.sh ${{parameters.group}}
       displayName: Run e2e tests
       env:
         IT_CASE_S3_BUCKET: $(SECRET_S3_BUCKET)
@@ -145,7 +145,7 @@ jobs:
       inputs:
         targetPath: $(DEBUG_FILES_OUTPUT_DIR)
         artifact: logs-${{parameters.stage_name}}-$(DEBUG_FILES_NAME)
-    - script: ./tools/azure-pipelines/cache_docker_images.sh save
+    - script: ./tools/ci/cache_docker_images.sh save
       displayName: Save docker images
       condition: and(not(eq(variables['SKIP'], '1')), ne(variables.DOCKER_IMAGES_CACHE_HIT, 'true'))
       continueOnError: true

tools/azure-pipelines/jobs-template.yml

@@ -37,7 +37,7 @@ jobs:
 
   steps:
   # if on Azure, free up disk space
-  - script: ./tools/azure-pipelines/free_disk_space.sh
+  - script: ./tools/ci/free_disk_space.sh
     target: host
     condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
     displayName: Free up disk space
@@ -69,7 +69,7 @@ jobs:
   # Compile
   - script: |
       ${{parameters.environment}} ./tools/ci/compile_ci.sh || exit $?
-      ./tools/azure-pipelines/create_build_artifact.sh
+      ./tools/ci/create_build_artifact.sh
     displayName: Compile
 
   # upload artifacts for next stage
@@ -103,7 +103,7 @@ jobs:
         module: misc
   steps:
   # if on Azure, free up disk space
-  - script: ./tools/azure-pipelines/free_disk_space.sh
+  - script: ./tools/ci/free_disk_space.sh
     target: host
     condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
     displayName: Free up disk space
@@ -121,7 +121,7 @@ jobs:
       path: $(FLINK_ARTIFACT_DIR)
       artifact: FlinkCompileArtifact-${{parameters.stage_name}}
 
-  - script: ./tools/azure-pipelines/unpack_build_artifact.sh
+  - script: ./tools/ci/unpack_build_artifact.sh
     displayName: "Unpack Build artifact"
 
   - task: Cache@2
@@ -142,7 +142,7 @@ jobs:
     condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
     displayName: Cache docker images
 
-  - script: ./tools/azure-pipelines/cache_docker_images.sh load
+  - script: ./tools/ci/cache_docker_images.sh load
     displayName: Restore docker images
     condition: and(not(canceled()), eq(variables.DOCKER_IMAGES_CACHE_HIT, 'true'))
     continueOnError: true
@@ -156,7 +156,7 @@ jobs:
     displayName: Set coredump pattern
 
   # Test
-  - script: ${{parameters.environment}} ./tools/azure-pipelines/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module)
+  - script: ${{parameters.environment}} ./tools/ci/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module)
     displayName: Test - $(module)
     env:
       IT_CASE_S3_BUCKET: $(SECRET_S3_BUCKET)
@@ -176,7 +176,7 @@ jobs:
       targetPath: $(DEBUG_FILES_OUTPUT_DIR)
       artifact: logs-${{parameters.stage_name}}-$(DEBUG_FILES_NAME)
 
-  - script: ./tools/azure-pipelines/cache_docker_images.sh save
+  - script: ./tools/ci/cache_docker_images.sh save
     displayName: Save docker images
     condition: and(not(canceled()), or(failed(), ne(variables.DOCKER_IMAGES_CACHE_HIT, 'true')))
     continueOnError: true

tools/azure-pipelines/uploading_watchdog.sh → tools/ci/uploading_watchdog.sh

@@ -58,7 +58,7 @@ fi
 
 source "${HERE}/../ci/controller_utils.sh"
 
-source ./tools/azure-pipelines/debug_files_utils.sh
+source ./tools/ci/debug_files_utils.sh
 prepare_debug_files "${temporary_folder}" "${job_name}"
 export FLINK_LOG_DIR="$DEBUG_FILES_OUTPUT_DIR/flink-logs"
 mkdir $FLINK_LOG_DIR || { echo "FAILURE: cannot create log directory '${FLINK_LOG_DIR}'." ; exit 1; }