diff --git a/.github/actions/job_init/action.yml b/.github/actions/job_init/action.yml index ea44331507dcf..eb9bb427e45e1 100644 --- a/.github/actions/job_init/action.yml +++ b/.github/actions/job_init/action.yml @@ -35,7 +35,7 @@ inputs: required: false default: "" free_disk_space: - description: "Runs tools/azure-pipelines/free_disk_space.sh to remove unneeded pre-installed software from the runner (only works for jobs that run directly on the host, not in a container)." + description: "Runs tools/ci/free_disk_space.sh to remove unneeded pre-installed software from the runner (only works for jobs that run directly on the host, not in a container)." required: false default: "false" outputs: @@ -62,7 +62,7 @@ runs: - name: "Free up disk space" if: ${{ inputs.free_disk_space == 'true' }} shell: bash - run: ./tools/azure-pipelines/free_disk_space.sh + run: ./tools/ci/free_disk_space.sh - name: "Delete unused binaries" if: ${{ inputs.free_disk_space != 'true' }} diff --git a/.github/workflows/template.flink-ci.yml b/.github/workflows/template.flink-ci.yml index 38fde30363367..02e5ae33d3587 100644 --- a/.github/workflows/template.flink-ci.yml +++ b/.github/workflows/template.flink-ci.yml @@ -64,7 +64,7 @@ env: MAVEN_REPO_FOLDER: /root/.m2/repository MAVEN_ARGS: -Dmaven.repo.local=/root/.m2/repository - # required by tools/azure-pipelines/cache_docker_images.sh + # required by tools/ci/cache_docker_images.sh DOCKER_IMAGES_CACHE_FOLDER: /root/.docker-cache jobs: @@ -113,7 +113,7 @@ jobs: - name: "Collect build artifacts" working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} run: | - ./tools/azure-pipelines/create_build_artifact.sh + ./tools/ci/create_build_artifact.sh - name: "Upload artifacts to make them available in downstream jobs" uses: actions/upload-artifact@v7 @@ -172,7 +172,7 @@ jobs: - name: "Unpack build artifact" working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} run: | - ./tools/azure-pipelines/unpack_build_artifact.sh + ./tools/ci/unpack_build_artifact.sh - name: "Test" working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} @@ -261,7 +261,7 @@ jobs: - name: "Unpack build artifact" working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} run: | - ./tools/azure-pipelines/unpack_build_artifact.sh + ./tools/ci/unpack_build_artifact.sh - name: "Try loading Docker images from Cache" id: docker-cache @@ -274,7 +274,7 @@ jobs: - name: "Load Docker images if not present in cache, yet" if: ${{ !cancelled() && steps.docker-cache.outputs.cache-hit != 'true' }} working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} - run: ./tools/azure-pipelines/cache_docker_images.sh load + run: ./tools/ci/cache_docker_images.sh load - name: "Create non-root user for test execution" run: | @@ -304,7 +304,7 @@ jobs: DOCKER_HOST: unix:///var/run/docker.sock timeout-minutes: ${{ fromJSON(env.GHA_JOB_TIMEOUT) }} run: | - runuser -u flink -- bash -c '${{ inputs.environment }} PROFILE="$PROFILE -Pgithub-actions" ./tools/azure-pipelines/uploading_watchdog.sh \ + runuser -u flink -- bash -c '${{ inputs.environment }} PROFILE="$PROFILE -Pgithub-actions" ./tools/ci/uploading_watchdog.sh \ ./tools/ci/test_controller.sh ${{ matrix.module }}' - name: "Post-build Disk Info" @@ -332,7 +332,7 @@ jobs: - name: "Save Docker images to cache" working-directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }} if: ${{ !cancelled() && (failure() || steps.docker-cache.outputs.cache-hit != 'true') }} - run: ./tools/azure-pipelines/cache_docker_images.sh save + run: ./tools/ci/cache_docker_images.sh save - name: "Save Maven package cache" if: ${{ !cancelled() && steps.job-init.outputs.maven-cache-hit != 'true' }} @@ -412,7 +412,7 @@ jobs: path: ${{ env.FLINK_ARTIFACT_DIR }} - name: "Unpack build artifact" - run: ./tools/azure-pipelines/unpack_build_artifact.sh + run: ./tools/ci/unpack_build_artifact.sh # the cache task does not create directories on a cache miss, and can later fail when trying to tar the directory if the test haven't created it # this may for example happen if a given directory is only used by a subset of tests, which are run in a different 'group' @@ -444,7 +444,7 @@ jobs: - name: "Load Docker images if not present in Cache, yet" if: ${{ !cancelled() && steps.docker-cache.outputs.cache-hit != 'true' }} - run: ./tools/azure-pipelines/cache_docker_images.sh load + run: ./tools/ci/cache_docker_images.sh load - name: "Build Flink" uses: "./.github/actions/run_mvn" @@ -460,7 +460,7 @@ jobs: IT_CASE_S3_SECRET_KEY: ${{ secrets.s3_secret_key }} timeout-minutes: ${{ fromJSON(env.GHA_JOB_TIMEOUT) }} run: | - ${{ inputs.environment }} FLINK_DIR=`pwd`/build-target PROFILE="$PROFILE -Pgithub-actions" ./tools/azure-pipelines/uploading_watchdog.sh \ + ${{ inputs.environment }} FLINK_DIR=`pwd`/build-target PROFILE="$PROFILE -Pgithub-actions" ./tools/ci/uploading_watchdog.sh \ flink-end-to-end-tests/run-nightly-tests.sh ${{ matrix.group }} - name: "Upload Logs" @@ -472,7 +472,7 @@ jobs: - name: "Save Docker images to Cache" if: ${{ !cancelled() && (failure() || steps.docker-cache.outputs.cache-hit != 'true') }} - run: ./tools/azure-pipelines/cache_docker_images.sh save + run: ./tools/ci/cache_docker_images.sh save - name: "Save Maven package cache" if: ${{ !cancelled() && steps.job-init.outputs.maven-cache-hit != 'true' }} diff --git a/tools/azure-pipelines/e2e-template.yml b/tools/azure-pipelines/e2e-template.yml index 3b1229af1a069..8045525156c6c 100644 --- a/tools/azure-pipelines/e2e-template.yml +++ b/tools/azure-pipelines/e2e-template.yml @@ -45,7 +45,7 @@ jobs: fi displayName: Check if Docs only PR # free up disk space before running anything caching related. Caching has proven to fail in the past, due to lacking disk space. - - script: ./tools/azure-pipelines/free_disk_space.sh + - script: ./tools/ci/free_disk_space.sh target: host displayName: Free up disk space - task: UsePythonVersion@0 @@ -101,7 +101,7 @@ jobs: continueOnError: true condition: not(eq(variables['SKIP'], '1')) displayName: Cache docker images - - script: ./tools/azure-pipelines/cache_docker_images.sh load + - script: ./tools/ci/cache_docker_images.sh load displayName: Restore docker images condition: and(not(canceled()), eq(variables.DOCKER_IMAGES_CACHE_HIT, 'true')) continueOnError: true @@ -131,7 +131,7 @@ jobs: - script: ${{parameters.environment}} PROFILE="$PROFILE -Dfast -Pskip-webui-build" ./tools/ci/compile_ci.sh displayName: Build Flink condition: not(eq(variables['SKIP'], '1')) - - script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target ./tools/azure-pipelines/uploading_watchdog.sh flink-end-to-end-tests/run-nightly-tests.sh ${{parameters.group}} + - script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target ./tools/ci/uploading_watchdog.sh flink-end-to-end-tests/run-nightly-tests.sh ${{parameters.group}} displayName: Run e2e tests env: IT_CASE_S3_BUCKET: $(SECRET_S3_BUCKET) @@ -145,7 +145,7 @@ jobs: inputs: targetPath: $(DEBUG_FILES_OUTPUT_DIR) artifact: logs-${{parameters.stage_name}}-$(DEBUG_FILES_NAME) - - script: ./tools/azure-pipelines/cache_docker_images.sh save + - script: ./tools/ci/cache_docker_images.sh save displayName: Save docker images condition: and(not(eq(variables['SKIP'], '1')), ne(variables.DOCKER_IMAGES_CACHE_HIT, 'true')) continueOnError: true diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml index a9d97ba436533..d86b5210465ed 100644 --- a/tools/azure-pipelines/jobs-template.yml +++ b/tools/azure-pipelines/jobs-template.yml @@ -37,7 +37,7 @@ jobs: steps: # if on Azure, free up disk space - - script: ./tools/azure-pipelines/free_disk_space.sh + - script: ./tools/ci/free_disk_space.sh target: host condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default')) displayName: Free up disk space @@ -69,7 +69,7 @@ jobs: # Compile - script: | ${{parameters.environment}} ./tools/ci/compile_ci.sh || exit $? - ./tools/azure-pipelines/create_build_artifact.sh + ./tools/ci/create_build_artifact.sh displayName: Compile # upload artifacts for next stage @@ -103,7 +103,7 @@ jobs: module: misc steps: # if on Azure, free up disk space - - script: ./tools/azure-pipelines/free_disk_space.sh + - script: ./tools/ci/free_disk_space.sh target: host condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default')) displayName: Free up disk space @@ -121,7 +121,7 @@ jobs: path: $(FLINK_ARTIFACT_DIR) artifact: FlinkCompileArtifact-${{parameters.stage_name}} - - script: ./tools/azure-pipelines/unpack_build_artifact.sh + - script: ./tools/ci/unpack_build_artifact.sh displayName: "Unpack Build artifact" - task: Cache@2 @@ -142,7 +142,7 @@ jobs: condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default')) displayName: Cache docker images - - script: ./tools/azure-pipelines/cache_docker_images.sh load + - script: ./tools/ci/cache_docker_images.sh load displayName: Restore docker images condition: and(not(canceled()), eq(variables.DOCKER_IMAGES_CACHE_HIT, 'true')) continueOnError: true @@ -156,7 +156,7 @@ jobs: displayName: Set coredump pattern # Test - - script: ${{parameters.environment}} ./tools/azure-pipelines/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module) + - script: ${{parameters.environment}} ./tools/ci/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module) displayName: Test - $(module) env: IT_CASE_S3_BUCKET: $(SECRET_S3_BUCKET) @@ -176,7 +176,7 @@ jobs: targetPath: $(DEBUG_FILES_OUTPUT_DIR) artifact: logs-${{parameters.stage_name}}-$(DEBUG_FILES_NAME) - - script: ./tools/azure-pipelines/cache_docker_images.sh save + - script: ./tools/ci/cache_docker_images.sh save displayName: Save docker images condition: and(not(canceled()), or(failed(), ne(variables.DOCKER_IMAGES_CACHE_HIT, 'true'))) continueOnError: true diff --git a/tools/azure-pipelines/cache_docker_images.sh b/tools/ci/cache_docker_images.sh similarity index 100% rename from tools/azure-pipelines/cache_docker_images.sh rename to tools/ci/cache_docker_images.sh diff --git a/tools/azure-pipelines/create_build_artifact.sh b/tools/ci/create_build_artifact.sh similarity index 100% rename from tools/azure-pipelines/create_build_artifact.sh rename to tools/ci/create_build_artifact.sh diff --git a/tools/azure-pipelines/debug_files_utils.sh b/tools/ci/debug_files_utils.sh similarity index 100% rename from tools/azure-pipelines/debug_files_utils.sh rename to tools/ci/debug_files_utils.sh diff --git a/tools/azure-pipelines/free_disk_space.sh b/tools/ci/free_disk_space.sh similarity index 100% rename from tools/azure-pipelines/free_disk_space.sh rename to tools/ci/free_disk_space.sh diff --git a/tools/azure-pipelines/unpack_build_artifact.sh b/tools/ci/unpack_build_artifact.sh similarity index 100% rename from tools/azure-pipelines/unpack_build_artifact.sh rename to tools/ci/unpack_build_artifact.sh diff --git a/tools/azure-pipelines/uploading_watchdog.sh b/tools/ci/uploading_watchdog.sh similarity index 98% rename from tools/azure-pipelines/uploading_watchdog.sh rename to tools/ci/uploading_watchdog.sh index 5b5963782f425..cce732013eacc 100755 --- a/tools/azure-pipelines/uploading_watchdog.sh +++ b/tools/ci/uploading_watchdog.sh @@ -58,7 +58,7 @@ fi source "${HERE}/../ci/controller_utils.sh" -source ./tools/azure-pipelines/debug_files_utils.sh +source ./tools/ci/debug_files_utils.sh prepare_debug_files "${temporary_folder}" "${job_name}" export FLINK_LOG_DIR="$DEBUG_FILES_OUTPUT_DIR/flink-logs" mkdir $FLINK_LOG_DIR || { echo "FAILURE: cannot create log directory '${FLINK_LOG_DIR}'." ; exit 1; }