diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 021c674d08..a716b2f648 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -82,7 +82,7 @@ See [`?test`](https://rdatatable.gitlab.io/data.table/reference/test.html). #### Performance testing -If your PR may have an effect on time/memory usage, please consider adding a performance test, either in the same PR, or a follow-up PR. Note that first-time contributors _must_ do so in a follow-up PR, since the tests are only run on PRs from branches created directly in the Rdatatable/data.table repo. See the [Performance testing](https://github.com/Rdatatable/data.table/wiki/Performance-testing) wiki page for details. +If your PR may have an effect on time/memory usage, please consider adding a performance test, either in the same PR, or a follow-up PR. On PRs opened from a fork, the `atime` results and comparison plots will be published to the GitHub Actions job summary (and downloadable artifact) rather than posted as a PR comment. See the [Performance testing](https://github.com/Rdatatable/data.table/wiki/Performance-testing) wiki page for details. Minimal first time PR --------------------- diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index 7c76af6981..8b3204f6fc 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -2,140 +2,111 @@ name: atime performance tests on: pull_request: - types: - - opened - - reopened - - synchronize paths: - 'R/**' - 'src/**' - '.ci/atime/**' - '.github/workflows/performance-tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + jobs: comment: runs-on: ubuntu-latest steps: - - name: Start timer - run: echo "START_SETUP_TIME=$(date +%s)" >> $GITHUB_ENV - shell: bash - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - name: Retrieve branch references, necessary for perf testing past versions - shell: bash - run: | - git switch "${GITHUB_BASE_REF}" - git switch "${GITHUB_HEAD_REF}" - - uses: actions/setup-node@v6 - name: setup-node https://github.com/actions/setup-node?tab=readme-ov-file#usage - - name: Safe directory exception - run: | - git config --global --add safe.directory '*' - shell: bash - - name: Setup - uses: eddelbuettel/github-actions/r-ci@master - - name: Dependencies - run: ./run.sh install_all - - name: Adjust compilation flags for old data.table versions - run: sudo sed -i 's|-Werror=format-security||' `Rscript -e 'writeLines(R.home("etc"))'`/Makeconf - - name: Install and run atime - run: | - echo "options(repos = c(CRAN = 'https://cloud.r-project.org'))" >> .Rprofile - Rscript -e 'install.packages(c("atime", "ggplot2", "directlabels"))' - echo "END_SETUP_START_TESTS_TIME=$(date +%s)" >> $GITHUB_ENV - Rscript -e 'atime::atime_pkg(Sys.getenv("GITHUB_WORKSPACE"), tests.dir = ".ci");' - echo "END_TESTS_TIME=$(date +%s)" >> $GITHUB_ENV - shell: bash - - name: Upload artifact - id: artifact-upload-step - uses: actions/upload-artifact@v4 - with: - name: atime-results - path: .ci/atime/ - - name: comments.json - if: ${{ github.event_name == 'pull_request' }} - env: - PR_NUMBER: ${{ github.event.number }} - TAG: atime - shell: bash - run: | - curl --no-progress-meter -L -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2026-03-10" https://api.github.com/repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments > comments.json - cat comments.json - echo > comment-number.txt - echo "" > comment.md - cat comment.md - - name: comment-number.txt - if: ${{ github.event_name == 'pull_request' }} - shell: node {0} - run: | - fs = require('fs'); - tag = fs.readFileSync('comment.md', 'utf-8'); - console.log(tag); - content = fs.readFileSync('comments.json', 'utf-8').replaceAll("\r", ""); - console.log(content); - prc = JSON.parse(content); - flist = prc.filter((o) => { return o.body.startsWith(tag); }); - console.log(flist); - if(flist.length == 0) process.exit(0); - comment_id = flist[0].id; - console.log(comment_id); - fs.writeFileSync('comment-number.txt', comment_id+""); - - name: add images - shell: bash - run: | - uploadImage() { - local png=$1 - curl -F "file=@$png" -F "visibility=unlisted" https://picrd.com/api/upload | sed 's/.*image_url":"//' | sed 's/",".*//' - } - cat comment-number.txt - PREVIEW_URL=$(uploadImage .ci/atime/tests_preview_facet.png) - ALL_URL=$(uploadImage .ci/atime/tests_all_facet.png) - echo $PREVIEW_URL $ALL_URL - cat .ci/atime/HEAD_issues.md >> comment.md - echo -e "\n[![Comparison Plot]($PREVIEW_URL)]($ALL_URL)" >> comment.md - echo -e "\nGenerated via commit ${{ github.event.pull_request.head.sha }}" >> comment.md - echo -e "\nDownload link for the artifact containing the test results: [↓ atime-results.zip](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/${{ steps.artifact-upload-step.outputs.artifact-id }})" >> comment.md - - name: add content to comment.md - run: | - R_SETUP_DURATION=$((END_SETUP_START_TESTS_TIME - START_SETUP_TIME)) - VERSION_SETUP_DURATION=$(cut -d '.' -f 1 ./.ci/atime/install_seconds.txt) - ATIME_TESTS_DURATION=$((END_TESTS_TIME - END_SETUP_START_TESTS_TIME - VERSION_SETUP_DURATION)) - formatTime() { + - name: Start timer + shell: bash + run: echo "START_SETUP_TIME=$(date +%s)" >> "$GITHUB_ENV" + + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Retrieve branch references, necessary for perf testing past versions + shell: bash + run: | + git branch -f "${GITHUB_BASE_REF}" "origin/${GITHUB_BASE_REF}" + HEAD_BRANCH="${GITHUB_HEAD_REF}" + if [ "$HEAD_BRANCH" = "$GITHUB_BASE_REF" ]; then + HEAD_BRANCH="pr-${{ github.event.number }}" + fi + git switch -C "$HEAD_BRANCH" + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + cache-version: ${{ hashFiles('.ci/atime/tests.R') }} + extra-packages: cran::data.table, any::atime, any::ggplot2, any::directlabels + + - name: Run atime + shell: bash + run: | + echo "END_SETUP_START_TESTS_TIME=$(date +%s)" >> "$GITHUB_ENV" + Rscript -e 'atime::atime_pkg(Sys.getenv("GITHUB_WORKSPACE"), tests.dir = ".ci")' + echo "END_TESTS_TIME=$(date +%s)" >> "$GITHUB_ENV" + + - name: Upload artifact + id: artifact-upload-step + uses: actions/upload-artifact@v7 + with: + name: atime-results + path: .ci/atime/ + + - name: Post or update PR comment + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.number }} + shell: bash + run: | + uploadImage() { + curl --fail --retry 3 -sS -F "file=@$1" -F "visibility=unlisted" https://picrd.com/api/upload | jq -r '.image_url' + } + PREVIEW_URL=$(uploadImage .ci/atime/tests_preview_facet.png) + ALL_URL=$(uploadImage .ci/atime/tests_all_facet.png) + + R_SETUP_DURATION=$((END_SETUP_START_TESTS_TIME - START_SETUP_TIME)) + VERSION_SETUP_DURATION=$(cut -d '.' -f 1 ./.ci/atime/install_seconds.txt) + ATIME_TESTS_DURATION=$((END_TESTS_TIME - END_SETUP_START_TESTS_TIME - VERSION_SETUP_DURATION)) + formatTime() { local totalSeconds=$1 - if [ $totalSeconds -lt 0 ]; then - echo "Time calculation error: negative duration" - return - fi - if [ $totalSeconds -lt 60 ]; then - echo "${totalSeconds} seconds" + if [ "$totalSeconds" -lt 0 ]; then + echo "Time calculation error: negative duration" + elif [ "$totalSeconds" -lt 60 ]; then + echo "${totalSeconds} seconds" else - local minutes=$((totalSeconds / 60)) - local seconds=$((totalSeconds % 60)) - echo "${minutes} minutes and ${seconds} seconds" + echo "$((totalSeconds / 60)) minutes and $((totalSeconds % 60)) seconds" fi - } - echo -e "\n| Task | Duration |" >> comment.md - echo -e "|---|---|" >> comment.md - echo -e "| R setup and installing dependencies | $(formatTime $R_SETUP_DURATION) |" >> comment.md - echo -e "| Installing different package versions | $(formatTime $VERSION_SETUP_DURATION) |" >> comment.md - echo -e "| Running and plotting the test cases | $(formatTime $ATIME_TESTS_DURATION) |" >> comment.md - - name: convert comment to json https://stackoverflow.com/questions/1251999/how-can-i-replace-each-newline-n-with-a-space-using-sed/1252191#1252191 then post - if: ${{ github.event_name == 'pull_request' }} - env: - PR_COMMENT_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.number }} - shell: bash - run: | - ISSUES_API=https://api.github.com/repos/$GITHUB_REPOSITORY/issues - PR_API=$ISSUES_API/$PR_NUMBER/comments - VERSION=2026-03-10 - ACCEPT=application/vnd.github+json - COMMENT_NUMBER=$(cat comment-number.txt) - if [ "$COMMENT_NUMBER" == "" ]; then CMD=$PR_API; else CMD=$ISSUES_API/comments/$COMMENT_NUMBER; fi; echo $CMD - cat comment.md - MD=$(sed 's/"/\\"/g' comment.md | sed ':a;N;$!ba;s/\n/\\n/g') - echo '{"body":"'$MD'"}' > pr-comment.json - cat pr-comment.json - curl --no-progress-meter -L -H "Accept: $ACCEPT" -H "X-GitHub-Api-Version: $VERSION" -X POST -H "Authorization: Bearer $PR_COMMENT_TOKEN" $CMD -d "@pr-comment.json" + } + + ARTIFACT_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/${{ steps.artifact-upload-step.outputs.artifact-id }}" + cat < comment.md + $(cat .ci/atime/HEAD_issues.md) + + [![Comparison Plot]($PREVIEW_URL)]($ALL_URL) + + Generated via commit ${{ github.event.pull_request.head.sha }} + + Download link for the artifact containing the test results: [↓ atime-results.zip]($ARTIFACT_URL) + + | Task | Duration | + |---|---| + | R setup and installing dependencies | $(formatTime "$R_SETUP_DURATION") | + | Installing different package versions | $(formatTime "$VERSION_SETUP_DURATION") | + | Running and plotting the test cases | $(formatTime "$ATIME_TESTS_DURATION") | + EOF + + cat comment.md >> "$GITHUB_STEP_SUMMARY" + + if [ "${{ github.event.pull_request.head.repo.full_name }}" = "$GITHUB_REPOSITORY" ]; then + gh pr comment "$PR_NUMBER" --edit-last --create-if-none -F comment.md + fi