From 9499e2ba361a78adb462c9c180003b6f7bbdb8de Mon Sep 17 00:00:00 2001 From: Toby Dylan Hocking Date: Fri, 18 Sep 2026 09:09:36 -0400 Subject: [PATCH 1/8] new atime ci, no more old cml container --- .github/workflows/performance-tests.yml | 127 ++++++++++++++++++++++-- 1 file changed, 121 insertions(+), 6 deletions(-) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index 0c42513e8c..56680ac20e 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -14,11 +14,126 @@ on: jobs: comment: - if: github.repository == 'Rdatatable/data.table' runs-on: ubuntu-latest - container: ghcr.io/iterative/cml:0-dvc2-base1 - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - repo_token: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: Anirban166/Autocomment-atime-results@v1.4.3 \ No newline at end of file + - name: Start timer + run: echo "START_SETUP_TIME=$(date +%s)" >> $GITHUB_ENV + shell: bash + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Retrieve branch references, necessary for perf testing past versions + shell: bash + run: | + git switch "${GITHUB_BASE_REF}" + git switch "${GITHUB_HEAD_REF}" + - uses: actions/setup-node@v6 + name: setup-node https://github.com/actions/setup-node?tab=readme-ov-file#usage + - name: Safe directory exception + run: | + git config --global --add safe.directory '*' + shell: bash + - name: Setup + uses: eddelbuettel/github-actions/r-ci@master + - name: Dependencies + run: ./run.sh install_all + - name: Install and run atime@main + run: | + echo "options(repos = c(CRAN = 'https://cloud.r-project.org'))" >> .Rprofile + Rscript -e 'install.packages(c("atime", "ggplot2", "directlabels"))' + echo "END_SETUP_START_TESTS_TIME=$(date +%s)" >> $GITHUB_ENV + Rscript -e 'atime::atime_pkg(Sys.getenv("GITHUB_WORKSPACE"), tests.dir = ".ci");' + echo "END_TESTS_TIME=$(date +%s)" >> $GITHUB_ENV + shell: bash + - name: Upload artifact + id: artifact-upload-step + uses: actions/upload-artifact@v4 + with: + name: atime-results + path: .ci/atime/ + - name: comments.json + if: ${{ github.event_name == 'pull_request' }} + env: + PR_NUMBER: ${{ github.event.number }} + TAG: atime + shell: bash + run: | + curl --no-progress-meter -L -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2026-03-10" https://api.github.com/repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments > comments.json + cat comments.json + echo > comment-number.txt + echo "" > comment.md + cat comment.md + - name: comment-number.txt + if: ${{ github.event_name == 'pull_request' }} + shell: node {0} + run: | + fs = require('fs'); + tag = fs.readFileSync('comment.md', 'utf-8'); + console.log(tag); + content = fs.readFileSync('comments.json', 'utf-8').replaceAll("\r", ""); + console.log(content); + prc = JSON.parse(content); + flist = prc.filter((o) => { return o.body.startsWith(tag); }); + console.log(flist); + if(flist.length == 0) process.exit(0); + comment_id = flist[0].id; + console.log(comment_id); + fs.writeFileSync('comment-number.txt', comment_id+""); + - name: add images + shell: bash + run: | + uploadImage() { + local png=$1 + curl -F "file=@$png" -F "visibility=unlisted" https://picrd.com/api/upload | sed 's/.*image_url":"//' | sed 's/",".*//' + } + cat comment-number.txt + PREVIEW_URL=$(uploadImage .ci/atime/tests_preview_facet.png) + ALL_URL=$(uploadImage .ci/atime/tests_all_facet.png) + echo $PREVIEW_URL $ALL_URL + cat .ci/atime/HEAD_issues.md >> comment.md + echo -e "\n[![Comparison Plot]($PREVIEW_URL)]($ALL_URL)" >> comment.md + echo -e "\nGenerated via commit ${{ github.event.pull_request.head.sha }}" >> comment.md + echo -e "\nDownload link for the artifact containing the test results: [↓ atime-results.zip](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/${{ steps.artifact-upload-step.outputs.artifact-id }})" >> comment.md + - name: add content to comment.md + run: | + R_SETUP_DURATION=$((END_SETUP_START_TESTS_TIME - START_SETUP_TIME)) + VERSION_SETUP_DURATION=$(cut -d '.' -f 1 ./.ci/atime/install_seconds.txt) + ATIME_TESTS_DURATION=$((END_TESTS_TIME - END_SETUP_START_TESTS_TIME - VERSION_SETUP_DURATION)) + formatTime() { + local totalSeconds=$1 + if [ $totalSeconds -lt 0 ]; then + echo "Time calculation error: negative duration" + return + fi + if [ $totalSeconds -lt 60 ]; then + echo "${totalSeconds} seconds" + else + local minutes=$((totalSeconds / 60)) + local seconds=$((totalSeconds % 60)) + echo "${minutes} minutes and ${seconds} seconds" + fi + } + echo -e "\n| Task | Duration |" >> comment.md + echo -e "|---|---|" >> comment.md + echo -e "| R setup and installing dependencies | $(formatTime $R_SETUP_DURATION) |" >> comment.md + echo -e "| Installing different package versions | $(formatTime $VERSION_SETUP_DURATION) |" >> comment.md + echo -e "| Running and plotting the test cases | $(formatTime $ATIME_TESTS_DURATION) |" >> comment.md + - name: convert comment to json https://stackoverflow.com/questions/1251999/how-can-i-replace-each-newline-n-with-a-space-using-sed/1252191#1252191 then post + if: ${{ github.event_name == 'pull_request' }} + env: + PR_COMMENT_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.number }} + shell: bash + run: | + ISSUES_API=https://api.github.com/repos/$GITHUB_REPOSITORY/issues + PR_API=$ISSUES_API/$PR_NUMBER/comments + VERSION=2026-03-10 + ACCEPT=application/vnd.github+json + COMMENT_NUMBER=$(cat comment-number.txt) + if [ "$COMMENT_NUMBER" == "" ]; then CMD=$PR_API; else CMD=$ISSUES_API/comments/$COMMENT_NUMBER; fi; echo $CMD + cat comment.md + MD=$(sed 's/"/\\"/g' comment.md | sed ':a;N;$!ba;s/\n/\\n/g') + echo '{"body":"'$MD'"}' > pr-comment.json + cat pr-comment.json + curl --no-progress-meter -L -H "Accept: $ACCEPT" -H "X-GitHub-Api-Version: $VERSION" -X POST -H "Authorization: Bearer $PR_COMMENT_TOKEN" $CMD -d "@pr-comment.json" From cbeb58bdb19e620898de5973cec325831e824496 Mon Sep 17 00:00:00 2001 From: Toby Dylan Hocking Date: Fri, 18 Sep 2026 09:15:28 -0400 Subject: [PATCH 2/8] run ci if yml config modified --- .github/workflows/performance-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index 56680ac20e..c3c7644dad 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -10,6 +10,7 @@ on: - 'R/**' - 'src/**' - '.ci/atime/**' + - '.github/workflows/performance-tests.yml' workflow_dispatch: jobs: From 4b52f282fdc92cdba3bbd11e4c117e89085ae2a8 Mon Sep 17 00:00:00 2001 From: Toby Dylan Hocking Date: Fri, 18 Sep 2026 09:18:48 -0400 Subject: [PATCH 3/8] rm workflow_dispatch which did not work anyway --- .github/workflows/performance-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index c3c7644dad..ff08fd67de 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -11,7 +11,6 @@ on: - 'src/**' - '.ci/atime/**' - '.github/workflows/performance-tests.yml' - workflow_dispatch: jobs: comment: From c19f1c2a953cfcac0c76c5e0d688814ce080ff7b Mon Sep 17 00:00:00 2001 From: Toby Dylan Hocking Date: Fri, 18 Sep 2026 09:21:40 -0400 Subject: [PATCH 4/8] rm @main --- .github/workflows/performance-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index ff08fd67de..dbdaca986e 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -38,7 +38,7 @@ jobs: uses: eddelbuettel/github-actions/r-ci@master - name: Dependencies run: ./run.sh install_all - - name: Install and run atime@main + - name: Install and run atime run: | echo "options(repos = c(CRAN = 'https://cloud.r-project.org'))" >> .Rprofile Rscript -e 'install.packages(c("atime", "ggplot2", "directlabels"))' From 6e34882ff27dc8eff2aa373160e421bf95b0a95f Mon Sep 17 00:00:00 2001 From: Ivan K Date: Sat, 19 Sep 2026 00:12:26 +0300 Subject: [PATCH 5/8] Compile without -Werror=format-security --- .github/workflows/performance-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index dbdaca986e..7c76af6981 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -38,6 +38,8 @@ jobs: uses: eddelbuettel/github-actions/r-ci@master - name: Dependencies run: ./run.sh install_all + - name: Adjust compilation flags for old data.table versions + run: sudo sed -i 's|-Werror=format-security||' `Rscript -e 'writeLines(R.home("etc"))'`/Makeconf - name: Install and run atime run: | echo "options(repos = c(CRAN = 'https://cloud.r-project.org'))" >> .Rprofile From 85df0c65311aa144204cc53708a3b059ce24e2be Mon Sep 17 00:00:00 2001 From: Ivan K Date: Sat, 19 Sep 2026 00:14:11 +0300 Subject: [PATCH 6/8] Allow manual dispatch --- .github/workflows/performance-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index 7c76af6981..fb164b0178 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -1,6 +1,7 @@ name: atime performance tests on: + workflow_dispatch: pull_request: types: - opened From b2d3a5e41f965274cf5c4c03fbfb77505e64f036 Mon Sep 17 00:00:00 2001 From: Ivan K Date: Wed, 23 Sep 2026 18:28:31 +0300 Subject: [PATCH 7/8] Revert "Allow manual dispatch" This reverts commit e1fa82b0b5dfadde4a2e0686cea6424ec0005c03. --- .github/workflows/performance-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index fb164b0178..7c76af6981 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -1,7 +1,6 @@ name: atime performance tests on: - workflow_dispatch: pull_request: types: - opened From 49a22ca5dab0a4699214572cfeee904fe000a90d Mon Sep 17 00:00:00 2001 From: Michael Chirico Date: Fri, 16 Oct 2026 08:04:36 -0700 Subject: [PATCH 8/8] Simplify atime GHA workflow using r-lib/actions, gh, and jq (#7921) --- .github/CONTRIBUTING.md | 2 +- .github/workflows/performance-tests.yml | 219 ++++++++++-------------- 2 files changed, 96 insertions(+), 125 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 021c674d08..a716b2f648 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -82,7 +82,7 @@ See [`?test`](https://rdatatable.gitlab.io/data.table/reference/test.html). #### Performance testing -If your PR may have an effect on time/memory usage, please consider adding a performance test, either in the same PR, or a follow-up PR. Note that first-time contributors _must_ do so in a follow-up PR, since the tests are only run on PRs from branches created directly in the Rdatatable/data.table repo. See the [Performance testing](https://github.com/Rdatatable/data.table/wiki/Performance-testing) wiki page for details. +If your PR may have an effect on time/memory usage, please consider adding a performance test, either in the same PR, or a follow-up PR. On PRs opened from a fork, the `atime` results and comparison plots will be published to the GitHub Actions job summary (and downloadable artifact) rather than posted as a PR comment. See the [Performance testing](https://github.com/Rdatatable/data.table/wiki/Performance-testing) wiki page for details. Minimal first time PR --------------------- diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index 7c76af6981..8b3204f6fc 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -2,140 +2,111 @@ name: atime performance tests on: pull_request: - types: - - opened - - reopened - - synchronize paths: - 'R/**' - 'src/**' - '.ci/atime/**' - '.github/workflows/performance-tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + jobs: comment: runs-on: ubuntu-latest steps: - - name: Start timer - run: echo "START_SETUP_TIME=$(date +%s)" >> $GITHUB_ENV - shell: bash - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - name: Retrieve branch references, necessary for perf testing past versions - shell: bash - run: | - git switch "${GITHUB_BASE_REF}" - git switch "${GITHUB_HEAD_REF}" - - uses: actions/setup-node@v6 - name: setup-node https://github.com/actions/setup-node?tab=readme-ov-file#usage - - name: Safe directory exception - run: | - git config --global --add safe.directory '*' - shell: bash - - name: Setup - uses: eddelbuettel/github-actions/r-ci@master - - name: Dependencies - run: ./run.sh install_all - - name: Adjust compilation flags for old data.table versions - run: sudo sed -i 's|-Werror=format-security||' `Rscript -e 'writeLines(R.home("etc"))'`/Makeconf - - name: Install and run atime - run: | - echo "options(repos = c(CRAN = 'https://cloud.r-project.org'))" >> .Rprofile - Rscript -e 'install.packages(c("atime", "ggplot2", "directlabels"))' - echo "END_SETUP_START_TESTS_TIME=$(date +%s)" >> $GITHUB_ENV - Rscript -e 'atime::atime_pkg(Sys.getenv("GITHUB_WORKSPACE"), tests.dir = ".ci");' - echo "END_TESTS_TIME=$(date +%s)" >> $GITHUB_ENV - shell: bash - - name: Upload artifact - id: artifact-upload-step - uses: actions/upload-artifact@v4 - with: - name: atime-results - path: .ci/atime/ - - name: comments.json - if: ${{ github.event_name == 'pull_request' }} - env: - PR_NUMBER: ${{ github.event.number }} - TAG: atime - shell: bash - run: | - curl --no-progress-meter -L -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2026-03-10" https://api.github.com/repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments > comments.json - cat comments.json - echo > comment-number.txt - echo "" > comment.md - cat comment.md - - name: comment-number.txt - if: ${{ github.event_name == 'pull_request' }} - shell: node {0} - run: | - fs = require('fs'); - tag = fs.readFileSync('comment.md', 'utf-8'); - console.log(tag); - content = fs.readFileSync('comments.json', 'utf-8').replaceAll("\r", ""); - console.log(content); - prc = JSON.parse(content); - flist = prc.filter((o) => { return o.body.startsWith(tag); }); - console.log(flist); - if(flist.length == 0) process.exit(0); - comment_id = flist[0].id; - console.log(comment_id); - fs.writeFileSync('comment-number.txt', comment_id+""); - - name: add images - shell: bash - run: | - uploadImage() { - local png=$1 - curl -F "file=@$png" -F "visibility=unlisted" https://picrd.com/api/upload | sed 's/.*image_url":"//' | sed 's/",".*//' - } - cat comment-number.txt - PREVIEW_URL=$(uploadImage .ci/atime/tests_preview_facet.png) - ALL_URL=$(uploadImage .ci/atime/tests_all_facet.png) - echo $PREVIEW_URL $ALL_URL - cat .ci/atime/HEAD_issues.md >> comment.md - echo -e "\n[![Comparison Plot]($PREVIEW_URL)]($ALL_URL)" >> comment.md - echo -e "\nGenerated via commit ${{ github.event.pull_request.head.sha }}" >> comment.md - echo -e "\nDownload link for the artifact containing the test results: [↓ atime-results.zip](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/${{ steps.artifact-upload-step.outputs.artifact-id }})" >> comment.md - - name: add content to comment.md - run: | - R_SETUP_DURATION=$((END_SETUP_START_TESTS_TIME - START_SETUP_TIME)) - VERSION_SETUP_DURATION=$(cut -d '.' -f 1 ./.ci/atime/install_seconds.txt) - ATIME_TESTS_DURATION=$((END_TESTS_TIME - END_SETUP_START_TESTS_TIME - VERSION_SETUP_DURATION)) - formatTime() { + - name: Start timer + shell: bash + run: echo "START_SETUP_TIME=$(date +%s)" >> "$GITHUB_ENV" + + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Retrieve branch references, necessary for perf testing past versions + shell: bash + run: | + git branch -f "${GITHUB_BASE_REF}" "origin/${GITHUB_BASE_REF}" + HEAD_BRANCH="${GITHUB_HEAD_REF}" + if [ "$HEAD_BRANCH" = "$GITHUB_BASE_REF" ]; then + HEAD_BRANCH="pr-${{ github.event.number }}" + fi + git switch -C "$HEAD_BRANCH" + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + cache-version: ${{ hashFiles('.ci/atime/tests.R') }} + extra-packages: cran::data.table, any::atime, any::ggplot2, any::directlabels + + - name: Run atime + shell: bash + run: | + echo "END_SETUP_START_TESTS_TIME=$(date +%s)" >> "$GITHUB_ENV" + Rscript -e 'atime::atime_pkg(Sys.getenv("GITHUB_WORKSPACE"), tests.dir = ".ci")' + echo "END_TESTS_TIME=$(date +%s)" >> "$GITHUB_ENV" + + - name: Upload artifact + id: artifact-upload-step + uses: actions/upload-artifact@v7 + with: + name: atime-results + path: .ci/atime/ + + - name: Post or update PR comment + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.number }} + shell: bash + run: | + uploadImage() { + curl --fail --retry 3 -sS -F "file=@$1" -F "visibility=unlisted" https://picrd.com/api/upload | jq -r '.image_url' + } + PREVIEW_URL=$(uploadImage .ci/atime/tests_preview_facet.png) + ALL_URL=$(uploadImage .ci/atime/tests_all_facet.png) + + R_SETUP_DURATION=$((END_SETUP_START_TESTS_TIME - START_SETUP_TIME)) + VERSION_SETUP_DURATION=$(cut -d '.' -f 1 ./.ci/atime/install_seconds.txt) + ATIME_TESTS_DURATION=$((END_TESTS_TIME - END_SETUP_START_TESTS_TIME - VERSION_SETUP_DURATION)) + formatTime() { local totalSeconds=$1 - if [ $totalSeconds -lt 0 ]; then - echo "Time calculation error: negative duration" - return - fi - if [ $totalSeconds -lt 60 ]; then - echo "${totalSeconds} seconds" + if [ "$totalSeconds" -lt 0 ]; then + echo "Time calculation error: negative duration" + elif [ "$totalSeconds" -lt 60 ]; then + echo "${totalSeconds} seconds" else - local minutes=$((totalSeconds / 60)) - local seconds=$((totalSeconds % 60)) - echo "${minutes} minutes and ${seconds} seconds" + echo "$((totalSeconds / 60)) minutes and $((totalSeconds % 60)) seconds" fi - } - echo -e "\n| Task | Duration |" >> comment.md - echo -e "|---|---|" >> comment.md - echo -e "| R setup and installing dependencies | $(formatTime $R_SETUP_DURATION) |" >> comment.md - echo -e "| Installing different package versions | $(formatTime $VERSION_SETUP_DURATION) |" >> comment.md - echo -e "| Running and plotting the test cases | $(formatTime $ATIME_TESTS_DURATION) |" >> comment.md - - name: convert comment to json https://stackoverflow.com/questions/1251999/how-can-i-replace-each-newline-n-with-a-space-using-sed/1252191#1252191 then post - if: ${{ github.event_name == 'pull_request' }} - env: - PR_COMMENT_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.number }} - shell: bash - run: | - ISSUES_API=https://api.github.com/repos/$GITHUB_REPOSITORY/issues - PR_API=$ISSUES_API/$PR_NUMBER/comments - VERSION=2026-03-10 - ACCEPT=application/vnd.github+json - COMMENT_NUMBER=$(cat comment-number.txt) - if [ "$COMMENT_NUMBER" == "" ]; then CMD=$PR_API; else CMD=$ISSUES_API/comments/$COMMENT_NUMBER; fi; echo $CMD - cat comment.md - MD=$(sed 's/"/\\"/g' comment.md | sed ':a;N;$!ba;s/\n/\\n/g') - echo '{"body":"'$MD'"}' > pr-comment.json - cat pr-comment.json - curl --no-progress-meter -L -H "Accept: $ACCEPT" -H "X-GitHub-Api-Version: $VERSION" -X POST -H "Authorization: Bearer $PR_COMMENT_TOKEN" $CMD -d "@pr-comment.json" + } + + ARTIFACT_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/${{ steps.artifact-upload-step.outputs.artifact-id }}" + cat < comment.md + $(cat .ci/atime/HEAD_issues.md) + + [![Comparison Plot]($PREVIEW_URL)]($ALL_URL) + + Generated via commit ${{ github.event.pull_request.head.sha }} + + Download link for the artifact containing the test results: [↓ atime-results.zip]($ARTIFACT_URL) + + | Task | Duration | + |---|---| + | R setup and installing dependencies | $(formatTime "$R_SETUP_DURATION") | + | Installing different package versions | $(formatTime "$VERSION_SETUP_DURATION") | + | Running and plotting the test cases | $(formatTime "$ATIME_TESTS_DURATION") | + EOF + + cat comment.md >> "$GITHUB_STEP_SUMMARY" + + if [ "${{ github.event.pull_request.head.repo.full_name }}" = "$GITHUB_REPOSITORY" ]; then + gh pr comment "$PR_NUMBER" --edit-last --create-if-none -F comment.md + fi