feat(ci): restore benchmark PR comments + add reliability/chaos reporting

.github/workflows/ci.yml

@@ -129,6 +129,7 @@ jobs:
     outputs:
       configurations: ${{ steps.compute.outputs.configurations }}
       run_fuzz: ${{ steps.compute.outputs.run_fuzz }}
+      run_reliability: ${{ steps.compute.outputs.run_reliability }}
     steps:
       - name: Debounce label events
         if: github.event.action == 'labeled'
@@ -161,8 +162,14 @@ jobs:
             else
               echo "run_fuzz=false" >> $GITHUB_OUTPUT
             fi
+            if echo "$labels" | grep -Fq "test:reliability"; then
+              echo "run_reliability=true" >> $GITHUB_OUTPUT
+            else
+              echo "run_reliability=false" >> $GITHUB_OUTPUT
+            fi
           else
             echo "run_fuzz=false" >> $GITHUB_OUTPUT
+            echo "run_reliability=false" >> $GITHUB_OUTPUT
           fi
 
           configs="$configs]"

.gitlab-ci.yml

@@ -8,6 +8,9 @@ variables:
   FORCE_BUILD:
     value: ""
     description: "Force build even if no new commits (any non-empty value)"
+  RUN_RELIABILITY:
+    value: "false"
+    description: "Run reliability and chaos tests. Set automatically when the test:reliability label is on the PR."
   MAVEN_REPOSITORY_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/"
 
 default:
@@ -153,6 +156,66 @@ jdk-integration-test:
     forward:
       pipeline_variables: true
 
+# Generates a child pipeline YAML for reliability/chaos tests when the PR
+# carries the test:reliability label (RUN_RELIABILITY=true in build.env).
+generate-reliability-child-pipeline:
+  stage: reliability
+  tags: ["arch:amd64"]
+  image: $PREPARE_IMAGE
+  needs:
+    - job: prepare:start
+      artifacts: true
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: never
+    - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null'
+      when: never
+    - when: on_success
+  script:
+    - |
+      if [ "${RUN_RELIABILITY:-}" = "true" ]; then
+        echo "Label test:reliability detected — enabling reliability child pipeline"
+        cp .gitlab/reliability/pr-child.gitlab-ci.yml generated-reliability.yml
+      else
+        cat > generated-reliability.yml << 'NOOP'
+      skip-reliability:
+        image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest
+        tags: ["arch:amd64"]
+        script:
+          - echo "Label test:reliability not set — skipping"
+        rules:
+          - when: always
+      NOOP
+      fi
+  artifacts:
+    paths:
+      - generated-reliability.yml
+    expire_in: 1 day
+
+run-reliability-tests:
+  stage: reliability
+  variables:
+    DDPROF_COMMIT_BRANCH: "$DDPROF_COMMIT_BRANCH"
+    DDPROF_COMMIT_SHA: "$DDPROF_COMMIT_SHA"
+  needs:
+    - job: generate-reliability-child-pipeline
+      artifacts: true
+    - job: prepare:start
+      artifacts: true
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: never
+    - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null'
+      when: never
+    - when: on_success
+  trigger:
+    include:
+      - artifact: generated-reliability.yml
+        job: generate-reliability-child-pipeline
+    strategy: depend
+    forward:
+      pipeline_variables: true
+
 include:
   - local: .gitlab/common.yml
   - local: .adms/python/gitlab.yaml

.gitlab/benchmarks/.gitlab-ci.yml

@@ -17,13 +17,17 @@ variables:
   rules:
     - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null'
       when: never
-    - if: '$CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "pipeline"'
-      when: on_success
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: never
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: never
     - if: '$CI_PIPELINE_SOURCE == "web"'
       when: manual
       allow_failure: true
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-      when: manual
+    # Run automatically and non-blocking on any other source (push/trigger/api/
+    # etc.) — mirrors the integration-test rules. The before_script CANCELLED
+    # gate skips branches with no open PR.
+    - when: on_success
       allow_failure: true
   script: |
     # setup the env
@@ -36,8 +40,8 @@ variables:
     if [ -z "${CANDIDATE_VERSION}" ]; then echo "Missing candidate version. Skipping."; exit 0; fi
 
     # fetch the common platform scripts
-    git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
-    git clone --branch dd-trace-go https://github.com/DataDog/benchmarking-platform ${PLATFORM_DIR}
+    git -c url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf="https://github.com/DataDog/" \
+      clone --branch dd-trace-go https://github.com/DataDog/benchmarking-platform ${PLATFORM_DIR}
 
     # apply the specific step scripts
     cp -r .gitlab/benchmarks/steps/* ${PLATFORM_DIR}/steps/
@@ -52,7 +56,6 @@ variables:
     ${PLATFORM_DIR}/steps/run-benchmarks.sh
     ${PLATFORM_DIR}/steps/analyze-results.sh
     ${PLATFORM_DIR}/steps/upload-results-to-s3.sh
-    ${PLATFORM_DIR}/steps/post-pr-comment.sh
   parallel:
     matrix:
       - RUN_MODE: ["cpu", "wall", "alloc", "memleak", "cpu,wall", "memleak,alloc", "cpu,wall,alloc,memleak"]
@@ -76,6 +79,36 @@ benchmarks-candidate-aarch64:
     KUBERNETES_MEMORY_REQUEST: 200Gi
     KUBERNETES_MEMORY_LIMIT: 200Gi
 
+post-benchmarks-pr-comment:
+  extends: .retry-config
+  stage: benchmarks
+  tags: ["arch:arm64"]
+  image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
+  id_tokens:
+    DDOCTOSTS_ID_TOKEN:
+      aud: dd-octo-sts
+  needs:
+    - job: prepare:start
+      artifacts: true
+    - job: benchmarks-candidate-amd64
+      artifacts: true
+    - job: benchmarks-candidate-aarch64
+      artifacts: true
+  rules:
+    - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null'
+      when: never
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: never
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: never
+    # Always run when the candidate jobs ran, regardless of source, so results
+    # are posted back to the PR.
+    - when: always
+  timeout: 5m
+  script:
+    - .gitlab/benchmarks/post-pr-comment.sh reports
+  allow_failure: true
+
 publish-benchmark-gh-pages:
   stage: benchmarks
   tags: ["arch:arm64"]

.gitlab/benchmarks/post-pr-comment.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Post aggregated benchmark comparison results as a single PR comment.
+#
+# Expects all per-cell comparison-baseline-vs-candidate_*.md reports to be
+# present under REPORTS_DIR (default: reports/).
+#
+# Required env:
+#   DDPROF_COMMIT_BRANCH  – branch name used to locate the open PR
+# Optional env:
+#   CI_PIPELINE_URL, DDPROF_COMMIT_SHA
+
+set -euo pipefail
+
+REPORTS_DIR="${1:-reports}"
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Aggregate all per-cell reports into a single comment body
+SECTIONS=""
+for md in "${REPORTS_DIR}"/comparison-baseline-vs-candidate_*.md; do
+  [ -f "${md}" ] || continue
+  label=$(basename "${md}" .md | sed 's/comparison-baseline-vs-candidate_//')
+  SECTIONS="${SECTIONS}
+<details><summary>${label}</summary>
+
+$(cat "${md}")
+
+</details>
+"
+done
+
+if [ -z "${SECTIONS}" ]; then
+  echo "No benchmark reports found under ${REPORTS_DIR} — skipping comment"
+  exit 0
+fi
+
+BODY_FILE=$(mktemp)
+trap 'rm -f "${BODY_FILE}"' EXIT
+cat > "${BODY_FILE}" <<EOF
+## Benchmark Results
+
+Pipeline: ${CI_PIPELINE_URL:-}  Commit: \`${DDPROF_COMMIT_SHA:-unknown}\`
+
+${SECTIONS}
+EOF
+
+"${HERE}/../scripts/upsert-github-pr-comment.sh" \
+  "benchmark-results" "${DDPROF_COMMIT_BRANCH:-}" "${BODY_FILE}"

.gitlab/reliability/chaos_check.sh

@@ -160,6 +160,14 @@ RC=$?
 echo "RC=$RC"
 
 if [ $RC -ne 0 ]; then
-  echo "FAIL:Chaos harness crashed (RC=$RC)" >&2
+  CRASH_MSG="Chaos harness crashed (RC=${RC})"
+  HS_ERR="${HERE}/../../hs_err.log"
+  if [ -f "${HS_ERR}" ]; then
+    SIG=$(grep -m1 '^siginfo:' "${HS_ERR}" 2>/dev/null | tr -d '\n' | cut -c1-120)
+    FRAME=$(grep -m1 'libjavaProfiler\|AsyncProfiler' "${HS_ERR}" 2>/dev/null | sed 's/^[[:space:]]*//' | tr -d '\n' | cut -c1-120)
+    [ -n "${SIG}" ]   && CRASH_MSG="${CRASH_MSG};${SIG}"
+    [ -n "${FRAME}" ] && CRASH_MSG="${CRASH_MSG};${FRAME}"
+  fi
+  echo "FAIL:${CRASH_MSG}" >&2
   exit 1
 fi

.gitlab/reliability/post-pr-comment.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Post aggregated reliability + chaos test results as a single PR comment.
+#
+# Reads REASON_* variables written to build.env by the reliability/chaos jobs
+# and emits a ✅/❌ matrix with failure <details> blocks.
+#
+# Required env:
+#   DDPROF_COMMIT_BRANCH  – branch name used to locate the open PR
+# Optional env:
+#   CI_PIPELINE_URL
+
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# ── Collect failures from REASON_* env vars ────────────────────────────────────
+rel_fail=0; rel_failures=""
+chaos_fail=0; chaos_failures=""
+
+for key in $(compgen -v | grep -E '^REASON_.*X(jit|memory)$' | sort); do
+  reason="${!key}"
+  label="${key#REASON_}"
+  rel_fail=$((rel_fail + 1))
+  detail=$(printf '%s' "${reason//\`/}" | tr ';' '\n')
+  rel_failures="${rel_failures}
+<details><summary>❌ ${label//_/ }</summary>
+
+\`\`\`
+${detail}
+\`\`\`
+
+</details>"
+done
+
+for key in $(compgen -v | grep -E '^REASON_.*Xchaos$' | sort); do
+  reason="${!key}"
+  label="${key#REASON_}"
+  chaos_fail=$((chaos_fail + 1))
+  detail=$(printf '%s' "${reason//\`/}" | tr ';' '\n')
+  chaos_failures="${chaos_failures}
+<details><summary>❌ chaos: ${label//_/ }</summary>
+
+\`\`\`
+${detail}
+\`\`\`
+
+</details>"
+done
+
+# ── Assemble comment ────────────────────────────────────────────────────────────
+total_fail=$((rel_fail + chaos_fail))
+if [ "${total_fail}" -gt 0 ]; then
+  overall="❌ **${total_fail} failure(s) detected**"
+else
+  overall="✅ **All reliability & chaos checks passed**"
+fi
+
+BODY_FILE=$(mktemp)
+trap 'rm -f "${BODY_FILE}"' EXIT
+cat > "${BODY_FILE}" <<EOF
+## Reliability & Chaos Results
+
+${overall}  Pipeline: ${CI_PIPELINE_URL:-}
+${rel_failures}${chaos_failures}
+EOF
+
+"${HERE}/../scripts/upsert-github-pr-comment.sh" \
+  "reliability-results" "${DDPROF_COMMIT_BRANCH:-}" "${BODY_FILE}"