From fc5fa145e70f250a4c3acff7347bfce44950d799 Mon Sep 17 00:00:00 2001 From: Dmitriy Aleksandrov Date: Fri, 16 Oct 2026 15:13:51 +0300 Subject: [PATCH] ci: add bench-diff: benchmarks of the pull request and the target branch compared by benchstat Signed-off-by: Dmitriy Aleksandrov --- .github/workflows/bench-diff.yml | 38 ++++++++++++++++ drivers/bench_test.go | 73 ++++++++++++++++++++++++++++++ scripts/bench-diff.sh | 76 ++++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+) create mode 100644 .github/workflows/bench-diff.yml create mode 100644 drivers/bench_test.go create mode 100755 scripts/bench-diff.sh diff --git a/.github/workflows/bench-diff.yml b/.github/workflows/bench-diff.yml new file mode 100644 index 0000000..1736ce6 --- /dev/null +++ b/.github/workflows/bench-diff.yml @@ -0,0 +1,38 @@ +name: Bench diff + +on: + pull_request: + branches: [ "main" ] + +permissions: + contents: read + pull-requests: write + +jobs: + bench-diff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 50 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: stable + + - name: Run bench-diff + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BENCH_COUNT: '10' + run: bash scripts/bench-diff.sh + + - name: Upload bench logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: bench-results + path: | + base.txt + head.txt + retention-days: 14 diff --git a/drivers/bench_test.go b/drivers/bench_test.go new file mode 100644 index 0000000..d73ce72 --- /dev/null +++ b/drivers/bench_test.go @@ -0,0 +1,73 @@ +package drivers_test + +import ( + "testing" + "time" + + "github.com/insei/tinyconf" + "github.com/insei/tinyconf/drivers/env" + "github.com/insei/tinyconf/drivers/tag" + "github.com/insei/tinyconf/drivers/yaml" +) + +type benchServer struct { + Host string `yaml:"host"` + Port int `yaml:"port"` +} + +type benchConfig struct { + Name string `initial:"app" yaml:"name" env:"BENCH_NAME"` + HTTP struct { + Host string `initial:"localhost" yaml:"host" env:"BENCH_HTTP_HOST"` + Port int `initial:"8080" yaml:"port" env:"BENCH_HTTP_PORT"` + Timeout time.Duration `initial:"30s" yaml:"timeout" env:"BENCH_HTTP_TIMEOUT"` + } `yaml:"http"` + Hosts []string `initial:"a,b" yaml:"hosts" env:"BENCH_HOSTS"` + Labels map[string]string `yaml:"labels" env:"BENCH_LABELS"` + Servers []benchServer `yaml:"servers"` + Secret tinyconf.Secret `yaml:"secret" env:"BENCH_SECRET"` +} + +const benchYAML = `name: bench +http: + port: 9090 + timeout: 1m +hosts: [c, d] +labels: {env: prod, team: core} +servers: + - {host: a, port: 1} + - {host: b, port: 2} +secret: password +` + +// BenchmarkParse_Drivers parses the config by the usual chain of the drivers: tags, a yaml document and environment. +func BenchmarkParse_Drivers(b *testing.B) { + b.Setenv("BENCH_HTTP_HOST", "example.com") + b.Setenv("BENCH_LABELS", "env:stage") + config, err := tinyconf.New( + tinyconf.WithDriver(must(tag.New("initial"))), + tinyconf.WithDriver(must(yaml.New("", yaml.WithData("bench.yaml", []byte(benchYAML))))), + tinyconf.WithDriver(must(env.New(env.WithFiles()))), + ) + if err != nil { + b.Fatal(err) + } + conf := &benchConfig{} + if err = config.Register(conf); err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err = config.Parse(conf); err != nil { + b.Fatal(err) + } + } +} + +func must(driver tinyconf.Driver, err error) tinyconf.Driver { + if err != nil { + panic(err) + } + return driver +} diff --git a/scripts/bench-diff.sh b/scripts/bench-diff.sh new file mode 100755 index 0000000..cc5968b --- /dev/null +++ b/scripts/bench-diff.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# Runs the benchmarks on the head of the pull request and on the target branch, compares them by benchstat +# and posts the summary to the pull request: one comment, updated after every push. +# +# It is used by .github/workflows/bench-diff.yml on pull_request events and expects the variables of GitHub Actions: +# GITHUB_BASE_REF, GITHUB_REPOSITORY, GITHUB_EVENT_PATH, GITHUB_API_URL, GITHUB_TOKEN (pull-requests: write). +# Without GITHUB_TOKEN the report is only in the logs and in the summary of the job. + +set -euo pipefail + +COUNT="${BENCH_COUNT:-10}" +BENCH_PATTERN='^Benchmark' +BENCH_PKGS=(. ./drivers) +MARKER='' + +: "${GITHUB_BASE_REF:?must run from pull_request event}" +: "${GITHUB_REPOSITORY:?}" +: "${GITHUB_EVENT_PATH:?}" + +PR_NUMBER="$(jq -r '.pull_request.number // empty' "$GITHUB_EVENT_PATH")" +API_URL="${GITHUB_API_URL:-https://api.github.com}" +HEAD_SHA="$(git rev-parse HEAD)" +HEAD_SHORT="$(git rev-parse --short HEAD)" + +bench() { + go test -run='^$' -bench="$BENCH_PATTERN" -benchmem -count="$COUNT" "${BENCH_PKGS[@]}" +} + +echo "=== Benchmarking head ($HEAD_SHORT) ===" +bench | tee head.txt + +git fetch --depth=50 origin "$GITHUB_BASE_REF" +BASE_SHORT="$(git rev-parse --short "origin/$GITHUB_BASE_REF")" +git checkout --detach "origin/$GITHUB_BASE_REF" +echo "=== Benchmarking base ($BASE_SHORT) ===" +BASE_OK=1 +if ! bench | tee base.txt; then + echo "base bench run failed; diff will be skipped" >&2 + BASE_OK=0 +fi +git checkout --detach "$HEAD_SHA" + +go install golang.org/x/perf/cmd/benchstat@latest +BENCHSTAT="$(go env GOPATH)/bin/benchstat" +if [[ "$BASE_OK" == "1" ]]; then + SUMMARY="$("$BENCHSTAT" base.txt head.txt)" + HEADER="Benchmark diff: \`$BASE_SHORT\` → \`$HEAD_SHORT\` ($COUNT runs)" +else + SUMMARY="$("$BENCHSTAT" head.txt)" + HEADER="Base benchmarks did not run on \`$BASE_SHORT\`; head-only results for \`$HEAD_SHORT\`" +fi +echo "$SUMMARY" +BODY=$(printf "%s\n### %s\n\n\`\`\`\n%s\n\`\`\`\n" "$MARKER" "$HEADER" "$SUMMARY") +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + echo "$BODY" >> "$GITHUB_STEP_SUMMARY" +fi + +if [[ -z "${GITHUB_TOKEN:-}" || -z "$PR_NUMBER" ]]; then + echo "GITHUB_TOKEN or the number of the pull request is not set, the comment is skipped" >&2 + exit 0 +fi +api() { + curl --fail-with-body -sS -H "Authorization: Bearer $GITHUB_TOKEN" -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" "$@" +} +PAYLOAD=$(jq -n --arg body "$BODY" '{body: $body}') +# the comment of the previous push is updated +COMMENT_ID="$(api "$API_URL/repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments?per_page=100" | + jq -r --arg marker "$MARKER" '[.[] | select(.body | startswith($marker))][0].id // empty')" +if [[ -n "$COMMENT_ID" ]]; then + api -X PATCH --data "$PAYLOAD" "$API_URL/repos/$GITHUB_REPOSITORY/issues/comments/$COMMENT_ID" >/dev/null +else + api -X POST --data "$PAYLOAD" "$API_URL/repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" >/dev/null +fi +echo "Posted benchmark diff to PR #$PR_NUMBER"