ci: consumable, contestant-focused benchmark reporting

.github/scripts/format-contestants.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""Format one JMH JSON result file into contestant-comparison tables.
+
+Used by benchmark.yml to report zstd-java (byte[] and MemorySegment)
+against zstd-jni and aircompressor on the same build, plus the cost of
+creating a fresh native context per call versus reusing one across calls.
+
+Usage:
+  format-contestants.py <results.json>
+"""
+import json
+import sys
+
+# Windows' default console code page isn't UTF-8, so stdout would otherwise
+# mangle the '±' in the tables below (garbles to '?').
+sys.stdout.reconfigure(encoding="utf-8")
+
+CONTESTANTS = ["zstdJavaSegment", "zstdJavaBytes", "zstdJni", "aircompressor"]
+CONTEXT_PAIRS = [
+    ("zstdJavaSegment", "zstdJavaSegmentFreshContext", "MemorySegment"),
+    ("zstdJavaBytes", "zstdJavaBytesFreshContext", "byte[]"),
+]
+
+
+def load(path):
+    with open(path) as f:
+        results = json.load(f)
+
+    by_class = {}
+    for r in results:
+        fqn = r["benchmark"]
+        cls, method = fqn.rsplit(".", 1)
+        cls = cls.rsplit(".", 1)[-1]
+        size = r.get("params", {}).get("size")
+        if size is None:
+            continue
+        by_class.setdefault(cls, {}).setdefault(size, {})[method] = r["primaryMetric"]
+    return by_class
+
+
+def fmt(metric):
+    if metric is None:
+        return "-"
+    return f"{metric['score']:.3f} ± {float(metric['scoreError']):.3f} {metric['scoreUnit']}"
+
+
+def print_contestants_table(cls, by_size):
+    print(f"#### {cls}: contestants (ops/ms, higher is better)")
+    print()
+    print("| size | " + " | ".join(CONTESTANTS) + " |")
+    print("|---|" + "---:|" * len(CONTESTANTS))
+    for size in sorted(by_size, key=int):
+        row = by_size[size]
+        cells = [fmt(row.get(c)) for c in CONTESTANTS]
+        print(f"| {int(size):,} | " + " | ".join(cells) + " |")
+    print()
+
+
+def print_context_table(cls, by_size):
+    rows = []
+    for size in sorted(by_size, key=int):
+        row = by_size[size]
+        for reused, fresh, label in CONTEXT_PAIRS:
+            r, f = row.get(reused), row.get(fresh)
+            if r is None or f is None:
+                continue
+            delta = (f["score"] - r["score"]) / r["score"] * 100 if r["score"] else 0.0
+            rows.append((size, label, r, f, delta))
+
+    if not rows:
+        return
+
+    print(f"#### {cls}: context reuse (reused across calls vs fresh per call)")
+    print()
+    print("| size | mode | reused context | fresh context per call | delta |")
+    print("|---|---|---:|---:|---:|")
+    for size, label, r, f, delta in rows:
+        print(f"| {int(size):,} | {label} | {fmt(r)} | {fmt(f)} | {delta:+.1f}% |")
+    print()
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("usage: format-contestants.py <results.json>", file=sys.stderr)
+        return 1
+
+    by_class = load(sys.argv[1])
+    for cls in sorted(by_class):
+        print_contestants_table(cls, by_class[cls])
+        print_context_table(cls, by_class[cls])
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/workflows/benchmark.yml

@@ -0,0 +1,80 @@
+name: Benchmark
+
+# Manual, exploratory, on every runner OS the repo supports. Not part of
+# ci.yml: JMH runs are too slow to gate every push/PR.
+#
+# Reports zstd-java (byte[] and MemorySegment) against zstd-jni and
+# aircompressor on identical input, plus reused-vs-fresh-per-call native
+# context overhead - the "is this library actually fast" evidence.
+#
+# Results are consumable two ways: the table is both printed to the job log
+# (no sign-in needed, unlike the Step Summary UI) and uploaded as a
+# per-classifier artifact - `gh run download <run-id>` gets the raw JMH JSON
+# plus the rendered table for offline diffing or archiving across runs.
+
+on:
+  workflow_dispatch:
+
+jobs:
+  benchmark:
+    name: ${{ matrix.classifier }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { os: ubuntu-latest,    classifier: linux-x86_64 }
+          - { os: ubuntu-24.04-arm, classifier: linux-aarch64 }
+          - { os: macos-14,         classifier: osx-aarch64 }
+          - { os: windows-latest,   classifier: windows-x86_64 }
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout (with zstd submodule)
+        uses: actions/checkout@v7
+        with:
+          submodules: recursive
+
+      - name: Set up JDK 25
+        uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: '25'
+          cache: maven
+
+      - name: Set up Zig
+        uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
+        with:
+          version: 0.16.0
+
+      # Anchored on the package-separator dot so `CompressBenchmark` doesn't
+      # also match `MultiThreadCompressBenchmark` (JMH's filter is a
+      # substring/regex search) - that class has no zstdJni/aircompressor
+      # peers, so it only pollutes the contestants table with empty cells.
+      # GoldenCorpusBenchmark stays excluded - keeps this to the quick
+      # synthetic-payload suite, not the slower real-corpus one.
+      - name: Build + run benchmark
+        shell: bash
+        run: |
+          ./mvnw -B -ntp -q -pl benchmark -am package -DskipTests
+          java -jar benchmark/target/benchmarks.jar '\.CompressBenchmark\.' '\.DecompressBenchmark\.' \
+            -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff results.json
+
+      - name: Report contestants (zstd-java vs zstd-jni vs aircompressor)
+        if: always()
+        shell: bash
+        run: |
+          {
+            echo "### ${{ matrix.classifier }}"
+            echo
+            python3 .github/scripts/format-contestants.py results.json
+          } | tee contestants.md | tee -a "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload benchmark results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results-${{ matrix.classifier }}
+          path: |
+            results.json
+            contestants.md
+          if-no-files-found: ignore
+          retention-days: 30

benchmark/src/main/java/io/github/dfa1/zstd/bench/CompressBenchmark.java

@@ -88,6 +88,22 @@ public long zstdJavaSegment() {
         return ffmCtx.compress(dstSeg, srcSeg);
     }
 
+    // Fresh context per call (vs ffmCtx, reused above) isolates the native
+    // ZSTD_createCCtx/ZSTD_freeCCtx cost that context reuse avoids.
+    @Benchmark
+    public byte[] zstdJavaBytesFreshContext() {
+        try (ZstdCompressContext ctx = new ZstdCompressContext().level(new ZstdCompressionLevel(level))) {
+            return ctx.compress(src);
+        }
+    }
+
+    @Benchmark
+    public long zstdJavaSegmentFreshContext() {
+        try (ZstdCompressContext ctx = new ZstdCompressContext().level(new ZstdCompressionLevel(level))) {
+            return ctx.compress(dstSeg, srcSeg);
+        }
+    }
+
     @Benchmark
     public byte[] zstdJni() {
         return com.github.luben.zstd.Zstd.compress(src, level);

benchmark/src/main/java/io/github/dfa1/zstd/bench/DecompressBenchmark.java

@@ -85,6 +85,22 @@ public long zstdJavaSegment() {
         return ffmCtx.decompress(dstSeg, frameSeg);
     }
 
+    // Fresh context per call (vs ffmCtx, reused above) isolates the native
+    // ZSTD_createDCtx/ZSTD_freeDCtx cost that context reuse avoids.
+    @Benchmark
+    public byte[] zstdJavaBytesFreshContext() {
+        try (ZstdDecompressContext ctx = new ZstdDecompressContext()) {
+            return ctx.decompress(frame, new ZstdByteSize(originalSize));
+        }
+    }
+
+    @Benchmark
+    public long zstdJavaSegmentFreshContext() {
+        try (ZstdDecompressContext ctx = new ZstdDecompressContext()) {
+            return ctx.decompress(dstSeg, frameSeg);
+        }
+    }
+
     @Benchmark
     public byte[] zstdJni() {
         return com.github.luben.zstd.Zstd.decompress(frame, originalSize);