From ad864dd0ad5e0c7f9e97fcdd9fd96c6a3004015e Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 26 Jul 2026 20:36:11 +0200 Subject: [PATCH 1/5] ci: surface benchmark-lto results as log output + artifacts Step Summary alone requires a signed-in browser session to view and isn't retrievable via the API. Tee the comparison table into the job log too, and upload the raw JMH JSON + rendered table as a per-classifier artifact for offline diffing. --- .github/workflows/benchmark-lto.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-lto.yml b/.github/workflows/benchmark-lto.yml index 3541a98..c4e360d 100644 --- a/.github/workflows/benchmark-lto.yml +++ b/.github/workflows/benchmark-lto.yml @@ -7,6 +7,12 @@ name: Benchmark LTO impact # instead of a manual claim. Not part of ci.yml: this is throughput # exploration, not a correctness gate, and JMH runs are too slow to run on # every push/PR. +# +# Results are consumable two ways: the comparison table is both printed to +# the job log (no sign-in needed, unlike the Step Summary UI) and uploaded +# as a per-classifier artifact - `gh run download ` gets the raw JMH +# JSON (baseline-results.json/lto-results.json) plus the rendered +# comparison.md for offline diffing or archiving across runs. on: workflow_dispatch: @@ -75,6 +81,7 @@ jobs: -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff ../lto-results.json - name: Summarize + if: always() shell: bash run: | { @@ -83,4 +90,16 @@ jobs: python3 baseline/.github/scripts/compare-benchmarks.py \ baseline-results.json lto-results.json echo - } >> "$GITHUB_STEP_SUMMARY" + } | tee comparison.md | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-${{ matrix.classifier }} + path: | + baseline-results.json + lto-results.json + comparison.md + if-no-files-found: ignore + retention-days: 30 From 6513829b1e53a578d22d53a1940d7d41c86c6956 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 27 Jul 2026 07:15:22 +0200 Subject: [PATCH 2/5] fix: force UTF-8 stdout in compare-benchmarks.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windows' console code page isn't UTF-8 by default, so the '±' in the comparison table garbled to '?' on windows-x86_64 runs. --- .github/scripts/compare-benchmarks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/scripts/compare-benchmarks.py b/.github/scripts/compare-benchmarks.py index e513f1a..e3d6793 100755 --- a/.github/scripts/compare-benchmarks.py +++ b/.github/scripts/compare-benchmarks.py @@ -7,6 +7,10 @@ import json import sys +# Windows' default console code page isn't UTF-8, so stdout would otherwise +# mangle the '±' in the table below (garbles to '?'). +sys.stdout.reconfigure(encoding="utf-8") + def load(path): with open(path) as f: From 8267389001a95132266aee892b8e16d1bce4e6eb Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 27 Jul 2026 07:40:18 +0200 Subject: [PATCH 3/5] ci: report zstd-java vs zstd-jni vs aircompressor, and context-reuse cost The LTO-diff workflow only ever compared the same code built two ways. Add a contestants report (zstd-java byte[]/MemorySegment vs zstd-jni vs aircompressor) from the baseline build, plus reused-vs-fresh-per-call native context overhead via two new JMH benchmark methods per class. Tighten the JMH filter to anchor on the package-separator dot so CompressBenchmark no longer also matches MultiThreadCompressBenchmark, which has no zstdJni/aircompressor peers and only produced empty cells. --- .github/scripts/format-contestants.py | 96 +++++++++++++++++++ .github/workflows/benchmark-lto.yml | 54 +++++++---- .../dfa1/zstd/bench/CompressBenchmark.java | 16 ++++ .../dfa1/zstd/bench/DecompressBenchmark.java | 16 ++++ 4 files changed, 163 insertions(+), 19 deletions(-) create mode 100644 .github/scripts/format-contestants.py diff --git a/.github/scripts/format-contestants.py b/.github/scripts/format-contestants.py new file mode 100644 index 0000000..cb73de3 --- /dev/null +++ b/.github/scripts/format-contestants.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Format one JMH JSON result file into contestant-comparison tables. + +Used by benchmark-lto.yml to report zstd-java (byte[] and MemorySegment) +against zstd-jni and aircompressor on the same build, plus the cost of +creating a fresh native context per call versus reusing one across calls. + +Usage: + format-contestants.py +""" +import json +import sys + +# Windows' default console code page isn't UTF-8, so stdout would otherwise +# mangle the '±' in the tables below (garbles to '?'). +sys.stdout.reconfigure(encoding="utf-8") + +CONTESTANTS = ["zstdJavaSegment", "zstdJavaBytes", "zstdJni", "aircompressor"] +CONTEXT_PAIRS = [ + ("zstdJavaSegment", "zstdJavaSegmentFreshContext", "MemorySegment"), + ("zstdJavaBytes", "zstdJavaBytesFreshContext", "byte[]"), +] + + +def load(path): + with open(path) as f: + results = json.load(f) + + by_class = {} + for r in results: + fqn = r["benchmark"] + cls, method = fqn.rsplit(".", 1) + cls = cls.rsplit(".", 1)[-1] + size = r.get("params", {}).get("size") + if size is None: + continue + by_class.setdefault(cls, {}).setdefault(size, {})[method] = r["primaryMetric"] + return by_class + + +def fmt(metric): + if metric is None: + return "-" + return f"{metric['score']:.3f} ± {float(metric['scoreError']):.3f} {metric['scoreUnit']}" + + +def print_contestants_table(cls, by_size): + print(f"#### {cls}: contestants (ops/ms, higher is better)") + print() + print("| size | " + " | ".join(CONTESTANTS) + " |") + print("|---|" + "---:|" * len(CONTESTANTS)) + for size in sorted(by_size, key=int): + row = by_size[size] + cells = [fmt(row.get(c)) for c in CONTESTANTS] + print(f"| {int(size):,} | " + " | ".join(cells) + " |") + print() + + +def print_context_table(cls, by_size): + rows = [] + for size in sorted(by_size, key=int): + row = by_size[size] + for reused, fresh, label in CONTEXT_PAIRS: + r, f = row.get(reused), row.get(fresh) + if r is None or f is None: + continue + delta = (f["score"] - r["score"]) / r["score"] * 100 if r["score"] else 0.0 + rows.append((size, label, r, f, delta)) + + if not rows: + return + + print(f"#### {cls}: context reuse (reused across calls vs fresh per call)") + print() + print("| size | mode | reused context | fresh context per call | delta |") + print("|---|---|---:|---:|---:|") + for size, label, r, f, delta in rows: + print(f"| {int(size):,} | {label} | {fmt(r)} | {fmt(f)} | {delta:+.1f}% |") + print() + + +def main(): + if len(sys.argv) != 2: + print("usage: format-contestants.py ", file=sys.stderr) + return 1 + + by_class = load(sys.argv[1]) + for cls in sorted(by_class): + print_contestants_table(cls, by_class[cls]) + print_context_table(cls, by_class[cls]) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/benchmark-lto.yml b/.github/workflows/benchmark-lto.yml index c4e360d..f3deb71 100644 --- a/.github/workflows/benchmark-lto.yml +++ b/.github/workflows/benchmark-lto.yml @@ -1,18 +1,21 @@ -name: Benchmark LTO impact +name: Benchmark -# Manual, exploratory: compares JMH throughput between main and an LTO -# variant branch (see scripts/build-zstd.sh - currently -flto is Linux-only, -# zig's Mach-O backend doesn't support it) on every runner OS the repo -# supports, so the real-vs-no-op split shows up as CI evidence per platform -# instead of a manual claim. Not part of ci.yml: this is throughput -# exploration, not a correctness gate, and JMH runs are too slow to run on -# every push/PR. +# Manual, exploratory, on every runner OS the repo supports. Not part of +# ci.yml: JMH runs are too slow to gate every push/PR. Two reports come out +# of the same baseline (main) build: # -# Results are consumable two ways: the comparison table is both printed to -# the job log (no sign-in needed, unlike the Step Summary UI) and uploaded -# as a per-classifier artifact - `gh run download ` gets the raw JMH -# JSON (baseline-results.json/lto-results.json) plus the rendered -# comparison.md for offline diffing or archiving across runs. +# - Contestants: zstd-java (byte[] and MemorySegment) vs zstd-jni vs +# aircompressor on identical input, plus reused-vs-fresh-per-call native +# context overhead - the "is this library actually fast" evidence. +# - LTO impact: baseline (main) vs an LTO variant branch (see +# scripts/build-zstd.sh - currently -flto is Linux-only, zig's Mach-O +# backend doesn't support it), so the real-vs-no-op split shows up as CI +# evidence per platform instead of a manual claim. +# +# Results are consumable two ways: every table is both printed to the job log +# (no sign-in needed, unlike the Step Summary UI) and uploaded as a +# per-classifier artifact - `gh run download ` gets the raw JMH JSON +# plus the rendered tables for offline diffing or archiving across runs. on: workflow_dispatch: @@ -60,16 +63,18 @@ jobs: with: version: 0.16.0 - # `ompressBenchmark` matches both CompressBenchmark and - # DecompressBenchmark method FQNs (JMH's filter is a substring/regex - # search), but not GoldenCorpusBenchmark - keeps this to the quick + # Anchored on the package-separator dot so `CompressBenchmark` doesn't + # also match `MultiThreadCompressBenchmark` (JMH's filter is a + # substring/regex search) - that class has no zstdJni/aircompressor + # peers, so it only pollutes the contestants table with empty cells. + # GoldenCorpusBenchmark stays excluded - keeps this to the quick # synthetic-payload suite, not the slower real-corpus one. - name: Build + run baseline benchmark shell: bash working-directory: baseline run: | ./mvnw -B -ntp -q -pl benchmark -am package -DskipTests - java -jar benchmark/target/benchmarks.jar ompressBenchmark \ + java -jar benchmark/target/benchmarks.jar '\.CompressBenchmark\.' '\.DecompressBenchmark\.' \ -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff ../baseline-results.json - name: Build + run LTO-variant benchmark @@ -77,10 +82,20 @@ jobs: working-directory: lto run: | ./mvnw -B -ntp -q -pl benchmark -am package -DskipTests - java -jar benchmark/target/benchmarks.jar ompressBenchmark \ + java -jar benchmark/target/benchmarks.jar '\.CompressBenchmark\.' '\.DecompressBenchmark\.' \ -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff ../lto-results.json - - name: Summarize + - name: Report contestants (zstd-java vs zstd-jni vs aircompressor) + if: always() + shell: bash + run: | + { + echo "### ${{ matrix.classifier }}" + echo + python3 baseline/.github/scripts/format-contestants.py baseline-results.json + } | tee contestants.md | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Report LTO impact (baseline vs variant) if: always() shell: bash run: | @@ -100,6 +115,7 @@ jobs: path: | baseline-results.json lto-results.json + contestants.md comparison.md if-no-files-found: ignore retention-days: 30 diff --git a/benchmark/src/main/java/io/github/dfa1/zstd/bench/CompressBenchmark.java b/benchmark/src/main/java/io/github/dfa1/zstd/bench/CompressBenchmark.java index a593d2a..f3be644 100644 --- a/benchmark/src/main/java/io/github/dfa1/zstd/bench/CompressBenchmark.java +++ b/benchmark/src/main/java/io/github/dfa1/zstd/bench/CompressBenchmark.java @@ -88,6 +88,22 @@ public long zstdJavaSegment() { return ffmCtx.compress(dstSeg, srcSeg); } + // Fresh context per call (vs ffmCtx, reused above) isolates the native + // ZSTD_createCCtx/ZSTD_freeCCtx cost that context reuse avoids. + @Benchmark + public byte[] zstdJavaBytesFreshContext() { + try (ZstdCompressContext ctx = new ZstdCompressContext().level(new ZstdCompressionLevel(level))) { + return ctx.compress(src); + } + } + + @Benchmark + public long zstdJavaSegmentFreshContext() { + try (ZstdCompressContext ctx = new ZstdCompressContext().level(new ZstdCompressionLevel(level))) { + return ctx.compress(dstSeg, srcSeg); + } + } + @Benchmark public byte[] zstdJni() { return com.github.luben.zstd.Zstd.compress(src, level); diff --git a/benchmark/src/main/java/io/github/dfa1/zstd/bench/DecompressBenchmark.java b/benchmark/src/main/java/io/github/dfa1/zstd/bench/DecompressBenchmark.java index 318e326..e90d75a 100644 --- a/benchmark/src/main/java/io/github/dfa1/zstd/bench/DecompressBenchmark.java +++ b/benchmark/src/main/java/io/github/dfa1/zstd/bench/DecompressBenchmark.java @@ -85,6 +85,22 @@ public long zstdJavaSegment() { return ffmCtx.decompress(dstSeg, frameSeg); } + // Fresh context per call (vs ffmCtx, reused above) isolates the native + // ZSTD_createDCtx/ZSTD_freeDCtx cost that context reuse avoids. + @Benchmark + public byte[] zstdJavaBytesFreshContext() { + try (ZstdDecompressContext ctx = new ZstdDecompressContext()) { + return ctx.decompress(frame, new ZstdByteSize(originalSize)); + } + } + + @Benchmark + public long zstdJavaSegmentFreshContext() { + try (ZstdDecompressContext ctx = new ZstdDecompressContext()) { + return ctx.decompress(dstSeg, frameSeg); + } + } + @Benchmark public byte[] zstdJni() { return com.github.luben.zstd.Zstd.decompress(frame, originalSize); From 81759bae8a3044ac5dfb32e2752264337b2c38c9 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 27 Jul 2026 07:53:13 +0200 Subject: [PATCH 4/5] ci: drop the LTO comparison from the benchmark workflow LTO was already investigated and rejected (CHANGELOG.md: regression on x86_64, and zig's Mach-O linker doesn't support it on macOS at all, issues #70/#77) - this was leftover machinery from that closed investigation, not something anyone runs against a real branch (experiment/lto-linux never existed). Renamed benchmark-lto.yml -> benchmark.yml since it's no longer LTO-specific, dropped compare-benchmarks.py (its only caller), and simplified back to a single checkout/build/run per platform. --- .github/scripts/compare-benchmarks.py | 54 ------------ .github/scripts/format-contestants.py | 2 +- .github/workflows/benchmark-lto.yml | 121 -------------------------- .github/workflows/benchmark.yml | 80 +++++++++++++++++ 4 files changed, 81 insertions(+), 176 deletions(-) delete mode 100755 .github/scripts/compare-benchmarks.py delete mode 100644 .github/workflows/benchmark-lto.yml create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/scripts/compare-benchmarks.py b/.github/scripts/compare-benchmarks.py deleted file mode 100755 index e3d6793..0000000 --- a/.github/scripts/compare-benchmarks.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -"""Compare two JMH JSON result files, print a Markdown table. - -Used by benchmark-lto.yml to summarize LTO's throughput impact per platform: -compare-benchmarks.py -""" -import json -import sys - -# Windows' default console code page isn't UTF-8, so stdout would otherwise -# mangle the '±' in the table below (garbles to '?'). -sys.stdout.reconfigure(encoding="utf-8") - - -def load(path): - with open(path) as f: - results = json.load(f) - return { - (r["benchmark"], tuple(sorted(r.get("params", {}).items()))): r - for r in results - } - - -def main(): - if len(sys.argv) != 3: - print("usage: compare-benchmarks.py ", file=sys.stderr) - return 1 - - baseline = load(sys.argv[1]) - variant = load(sys.argv[2]) - - print("| Benchmark | Params | Baseline | Variant | Delta |") - print("|---|---|---:|---:|---:|") - for key in sorted(baseline): - if key not in variant: - continue - b, v = baseline[key]["primaryMetric"], variant[key]["primaryMetric"] - # scoreError is "NaN" (a JSON string, not a number) when JMH can't - # compute a confidence interval - e.g. a single measurement iteration. - b_score, b_err, unit = b["score"], float(b["scoreError"]), b["scoreUnit"] - v_score, v_err = v["score"], float(v["scoreError"]) - delta = (v_score - b_score) / b_score * 100 if b_score else 0.0 - name = key[0].rsplit(".", 1)[-1] - params = ", ".join(f"{k}={pv}" for k, pv in key[1]) or "-" - print( - f"| {name} | {params} | {b_score:.3f} ± {b_err:.3f} {unit} " - f"| {v_score:.3f} ± {v_err:.3f} {unit} | {delta:+.1f}% |" - ) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.github/scripts/format-contestants.py b/.github/scripts/format-contestants.py index cb73de3..480bdc5 100644 --- a/.github/scripts/format-contestants.py +++ b/.github/scripts/format-contestants.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Format one JMH JSON result file into contestant-comparison tables. -Used by benchmark-lto.yml to report zstd-java (byte[] and MemorySegment) +Used by benchmark.yml to report zstd-java (byte[] and MemorySegment) against zstd-jni and aircompressor on the same build, plus the cost of creating a fresh native context per call versus reusing one across calls. diff --git a/.github/workflows/benchmark-lto.yml b/.github/workflows/benchmark-lto.yml deleted file mode 100644 index f3deb71..0000000 --- a/.github/workflows/benchmark-lto.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: Benchmark - -# Manual, exploratory, on every runner OS the repo supports. Not part of -# ci.yml: JMH runs are too slow to gate every push/PR. Two reports come out -# of the same baseline (main) build: -# -# - Contestants: zstd-java (byte[] and MemorySegment) vs zstd-jni vs -# aircompressor on identical input, plus reused-vs-fresh-per-call native -# context overhead - the "is this library actually fast" evidence. -# - LTO impact: baseline (main) vs an LTO variant branch (see -# scripts/build-zstd.sh - currently -flto is Linux-only, zig's Mach-O -# backend doesn't support it), so the real-vs-no-op split shows up as CI -# evidence per platform instead of a manual claim. -# -# Results are consumable two ways: every table is both printed to the job log -# (no sign-in needed, unlike the Step Summary UI) and uploaded as a -# per-classifier artifact - `gh run download ` gets the raw JMH JSON -# plus the rendered tables for offline diffing or archiving across runs. - -on: - workflow_dispatch: - inputs: - lto_ref: - description: Branch/ref with the LTO change to compare against main. - required: false - default: experiment/lto-linux - -jobs: - benchmark: - name: ${{ matrix.classifier }} - strategy: - fail-fast: false - matrix: - include: - - { os: ubuntu-latest, classifier: linux-x86_64 } - - { os: ubuntu-24.04-arm, classifier: linux-aarch64 } - - { os: macos-14, classifier: osx-aarch64 } - - { os: windows-latest, classifier: windows-x86_64 } - runs-on: ${{ matrix.os }} - steps: - - name: Checkout baseline (main, with zstd submodule) - uses: actions/checkout@v7 - with: - submodules: recursive - path: baseline - - - name: Checkout LTO variant (${{ inputs.lto_ref }}, with zstd submodule) - uses: actions/checkout@v7 - with: - submodules: recursive - ref: ${{ inputs.lto_ref }} - path: lto - - - name: Set up JDK 25 - uses: actions/setup-java@v5 - with: - distribution: temurin - java-version: '25' - cache: maven - - - name: Set up Zig - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2 - with: - version: 0.16.0 - - # Anchored on the package-separator dot so `CompressBenchmark` doesn't - # also match `MultiThreadCompressBenchmark` (JMH's filter is a - # substring/regex search) - that class has no zstdJni/aircompressor - # peers, so it only pollutes the contestants table with empty cells. - # GoldenCorpusBenchmark stays excluded - keeps this to the quick - # synthetic-payload suite, not the slower real-corpus one. - - name: Build + run baseline benchmark - shell: bash - working-directory: baseline - run: | - ./mvnw -B -ntp -q -pl benchmark -am package -DskipTests - java -jar benchmark/target/benchmarks.jar '\.CompressBenchmark\.' '\.DecompressBenchmark\.' \ - -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff ../baseline-results.json - - - name: Build + run LTO-variant benchmark - shell: bash - working-directory: lto - run: | - ./mvnw -B -ntp -q -pl benchmark -am package -DskipTests - java -jar benchmark/target/benchmarks.jar '\.CompressBenchmark\.' '\.DecompressBenchmark\.' \ - -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff ../lto-results.json - - - name: Report contestants (zstd-java vs zstd-jni vs aircompressor) - if: always() - shell: bash - run: | - { - echo "### ${{ matrix.classifier }}" - echo - python3 baseline/.github/scripts/format-contestants.py baseline-results.json - } | tee contestants.md | tee -a "$GITHUB_STEP_SUMMARY" - - - name: Report LTO impact (baseline vs variant) - if: always() - shell: bash - run: | - { - echo "### ${{ matrix.classifier }}" - echo - python3 baseline/.github/scripts/compare-benchmarks.py \ - baseline-results.json lto-results.json - echo - } | tee comparison.md | tee -a "$GITHUB_STEP_SUMMARY" - - - name: Upload benchmark results - if: always() - uses: actions/upload-artifact@v4 - with: - name: benchmark-results-${{ matrix.classifier }} - path: | - baseline-results.json - lto-results.json - contestants.md - comparison.md - if-no-files-found: ignore - retention-days: 30 diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..aa3d2e1 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,80 @@ +name: Benchmark + +# Manual, exploratory, on every runner OS the repo supports. Not part of +# ci.yml: JMH runs are too slow to gate every push/PR. +# +# Reports zstd-java (byte[] and MemorySegment) against zstd-jni and +# aircompressor on identical input, plus reused-vs-fresh-per-call native +# context overhead - the "is this library actually fast" evidence. +# +# Results are consumable two ways: the table is both printed to the job log +# (no sign-in needed, unlike the Step Summary UI) and uploaded as a +# per-classifier artifact - `gh run download ` gets the raw JMH JSON +# plus the rendered table for offline diffing or archiving across runs. + +on: + workflow_dispatch: + +jobs: + benchmark: + name: ${{ matrix.classifier }} + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, classifier: linux-x86_64 } + - { os: ubuntu-24.04-arm, classifier: linux-aarch64 } + - { os: macos-14, classifier: osx-aarch64 } + - { os: windows-latest, classifier: windows-x86_64 } + runs-on: ${{ matrix.os }} + steps: + - name: Checkout (with zstd submodule) + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Set up JDK 25 + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: '25' + cache: maven + + - name: Set up Zig + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2 + with: + version: 0.16.0 + + # Anchored on the package-separator dot so `CompressBenchmark` doesn't + # also match `MultiThreadCompressBenchmark` (JMH's filter is a + # substring/regex search) - that class has no zstdJni/aircompressor + # peers, so it only pollutes the contestants table with empty cells. + # GoldenCorpusBenchmark stays excluded - keeps this to the quick + # synthetic-payload suite, not the slower real-corpus one. + - name: Build + run benchmark + shell: bash + run: | + ./mvnw -B -ntp -q -pl benchmark -am package -DskipTests + java -jar benchmark/target/benchmarks.jar '\.CompressBenchmark\.' '\.DecompressBenchmark\.' \ + -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff results.json + + - name: Report contestants (zstd-java vs zstd-jni vs aircompressor) + if: always() + shell: bash + run: | + { + echo "### ${{ matrix.classifier }}" + echo + python3 .github/scripts/format-contestants.py results.json + } | tee contestants.md | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-${{ matrix.classifier }} + path: | + results.json + contestants.md + if-no-files-found: ignore + retention-days: 30 From 5707c227279c3f7d4c80cb1332a7e4acde566383 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 27 Jul 2026 08:02:26 +0200 Subject: [PATCH 5/5] bench: add zstdJavaStream to LargeFileBenchmark LargeFileBenchmark only ever compared the mmap + MemorySegment path against zstd-jni's stream API, never against this library's own ZstdOutputStream - so the mmap path's gain over our own conventional stream API was invisible. Add zstdJavaStream (same buffered-read/ heap-write shape as zstdJniStream) to close that gap. Verified: compiles, runs clean at size=4194304, and the exact write()-loop-through-ZstdOutputStream pattern round-trips byte-for-byte through Zstd.decompress (checked standalone, not asserted in the benchmark itself - JMH benchmarks here aren't correctness tests, ZstdOutputStream's correctness is already covered by ZstdStreamTest). --- .../dfa1/zstd/bench/LargeFileBenchmark.java | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/benchmark/src/main/java/io/github/dfa1/zstd/bench/LargeFileBenchmark.java b/benchmark/src/main/java/io/github/dfa1/zstd/bench/LargeFileBenchmark.java index f1f176e..48e5ea9 100644 --- a/benchmark/src/main/java/io/github/dfa1/zstd/bench/LargeFileBenchmark.java +++ b/benchmark/src/main/java/io/github/dfa1/zstd/bench/LargeFileBenchmark.java @@ -3,6 +3,7 @@ import io.github.dfa1.zstd.ZstdCompressStream; import io.github.dfa1.zstd.ZstdCompressionLevel; import io.github.dfa1.zstd.ZstdEndDirective; +import io.github.dfa1.zstd.ZstdOutputStream; import io.github.dfa1.zstd.ZstdStreamResult; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -39,9 +40,11 @@ import static java.lang.foreign.ValueLayout.JAVA_LONG; /// Whole-file compression: this library's `mmap` + `MemorySegment` path -/// (with and without a `posix_madvise(WILLNEED)` readahead hint) against -/// zstd-jni's classic `ZstdOutputStream` over a buffered `FileInputStream` — -/// the comparison behind the numbers in `docs/zero-copy.md`. +/// (with and without a `posix_madvise(WILLNEED)` readahead hint) against two +/// conventional buffered-stream paths - this library's own [ZstdOutputStream] +/// and zstd-jni's classic `ZstdOutputStream`, both over a buffered +/// `FileInputStream` - the comparison behind the numbers in +/// `docs/zero-copy.md`. /// /// Sizes run from 4 MiB up to 10 GiB, so each `@Benchmark` invocation is one /// full-file compression (seconds to tens of seconds), not a tight throughput @@ -49,12 +52,12 @@ /// counting iterations in a fixed window. /// /// The source file is generated once per size and cached under the system -/// temp directory, reused across all three variants' trials rather than +/// temp directory, reused across all four variants' trials rather than /// regenerated per (variant, size) — each variant's JMH fork otherwise /// rewrites an identical multi-gigabyte file for no reason. Only the timed -/// compress-and-write work is measured. Both the mmap path and zstd-jni's -/// stream path write their compressed output to a real file, matching real -/// usage rather than discarding output to a null sink. +/// compress-and-write work is measured. Every variant writes its compressed +/// output to a real file, matching real usage rather than discarding output +/// to a null sink. @BenchmarkMode(Mode.SingleShotTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Thread) @@ -105,6 +108,24 @@ public long mmapWithAdvise() throws IOException { return mmapCompress(true); } + // Same buffered-read/heap-write shape as zstdJniStream below, but through + // this library's own ZstdOutputStream - isolates the mmap+MemorySegment + // path's gain over our own conventional stream API, not just zstd-jni's. + @Benchmark + public long zstdJavaStream() throws IOException { + try (InputStream in = new BufferedInputStream(Files.newInputStream(sourceFile), CHUNK); + ZstdOutputStream out = new ZstdOutputStream(Files.newOutputStream(destFile), new ZstdCompressionLevel(level))) { + byte[] buffer = new byte[CHUNK]; + long total = 0; + int n; + while ((n = in.read(buffer)) != -1) { + out.write(buffer, 0, n); + total += n; + } + return total; + } + } + @Benchmark public long zstdJniStream() throws IOException { try (InputStream in = new BufferedInputStream(Files.newInputStream(sourceFile), CHUNK);