diff --git a/.github/scripts/compare-benchmarks.py b/.github/scripts/compare-benchmarks.py deleted file mode 100755 index e513f1a..0000000 --- a/.github/scripts/compare-benchmarks.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -"""Compare two JMH JSON result files, print a Markdown table. - -Used by benchmark-lto.yml to summarize LTO's throughput impact per platform: -compare-benchmarks.py -""" -import json -import sys - - -def load(path): - with open(path) as f: - results = json.load(f) - return { - (r["benchmark"], tuple(sorted(r.get("params", {}).items()))): r - for r in results - } - - -def main(): - if len(sys.argv) != 3: - print("usage: compare-benchmarks.py ", file=sys.stderr) - return 1 - - baseline = load(sys.argv[1]) - variant = load(sys.argv[2]) - - print("| Benchmark | Params | Baseline | Variant | Delta |") - print("|---|---|---:|---:|---:|") - for key in sorted(baseline): - if key not in variant: - continue - b, v = baseline[key]["primaryMetric"], variant[key]["primaryMetric"] - # scoreError is "NaN" (a JSON string, not a number) when JMH can't - # compute a confidence interval - e.g. a single measurement iteration. - b_score, b_err, unit = b["score"], float(b["scoreError"]), b["scoreUnit"] - v_score, v_err = v["score"], float(v["scoreError"]) - delta = (v_score - b_score) / b_score * 100 if b_score else 0.0 - name = key[0].rsplit(".", 1)[-1] - params = ", ".join(f"{k}={pv}" for k, pv in key[1]) or "-" - print( - f"| {name} | {params} | {b_score:.3f} ± {b_err:.3f} {unit} " - f"| {v_score:.3f} ± {v_err:.3f} {unit} | {delta:+.1f}% |" - ) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.github/scripts/format-contestants.py b/.github/scripts/format-contestants.py new file mode 100644 index 0000000..480bdc5 --- /dev/null +++ b/.github/scripts/format-contestants.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Format one JMH JSON result file into contestant-comparison tables. + +Used by benchmark.yml to report zstd-java (byte[] and MemorySegment) +against zstd-jni and aircompressor on the same build, plus the cost of +creating a fresh native context per call versus reusing one across calls. + +Usage: + format-contestants.py +""" +import json +import sys + +# Windows' default console code page isn't UTF-8, so stdout would otherwise +# mangle the '±' in the tables below (garbles to '?'). +sys.stdout.reconfigure(encoding="utf-8") + +CONTESTANTS = ["zstdJavaSegment", "zstdJavaBytes", "zstdJni", "aircompressor"] +CONTEXT_PAIRS = [ + ("zstdJavaSegment", "zstdJavaSegmentFreshContext", "MemorySegment"), + ("zstdJavaBytes", "zstdJavaBytesFreshContext", "byte[]"), +] + + +def load(path): + with open(path) as f: + results = json.load(f) + + by_class = {} + for r in results: + fqn = r["benchmark"] + cls, method = fqn.rsplit(".", 1) + cls = cls.rsplit(".", 1)[-1] + size = r.get("params", {}).get("size") + if size is None: + continue + by_class.setdefault(cls, {}).setdefault(size, {})[method] = r["primaryMetric"] + return by_class + + +def fmt(metric): + if metric is None: + return "-" + return f"{metric['score']:.3f} ± {float(metric['scoreError']):.3f} {metric['scoreUnit']}" + + +def print_contestants_table(cls, by_size): + print(f"#### {cls}: contestants (ops/ms, higher is better)") + print() + print("| size | " + " | ".join(CONTESTANTS) + " |") + print("|---|" + "---:|" * len(CONTESTANTS)) + for size in sorted(by_size, key=int): + row = by_size[size] + cells = [fmt(row.get(c)) for c in CONTESTANTS] + print(f"| {int(size):,} | " + " | ".join(cells) + " |") + print() + + +def print_context_table(cls, by_size): + rows = [] + for size in sorted(by_size, key=int): + row = by_size[size] + for reused, fresh, label in CONTEXT_PAIRS: + r, f = row.get(reused), row.get(fresh) + if r is None or f is None: + continue + delta = (f["score"] - r["score"]) / r["score"] * 100 if r["score"] else 0.0 + rows.append((size, label, r, f, delta)) + + if not rows: + return + + print(f"#### {cls}: context reuse (reused across calls vs fresh per call)") + print() + print("| size | mode | reused context | fresh context per call | delta |") + print("|---|---|---:|---:|---:|") + for size, label, r, f, delta in rows: + print(f"| {int(size):,} | {label} | {fmt(r)} | {fmt(f)} | {delta:+.1f}% |") + print() + + +def main(): + if len(sys.argv) != 2: + print("usage: format-contestants.py ", file=sys.stderr) + return 1 + + by_class = load(sys.argv[1]) + for cls in sorted(by_class): + print_contestants_table(cls, by_class[cls]) + print_context_table(cls, by_class[cls]) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/benchmark-lto.yml b/.github/workflows/benchmark-lto.yml deleted file mode 100644 index 3541a98..0000000 --- a/.github/workflows/benchmark-lto.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Benchmark LTO impact - -# Manual, exploratory: compares JMH throughput between main and an LTO -# variant branch (see scripts/build-zstd.sh - currently -flto is Linux-only, -# zig's Mach-O backend doesn't support it) on every runner OS the repo -# supports, so the real-vs-no-op split shows up as CI evidence per platform -# instead of a manual claim. Not part of ci.yml: this is throughput -# exploration, not a correctness gate, and JMH runs are too slow to run on -# every push/PR. - -on: - workflow_dispatch: - inputs: - lto_ref: - description: Branch/ref with the LTO change to compare against main. - required: false - default: experiment/lto-linux - -jobs: - benchmark: - name: ${{ matrix.classifier }} - strategy: - fail-fast: false - matrix: - include: - - { os: ubuntu-latest, classifier: linux-x86_64 } - - { os: ubuntu-24.04-arm, classifier: linux-aarch64 } - - { os: macos-14, classifier: osx-aarch64 } - - { os: windows-latest, classifier: windows-x86_64 } - runs-on: ${{ matrix.os }} - steps: - - name: Checkout baseline (main, with zstd submodule) - uses: actions/checkout@v7 - with: - submodules: recursive - path: baseline - - - name: Checkout LTO variant (${{ inputs.lto_ref }}, with zstd submodule) - uses: actions/checkout@v7 - with: - submodules: recursive - ref: ${{ inputs.lto_ref }} - path: lto - - - name: Set up JDK 25 - uses: actions/setup-java@v5 - with: - distribution: temurin - java-version: '25' - cache: maven - - - name: Set up Zig - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2 - with: - version: 0.16.0 - - # `ompressBenchmark` matches both CompressBenchmark and - # DecompressBenchmark method FQNs (JMH's filter is a substring/regex - # search), but not GoldenCorpusBenchmark - keeps this to the quick - # synthetic-payload suite, not the slower real-corpus one. - - name: Build + run baseline benchmark - shell: bash - working-directory: baseline - run: | - ./mvnw -B -ntp -q -pl benchmark -am package -DskipTests - java -jar benchmark/target/benchmarks.jar ompressBenchmark \ - -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff ../baseline-results.json - - - name: Build + run LTO-variant benchmark - shell: bash - working-directory: lto - run: | - ./mvnw -B -ntp -q -pl benchmark -am package -DskipTests - java -jar benchmark/target/benchmarks.jar ompressBenchmark \ - -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff ../lto-results.json - - - name: Summarize - shell: bash - run: | - { - echo "### ${{ matrix.classifier }}" - echo - python3 baseline/.github/scripts/compare-benchmarks.py \ - baseline-results.json lto-results.json - echo - } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..aa3d2e1 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,80 @@ +name: Benchmark + +# Manual, exploratory, on every runner OS the repo supports. Not part of +# ci.yml: JMH runs are too slow to gate every push/PR. +# +# Reports zstd-java (byte[] and MemorySegment) against zstd-jni and +# aircompressor on identical input, plus reused-vs-fresh-per-call native +# context overhead - the "is this library actually fast" evidence. +# +# Results are consumable two ways: the table is both printed to the job log +# (no sign-in needed, unlike the Step Summary UI) and uploaded as a +# per-classifier artifact - `gh run download ` gets the raw JMH JSON +# plus the rendered table for offline diffing or archiving across runs. + +on: + workflow_dispatch: + +jobs: + benchmark: + name: ${{ matrix.classifier }} + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, classifier: linux-x86_64 } + - { os: ubuntu-24.04-arm, classifier: linux-aarch64 } + - { os: macos-14, classifier: osx-aarch64 } + - { os: windows-latest, classifier: windows-x86_64 } + runs-on: ${{ matrix.os }} + steps: + - name: Checkout (with zstd submodule) + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Set up JDK 25 + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: '25' + cache: maven + + - name: Set up Zig + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2 + with: + version: 0.16.0 + + # Anchored on the package-separator dot so `CompressBenchmark` doesn't + # also match `MultiThreadCompressBenchmark` (JMH's filter is a + # substring/regex search) - that class has no zstdJni/aircompressor + # peers, so it only pollutes the contestants table with empty cells. + # GoldenCorpusBenchmark stays excluded - keeps this to the quick + # synthetic-payload suite, not the slower real-corpus one. + - name: Build + run benchmark + shell: bash + run: | + ./mvnw -B -ntp -q -pl benchmark -am package -DskipTests + java -jar benchmark/target/benchmarks.jar '\.CompressBenchmark\.' '\.DecompressBenchmark\.' \ + -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff results.json + + - name: Report contestants (zstd-java vs zstd-jni vs aircompressor) + if: always() + shell: bash + run: | + { + echo "### ${{ matrix.classifier }}" + echo + python3 .github/scripts/format-contestants.py results.json + } | tee contestants.md | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-${{ matrix.classifier }} + path: | + results.json + contestants.md + if-no-files-found: ignore + retention-days: 30 diff --git a/benchmark/src/main/java/io/github/dfa1/zstd/bench/CompressBenchmark.java b/benchmark/src/main/java/io/github/dfa1/zstd/bench/CompressBenchmark.java index a593d2a..f3be644 100644 --- a/benchmark/src/main/java/io/github/dfa1/zstd/bench/CompressBenchmark.java +++ b/benchmark/src/main/java/io/github/dfa1/zstd/bench/CompressBenchmark.java @@ -88,6 +88,22 @@ public long zstdJavaSegment() { return ffmCtx.compress(dstSeg, srcSeg); } + // Fresh context per call (vs ffmCtx, reused above) isolates the native + // ZSTD_createCCtx/ZSTD_freeCCtx cost that context reuse avoids. + @Benchmark + public byte[] zstdJavaBytesFreshContext() { + try (ZstdCompressContext ctx = new ZstdCompressContext().level(new ZstdCompressionLevel(level))) { + return ctx.compress(src); + } + } + + @Benchmark + public long zstdJavaSegmentFreshContext() { + try (ZstdCompressContext ctx = new ZstdCompressContext().level(new ZstdCompressionLevel(level))) { + return ctx.compress(dstSeg, srcSeg); + } + } + @Benchmark public byte[] zstdJni() { return com.github.luben.zstd.Zstd.compress(src, level); diff --git a/benchmark/src/main/java/io/github/dfa1/zstd/bench/DecompressBenchmark.java b/benchmark/src/main/java/io/github/dfa1/zstd/bench/DecompressBenchmark.java index 318e326..e90d75a 100644 --- a/benchmark/src/main/java/io/github/dfa1/zstd/bench/DecompressBenchmark.java +++ b/benchmark/src/main/java/io/github/dfa1/zstd/bench/DecompressBenchmark.java @@ -85,6 +85,22 @@ public long zstdJavaSegment() { return ffmCtx.decompress(dstSeg, frameSeg); } + // Fresh context per call (vs ffmCtx, reused above) isolates the native + // ZSTD_createDCtx/ZSTD_freeDCtx cost that context reuse avoids. + @Benchmark + public byte[] zstdJavaBytesFreshContext() { + try (ZstdDecompressContext ctx = new ZstdDecompressContext()) { + return ctx.decompress(frame, new ZstdByteSize(originalSize)); + } + } + + @Benchmark + public long zstdJavaSegmentFreshContext() { + try (ZstdDecompressContext ctx = new ZstdDecompressContext()) { + return ctx.decompress(dstSeg, frameSeg); + } + } + @Benchmark public byte[] zstdJni() { return com.github.luben.zstd.Zstd.decompress(frame, originalSize); diff --git a/benchmark/src/main/java/io/github/dfa1/zstd/bench/LargeFileBenchmark.java b/benchmark/src/main/java/io/github/dfa1/zstd/bench/LargeFileBenchmark.java index f1f176e..48e5ea9 100644 --- a/benchmark/src/main/java/io/github/dfa1/zstd/bench/LargeFileBenchmark.java +++ b/benchmark/src/main/java/io/github/dfa1/zstd/bench/LargeFileBenchmark.java @@ -3,6 +3,7 @@ import io.github.dfa1.zstd.ZstdCompressStream; import io.github.dfa1.zstd.ZstdCompressionLevel; import io.github.dfa1.zstd.ZstdEndDirective; +import io.github.dfa1.zstd.ZstdOutputStream; import io.github.dfa1.zstd.ZstdStreamResult; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -39,9 +40,11 @@ import static java.lang.foreign.ValueLayout.JAVA_LONG; /// Whole-file compression: this library's `mmap` + `MemorySegment` path -/// (with and without a `posix_madvise(WILLNEED)` readahead hint) against -/// zstd-jni's classic `ZstdOutputStream` over a buffered `FileInputStream` — -/// the comparison behind the numbers in `docs/zero-copy.md`. +/// (with and without a `posix_madvise(WILLNEED)` readahead hint) against two +/// conventional buffered-stream paths - this library's own [ZstdOutputStream] +/// and zstd-jni's classic `ZstdOutputStream`, both over a buffered +/// `FileInputStream` - the comparison behind the numbers in +/// `docs/zero-copy.md`. /// /// Sizes run from 4 MiB up to 10 GiB, so each `@Benchmark` invocation is one /// full-file compression (seconds to tens of seconds), not a tight throughput @@ -49,12 +52,12 @@ /// counting iterations in a fixed window. /// /// The source file is generated once per size and cached under the system -/// temp directory, reused across all three variants' trials rather than +/// temp directory, reused across all four variants' trials rather than /// regenerated per (variant, size) — each variant's JMH fork otherwise /// rewrites an identical multi-gigabyte file for no reason. Only the timed -/// compress-and-write work is measured. Both the mmap path and zstd-jni's -/// stream path write their compressed output to a real file, matching real -/// usage rather than discarding output to a null sink. +/// compress-and-write work is measured. Every variant writes its compressed +/// output to a real file, matching real usage rather than discarding output +/// to a null sink. @BenchmarkMode(Mode.SingleShotTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Thread) @@ -105,6 +108,24 @@ public long mmapWithAdvise() throws IOException { return mmapCompress(true); } + // Same buffered-read/heap-write shape as zstdJniStream below, but through + // this library's own ZstdOutputStream - isolates the mmap+MemorySegment + // path's gain over our own conventional stream API, not just zstd-jni's. + @Benchmark + public long zstdJavaStream() throws IOException { + try (InputStream in = new BufferedInputStream(Files.newInputStream(sourceFile), CHUNK); + ZstdOutputStream out = new ZstdOutputStream(Files.newOutputStream(destFile), new ZstdCompressionLevel(level))) { + byte[] buffer = new byte[CHUNK]; + long total = 0; + int n; + while ((n = in.read(buffer)) != -1) { + out.write(buffer, 0, n); + total += n; + } + return total; + } + } + @Benchmark public long zstdJniStream() throws IOException { try (InputStream in = new BufferedInputStream(Files.newInputStream(sourceFile), CHUNK);