diff --git a/README.md b/README.md index 0bfba7941a..b5221d5f30 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,8 @@ To introduce a new result for an existing system with a different usage scenario `index.html` can be re-generated using `./generate-results.sh`. The CI (GitHub Actions) does this automatically, this step is optional. +The shared driver (`lib/benchmark-common.sh`) runs a supplementary concurrent-QPS test (`BENCH_CONCURRENT_CONNECTIONS` workers for `BENCH_CONCURRENT_DURATION` seconds) after the main sweep. Single-process engines must set `BENCH_CONCURRENT_DURATION=0` in their `benchmark.sh` to skip it: each query forks a fresh full-machine process with no shared scheduler, so concurrent connections only oversubscribe RAM (and can OOM the run) instead of measuring throughput. Rule of thumb — skip when `./start` launches no shared server (the embedded CLIs and Spark variants); keep it for daemons and the in-process server wrappers (pandas/polars/`*-dataframe`), which share one process. See issue #946. + All tests were originally run on AWS c6a.4xlarge EC2 VMs with 500 GB gp2 disks. With better automation, more EC2 machines were added later: c6a.2xlarge, c6a.metal, c8g.4xlarge, c6a.xlarge, c7a.metal-48xl, c6a.large, c8g.metal-48xl, and t3a.small. These represent older and modern machines, and small / medium / large systems (CPU and main memory). diff --git a/bqn/benchmark.sh b/bqn/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/bqn/benchmark.sh +++ b/bqn/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/bqn/results/20260529/c6a.4xlarge.json b/bqn/results/20260529/c6a.4xlarge.json index 30d42d51d2..ed57253aa6 100644 --- a/bqn/results/20260529/c6a.4xlarge.json +++ b/bqn/results/20260529/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C","embedded","array language"], "load_time": 526, "data_size": 118169498884, - "concurrent_qps": 0.042, - "concurrent_error_ratio": 0.554, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.002, 0.001, 0.001], [3.551, 1.308, 1.306], diff --git a/bqn/results/20260529/c6a.metal.json b/bqn/results/20260529/c6a.metal.json index 42e3e4acb6..fdabbd82cb 100644 --- a/bqn/results/20260529/c6a.metal.json +++ b/bqn/results/20260529/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C","embedded","array language"], "load_time": 472, "data_size": 118169498884, - "concurrent_qps": 0.223, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.001, 0.001, 0.001], [4.181, 1.272, 1.22], diff --git a/bqn/results/20260529/c7a.metal-48xl.json b/bqn/results/20260529/c7a.metal-48xl.json index 921b88b447..6541120d27 100644 --- a/bqn/results/20260529/c7a.metal-48xl.json +++ b/bqn/results/20260529/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C","embedded","array language"], "load_time": 473, "data_size": 118169498884, - "concurrent_qps": 0.235, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.001, 0.001, 0.001], [4.122, 1.161, 1.094], diff --git a/bqn/results/20260529/c8g.4xlarge.json b/bqn/results/20260529/c8g.4xlarge.json index a3709b9501..9a15aa6ca5 100644 --- a/bqn/results/20260529/c8g.4xlarge.json +++ b/bqn/results/20260529/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C","embedded","array language"], "load_time": 518, "data_size": 118169498884, - "concurrent_qps": 0.042, - "concurrent_error_ratio": 0.597, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.001, 0, 0], [3.349, 0.676, 0.693], diff --git a/bqn/results/20260529/c8g.metal-48xl.json b/bqn/results/20260529/c8g.metal-48xl.json index aaf7f4d1c1..9450d37098 100644 --- a/bqn/results/20260529/c8g.metal-48xl.json +++ b/bqn/results/20260529/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C","embedded","array language"], "load_time": 472, "data_size": 118169498884, - "concurrent_qps": 0.237, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.001, 0, 0], [3.657, 0.702, 0.699], diff --git a/chdb-parquet-partitioned/benchmark.sh b/chdb-parquet-partitioned/benchmark.sh index e236dbbd21..5972a9845b 100755 --- a/chdb-parquet-partitioned/benchmark.sh +++ b/chdb-parquet-partitioned/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/chdb-parquet-partitioned/results/20260511/c6a.2xlarge.json b/chdb-parquet-partitioned/results/20260511/c6a.2xlarge.json index fb8c499658..ad6ab77aae 100644 --- a/chdb-parquet-partitioned/results/20260511/c6a.2xlarge.json +++ b/chdb-parquet-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 9, "data_size": 14737666736, - "concurrent_qps": 0.045, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.061, 0.027, 0.026], [0.151, 0.058, 0.058], diff --git a/chdb-parquet-partitioned/results/20260511/c6a.4xlarge.json b/chdb-parquet-partitioned/results/20260511/c6a.4xlarge.json index 5f6ff9c9a5..ef3f0be45f 100644 --- a/chdb-parquet-partitioned/results/20260511/c6a.4xlarge.json +++ b/chdb-parquet-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.108, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.049, 0.026, 0.026], [0.112, 0.045, 0.045], diff --git a/chdb-parquet-partitioned/results/20260511/c6a.large.json b/chdb-parquet-partitioned/results/20260511/c6a.large.json index 1f54d9cd94..67072e2cdf 100644 --- a/chdb-parquet-partitioned/results/20260511/c6a.large.json +++ b/chdb-parquet-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.035, - "concurrent_error_ratio": 0.045, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.17, 0.044, 0.041], [0.352, 0.141, 0.144], diff --git a/chdb-parquet-partitioned/results/20260511/c6a.metal.json b/chdb-parquet-partitioned/results/20260511/c6a.metal.json index 9ebc69ea06..af5abfdec2 100644 --- a/chdb-parquet-partitioned/results/20260511/c6a.metal.json +++ b/chdb-parquet-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 66, "data_size": 14737666736, - "concurrent_qps": 3.643, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.045, 0.012, 0.018], [0.114, 0.054, 0.052], diff --git a/chdb-parquet-partitioned/results/20260511/c6a.xlarge.json b/chdb-parquet-partitioned/results/20260511/c6a.xlarge.json index 009709246a..5473e5b5b2 100644 --- a/chdb-parquet-partitioned/results/20260511/c6a.xlarge.json +++ b/chdb-parquet-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 4, "data_size": 14737666736, - "concurrent_qps": 0.04, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.101, 0.026, 0.027], [0.217, 0.083, 0.082], diff --git a/chdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json b/chdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json index 0f34b61895..020657e934 100644 --- a/chdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json +++ b/chdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 62, "data_size": 14737666736, - "concurrent_qps": 4.03, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.066, 0.015, 0.015], [0.134, 0.088, 0.089], diff --git a/chdb-parquet-partitioned/results/20260511/c8g.4xlarge.json b/chdb-parquet-partitioned/results/20260511/c8g.4xlarge.json index 3b3d43e24e..73c4e8d84b 100644 --- a/chdb-parquet-partitioned/results/20260511/c8g.4xlarge.json +++ b/chdb-parquet-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.072, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.033, 0.017, 0.018], [0.082, 0.034, 0.036], diff --git a/chdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json b/chdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json index 5b090867c6..09dfde1fb9 100644 --- a/chdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json +++ b/chdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 66, "data_size": 14737666736, - "concurrent_qps": 4.042, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.037, 0.011, 0.012], [0.076, 0.033, 0.029], diff --git a/chdb-parquet-partitioned/results/20260511/t3a.small.json b/chdb-parquet-partitioned/results/20260511/t3a.small.json index 777b8bd5fb..49ac7f2293 100644 --- a/chdb-parquet-partitioned/results/20260511/t3a.small.json +++ b/chdb-parquet-partitioned/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.023, - "concurrent_error_ratio": 0.176, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.224, 0.069, 0.068], [0.515, 0.286, 0.268], diff --git a/chdb/benchmark.sh b/chdb/benchmark.sh index 2e5741b394..bf02868043 100755 --- a/chdb/benchmark.sh +++ b/chdb/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/chdb/results/20260511/c6a.2xlarge.json b/chdb/results/20260511/c6a.2xlarge.json index e81217f1c2..1447746968 100644 --- a/chdb/results/20260511/c6a.2xlarge.json +++ b/chdb/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 573, "data_size": 15259766974, - "concurrent_qps": 0.307, - "concurrent_error_ratio": 0.959, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.014, 0.003, 0.003], [0.064, 0.014, 0.015], diff --git a/chdb/results/20260511/c6a.4xlarge.json b/chdb/results/20260511/c6a.4xlarge.json index db5f829e68..5b87eb65d9 100644 --- a/chdb/results/20260511/c6a.4xlarge.json +++ b/chdb/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 532, "data_size": 15261827371, - "concurrent_qps": 0.58, - "concurrent_error_ratio": 0.94, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.015, 0.003, 0.003], [0.054, 0.012, 0.011], diff --git a/chdb/results/20260511/c6a.large.json b/chdb/results/20260511/c6a.large.json index 77d96d613d..edde078432 100644 --- a/chdb/results/20260511/c6a.large.json +++ b/chdb/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 774, "data_size": 15257484538, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.015, 0.005, 0.005], [0.132, 0.069, 0.052], diff --git a/chdb/results/20260511/c6a.metal.json b/chdb/results/20260511/c6a.metal.json index 3084da182e..f6f6b7a80c 100644 --- a/chdb/results/20260511/c6a.metal.json +++ b/chdb/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 163, "data_size": 15266676267, - "concurrent_qps": 1.358, - "concurrent_error_ratio": 0.942, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.019, 0.007, 0.007], [0.096, 0.06, 0.053], diff --git a/chdb/results/20260511/c6a.xlarge.json b/chdb/results/20260511/c6a.xlarge.json index dbe122e20f..ecc7ca0540 100644 --- a/chdb/results/20260511/c6a.xlarge.json +++ b/chdb/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 602, "data_size": 15261958294, - "concurrent_qps": 0.13, - "concurrent_error_ratio": 0.97, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.014, 0.002, 0.002], [0.072, 0.023, 0.022], diff --git a/chdb/results/20260511/c7a.metal-48xl.json b/chdb/results/20260511/c7a.metal-48xl.json index a496a3a795..013fc8990a 100644 --- a/chdb/results/20260511/c7a.metal-48xl.json +++ b/chdb/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 156, "data_size": 15266229827, - "concurrent_qps": 1.185, - "concurrent_error_ratio": 0.96, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.02, 0.004, 0.004], [0.132, 0.058, 0.075], diff --git a/chdb/results/20260511/c8g.4xlarge.json b/chdb/results/20260511/c8g.4xlarge.json index 1096197e11..1b4d7669aa 100644 --- a/chdb/results/20260511/c8g.4xlarge.json +++ b/chdb/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 602, "data_size": 15260683017, - "concurrent_qps": 1.028, - "concurrent_error_ratio": 0.942, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.01, 0.002, 0.002], [0.046, 0.014, 0.01], diff --git a/chdb/results/20260511/c8g.metal-48xl.json b/chdb/results/20260511/c8g.metal-48xl.json index 9012a0a21b..fc81ec78ed 100644 --- a/chdb/results/20260511/c8g.metal-48xl.json +++ b/chdb/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 149, "data_size": 15264739392, - "concurrent_qps": 1.5150000000000001, - "concurrent_error_ratio": 0.954, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.012, 0.007, 0.007], [0.064, 0.027, 0.027], diff --git a/chdb/results/20260511/t3a.small.json b/chdb/results/20260511/t3a.small.json index a0051cd52e..b00f250769 100644 --- a/chdb/results/20260511/t3a.small.json +++ b/chdb/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 1516, "data_size": 15257125380, - "concurrent_qps": 0.007, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.025, 0.009, 0.008], [0.239, 0.12, 0.118], diff --git a/clickhouse-datalake-partitioned/benchmark.sh b/clickhouse-datalake-partitioned/benchmark.sh index 08d52c39fc..9c514c4803 100755 --- a/clickhouse-datalake-partitioned/benchmark.sh +++ b/clickhouse-datalake-partitioned/benchmark.sh @@ -4,4 +4,9 @@ export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.2xlarge.json b/clickhouse-datalake-partitioned/results/20260511/c6a.2xlarge.json index dba6ed4202..bf6b2ec073 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c6a.2xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.13, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.422, 0.378, 0.376], [1.431, 1.414, 1.311], diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.4xlarge.json b/clickhouse-datalake-partitioned/results/20260511/c6a.4xlarge.json index 6e5211d1a5..91f938b96a 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c6a.4xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.378, - "concurrent_error_ratio": 0.995, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.453, 0.228, 0.224], [2.036, 0.904, 0.781], diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.large.json b/clickhouse-datalake-partitioned/results/20260511/c6a.large.json index 4eb2f06b60..f3f7e0ad53 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c6a.large.json +++ b/clickhouse-datalake-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.008, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [1.478, 1.47, 1.411], [4.133, 3.975, 4.106], diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.metal.json b/clickhouse-datalake-partitioned/results/20260511/c6a.metal.json index 5dfd90b4f2..4f14353cb8 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c6a.metal.json +++ b/clickhouse-datalake-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 1.008, - "concurrent_error_ratio": 0.992, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.181, 0.141, 0.164], [0.382, 0.3, 0.308], diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.xlarge.json b/clickhouse-datalake-partitioned/results/20260511/c6a.xlarge.json index 59388834d6..ff4521330c 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c6a.xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.042, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.72, 0.742, 0.753], [2.651, 2.642, 2.479], diff --git a/clickhouse-datalake-partitioned/results/20260511/c7a.metal-48xl.json b/clickhouse-datalake-partitioned/results/20260511/c7a.metal-48xl.json index 991e7a0682..d6dfbbb78a 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c7a.metal-48xl.json +++ b/clickhouse-datalake-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 1.102, - "concurrent_error_ratio": 0.991, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.199, 0.165, 0.181], [0.337, 0.296, 0.297], diff --git a/clickhouse-datalake-partitioned/results/20260511/c8g.4xlarge.json b/clickhouse-datalake-partitioned/results/20260511/c8g.4xlarge.json index 814817e729..9e15c1bce9 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c8g.4xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.74, - "concurrent_error_ratio": 0.994, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.239, 0.222, 0.223], [0.726, 0.75, 0.721], diff --git a/clickhouse-datalake-partitioned/results/20260511/c8g.metal-48xl.json b/clickhouse-datalake-partitioned/results/20260511/c8g.metal-48xl.json index 4993286c89..067d6dcfcf 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c8g.metal-48xl.json +++ b/clickhouse-datalake-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 1.143, - "concurrent_error_ratio": 0.992, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.183, 0.145, 0.146], [0.307, 0.312, 0.296], diff --git a/clickhouse-datalake-partitioned/results/20260511/t3a.small.json b/clickhouse-datalake-partitioned/results/20260511/t3a.small.json index 4cf59e4f3c..4fef9ba5b2 100644 --- a/clickhouse-datalake-partitioned/results/20260511/t3a.small.json +++ b/clickhouse-datalake-partitioned/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.012, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.045, 3.076, 3.281], [7.062, 7.301, 7.262], diff --git a/clickhouse-datalake-partitioned/results/20260516/c6a.2xlarge.json b/clickhouse-datalake-partitioned/results/20260516/c6a.2xlarge.json index 354d0418c4..f5204b1c6e 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c6a.2xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260516/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.135, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.482, 0.385, 0.394], [1.442, 1.362, 1.311], diff --git a/clickhouse-datalake-partitioned/results/20260516/c6a.4xlarge.json b/clickhouse-datalake-partitioned/results/20260516/c6a.4xlarge.json index 03e3a99d2b..e726cb9e27 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c6a.4xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260516/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.392, - "concurrent_error_ratio": 0.995, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.643, 0.299, 0.235], [2.039, 1.078, 0.76], diff --git a/clickhouse-datalake-partitioned/results/20260516/c6a.metal.json b/clickhouse-datalake-partitioned/results/20260516/c6a.metal.json index 7e9c1ebac8..463704de75 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c6a.metal.json +++ b/clickhouse-datalake-partitioned/results/20260516/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.958, - "concurrent_error_ratio": 0.992, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.238, 0.132, 0.119], [0.379, 0.317, 0.28], diff --git a/clickhouse-datalake-partitioned/results/20260516/c6a.xlarge.json b/clickhouse-datalake-partitioned/results/20260516/c6a.xlarge.json index 122b0fb8aa..101ead2f3d 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c6a.xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260516/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.032, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.788, 0.709, 0.716], [2.651, 2.555, 2.516], diff --git a/clickhouse-datalake-partitioned/results/20260516/c7a.metal-48xl.json b/clickhouse-datalake-partitioned/results/20260516/c7a.metal-48xl.json index 6b72b3cfeb..65983b5ddf 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c7a.metal-48xl.json +++ b/clickhouse-datalake-partitioned/results/20260516/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 1.078, - "concurrent_error_ratio": 0.991, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.244, 0.131, 0.14], [0.373, 0.267, 0.271], diff --git a/clickhouse-datalake-partitioned/results/20260516/c8g.4xlarge.json b/clickhouse-datalake-partitioned/results/20260516/c8g.4xlarge.json index 7185b4c335..d7649a5ce4 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c8g.4xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260516/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.487, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.279, 0.219, 0.229], [0.792, 0.715, 0.742], diff --git a/clickhouse-datalake-partitioned/results/20260516/c8g.metal-48xl.json b/clickhouse-datalake-partitioned/results/20260516/c8g.metal-48xl.json index 71ab6a57ce..27432caad2 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c8g.metal-48xl.json +++ b/clickhouse-datalake-partitioned/results/20260516/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 1.272, - "concurrent_error_ratio": 0.991, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.174, 0.153, 0.157], [0.315, 0.282, 0.28], diff --git a/clickhouse-datalake-partitioned/results/20260517/c6a.large.json b/clickhouse-datalake-partitioned/results/20260517/c6a.large.json index 8bf816417d..6a8dd4bb25 100644 --- a/clickhouse-datalake-partitioned/results/20260517/c6a.large.json +++ b/clickhouse-datalake-partitioned/results/20260517/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.018, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [1.496, 1.432, 1.405], [4.177, 3.961, 3.944], diff --git a/clickhouse-datalake-partitioned/results/20260517/t3a.small.json b/clickhouse-datalake-partitioned/results/20260517/t3a.small.json index 59a08bf37e..70fcd44eac 100644 --- a/clickhouse-datalake-partitioned/results/20260517/t3a.small.json +++ b/clickhouse-datalake-partitioned/results/20260517/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.007, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.155, 2.982, 2.899], [7.073, 6.892, 6.605], diff --git a/clickhouse-datalake/benchmark.sh b/clickhouse-datalake/benchmark.sh index 08d52c39fc..9c514c4803 100755 --- a/clickhouse-datalake/benchmark.sh +++ b/clickhouse-datalake/benchmark.sh @@ -4,4 +4,9 @@ export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/clickhouse-datalake/results/20260511/c6a.2xlarge.json b/clickhouse-datalake/results/20260511/c6a.2xlarge.json index ef68c7a32f..746f03b339 100644 --- a/clickhouse-datalake/results/20260511/c6a.2xlarge.json +++ b/clickhouse-datalake/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.197, - "concurrent_error_ratio": 0.996, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.311, 0.21, 0.211], [0.987, 0.864, 0.926], diff --git a/clickhouse-datalake/results/20260511/c6a.4xlarge.json b/clickhouse-datalake/results/20260511/c6a.4xlarge.json index d259d68f09..6eab8180fe 100644 --- a/clickhouse-datalake/results/20260511/c6a.4xlarge.json +++ b/clickhouse-datalake/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.37, - "concurrent_error_ratio": 0.996, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.285, 0.205, 0.206], [0.746, 0.536, 0.52], diff --git a/clickhouse-datalake/results/20260511/c6a.large.json b/clickhouse-datalake/results/20260511/c6a.large.json index 1d42988e4f..9c78be1c5f 100644 --- a/clickhouse-datalake/results/20260511/c6a.large.json +++ b/clickhouse-datalake/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.035, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.333, 0.231, 0.223], [2.554, 2.373, 2.309], diff --git a/clickhouse-datalake/results/20260511/c6a.metal.json b/clickhouse-datalake/results/20260511/c6a.metal.json index b163df750e..d6e7be9d3f 100644 --- a/clickhouse-datalake/results/20260511/c6a.metal.json +++ b/clickhouse-datalake/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.88, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.315, 0.201, 0.204], [0.389, 0.251, 0.269], diff --git a/clickhouse-datalake/results/20260511/c6a.xlarge.json b/clickhouse-datalake/results/20260511/c6a.xlarge.json index 064d156dcf..f8d32220a6 100644 --- a/clickhouse-datalake/results/20260511/c6a.xlarge.json +++ b/clickhouse-datalake/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.038, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.291, 0.206, 0.201], [1.599, 1.464, 1.505], diff --git a/clickhouse-datalake/results/20260511/c7a.metal-48xl.json b/clickhouse-datalake/results/20260511/c7a.metal-48xl.json index b39e031b15..ed1640fe6f 100644 --- a/clickhouse-datalake/results/20260511/c7a.metal-48xl.json +++ b/clickhouse-datalake/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.798, - "concurrent_error_ratio": 0.994, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.338, 0.2, 0.195], [0.411, 0.289, 0.274], diff --git a/clickhouse-datalake/results/20260511/c8g.4xlarge.json b/clickhouse-datalake/results/20260511/c8g.4xlarge.json index 28c0213bf1..8bbc318fe8 100644 --- a/clickhouse-datalake/results/20260511/c8g.4xlarge.json +++ b/clickhouse-datalake/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.477, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.215, 0.169, 0.148], [0.593, 0.527, 0.568], diff --git a/clickhouse-datalake/results/20260511/c8g.metal-48xl.json b/clickhouse-datalake/results/20260511/c8g.metal-48xl.json index 9269c5e597..54dfd22034 100644 --- a/clickhouse-datalake/results/20260511/c8g.metal-48xl.json +++ b/clickhouse-datalake/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.855, - "concurrent_error_ratio": 0.995, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.217, 0.174, 0.161], [0.367, 0.259, 0.265], diff --git a/clickhouse-datalake/results/20260511/t3a.small.json b/clickhouse-datalake/results/20260511/t3a.small.json index 8262e358ac..6225bee0d3 100644 --- a/clickhouse-datalake/results/20260511/t3a.small.json +++ b/clickhouse-datalake/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.007, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.449, 0.344, 0.36], [2.773, 2.534, 2.698], diff --git a/clickhouse-datalake/results/20260516/c6a.2xlarge.json b/clickhouse-datalake/results/20260516/c6a.2xlarge.json index f34cac4342..7d4f3d9729 100644 --- a/clickhouse-datalake/results/20260516/c6a.2xlarge.json +++ b/clickhouse-datalake/results/20260516/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.115, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.325, 0.213, 0.217], [1.161, 0.908, 0.94], diff --git a/clickhouse-datalake/results/20260516/c6a.4xlarge.json b/clickhouse-datalake/results/20260516/c6a.4xlarge.json index f661269e15..507b03d992 100644 --- a/clickhouse-datalake/results/20260516/c6a.4xlarge.json +++ b/clickhouse-datalake/results/20260516/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.287, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.36, 0.208, 0.213], [0.998, 0.581, 0.601], diff --git a/clickhouse-datalake/results/20260516/c6a.metal.json b/clickhouse-datalake/results/20260516/c6a.metal.json index bcd4fe5ab2..f8ce94751b 100644 --- a/clickhouse-datalake/results/20260516/c6a.metal.json +++ b/clickhouse-datalake/results/20260516/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.942, - "concurrent_error_ratio": 0.992, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.37, 0.222, 0.219], [0.481, 0.376, 0.288], diff --git a/clickhouse-datalake/results/20260516/c6a.xlarge.json b/clickhouse-datalake/results/20260516/c6a.xlarge.json index 25d1e7efcc..ac0a137c79 100644 --- a/clickhouse-datalake/results/20260516/c6a.xlarge.json +++ b/clickhouse-datalake/results/20260516/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.032, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.341, 0.217, 0.209], [1.641, 1.451, 1.486], diff --git a/clickhouse-datalake/results/20260516/c7a.metal-48xl.json b/clickhouse-datalake/results/20260516/c7a.metal-48xl.json index 8071f13388..933e407dc9 100644 --- a/clickhouse-datalake/results/20260516/c7a.metal-48xl.json +++ b/clickhouse-datalake/results/20260516/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.91, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.455, 0.207, 0.222], [0.639, 0.347, 0.271], diff --git a/clickhouse-datalake/results/20260516/c8g.4xlarge.json b/clickhouse-datalake/results/20260516/c8g.4xlarge.json index ccfa132766..c6b1e6a927 100644 --- a/clickhouse-datalake/results/20260516/c8g.4xlarge.json +++ b/clickhouse-datalake/results/20260516/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.453, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.264, 0.167, 0.168], [0.682, 0.556, 0.542], diff --git a/clickhouse-datalake/results/20260516/c8g.metal-48xl.json b/clickhouse-datalake/results/20260516/c8g.metal-48xl.json index aadb1f4aac..8f30654486 100644 --- a/clickhouse-datalake/results/20260516/c8g.metal-48xl.json +++ b/clickhouse-datalake/results/20260516/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.955, - "concurrent_error_ratio": 0.994, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.255, 0.167, 0.174], [0.427, 0.28, 0.289], diff --git a/clickhouse-datalake/results/20260517/c6a.large.json b/clickhouse-datalake/results/20260517/c6a.large.json index 4c8d29bf19..97d76dd223 100644 --- a/clickhouse-datalake/results/20260517/c6a.large.json +++ b/clickhouse-datalake/results/20260517/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.005, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.408, 0.228, 0.224], [2.574, 2.408, 2.455], diff --git a/clickhouse-datalake/results/20260517/t3a.small.json b/clickhouse-datalake/results/20260517/t3a.small.json index 8cf759f7a8..584455b517 100644 --- a/clickhouse-datalake/results/20260517/t3a.small.json +++ b/clickhouse-datalake/results/20260517/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.01, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.541, 0.375, 0.379], [2.806, 2.65, 2.682], diff --git a/clickhouse-parquet-partitioned/benchmark.sh b/clickhouse-parquet-partitioned/benchmark.sh index e236dbbd21..5972a9845b 100755 --- a/clickhouse-parquet-partitioned/benchmark.sh +++ b/clickhouse-parquet-partitioned/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/clickhouse-parquet-partitioned/results/20260511/c6a.2xlarge.json b/clickhouse-parquet-partitioned/results/20260511/c6a.2xlarge.json index 4df9a61751..6b83acc52a 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c6a.2xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 9, "data_size": 14737666736, - "concurrent_qps": 0.048, - "concurrent_error_ratio": 0.237, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.061, 0.015, 0.015], [0.136, 0.045, 0.043], diff --git a/clickhouse-parquet-partitioned/results/20260511/c6a.4xlarge.json b/clickhouse-parquet-partitioned/results/20260511/c6a.4xlarge.json index dc5f2f8e02..5d93736610 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c6a.4xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.088, - "concurrent_error_ratio": 0.086, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.065, 0.011, 0.011], [0.117, 0.03, 0.029], diff --git a/clickhouse-parquet-partitioned/results/20260511/c6a.large.json b/clickhouse-parquet-partitioned/results/20260511/c6a.large.json index b342c90eff..3839039479 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c6a.large.json +++ b/clickhouse-parquet-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.035, - "concurrent_error_ratio": 0.087, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.191, 0.063, 0.065], [0.484, 0.308, 0.297], diff --git a/clickhouse-parquet-partitioned/results/20260511/c6a.metal.json b/clickhouse-parquet-partitioned/results/20260511/c6a.metal.json index 7af2ff74d1..d2fa3379ec 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c6a.metal.json +++ b/clickhouse-parquet-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.88, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.05, 0.012, 0.012], [0.096, 0.055, 0.041], diff --git a/clickhouse-parquet-partitioned/results/20260511/c6a.xlarge.json b/clickhouse-parquet-partitioned/results/20260511/c6a.xlarge.json index 8fa6a76496..64c42f28cc 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c6a.xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 4, "data_size": 14737666736, - "concurrent_qps": 0.047, - "concurrent_error_ratio": 0.067, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.119, 0.024, 0.024], [0.229, 0.072, 0.073], diff --git a/clickhouse-parquet-partitioned/results/20260511/c7a.metal-48xl.json b/clickhouse-parquet-partitioned/results/20260511/c7a.metal-48xl.json index 7c6ff35226..c30bc2b647 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c7a.metal-48xl.json +++ b/clickhouse-parquet-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.855, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.088, 0.015, 0.014], [0.153, 0.069, 0.066], diff --git a/clickhouse-parquet-partitioned/results/20260511/c8g.4xlarge.json b/clickhouse-parquet-partitioned/results/20260511/c8g.4xlarge.json index e1e065c194..b19e5be951 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c8g.4xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.137, - "concurrent_error_ratio": 0.079, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.041, 0.006, 0.006], [0.078, 0.023, 0.023], diff --git a/clickhouse-parquet-partitioned/results/20260511/c8g.metal-48xl.json b/clickhouse-parquet-partitioned/results/20260511/c8g.metal-48xl.json index 018aebd019..f4d951b9c8 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c8g.metal-48xl.json +++ b/clickhouse-parquet-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.732, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.047, 0.026, 0.033], [0.078, 0.065, 0.041], diff --git a/clickhouse-parquet-partitioned/results/20260511/t3a.small.json b/clickhouse-parquet-partitioned/results/20260511/t3a.small.json index c2375896fa..1a8a5fadd1 100644 --- a/clickhouse-parquet-partitioned/results/20260511/t3a.small.json +++ b/clickhouse-parquet-partitioned/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.04, - "concurrent_error_ratio": 0.111, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.427, 0.15, 0.15], [1.2, 0.765, 0.703], diff --git a/clickhouse-parquet-partitioned/results/20260516/c6a.2xlarge.json b/clickhouse-parquet-partitioned/results/20260516/c6a.2xlarge.json index 9a5dbd9ac4..796e3de8a1 100644 --- a/clickhouse-parquet-partitioned/results/20260516/c6a.2xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260516/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 10, "data_size": 14737666736, - "concurrent_qps": 0.047, - "concurrent_error_ratio": 0.067, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.108, 0.015, 0.016], [0.2, 0.045, 0.045], diff --git a/clickhouse-parquet-partitioned/results/20260516/c6a.4xlarge.json b/clickhouse-parquet-partitioned/results/20260516/c6a.4xlarge.json index fdedfebcfc..100a94d92b 100644 --- a/clickhouse-parquet-partitioned/results/20260516/c6a.4xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260516/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 22, "data_size": 14737666736, - "concurrent_qps": 0.1, - "concurrent_error_ratio": 0.062, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.09, 0.012, 0.012], [0.169, 0.032, 0.03], diff --git a/clickhouse-parquet-partitioned/results/20260516/c6a.metal.json b/clickhouse-parquet-partitioned/results/20260516/c6a.metal.json index bd45e30580..9ecdda9e0a 100644 --- a/clickhouse-parquet-partitioned/results/20260516/c6a.metal.json +++ b/clickhouse-parquet-partitioned/results/20260516/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.842, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.109, 0.011, 0.012], [0.159, 0.064, 0.059], diff --git a/clickhouse-parquet-partitioned/results/20260516/c7a.metal-48xl.json b/clickhouse-parquet-partitioned/results/20260516/c7a.metal-48xl.json index aab2fc8e65..447a245e6a 100644 --- a/clickhouse-parquet-partitioned/results/20260516/c7a.metal-48xl.json +++ b/clickhouse-parquet-partitioned/results/20260516/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.822, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.148, 0.015, 0.014], [0.229, 0.083, 0.064], diff --git a/clickhouse-parquet-partitioned/results/20260516/c8g.4xlarge.json b/clickhouse-parquet-partitioned/results/20260516/c8g.4xlarge.json index 935828cbe5..d0a9367c46 100644 --- a/clickhouse-parquet-partitioned/results/20260516/c8g.4xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260516/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.188, - "concurrent_error_ratio": 0.066, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.076, 0.006, 0.007], [0.144, 0.024, 0.024], diff --git a/clickhouse-parquet-partitioned/results/20260516/c8g.metal-48xl.json b/clickhouse-parquet-partitioned/results/20260516/c8g.metal-48xl.json index 73aee9b908..7bba344cfe 100644 --- a/clickhouse-parquet-partitioned/results/20260516/c8g.metal-48xl.json +++ b/clickhouse-parquet-partitioned/results/20260516/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 1.9449999999999998, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.083, 0.025, 0.022], [0.118, 0.046, 0.043], diff --git a/clickhouse-parquet-partitioned/results/20260517/c6a.large.json b/clickhouse-parquet-partitioned/results/20260517/c6a.large.json index f93ae6876f..49416b858a 100644 --- a/clickhouse-parquet-partitioned/results/20260517/c6a.large.json +++ b/clickhouse-parquet-partitioned/results/20260517/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.035, - "concurrent_error_ratio": 0.16, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.219, 0.062, 0.063], [0.568, 0.313, 0.3], diff --git a/clickhouse-parquet-partitioned/results/20260517/c6a.xlarge.json b/clickhouse-parquet-partitioned/results/20260517/c6a.xlarge.json index bb43cf976e..b453f6af25 100644 --- a/clickhouse-parquet-partitioned/results/20260517/c6a.xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260517/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 4, "data_size": 14737666736, - "concurrent_qps": 0.057, - "concurrent_error_ratio": 0.056, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.167, 0.024, 0.024], [0.286, 0.088, 0.077], diff --git a/clickhouse-parquet-partitioned/results/20260517/t3a.small.json b/clickhouse-parquet-partitioned/results/20260517/t3a.small.json index 108afc5fe6..135060bee1 100644 --- a/clickhouse-parquet-partitioned/results/20260517/t3a.small.json +++ b/clickhouse-parquet-partitioned/results/20260517/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.028, - "concurrent_error_ratio": 0.227, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.429, 0.132, 0.155], [1.219, 0.633, 0.66], diff --git a/clickhouse-parquet/benchmark.sh b/clickhouse-parquet/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/clickhouse-parquet/benchmark.sh +++ b/clickhouse-parquet/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/clickhouse-parquet/results/20260511/c6a.2xlarge.json b/clickhouse-parquet/results/20260511/c6a.2xlarge.json index 17eca5c1c3..f63bf5554d 100644 --- a/clickhouse-parquet/results/20260511/c6a.2xlarge.json +++ b/clickhouse-parquet/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.08, - "concurrent_error_ratio": 0.059, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.081, 0.037, 0.037], [0.159, 0.078, 0.08], diff --git a/clickhouse-parquet/results/20260511/c6a.4xlarge.json b/clickhouse-parquet/results/20260511/c6a.4xlarge.json index ede89e7034..0c76a42bff 100644 --- a/clickhouse-parquet/results/20260511/c6a.4xlarge.json +++ b/clickhouse-parquet/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.103, - "concurrent_error_ratio": 0.061, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.084, 0.037, 0.037], [0.143, 0.067, 0.065], diff --git a/clickhouse-parquet/results/20260511/c6a.large.json b/clickhouse-parquet/results/20260511/c6a.large.json index 2ad80a56f8..cc7d15b915 100644 --- a/clickhouse-parquet/results/20260511/c6a.large.json +++ b/clickhouse-parquet/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.027, - "concurrent_error_ratio": 0.111, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.088, 0.048, 0.042], [0.381, 0.281, 0.279], diff --git a/clickhouse-parquet/results/20260511/c6a.metal.json b/clickhouse-parquet/results/20260511/c6a.metal.json index 666e24c210..92fed31a83 100644 --- a/clickhouse-parquet/results/20260511/c6a.metal.json +++ b/clickhouse-parquet/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 2.013, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.079, 0.038, 0.039], [0.139, 0.076, 0.08], diff --git a/clickhouse-parquet/results/20260511/c6a.xlarge.json b/clickhouse-parquet/results/20260511/c6a.xlarge.json index 7ce2c12137..447e0d35e6 100644 --- a/clickhouse-parquet/results/20260511/c6a.xlarge.json +++ b/clickhouse-parquet/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.045, - "concurrent_error_ratio": 0.069, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.084, 0.038, 0.038], [0.2, 0.104, 0.102], diff --git a/clickhouse-parquet/results/20260511/c7a.metal-48xl.json b/clickhouse-parquet/results/20260511/c7a.metal-48xl.json index 69c11d9653..f7e68eaafd 100644 --- a/clickhouse-parquet/results/20260511/c7a.metal-48xl.json +++ b/clickhouse-parquet/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 2.04, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.078, 0.039, 0.038], [0.143, 0.087, 0.082], diff --git a/clickhouse-parquet/results/20260511/c8g.4xlarge.json b/clickhouse-parquet/results/20260511/c8g.4xlarge.json index 532bfee46a..32178d8abe 100644 --- a/clickhouse-parquet/results/20260511/c8g.4xlarge.json +++ b/clickhouse-parquet/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.082, - "concurrent_error_ratio": 0.058, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.066, 0.029, 0.029], [0.113, 0.054, 0.054], diff --git a/clickhouse-parquet/results/20260511/c8g.metal-48xl.json b/clickhouse-parquet/results/20260511/c8g.metal-48xl.json index 84f1244bd8..1cdf9d57db 100644 --- a/clickhouse-parquet/results/20260511/c8g.metal-48xl.json +++ b/clickhouse-parquet/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 2.078, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.055, 0.034, 0.035], [0.104, 0.061, 0.062], diff --git a/clickhouse-parquet/results/20260511/t3a.small.json b/clickhouse-parquet/results/20260511/t3a.small.json index e27e135a3c..5d345e082d 100644 --- a/clickhouse-parquet/results/20260511/t3a.small.json +++ b/clickhouse-parquet/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 0.118, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.136, 0.105, 0.087], [0.609, 0.53, 0.494], diff --git a/clickhouse-parquet/results/20260516/c6a.2xlarge.json b/clickhouse-parquet/results/20260516/c6a.2xlarge.json index 9bacce6536..9592f8d58b 100644 --- a/clickhouse-parquet/results/20260516/c6a.2xlarge.json +++ b/clickhouse-parquet/results/20260516/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.06, - "concurrent_error_ratio": 0.077, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.119, 0.036, 0.036], [0.245, 0.079, 0.078], diff --git a/clickhouse-parquet/results/20260516/c6a.metal.json b/clickhouse-parquet/results/20260516/c6a.metal.json index c75d926d35..5e3a0d5af1 100644 --- a/clickhouse-parquet/results/20260516/c6a.metal.json +++ b/clickhouse-parquet/results/20260516/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 3, "data_size": 14779976446, - "concurrent_qps": 1.995, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.143, 0.039, 0.038], [0.204, 0.08, 0.079], diff --git a/clickhouse-parquet/results/20260516/c6a.xlarge.json b/clickhouse-parquet/results/20260516/c6a.xlarge.json index f2112444ea..b04425cbab 100644 --- a/clickhouse-parquet/results/20260516/c6a.xlarge.json +++ b/clickhouse-parquet/results/20260516/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.043, - "concurrent_error_ratio": 0.103, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.119, 0.036, 0.036], [0.267, 0.11, 0.108], diff --git a/clickhouse-parquet/results/20260516/c7a.metal-48xl.json b/clickhouse-parquet/results/20260516/c7a.metal-48xl.json index 2213760f79..808a78ecdd 100644 --- a/clickhouse-parquet/results/20260516/c7a.metal-48xl.json +++ b/clickhouse-parquet/results/20260516/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 2.052, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.137, 0.04, 0.038], [0.215, 0.083, 0.084], diff --git a/clickhouse-parquet/results/20260516/c8g.4xlarge.json b/clickhouse-parquet/results/20260516/c8g.4xlarge.json index 258298c203..5744759c5f 100644 --- a/clickhouse-parquet/results/20260516/c8g.4xlarge.json +++ b/clickhouse-parquet/results/20260516/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.12, - "concurrent_error_ratio": 0.065, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.101, 0.031, 0.031], [0.159, 0.056, 0.058], diff --git a/clickhouse-parquet/results/20260516/c8g.metal-48xl.json b/clickhouse-parquet/results/20260516/c8g.metal-48xl.json index dfe3114be6..ec7ff2a263 100644 --- a/clickhouse-parquet/results/20260516/c8g.metal-48xl.json +++ b/clickhouse-parquet/results/20260516/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 2.135, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.105, 0.034, 0.034], [0.154, 0.061, 0.06], diff --git a/clickhouse-parquet/results/20260517/c6a.4xlarge.json b/clickhouse-parquet/results/20260517/c6a.4xlarge.json index 953ae5921f..e7199f3bca 100644 --- a/clickhouse-parquet/results/20260517/c6a.4xlarge.json +++ b/clickhouse-parquet/results/20260517/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 8, "data_size": 14779976446, - "concurrent_qps": 0.22, - "concurrent_error_ratio": 0.05, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.137, 0.037, 0.037], [0.205, 0.065, 0.067], diff --git a/clickhouse-parquet/results/20260517/c6a.large.json b/clickhouse-parquet/results/20260517/c6a.large.json index c8dd52b5bf..422f7f0159 100644 --- a/clickhouse-parquet/results/20260517/c6a.large.json +++ b/clickhouse-parquet/results/20260517/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.033, - "concurrent_error_ratio": 0.091, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.148, 0.041, 0.041], [0.444, 0.282, 0.289], diff --git a/clickhouse-parquet/results/20260517/t3a.small.json b/clickhouse-parquet/results/20260517/t3a.small.json index 138078ab15..87d115fe74 100644 --- a/clickhouse-parquet/results/20260517/t3a.small.json +++ b/clickhouse-parquet/results/20260517/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.03, - "concurrent_error_ratio": 0.053, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.205, 0.076, 0.086], [0.694, 0.479, 0.307], diff --git a/datafusion-partitioned/results/20260511/c6a.2xlarge.json b/datafusion-partitioned/results/20260511/c6a.2xlarge.json index 5ffea5fe28..5a9e1d1a42 100644 --- a/datafusion-partitioned/results/20260511/c6a.2xlarge.json +++ b/datafusion-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 8, "data_size": 14737666736, - "concurrent_qps": 0.062, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.024, 0.002, 0.002], [0.114, 0.036, 0.038], diff --git a/datafusion-partitioned/results/20260511/c6a.4xlarge.json b/datafusion-partitioned/results/20260511/c6a.4xlarge.json index 366d9d09e2..daa19b16dc 100644 --- a/datafusion-partitioned/results/20260511/c6a.4xlarge.json +++ b/datafusion-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 17, "data_size": 14737666736, - "concurrent_qps": 0.097, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.02, 0.002, 0.002], [0.079, 0.021, 0.022], diff --git a/datafusion-partitioned/results/20260511/c6a.metal.json b/datafusion-partitioned/results/20260511/c6a.metal.json index bf0f0bac7e..b1e13f47f2 100644 --- a/datafusion-partitioned/results/20260511/c6a.metal.json +++ b/datafusion-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 51, "data_size": 14737666736, - "concurrent_qps": 4.915, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.037, 0.003, 0.002], [0.105, 0.036, 0.036], diff --git a/datafusion-partitioned/results/20260511/c6a.xlarge.json b/datafusion-partitioned/results/20260511/c6a.xlarge.json index 6bfb4e4288..7e0558b5fb 100644 --- a/datafusion-partitioned/results/20260511/c6a.xlarge.json +++ b/datafusion-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 3, "data_size": 14737666736, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.022, 0.002, 0.002], [0.139, 0.063, 0.062], diff --git a/datafusion-partitioned/results/20260511/c7a.metal-48xl.json b/datafusion-partitioned/results/20260511/c7a.metal-48xl.json index 649e626a61..ec4dec3f18 100644 --- a/datafusion-partitioned/results/20260511/c7a.metal-48xl.json +++ b/datafusion-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 54, "data_size": 14737666736, - "concurrent_qps": 7.017, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.018, 0.003, 0.002], [0.07, 0.039, 0.042], diff --git a/datafusion-partitioned/results/20260511/c8g.4xlarge.json b/datafusion-partitioned/results/20260511/c8g.4xlarge.json index 5b79c7315f..22db69ba25 100644 --- a/datafusion-partitioned/results/20260511/c8g.4xlarge.json +++ b/datafusion-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 18, "data_size": 14737666736, - "concurrent_qps": 0.095, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.015, 0.002, 0.002], [0.075, 0.014, 0.012], diff --git a/datafusion-partitioned/results/20260511/c8g.metal-48xl.json b/datafusion-partitioned/results/20260511/c8g.metal-48xl.json index e9f2981bd1..7502a879fc 100644 --- a/datafusion-partitioned/results/20260511/c8g.metal-48xl.json +++ b/datafusion-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 45, "data_size": 14737666736, - "concurrent_qps": 6.498, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.013, 0.002, 0.002], [0.058, 0.018, 0.019], diff --git a/datafusion-vortex-partitioned/benchmark.sh b/datafusion-vortex-partitioned/benchmark.sh index 51153ec49e..6bbd1cebbe 100755 --- a/datafusion-vortex-partitioned/benchmark.sh +++ b/datafusion-vortex-partitioned/benchmark.sh @@ -4,4 +4,9 @@ export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/datafusion-vortex-partitioned/results/20260511/c6a.2xlarge.json b/datafusion-vortex-partitioned/results/20260511/c6a.2xlarge.json index 9899ec8c22..d79c49726c 100644 --- a/datafusion-vortex-partitioned/results/20260511/c6a.2xlarge.json +++ b/datafusion-vortex-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 299, "data_size": 15812419580, - "concurrent_qps": 0.022, - "concurrent_error_ratio": 0.519, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.223, 0.026, 0.025], [0.273, 0.049, 0.048], diff --git a/datafusion-vortex-partitioned/results/20260511/c6a.4xlarge.json b/datafusion-vortex-partitioned/results/20260511/c6a.4xlarge.json index fd0f93c816..e72b8ad28b 100644 --- a/datafusion-vortex-partitioned/results/20260511/c6a.4xlarge.json +++ b/datafusion-vortex-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 157, "data_size": 15812419580, - "concurrent_qps": 0.022, - "concurrent_error_ratio": 0.567, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.208, 0.026, 0.026], [0.205, 0.039, 0.04], diff --git a/datafusion-vortex-partitioned/results/20260511/c6a.metal.json b/datafusion-vortex-partitioned/results/20260511/c6a.metal.json index 8ec6088aa5..85e127a69c 100644 --- a/datafusion-vortex-partitioned/results/20260511/c6a.metal.json +++ b/datafusion-vortex-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 129, "data_size": 15812419580, - "concurrent_qps": 0.952, - "concurrent_error_ratio": 0.789, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.186, 0.018, 0.019], [0.165, 0.027, 0.029], diff --git a/datafusion-vortex-partitioned/results/20260511/c6a.xlarge.json b/datafusion-vortex-partitioned/results/20260511/c6a.xlarge.json index cf9d4c08c7..7f6bc39511 100644 --- a/datafusion-vortex-partitioned/results/20260511/c6a.xlarge.json +++ b/datafusion-vortex-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 969, "data_size": 15812419580, - "concurrent_qps": 0.013, - "concurrent_error_ratio": 0.5, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.216, 0.028, 0.027], [0.251, 0.069, 0.069], diff --git a/datafusion-vortex-partitioned/results/20260511/c7a.metal-48xl.json b/datafusion-vortex-partitioned/results/20260511/c7a.metal-48xl.json index c534a95f36..e63a8b7e98 100644 --- a/datafusion-vortex-partitioned/results/20260511/c7a.metal-48xl.json +++ b/datafusion-vortex-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 130, "data_size": 15812419580, - "concurrent_qps": 1.345, - "concurrent_error_ratio": 0.79, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.216, 0.017, 0.016], [0.206, 0.026, 0.029], diff --git a/datafusion-vortex-partitioned/results/20260511/c8g.4xlarge.json b/datafusion-vortex-partitioned/results/20260511/c8g.4xlarge.json index 3d07cea9d0..82deff4eb0 100644 --- a/datafusion-vortex-partitioned/results/20260511/c8g.4xlarge.json +++ b/datafusion-vortex-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 239, "data_size": 15812419580, - "concurrent_qps": 0.028, - "concurrent_error_ratio": 0.646, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.217, 0.009, 0.01], [0.268, 0.018, 0.019], diff --git a/datafusion-vortex/benchmark.sh b/datafusion-vortex/benchmark.sh index dce465f2fc..6dd84d0d38 100755 --- a/datafusion-vortex/benchmark.sh +++ b/datafusion-vortex/benchmark.sh @@ -4,4 +4,9 @@ export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/datafusion/results/20260511/c6a.2xlarge.json b/datafusion/results/20260511/c6a.2xlarge.json index c260fc1fac..d009f18454 100644 --- a/datafusion/results/20260511/c6a.2xlarge.json +++ b/datafusion/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.042, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.026, 0.001, 0.001], [0.121, 0.039, 0.039], diff --git a/datafusion/results/20260511/c6a.4xlarge.json b/datafusion/results/20260511/c6a.4xlarge.json index 8d65274f62..20f9c2abb7 100644 --- a/datafusion/results/20260511/c6a.4xlarge.json +++ b/datafusion/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 10, "data_size": 14779976446, - "concurrent_qps": 0.11, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.027, 0.001, 0.001], [0.114, 0.035, 0.036], diff --git a/datafusion/results/20260511/c6a.metal.json b/datafusion/results/20260511/c6a.metal.json index 42ef6f7a5c..002ccf020e 100644 --- a/datafusion/results/20260511/c6a.metal.json +++ b/datafusion/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 4.017, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.023, 0.001, 0.002], [0.136, 0.076, 0.096], diff --git a/datafusion/results/20260511/c6a.xlarge.json b/datafusion/results/20260511/c6a.xlarge.json index 4245154e85..b40baff230 100644 --- a/datafusion/results/20260511/c6a.xlarge.json +++ b/datafusion/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.026, 0.001, 0.001], [0.165, 0.059, 0.059], diff --git a/datafusion/results/20260511/c7a.metal-48xl.json b/datafusion/results/20260511/c7a.metal-48xl.json index 27c3c0a093..62845a0384 100644 --- a/datafusion/results/20260511/c7a.metal-48xl.json +++ b/datafusion/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 5.888, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.024, 0.001, 0.001], [0.147, 0.101, 0.103], diff --git a/datafusion/results/20260511/c8g.4xlarge.json b/datafusion/results/20260511/c8g.4xlarge.json index f959b3d85f..d00a01b613 100644 --- a/datafusion/results/20260511/c8g.4xlarge.json +++ b/datafusion/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 10, "data_size": 14779976446, - "concurrent_qps": 0.092, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.013, 0.001, 0.001], [0.068, 0.018, 0.017], diff --git a/datafusion/results/20260511/c8g.metal-48xl.json b/datafusion/results/20260511/c8g.metal-48xl.json index 8b28d3f032..5d6cf28b2a 100644 --- a/datafusion/results/20260511/c8g.metal-48xl.json +++ b/datafusion/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 5.388, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.013, 0.001, 0.001], [0.101, 0.067, 0.079], diff --git a/drill/benchmark.sh b/drill/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/drill/benchmark.sh +++ b/drill/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-datalake-partitioned/benchmark.sh b/duckdb-datalake-partitioned/benchmark.sh index 08d52c39fc..9c514c4803 100755 --- a/duckdb-datalake-partitioned/benchmark.sh +++ b/duckdb-datalake-partitioned/benchmark.sh @@ -4,4 +4,9 @@ export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-datalake-partitioned/results/20260511/c6a.2xlarge.json b/duckdb-datalake-partitioned/results/20260511/c6a.2xlarge.json index 96c23dca1a..bdd428bc13 100644 --- a/duckdb-datalake-partitioned/results/20260511/c6a.2xlarge.json +++ b/duckdb-datalake-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.18, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.896, 0.752, 0.808], [2.036, 2.01, 2.055], diff --git a/duckdb-datalake-partitioned/results/20260511/c6a.4xlarge.json b/duckdb-datalake-partitioned/results/20260511/c6a.4xlarge.json index c815c397b1..cf6ca89caf 100644 --- a/duckdb-datalake-partitioned/results/20260511/c6a.4xlarge.json +++ b/duckdb-datalake-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.378, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.659, 0.518, 0.436], [1.207, 1.098, 1.039], diff --git a/duckdb-datalake-partitioned/results/20260511/c6a.large.json b/duckdb-datalake-partitioned/results/20260511/c6a.large.json index d853d06254..8c9aff1155 100644 --- a/duckdb-datalake-partitioned/results/20260511/c6a.large.json +++ b/duckdb-datalake-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14737666736, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [2.504, 2.324, 2.666], [7.67, 7.597, 7.455], diff --git a/duckdb-datalake-partitioned/results/20260511/c6a.metal.json b/duckdb-datalake-partitioned/results/20260511/c6a.metal.json index 3d069c3a14..fc9a581b6d 100644 --- a/duckdb-datalake-partitioned/results/20260511/c6a.metal.json +++ b/duckdb-datalake-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.628, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.619, 0.558, 0.582], [5.483, 0.676, 0.621], diff --git a/duckdb-datalake-partitioned/results/20260511/c6a.xlarge.json b/duckdb-datalake-partitioned/results/20260511/c6a.xlarge.json index 202fea0d7e..e0491519da 100644 --- a/duckdb-datalake-partitioned/results/20260511/c6a.xlarge.json +++ b/duckdb-datalake-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.067, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [1.486, 1.281, 1.474], [4.146, 3.676, 3.939], diff --git a/duckdb-datalake-partitioned/results/20260511/c7a.metal-48xl.json b/duckdb-datalake-partitioned/results/20260511/c7a.metal-48xl.json index c9ec0c2846..d9c282d95a 100644 --- a/duckdb-datalake-partitioned/results/20260511/c7a.metal-48xl.json +++ b/duckdb-datalake-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.76, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.586, 0.547, 0.603], [0.661, 5.358, 5.35], diff --git a/duckdb-datalake-partitioned/results/20260511/c8g.4xlarge.json b/duckdb-datalake-partitioned/results/20260511/c8g.4xlarge.json index cafb0e56aa..926d4f4036 100644 --- a/duckdb-datalake-partitioned/results/20260511/c8g.4xlarge.json +++ b/duckdb-datalake-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14737666736, - "concurrent_qps": 0.482, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.488, 0.484, 0.392], [1.101, 0.922, 0.907], diff --git a/duckdb-datalake-partitioned/results/20260511/c8g.metal-48xl.json b/duckdb-datalake-partitioned/results/20260511/c8g.metal-48xl.json index 1afa745266..cc49d8fac6 100644 --- a/duckdb-datalake-partitioned/results/20260511/c8g.metal-48xl.json +++ b/duckdb-datalake-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14737666736, - "concurrent_qps": 0.842, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.419, 0.402, 0.459], [0.476, 0.439, 0.445], diff --git a/duckdb-datalake-partitioned/results/20260511/t3a.small.json b/duckdb-datalake-partitioned/results/20260511/t3a.small.json index 196ba09b0a..c7d934c3a8 100644 --- a/duckdb-datalake-partitioned/results/20260511/t3a.small.json +++ b/duckdb-datalake-partitioned/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.012, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.187, 3.345, 3.31], [9.425, 9.408, 9.526], diff --git a/duckdb-datalake/benchmark.sh b/duckdb-datalake/benchmark.sh index 08d52c39fc..9c514c4803 100755 --- a/duckdb-datalake/benchmark.sh +++ b/duckdb-datalake/benchmark.sh @@ -4,4 +4,9 @@ export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-datalake/results/20260511/c6a.2xlarge.json b/duckdb-datalake/results/20260511/c6a.2xlarge.json index b0b4975446..b774238688 100644 --- a/duckdb-datalake/results/20260511/c6a.2xlarge.json +++ b/duckdb-datalake/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.245, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.182, 0.162, 0.151], [0.885, 0.822, 0.822], diff --git a/duckdb-datalake/results/20260511/c6a.4xlarge.json b/duckdb-datalake/results/20260511/c6a.4xlarge.json index 0f84b3f723..5853557112 100644 --- a/duckdb-datalake/results/20260511/c6a.4xlarge.json +++ b/duckdb-datalake/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.475, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.184, 0.142, 0.15], [0.567, 0.51, 0.506], diff --git a/duckdb-datalake/results/20260511/c6a.large.json b/duckdb-datalake/results/20260511/c6a.large.json index d13b7affed..5aa3bb7840 100644 --- a/duckdb-datalake/results/20260511/c6a.large.json +++ b/duckdb-datalake/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.043, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.239, 0.216, 0.191], [2.835, 2.884, 2.863], diff --git a/duckdb-datalake/results/20260511/c6a.metal.json b/duckdb-datalake/results/20260511/c6a.metal.json index 05553b8980..2a7a16bca0 100644 --- a/duckdb-datalake/results/20260511/c6a.metal.json +++ b/duckdb-datalake/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.645, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.178, 0.146, 0.124], [5.334, 0.492, 5.279], diff --git a/duckdb-datalake/results/20260511/c6a.xlarge.json b/duckdb-datalake/results/20260511/c6a.xlarge.json index 45866ee4a4..e85f403ca6 100644 --- a/duckdb-datalake/results/20260511/c6a.xlarge.json +++ b/duckdb-datalake/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.045, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.212, 0.169, 0.168], [1.537, 1.508, 1.512], diff --git a/duckdb-datalake/results/20260511/c7a.metal-48xl.json b/duckdb-datalake/results/20260511/c7a.metal-48xl.json index 084e1b9360..2a83920011 100644 --- a/duckdb-datalake/results/20260511/c7a.metal-48xl.json +++ b/duckdb-datalake/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.838, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.165, 0.158, 0.128], [0.53, 0.448, 0.446], diff --git a/duckdb-datalake/results/20260511/c8g.4xlarge.json b/duckdb-datalake/results/20260511/c8g.4xlarge.json index b8ef59ca21..6cb34a397d 100644 --- a/duckdb-datalake/results/20260511/c8g.4xlarge.json +++ b/duckdb-datalake/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.66, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.189, 0.133, 0.124], [0.528, 0.495, 0.487], diff --git a/duckdb-datalake/results/20260511/c8g.metal-48xl.json b/duckdb-datalake/results/20260511/c8g.metal-48xl.json index 79740ca165..0ff3f954be 100644 --- a/duckdb-datalake/results/20260511/c8g.metal-48xl.json +++ b/duckdb-datalake/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.547, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.15, 0.111, 0.147], [5.271, 0.355, 0.342], diff --git a/duckdb-datalake/results/20260511/t3a.small.json b/duckdb-datalake/results/20260511/t3a.small.json index 2864184c74..351ed7db33 100644 --- a/duckdb-datalake/results/20260511/t3a.small.json +++ b/duckdb-datalake/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.013, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.382, 0.338, 0.351], [3.213, 3.143, 3.205], diff --git a/duckdb-parquet-partitioned/benchmark.sh b/duckdb-parquet-partitioned/benchmark.sh index e236dbbd21..5972a9845b 100755 --- a/duckdb-parquet-partitioned/benchmark.sh +++ b/duckdb-parquet-partitioned/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-parquet-partitioned/results/20260511/c6a.2xlarge.json b/duckdb-parquet-partitioned/results/20260511/c6a.2xlarge.json index d6d47f33f2..38b4645cb2 100644 --- a/duckdb-parquet-partitioned/results/20260511/c6a.2xlarge.json +++ b/duckdb-parquet-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 10, "data_size": 14737666736, - "concurrent_qps": 0.383, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.073, 0.038, 0.036], [0.118, 0.048, 0.047], diff --git a/duckdb-parquet-partitioned/results/20260511/c6a.4xlarge.json b/duckdb-parquet-partitioned/results/20260511/c6a.4xlarge.json index 4593218e00..dbce9f8060 100644 --- a/duckdb-parquet-partitioned/results/20260511/c6a.4xlarge.json +++ b/duckdb-parquet-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.917, - "concurrent_error_ratio": 0.996, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.065, 0.034, 0.037], [0.076, 0.027, 0.028], diff --git a/duckdb-parquet-partitioned/results/20260511/c6a.large.json b/duckdb-parquet-partitioned/results/20260511/c6a.large.json index ec3d8050c0..70482ccecc 100644 --- a/duckdb-parquet-partitioned/results/20260511/c6a.large.json +++ b/duckdb-parquet-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.018, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.241, 0.104, 0.107], [0.427, 0.174, 0.176], diff --git a/duckdb-parquet-partitioned/results/20260511/c6a.metal.json b/duckdb-parquet-partitioned/results/20260511/c6a.metal.json index 7262b301a8..0c474c13c3 100644 --- a/duckdb-parquet-partitioned/results/20260511/c6a.metal.json +++ b/duckdb-parquet-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.865, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.378, 0.363, 0.331], [0.076, 0.055, 0.066], diff --git a/duckdb-parquet-partitioned/results/20260511/c6a.xlarge.json b/duckdb-parquet-partitioned/results/20260511/c6a.xlarge.json index a677357011..203ebe03c4 100644 --- a/duckdb-parquet-partitioned/results/20260511/c6a.xlarge.json +++ b/duckdb-parquet-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 5, "data_size": 14737666736, - "concurrent_qps": 0.097, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.13, 0.06, 0.061], [0.212, 0.092, 0.09], diff --git a/duckdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json b/duckdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json index 6691643773..9e848738a7 100644 --- a/duckdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json +++ b/duckdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 2.078, - "concurrent_error_ratio": 0.992, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.308, 0.329, 0.285], [0.083, 0.059, 0.053], diff --git a/duckdb-parquet-partitioned/results/20260511/c8g.4xlarge.json b/duckdb-parquet-partitioned/results/20260511/c8g.4xlarge.json index 0daec49863..310bb94bde 100644 --- a/duckdb-parquet-partitioned/results/20260511/c8g.4xlarge.json +++ b/duckdb-parquet-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 1.597, - "concurrent_error_ratio": 0.996, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.044, 0.02, 0.022], [0.06, 0.016, 0.017], diff --git a/duckdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json b/duckdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json index 0b92bcd99a..a2467e1fc0 100644 --- a/duckdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json +++ b/duckdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 2.442, - "concurrent_error_ratio": 0.994, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.189, 0.18, 0.223], [0.067, 0.057, 0.066], diff --git a/duckdb-parquet-partitioned/results/20260511/t3a.small.json b/duckdb-parquet-partitioned/results/20260511/t3a.small.json index 59f3900f76..2e4033d411 100644 --- a/duckdb-parquet-partitioned/results/20260511/t3a.small.json +++ b/duckdb-parquet-partitioned/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14737666736, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.309, 0.185, 0.195], [0.522, 0.28, 0.285], diff --git a/duckdb-parquet/benchmark.sh b/duckdb-parquet/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/duckdb-parquet/benchmark.sh +++ b/duckdb-parquet/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-parquet/results/20260511/c6a.2xlarge.json b/duckdb-parquet/results/20260511/c6a.2xlarge.json index fadc9dec9d..dd86a1a5ae 100644 --- a/duckdb-parquet/results/20260511/c6a.2xlarge.json +++ b/duckdb-parquet/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.387, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.076, 0.051, 0.051], [0.146, 0.083, 0.081], diff --git a/duckdb-parquet/results/20260511/c6a.4xlarge.json b/duckdb-parquet/results/20260511/c6a.4xlarge.json index 1609298683..bcbf648c83 100644 --- a/duckdb-parquet/results/20260511/c6a.4xlarge.json +++ b/duckdb-parquet/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 0.88, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.079, 0.053, 0.052], [0.12, 0.069, 0.07], diff --git a/duckdb-parquet/results/20260511/c6a.large.json b/duckdb-parquet/results/20260511/c6a.large.json index 1b87e3f105..234fab3f24 100644 --- a/duckdb-parquet/results/20260511/c6a.large.json +++ b/duckdb-parquet/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.079, 0.053, 0.052], [0.29, 0.168, 0.165], diff --git a/duckdb-parquet/results/20260511/c6a.metal.json b/duckdb-parquet/results/20260511/c6a.metal.json index 8a275137e2..8ed389f4f6 100644 --- a/duckdb-parquet/results/20260511/c6a.metal.json +++ b/duckdb-parquet/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 1.815, - "concurrent_error_ratio": 0.994, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.072, 0.05, 0.051], [0.122, 0.079, 0.077], diff --git a/duckdb-parquet/results/20260511/c6a.xlarge.json b/duckdb-parquet/results/20260511/c6a.xlarge.json index 453fe02b5c..48802cb44f 100644 --- a/duckdb-parquet/results/20260511/c6a.xlarge.json +++ b/duckdb-parquet/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.045, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.073, 0.055, 0.055], [0.193, 0.114, 0.113], diff --git a/duckdb-parquet/results/20260511/c7a.metal-48xl.json b/duckdb-parquet/results/20260511/c7a.metal-48xl.json index 7e23cbc8b2..cb6d5b01fb 100644 --- a/duckdb-parquet/results/20260511/c7a.metal-48xl.json +++ b/duckdb-parquet/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 2.057, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.072, 0.044, 0.046], [0.112, 0.07, 0.073], diff --git a/duckdb-parquet/results/20260511/c8g.4xlarge.json b/duckdb-parquet/results/20260511/c8g.4xlarge.json index 832c26886a..89719e70ba 100644 --- a/duckdb-parquet/results/20260511/c8g.4xlarge.json +++ b/duckdb-parquet/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 1.463, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.055, 0.039, 0.039], [0.097, 0.049, 0.049], diff --git a/duckdb-parquet/results/20260511/c8g.metal-48xl.json b/duckdb-parquet/results/20260511/c8g.metal-48xl.json index c10a8750d0..4d4c2e9dfb 100644 --- a/duckdb-parquet/results/20260511/c8g.metal-48xl.json +++ b/duckdb-parquet/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 2.61, - "concurrent_error_ratio": 0.994, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.051, 0.043, 0.04], [0.094, 0.07, 0.076], diff --git a/duckdb-parquet/results/20260511/t3a.small.json b/duckdb-parquet/results/20260511/t3a.small.json index 6b284d92bf..bd98cdb3f5 100644 --- a/duckdb-parquet/results/20260511/t3a.small.json +++ b/duckdb-parquet/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.018, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.145, 0.114, 0.114], [0.473, 0.301, 0.304], diff --git a/duckdb-vortex-partitioned/benchmark.sh b/duckdb-vortex-partitioned/benchmark.sh index e236dbbd21..5972a9845b 100755 --- a/duckdb-vortex-partitioned/benchmark.sh +++ b/duckdb-vortex-partitioned/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-vortex/benchmark.sh b/duckdb-vortex/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/duckdb-vortex/benchmark.sh +++ b/duckdb-vortex/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-vortex/results/20260511/c6a.2xlarge.json b/duckdb-vortex/results/20260511/c6a.2xlarge.json index c83e07ead4..b816f92e58 100644 --- a/duckdb-vortex/results/20260511/c6a.2xlarge.json +++ b/duckdb-vortex/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 197, "data_size": 15731820628, - "concurrent_qps": 0.198, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.042, 0.027, 0.027], [0.062, 0.04, 0.04], diff --git a/duckdb-vortex/results/20260511/c6a.4xlarge.json b/duckdb-vortex/results/20260511/c6a.4xlarge.json index 5dd6110744..ddf7d6c98b 100644 --- a/duckdb-vortex/results/20260511/c6a.4xlarge.json +++ b/duckdb-vortex/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 141, "data_size": 15731820628, - "concurrent_qps": 0.735, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.043, 0.027, 0.027], [0.054, 0.032, 0.032], diff --git a/duckdb-vortex/results/20260511/c6a.metal.json b/duckdb-vortex/results/20260511/c6a.metal.json index 1d8ee97a39..0c83d39479 100644 --- a/duckdb-vortex/results/20260511/c6a.metal.json +++ b/duckdb-vortex/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 222, "data_size": 15731820628, - "concurrent_qps": 0.665, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.048, 0.031, 0.032], [0.082, 0.059, 0.061], diff --git a/duckdb-vortex/results/20260511/c7a.metal-48xl.json b/duckdb-vortex/results/20260511/c7a.metal-48xl.json index 078b2c45d6..0fb50ab47b 100644 --- a/duckdb-vortex/results/20260511/c7a.metal-48xl.json +++ b/duckdb-vortex/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 292, "data_size": 15731820628, - "concurrent_qps": 0.585, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.057, 0.029, 0.03], [0.095, 0.057, 0.057], diff --git a/duckdb-vortex/results/20260511/c8g.4xlarge.json b/duckdb-vortex/results/20260511/c8g.4xlarge.json index af970a7b08..cd32923912 100644 --- a/duckdb-vortex/results/20260511/c8g.4xlarge.json +++ b/duckdb-vortex/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 118, "data_size": 15731820628, - "concurrent_qps": 0.977, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.042, 0.022, 0.022], [0.05, 0.021, 0.021], diff --git a/duckdb-vortex/results/20260511/c8g.metal-48xl.json b/duckdb-vortex/results/20260511/c8g.metal-48xl.json index a464de31d2..c237ffc1a6 100644 --- a/duckdb-vortex/results/20260511/c8g.metal-48xl.json +++ b/duckdb-vortex/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 148, "data_size": 15731820628, - "concurrent_qps": 0.913, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.037, 0.024, 0.025], [0.06, 0.046, 0.04], diff --git a/duckdb/benchmark.sh b/duckdb/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/duckdb/benchmark.sh +++ b/duckdb/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb/results/20260511/c6a.2xlarge.json b/duckdb/results/20260511/c6a.2xlarge.json index 18f563e1aa..bf9b68850e 100644 --- a/duckdb/results/20260511/c6a.2xlarge.json +++ b/duckdb/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 208, "data_size": 20436758528, - "concurrent_qps": 0.337, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.027, 0.018, 0.018], [0.202, 0.052, 0.051], diff --git a/duckdb/results/20260511/c6a.4xlarge.json b/duckdb/results/20260511/c6a.4xlarge.json index e0c5d0e34f..8f7dbe2d39 100644 --- a/duckdb/results/20260511/c6a.4xlarge.json +++ b/duckdb/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 126, "data_size": 20457467904, - "concurrent_qps": 0.705, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.03, 0.018, 0.018], [0.132, 0.041, 0.041], diff --git a/duckdb/results/20260511/c6a.large.json b/duckdb/results/20260511/c6a.large.json index 7c90fdd4cb..308ddeae2e 100644 --- a/duckdb/results/20260511/c6a.large.json +++ b/duckdb/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 508, "data_size": 20422340608, - "concurrent_qps": 0.03, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.026, 0.018, 0.018], [0.559, 0.092, 0.092], diff --git a/duckdb/results/20260511/c6a.metal.json b/duckdb/results/20260511/c6a.metal.json index 91d377e83c..429ed173d7 100644 --- a/duckdb/results/20260511/c6a.metal.json +++ b/duckdb/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 102, "data_size": 20635725824, - "concurrent_qps": 1.258, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.024, 0.017, 0.018], [0.082, 0.045, 0.043], diff --git a/duckdb/results/20260511/c6a.xlarge.json b/duckdb/results/20260511/c6a.xlarge.json index 5181050654..e4cbb8da1e 100644 --- a/duckdb/results/20260511/c6a.xlarge.json +++ b/duckdb/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 279, "data_size": 20429680640, - "concurrent_qps": 0.138, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.029, 0.018, 0.019], [0.324, 0.067, 0.068], diff --git a/duckdb/results/20260511/c7a.metal-48xl.json b/duckdb/results/20260511/c7a.metal-48xl.json index 6f24176b5e..f69b4c5a67 100644 --- a/duckdb/results/20260511/c7a.metal-48xl.json +++ b/duckdb/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 98, "data_size": 20636512256, - "concurrent_qps": 1.553, - "concurrent_error_ratio": 0.992, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.025, 0.017, 0.017], [0.098, 0.041, 0.04], diff --git a/duckdb/results/20260511/c8g.4xlarge.json b/duckdb/results/20260511/c8g.4xlarge.json index 9159fc9f67..e389a496a9 100644 --- a/duckdb/results/20260511/c8g.4xlarge.json +++ b/duckdb/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 120, "data_size": 20462448640, - "concurrent_qps": 1.067, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.021, 0.013, 0.013], [0.083, 0.025, 0.026], diff --git a/duckdb/results/20260511/c8g.metal-48xl.json b/duckdb/results/20260511/c8g.metal-48xl.json index d5bc8ca9c9..ca2540849d 100644 --- a/duckdb/results/20260511/c8g.metal-48xl.json +++ b/duckdb/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 96, "data_size": 20639133696, - "concurrent_qps": 2.1, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.017, 0.014, 0.015], [0.059, 0.032, 0.034], diff --git a/gendb/benchmark.sh b/gendb/benchmark.sh index 31b5bc717a..68314250b9 100755 --- a/gendb/benchmark.sh +++ b/gendb/benchmark.sh @@ -12,4 +12,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no export BENCH_DURABLE=yes +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/gendb/results/20260518/c6a.2xlarge.json b/gendb/results/20260518/c6a.2xlarge.json index 291ae31804..1ead20d209 100644 --- a/gendb/results/20260518/c6a.2xlarge.json +++ b/gendb/results/20260518/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 180, "data_size": 37351308970, - "concurrent_qps": 0.047, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0, 0, 0], [0.738, 0.015, 0.015], diff --git a/gendb/results/20260518/c6a.4xlarge.json b/gendb/results/20260518/c6a.4xlarge.json index 7221abb721..14c674deef 100644 --- a/gendb/results/20260518/c6a.4xlarge.json +++ b/gendb/results/20260518/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 166, "data_size": 37351308970, - "concurrent_qps": 0.093, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0, 0, 0], [0.638, 0.016, 0.015], diff --git a/gendb/results/20260518/c6a.metal.json b/gendb/results/20260518/c6a.metal.json index 818e9aa7f2..2a251de1dd 100644 --- a/gendb/results/20260518/c6a.metal.json +++ b/gendb/results/20260518/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 177, "data_size": 37351308970, - "concurrent_qps": 2.002, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0, 0, 0], [0.663, 0.027, 0.031], diff --git a/gendb/results/20260518/c7a.metal-48xl.json b/gendb/results/20260518/c7a.metal-48xl.json index 4f4940b576..766c6c40db 100644 --- a/gendb/results/20260518/c7a.metal-48xl.json +++ b/gendb/results/20260518/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 181, "data_size": 37351308970, - "concurrent_qps": 2.4, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0, 0, 0], [0.791, 0.022, 0.022], diff --git a/gendb/results/20260518/c8g.4xlarge.json b/gendb/results/20260518/c8g.4xlarge.json index 5e6ed85260..80ad9f233f 100644 --- a/gendb/results/20260518/c8g.4xlarge.json +++ b/gendb/results/20260518/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 162, "data_size": 37351308970, - "concurrent_qps": 0.175, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0, 0, 0], [0.887, 0.006, 0.005], diff --git a/gendb/results/20260518/c8g.metal-48xl.json b/gendb/results/20260518/c8g.metal-48xl.json index 336684bbcb..706301c352 100644 --- a/gendb/results/20260518/c8g.metal-48xl.json +++ b/gendb/results/20260518/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 173, "data_size": 37351308970, - "concurrent_qps": 3.41, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0, 0, 0], [0.613, 0.008, 0.008], diff --git a/glaredb-partitioned/benchmark.sh b/glaredb-partitioned/benchmark.sh index e236dbbd21..5972a9845b 100755 --- a/glaredb-partitioned/benchmark.sh +++ b/glaredb-partitioned/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/glaredb-partitioned/results/20260511/c6a.2xlarge.json b/glaredb-partitioned/results/20260511/c6a.2xlarge.json index 51c7287e2d..5e9109a673 100644 --- a/glaredb-partitioned/results/20260511/c6a.2xlarge.json +++ b/glaredb-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 9, "data_size": 14737666736, - "concurrent_qps": 0.04, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.06, 0.046, 0.042], [0.208, 0.184, 0.183], diff --git a/glaredb-partitioned/results/20260511/c6a.4xlarge.json b/glaredb-partitioned/results/20260511/c6a.4xlarge.json index dc1dba0f0e..cd8cb04fb0 100644 --- a/glaredb-partitioned/results/20260511/c6a.4xlarge.json +++ b/glaredb-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.052, - "concurrent_error_ratio": 0.031, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.03, 0.02, 0.028], [0.116, 0.098, 0.12], diff --git a/glaredb-partitioned/results/20260511/c6a.large.json b/glaredb-partitioned/results/20260511/c6a.large.json index abc75a0157..0be07f7d02 100644 --- a/glaredb-partitioned/results/20260511/c6a.large.json +++ b/glaredb-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.027, - "concurrent_error_ratio": 0.059, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.235, 0.13, 0.129], [0.817, 0.711, 0.712], diff --git a/glaredb-partitioned/results/20260511/c6a.metal.json b/glaredb-partitioned/results/20260511/c6a.metal.json index 97a82f5adf..1052e25c90 100644 --- a/glaredb-partitioned/results/20260511/c6a.metal.json +++ b/glaredb-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.705, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.047, 0.062, 0.057], [0.081, 0.077, 0.077], diff --git a/glaredb-partitioned/results/20260511/c7a.metal-48xl.json b/glaredb-partitioned/results/20260511/c7a.metal-48xl.json index faeba470ea..f479d7dc1a 100644 --- a/glaredb-partitioned/results/20260511/c7a.metal-48xl.json +++ b/glaredb-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 40, "data_size": 14737666736, - "concurrent_qps": 1.987, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.065, 0.06, 0.068], [0.082, 0.072, 0.081], diff --git a/glaredb-partitioned/results/20260511/c8g.4xlarge.json b/glaredb-partitioned/results/20260511/c8g.4xlarge.json index 3ca7be616c..3bc2a52975 100644 --- a/glaredb-partitioned/results/20260511/c8g.4xlarge.json +++ b/glaredb-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.07, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.045, 0.038, 0.036], [0.109, 0.058, 0.075], diff --git a/glaredb-partitioned/results/20260511/c8g.metal-48xl.json b/glaredb-partitioned/results/20260511/c8g.metal-48xl.json index 24e7db0671..15bf3d3da7 100644 --- a/glaredb-partitioned/results/20260511/c8g.metal-48xl.json +++ b/glaredb-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.373, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.071, 0.068, 0.065], [0.067, 0.075, 0.073], diff --git a/glaredb/benchmark.sh b/glaredb/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/glaredb/benchmark.sh +++ b/glaredb/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/glaredb/results/20260511/c6a.2xlarge.json b/glaredb/results/20260511/c6a.2xlarge.json index 279c0c8e3f..66821a9212 100644 --- a/glaredb/results/20260511/c6a.2xlarge.json +++ b/glaredb/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.037, - "concurrent_error_ratio": 0.043, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.062, 0.057, 0.053], [0.208, 0.186, 0.203], diff --git a/glaredb/results/20260511/c6a.4xlarge.json b/glaredb/results/20260511/c6a.4xlarge.json index 3e3a1e2bda..d97574d6ac 100644 --- a/glaredb/results/20260511/c6a.4xlarge.json +++ b/glaredb/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.052, - "concurrent_error_ratio": 0.061, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.045, 0.036, 0.037], [0.119, 0.126, 0.121], diff --git a/glaredb/results/20260511/c6a.large.json b/glaredb/results/20260511/c6a.large.json index 5ebc43ae95..4488056e8e 100644 --- a/glaredb/results/20260511/c6a.large.json +++ b/glaredb/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.028, - "concurrent_error_ratio": 0.056, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.121, 0.114, 0.115], [0.698, 0.643, 0.645], diff --git a/glaredb/results/20260511/c6a.metal.json b/glaredb/results/20260511/c6a.metal.json index 2a82a2b523..13b0eca240 100644 --- a/glaredb/results/20260511/c6a.metal.json +++ b/glaredb/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 1.6680000000000001, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.052, 0.066, 0.046], [0.077, 0.081, 0.071], diff --git a/glaredb/results/20260511/c7a.metal-48xl.json b/glaredb/results/20260511/c7a.metal-48xl.json index c0cba3770e..aab081cc5d 100644 --- a/glaredb/results/20260511/c7a.metal-48xl.json +++ b/glaredb/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 1.817, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.045, 0.058, 0.057], [0.063, 0.063, 0.073], diff --git a/glaredb/results/20260511/c8g.4xlarge.json b/glaredb/results/20260511/c8g.4xlarge.json index 7c7daa0086..9b889cfaed 100644 --- a/glaredb/results/20260511/c8g.4xlarge.json +++ b/glaredb/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.063, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.053, 0.05, 0.057], [0.087, 0.072, 0.071], diff --git a/glaredb/results/20260511/c8g.metal-48xl.json b/glaredb/results/20260511/c8g.metal-48xl.json index 13f94099cd..f5acc6c224 100644 --- a/glaredb/results/20260511/c8g.metal-48xl.json +++ b/glaredb/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 1.383, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.054, 0.053, 0.057], [0.059, 0.056, 0.058], diff --git a/hyper-parquet/benchmark.sh b/hyper-parquet/benchmark.sh index e236dbbd21..5972a9845b 100755 --- a/hyper-parquet/benchmark.sh +++ b/hyper-parquet/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/hyper-parquet/results/20260511/c6a.2xlarge.json b/hyper-parquet/results/20260511/c6a.2xlarge.json index 0065cb4691..41074c43ff 100644 --- a/hyper-parquet/results/20260511/c6a.2xlarge.json +++ b/hyper-parquet/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","stateless"], "load_time": 10, "data_size": 14737666736, - "concurrent_qps": 0.04, - "concurrent_error_ratio": 0.226, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.867, 0.397, 0.386], [0.916, 0.413, 0.415], diff --git a/hyper-parquet/results/20260511/c6a.4xlarge.json b/hyper-parquet/results/20260511/c6a.4xlarge.json index 5cd1500acc..24b68a6ba8 100644 --- a/hyper-parquet/results/20260511/c6a.4xlarge.json +++ b/hyper-parquet/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.095, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.668, 0.321, 0.309], [0.696, 0.305, 0.345], diff --git a/hyper-parquet/results/20260511/c6a.metal.json b/hyper-parquet/results/20260511/c6a.metal.json index 33d48d6166..eb2842709a 100644 --- a/hyper-parquet/results/20260511/c6a.metal.json +++ b/hyper-parquet/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","stateless"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 5.122, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.599, 0.261, 0.287], [0.649, 0.294, 0.297], diff --git a/hyper-parquet/results/20260511/c6a.xlarge.json b/hyper-parquet/results/20260511/c6a.xlarge.json index 9bd89c84a6..033fbc6ed7 100644 --- a/hyper-parquet/results/20260511/c6a.xlarge.json +++ b/hyper-parquet/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","stateless"], "load_time": 4, "data_size": 14737666736, - "concurrent_qps": 0.018, - "concurrent_error_ratio": 0.389, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [1.204, 0.55, 0.604], [1.339, 0.659, 0.622], diff --git a/hyper-parquet/results/20260511/c7a.metal-48xl.json b/hyper-parquet/results/20260511/c7a.metal-48xl.json index 91db482708..db84ffb0c9 100644 --- a/hyper-parquet/results/20260511/c7a.metal-48xl.json +++ b/hyper-parquet/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","stateless"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 6.622, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.628, 0.252, 0.247], [0.644, 0.252, 0.253], diff --git a/hyper/benchmark.sh b/hyper/benchmark.sh index 2e5741b394..bf02868043 100755 --- a/hyper/benchmark.sh +++ b/hyper/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/hyper/results/20260511/c6a.2xlarge.json b/hyper/results/20260511/c6a.2xlarge.json index 509e57d0de..371e0c65f9 100644 --- a/hyper/results/20260511/c6a.2xlarge.json +++ b/hyper/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 638, "data_size": 18959040512, - "concurrent_qps": 0.305, - "concurrent_error_ratio": 0.991, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.064, 0.041, 0.038], [0.173, 0.041, 0.04], diff --git a/hyper/results/20260511/c6a.4xlarge.json b/hyper/results/20260511/c6a.4xlarge.json index e9ae99397d..3f188d0c22 100644 --- a/hyper/results/20260511/c6a.4xlarge.json +++ b/hyper/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 655, "data_size": 18959040512, - "concurrent_qps": 0.793, - "concurrent_error_ratio": 0.981, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.06, 0.022, 0.022], [0.109, 0.03, 0.023], diff --git a/hyper/results/20260511/c6a.large.json b/hyper/results/20260511/c6a.large.json index bb33d932a0..128df8ac96 100644 --- a/hyper/results/20260511/c6a.large.json +++ b/hyper/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 750, "data_size": 18959040512, - "concurrent_qps": 0.01, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.094, 0.056, 0.054], [0.541, 0.105, 0.104], diff --git a/hyper/results/20260511/c6a.metal.json b/hyper/results/20260511/c6a.metal.json index a5b58206ff..4104b44892 100644 --- a/hyper/results/20260511/c6a.metal.json +++ b/hyper/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 428, "data_size": 18959040512, - "concurrent_qps": 1.74, - "concurrent_error_ratio": 0.963, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.044, 0.014, 0.014], [0.075, 0.029, 0.028], diff --git a/hyper/results/20260511/c6a.xlarge.json b/hyper/results/20260511/c6a.xlarge.json index 5c35cbccf0..a68c5ccc50 100644 --- a/hyper/results/20260511/c6a.xlarge.json +++ b/hyper/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 623, "data_size": 18959040512, - "concurrent_qps": 0.017, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.073, 0.035, 0.035], [0.285, 0.058, 0.059], diff --git a/hyper/results/20260511/c7a.metal-48xl.json b/hyper/results/20260511/c7a.metal-48xl.json index 84321e5693..219066b6c0 100644 --- a/hyper/results/20260511/c7a.metal-48xl.json +++ b/hyper/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 356, "data_size": 18959040512, - "concurrent_qps": 2.01, - "concurrent_error_ratio": 0.963, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.04, 0.011, 0.012], [0.083, 0.026, 0.025], diff --git a/kdb/benchmark.sh b/kdb/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/kdb/benchmark.sh +++ b/kdb/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/lib/benchmark-common.sh b/lib/benchmark-common.sh index 5e796ba6ff..49f47be4e3 100755 --- a/lib/benchmark-common.sh +++ b/lib/benchmark-common.sh @@ -55,7 +55,16 @@ # that runs after the cold/warm sweep. Default 10. # BENCH_CONCURRENT_DURATION # Wall-clock window for the QPS test, in seconds. -# Default 600. +# Default 600. Set to 0 to skip the test (emits +# null). Single-process engines should set 0: each +# query forks a fresh full-machine process with no +# shared scheduler, so N concurrent connections only +# oversubscribe RAM instead of measuring throughput. +# The rule: skip when ./start launches no shared +# server (the embedded CLIs and spark variants), keep +# for daemons and the in-process server wrappers +# (pandas/polars/*-dataframe), which do share one +# process. See issue #946. # BENCH_CONCURRENT_SEED Seed shared across systems so that connection # N hits the same query order on every engine # (the per-connection permutation is derived @@ -324,6 +333,15 @@ bench_concurrent_qps() { local duration="${BENCH_CONCURRENT_DURATION}" local seed="${BENCH_CONCURRENT_SEED}" + # Duration 0 means the test is disabled (single-process engines set this; + # see the BENCH_CONCURRENT_DURATION docs). Emit null and skip the restart, + # cache flush, and worker setup entirely. + if [ "$duration" -le 0 ]; then + echo "Concurrent QPS: null" + echo "Concurrent error ratio: null" + return 0 + fi + # Read the same query file that bench_run_query consumed. local queries=() q while IFS= read -r q; do diff --git a/octosql/benchmark.sh b/octosql/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/octosql/benchmark.sh +++ b/octosql/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/opteryx/benchmark.sh b/opteryx/benchmark.sh index e236dbbd21..5972a9845b 100755 --- a/opteryx/benchmark.sh +++ b/opteryx/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/opteryx/results/20260511/c6a.2xlarge.json b/opteryx/results/20260511/c6a.2xlarge.json index 398482794c..1c9951b158 100644 --- a/opteryx/results/20260511/c6a.2xlarge.json +++ b/opteryx/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["stateless","column-oriented","embedded"], "load_time": 10, "data_size": 14737666736, - "concurrent_qps": 0.035, - "concurrent_error_ratio": 0.087, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.628, 0.497, 0.499], [1.18, 0.723, 0.728], diff --git a/opteryx/results/20260511/c6a.4xlarge.json b/opteryx/results/20260511/c6a.4xlarge.json index 58b68bb466..b3a00faa06 100644 --- a/opteryx/results/20260511/c6a.4xlarge.json +++ b/opteryx/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["stateless","column-oriented","embedded"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.04, - "concurrent_error_ratio": 0.04, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.631, 0.496, 0.497], [1.181, 0.723, 0.714], diff --git a/opteryx/results/20260511/c6a.large.json b/opteryx/results/20260511/c6a.large.json index 370e668a29..10681a4663 100644 --- a/opteryx/results/20260511/c6a.large.json +++ b/opteryx/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["stateless","column-oriented","embedded"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.015, - "concurrent_error_ratio": 0.308, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.659, 0.494, 0.494], [1.289, 0.866, 0.871], diff --git a/opteryx/results/20260511/c6a.metal.json b/opteryx/results/20260511/c6a.metal.json index a5146ec2fc..4ab59bf091 100644 --- a/opteryx/results/20260511/c6a.metal.json +++ b/opteryx/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["stateless","column-oriented","embedded"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 0.875, - "concurrent_error_ratio": 0.047, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.58, 0.46, 0.473], [1.192, 0.767, 0.74], diff --git a/opteryx/results/20260511/c6a.xlarge.json b/opteryx/results/20260511/c6a.xlarge.json index 79d64349f8..f644feec27 100644 --- a/opteryx/results/20260511/c6a.xlarge.json +++ b/opteryx/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["stateless","column-oriented","embedded"], "load_time": 4, "data_size": 14737666736, - "concurrent_qps": 0.023, - "concurrent_error_ratio": 0.125, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.705, 0.511, 0.502], [1.321, 0.768, 0.764], diff --git a/opteryx/results/20260511/c7a.metal-48xl.json b/opteryx/results/20260511/c7a.metal-48xl.json index a7e76e339f..ec645afdc7 100644 --- a/opteryx/results/20260511/c7a.metal-48xl.json +++ b/opteryx/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["stateless","column-oriented","embedded"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 0.993, - "concurrent_error_ratio": 0.046, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.531, 0.435, 0.434], [1.135, 0.678, 0.702], diff --git a/sail-partitioned/benchmark.sh b/sail-partitioned/benchmark.sh index e236dbbd21..5972a9845b 100755 --- a/sail-partitioned/benchmark.sh +++ b/sail-partitioned/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/sail-partitioned/results/20260511/c6a.2xlarge.json b/sail-partitioned/results/20260511/c6a.2xlarge.json index 741f73159b..bb227ae2c2 100644 --- a/sail-partitioned/results/20260511/c6a.2xlarge.json +++ b/sail-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 10, "data_size": 14737666736, - "concurrent_qps": 0.045, - "concurrent_error_ratio": 0.036, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.257, 0.082, 0.083], [0.338, 0.119, 0.118], diff --git a/sail-partitioned/results/20260511/c6a.4xlarge.json b/sail-partitioned/results/20260511/c6a.4xlarge.json index 1b077b8d0e..35da4a0c0e 100644 --- a/sail-partitioned/results/20260511/c6a.4xlarge.json +++ b/sail-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.223, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.258, 0.086, 0.08], [0.325, 0.102, 0.102], diff --git a/sail-partitioned/results/20260511/c6a.large.json b/sail-partitioned/results/20260511/c6a.large.json index aec3b7dfb4..fb12c80852 100644 --- a/sail-partitioned/results/20260511/c6a.large.json +++ b/sail-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 0.167, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.262, 0.089, 0.087], [0.477, 0.213, 0.211], diff --git a/sail-partitioned/results/20260511/c6a.metal.json b/sail-partitioned/results/20260511/c6a.metal.json index 3cdd77b5ca..705ff53c6e 100644 --- a/sail-partitioned/results/20260511/c6a.metal.json +++ b/sail-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 3.038, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.248, 0.086, 0.082], [0.297, 0.12, 0.124], diff --git a/sail-partitioned/results/20260511/c6a.xlarge.json b/sail-partitioned/results/20260511/c6a.xlarge.json index 17028d4665..7de462da60 100644 --- a/sail-partitioned/results/20260511/c6a.xlarge.json +++ b/sail-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 4, "data_size": 14737666736, - "concurrent_qps": 0.035, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.34, 0.084, 0.084], [0.473, 0.152, 0.156], diff --git a/sail-partitioned/results/20260511/c7a.metal-48xl.json b/sail-partitioned/results/20260511/c7a.metal-48xl.json index be9a605f48..0def127fe0 100644 --- a/sail-partitioned/results/20260511/c7a.metal-48xl.json +++ b/sail-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 3.735, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.23, 0.075, 0.075], [0.301, 0.11, 0.107], diff --git a/sail-partitioned/results/20260511/c8g.4xlarge.json b/sail-partitioned/results/20260511/c8g.4xlarge.json index d829f15d15..300ea1de67 100644 --- a/sail-partitioned/results/20260511/c8g.4xlarge.json +++ b/sail-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.317, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.215, 0.072, 0.076], [0.275, 0.09, 0.093], diff --git a/sail-partitioned/results/20260511/c8g.metal-48xl.json b/sail-partitioned/results/20260511/c8g.metal-48xl.json index c3613c2d7c..ca427c54fd 100644 --- a/sail-partitioned/results/20260511/c8g.metal-48xl.json +++ b/sail-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 4.01, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.207, 0.073, 0.073], [0.242, 0.096, 0.092], diff --git a/sail-partitioned/results/20260511/t3a.small.json b/sail-partitioned/results/20260511/t3a.small.json index 6790683855..245cbbf5ad 100644 --- a/sail-partitioned/results/20260511/t3a.small.json +++ b/sail-partitioned/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.015, - "concurrent_error_ratio": 0.471, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.365, 0.174, 0.181], [0.681, 0.398, 0.413], diff --git a/sail/benchmark.sh b/sail/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/sail/benchmark.sh +++ b/sail/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/sail/results/20260511/c6a.2xlarge.json b/sail/results/20260511/c6a.2xlarge.json index 09d0cae9f0..b94548d406 100644 --- a/sail/results/20260511/c6a.2xlarge.json +++ b/sail/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.047, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.208, 0.014, 0.014], [0.306, 0.058, 0.058], diff --git a/sail/results/20260511/c6a.4xlarge.json b/sail/results/20260511/c6a.4xlarge.json index 532915b87a..1fce537cf7 100644 --- a/sail/results/20260511/c6a.4xlarge.json +++ b/sail/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.205, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.213, 0.014, 0.014], [0.296, 0.05, 0.051], diff --git a/sail/results/20260511/c6a.large.json b/sail/results/20260511/c6a.large.json index 6f068ebf4b..344fb6a32d 100644 --- a/sail/results/20260511/c6a.large.json +++ b/sail/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.023, - "concurrent_error_ratio": 0.222, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.226, 0.015, 0.019], [0.443, 0.13, 0.128], diff --git a/sail/results/20260511/c6a.metal.json b/sail/results/20260511/c6a.metal.json index 0a1fc1ecf4..1a5fabc9d4 100644 --- a/sail/results/20260511/c6a.metal.json +++ b/sail/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 6, "data_size": 14779976446, - "concurrent_qps": 2.575, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.196, 0.015, 0.017], [0.713, 0.48, 0.475], diff --git a/sail/results/20260511/c6a.xlarge.json b/sail/results/20260511/c6a.xlarge.json index 4194495ad1..65e9a32ed5 100644 --- a/sail/results/20260511/c6a.xlarge.json +++ b/sail/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.21, 0.017, 0.017], [0.354, 0.084, 0.083], diff --git a/sail/results/20260511/c7a.metal-48xl.json b/sail/results/20260511/c7a.metal-48xl.json index c7b5e606d8..af6c02dd5e 100644 --- a/sail/results/20260511/c7a.metal-48xl.json +++ b/sail/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 3.217, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.214, 0.017, 0.015], [0.643, 0.43, 0.428], diff --git a/sail/results/20260511/c8g.4xlarge.json b/sail/results/20260511/c8g.4xlarge.json index 9e9a80222f..22802dbcd1 100644 --- a/sail/results/20260511/c8g.4xlarge.json +++ b/sail/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.275, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.146, 0.013, 0.013], [0.212, 0.041, 0.039], diff --git a/sail/results/20260511/c8g.metal-48xl.json b/sail/results/20260511/c8g.metal-48xl.json index 4a1ff2a775..9167309e3f 100644 --- a/sail/results/20260511/c8g.metal-48xl.json +++ b/sail/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 7, "data_size": 14779976446, - "concurrent_qps": 3.477, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.089, 0.011, 0.011], [0.276, 0.221, 0.274], diff --git a/sail/results/20260511/t3a.small.json b/sail/results/20260511/t3a.small.json index 7b6ccd3366..2a0579bc4a 100644 --- a/sail/results/20260511/t3a.small.json +++ b/sail/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.018, - "concurrent_error_ratio": 0.353, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.249, 0.029, 0.037], [0.679, 0.237, 0.228], diff --git a/spark-auron/benchmark.sh b/spark-auron/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/spark-auron/benchmark.sh +++ b/spark-auron/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark-auron/results/20260511/c6a.2xlarge.json b/spark-auron/results/20260511/c6a.2xlarge.json index 0f73988e60..be77886eb0 100644 --- a/spark-auron/results/20260511/c6a.2xlarge.json +++ b/spark-auron/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.107, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [58.052, 47.978, 47.209], [3.761, 3.519, 3.612], diff --git a/spark-auron/results/20260511/c6a.4xlarge.json b/spark-auron/results/20260511/c6a.4xlarge.json index ca6152fa31..a86f24c0c8 100644 --- a/spark-auron/results/20260511/c6a.4xlarge.json +++ b/spark-auron/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.177, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [60.775, 12.96, 13.058], [3.135, 2.89, 2.951], diff --git a/spark-auron/results/20260511/c6a.large.json b/spark-auron/results/20260511/c6a.large.json index 41b3d7f656..008fef7051 100644 --- a/spark-auron/results/20260511/c6a.large.json +++ b/spark-auron/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.027, - "concurrent_error_ratio": 0.673, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [85.079, 84.822, 86.685], [10.121, 9.197, 9.338], diff --git a/spark-auron/results/20260511/c6a.metal.json b/spark-auron/results/20260511/c6a.metal.json index 6caa46b24c..d61665cb0b 100644 --- a/spark-auron/results/20260511/c6a.metal.json +++ b/spark-auron/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.103, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [75.476, 40.827, 40.45], [39.232, 39.05, 39.061], diff --git a/spark-auron/results/20260511/c6a.xlarge.json b/spark-auron/results/20260511/c6a.xlarge.json index f2a0316ba8..fd6874b142 100644 --- a/spark-auron/results/20260511/c6a.xlarge.json +++ b/spark-auron/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.043, - "concurrent_error_ratio": 0.297, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [65.718, 51.592, 51.302], [5.491, 5.624, 5.38], diff --git a/spark-auron/results/20260511/c7a.metal-48xl.json b/spark-auron/results/20260511/c7a.metal-48xl.json index c277f91722..97eb943d03 100644 --- a/spark-auron/results/20260511/c7a.metal-48xl.json +++ b/spark-auron/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.13, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [73.191, 42.478, 42.628], [40.636, 39.718, 40.684], diff --git a/spark-comet/benchmark.sh b/spark-comet/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/spark-comet/benchmark.sh +++ b/spark-comet/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark-comet/results/20260511/c6a.2xlarge.json b/spark-comet/results/20260511/c6a.2xlarge.json index 6425de687c..a3cbeddedf 100644 --- a/spark-comet/results/20260511/c6a.2xlarge.json +++ b/spark-comet/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.013, - "concurrent_error_ratio": 0.2, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [6.946, 6.925, 6.644], [7.363, 6.723, 7.095], diff --git a/spark-comet/results/20260511/c6a.4xlarge.json b/spark-comet/results/20260511/c6a.4xlarge.json index 27ed3f2046..d07cc8b19a 100644 --- a/spark-comet/results/20260511/c6a.4xlarge.json +++ b/spark-comet/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 10, "data_size": 14779976446, - "concurrent_qps": 0.008, - "concurrent_error_ratio": 0.5, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [4.677, 4.421, 4.342], [4.763, 4.434, 4.607], diff --git a/spark-comet/results/20260511/c6a.large.json b/spark-comet/results/20260511/c6a.large.json index 55a48ef48b..7cf9ed2c7c 100644 --- a/spark-comet/results/20260511/c6a.large.json +++ b/spark-comet/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.01, - "concurrent_error_ratio": 0.4, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [17.778, 17.718, 17.454], [18.684, 18.37, 18.418], diff --git a/spark-comet/results/20260511/c6a.metal.json b/spark-comet/results/20260511/c6a.metal.json index f15016a752..5871048f0e 100644 --- a/spark-comet/results/20260511/c6a.metal.json +++ b/spark-comet/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 0.3, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [5.209, 5.429, 5.368], [5.248, 5.338, 5.354], diff --git a/spark-comet/results/20260511/c6a.xlarge.json b/spark-comet/results/20260511/c6a.xlarge.json index ffa076ce80..387afb4854 100644 --- a/spark-comet/results/20260511/c6a.xlarge.json +++ b/spark-comet/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.013, - "concurrent_error_ratio": 0.2, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [10.699, 9.82, 9.716], [10.407, 10.268, 10.193], diff --git a/spark-comet/results/20260511/c7a.metal-48xl.json b/spark-comet/results/20260511/c7a.metal-48xl.json index 1845d11ce5..0a755c63da 100644 --- a/spark-comet/results/20260511/c7a.metal-48xl.json +++ b/spark-comet/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 3, "data_size": 14779976446, - "concurrent_qps": 0.385, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [5.288, 4.643, 4.884], [5.335, 5.637, 5.122], diff --git a/spark-comet/results/20260511/c8g.4xlarge.json b/spark-comet/results/20260511/c8g.4xlarge.json index 771b2459ee..ec8dfa8d01 100644 --- a/spark-comet/results/20260511/c8g.4xlarge.json +++ b/spark-comet/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 10, "data_size": 14779976446, - "concurrent_qps": 0.005, - "concurrent_error_ratio": 0.7, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.331, 3.082, 3.007], [3.455, 3.137, 3.095], diff --git a/spark-comet/results/20260511/c8g.metal-48xl.json b/spark-comet/results/20260511/c8g.metal-48xl.json index 4eec42226c..641e810cbf 100644 --- a/spark-comet/results/20260511/c8g.metal-48xl.json +++ b/spark-comet/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.342, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [4.542, 4.4, 5.007], [5.498, 5.086, 4.329], diff --git a/spark-gluten/benchmark.sh b/spark-gluten/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/spark-gluten/benchmark.sh +++ b/spark-gluten/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark-gluten/results/20260511/c6a.2xlarge.json b/spark-gluten/results/20260511/c6a.2xlarge.json index 8775fdf0de..bec0041356 100644 --- a/spark-gluten/results/20260511/c6a.2xlarge.json +++ b/spark-gluten/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.11, - "concurrent_error_ratio": 0.029, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [4.095, 3.583, 3.521], [4.278, 3.792, 3.757], diff --git a/spark-gluten/results/20260511/c6a.4xlarge.json b/spark-gluten/results/20260511/c6a.4xlarge.json index 242305549b..83d2781a94 100644 --- a/spark-gluten/results/20260511/c6a.4xlarge.json +++ b/spark-gluten/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 11, "data_size": 14779976446, - "concurrent_qps": 0.183, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.441, 2.905, 2.934], [3.577, 3.132, 3.003], diff --git a/spark-gluten/results/20260511/c6a.large.json b/spark-gluten/results/20260511/c6a.large.json index afea681fa6..36ab22ca2f 100644 --- a/spark-gluten/results/20260511/c6a.large.json +++ b/spark-gluten/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.017, - "concurrent_error_ratio": 0.231, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [10.073, 9.513, 9.553], [10.725, 10.379, 10.318], diff --git a/spark-gluten/results/20260511/c6a.metal.json b/spark-gluten/results/20260511/c6a.metal.json index ac5dc6e2cb..e9ca52372c 100644 --- a/spark-gluten/results/20260511/c6a.metal.json +++ b/spark-gluten/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 0.512, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [5.086, 4.489, 4.714], [5.324, 4.703, 4.831], diff --git a/spark-gluten/results/20260511/c6a.xlarge.json b/spark-gluten/results/20260511/c6a.xlarge.json index d82b516017..fcfb8d38bc 100644 --- a/spark-gluten/results/20260511/c6a.xlarge.json +++ b/spark-gluten/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.032, - "concurrent_error_ratio": 0.345, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [6.157, 5.598, 5.608], [6.366, 5.885, 5.763], diff --git a/spark-gluten/results/20260511/c7a.metal-48xl.json b/spark-gluten/results/20260511/c7a.metal-48xl.json index 2580b3c8e2..edfb90f646 100644 --- a/spark-gluten/results/20260511/c7a.metal-48xl.json +++ b/spark-gluten/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 3, "data_size": 14779976446, - "concurrent_qps": 0.605, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [4.551, 4.001, 3.975], [4.62, 4.068, 4.141], diff --git a/spark-gluten/results/20260511/t3a.small.json b/spark-gluten/results/20260511/t3a.small.json index 94cd0ccbd0..c8e7299b97 100644 --- a/spark-gluten/results/20260511/t3a.small.json +++ b/spark-gluten/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.015, - "concurrent_error_ratio": 0.1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [17.782, 17.036, 17.335], [18.613, 18.098, 17.83], diff --git a/spark-velox/benchmark.sh b/spark-velox/benchmark.sh index 617422ddc2..5427511a01 100755 --- a/spark-velox/benchmark.sh +++ b/spark-velox/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark-velox/results/20260511/c6a.2xlarge.json b/spark-velox/results/20260511/c6a.2xlarge.json index 18d92e93d9..841319da52 100644 --- a/spark-velox/results/20260511/c6a.2xlarge.json +++ b/spark-velox/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 0.108, - "concurrent_error_ratio": 0.03, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [4.092, 3.645, 3.659], [4.371, 3.824, 3.749], diff --git a/spark-velox/results/20260511/c6a.4xlarge.json b/spark-velox/results/20260511/c6a.4xlarge.json index ccb3f1a4fa..cd067734c0 100644 --- a/spark-velox/results/20260511/c6a.4xlarge.json +++ b/spark-velox/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 10, "data_size": 14779976446, - "concurrent_qps": 0.18, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.441, 2.95, 2.937], [3.573, 3.001, 2.978], diff --git a/spark-velox/results/20260511/c6a.large.json b/spark-velox/results/20260511/c6a.large.json index 7eb6a5efdb..5fe86e958b 100644 --- a/spark-velox/results/20260511/c6a.large.json +++ b/spark-velox/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.015, - "concurrent_error_ratio": 0.1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [9.776, 9.458, 9.537], [10.542, 10.205, 10.015], diff --git a/spark-velox/results/20260511/c6a.metal.json b/spark-velox/results/20260511/c6a.metal.json index 3bed8fc13c..f67822c00c 100644 --- a/spark-velox/results/20260511/c6a.metal.json +++ b/spark-velox/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 7, "data_size": 14779976446, - "concurrent_qps": 0.513, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [5.104, 4.608, 4.638], [5.057, 4.653, 4.661], diff --git a/spark-velox/results/20260511/c6a.xlarge.json b/spark-velox/results/20260511/c6a.xlarge.json index 7c57046998..6e5c20da2f 100644 --- a/spark-velox/results/20260511/c6a.xlarge.json +++ b/spark-velox/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.032, - "concurrent_error_ratio": 0.472, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [6.052, 5.51, 5.528], [6.261, 5.842, 5.874], diff --git a/spark-velox/results/20260511/c7a.metal-48xl.json b/spark-velox/results/20260511/c7a.metal-48xl.json index 4a98121608..36aaf20a56 100644 --- a/spark-velox/results/20260511/c7a.metal-48xl.json +++ b/spark-velox/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 0.607, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [4.522, 4.082, 4.224], [4.674, 4.142, 4.258], diff --git a/spark-velox/results/20260511/t3a.small.json b/spark-velox/results/20260511/t3a.small.json index 421d3d7795..230645127b 100644 --- a/spark-velox/results/20260511/t3a.small.json +++ b/spark-velox/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.017, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [19.718, 20.133, 19.386], [21.277, 20.722, 20.455], diff --git a/spark/benchmark.sh b/spark/benchmark.sh index 5c71433ad1..3d8a0f4861 100755 --- a/spark/benchmark.sh +++ b/spark/benchmark.sh @@ -5,4 +5,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" # (and would re-download nothing). Skip restart. export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark/results/20260511/c6a.2xlarge.json b/spark/results/20260511/c6a.2xlarge.json index 80b6e51687..5770597af5 100644 --- a/spark/results/20260511/c6a.2xlarge.json +++ b/spark/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.01, - "concurrent_error_ratio": 0.4, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.912, 3.65, 3.675], [4.21, 3.852, 3.985], diff --git a/spark/results/20260511/c6a.4xlarge.json b/spark/results/20260511/c6a.4xlarge.json index 5b75c4f4ea..e1a4c0141c 100644 --- a/spark/results/20260511/c6a.4xlarge.json +++ b/spark/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.003, - "concurrent_error_ratio": 0.8, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.275, 3.111, 2.948], [3.512, 3.224, 3.245], diff --git a/spark/results/20260511/c6a.large.json b/spark/results/20260511/c6a.large.json index d62d022527..a60b711757 100644 --- a/spark/results/20260511/c6a.large.json +++ b/spark/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.007, - "concurrent_error_ratio": 0.6, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [10.198, 10.136, 9.943], [11.14, 10.324, 10.688], diff --git a/spark/results/20260511/c6a.metal.json b/spark/results/20260511/c6a.metal.json index 236c5e6780..e0f7707eb4 100644 --- a/spark/results/20260511/c6a.metal.json +++ b/spark/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.203, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.511, 3.602, 3.089], [5.46, 17.568, 5.541], diff --git a/spark/results/20260511/c6a.xlarge.json b/spark/results/20260511/c6a.xlarge.json index c68527c916..808cccf706 100644 --- a/spark/results/20260511/c6a.xlarge.json +++ b/spark/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.013, - "concurrent_error_ratio": 0.2, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [5.962, 5.576, 5.768], [6.188, 6.054, 6.545], diff --git a/spark/results/20260511/c7a.metal-48xl.json b/spark/results/20260511/c7a.metal-48xl.json index 286fe6156c..1f85d63be2 100644 --- a/spark/results/20260511/c7a.metal-48xl.json +++ b/spark/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 3, "data_size": 14779976446, - "concurrent_qps": 0.225, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [2.893, 2.672, 2.863], [5.848, 10.434, 3.3], diff --git a/spark/results/20260511/c8g.4xlarge.json b/spark/results/20260511/c8g.4xlarge.json index a9282ae91e..5951a1fed0 100644 --- a/spark/results/20260511/c8g.4xlarge.json +++ b/spark/results/20260511/c8g.4xlarge.json @@ -10,7 +10,7 @@ "load_time": 1, "data_size": 14779976446, "concurrent_qps": null, - "concurrent_error_ratio": 1, + "concurrent_error_ratio": null, "result": [ [2.517, 2.265, 2.269], [2.867, 2.517, 2.554], diff --git a/spark/results/20260511/c8g.metal-48xl.json b/spark/results/20260511/c8g.metal-48xl.json index 67d8288719..a70450e99c 100644 --- a/spark/results/20260511/c8g.metal-48xl.json +++ b/spark/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 3, "data_size": 14779976446, - "concurrent_qps": 0.078, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [2.984, 3.11, 2.639], [5.773, 4.555, 4.157], diff --git a/sqlite/benchmark.sh b/sqlite/benchmark.sh index 2e5741b394..bf02868043 100755 --- a/sqlite/benchmark.sh +++ b/sqlite/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/turso/benchmark.sh b/turso/benchmark.sh index 2e5741b394..bf02868043 100755 --- a/turso/benchmark.sh +++ b/turso/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/zighouse/benchmark.sh b/zighouse/benchmark.sh index c722454d56..8f6103ef31 100755 --- a/zighouse/benchmark.sh +++ b/zighouse/benchmark.sh @@ -3,4 +3,9 @@ set -e export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/zighouse/results/20260511/c6a.4xlarge.json b/zighouse/results/20260511/c6a.4xlarge.json index cffe5a5d3b..018479e3d5 100644 --- a/zighouse/results/20260511/c6a.4xlarge.json +++ b/zighouse/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Zig","column-oriented","embedded","parquet","tuned"], "load_time": 86, "data_size": 10907660070, - "concurrent_qps": 3.71, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.003, 0, 0], [0.626, 0.017, 0.017],